rda-python-dsupdt 2.0.2__tar.gz → 2.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/PKG-INFO +1 -1
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/pyproject.toml +1 -1
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt/dsupdt.py +25 -14
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt/dsupdt.usg +1 -11
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt/pg_updt.py +217 -219
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt.egg-info/PKG-INFO +1 -1
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/LICENSE +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/MANIFEST.in +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/README.md +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/setup.cfg +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt/PgUpdt.py +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt/__init__.py +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt/ds_updt.py +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt.egg-info/SOURCES.txt +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt.egg-info/dependency_links.txt +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt.egg-info/entry_points.txt +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt.egg-info/requires.txt +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt.egg-info/top_level.txt +0 -0
- {rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/tests/test_dsupdt.py +0 -0
|
@@ -26,7 +26,7 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
26
26
|
self.TEMPINFO = {}
|
|
27
27
|
self.TOPMSG = self.SUBJECT = self.ACTSTR = None
|
|
28
28
|
self.ALLCNT = 0
|
|
29
|
-
self.DEFTYPES = {'WT'
|
|
29
|
+
self.DEFTYPES = {'WT': 'D', 'ST': 'P', 'QT': 'B'}
|
|
30
30
|
|
|
31
31
|
# main function to run dsupdt
|
|
32
32
|
def read_parameters(self):
|
|
@@ -387,7 +387,7 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
387
387
|
# get update info of local and remote files owned by login name
|
|
388
388
|
def get_update_info(self):
|
|
389
389
|
if 'DS' in self.params:
|
|
390
|
-
dsids = {'dsid'
|
|
390
|
+
dsids = {'dsid': [self.params['DS']]}
|
|
391
391
|
dscnt = 1
|
|
392
392
|
else:
|
|
393
393
|
tname = "dlupdt"
|
|
@@ -639,7 +639,7 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
639
639
|
return self.pglog("{}: NO remote file record matched for {}".format(locinfo, rcnd), self.PGOPT['emlerr'])
|
|
640
640
|
# create a empty record remote file
|
|
641
641
|
rcnt = 1
|
|
642
|
-
rmtrecs = {'lindex'
|
|
642
|
+
rmtrecs = {'lindex': [lindex], 'dindex': [0]}
|
|
643
643
|
rflds = ['remotefile', 'serverfile', 'download', 'begintime', 'endtime', 'tinterval']
|
|
644
644
|
for rfld in rflds: rmtrecs[rfld] = [None]
|
|
645
645
|
if rcnt == 1:
|
|
@@ -1291,7 +1291,7 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
1291
1291
|
if ainfo['vindex']: growing = self.is_growing_file(locrec['locfile'], tempinfo['FQ'])
|
|
1292
1292
|
tempinfo['ainfo'] = None # clean the archive info recorded earlier
|
|
1293
1293
|
else:
|
|
1294
|
-
ainfo = {'archived'
|
|
1294
|
+
ainfo = {'archived': 0, 'note': None} # reference to empty hash
|
|
1295
1295
|
self.pglog("{}: start {} for {}".format(lfile, locrec['action'], tempinfo['einfo']), self.PGOPT['emllog'])
|
|
1296
1296
|
options = locrec['options'] if locrec['options'] else ""
|
|
1297
1297
|
act = locrec['action']
|
|
@@ -1324,6 +1324,7 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
1324
1324
|
else:
|
|
1325
1325
|
ifopt = 'LF'
|
|
1326
1326
|
acmd = "dsarch {} {} -{} {}".format(self.params['DS'], act, ifopt, lfile)
|
|
1327
|
+
gcmd = None
|
|
1327
1328
|
if 'wfile' in ainfo: acmd += " -WF " + ainfo['wfile']
|
|
1328
1329
|
if 'sfile' in ainfo: acmd += " -SF " + ainfo['sfile']
|
|
1329
1330
|
if 'bfile' in ainfo: acmd += " -QF " + ainfo['bfile']
|
|
@@ -1334,8 +1335,16 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
1334
1335
|
if tempinfo['gotnew'] and not re.search(r'(^|\s)-OE(\s|$)', options, re.I): acmd += " -OE"
|
|
1335
1336
|
if 'VS' in self.params:
|
|
1336
1337
|
acmd += " -VS {}".format(self.params['VS'])
|
|
1337
|
-
if 'VS' in tempinfo: options = re.sub(r'-VS\s+\d
|
|
1338
|
-
if
|
|
1338
|
+
if 'VS' in tempinfo: options = re.sub(r'-VS\s+\d+\s*', '', options, flags=re.I)
|
|
1339
|
+
if re.search(r'(^|\s)-GX(\s|$)', options, re.I):
|
|
1340
|
+
wfile = ainfo['wfile'] if 'wfile' in ainfo else ainfo['afile']
|
|
1341
|
+
ms = re.search(r'(^|\s)-DF (\w+)(\s|$)', options, re.I)
|
|
1342
|
+
fmt = ms.ms.group(2).lower() if ms else None
|
|
1343
|
+
if wfile and fmt:
|
|
1344
|
+
if fmt == "netcdf": fmt = "cf" + fmt
|
|
1345
|
+
rs = " -R -S" if tempinfo['RS'] == 1 else ''
|
|
1346
|
+
gcmd = "gatherxml -d {} -f {}{} {}".format(self.params['DS'], fmt, rs, wfile)
|
|
1347
|
+
options = re.sub(r'-GX\s*', '', options, flags=re.I)
|
|
1339
1348
|
fnote = None
|
|
1340
1349
|
if locrec['note'] and not re.search(r'(^|\s)-DE(\s|$)', options, re.I):
|
|
1341
1350
|
note = self.build_data_note(ainfo['note'], lfile, locrec, tempinfo)
|
|
@@ -1352,6 +1361,7 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
1352
1361
|
if locrec['cleancmd']: options = re.sub(r'(^-NW\s+|\s+-NW$)', '', options, 1, re.I)
|
|
1353
1362
|
acmd += " " + self.replace_pattern(options, tempinfo['edate'], tempinfo['ehour'], tempinfo['FQ'])
|
|
1354
1363
|
ret = self.pgsystem(acmd, self.PGOPT['emerol'], 69) # 1 + 4 + 64
|
|
1364
|
+
if gcmd: self.pgsystem(gcmd, self.PGOPT['emerol'], 5)
|
|
1355
1365
|
if fnote: self.pgsystem("rm -f " + fnote, self.PGOPT['emerol'], 4)
|
|
1356
1366
|
tempinfo['ainfo'] = self.file_archive_info(lfile, locrec, tempinfo)
|
|
1357
1367
|
note = self.count_update_files(ainfo, tempinfo['ainfo'], ret, tempinfo['RS'])
|
|
@@ -1428,9 +1438,9 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
1428
1438
|
val = self.PGOPT['UCNTL']['validint']
|
|
1429
1439
|
else:
|
|
1430
1440
|
val = None
|
|
1431
|
-
tempinfo = {'AT'
|
|
1432
|
-
'VD'
|
|
1433
|
-
'QU'
|
|
1441
|
+
tempinfo = {'AT': None, 'DC': None, 'ED': [], 'EH': [], 'VI': None,
|
|
1442
|
+
'VD': None, 'VH': None, 'CVD': None, 'NX': None, 'FQ': None,
|
|
1443
|
+
'QU': None, 'EP': 0, 'RS': -1, 'AQ': None}
|
|
1434
1444
|
if val: val = self.get_control_time(val, "Valid Internal")
|
|
1435
1445
|
if val:
|
|
1436
1446
|
tempinfo['VI'] = val
|
|
@@ -1506,7 +1516,7 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
1506
1516
|
dhour = self.diffdatehour(vdate, vhour, locrec['missdate'], locrec['misshour'])
|
|
1507
1517
|
if dhour > 0:
|
|
1508
1518
|
if dhour > 240:
|
|
1509
|
-
record = {'missdate'
|
|
1519
|
+
record = {'missdate': None, 'misshour': None}
|
|
1510
1520
|
self.pgupdt("dlupdt", record, "lindex = {}".format(locrec['lindex']))
|
|
1511
1521
|
else:
|
|
1512
1522
|
vdate = locrec['missdate']
|
|
@@ -1576,7 +1586,7 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
1576
1586
|
if tempinfo['ainfo'] != None: return tempinfo['ainfo']
|
|
1577
1587
|
edate = tempinfo['edate']
|
|
1578
1588
|
ehour = tempinfo['ehour']
|
|
1579
|
-
ainfo = {'archcnt'
|
|
1589
|
+
ainfo = {'archcnt': 0, 'archived': 0, 'present': 0, 'vindex': 0, 'types': {}, 'note': None, 'afile' : None}
|
|
1580
1590
|
growing = self.is_growing_file(locrec['locfile'], tempinfo['FQ'])
|
|
1581
1591
|
if growing:
|
|
1582
1592
|
if tempinfo['NX']:
|
|
@@ -1607,6 +1617,7 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
1607
1617
|
else:
|
|
1608
1618
|
path = self.get_group_field_path(locrec['gindex'], dsid, 'webpath')
|
|
1609
1619
|
if path: afile = self.join_paths(path, afile)
|
|
1620
|
+
ainfo['afile'] = afile
|
|
1610
1621
|
wrec = self.pgget_wfile(dsid, "*", "{} AND type = '{}' AND wfile = '{}'".format(gcnd, type, afile), self.PGOPT['extlog'])
|
|
1611
1622
|
if wrec:
|
|
1612
1623
|
ainfo['wfile'] = wrec['wfile']
|
|
@@ -1790,8 +1801,8 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
1790
1801
|
finfo = self.check_local_file(cfile, 33, self.PGOPT['wrnlog'])
|
|
1791
1802
|
if not finfo: return 3 # download if can not check newer
|
|
1792
1803
|
else:
|
|
1793
|
-
finfo = {'isfile'
|
|
1794
|
-
'date_modified'
|
|
1804
|
+
finfo = {'isfile': 0, 'checksum': ainfo['chksm'], 'data_size': ainfo['asize'],
|
|
1805
|
+
'date_modified': ainfo['adate'], 'time_modified': ainfo['atime']}
|
|
1795
1806
|
cinfo = self.check_server_file(dcmd, 33, cfile)
|
|
1796
1807
|
if not cinfo:
|
|
1797
1808
|
sact = self.get_download_action(dcmd)
|
|
@@ -1938,7 +1949,7 @@ class DsUpdt(PgUpdt, PgSplit):
|
|
|
1938
1949
|
if rcnd and self.pgget("drupdt", "", loccnd):
|
|
1939
1950
|
return self.pglog("{}: NO remote file record matched for {}".format(locinfo, rcnd), self.PGOPT['emlerr'])
|
|
1940
1951
|
rcnt = 1 # create a empty record remote file
|
|
1941
|
-
rmtrecs = {'lindex'
|
|
1952
|
+
rmtrecs = {'lindex': locrec['lindex'], 'remotefile': None, 'serverfile': None}
|
|
1942
1953
|
if rcnt == 1:
|
|
1943
1954
|
if 'RF' in self.params and len(self.params['RF']) == 1 and not (rmtrecs['remotefile'][0] and self.params['RF'][0] == rmtrecs['remotefile'][0]):
|
|
1944
1955
|
rmtrecs['remotefile'][0] = self.params['RF'][0]
|
|
@@ -275,7 +275,7 @@ ControlIndex<:>Specialist<:>ParentIndex<:>Action<:>Frequency<:>ControlOffset<:>C
|
|
|
275
275
|
D -(DT|DataTime) time data updated to, format: YYY-MM-DD HH:NN:SS
|
|
276
276
|
H -(HN|HostName) hostnames this cindex can/cannot be processed on
|
|
277
277
|
S -(SB|SBatchOptions) additional PBS batch options
|
|
278
|
-
Q -(QS|QSubOptions) additional
|
|
278
|
+
Q -(QS|QSubOptions) additional PBS batch options
|
|
279
279
|
Y -(CC|CarbonCopy) carbon copies for additional email addresses
|
|
280
280
|
X -(XC|ExecuteCommand) Command to be executed after successful update
|
|
281
281
|
|
|
@@ -687,7 +687,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
687
687
|
[-(VS|ValidSize) MinSizeForValidFile]
|
|
688
688
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
689
689
|
[-(HO|HourOffset) TimeZoneHourOffset]
|
|
690
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
691
690
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
692
691
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
693
692
|
[-(CC|CarbonCopy) Cc'dEmailAddresses]
|
|
@@ -811,7 +810,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
811
810
|
[-(VS|ValidSize) MinSizeForValidFile]
|
|
812
811
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
813
812
|
[-(HO|HourOffset) TimeZoneHourOffset]
|
|
814
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
815
813
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
816
814
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
817
815
|
[-(CC|CarbonCopy) Cc'dEmailAddresses]
|
|
@@ -883,7 +881,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
883
881
|
[-(CH|CurrentHour) CurrentHour] # provide if different than current
|
|
884
882
|
[-(VS|ValidSize) MinSizeForValidFile]
|
|
885
883
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
886
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
887
884
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
888
885
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
889
886
|
[-(CC|CarbonCopy) Cc'dEmailAddresses]
|
|
@@ -959,7 +956,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
959
956
|
[-(CH|CurrentHour) CurrentHour] # provide if different than current
|
|
960
957
|
[-(VS|ValidSize) MinSizeForValidFile]
|
|
961
958
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
962
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
963
959
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
964
960
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
965
961
|
[-(HO|HourOffset) TimeZoneHourOffset]
|
|
@@ -1024,7 +1020,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
1024
1020
|
[-(CH|CurrentHour) CurrentHour] # provide if different than current
|
|
1025
1021
|
[-(VS|ValidSize) MinSizeForValidFile]
|
|
1026
1022
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
1027
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
1028
1023
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
1029
1024
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
1030
1025
|
[-(CC|CarbonCopy) Cc'dEmailAddresses]
|
|
@@ -1087,7 +1082,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
1087
1082
|
[-(CD|CurrentDate) CurrentDate] # provide if different than current
|
|
1088
1083
|
[-(CH|CurrentHour) CurrentHour] # provide if different than current
|
|
1089
1084
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
1090
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
1091
1085
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
1092
1086
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
1093
1087
|
[-(CC|CarbonCopy) Cc'dEmailAddresses]
|
|
@@ -1717,10 +1711,6 @@ Information options are used to pass information, one or multiple values, into
|
|
|
1717
1711
|
actions -SL (-SetLocalFile). Another way of reordering the local files is to
|
|
1718
1712
|
provide explicitly order index values per Info option -XO (-ExecOrder).
|
|
1719
1713
|
|
|
1720
|
-
-SB or -SbatchOptions, (Alias: -SlurmOptions), specifies options to execute dsupdt
|
|
1721
|
-
as a batch job via sbatch on SLURM nodes. The sbatch options must be quoted when
|
|
1722
|
-
prsented on command line, such as, -QS '-t 24:00:00'.
|
|
1723
|
-
|
|
1724
1714
|
-SF or -ServerFile, only used to set file names on Remote servers when they
|
|
1725
1715
|
are different from the remote file names.
|
|
1726
1716
|
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
#
|
|
2
2
|
###############################################################################
|
|
3
3
|
#
|
|
4
|
-
# Title
|
|
5
|
-
# Author
|
|
6
|
-
# Date
|
|
4
|
+
# Title: pg_updt.py
|
|
5
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
6
|
+
# Date: 09/23/2020
|
|
7
7
|
# 2025-02-07 transferred to package rda_python_dsupdt from
|
|
8
8
|
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
9
|
# 2025-12-08 transfer to class PgUpdt
|
|
10
|
-
# Purpose
|
|
10
|
+
# Purpose: python library module to help rountinely updates of new data
|
|
11
11
|
# for one or multiple datasets
|
|
12
12
|
#
|
|
13
|
-
# Github
|
|
13
|
+
# Github: https://github.com/NCAR/rda-python-dsupdt.git
|
|
14
14
|
#
|
|
15
15
|
###############################################################################
|
|
16
16
|
#
|
|
@@ -27,226 +27,224 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
27
27
|
super().__init__() # initialize parent class
|
|
28
28
|
self.CORDERS = {}
|
|
29
29
|
self.OPTS.update({
|
|
30
|
-
'DR'
|
|
31
|
-
'BL'
|
|
32
|
-
'PB'
|
|
33
|
-
'AF'
|
|
34
|
-
'CF'
|
|
35
|
-
'UF'
|
|
36
|
-
'CU'
|
|
37
|
-
'GC'
|
|
38
|
-
'GL'
|
|
39
|
-
'GR'
|
|
40
|
-
'GA'
|
|
41
|
-
'SC'
|
|
42
|
-
'SL'
|
|
43
|
-
'SR'
|
|
44
|
-
'SA'
|
|
45
|
-
'DL'
|
|
46
|
-
'UL'
|
|
47
|
-
'AW'
|
|
48
|
-
'BG'
|
|
49
|
-
'CA'
|
|
50
|
-
'CN'
|
|
51
|
-
'CP'
|
|
52
|
-
'EE'
|
|
53
|
-
'FO'
|
|
54
|
-
'FU'
|
|
55
|
-
'GZ'
|
|
56
|
-
'HU'
|
|
57
|
-
'IE'
|
|
58
|
-
'KR'
|
|
59
|
-
'KS'
|
|
60
|
-
'LO'
|
|
61
|
-
'MD'
|
|
62
|
-
'MO'
|
|
63
|
-
'MU'
|
|
64
|
-
'NC'
|
|
65
|
-
'NE'
|
|
66
|
-
'NL'
|
|
67
|
-
'NY'
|
|
68
|
-
'QE'
|
|
69
|
-
'RA'
|
|
70
|
-
'RD'
|
|
71
|
-
'RE'
|
|
72
|
-
'RO'
|
|
73
|
-
'SE'
|
|
74
|
-
'UB'
|
|
75
|
-
'UT'
|
|
76
|
-
'AO'
|
|
77
|
-
'CD'
|
|
78
|
-
'CH'
|
|
79
|
-
'DS'
|
|
80
|
-
'DV'
|
|
81
|
-
'ES'
|
|
82
|
-
'FN'
|
|
83
|
-
'LN'
|
|
84
|
-
'OF'
|
|
85
|
-
'ON'
|
|
86
|
-
'PL'
|
|
87
|
-
'VS'
|
|
88
|
-
'AN'
|
|
89
|
-
'AT'
|
|
90
|
-
'BC'
|
|
91
|
-
'BP'
|
|
92
|
-
'BT'
|
|
93
|
-
'CC'
|
|
94
|
-
'CI'
|
|
95
|
-
'CL'
|
|
96
|
-
'CO'
|
|
97
|
-
'CT'
|
|
98
|
-
'DB'
|
|
99
|
-
'DC'
|
|
100
|
-
'DE'
|
|
101
|
-
'DO'
|
|
102
|
-
'DT'
|
|
103
|
-
'EC'
|
|
104
|
-
'ED'
|
|
105
|
-
'EH'
|
|
106
|
-
'EP'
|
|
107
|
-
'ET'
|
|
108
|
-
'FA'
|
|
109
|
-
'FQ'
|
|
110
|
-
'GP'
|
|
111
|
-
'HN'
|
|
112
|
-
'HO'
|
|
113
|
-
'ID'
|
|
114
|
-
'IF'
|
|
115
|
-
'KF'
|
|
116
|
-
'LF'
|
|
117
|
-
'LI'
|
|
118
|
-
'MC'
|
|
119
|
-
'MR'
|
|
120
|
-
'DI'
|
|
121
|
-
'OP'
|
|
122
|
-
'PD'
|
|
123
|
-
'PI'
|
|
124
|
-
'PR'
|
|
125
|
-
'QS'
|
|
126
|
-
'RF'
|
|
127
|
-
'RI'
|
|
128
|
-
'
|
|
129
|
-
'
|
|
130
|
-
'
|
|
131
|
-
'
|
|
132
|
-
'
|
|
133
|
-
'
|
|
134
|
-
'
|
|
135
|
-
'
|
|
136
|
-
'XO' : [2, 'ExecOrder', 16],
|
|
30
|
+
'DR': [0x00010, 'DownloadRemote',2],
|
|
31
|
+
'BL': [0x00020, 'BuildLocal', 2],
|
|
32
|
+
'PB': [0x00030, 'ProcessBoth', 2], # DR & BL
|
|
33
|
+
'AF': [0x00040, 'ArchiveFile', 2],
|
|
34
|
+
'CF': [0x00080, 'CleanFile', 2],
|
|
35
|
+
'UF': [0x000F0, 'UpdateFile', 2], # DR & BL & AF & CF
|
|
36
|
+
'CU': [0x00200, 'CheckUpdate', 0],
|
|
37
|
+
'GC': [0x00400, 'GetControl', 0],
|
|
38
|
+
'GL': [0x00800, 'GetLocalFile', 0],
|
|
39
|
+
'GR': [0x01000, 'GetRemoteFile', 0],
|
|
40
|
+
'GA': [0x01C00, 'GetALL', 0], # GC & GL & GR
|
|
41
|
+
'SC': [0x02000, 'SetControl', 1],
|
|
42
|
+
'SL': [0x04000, 'SetLocalFile', 1],
|
|
43
|
+
'SR': [0x08000, 'SetRemoteFile', 1],
|
|
44
|
+
'SA': [0x0E000, 'SetALL', 4], # SC & SL & SR
|
|
45
|
+
'DL': [0x20000, 'Delete', 1],
|
|
46
|
+
'UL': [0x40000, 'UnLock', 1],
|
|
47
|
+
'AW': [0, 'AnyWhere'],
|
|
48
|
+
'BG': [0, 'BackGround'],
|
|
49
|
+
'CA': [0, 'CheckAll'],
|
|
50
|
+
'CN': [0, 'CheckNew'],
|
|
51
|
+
'CP': [0, 'CurrrentPeriod'],
|
|
52
|
+
'EE': [0, 'ErrorEmail'], # send email when error happens only
|
|
53
|
+
'FO': [0, 'FormatOutput'],
|
|
54
|
+
'FU': [0, 'FutureUpdate'],
|
|
55
|
+
'GZ': [0, 'GMTZone'],
|
|
56
|
+
'HU': [0, 'HourlyUpdate'],
|
|
57
|
+
'IE': [0, 'IgnoreError'],
|
|
58
|
+
'KR': [0, 'KeepRemote'],
|
|
59
|
+
'KS': [0, 'KeepServer'],
|
|
60
|
+
'LO': [0, 'LogOn'],
|
|
61
|
+
'MD': [0, 'PgDataset'],
|
|
62
|
+
'MO': [0, 'MissedOnly'],
|
|
63
|
+
'MU': [0, 'MultipleUpdate'],
|
|
64
|
+
'NC': [0, 'NewControl'],
|
|
65
|
+
'NE': [0, 'NoEmail'],
|
|
66
|
+
'NL': [0, 'NewLocfile'],
|
|
67
|
+
'NY': [0, 'NoLeapYear'],
|
|
68
|
+
'QE': [0, 'QuitError'],
|
|
69
|
+
'RA': [0, 'RetryArchive'],
|
|
70
|
+
'RD': [0, 'RetryDownload'],
|
|
71
|
+
'RE': [0, 'ResetEndTime'],
|
|
72
|
+
'RO': [0, 'ResetOrder'],
|
|
73
|
+
'SE': [0, 'SummaryEmail'], # send summary email only
|
|
74
|
+
'UB': [0, 'UseBeginTime'],
|
|
75
|
+
'UT': [0, 'UpdateTime'],
|
|
76
|
+
'AO': [1, 'ActOption', 1], # default to <!>
|
|
77
|
+
'CD': [1, 'CurrentDate', 256], # used this instead of curdate()
|
|
78
|
+
'CH': [1, 'CurrentHour', 16], # used this instead of (localtime)[2]
|
|
79
|
+
'DS': [1, 'Dataset', 0],
|
|
80
|
+
'DV': [1, 'Divider', 1], # default to <:>
|
|
81
|
+
'ES': [1, 'EqualSign', 1], # default to <=>
|
|
82
|
+
'FN': [1, 'FieldNames', 0],
|
|
83
|
+
'LN': [1, 'LoginName', 1],
|
|
84
|
+
'OF': [1, 'OutputFile', 0],
|
|
85
|
+
'ON': [1, 'OrderNames', 0],
|
|
86
|
+
'PL': [1, 'ProcessLimit', 17],
|
|
87
|
+
'VS': [1, 'ValidSize', 17], # default to self.PGLOG['MINSIZE']
|
|
88
|
+
'AN': [2, 'ActionName', 1],
|
|
89
|
+
'AT': [2, 'AgeTime', 1],
|
|
90
|
+
'BC': [2, 'BuildCommand', 1],
|
|
91
|
+
'BP': [2, 'BatchProcess', 0, ''],
|
|
92
|
+
'BT': [2, 'BeginTime', 1],
|
|
93
|
+
'CC': [2, 'CarbonCopy', 0],
|
|
94
|
+
'CI': [2, 'ControlIndex', 16],
|
|
95
|
+
'CL': [2, 'CleanCommand', 1],
|
|
96
|
+
'CO': [2, "ControlOffset", 1],
|
|
97
|
+
'CT': [2, 'ControlTime', 32+356],
|
|
98
|
+
'DB': [2, 'Debug', 0],
|
|
99
|
+
'DC': [2, 'DownloadCommand', 1],
|
|
100
|
+
'DE': [2, 'Description', 64],
|
|
101
|
+
'DO': [2, 'DownloadOrder', 16],
|
|
102
|
+
'DT': [2, 'DataTime', 1+32+256],
|
|
103
|
+
'EC': [2, 'ErrorControl', 1, "NIQ"],
|
|
104
|
+
'ED': [2, 'EndDate', 257],
|
|
105
|
+
'EH': [2, 'EndHour', 33],
|
|
106
|
+
'EP': [2, 'EndPeriod', 1],
|
|
107
|
+
'ET': [2, 'EndTime', 33],
|
|
108
|
+
'FA': [2, 'FileArchived', 0],
|
|
109
|
+
'FQ': [2, 'Frequency', 1],
|
|
110
|
+
'GP': [2, 'GenericPattern', 0],
|
|
111
|
+
'HN': [2, "HostName", 1],
|
|
112
|
+
'HO': [2, 'HourOffset', 17],
|
|
113
|
+
'ID': [2, 'ControlID', 0],
|
|
114
|
+
'IF': [2, 'InputFile', 0],
|
|
115
|
+
'KF': [2, 'KeepFile', 1, "NRSB"],
|
|
116
|
+
'LF': [2, 'LocalFile', 0],
|
|
117
|
+
'LI': [2, 'LocalIndex', 17],
|
|
118
|
+
'MC': [2, 'EMailControl', 1, "ASNEB"],
|
|
119
|
+
'MR': [2, 'MissRemote', 128, "NY"],
|
|
120
|
+
'DI': [2, 'DueInterval', 1],
|
|
121
|
+
'OP': [2, 'Options', 1],
|
|
122
|
+
'PD': [2, 'PatternDelimiter', 2], # pattern delimiters, default to ["<", ">"]
|
|
123
|
+
'PI': [2, 'ParentIndex', 17],
|
|
124
|
+
'PR': [2, 'ProcessRemote', 1],
|
|
125
|
+
'QS': [2, 'QSubOptions', 0],
|
|
126
|
+
'RF': [2, 'RemoteFile', 0],
|
|
127
|
+
'RI': [2, 'RetryInterval', 1],
|
|
128
|
+
'SF': [2, 'ServerFile', 0],
|
|
129
|
+
'SN': [2, 'Specialist', 1],
|
|
130
|
+
'TI': [2, 'TimeInterval', 1],
|
|
131
|
+
'UC': [2, 'UpdateControl', 1],
|
|
132
|
+
'VI': [2, 'ValidInterval', 1],
|
|
133
|
+
'WD': [2, 'WorkDir', 1],
|
|
134
|
+
'XC': [2, 'ExecuteCommand', 1],
|
|
135
|
+
'XO': [2, 'ExecOrder', 16],
|
|
137
136
|
})
|
|
138
137
|
self.ALIAS.update({
|
|
139
|
-
'AN'
|
|
140
|
-
'AT'
|
|
141
|
-
'BC'
|
|
142
|
-
'BG'
|
|
143
|
-
'BL'
|
|
144
|
-
'BP'
|
|
145
|
-
'BT'
|
|
146
|
-
'CI'
|
|
147
|
-
'CL'
|
|
148
|
-
'CN'
|
|
149
|
-
'DC'
|
|
150
|
-
'DE'
|
|
151
|
-
'DI'
|
|
152
|
-
'DL'
|
|
153
|
-
'DR'
|
|
154
|
-
'DS'
|
|
155
|
-
'DV'
|
|
156
|
-
'ED'
|
|
157
|
-
'EH'
|
|
158
|
-
'EP'
|
|
159
|
-
'FA'
|
|
160
|
-
'FQ'
|
|
161
|
-
'FU'
|
|
162
|
-
'GC'
|
|
163
|
-
'GL'
|
|
164
|
-
'GN'
|
|
165
|
-
'GP'
|
|
166
|
-
'GR'
|
|
167
|
-
'GZ'
|
|
168
|
-
'HN'
|
|
169
|
-
'KR'
|
|
170
|
-
'KS'
|
|
171
|
-
'LF'
|
|
172
|
-
'LI'
|
|
173
|
-
'LO'
|
|
174
|
-
'OP'
|
|
175
|
-
'NC'
|
|
176
|
-
'NL'
|
|
177
|
-
'PD'
|
|
178
|
-
'QE'
|
|
179
|
-
'QS'
|
|
180
|
-
'RD'
|
|
181
|
-
'RO'
|
|
182
|
-
'
|
|
183
|
-
'
|
|
184
|
-
'
|
|
185
|
-
'
|
|
186
|
-
'
|
|
187
|
-
'
|
|
188
|
-
'
|
|
189
|
-
'
|
|
190
|
-
'XO' : ['ExecuteOrder']
|
|
138
|
+
'AN': ['Action', "AC"],
|
|
139
|
+
'AT': ['FileAge', "FileAgeTime"],
|
|
140
|
+
'BC': ['BuildCmd'],
|
|
141
|
+
'BG': ['b'],
|
|
142
|
+
'BL': ['BuildLocalfile'],
|
|
143
|
+
'BP': ['d', 'DelayedMode'],
|
|
144
|
+
'BT': ['IT', 'InitialTime'],
|
|
145
|
+
'CI': ['UpdateControlIndex'],
|
|
146
|
+
'CL': ['CleanFile'],
|
|
147
|
+
'CN': ['CheckNewFile'],
|
|
148
|
+
'DC': ['Command', 'Download'],
|
|
149
|
+
'DE': ['Desc', 'Note', 'FileDesc', 'FileDescription'],
|
|
150
|
+
'DI': ['NextDue'],
|
|
151
|
+
'DL': ['RM', 'Remove'],
|
|
152
|
+
'DR': ['DownloadRemoteFile'],
|
|
153
|
+
'DS': ['Dsid', 'DatasetID'],
|
|
154
|
+
'DV': ['Delimiter', 'Separator'],
|
|
155
|
+
'ED': ['UpdateEndDate'],
|
|
156
|
+
'EH': ['UpdateEndHour'],
|
|
157
|
+
'EP': ['EndPeriodDay'],
|
|
158
|
+
'FA': ['SF', 'WF', 'QF'],
|
|
159
|
+
'FQ': ['UpdateFrequency'],
|
|
160
|
+
'FU': ["ForceUpdate"],
|
|
161
|
+
'GC': ['GetUpdateControl'],
|
|
162
|
+
'GL': ['GetLocal'],
|
|
163
|
+
'GN': ['GroupID'],
|
|
164
|
+
'GP': ['GeneralPattern'],
|
|
165
|
+
'GR': ['GetRemote'],
|
|
166
|
+
'GZ': ['GMT', 'GreenwichZone', 'UTC'],
|
|
167
|
+
'HN': ['HostMachine'],
|
|
168
|
+
'KR': ['KeepRemoteFile'],
|
|
169
|
+
'KS': ['KeepServerFile'],
|
|
170
|
+
'LF': ['LocalFileIndex'],
|
|
171
|
+
'LI': ['LocIndex', "UpdateIndex"],
|
|
172
|
+
'LO': ['LoggingOn'],
|
|
173
|
+
'OP': ['DsarchOption'],
|
|
174
|
+
'NC': ['NewUpdateControl'],
|
|
175
|
+
'NL': ['NewLocalFile'],
|
|
176
|
+
'PD': ['TD', 'TemporalDelimiter'],
|
|
177
|
+
'QE': ['QuitOnError'],
|
|
178
|
+
'QS': ['PBSOptions'],
|
|
179
|
+
'RD': ['Redownlaod'],
|
|
180
|
+
'RO': ['Reorder'],
|
|
181
|
+
'SC': ['SetUpdateControl'],
|
|
182
|
+
'SL': ['SetLocal'],
|
|
183
|
+
'SN': ['SpecialistName'],
|
|
184
|
+
'SR': ['SetRemote'],
|
|
185
|
+
'TI': ['Interval'],
|
|
186
|
+
'UL': ["UnLockUpdate"],
|
|
187
|
+
'XC': ['ExecCmd'],
|
|
188
|
+
'XO': ['ExecuteOrder']
|
|
191
189
|
})
|
|
192
190
|
# single letter short names for option 'FN' (Field Names) to retrieve info
|
|
193
191
|
# from RDADB; only the fields can be manipulated by this application are listed
|
|
194
192
|
# SHORTNM KEYS(self.OPTS) DBFIELD
|
|
195
193
|
self.TBLHASH['dlupdt'] = { # condition flag, 0-int, 1-string, -1-exclude
|
|
196
|
-
'L'
|
|
197
|
-
'F'
|
|
198
|
-
'A'
|
|
199
|
-
'I'
|
|
200
|
-
'U'
|
|
201
|
-
'X'
|
|
202
|
-
'S'
|
|
203
|
-
'M'
|
|
204
|
-
'W'
|
|
205
|
-
'O'
|
|
206
|
-
'C'
|
|
207
|
-
'Q'
|
|
208
|
-
'E'
|
|
209
|
-
'J'
|
|
210
|
-
'K'
|
|
211
|
-
'N'
|
|
212
|
-
'V'
|
|
213
|
-
'T'
|
|
214
|
-
'R'
|
|
215
|
-
'B'
|
|
216
|
-
'Z'
|
|
217
|
-
'D'
|
|
194
|
+
'L': ['LI', "lindex", 0],
|
|
195
|
+
'F': ['LF', "locfile", 1],
|
|
196
|
+
'A': ['AN', "action", 1], # dsarch action
|
|
197
|
+
'I': ['CI', "cindex", 0],
|
|
198
|
+
'U': ['FA', "archfile", 1],
|
|
199
|
+
'X': ['XO', "execorder", 1],
|
|
200
|
+
'S': ['SN', "specialist", 1],
|
|
201
|
+
'M': ['MR', "missremote", 1],
|
|
202
|
+
'W': ['WD', "workdir", 1],
|
|
203
|
+
'O': ['OP', "options", 1],
|
|
204
|
+
'C': ['DC', "download", 1],
|
|
205
|
+
'Q': ['FQ', "frequency", 1],
|
|
206
|
+
'E': ['EP', "endperiod", 0],
|
|
207
|
+
'J': ['ED', "enddate", 1],
|
|
208
|
+
'K': ['EH', "endhour", 0],
|
|
209
|
+
'N': ['DI', "nextdue", 1],
|
|
210
|
+
'V': ['VI', "validint", 1],
|
|
211
|
+
'T': ['AT', "agetime", 1],
|
|
212
|
+
'R': ['PR', "processremote", 1],
|
|
213
|
+
'B': ['BC', "buildcmd", 1],
|
|
214
|
+
'Z': ['CL', "cleancmd", 1],
|
|
215
|
+
'D': ['DE', "note", 1],
|
|
218
216
|
}
|
|
219
217
|
self.TBLHASH['drupdt'] = {
|
|
220
|
-
'L'
|
|
221
|
-
'F'
|
|
222
|
-
'D'
|
|
223
|
-
'S'
|
|
224
|
-
'C'
|
|
225
|
-
'B'
|
|
226
|
-
'E'
|
|
227
|
-
'T'
|
|
218
|
+
'L': ['LI', "lindex", 0], # same as dlupdt.lindex
|
|
219
|
+
'F': ['RF', "remotefile", 1],
|
|
220
|
+
'D': ['DO', "dindex", 0],
|
|
221
|
+
'S': ['SF', "serverfile", 1],
|
|
222
|
+
'C': ['DC', "download", 1],
|
|
223
|
+
'B': ['BT', "begintime", 1],
|
|
224
|
+
'E': ['ET', "endtime", 1],
|
|
225
|
+
'T': ['TI', "tinterval", 1],
|
|
228
226
|
}
|
|
229
227
|
self.TBLHASH['dcupdt'] = {
|
|
230
|
-
'C'
|
|
231
|
-
'L'
|
|
232
|
-
'N'
|
|
233
|
-
'P'
|
|
234
|
-
'A'
|
|
235
|
-
'F'
|
|
236
|
-
'O'
|
|
237
|
-
'T'
|
|
238
|
-
'R'
|
|
239
|
-
'V'
|
|
240
|
-
'U'
|
|
241
|
-
'J'
|
|
242
|
-
'E'
|
|
243
|
-
'K'
|
|
244
|
-
'Z'
|
|
245
|
-
'D'
|
|
246
|
-
'H'
|
|
247
|
-
'Q'
|
|
248
|
-
'Y'
|
|
249
|
-
'X'
|
|
228
|
+
'C': ['CI', "cindex", 0],
|
|
229
|
+
'L': ['ID', "cntlid", 1],
|
|
230
|
+
'N': ['SN', "specialist", 1],
|
|
231
|
+
'P': ['PI', "pindex", 0], # if not 0, refer to another dcupdt.cindex
|
|
232
|
+
'A': ['AN', "action", 1], # dsupdt action
|
|
233
|
+
'F': ['FQ', "frequency", 1],
|
|
234
|
+
'O': ['CO', "cntloffset", 1],
|
|
235
|
+
'T': ['CT', "cntltime", 1],
|
|
236
|
+
'R': ['RI', "retryint", 1],
|
|
237
|
+
'V': ['VI', "validint", 1],
|
|
238
|
+
'U': ['UC', "updtcntl", 1],
|
|
239
|
+
'J': ['MC', "emailcntl", 1],
|
|
240
|
+
'E': ['EC', "errorcntl", 1],
|
|
241
|
+
'K': ['KF', "keepfile", 1],
|
|
242
|
+
'Z': ['HO', "houroffset", 1],
|
|
243
|
+
'D': ['DT', "datatime", 1],
|
|
244
|
+
'H': ['HN', "hostname", 1],
|
|
245
|
+
'Q': ['QS', "qoptions", 1],
|
|
246
|
+
'Y': ['CC', "emails", 1],
|
|
247
|
+
'X': ['XC', "execcmd", 1],
|
|
250
248
|
}
|
|
251
249
|
# global info to be used by the whole application
|
|
252
250
|
self.PGOPT['updated'] = 0
|
|
@@ -278,7 +276,7 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
278
276
|
self.PGOPT['uwcnt'] = self.PGOPT['udcnt'] = self.PGOPT['uncnt'] = self.PGOPT['rdcnt'] = 0
|
|
279
277
|
self.PGOPT['lindex'] = 0 # the current lindex is under updating
|
|
280
278
|
self.WSLOWS = {
|
|
281
|
-
'nomads.ncep.noaa.gov'
|
|
279
|
+
'nomads.ncep.noaa.gov': 8
|
|
282
280
|
}
|
|
283
281
|
# set default parameters
|
|
284
282
|
self.params['PD'] = ["<" , ">"] # temporal pattern delimiters
|
|
@@ -396,8 +394,8 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
396
394
|
pcnt = len(patterns)
|
|
397
395
|
if pcnt == 0: return fname # return original name if no pattern
|
|
398
396
|
if limit and pcnt > limit: pcnt = limit
|
|
399
|
-
mps = {'b'
|
|
400
|
-
'n'
|
|
397
|
+
mps = {'b': r'^B(.+)B$', 'c': r'^C(.+)C$', 'd': r'(\d+)$', 'm': r'^M([NC])M$',
|
|
398
|
+
'n': r'^N(H+|D+)N$', 'p': r'^P(\d+)$', 's': r'^S[\d:]+S$', 'w': r'^W(.+)W$'}
|
|
401
399
|
for i in range(pcnt):
|
|
402
400
|
pattern = patterns[i]
|
|
403
401
|
replace = "{}{}{}".format(seps[0], pattern, seps[1])
|
|
@@ -815,7 +813,7 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
815
813
|
(date, hour) = self.addfrequency(date, hour, tempinfo['FQ'], 1)
|
|
816
814
|
date = self.enddate(date, tempinfo['EP'], tempinfo['QU'], tempinfo['FQ'][6])
|
|
817
815
|
if 'UT' in self.params or not locrec['enddate'] or self.diffdatehour(date, hour, locrec['enddate'], locrec['endhour']) > 0:
|
|
818
|
-
record = {'enddate'
|
|
816
|
+
record = {'enddate': date}
|
|
819
817
|
if hour != None:
|
|
820
818
|
record['endhour'] = hour
|
|
821
819
|
einfo = "end data date:hour {}:{:02}".format(date, hour)
|
|
@@ -1538,7 +1536,7 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
1538
1536
|
# reset updated data time
|
|
1539
1537
|
def reset_data_time(self, qu, ddate, dhour, lidx):
|
|
1540
1538
|
pgrec = self.PGOPT['UCNTL']
|
|
1541
|
-
record = {'chktime'
|
|
1539
|
+
record = {'chktime': int(time.time())}
|
|
1542
1540
|
if ddate:
|
|
1543
1541
|
if dhour is None: dhour = 0 if qu == 'H' else 23
|
|
1544
1542
|
dtime = "{} {:02}:59:59".format(ddate, dhour)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
{rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt.egg-info/entry_points.txt
RENAMED
|
File without changes
|
{rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt.egg-info/requires.txt
RENAMED
|
File without changes
|
{rda_python_dsupdt-2.0.2 → rda_python_dsupdt-2.0.4}/src/rda_python_dsupdt.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|