rda-python-dsupdt 2.0.1__tar.gz → 2.0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/PKG-INFO +1 -1
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/pyproject.toml +1 -1
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt/dsupdt.py +16 -15
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt/dsupdt.usg +1 -11
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt/pg_updt.py +218 -220
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt.egg-info/PKG-INFO +1 -1
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/LICENSE +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/MANIFEST.in +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/README.md +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/setup.cfg +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt/PgUpdt.py +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt/__init__.py +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt/ds_updt.py +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt.egg-info/SOURCES.txt +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt.egg-info/dependency_links.txt +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt.egg-info/entry_points.txt +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt.egg-info/requires.txt +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt.egg-info/top_level.txt +0 -0
- {rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/tests/test_dsupdt.py +0 -0
|
@@ -18,14 +18,15 @@ import os
|
|
|
18
18
|
import re
|
|
19
19
|
from os import path as op
|
|
20
20
|
from .pg_updt import PgUpdt
|
|
21
|
+
from rda_python_common.pg_split import PgSplit
|
|
21
22
|
|
|
22
|
-
class DsUpdt(PgUpdt):
|
|
23
|
+
class DsUpdt(PgUpdt, PgSplit):
|
|
23
24
|
def __init__(self):
|
|
24
25
|
super().__init__() # initialize parent class
|
|
25
26
|
self.TEMPINFO = {}
|
|
26
27
|
self.TOPMSG = self.SUBJECT = self.ACTSTR = None
|
|
27
28
|
self.ALLCNT = 0
|
|
28
|
-
self.DEFTYPES = {'WT'
|
|
29
|
+
self.DEFTYPES = {'WT': 'D', 'ST': 'P', 'QT': 'B'}
|
|
29
30
|
|
|
30
31
|
# main function to run dsupdt
|
|
31
32
|
def read_parameters(self):
|
|
@@ -386,7 +387,7 @@ class DsUpdt(PgUpdt):
|
|
|
386
387
|
# get update info of local and remote files owned by login name
|
|
387
388
|
def get_update_info(self):
|
|
388
389
|
if 'DS' in self.params:
|
|
389
|
-
dsids = {'dsid'
|
|
390
|
+
dsids = {'dsid': [self.params['DS']]}
|
|
390
391
|
dscnt = 1
|
|
391
392
|
else:
|
|
392
393
|
tname = "dlupdt"
|
|
@@ -638,7 +639,7 @@ class DsUpdt(PgUpdt):
|
|
|
638
639
|
return self.pglog("{}: NO remote file record matched for {}".format(locinfo, rcnd), self.PGOPT['emlerr'])
|
|
639
640
|
# create a empty record remote file
|
|
640
641
|
rcnt = 1
|
|
641
|
-
rmtrecs = {'lindex'
|
|
642
|
+
rmtrecs = {'lindex': [lindex], 'dindex': [0]}
|
|
642
643
|
rflds = ['remotefile', 'serverfile', 'download', 'begintime', 'endtime', 'tinterval']
|
|
643
644
|
for rfld in rflds: rmtrecs[rfld] = [None]
|
|
644
645
|
if rcnt == 1:
|
|
@@ -1290,7 +1291,7 @@ class DsUpdt(PgUpdt):
|
|
|
1290
1291
|
if ainfo['vindex']: growing = self.is_growing_file(locrec['locfile'], tempinfo['FQ'])
|
|
1291
1292
|
tempinfo['ainfo'] = None # clean the archive info recorded earlier
|
|
1292
1293
|
else:
|
|
1293
|
-
ainfo = {'archived'
|
|
1294
|
+
ainfo = {'archived': 0, 'note': None} # reference to empty hash
|
|
1294
1295
|
self.pglog("{}: start {} for {}".format(lfile, locrec['action'], tempinfo['einfo']), self.PGOPT['emllog'])
|
|
1295
1296
|
options = locrec['options'] if locrec['options'] else ""
|
|
1296
1297
|
act = locrec['action']
|
|
@@ -1333,7 +1334,7 @@ class DsUpdt(PgUpdt):
|
|
|
1333
1334
|
if tempinfo['gotnew'] and not re.search(r'(^|\s)-OE(\s|$)', options, re.I): acmd += " -OE"
|
|
1334
1335
|
if 'VS' in self.params:
|
|
1335
1336
|
acmd += " -VS {}".format(self.params['VS'])
|
|
1336
|
-
if 'VS' in tempinfo: options = re.sub('-VS\s+\d+(\s+|$)', '', options, flags=re.I)
|
|
1337
|
+
if 'VS' in tempinfo: options = re.sub(r'-VS\s+\d+(\s+|$)', '', options, flags=re.I)
|
|
1337
1338
|
if tempinfo['RS'] == 1: acmd += " -RS"
|
|
1338
1339
|
fnote = None
|
|
1339
1340
|
if locrec['note'] and not re.search(r'(^|\s)-DE(\s|$)', options, re.I):
|
|
@@ -1427,9 +1428,9 @@ class DsUpdt(PgUpdt):
|
|
|
1427
1428
|
val = self.PGOPT['UCNTL']['validint']
|
|
1428
1429
|
else:
|
|
1429
1430
|
val = None
|
|
1430
|
-
tempinfo = {'AT'
|
|
1431
|
-
'VD'
|
|
1432
|
-
'QU'
|
|
1431
|
+
tempinfo = {'AT': None, 'DC': None, 'ED': [], 'EH': [], 'VI': None,
|
|
1432
|
+
'VD': None, 'VH': None, 'CVD': None, 'NX': None, 'FQ': None,
|
|
1433
|
+
'QU': None, 'EP': 0, 'RS': -1, 'AQ': None}
|
|
1433
1434
|
if val: val = self.get_control_time(val, "Valid Internal")
|
|
1434
1435
|
if val:
|
|
1435
1436
|
tempinfo['VI'] = val
|
|
@@ -1505,7 +1506,7 @@ class DsUpdt(PgUpdt):
|
|
|
1505
1506
|
dhour = self.diffdatehour(vdate, vhour, locrec['missdate'], locrec['misshour'])
|
|
1506
1507
|
if dhour > 0:
|
|
1507
1508
|
if dhour > 240:
|
|
1508
|
-
record = {'missdate'
|
|
1509
|
+
record = {'missdate': None, 'misshour': None}
|
|
1509
1510
|
self.pgupdt("dlupdt", record, "lindex = {}".format(locrec['lindex']))
|
|
1510
1511
|
else:
|
|
1511
1512
|
vdate = locrec['missdate']
|
|
@@ -1575,7 +1576,7 @@ class DsUpdt(PgUpdt):
|
|
|
1575
1576
|
if tempinfo['ainfo'] != None: return tempinfo['ainfo']
|
|
1576
1577
|
edate = tempinfo['edate']
|
|
1577
1578
|
ehour = tempinfo['ehour']
|
|
1578
|
-
ainfo = {'archcnt'
|
|
1579
|
+
ainfo = {'archcnt': 0, 'archived': 0, 'present': 0, 'vindex': 0, 'types': {}, 'note': None}
|
|
1579
1580
|
growing = self.is_growing_file(locrec['locfile'], tempinfo['FQ'])
|
|
1580
1581
|
if growing:
|
|
1581
1582
|
if tempinfo['NX']:
|
|
@@ -1705,7 +1706,7 @@ class DsUpdt(PgUpdt):
|
|
|
1705
1706
|
return onote
|
|
1706
1707
|
if pcnt == 2: # replace start time
|
|
1707
1708
|
if onote: # get start time from existing note
|
|
1708
|
-
replace = "{}{}{}".format(seps[0], patterns[0], seps[1])
|
|
1709
|
+
replace = r"{}{}{}".format(seps[0], patterns[0], seps[1])
|
|
1709
1710
|
ms = re.match(r'^(.*){}(.*){}'.format(replace, self.params['PD'][0]), note)
|
|
1710
1711
|
if ms:
|
|
1711
1712
|
init = ms.group(1)
|
|
@@ -1789,8 +1790,8 @@ class DsUpdt(PgUpdt):
|
|
|
1789
1790
|
finfo = self.check_local_file(cfile, 33, self.PGOPT['wrnlog'])
|
|
1790
1791
|
if not finfo: return 3 # download if can not check newer
|
|
1791
1792
|
else:
|
|
1792
|
-
finfo = {'isfile'
|
|
1793
|
-
'date_modified'
|
|
1793
|
+
finfo = {'isfile': 0, 'checksum': ainfo['chksm'], 'data_size': ainfo['asize'],
|
|
1794
|
+
'date_modified': ainfo['adate'], 'time_modified': ainfo['atime']}
|
|
1794
1795
|
cinfo = self.check_server_file(dcmd, 33, cfile)
|
|
1795
1796
|
if not cinfo:
|
|
1796
1797
|
sact = self.get_download_action(dcmd)
|
|
@@ -1937,7 +1938,7 @@ class DsUpdt(PgUpdt):
|
|
|
1937
1938
|
if rcnd and self.pgget("drupdt", "", loccnd):
|
|
1938
1939
|
return self.pglog("{}: NO remote file record matched for {}".format(locinfo, rcnd), self.PGOPT['emlerr'])
|
|
1939
1940
|
rcnt = 1 # create a empty record remote file
|
|
1940
|
-
rmtrecs = {'lindex'
|
|
1941
|
+
rmtrecs = {'lindex': locrec['lindex'], 'remotefile': None, 'serverfile': None}
|
|
1941
1942
|
if rcnt == 1:
|
|
1942
1943
|
if 'RF' in self.params and len(self.params['RF']) == 1 and not (rmtrecs['remotefile'][0] and self.params['RF'][0] == rmtrecs['remotefile'][0]):
|
|
1943
1944
|
rmtrecs['remotefile'][0] = self.params['RF'][0]
|
|
@@ -275,7 +275,7 @@ ControlIndex<:>Specialist<:>ParentIndex<:>Action<:>Frequency<:>ControlOffset<:>C
|
|
|
275
275
|
D -(DT|DataTime) time data updated to, format: YYY-MM-DD HH:NN:SS
|
|
276
276
|
H -(HN|HostName) hostnames this cindex can/cannot be processed on
|
|
277
277
|
S -(SB|SBatchOptions) additional PBS batch options
|
|
278
|
-
Q -(QS|QSubOptions) additional
|
|
278
|
+
Q -(QS|QSubOptions) additional PBS batch options
|
|
279
279
|
Y -(CC|CarbonCopy) carbon copies for additional email addresses
|
|
280
280
|
X -(XC|ExecuteCommand) Command to be executed after successful update
|
|
281
281
|
|
|
@@ -687,7 +687,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
687
687
|
[-(VS|ValidSize) MinSizeForValidFile]
|
|
688
688
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
689
689
|
[-(HO|HourOffset) TimeZoneHourOffset]
|
|
690
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
691
690
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
692
691
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
693
692
|
[-(CC|CarbonCopy) Cc'dEmailAddresses]
|
|
@@ -811,7 +810,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
811
810
|
[-(VS|ValidSize) MinSizeForValidFile]
|
|
812
811
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
813
812
|
[-(HO|HourOffset) TimeZoneHourOffset]
|
|
814
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
815
813
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
816
814
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
817
815
|
[-(CC|CarbonCopy) Cc'dEmailAddresses]
|
|
@@ -883,7 +881,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
883
881
|
[-(CH|CurrentHour) CurrentHour] # provide if different than current
|
|
884
882
|
[-(VS|ValidSize) MinSizeForValidFile]
|
|
885
883
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
886
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
887
884
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
888
885
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
889
886
|
[-(CC|CarbonCopy) Cc'dEmailAddresses]
|
|
@@ -959,7 +956,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
959
956
|
[-(CH|CurrentHour) CurrentHour] # provide if different than current
|
|
960
957
|
[-(VS|ValidSize) MinSizeForValidFile]
|
|
961
958
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
962
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
963
959
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
964
960
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
965
961
|
[-(HO|HourOffset) TimeZoneHourOffset]
|
|
@@ -1024,7 +1020,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
1024
1020
|
[-(CH|CurrentHour) CurrentHour] # provide if different than current
|
|
1025
1021
|
[-(VS|ValidSize) MinSizeForValidFile]
|
|
1026
1022
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
1027
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
1028
1023
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
1029
1024
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
1030
1025
|
[-(CC|CarbonCopy) Cc'dEmailAddresses]
|
|
@@ -1087,7 +1082,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
|
|
|
1087
1082
|
[-(CD|CurrentDate) CurrentDate] # provide if different than current
|
|
1088
1083
|
[-(CH|CurrentHour) CurrentHour] # provide if different than current
|
|
1089
1084
|
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
|
|
1090
|
-
[-(SB|SbatchOptions) SlurmBatchOptions]
|
|
1091
1085
|
[-(QS|QsubOptions) PBSBatchOptions]
|
|
1092
1086
|
[-(BP|BatchProcess) [BatchProcessHosts]]
|
|
1093
1087
|
[-(CC|CarbonCopy) Cc'dEmailAddresses]
|
|
@@ -1717,10 +1711,6 @@ Information options are used to pass information, one or multiple values, into
|
|
|
1717
1711
|
actions -SL (-SetLocalFile). Another way of reordering the local files is to
|
|
1718
1712
|
provide explicitly order index values per Info option -XO (-ExecOrder).
|
|
1719
1713
|
|
|
1720
|
-
-SB or -SbatchOptions, (Alias: -SlurmOptions), specifies options to execute dsupdt
|
|
1721
|
-
as a batch job via sbatch on SLURM nodes. The sbatch options must be quoted when
|
|
1722
|
-
prsented on command line, such as, -QS '-t 24:00:00'.
|
|
1723
|
-
|
|
1724
1714
|
-SF or -ServerFile, only used to set file names on Remote servers when they
|
|
1725
1715
|
are different from the remote file names.
|
|
1726
1716
|
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
#
|
|
2
2
|
###############################################################################
|
|
3
3
|
#
|
|
4
|
-
# Title
|
|
5
|
-
# Author
|
|
6
|
-
# Date
|
|
4
|
+
# Title: pg_updt.py
|
|
5
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
6
|
+
# Date: 09/23/2020
|
|
7
7
|
# 2025-02-07 transferred to package rda_python_dsupdt from
|
|
8
8
|
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
9
|
# 2025-12-08 transfer to class PgUpdt
|
|
10
|
-
# Purpose
|
|
10
|
+
# Purpose: python library module to help rountinely updates of new data
|
|
11
11
|
# for one or multiple datasets
|
|
12
12
|
#
|
|
13
|
-
# Github
|
|
13
|
+
# Github: https://github.com/NCAR/rda-python-dsupdt.git
|
|
14
14
|
#
|
|
15
15
|
###############################################################################
|
|
16
16
|
#
|
|
@@ -27,226 +27,224 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
27
27
|
super().__init__() # initialize parent class
|
|
28
28
|
self.CORDERS = {}
|
|
29
29
|
self.OPTS.update({
|
|
30
|
-
'DR'
|
|
31
|
-
'BL'
|
|
32
|
-
'PB'
|
|
33
|
-
'AF'
|
|
34
|
-
'CF'
|
|
35
|
-
'UF'
|
|
36
|
-
'CU'
|
|
37
|
-
'GC'
|
|
38
|
-
'GL'
|
|
39
|
-
'GR'
|
|
40
|
-
'GA'
|
|
41
|
-
'SC'
|
|
42
|
-
'SL'
|
|
43
|
-
'SR'
|
|
44
|
-
'SA'
|
|
45
|
-
'DL'
|
|
46
|
-
'UL'
|
|
47
|
-
'AW'
|
|
48
|
-
'BG'
|
|
49
|
-
'CA'
|
|
50
|
-
'CN'
|
|
51
|
-
'CP'
|
|
52
|
-
'EE'
|
|
53
|
-
'FO'
|
|
54
|
-
'FU'
|
|
55
|
-
'GZ'
|
|
56
|
-
'HU'
|
|
57
|
-
'IE'
|
|
58
|
-
'KR'
|
|
59
|
-
'KS'
|
|
60
|
-
'LO'
|
|
61
|
-
'MD'
|
|
62
|
-
'MO'
|
|
63
|
-
'MU'
|
|
64
|
-
'NC'
|
|
65
|
-
'NE'
|
|
66
|
-
'NL'
|
|
67
|
-
'NY'
|
|
68
|
-
'QE'
|
|
69
|
-
'RA'
|
|
70
|
-
'RD'
|
|
71
|
-
'RE'
|
|
72
|
-
'RO'
|
|
73
|
-
'SE'
|
|
74
|
-
'UB'
|
|
75
|
-
'UT'
|
|
76
|
-
'AO'
|
|
77
|
-
'CD'
|
|
78
|
-
'CH'
|
|
79
|
-
'DS'
|
|
80
|
-
'DV'
|
|
81
|
-
'ES'
|
|
82
|
-
'FN'
|
|
83
|
-
'LN'
|
|
84
|
-
'OF'
|
|
85
|
-
'ON'
|
|
86
|
-
'PL'
|
|
87
|
-
'VS'
|
|
88
|
-
'AN'
|
|
89
|
-
'AT'
|
|
90
|
-
'BC'
|
|
91
|
-
'BP'
|
|
92
|
-
'BT'
|
|
93
|
-
'CC'
|
|
94
|
-
'CI'
|
|
95
|
-
'CL'
|
|
96
|
-
'CO'
|
|
97
|
-
'CT'
|
|
98
|
-
'DB'
|
|
99
|
-
'DC'
|
|
100
|
-
'DE'
|
|
101
|
-
'DO'
|
|
102
|
-
'DT'
|
|
103
|
-
'EC'
|
|
104
|
-
'ED'
|
|
105
|
-
'EH'
|
|
106
|
-
'EP'
|
|
107
|
-
'ET'
|
|
108
|
-
'FA'
|
|
109
|
-
'FQ'
|
|
110
|
-
'GP'
|
|
111
|
-
'HN'
|
|
112
|
-
'HO'
|
|
113
|
-
'ID'
|
|
114
|
-
'IF'
|
|
115
|
-
'KF'
|
|
116
|
-
'LF'
|
|
117
|
-
'LI'
|
|
118
|
-
'MC'
|
|
119
|
-
'MR'
|
|
120
|
-
'DI'
|
|
121
|
-
'OP'
|
|
122
|
-
'PD'
|
|
123
|
-
'PI'
|
|
124
|
-
'PR'
|
|
125
|
-
'QS'
|
|
126
|
-
'RF'
|
|
127
|
-
'RI'
|
|
128
|
-
'
|
|
129
|
-
'
|
|
130
|
-
'
|
|
131
|
-
'
|
|
132
|
-
'
|
|
133
|
-
'
|
|
134
|
-
'
|
|
135
|
-
'
|
|
136
|
-
'XO' : [2, 'ExecOrder', 16],
|
|
30
|
+
'DR': [0x00010, 'DownloadRemote',2],
|
|
31
|
+
'BL': [0x00020, 'BuildLocal', 2],
|
|
32
|
+
'PB': [0x00030, 'ProcessBoth', 2], # DR & BL
|
|
33
|
+
'AF': [0x00040, 'ArchiveFile', 2],
|
|
34
|
+
'CF': [0x00080, 'CleanFile', 2],
|
|
35
|
+
'UF': [0x000F0, 'UpdateFile', 2], # DR & BL & AF & CF
|
|
36
|
+
'CU': [0x00200, 'CheckUpdate', 0],
|
|
37
|
+
'GC': [0x00400, 'GetControl', 0],
|
|
38
|
+
'GL': [0x00800, 'GetLocalFile', 0],
|
|
39
|
+
'GR': [0x01000, 'GetRemoteFile', 0],
|
|
40
|
+
'GA': [0x01C00, 'GetALL', 0], # GC & GL & GR
|
|
41
|
+
'SC': [0x02000, 'SetControl', 1],
|
|
42
|
+
'SL': [0x04000, 'SetLocalFile', 1],
|
|
43
|
+
'SR': [0x08000, 'SetRemoteFile', 1],
|
|
44
|
+
'SA': [0x0E000, 'SetALL', 4], # SC & SL & SR
|
|
45
|
+
'DL': [0x20000, 'Delete', 1],
|
|
46
|
+
'UL': [0x40000, 'UnLock', 1],
|
|
47
|
+
'AW': [0, 'AnyWhere'],
|
|
48
|
+
'BG': [0, 'BackGround'],
|
|
49
|
+
'CA': [0, 'CheckAll'],
|
|
50
|
+
'CN': [0, 'CheckNew'],
|
|
51
|
+
'CP': [0, 'CurrrentPeriod'],
|
|
52
|
+
'EE': [0, 'ErrorEmail'], # send email when error happens only
|
|
53
|
+
'FO': [0, 'FormatOutput'],
|
|
54
|
+
'FU': [0, 'FutureUpdate'],
|
|
55
|
+
'GZ': [0, 'GMTZone'],
|
|
56
|
+
'HU': [0, 'HourlyUpdate'],
|
|
57
|
+
'IE': [0, 'IgnoreError'],
|
|
58
|
+
'KR': [0, 'KeepRemote'],
|
|
59
|
+
'KS': [0, 'KeepServer'],
|
|
60
|
+
'LO': [0, 'LogOn'],
|
|
61
|
+
'MD': [0, 'PgDataset'],
|
|
62
|
+
'MO': [0, 'MissedOnly'],
|
|
63
|
+
'MU': [0, 'MultipleUpdate'],
|
|
64
|
+
'NC': [0, 'NewControl'],
|
|
65
|
+
'NE': [0, 'NoEmail'],
|
|
66
|
+
'NL': [0, 'NewLocfile'],
|
|
67
|
+
'NY': [0, 'NoLeapYear'],
|
|
68
|
+
'QE': [0, 'QuitError'],
|
|
69
|
+
'RA': [0, 'RetryArchive'],
|
|
70
|
+
'RD': [0, 'RetryDownload'],
|
|
71
|
+
'RE': [0, 'ResetEndTime'],
|
|
72
|
+
'RO': [0, 'ResetOrder'],
|
|
73
|
+
'SE': [0, 'SummaryEmail'], # send summary email only
|
|
74
|
+
'UB': [0, 'UseBeginTime'],
|
|
75
|
+
'UT': [0, 'UpdateTime'],
|
|
76
|
+
'AO': [1, 'ActOption', 1], # default to <!>
|
|
77
|
+
'CD': [1, 'CurrentDate', 256], # used this instead of curdate()
|
|
78
|
+
'CH': [1, 'CurrentHour', 16], # used this instead of (localtime)[2]
|
|
79
|
+
'DS': [1, 'Dataset', 0],
|
|
80
|
+
'DV': [1, 'Divider', 1], # default to <:>
|
|
81
|
+
'ES': [1, 'EqualSign', 1], # default to <=>
|
|
82
|
+
'FN': [1, 'FieldNames', 0],
|
|
83
|
+
'LN': [1, 'LoginName', 1],
|
|
84
|
+
'OF': [1, 'OutputFile', 0],
|
|
85
|
+
'ON': [1, 'OrderNames', 0],
|
|
86
|
+
'PL': [1, 'ProcessLimit', 17],
|
|
87
|
+
'VS': [1, 'ValidSize', 17], # default to self.PGLOG['MINSIZE']
|
|
88
|
+
'AN': [2, 'ActionName', 1],
|
|
89
|
+
'AT': [2, 'AgeTime', 1],
|
|
90
|
+
'BC': [2, 'BuildCommand', 1],
|
|
91
|
+
'BP': [2, 'BatchProcess', 0, ''],
|
|
92
|
+
'BT': [2, 'BeginTime', 1],
|
|
93
|
+
'CC': [2, 'CarbonCopy', 0],
|
|
94
|
+
'CI': [2, 'ControlIndex', 16],
|
|
95
|
+
'CL': [2, 'CleanCommand', 1],
|
|
96
|
+
'CO': [2, "ControlOffset", 1],
|
|
97
|
+
'CT': [2, 'ControlTime', 32+356],
|
|
98
|
+
'DB': [2, 'Debug', 0],
|
|
99
|
+
'DC': [2, 'DownloadCommand', 1],
|
|
100
|
+
'DE': [2, 'Description', 64],
|
|
101
|
+
'DO': [2, 'DownloadOrder', 16],
|
|
102
|
+
'DT': [2, 'DataTime', 1+32+256],
|
|
103
|
+
'EC': [2, 'ErrorControl', 1, "NIQ"],
|
|
104
|
+
'ED': [2, 'EndDate', 257],
|
|
105
|
+
'EH': [2, 'EndHour', 33],
|
|
106
|
+
'EP': [2, 'EndPeriod', 1],
|
|
107
|
+
'ET': [2, 'EndTime', 33],
|
|
108
|
+
'FA': [2, 'FileArchived', 0],
|
|
109
|
+
'FQ': [2, 'Frequency', 1],
|
|
110
|
+
'GP': [2, 'GenericPattern', 0],
|
|
111
|
+
'HN': [2, "HostName", 1],
|
|
112
|
+
'HO': [2, 'HourOffset', 17],
|
|
113
|
+
'ID': [2, 'ControlID', 0],
|
|
114
|
+
'IF': [2, 'InputFile', 0],
|
|
115
|
+
'KF': [2, 'KeepFile', 1, "NRSB"],
|
|
116
|
+
'LF': [2, 'LocalFile', 0],
|
|
117
|
+
'LI': [2, 'LocalIndex', 17],
|
|
118
|
+
'MC': [2, 'EMailControl', 1, "ASNEB"],
|
|
119
|
+
'MR': [2, 'MissRemote', 128, "NY"],
|
|
120
|
+
'DI': [2, 'DueInterval', 1],
|
|
121
|
+
'OP': [2, 'Options', 1],
|
|
122
|
+
'PD': [2, 'PatternDelimiter', 2], # pattern delimiters, default to ["<", ">"]
|
|
123
|
+
'PI': [2, 'ParentIndex', 17],
|
|
124
|
+
'PR': [2, 'ProcessRemote', 1],
|
|
125
|
+
'QS': [2, 'QSubOptions', 0],
|
|
126
|
+
'RF': [2, 'RemoteFile', 0],
|
|
127
|
+
'RI': [2, 'RetryInterval', 1],
|
|
128
|
+
'SF': [2, 'ServerFile', 0],
|
|
129
|
+
'SN': [2, 'Specialist', 1],
|
|
130
|
+
'TI': [2, 'TimeInterval', 1],
|
|
131
|
+
'UC': [2, 'UpdateControl', 1],
|
|
132
|
+
'VI': [2, 'ValidInterval', 1],
|
|
133
|
+
'WD': [2, 'WorkDir', 1],
|
|
134
|
+
'XC': [2, 'ExecuteCommand', 1],
|
|
135
|
+
'XO': [2, 'ExecOrder', 16],
|
|
137
136
|
})
|
|
138
137
|
self.ALIAS.update({
|
|
139
|
-
'AN'
|
|
140
|
-
'AT'
|
|
141
|
-
'BC'
|
|
142
|
-
'BG'
|
|
143
|
-
'BL'
|
|
144
|
-
'BP'
|
|
145
|
-
'BT'
|
|
146
|
-
'CI'
|
|
147
|
-
'CL'
|
|
148
|
-
'CN'
|
|
149
|
-
'DC'
|
|
150
|
-
'DE'
|
|
151
|
-
'DI'
|
|
152
|
-
'DL'
|
|
153
|
-
'DR'
|
|
154
|
-
'DS'
|
|
155
|
-
'DV'
|
|
156
|
-
'ED'
|
|
157
|
-
'EH'
|
|
158
|
-
'EP'
|
|
159
|
-
'FA'
|
|
160
|
-
'FQ'
|
|
161
|
-
'FU'
|
|
162
|
-
'GC'
|
|
163
|
-
'GL'
|
|
164
|
-
'GN'
|
|
165
|
-
'GP'
|
|
166
|
-
'GR'
|
|
167
|
-
'GZ'
|
|
168
|
-
'HN'
|
|
169
|
-
'KR'
|
|
170
|
-
'KS'
|
|
171
|
-
'LF'
|
|
172
|
-
'LI'
|
|
173
|
-
'LO'
|
|
174
|
-
'OP'
|
|
175
|
-
'NC'
|
|
176
|
-
'NL'
|
|
177
|
-
'PD'
|
|
178
|
-
'QE'
|
|
179
|
-
'QS'
|
|
180
|
-
'RD'
|
|
181
|
-
'RO'
|
|
182
|
-
'
|
|
183
|
-
'
|
|
184
|
-
'
|
|
185
|
-
'
|
|
186
|
-
'
|
|
187
|
-
'
|
|
188
|
-
'
|
|
189
|
-
'
|
|
190
|
-
'XO' : ['ExecuteOrder']
|
|
138
|
+
'AN': ['Action', "AC"],
|
|
139
|
+
'AT': ['FileAge', "FileAgeTime"],
|
|
140
|
+
'BC': ['BuildCmd'],
|
|
141
|
+
'BG': ['b'],
|
|
142
|
+
'BL': ['BuildLocalfile'],
|
|
143
|
+
'BP': ['d', 'DelayedMode'],
|
|
144
|
+
'BT': ['IT', 'InitialTime'],
|
|
145
|
+
'CI': ['UpdateControlIndex'],
|
|
146
|
+
'CL': ['CleanFile'],
|
|
147
|
+
'CN': ['CheckNewFile'],
|
|
148
|
+
'DC': ['Command', 'Download'],
|
|
149
|
+
'DE': ['Desc', 'Note', 'FileDesc', 'FileDescription'],
|
|
150
|
+
'DI': ['NextDue'],
|
|
151
|
+
'DL': ['RM', 'Remove'],
|
|
152
|
+
'DR': ['DownloadRemoteFile'],
|
|
153
|
+
'DS': ['Dsid', 'DatasetID'],
|
|
154
|
+
'DV': ['Delimiter', 'Separator'],
|
|
155
|
+
'ED': ['UpdateEndDate'],
|
|
156
|
+
'EH': ['UpdateEndHour'],
|
|
157
|
+
'EP': ['EndPeriodDay'],
|
|
158
|
+
'FA': ['SF', 'WF', 'QF'],
|
|
159
|
+
'FQ': ['UpdateFrequency'],
|
|
160
|
+
'FU': ["ForceUpdate"],
|
|
161
|
+
'GC': ['GetUpdateControl'],
|
|
162
|
+
'GL': ['GetLocal'],
|
|
163
|
+
'GN': ['GroupID'],
|
|
164
|
+
'GP': ['GeneralPattern'],
|
|
165
|
+
'GR': ['GetRemote'],
|
|
166
|
+
'GZ': ['GMT', 'GreenwichZone', 'UTC'],
|
|
167
|
+
'HN': ['HostMachine'],
|
|
168
|
+
'KR': ['KeepRemoteFile'],
|
|
169
|
+
'KS': ['KeepServerFile'],
|
|
170
|
+
'LF': ['LocalFileIndex'],
|
|
171
|
+
'LI': ['LocIndex', "UpdateIndex"],
|
|
172
|
+
'LO': ['LoggingOn'],
|
|
173
|
+
'OP': ['DsarchOption'],
|
|
174
|
+
'NC': ['NewUpdateControl'],
|
|
175
|
+
'NL': ['NewLocalFile'],
|
|
176
|
+
'PD': ['TD', 'TemporalDelimiter'],
|
|
177
|
+
'QE': ['QuitOnError'],
|
|
178
|
+
'QS': ['PBSOptions'],
|
|
179
|
+
'RD': ['Redownlaod'],
|
|
180
|
+
'RO': ['Reorder'],
|
|
181
|
+
'SC': ['SetUpdateControl'],
|
|
182
|
+
'SL': ['SetLocal'],
|
|
183
|
+
'SN': ['SpecialistName'],
|
|
184
|
+
'SR': ['SetRemote'],
|
|
185
|
+
'TI': ['Interval'],
|
|
186
|
+
'UL': ["UnLockUpdate"],
|
|
187
|
+
'XC': ['ExecCmd'],
|
|
188
|
+
'XO': ['ExecuteOrder']
|
|
191
189
|
})
|
|
192
190
|
# single letter short names for option 'FN' (Field Names) to retrieve info
|
|
193
191
|
# from RDADB; only the fields can be manipulated by this application are listed
|
|
194
192
|
# SHORTNM KEYS(self.OPTS) DBFIELD
|
|
195
193
|
self.TBLHASH['dlupdt'] = { # condition flag, 0-int, 1-string, -1-exclude
|
|
196
|
-
'L'
|
|
197
|
-
'F'
|
|
198
|
-
'A'
|
|
199
|
-
'I'
|
|
200
|
-
'U'
|
|
201
|
-
'X'
|
|
202
|
-
'S'
|
|
203
|
-
'M'
|
|
204
|
-
'W'
|
|
205
|
-
'O'
|
|
206
|
-
'C'
|
|
207
|
-
'Q'
|
|
208
|
-
'E'
|
|
209
|
-
'J'
|
|
210
|
-
'K'
|
|
211
|
-
'N'
|
|
212
|
-
'V'
|
|
213
|
-
'T'
|
|
214
|
-
'R'
|
|
215
|
-
'B'
|
|
216
|
-
'Z'
|
|
217
|
-
'D'
|
|
194
|
+
'L': ['LI', "lindex", 0],
|
|
195
|
+
'F': ['LF', "locfile", 1],
|
|
196
|
+
'A': ['AN', "action", 1], # dsarch action
|
|
197
|
+
'I': ['CI', "cindex", 0],
|
|
198
|
+
'U': ['FA', "archfile", 1],
|
|
199
|
+
'X': ['XO', "execorder", 1],
|
|
200
|
+
'S': ['SN', "specialist", 1],
|
|
201
|
+
'M': ['MR', "missremote", 1],
|
|
202
|
+
'W': ['WD', "workdir", 1],
|
|
203
|
+
'O': ['OP', "options", 1],
|
|
204
|
+
'C': ['DC', "download", 1],
|
|
205
|
+
'Q': ['FQ', "frequency", 1],
|
|
206
|
+
'E': ['EP', "endperiod", 0],
|
|
207
|
+
'J': ['ED', "enddate", 1],
|
|
208
|
+
'K': ['EH', "endhour", 0],
|
|
209
|
+
'N': ['DI', "nextdue", 1],
|
|
210
|
+
'V': ['VI', "validint", 1],
|
|
211
|
+
'T': ['AT', "agetime", 1],
|
|
212
|
+
'R': ['PR', "processremote", 1],
|
|
213
|
+
'B': ['BC', "buildcmd", 1],
|
|
214
|
+
'Z': ['CL', "cleancmd", 1],
|
|
215
|
+
'D': ['DE', "note", 1],
|
|
218
216
|
}
|
|
219
217
|
self.TBLHASH['drupdt'] = {
|
|
220
|
-
'L'
|
|
221
|
-
'F'
|
|
222
|
-
'D'
|
|
223
|
-
'S'
|
|
224
|
-
'C'
|
|
225
|
-
'B'
|
|
226
|
-
'E'
|
|
227
|
-
'T'
|
|
218
|
+
'L': ['LI', "lindex", 0], # same as dlupdt.lindex
|
|
219
|
+
'F': ['RF', "remotefile", 1],
|
|
220
|
+
'D': ['DO', "dindex", 0],
|
|
221
|
+
'S': ['SF', "serverfile", 1],
|
|
222
|
+
'C': ['DC', "download", 1],
|
|
223
|
+
'B': ['BT', "begintime", 1],
|
|
224
|
+
'E': ['ET', "endtime", 1],
|
|
225
|
+
'T': ['TI', "tinterval", 1],
|
|
228
226
|
}
|
|
229
227
|
self.TBLHASH['dcupdt'] = {
|
|
230
|
-
'C'
|
|
231
|
-
'L'
|
|
232
|
-
'N'
|
|
233
|
-
'P'
|
|
234
|
-
'A'
|
|
235
|
-
'F'
|
|
236
|
-
'O'
|
|
237
|
-
'T'
|
|
238
|
-
'R'
|
|
239
|
-
'V'
|
|
240
|
-
'U'
|
|
241
|
-
'J'
|
|
242
|
-
'E'
|
|
243
|
-
'K'
|
|
244
|
-
'Z'
|
|
245
|
-
'D'
|
|
246
|
-
'H'
|
|
247
|
-
'Q'
|
|
248
|
-
'Y'
|
|
249
|
-
'X'
|
|
228
|
+
'C': ['CI', "cindex", 0],
|
|
229
|
+
'L': ['ID', "cntlid", 1],
|
|
230
|
+
'N': ['SN', "specialist", 1],
|
|
231
|
+
'P': ['PI', "pindex", 0], # if not 0, refer to another dcupdt.cindex
|
|
232
|
+
'A': ['AN', "action", 1], # dsupdt action
|
|
233
|
+
'F': ['FQ', "frequency", 1],
|
|
234
|
+
'O': ['CO', "cntloffset", 1],
|
|
235
|
+
'T': ['CT', "cntltime", 1],
|
|
236
|
+
'R': ['RI', "retryint", 1],
|
|
237
|
+
'V': ['VI', "validint", 1],
|
|
238
|
+
'U': ['UC', "updtcntl", 1],
|
|
239
|
+
'J': ['MC', "emailcntl", 1],
|
|
240
|
+
'E': ['EC', "errorcntl", 1],
|
|
241
|
+
'K': ['KF', "keepfile", 1],
|
|
242
|
+
'Z': ['HO', "houroffset", 1],
|
|
243
|
+
'D': ['DT', "datatime", 1],
|
|
244
|
+
'H': ['HN', "hostname", 1],
|
|
245
|
+
'Q': ['QS', "qoptions", 1],
|
|
246
|
+
'Y': ['CC', "emails", 1],
|
|
247
|
+
'X': ['XC', "execcmd", 1],
|
|
250
248
|
}
|
|
251
249
|
# global info to be used by the whole application
|
|
252
250
|
self.PGOPT['updated'] = 0
|
|
@@ -278,7 +276,7 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
278
276
|
self.PGOPT['uwcnt'] = self.PGOPT['udcnt'] = self.PGOPT['uncnt'] = self.PGOPT['rdcnt'] = 0
|
|
279
277
|
self.PGOPT['lindex'] = 0 # the current lindex is under updating
|
|
280
278
|
self.WSLOWS = {
|
|
281
|
-
'nomads.ncep.noaa.gov'
|
|
279
|
+
'nomads.ncep.noaa.gov': 8
|
|
282
280
|
}
|
|
283
281
|
# set default parameters
|
|
284
282
|
self.params['PD'] = ["<" , ">"] # temporal pattern delimiters
|
|
@@ -396,8 +394,8 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
396
394
|
pcnt = len(patterns)
|
|
397
395
|
if pcnt == 0: return fname # return original name if no pattern
|
|
398
396
|
if limit and pcnt > limit: pcnt = limit
|
|
399
|
-
mps = {'b'
|
|
400
|
-
'n'
|
|
397
|
+
mps = {'b': r'^B(.+)B$', 'c': r'^C(.+)C$', 'd': r'(\d+)$', 'm': r'^M([NC])M$',
|
|
398
|
+
'n': r'^N(H+|D+)N$', 'p': r'^P(\d+)$', 's': r'^S[\d:]+S$', 'w': r'^W(.+)W$'}
|
|
401
399
|
for i in range(pcnt):
|
|
402
400
|
pattern = patterns[i]
|
|
403
401
|
replace = "{}{}{}".format(seps[0], pattern, seps[1])
|
|
@@ -815,7 +813,7 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
815
813
|
(date, hour) = self.addfrequency(date, hour, tempinfo['FQ'], 1)
|
|
816
814
|
date = self.enddate(date, tempinfo['EP'], tempinfo['QU'], tempinfo['FQ'][6])
|
|
817
815
|
if 'UT' in self.params or not locrec['enddate'] or self.diffdatehour(date, hour, locrec['enddate'], locrec['endhour']) > 0:
|
|
818
|
-
record = {'enddate'
|
|
816
|
+
record = {'enddate': date}
|
|
819
817
|
if hour != None:
|
|
820
818
|
record['endhour'] = hour
|
|
821
819
|
einfo = "end data date:hour {}:{:02}".format(date, hour)
|
|
@@ -1071,7 +1069,7 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
1071
1069
|
ms = re.match(r'^(-\w+)', buf)
|
|
1072
1070
|
while ms:
|
|
1073
1071
|
flg = ms.group(1)
|
|
1074
|
-
buf = re.sub('^-\w+\s+'.format(flg), '', buf, 1) # remove options
|
|
1072
|
+
buf = re.sub(r'^-\w+\s+'.format(flg), '', buf, 1) # remove options
|
|
1075
1073
|
if flg != "-r": # no option value
|
|
1076
1074
|
m = re.match(r'^(\S+)\s', buf)
|
|
1077
1075
|
if not m: break
|
|
@@ -1538,7 +1536,7 @@ class PgUpdt(PgOPT, PgCMD):
|
|
|
1538
1536
|
# reset updated data time
|
|
1539
1537
|
def reset_data_time(self, qu, ddate, dhour, lidx):
|
|
1540
1538
|
pgrec = self.PGOPT['UCNTL']
|
|
1541
|
-
record = {'chktime'
|
|
1539
|
+
record = {'chktime': int(time.time())}
|
|
1542
1540
|
if ddate:
|
|
1543
1541
|
if dhour is None: dhour = 0 if qu == 'H' else 23
|
|
1544
1542
|
dtime = "{} {:02}:59:59".format(ddate, dhour)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
{rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt.egg-info/entry_points.txt
RENAMED
|
File without changes
|
{rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt.egg-info/requires.txt
RENAMED
|
File without changes
|
{rda_python_dsupdt-2.0.1 → rda_python_dsupdt-2.0.3}/src/rda_python_dsupdt.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|