rda-python-common 2.1.1__tar.gz → 2.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/PKG-INFO +1 -1
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/pyproject.toml +1 -1
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/pg_cmd.py +344 -71
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/pg_dbi.py +1007 -210
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/pg_file.py +1388 -21
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/pg_lock.py +244 -52
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/pg_log.py +733 -188
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/pg_opt.py +650 -142
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/pg_sig.py +462 -52
- rda_python_common-2.1.3/src/rda_python_common/pg_split.py +424 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/pg_util.py +887 -40
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common.egg-info/PKG-INFO +1 -1
- rda_python_common-2.1.1/src/rda_python_common/pg_split.py +0 -253
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/LICENSE +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/README.md +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/setup.cfg +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/PgCMD.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/PgDBI.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/PgFile.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/PgLOG.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/PgLock.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/PgOPT.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/PgSIG.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/PgSplit.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/PgUtil.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/__init__.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/pg_password.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common/pgpassword.py +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common.egg-info/SOURCES.txt +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common.egg-info/dependency_links.txt +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common.egg-info/entry_points.txt +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common.egg-info/requires.txt +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/src/rda_python_common.egg-info/top_level.txt +0 -0
- {rda_python_common-2.1.1 → rda_python_common-2.1.3}/test/test_common.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rda_python_common
|
|
3
|
-
Version: 2.1.
|
|
3
|
+
Version: 2.1.3
|
|
4
4
|
Summary: RDA Python common library codes shared by other RDA python packages
|
|
5
5
|
Author-email: Zaihua Ji <zji@ucar.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/NCAR/rda-python-common
|
|
@@ -15,8 +15,27 @@ import time
|
|
|
15
15
|
from .pg_lock import PgLock
|
|
16
16
|
|
|
17
17
|
class PgCMD(PgLock):
|
|
18
|
+
"""Manages batch/delayed-mode command execution records in the RDADB dscheck table.
|
|
19
|
+
|
|
20
|
+
Allows commands to be queued, tracked across runs, and cleaned up on
|
|
21
|
+
completion. Wraps the lower-level database and locking primitives provided
|
|
22
|
+
by PgLock to provide a complete command-lifecycle interface.
|
|
23
|
+
|
|
24
|
+
Attributes:
|
|
25
|
+
DSCHK (dict): Cached dscheck info dict for the currently running command.
|
|
26
|
+
BOPTIONS (dict): Batch options keyed by field name: hostname, qoptions,
|
|
27
|
+
modules, environments.
|
|
28
|
+
BFIELDS (str): Comma-joined string of BOPTIONS keys for use in SQL
|
|
29
|
+
SELECT clauses.
|
|
30
|
+
TRYLMTS (dict): Per-command retry limits, with a 'default' fallback.
|
|
31
|
+
DLYPTN (str): Compiled regex pattern string that matches the delayed-mode
|
|
32
|
+
flag in an argument string.
|
|
33
|
+
DLYOPT (dict): Per-command delayed-mode option strings appended to argv
|
|
34
|
+
when queuing a new dscheck record.
|
|
35
|
+
"""
|
|
18
36
|
|
|
19
37
|
def __init__(self):
|
|
38
|
+
"""Initializes PgCMD and all instance attributes."""
|
|
20
39
|
super().__init__() # initialize parent class
|
|
21
40
|
# cached dscheck info
|
|
22
41
|
self.DSCHK = {}
|
|
@@ -34,21 +53,29 @@ class PgCMD(PgLock):
|
|
|
34
53
|
'dsrqst': ' -d'
|
|
35
54
|
}
|
|
36
55
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
56
|
+
def set_batch_options(self, params, opt, addhost=0):
|
|
57
|
+
"""Sets batch options from a parsed parameter dictionary.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
params (dict): Dict holding option values, keyed by short option
|
|
61
|
+
names (e.g. 'QS', 'MO', 'EV', 'HN').
|
|
62
|
+
opt (int): If 2, each value in params is a list and index 0 is used;
|
|
63
|
+
otherwise the value is used directly.
|
|
64
|
+
addhost (int): If 1, also set the hostname option from params['HN'].
|
|
65
|
+
"""
|
|
42
66
|
if 'QS' in params: self.BOPTIONS['qoptions'] = (params['QS'][0] if opt == 2 else params['QS'])
|
|
43
67
|
if 'MO' in params: self.BOPTIONS['modules'] = (params['MO'][0] if opt == 2 else params['MO'])
|
|
44
68
|
if 'EV' in params: self.BOPTIONS['environments'] = (params['EV'][0] if opt == 2 else params['EV'])
|
|
45
69
|
if addhost and 'HN' in params: self.BOPTIONS['hostname'] = (params['HN'][0] if opt == 2 else params['HN'])
|
|
46
70
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
71
|
+
def fill_batch_options(self, boptions, refresh=0, checkkey=0):
|
|
72
|
+
"""Fills BOPTIONS from a dict of batch option values recorded in RDADB.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
boptions (dict): Dict mapping batch option field names to values.
|
|
76
|
+
refresh (int): If 1, reset all BOPTIONS values to None before filling.
|
|
77
|
+
checkkey (int): If 1, only copy keys that already exist in BOPTIONS.
|
|
78
|
+
"""
|
|
52
79
|
if refresh:
|
|
53
80
|
for bkey in self.BOPTIONS:
|
|
54
81
|
self.BOPTIONS[bkey] = None # clean the hash before filling it up
|
|
@@ -57,19 +84,33 @@ class PgCMD(PgLock):
|
|
|
57
84
|
if not checkkey or bkey in self.BOPTIONS:
|
|
58
85
|
self.BOPTIONS[bkey] = boptions[bkey]
|
|
59
86
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
87
|
+
def set_one_boption(self, bkey, bval, override=0):
|
|
88
|
+
"""Sets a single batch option value.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
bkey (str): Batch option field name (must be a key in BOPTIONS).
|
|
92
|
+
bval (str or None): The value to set.
|
|
93
|
+
override (int): If 1, overwrite an existing value; if 0, only set
|
|
94
|
+
when the option is currently unset. When bval is falsy and
|
|
95
|
+
override is 1, clears the existing value.
|
|
96
|
+
"""
|
|
65
97
|
if bval:
|
|
66
|
-
if override or not (
|
|
98
|
+
if override or not (bkey in self.BOPTIONS and self.BOPTIONS[bkey]): self.BOPTIONS[bkey] = bval
|
|
67
99
|
elif override and bkey in self.BOPTIONS and self.BOPTIONS[bkey]:
|
|
68
100
|
self.BOPTIONS[bkey] = None
|
|
69
101
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
102
|
+
def get_batch_options(self, pgrec=None):
|
|
103
|
+
"""Returns a dict of effective batch options, preferring pgrec values.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
pgrec (dict or None): Optional existing DB record whose non-empty
|
|
107
|
+
field values take precedence over the cached BOPTIONS.
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
dict: Mapping of option field names to their effective values.
|
|
111
|
+
Only fields with a non-None/non-empty value are included.
|
|
112
|
+
"""
|
|
113
|
+
record = {}
|
|
73
114
|
for bkey in self.BOPTIONS:
|
|
74
115
|
if pgrec and bkey in pgrec and pgrec[bkey]:
|
|
75
116
|
record[bkey] = pgrec[bkey]
|
|
@@ -77,15 +118,35 @@ class PgCMD(PgLock):
|
|
|
77
118
|
record[bkey] = self.BOPTIONS[bkey]
|
|
78
119
|
return record
|
|
79
120
|
|
|
80
|
-
# return delay mode option to append to argv string for a specified cmd
|
|
81
121
|
def append_delayed_mode(self, cmd, argv):
|
|
122
|
+
"""Returns the delayed-mode option string for cmd if not already in argv.
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
cmd (str): Command name to look up in DLYOPT.
|
|
126
|
+
argv (str): Current argument string to check for an existing flag.
|
|
127
|
+
|
|
128
|
+
Returns:
|
|
129
|
+
str: The delayed-mode option string (e.g. ' -d') if cmd is in DLYOPT
|
|
130
|
+
and no delayed-mode flag is already present; otherwise ''.
|
|
131
|
+
"""
|
|
82
132
|
if cmd in self.DLYOPT and not re.search(self.DLYPTN, argv, re.I):
|
|
83
133
|
return self.DLYOPT[cmd]
|
|
84
134
|
else:
|
|
85
135
|
return ''
|
|
86
136
|
|
|
87
|
-
# check given doptions and cmd, and return the try limit and specified hosts
|
|
88
137
|
def get_delay_options(self, doptions, cmd):
|
|
138
|
+
"""Parses delay options and returns the try count and optional host.
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
doptions (list or None): List of option strings; numeric strings set
|
|
142
|
+
the try count (capped at 99), non-numeric strings are treated as
|
|
143
|
+
a hostname override.
|
|
144
|
+
cmd (str): Command name used to look up the default try limit.
|
|
145
|
+
|
|
146
|
+
Returns:
|
|
147
|
+
tuple: A 2-tuple (mcount, hosts) where mcount (int) is the maximum
|
|
148
|
+
number of tries and hosts (str or None) is the hostname override.
|
|
149
|
+
"""
|
|
89
150
|
mcount = 0
|
|
90
151
|
hosts = None
|
|
91
152
|
if doptions:
|
|
@@ -99,8 +160,31 @@ class PgCMD(PgLock):
|
|
|
99
160
|
if hosts: self.set_one_boption('hostname', hosts, 1)
|
|
100
161
|
return (mcount, hosts)
|
|
101
162
|
|
|
102
|
-
|
|
103
|
-
|
|
163
|
+
def init_dscheck(self, oindex, otype, cmd, dsid, action, workdir=None, specialist=None, doptions=None, logact=0):
|
|
164
|
+
"""Finds or creates a dscheck record and prepares it for the current run.
|
|
165
|
+
|
|
166
|
+
Looks up an existing dscheck record matching the current command
|
|
167
|
+
invocation. If found and eligible to run, locks it and updates its
|
|
168
|
+
status to 'R'. If not found, inserts a new record and exits. Exits the
|
|
169
|
+
process in several terminal conditions (already running, finished,
|
|
170
|
+
lock failure).
|
|
171
|
+
|
|
172
|
+
Args:
|
|
173
|
+
oindex (int): Object index (e.g. rindex for dsrqst).
|
|
174
|
+
otype (str): Object type code (e.g. 'P', 'R', 'L').
|
|
175
|
+
cmd (str): Command name (e.g. 'dsrqst', 'dsarch').
|
|
176
|
+
dsid (str): Dataset identifier string.
|
|
177
|
+
action (str): Action code for this invocation.
|
|
178
|
+
workdir (str or None): Working directory; defaults to os.getcwd().
|
|
179
|
+
specialist (str or None): Specialist UID; defaults to PGLOG['CURUID'].
|
|
180
|
+
doptions (list or None): Delay options (try count and/or hostname).
|
|
181
|
+
logact (int): Logging action flags.
|
|
182
|
+
|
|
183
|
+
Returns:
|
|
184
|
+
int: The cindex (check index) of the locked dscheck record, or
|
|
185
|
+
exits the process if the record was newly created or a terminal
|
|
186
|
+
condition was reached.
|
|
187
|
+
"""
|
|
104
188
|
cidx = 0
|
|
105
189
|
argv = self.argv_to_string(sys.argv[1:], 0, "Process in Delayed Mode")
|
|
106
190
|
argextra = None
|
|
@@ -114,7 +198,7 @@ class PgCMD(PgLock):
|
|
|
114
198
|
bck = self.PGLOG['BCKGRND']
|
|
115
199
|
self.PGLOG['BCKGRND'] = 0
|
|
116
200
|
cinfo = "{}-{}-Chk".format(self.PGLOG['HOSTNAME'], self.current_datetime())
|
|
117
|
-
pgrec = self.get_dscheck(cmd, argv, workdir, specialist, argextra, logact)
|
|
201
|
+
pgrec = self.get_dscheck(cmd, argv, workdir, specialist, argextra, logact)
|
|
118
202
|
if pgrec: # found existing dscheck record
|
|
119
203
|
cidx = pgrec['cindex']
|
|
120
204
|
cmsg = "{}{}: {} batch process ".format(cinfo, cidx, self.get_command_info(pgrec))
|
|
@@ -135,7 +219,7 @@ class PgCMD(PgLock):
|
|
|
135
219
|
sys.exit(0)
|
|
136
220
|
if not cidx: # add new dscheck record
|
|
137
221
|
record = {}
|
|
138
|
-
if hosts and re.match(r'
|
|
222
|
+
if hosts and re.match(r'^\w\d\d\d\d\d\d$', hosts):
|
|
139
223
|
self.pglog(hosts + ": Cannot pass DSID for hostname to submit batch process", self.LGEREX)
|
|
140
224
|
if oindex: self.set_command_control(oindex, otype, cmd, logact)
|
|
141
225
|
record['oindex'] = oindex
|
|
@@ -155,14 +239,14 @@ class PgCMD(PgLock):
|
|
|
155
239
|
cmsg = "{}{}: {} Adds a new check".format(cinfo, cidx, self.get_command_info(record))
|
|
156
240
|
self.pglog(cmsg, self.LOGWRN)
|
|
157
241
|
sys.exit(0)
|
|
158
|
-
|
|
242
|
+
|
|
159
243
|
(chost, cpid) = self.current_process_info()
|
|
160
244
|
(rhost, rpid) = self.current_process_info(1)
|
|
161
|
-
|
|
245
|
+
|
|
162
246
|
if not self.check_command_specialist_host(hosts, chost, specialist, cmd, action, self.LOGERR):
|
|
163
247
|
self.lock_dscheck(cidx, 0, logact)
|
|
164
248
|
sys.exit(1)
|
|
165
|
-
|
|
249
|
+
|
|
166
250
|
record = {}
|
|
167
251
|
record['status'] = "R"
|
|
168
252
|
if mcount > 0: record['mcount'] = mcount
|
|
@@ -196,8 +280,21 @@ class PgCMD(PgLock):
|
|
|
196
280
|
self.PGLOG['BCKGRND'] = bck
|
|
197
281
|
return cidx
|
|
198
282
|
|
|
199
|
-
|
|
200
|
-
|
|
283
|
+
def check_command_specialist_host(self, hosts, chost, specialist, cmd, act=0, logact=0):
|
|
284
|
+
"""Checks whether the current host is configured to run cmd for specialist.
|
|
285
|
+
|
|
286
|
+
Args:
|
|
287
|
+
hosts (str or None): Allowed hostname pattern or None.
|
|
288
|
+
chost (str): Current host name.
|
|
289
|
+
specialist (str): Specialist UID.
|
|
290
|
+
cmd (str): Command name.
|
|
291
|
+
act (str or int): Action code; 'PR' triggers global match flag.
|
|
292
|
+
logact (int): Logging action flags.
|
|
293
|
+
|
|
294
|
+
Returns:
|
|
295
|
+
bool: True if the current host is permitted to run the command,
|
|
296
|
+
False otherwise.
|
|
297
|
+
"""
|
|
201
298
|
if cmd == 'dsrqst' and act == 'PR':
|
|
202
299
|
mflag = 'G'
|
|
203
300
|
else:
|
|
@@ -206,8 +303,18 @@ class PgCMD(PgLock):
|
|
|
206
303
|
mflag = (pgrec['matchhost'] if pgrec else 'G')
|
|
207
304
|
return self.check_process_host(hosts, chost, mflag, "{}-{}".format(specialist, cmd), logact)
|
|
208
305
|
|
|
209
|
-
|
|
210
|
-
|
|
306
|
+
def set_command_control(self, oindex, otype, cmd, logact=0):
|
|
307
|
+
"""Retrieves and applies batch control options from the database for cmd.
|
|
308
|
+
|
|
309
|
+
Looks up control records (e.g. rcrqst, dcupdt) associated with the
|
|
310
|
+
given object and merges their batch option fields into BOPTIONS.
|
|
311
|
+
|
|
312
|
+
Args:
|
|
313
|
+
oindex (int): Object index.
|
|
314
|
+
otype (str): Object type code.
|
|
315
|
+
cmd (str): Command name ('dsrqst' or 'dsupdt').
|
|
316
|
+
logact (int): Logging action flags.
|
|
317
|
+
"""
|
|
211
318
|
if not oindex: return
|
|
212
319
|
pgctl = None
|
|
213
320
|
if cmd == "dsrqst":
|
|
@@ -219,7 +326,7 @@ class PgCMD(PgLock):
|
|
|
219
326
|
if pgrec: pgctl = self.get_dsrqst_control(pgrec, logact)
|
|
220
327
|
elif cmd == "dsupdt":
|
|
221
328
|
if otype == 'L':
|
|
222
|
-
pgrec = self.pgget("dlupdt", "cindex", "lindex = {}".format(oindex),
|
|
329
|
+
pgrec = self.pgget("dlupdt", "cindex", "lindex = {}".format(oindex), logact)
|
|
223
330
|
if not (pgrec and pgrec['cindex']): return
|
|
224
331
|
oindex = pgrec['cindex']
|
|
225
332
|
pgctl = self.pgget("dcupdt", self.BFIELDS, "cindex = {}".format(oindex), logact)
|
|
@@ -227,8 +334,20 @@ class PgCMD(PgLock):
|
|
|
227
334
|
for bkey in pgctl:
|
|
228
335
|
self.set_one_boption(bkey, pgctl[bkey], 0)
|
|
229
336
|
|
|
230
|
-
|
|
231
|
-
|
|
337
|
+
def get_dsrqst_control(self, pgrqst, logact=0):
|
|
338
|
+
"""Retrieves the batch control record for a dsrqst request.
|
|
339
|
+
|
|
340
|
+
Walks the group hierarchy (via pindex) until a matching rcrqst record
|
|
341
|
+
is found or the root is reached.
|
|
342
|
+
|
|
343
|
+
Args:
|
|
344
|
+
pgrqst (dict): A dsrqst record containing at least dsid, gindex,
|
|
345
|
+
cindex, and rqsttype fields.
|
|
346
|
+
logact (int): Logging action flags.
|
|
347
|
+
|
|
348
|
+
Returns:
|
|
349
|
+
dict or None: The rcrqst control record, or None if not found.
|
|
350
|
+
"""
|
|
232
351
|
cflds = self.BFIELDS
|
|
233
352
|
if 'ptcount' in pgrqst and pgrqst['ptcount'] == 0: cflds += ", ptlimit, ptsize"
|
|
234
353
|
if pgrqst['cindex']:
|
|
@@ -250,26 +369,52 @@ class PgCMD(PgLock):
|
|
|
250
369
|
gindex = pgctl['pindex']
|
|
251
370
|
return pgctl
|
|
252
371
|
|
|
253
|
-
|
|
254
|
-
|
|
372
|
+
def get_partition_control(self, pgpart, pgrqst=None, pgctl=None, logact=0):
|
|
373
|
+
"""Retrieves batch control info for a dsrqst partition record.
|
|
374
|
+
|
|
375
|
+
Args:
|
|
376
|
+
pgpart (dict): Partition record containing at least rindex.
|
|
377
|
+
pgrqst (dict or None): Parent dsrqst record; fetched automatically
|
|
378
|
+
from pgpart['rindex'] if not provided.
|
|
379
|
+
pgctl (dict or None): Pre-fetched control record; skips DB lookup
|
|
380
|
+
if provided.
|
|
381
|
+
logact (int): Logging action flags.
|
|
382
|
+
|
|
383
|
+
Returns:
|
|
384
|
+
dict or None: The rcrqst control record, or None if not found.
|
|
385
|
+
"""
|
|
255
386
|
if not pgctl:
|
|
256
387
|
if not pgrqst and pgpart['rindex']:
|
|
257
388
|
pgrqst = self.pgget("dsrqst", "dsid, gindex, cindex, rqsttype", "rindex = {}".format(pgpart['rindex']), logact)
|
|
258
389
|
if pgrqst: pgctl = self.get_dsrqst_control(pgrqst, logact)
|
|
259
390
|
return pgctl
|
|
260
391
|
|
|
261
|
-
# build the dynamic options
|
|
262
392
|
def get_dynamic_options(self, cmd, oindex, otype):
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
393
|
+
"""Runs cmd to retrieve dynamic option strings, retrying on timeout.
|
|
394
|
+
|
|
395
|
+
Executes the command up to three times, retrying when a connection
|
|
396
|
+
timeout is detected in PGLOG['SYSERR']. Parses the output to extract
|
|
397
|
+
option strings in 'read/write' format (-opt1/-opt2) based on otype.
|
|
398
|
+
|
|
399
|
+
Args:
|
|
400
|
+
cmd (str): Base command to execute.
|
|
401
|
+
oindex (int or None): Object index appended to cmd when truthy.
|
|
402
|
+
otype (str or None): Object type appended to cmd when truthy;
|
|
403
|
+
'R' selects the first option from a slash-separated pair.
|
|
404
|
+
|
|
405
|
+
Returns:
|
|
406
|
+
str: The parsed option string, or '' if the command produced no
|
|
407
|
+
usable output (error details are appended to PGLOG['SYSERR']).
|
|
408
|
+
"""
|
|
409
|
+
if oindex: cmd += " {}".format(oindex)
|
|
410
|
+
if otype: cmd += ' ' + otype
|
|
266
411
|
ret = options = ''
|
|
267
412
|
for loop in range(3):
|
|
268
413
|
ret = self.pgsystem(cmd, self.LOGWRN, 1299) # 1+2+16+256+1024
|
|
269
414
|
if loop < 2 and self.PGLOG['SYSERR'] and 'Connection timed out' in self.PGLOG['SYSERR']:
|
|
270
415
|
time.sleep(self.PGSIG['ETIME'])
|
|
271
416
|
else:
|
|
272
|
-
break
|
|
417
|
+
break
|
|
273
418
|
if ret:
|
|
274
419
|
ret = ret.strip()
|
|
275
420
|
ms = re.match(r'^(-.+)/(-.+)$', ret)
|
|
@@ -278,13 +423,29 @@ class PgCMD(PgLock):
|
|
|
278
423
|
elif re.match(r'^(-.+)$', ret):
|
|
279
424
|
options = ret
|
|
280
425
|
if not options:
|
|
281
|
-
if ret: self.PGLOG['SYSERR']
|
|
282
|
-
self.PGLOG['SYSERR']
|
|
283
|
-
|
|
426
|
+
if ret: self.PGLOG['SYSERR'] = (self.PGLOG['SYSERR'] or '') + ret
|
|
427
|
+
self.PGLOG['SYSERR'] = (self.PGLOG['SYSERR'] or '') + " for {}".format(cmd)
|
|
428
|
+
|
|
284
429
|
return options
|
|
285
430
|
|
|
286
|
-
|
|
287
|
-
|
|
431
|
+
def get_dscheck(self, cmd, argv, workdir, specialist, argextra=None, logact=0):
|
|
432
|
+
"""Retrieves a dscheck record matching the given command invocation.
|
|
433
|
+
|
|
434
|
+
Searches for an existing dscheck record by command, specialist, argv,
|
|
435
|
+
workdir, and argextra. Also tries variants with the delayed-mode option
|
|
436
|
+
appended or stripped from argv.
|
|
437
|
+
|
|
438
|
+
Args:
|
|
439
|
+
cmd (str): Command name.
|
|
440
|
+
argv (str): Argument string (truncated to 100 chars).
|
|
441
|
+
workdir (str): Working directory path.
|
|
442
|
+
specialist (str): Specialist UID.
|
|
443
|
+
argextra (str or None): Remainder of argv beyond 100 chars.
|
|
444
|
+
logact (int): Logging action flags.
|
|
445
|
+
|
|
446
|
+
Returns:
|
|
447
|
+
dict or None: The matching dscheck record, or None if not found.
|
|
448
|
+
"""
|
|
288
449
|
cnd = "command = '{}' AND specialist = '{}' AND argv = '{}'".format(cmd, specialist, argv)
|
|
289
450
|
pgrecs = self.pgmget("dscheck", "*", cnd, logact)
|
|
290
451
|
cnt = len(pgrecs['cindex']) if pgrecs else 0
|
|
@@ -305,12 +466,28 @@ class PgCMD(PgLock):
|
|
|
305
466
|
for i in range(cnt):
|
|
306
467
|
pgrec = self.onerecord(pgrecs, i)
|
|
307
468
|
if pgrec['workdir'] and self.pgcmp(workdir, pgrec['workdir']): continue
|
|
308
|
-
if self.pgcmp(argextra, pgrec['argextra']): continue
|
|
469
|
+
if self.pgcmp(argextra, pgrec['argextra']): continue
|
|
309
470
|
return pgrec
|
|
310
471
|
return None
|
|
311
472
|
|
|
312
|
-
|
|
313
|
-
|
|
473
|
+
def delete_dscheck(self, pgrec, chkcnd, logact=0):
|
|
474
|
+
"""Archives and deletes a dscheck record into dschkhist.
|
|
475
|
+
|
|
476
|
+
Copies the dscheck record to dschkhist (inserting or updating), logs
|
|
477
|
+
the cleanup, deletes the original dscheck row, and logs any error
|
|
478
|
+
message when the final status is 'E'.
|
|
479
|
+
|
|
480
|
+
Args:
|
|
481
|
+
pgrec (dict or None): The dscheck record to delete; fetched from DB
|
|
482
|
+
using chkcnd if None.
|
|
483
|
+
chkcnd (str or None): SQL WHERE condition identifying the record;
|
|
484
|
+
derived from pgrec or DSCHK['chkcnd'] if None.
|
|
485
|
+
logact (int): Logging action flags.
|
|
486
|
+
|
|
487
|
+
Returns:
|
|
488
|
+
int: The return value of pgdel (non-zero on success), or 0 if there
|
|
489
|
+
was nothing to delete or the record was already gone.
|
|
490
|
+
"""
|
|
314
491
|
if not chkcnd:
|
|
315
492
|
if pgrec:
|
|
316
493
|
chkcnd = "cindex = {}".format(pgrec['cindex'])
|
|
@@ -349,7 +526,7 @@ class PgCMD(PgLock):
|
|
|
349
526
|
if self.pgget("dschkhist", "", chkcnd):
|
|
350
527
|
stat = self.pgupdt("dschkhist", record, chkcnd, logact)
|
|
351
528
|
else:
|
|
352
|
-
stat = self.pgadd("dschkhist", record, logact)
|
|
529
|
+
stat = self.pgadd("dschkhist", record, logact)
|
|
353
530
|
if stat:
|
|
354
531
|
cmsg = "{} cleaned as '{}' at {} on {}".format(self.get_command_info(pgrec), record['status'], self.current_datetime(), self.PGLOG['HOSTNAME'])
|
|
355
532
|
self.pglog("Chk{}: {}".format(pgrec['cindex'], cmsg), self.LOGWRN|self.FRCLOG)
|
|
@@ -358,8 +535,20 @@ class PgCMD(PgLock):
|
|
|
358
535
|
self.pglog("Chk{}: {} Exits with Error\n{}".format(pgrec['cindex'], self.get_command_info(pgrec), record['errmsg']), logact)
|
|
359
536
|
return stat
|
|
360
537
|
|
|
361
|
-
|
|
362
|
-
|
|
538
|
+
def get_dsrqst_counts(self, pgchk, logact=0):
|
|
539
|
+
"""Retrieves up-to-date fcount, dcount, and size for a dsrqst check.
|
|
540
|
+
|
|
541
|
+
Fetches current file counts and size from dsrqst or ptrqst and wfrqst
|
|
542
|
+
tables, falling back to the values already stored in pgchk.
|
|
543
|
+
|
|
544
|
+
Args:
|
|
545
|
+
pgchk (dict): A dscheck record with at least oindex, otype, fcount,
|
|
546
|
+
dcount, and size fields.
|
|
547
|
+
logact (int): Logging action flags.
|
|
548
|
+
|
|
549
|
+
Returns:
|
|
550
|
+
tuple: A 3-tuple (fcount, dcount, size) with the most current values.
|
|
551
|
+
"""
|
|
363
552
|
fcount = pgchk['fcount']
|
|
364
553
|
dcount = pgchk['dcount']
|
|
365
554
|
size = pgchk['size']
|
|
@@ -395,23 +584,49 @@ class PgCMD(PgLock):
|
|
|
395
584
|
if pgrec and pgrec['data_size']: size = pgrec['data_size']
|
|
396
585
|
return (fcount, dcount, size)
|
|
397
586
|
|
|
398
|
-
|
|
399
|
-
|
|
587
|
+
def set_dscheck_fcount(self, count, logact=0):
|
|
588
|
+
"""Updates the fcount field of the active dscheck record.
|
|
589
|
+
|
|
590
|
+
Args:
|
|
591
|
+
count (int): New file count value to store.
|
|
592
|
+
logact (int): Logging action flags.
|
|
593
|
+
|
|
594
|
+
Returns:
|
|
595
|
+
int: The current dcount value from the cached DSCHK dict.
|
|
596
|
+
"""
|
|
400
597
|
record = {'fcount': count, 'chktime': int(time.time())}
|
|
401
598
|
self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
402
599
|
self.DSCHK['fcount'] = count
|
|
403
600
|
return self.DSCHK['dcount'] # return Done count
|
|
404
601
|
|
|
405
|
-
|
|
406
|
-
|
|
602
|
+
def set_dscheck_dcount(self, count, size, logact=0):
|
|
603
|
+
"""Updates the dcount and size fields of the active dscheck record.
|
|
604
|
+
|
|
605
|
+
Args:
|
|
606
|
+
count (int): New done-file count value to store.
|
|
607
|
+
size (int): New total size value to store.
|
|
608
|
+
logact (int): Logging action flags.
|
|
609
|
+
|
|
610
|
+
Returns:
|
|
611
|
+
int: The current dcount value from the cached DSCHK dict.
|
|
612
|
+
"""
|
|
407
613
|
record = {'dcount': count, 'size': size, 'chktime': int(time.time())}
|
|
408
614
|
self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
409
615
|
self.DSCHK['dcount'] = count
|
|
410
616
|
self.DSCHK['size'] = size
|
|
411
617
|
return self.DSCHK['dcount'] # return Done count
|
|
412
618
|
|
|
413
|
-
|
|
414
|
-
|
|
619
|
+
def add_dscheck_dcount(self, count, size, logact=0):
|
|
620
|
+
"""Increments the dcount and size fields of the active dscheck record.
|
|
621
|
+
|
|
622
|
+
Args:
|
|
623
|
+
count (int): Amount to add to the current dcount.
|
|
624
|
+
size (int): Amount to add to the current size.
|
|
625
|
+
logact (int): Logging action flags.
|
|
626
|
+
|
|
627
|
+
Returns:
|
|
628
|
+
int: The updated dcount value from the cached DSCHK dict.
|
|
629
|
+
"""
|
|
415
630
|
record = {}
|
|
416
631
|
if count:
|
|
417
632
|
self.DSCHK['dcount'] += count
|
|
@@ -423,42 +638,86 @@ class PgCMD(PgLock):
|
|
|
423
638
|
self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
424
639
|
return self.DSCHK['dcount'] # return Done count
|
|
425
640
|
|
|
426
|
-
|
|
427
|
-
|
|
641
|
+
def set_dscheck_attribute(self, fname, value, logact=0):
|
|
642
|
+
"""Updates a single named field of the active dscheck record.
|
|
643
|
+
|
|
644
|
+
Args:
|
|
645
|
+
fname (str): Field name to update in dscheck.
|
|
646
|
+
value: Value to store; if falsy, only chktime is updated.
|
|
647
|
+
logact (int): Logging action flags.
|
|
648
|
+
"""
|
|
428
649
|
record = {}
|
|
429
650
|
if value: record[fname] = value
|
|
430
651
|
record['chktime'] = int(time.time())
|
|
431
652
|
self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
432
653
|
|
|
433
|
-
|
|
434
|
-
|
|
654
|
+
def record_dscheck_status(self, stat, logact=0):
|
|
655
|
+
"""Updates the dscheck status field if the record is still locked by this process.
|
|
656
|
+
|
|
657
|
+
Verifies that the current process still holds the lock on the dscheck
|
|
658
|
+
record before applying the status update, preventing stale updates.
|
|
659
|
+
|
|
660
|
+
Args:
|
|
661
|
+
stat (str): Status code to write (e.g. 'D', 'E').
|
|
662
|
+
logact (int): Logging action flags.
|
|
663
|
+
|
|
664
|
+
Returns:
|
|
665
|
+
int: Return value of pgupdt (non-zero on success), or 0 if the
|
|
666
|
+
record is missing, unlocked, or locked by a different process.
|
|
667
|
+
"""
|
|
435
668
|
pgrec = self.pgget("dscheck", "lockhost, pid", self.DSCHK['chkcnd'], logact)
|
|
436
669
|
if not pgrec: return 0
|
|
437
670
|
if not (pgrec['pid'] and pgrec['lockhost']): return 0
|
|
438
671
|
(chost, cpid) = self.current_process_info()
|
|
439
672
|
if pgrec['pid'] != cpid or pgrec['lockhost'] != chost: return 0
|
|
440
|
-
|
|
673
|
+
|
|
441
674
|
# update dscheck status only if it is still locked by the current process
|
|
442
675
|
record = {'status': stat, 'chktime': int(time.time()), 'pid': 0}
|
|
443
676
|
return self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
444
677
|
|
|
445
|
-
# get the number of tries to execute for a given cmd under dscheck control
|
|
446
678
|
def get_try_limit(self, cmd):
|
|
679
|
+
"""Returns the maximum number of execution attempts for a given command.
|
|
680
|
+
|
|
681
|
+
Args:
|
|
682
|
+
cmd (str): Command name to look up in TRYLMTS.
|
|
683
|
+
|
|
684
|
+
Returns:
|
|
685
|
+
int: The per-command try limit, or the 'default' value if cmd is
|
|
686
|
+
not explicitly listed in TRYLMTS.
|
|
687
|
+
"""
|
|
447
688
|
return (self.TRYLMTS[cmd] if cmd in self.TRYLMTS else self.TRYLMTS['default'])
|
|
448
689
|
|
|
449
|
-
# get the execution time for a dscheck command
|
|
450
690
|
@staticmethod
|
|
451
|
-
def get_dscheck_runtime(pgrec, current
|
|
691
|
+
def get_dscheck_runtime(pgrec, current=0):
|
|
692
|
+
"""Calculates cumulative total and queue times for a dscheck record.
|
|
693
|
+
|
|
694
|
+
Args:
|
|
695
|
+
pgrec (dict): A dscheck record containing subtime, stttime, chktime,
|
|
696
|
+
ttltime, and quetime fields.
|
|
697
|
+
current (int): If 1, ignore previously accumulated times and compute
|
|
698
|
+
only from the current run; if 0, add to existing accumulated totals.
|
|
699
|
+
|
|
700
|
+
Returns:
|
|
701
|
+
tuple: A 2-tuple (ttltime, quetime) of accumulated integer seconds.
|
|
702
|
+
"""
|
|
452
703
|
ttime = (0 if current else pgrec['ttltime'])
|
|
453
704
|
qtime = (0 if current else pgrec['quetime'])
|
|
454
705
|
if pgrec['subtime']:
|
|
455
706
|
ttime += (pgrec['chktime'] - pgrec['subtime'])
|
|
456
|
-
if pgrec['stttime']: qtime += (pgrec['stttime'] - pgrec['subtime'])
|
|
707
|
+
if pgrec['stttime']: qtime += (pgrec['stttime'] - pgrec['subtime'])
|
|
457
708
|
return (ttime, qtime)
|
|
458
709
|
|
|
459
|
-
# retrieve a command string from a given dscheck record
|
|
460
710
|
@staticmethod
|
|
461
711
|
def get_command_info(pgrec):
|
|
712
|
+
"""Builds a human-readable command identifier string from a dscheck record.
|
|
713
|
+
|
|
714
|
+
Args:
|
|
715
|
+
pgrec (dict): A dscheck record containing at least command, oindex,
|
|
716
|
+
otype, dsid, action, and specialist fields.
|
|
717
|
+
|
|
718
|
+
Returns:
|
|
719
|
+
str: A concise string such as 'Rqst42 d123000 ST of jsmith'.
|
|
720
|
+
"""
|
|
462
721
|
if pgrec['oindex']:
|
|
463
722
|
if pgrec['command'] == "dsupdt":
|
|
464
723
|
cinfo = "UC{}".format(pgrec['oindex'])
|
|
@@ -468,7 +727,7 @@ class PgCMD(PgLock):
|
|
|
468
727
|
else:
|
|
469
728
|
cinfo = "Rqst{}".format(pgrec['oindex'])
|
|
470
729
|
else:
|
|
471
|
-
cinfo ="{}-{}".format(pgrec['command'], pgrec['oindex'])
|
|
730
|
+
cinfo = "{}-{}".format(pgrec['command'], pgrec['oindex'])
|
|
472
731
|
else:
|
|
473
732
|
cinfo = pgrec['command']
|
|
474
733
|
if pgrec['dsid']: cinfo += " " + pgrec['dsid']
|
|
@@ -476,8 +735,22 @@ class PgCMD(PgLock):
|
|
|
476
735
|
cinfo += " of " + pgrec['specialist']
|
|
477
736
|
return cinfo
|
|
478
737
|
|
|
479
|
-
# change the dscheck original command information
|
|
480
738
|
def change_dscheck_oinfo(self, oidx, otype, nidx, ntype):
|
|
739
|
+
"""Updates the oindex and otype fields of a dscheck record.
|
|
740
|
+
|
|
741
|
+
Finds the dscheck record matching oidx/otype and replaces the object
|
|
742
|
+
reference with nidx/ntype, also updating the cached DSCHK dict.
|
|
743
|
+
|
|
744
|
+
Args:
|
|
745
|
+
oidx (int): Current object index to match.
|
|
746
|
+
otype (str): Current object type to match.
|
|
747
|
+
nidx (int): New object index value.
|
|
748
|
+
ntype (str): New object type value.
|
|
749
|
+
|
|
750
|
+
Returns:
|
|
751
|
+
int: Return value of pgupdt (non-zero on success), or 0 if no
|
|
752
|
+
matching dscheck record was found.
|
|
753
|
+
"""
|
|
481
754
|
cnd = "oindex = {} AND otype = '{}'".format(oidx, otype)
|
|
482
755
|
pgchk = self.pgget('dscheck', 'cindex, oindex, otype', cnd, self.LGEREX)
|
|
483
756
|
if not pgchk: return 0 # miss dscheck record to change
|