rda-python-common 2.0.10__tar.gz → 2.0.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/PKG-INFO +1 -1
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/pyproject.toml +1 -1
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/PgDBI.py +2 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pg_cmd.py +16 -21
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pg_dbi.py +59 -62
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pg_file.py +44 -49
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pg_lock.py +6 -12
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pg_log.py +87 -96
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pg_opt.py +84 -90
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pg_sig.py +23 -29
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pg_split.py +14 -21
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pg_util.py +15 -20
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pgpassword.py +10 -15
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/PKG-INFO +1 -1
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/LICENSE +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/README.md +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/setup.cfg +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/PgCMD.py +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/PgFile.py +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/PgLOG.py +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/PgLock.py +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/PgOPT.py +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/PgSIG.py +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/PgSplit.py +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/PgUtil.py +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/__init__.py +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common/pg_password.py +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/SOURCES.txt +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/dependency_links.txt +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/entry_points.txt +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/requires.txt +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/top_level.txt +0 -0
- {rda_python_common-2.0.10 → rda_python_common-2.0.14}/test/test_common.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rda_python_common
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.14
|
|
4
4
|
Summary: RDA Python common library codes shared by other RDA python packages
|
|
5
5
|
Author-email: Zaihua Ji <zji@ucar.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/NCAR/rda-python-common
|
|
@@ -468,6 +468,8 @@ def check_dberror(pgerr, pgcnt, sqlstr, ary, logact = PGDBI['ERRLOG']):
|
|
|
468
468
|
|
|
469
469
|
if logact&PgLOG.DOLOCK and pgcode and re.match(r'^55\w\w\w$', pgcode):
|
|
470
470
|
logact &= ~PgLOG.EXITLG # no exit for lock error
|
|
471
|
+
elif pgcnt > PgLOG.PGLOG['DBRETRY']:
|
|
472
|
+
logact |= PgLOG.EXITLG # exit for error count exceeds limit
|
|
471
473
|
return qelog(dberror, 0, sqlstr, ary, pgcnt, logact)
|
|
472
474
|
|
|
473
475
|
#
|
|
@@ -1,18 +1,13 @@
|
|
|
1
|
-
#
|
|
2
1
|
###############################################################################
|
|
3
|
-
#
|
|
4
|
-
#
|
|
5
|
-
#
|
|
6
|
-
# Date : 08/25/2020
|
|
2
|
+
# Title: pg_cmd.py
|
|
3
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
4
|
+
# Date: 08/25/2020
|
|
7
5
|
# 2025-01-10 transferred to package rda_python_common from
|
|
8
6
|
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
|
-
# Purpose
|
|
7
|
+
# Purpose: python library module for functions to record commands for delayed
|
|
10
8
|
# mode or command recovery
|
|
11
|
-
#
|
|
12
|
-
# Github : https://github.com/NCAR/rda-python-common.git
|
|
13
|
-
#
|
|
9
|
+
# Github: https://github.com/NCAR/rda-python-common.git
|
|
14
10
|
###############################################################################
|
|
15
|
-
#
|
|
16
11
|
import os
|
|
17
12
|
import re
|
|
18
13
|
import sys
|
|
@@ -25,18 +20,18 @@ class PgCMD(PgLock):
|
|
|
25
20
|
super().__init__() # initialize parent class
|
|
26
21
|
# cached dscheck info
|
|
27
22
|
self.DSCHK = {}
|
|
28
|
-
self.BOPTIONS = {"hostname"
|
|
23
|
+
self.BOPTIONS = {"hostname": None, "qoptions": None, "modules": None, "environments": None}
|
|
29
24
|
self.BFIELDS = ', '.join(self.BOPTIONS)
|
|
30
25
|
self.TRYLMTS = {
|
|
31
|
-
'dsquasar'
|
|
32
|
-
'dsarch'
|
|
33
|
-
'default'
|
|
26
|
+
'dsquasar': 3,
|
|
27
|
+
'dsarch': 2,
|
|
28
|
+
'default': 1
|
|
34
29
|
}
|
|
35
30
|
self.DLYPTN = r'(^|\s)-(d|BP|BatchProcess|DelayedMode)(\s|$)'
|
|
36
31
|
self.DLYOPT = {
|
|
37
|
-
'dsarch'
|
|
38
|
-
'dsupdt'
|
|
39
|
-
'dsrqst'
|
|
32
|
+
'dsarch': ' -d',
|
|
33
|
+
'dsupdt': ' -d',
|
|
34
|
+
'dsrqst': ' -d'
|
|
40
35
|
}
|
|
41
36
|
|
|
42
37
|
# params: dict array holding option values
|
|
@@ -381,7 +376,7 @@ class PgCMD(PgLock):
|
|
|
381
376
|
fcnt = pgrec['fcount']
|
|
382
377
|
else:
|
|
383
378
|
fcnt = 0
|
|
384
|
-
pgrec = {'fcount'
|
|
379
|
+
pgrec = {'fcount': 0}
|
|
385
380
|
if not fcnt: fcnt = self.pgget("wfrqst", "", cnd, logact)
|
|
386
381
|
if fcnt and fcount != fcnt: fcount = fcnt
|
|
387
382
|
if fcount:
|
|
@@ -402,14 +397,14 @@ class PgCMD(PgLock):
|
|
|
402
397
|
|
|
403
398
|
# set dscheck fcount
|
|
404
399
|
def set_dscheck_fcount(self, count, logact = 0):
|
|
405
|
-
record = {'fcount'
|
|
400
|
+
record = {'fcount': count, 'chktime': int(time.time())}
|
|
406
401
|
self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
407
402
|
self.DSCHK['fcount'] = count
|
|
408
403
|
return self.DSCHK['dcount'] # return Done count
|
|
409
404
|
|
|
410
405
|
# set dscheck dcount
|
|
411
406
|
def set_dscheck_dcount(self, count, size, logact = 0):
|
|
412
|
-
record = {'dcount'
|
|
407
|
+
record = {'dcount': count, 'size': size, 'chktime': int(time.time())}
|
|
413
408
|
self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
414
409
|
self.DSCHK['dcount'] = count
|
|
415
410
|
self.DSCHK['size'] = size
|
|
@@ -444,7 +439,7 @@ class PgCMD(PgLock):
|
|
|
444
439
|
if pgrec['pid'] != cpid or pgrec['lockhost'] != chost: return 0
|
|
445
440
|
|
|
446
441
|
# update dscheck status only if it is still locked by the current process
|
|
447
|
-
record = {'status'
|
|
442
|
+
record = {'status': stat, 'chktime': int(time.time()), 'pid': 0}
|
|
448
443
|
return self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
449
444
|
|
|
450
445
|
# get the number of tries to execute for a given cmd under dscheck control
|
|
@@ -1,18 +1,13 @@
|
|
|
1
|
-
#
|
|
2
1
|
###############################################################################
|
|
3
|
-
#
|
|
4
|
-
#
|
|
5
|
-
#
|
|
6
|
-
# Date : 06/07/2022
|
|
2
|
+
# Title: pg_dbi.py -- PostgreSQL DataBase Interface
|
|
3
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
4
|
+
# Date: 06/07/2022
|
|
7
5
|
# 2025-01-10 transferred to package rda_python_common from
|
|
8
6
|
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
7
|
# 2025-11-24 convert to class PgDBI
|
|
10
|
-
# Purpose
|
|
11
|
-
#
|
|
12
|
-
# Github : https://github.com/NCAR/rda-python-common.git
|
|
13
|
-
#
|
|
8
|
+
# Purpose: Python library module to handle query and manipulate PostgreSQL database
|
|
9
|
+
# Github: https://github.com/NCAR/rda-python-common.git
|
|
14
10
|
###############################################################################
|
|
15
|
-
|
|
16
11
|
import os
|
|
17
12
|
import re
|
|
18
13
|
import time
|
|
@@ -49,22 +44,22 @@ class PgDBI(PgLOG):
|
|
|
49
44
|
self.PGSIGNS = ['!', '<', '>', '<>']
|
|
50
45
|
self.CHCODE = 1042
|
|
51
46
|
# hard coded db ports for dbnames
|
|
52
|
-
self.DBPORTS = {'default'
|
|
47
|
+
self.DBPORTS = {'default': 0}
|
|
53
48
|
self.DBPASS = {}
|
|
54
49
|
self.DBBAOS = {}
|
|
55
50
|
# hard coded db names for given schema names
|
|
56
51
|
self.DBNAMES = {
|
|
57
|
-
'ivaddb'
|
|
58
|
-
'cntldb'
|
|
59
|
-
'cdmsdb'
|
|
60
|
-
'ispddb'
|
|
61
|
-
'obsua'
|
|
62
|
-
'default'
|
|
52
|
+
'ivaddb': 'ivaddb',
|
|
53
|
+
'cntldb': 'ivaddb',
|
|
54
|
+
'cdmsdb': 'ivaddb',
|
|
55
|
+
'ispddb': 'ispddb',
|
|
56
|
+
'obsua': 'upadb',
|
|
57
|
+
'default': 'rdadb',
|
|
63
58
|
}
|
|
64
59
|
# hard coded socket paths for machine_dbnames
|
|
65
|
-
self.DBSOCKS = {'default'
|
|
60
|
+
self.DBSOCKS = {'default': ''}
|
|
66
61
|
# home path for check db on alter host
|
|
67
|
-
self.VIEWHOMES = {'default'
|
|
62
|
+
self.VIEWHOMES = {'default': self.PGLOG['DSSDBHM']}
|
|
68
63
|
# add more to the list if used for names
|
|
69
64
|
self.PGRES = ['end', 'window']
|
|
70
65
|
self.SETPGDBI('DEFDB', 'rdadb')
|
|
@@ -366,6 +361,8 @@ class PgDBI(PgLOG):
|
|
|
366
361
|
return self.SUCCESS
|
|
367
362
|
if logact&self.DOLOCK and pgcode and re.match(r'^55\w\w\w$', pgcode):
|
|
368
363
|
logact &= ~self.EXITLG # no exit for lock error
|
|
364
|
+
elif pgcnt > self.PGLOG['DBRETRY']:
|
|
365
|
+
logact |= self.EXITLG # exit for error count exceeds limit
|
|
369
366
|
return self.qelog(dberror, 0, sqlstr, ary, pgcnt, logact)
|
|
370
367
|
|
|
371
368
|
# return hash reference to postgresql batch mode command and output file name
|
|
@@ -395,8 +392,8 @@ class PgDBI(PgLOG):
|
|
|
395
392
|
elif reconnect:
|
|
396
393
|
reconnect = 0 # initial connection
|
|
397
394
|
while True:
|
|
398
|
-
config = {'database'
|
|
399
|
-
'user'
|
|
395
|
+
config = {'database': self.PGDBI['DBNAME'],
|
|
396
|
+
'user': self.PGDBI['LNNAME']}
|
|
400
397
|
if self.PGDBI['DBSHOST'] == self.PGLOG['HOSTNAME']:
|
|
401
398
|
config['host'] = 'localhost'
|
|
402
399
|
else:
|
|
@@ -746,8 +743,8 @@ class PgDBI(PgLOG):
|
|
|
746
743
|
|
|
747
744
|
# tablenames: comma deliminated string of one or more tables
|
|
748
745
|
# fields: comma deliminated string of one or more field names,
|
|
749
|
-
# cnddict: condition dict with field names
|
|
750
|
-
# return a dict(field names
|
|
746
|
+
# cnddict: condition dict with field names: values
|
|
747
|
+
# return a dict(field names: values) upon success
|
|
751
748
|
# retrieve one records from tablenames condition dict
|
|
752
749
|
def pghget(self, tablenames, fields, cnddict, logact = None):
|
|
753
750
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -792,8 +789,8 @@ class PgDBI(PgLOG):
|
|
|
792
789
|
|
|
793
790
|
# tablenames: comma deliminated string of one or more tables
|
|
794
791
|
# fields: comma deliminated string of one or more field names,
|
|
795
|
-
# cnddicts: condition dict with field names
|
|
796
|
-
# return a dict(field names
|
|
792
|
+
# cnddicts: condition dict with field names: value lists
|
|
793
|
+
# return a dict(field names: value lists) upon success
|
|
797
794
|
# retrieve multiple records from tablenames for condition dict
|
|
798
795
|
def pgmhget(self, tablenames, fields, cnddicts, logact = None):
|
|
799
796
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -868,7 +865,7 @@ class PgDBI(PgLOG):
|
|
|
868
865
|
|
|
869
866
|
# update one or multiple rows in tablename
|
|
870
867
|
# tablename: update for one table name each call
|
|
871
|
-
# record: dict with field names
|
|
868
|
+
# record: dict with field names: values
|
|
872
869
|
# condition: update conditions for where clause)
|
|
873
870
|
# return number of rows undated upon success
|
|
874
871
|
def pgupdt(self, tablename, record, condition, logact = None):
|
|
@@ -902,8 +899,8 @@ class PgDBI(PgLOG):
|
|
|
902
899
|
|
|
903
900
|
# update one or multiple records in tablename
|
|
904
901
|
# tablename: update for one table name each call
|
|
905
|
-
# record: update values, dict with field names
|
|
906
|
-
# cnddict: condition dict with field names
|
|
902
|
+
# record: update values, dict with field names: values
|
|
903
|
+
# cnddict: condition dict with field names: values
|
|
907
904
|
# return number of records updated upon success
|
|
908
905
|
def pghupdt(self, tablename, record, cnddict, logact = None):
|
|
909
906
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -937,8 +934,8 @@ class PgDBI(PgLOG):
|
|
|
937
934
|
|
|
938
935
|
# update multiple records in tablename
|
|
939
936
|
# tablename: update for one table name each call
|
|
940
|
-
# records: update values, dict with field names
|
|
941
|
-
# cnddicts: condition dict with field names
|
|
937
|
+
# records: update values, dict with field names: value lists
|
|
938
|
+
# cnddicts: condition dict with field names: value lists
|
|
942
939
|
# return number of records updated upon success
|
|
943
940
|
def pgmupdt(self, tablename, records, cnddicts, logact = None):
|
|
944
941
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -1019,7 +1016,7 @@ class PgDBI(PgLOG):
|
|
|
1019
1016
|
|
|
1020
1017
|
# delete one or mutiple records in tablename according condition
|
|
1021
1018
|
# tablename: delete for one table name each call
|
|
1022
|
-
# cndict: delete condition dict for names
|
|
1019
|
+
# cndict: delete condition dict for names: values
|
|
1023
1020
|
# return number of records deleted upon success
|
|
1024
1021
|
def pghdel(self, tablename, cnddict, logact = None):
|
|
1025
1022
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -1050,7 +1047,7 @@ class PgDBI(PgLOG):
|
|
|
1050
1047
|
|
|
1051
1048
|
# delete mutiple records in tablename according condition
|
|
1052
1049
|
# tablename: delete for one table name each call
|
|
1053
|
-
# cndicts: delete condition dict for names
|
|
1050
|
+
# cndicts: delete condition dict for names: value lists
|
|
1054
1051
|
# return number of records deleted upon success
|
|
1055
1052
|
def pgmdel(self, tablename, cnddicts, logact = None):
|
|
1056
1053
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -1155,7 +1152,7 @@ class PgDBI(PgLOG):
|
|
|
1155
1152
|
pgrec = self.pgget("dssdb.user", "uid", "userno = {}".format(userno), self.PGDBI['ERRLOG'])
|
|
1156
1153
|
if pgrec: return pgrec['uid']
|
|
1157
1154
|
pgrec = self.ucar_user_info(userno)
|
|
1158
|
-
if not pgrec: pgrec = {'userno'
|
|
1155
|
+
if not pgrec: pgrec = {'userno': userno, 'stat_flag': 'M'}
|
|
1159
1156
|
uid = self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID))
|
|
1160
1157
|
if uid: self.pglog("{}: Scientist ID Added as user.uid = {}".format(userno, uid), self.LGWNEM)
|
|
1161
1158
|
return uid
|
|
@@ -1177,7 +1174,7 @@ class PgDBI(PgLOG):
|
|
|
1177
1174
|
pgrec = self.pgget("dssdb.user", "uid", "logname = '{}'".format(logname), self.PGDBI['ERRLOG'])
|
|
1178
1175
|
if pgrec: return pgrec['uid']
|
|
1179
1176
|
pgrec = self.ucar_user_info(0, logname)
|
|
1180
|
-
if not pgrec: pgrec = {'logname'
|
|
1177
|
+
if not pgrec: pgrec = {'logname': logname, 'stat_flag': 'M'}
|
|
1181
1178
|
uid = self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID))
|
|
1182
1179
|
if uid: self.pglog("{}: UCAR Login Name Added as user.uid = {}".format(logname, uid), self.LGWNEM)
|
|
1183
1180
|
return uid
|
|
@@ -1185,18 +1182,18 @@ class PgDBI(PgLOG):
|
|
|
1185
1182
|
# get ucar user info for given userno (scientist number) or logname (Ucar login)
|
|
1186
1183
|
def ucar_user_info(self, userno, logname = None):
|
|
1187
1184
|
matches = {
|
|
1188
|
-
'upid'
|
|
1189
|
-
'uid'
|
|
1190
|
-
'username'
|
|
1191
|
-
'lastName'
|
|
1192
|
-
'firstName'
|
|
1193
|
-
'active'
|
|
1194
|
-
'internalOrg'
|
|
1195
|
-
'externalOrg'
|
|
1196
|
-
'country'
|
|
1197
|
-
'forwardEmail'
|
|
1198
|
-
'email'
|
|
1199
|
-
'phone'
|
|
1185
|
+
'upid': "upid",
|
|
1186
|
+
'uid': "userno",
|
|
1187
|
+
'username': "logname",
|
|
1188
|
+
'lastName': "lstname",
|
|
1189
|
+
'firstName': "fstname",
|
|
1190
|
+
'active': "stat_flag",
|
|
1191
|
+
'internalOrg': "division",
|
|
1192
|
+
'externalOrg': "org_name",
|
|
1193
|
+
'country': "country",
|
|
1194
|
+
'forwardEmail': "email",
|
|
1195
|
+
'email': "ucaremail",
|
|
1196
|
+
'phone': "phoneno"
|
|
1200
1197
|
}
|
|
1201
1198
|
buf = self.pgsystem("pgperson " + ("-uid {}".format(userno) if userno else "-username {}".format(logname)), self.LOGWRN, 20)
|
|
1202
1199
|
if not buf: return None
|
|
@@ -1248,13 +1245,13 @@ class PgDBI(PgLOG):
|
|
|
1248
1245
|
# set country code for given coutry name or email address
|
|
1249
1246
|
def set_country_code(self, email, country = None):
|
|
1250
1247
|
codes = {
|
|
1251
|
-
'CHINA'
|
|
1252
|
-
'ENGLAND'
|
|
1253
|
-
'FR'
|
|
1254
|
-
'KOREA'
|
|
1255
|
-
'USSR'
|
|
1256
|
-
'US'
|
|
1257
|
-
'U.S.A.'
|
|
1248
|
+
'CHINA': "P.R.CHINA",
|
|
1249
|
+
'ENGLAND': "UNITED.KINGDOM",
|
|
1250
|
+
'FR': "FRANCE",
|
|
1251
|
+
'KOREA': "SOUTH.KOREA",
|
|
1252
|
+
'USSR': "RUSSIA",
|
|
1253
|
+
'US': "UNITED.STATES",
|
|
1254
|
+
'U.S.A.': "UNITED.STATES"
|
|
1258
1255
|
}
|
|
1259
1256
|
if country:
|
|
1260
1257
|
country = country.upper()
|
|
@@ -1282,7 +1279,7 @@ class PgDBI(PgLOG):
|
|
|
1282
1279
|
pgrec = self.pgget("wuser", "wuid", emcond, self.LOGERR)
|
|
1283
1280
|
if pgrec: return pgrec['wuid']
|
|
1284
1281
|
# now add one in
|
|
1285
|
-
record = {'email'
|
|
1282
|
+
record = {'email': email}
|
|
1286
1283
|
# check again if a ruser is on file
|
|
1287
1284
|
pgrec = self.pgget("ruser", "*", emcond + " AND end_date IS NULL", self.PGDBI['ERRLOG'])
|
|
1288
1285
|
if not pgrec: pgrec = self.pgget("ruser", "*", emcond, self.PGDBI['ERRLOG'])
|
|
@@ -1528,11 +1525,11 @@ class PgDBI(PgLOG):
|
|
|
1528
1525
|
|
|
1529
1526
|
# email: full user email address
|
|
1530
1527
|
# get user real name from table ruser for a given email address
|
|
1531
|
-
# opts == 1
|
|
1532
|
-
# opts == 2
|
|
1533
|
-
# opts == 4
|
|
1534
|
-
# opts == 8
|
|
1535
|
-
# opts == 16
|
|
1528
|
+
# opts == 1: include email
|
|
1529
|
+
# opts == 2: include org_type
|
|
1530
|
+
# opts == 4: include country
|
|
1531
|
+
# opts == 8: include valid_email
|
|
1532
|
+
# opts == 16: include org
|
|
1536
1533
|
def get_ruser_names(self, email, opts = 0, date = None):
|
|
1537
1534
|
fields = "lname lstname, fname fstname"
|
|
1538
1535
|
if opts&1: fields += ", email"
|
|
@@ -1857,9 +1854,9 @@ class PgDBI(PgLOG):
|
|
|
1857
1854
|
self.DBBAOS[dbname] = {}
|
|
1858
1855
|
url = 'https://bao.k8s.ucar.edu/'
|
|
1859
1856
|
baopath = {
|
|
1860
|
-
'ivaddb'
|
|
1861
|
-
'ispddb'
|
|
1862
|
-
'default'
|
|
1857
|
+
'ivaddb': 'gdex/pgdb03',
|
|
1858
|
+
'ispddb': 'gdex/pgdb03',
|
|
1859
|
+
'default': 'gdex/pgdb01'
|
|
1863
1860
|
}
|
|
1864
1861
|
dbpath = baopath[dbname] if dbname in baopath else baopath['default']
|
|
1865
1862
|
client = hvac.Client(url=self.PGDBI.get('BAOURL'))
|
|
@@ -1,19 +1,14 @@
|
|
|
1
|
-
#
|
|
2
1
|
###############################################################################
|
|
3
|
-
#
|
|
4
|
-
#
|
|
5
|
-
#
|
|
6
|
-
# Date : 08/05/2020
|
|
2
|
+
# Title: pg_file.py
|
|
3
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
4
|
+
# Date: 08/05/2020
|
|
7
5
|
# 2025-01-10 transferred to package rda_python_common from
|
|
8
6
|
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
7
|
# 2025-12-01 convert to class PgFile
|
|
10
|
-
# Purpose
|
|
8
|
+
# Purpose: python library module to copy, move and delete data files locally
|
|
11
9
|
# and remotely
|
|
12
|
-
#
|
|
13
|
-
# Github : https://github.com/NCAR/rda-python-common.git
|
|
14
|
-
#
|
|
10
|
+
# Github: https://github.com/NCAR/rda-python-common.git
|
|
15
11
|
###############################################################################
|
|
16
|
-
#
|
|
17
12
|
import sys
|
|
18
13
|
import os
|
|
19
14
|
from os import path as op
|
|
@@ -39,24 +34,24 @@ class PgFile(PgUtil, PgSIG):
|
|
|
39
34
|
super().__init__() # initialize parent class
|
|
40
35
|
self.PGCMPS = {
|
|
41
36
|
# extension Compress Uncompress ArchiveFormat
|
|
42
|
-
'Z'
|
|
43
|
-
'zip'
|
|
44
|
-
'gz'
|
|
45
|
-
'xz'
|
|
46
|
-
'bz2'
|
|
37
|
+
'Z': ['compress -f', 'uncompress -f', 'Z'],
|
|
38
|
+
'zip': ['zip', 'unzip', 'ZIP'],
|
|
39
|
+
'gz': ['gzip', 'gunzip', 'GZ'],
|
|
40
|
+
'xz': ['xz', 'unxz', 'XZ'],
|
|
41
|
+
'bz2': ['bzip2', 'bunzip2', 'BZ2']
|
|
47
42
|
}
|
|
48
43
|
self.CMPSTR = '|'.join(self.PGCMPS)
|
|
49
44
|
self.PGTARS = {
|
|
50
45
|
# extension Packing Unpacking ArchiveFormat
|
|
51
|
-
'tar'
|
|
52
|
-
'tar.Z'
|
|
53
|
-
'zip'
|
|
54
|
-
'tgz'
|
|
55
|
-
'tar.gz'
|
|
56
|
-
'txz'
|
|
57
|
-
'tar.xz'
|
|
58
|
-
'tbz2'
|
|
59
|
-
'tar.bz2'
|
|
46
|
+
'tar': ['tar -cvf', 'tar -xvf', 'TAR'],
|
|
47
|
+
'tar.Z': ['tar -Zcvf', 'tar -xvf', 'TAR.Z'],
|
|
48
|
+
'zip': ['zip -v', 'unzip -v', 'ZIP'],
|
|
49
|
+
'tgz': ['tar -zcvf', 'tar -xvf', 'TGZ'],
|
|
50
|
+
'tar.gz': ['tar -zcvf', 'tar -xvf', 'TAR.GZ'],
|
|
51
|
+
'txz': ['tar -cvJf', 'tar -xvf', 'TXZ'],
|
|
52
|
+
'tar.xz': ['tar -cvJf', 'tar -xvf', 'TAR.XZ'],
|
|
53
|
+
'tbz2': ['tar -cvjf', 'tar -xvf', 'TBZ2'],
|
|
54
|
+
'tar.bz2': ['tar -cvjf', 'tar -xvf', 'TAR.BZ2']
|
|
60
55
|
}
|
|
61
56
|
self.TARSTR = '|'.join(self.PGTARS)
|
|
62
57
|
self.DELDIRS = {}
|
|
@@ -70,42 +65,42 @@ class PgFile(PgUtil, PgSIG):
|
|
|
70
65
|
self.DIRLVLS = 0
|
|
71
66
|
self.BFILES = {} # cache backup file names and dates for each bid
|
|
72
67
|
# record how many errors happen for working with HPSS, local or remote machines
|
|
73
|
-
self.ECNTS = {'D'
|
|
68
|
+
self.ECNTS = {'D': 0, 'H': 0, 'L': 0, 'R': 0, 'O': 0, 'B': 0}
|
|
74
69
|
# up limits for how many continuing errors allowed
|
|
75
|
-
self.ELMTS = {'D'
|
|
70
|
+
self.ELMTS = {'D': 20, 'H': 20, 'L': 20, 'R': 20, 'O': 10, 'B': 10}
|
|
76
71
|
# down storage hostnames & paths
|
|
77
72
|
self.DHOSTS = {
|
|
78
|
-
'G'
|
|
79
|
-
'O'
|
|
80
|
-
'B'
|
|
81
|
-
'D'
|
|
73
|
+
'G': self.PGLOG['GPFSNAME'],
|
|
74
|
+
'O': self.OHOST,
|
|
75
|
+
'B': self.BHOST,
|
|
76
|
+
'D': self.DHOST
|
|
82
77
|
}
|
|
83
78
|
self.DPATHS = {
|
|
84
|
-
'G'
|
|
85
|
-
'O'
|
|
86
|
-
'B'
|
|
87
|
-
'D'
|
|
79
|
+
'G': self.PGLOG['DSSDATA'],
|
|
80
|
+
'O': self.PGLOG['OBJCTBKT'],
|
|
81
|
+
'B': '/' + self.PGLOG['DEFDSID'], # backup globus endpoint
|
|
82
|
+
'D': '/' + self.PGLOG['DEFDSID'] # disaster recovery globus endpoint
|
|
88
83
|
}
|
|
89
84
|
self.QSTATS = {
|
|
90
|
-
'A'
|
|
91
|
-
'I'
|
|
92
|
-
'S'
|
|
93
|
-
'F'
|
|
85
|
+
'A': 'ACTIVE',
|
|
86
|
+
'I': 'INACTIVE',
|
|
87
|
+
'S': 'SUCCEEDED',
|
|
88
|
+
'F': 'FAILED',
|
|
94
89
|
}
|
|
95
90
|
self.QPOINTS = {
|
|
96
|
-
'L'
|
|
97
|
-
'B'
|
|
98
|
-
'D'
|
|
91
|
+
'L': 'gdex-glade',
|
|
92
|
+
'B': 'gdex-quasar',
|
|
93
|
+
'D': 'gdex-quasar-drdata'
|
|
99
94
|
}
|
|
100
95
|
self.QHOSTS = {
|
|
101
|
-
'gdex-glade'
|
|
102
|
-
'gdex-quasar'
|
|
103
|
-
'gdex-quasar-drdata'
|
|
96
|
+
'gdex-glade': self.LHOST,
|
|
97
|
+
'gdex-quasar': self.BHOST,
|
|
98
|
+
'gdex-quasar-drdata': self.DHOST
|
|
104
99
|
}
|
|
105
100
|
self.ENDPOINTS = {
|
|
106
|
-
'gdex-glade'
|
|
107
|
-
'gdex-quasar'
|
|
108
|
-
'gdex-quasar-drdata'
|
|
101
|
+
'gdex-glade': "NCAR GDEX GLADE",
|
|
102
|
+
'gdex-quasar': "NCAR GDEX Quasar",
|
|
103
|
+
'gdex-quasar-drdata': "NCAR GDEX Quasar DRDATA"
|
|
109
104
|
}
|
|
110
105
|
|
|
111
106
|
# reset the up limit for a specified error type
|
|
@@ -339,7 +334,7 @@ class PgFile(PgUtil, PgSIG):
|
|
|
339
334
|
|
|
340
335
|
# submit a globus task and return a task id
|
|
341
336
|
def submit_globus_task(self, cmd, endpoint, logact = 0, qstr = None):
|
|
342
|
-
task = {'id'
|
|
337
|
+
task = {'id': None, 'stat': 'U'}
|
|
343
338
|
loop = reset = 0
|
|
344
339
|
while (loop-reset) < 2:
|
|
345
340
|
buf = self.pgsystem(cmd, logact, self.CMDGLB, qstr)
|
|
@@ -1465,7 +1460,7 @@ class PgFile(PgUtil, PgSIG):
|
|
|
1465
1460
|
|
|
1466
1461
|
# object store function to get file stat
|
|
1467
1462
|
def object_file_stat(self, hash, uhash, opt):
|
|
1468
|
-
info = {'isfile'
|
|
1463
|
+
info = {'isfile': 1, 'data_size': int(hash['Size']), 'fname': op.basename(hash['Key'])}
|
|
1469
1464
|
if not opt: return info
|
|
1470
1465
|
if opt&17:
|
|
1471
1466
|
ms = re.match(r'^(\d+-\d+-\d+)\s+(\d+:\d+:\d+)', hash['LastModified'])
|
|
@@ -1,18 +1,13 @@
|
|
|
1
|
-
#
|
|
2
1
|
###############################################################################
|
|
3
|
-
#
|
|
4
|
-
#
|
|
5
|
-
#
|
|
6
|
-
# Date : 08/118/2020
|
|
2
|
+
# Title: pg_lock.py
|
|
3
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
4
|
+
# Date: 08/118/2020
|
|
7
5
|
# 2025-01-10 transferred to package rda_python_common from
|
|
8
6
|
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
7
|
# 2025-12-01 convert to class PgLock
|
|
10
|
-
# Purpose
|
|
11
|
-
#
|
|
12
|
-
# Github : https://github.com/NCAR/rda-python-common.git
|
|
13
|
-
#
|
|
8
|
+
# Purpose: python library module for functions to lock RDADB records
|
|
9
|
+
# Github: https://github.com/NCAR/rda-python-common.git
|
|
14
10
|
###############################################################################
|
|
15
|
-
#
|
|
16
11
|
import re
|
|
17
12
|
import time
|
|
18
13
|
from .pg_file import PgFile
|
|
@@ -20,9 +15,8 @@ from .pg_file import PgFile
|
|
|
20
15
|
class PgLock(PgFile):
|
|
21
16
|
|
|
22
17
|
def __init__(self):
|
|
23
|
-
|
|
24
18
|
super().__init__() # initialize parent class
|
|
25
|
-
self.DOLOCKS = {-2
|
|
19
|
+
self.DOLOCKS = {-2: 'Force Unlock', -1: 'Unlock', 0: 'Unlock', 1: 'Relock', 2: 'Force Relock'}
|
|
26
20
|
|
|
27
21
|
def end_db_transaction(self, idx):
|
|
28
22
|
if idx > 0:
|