rda-python-common 2.0.12__tar.gz → 2.0.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/PKG-INFO +1 -1
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/pyproject.toml +1 -1
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pg_cmd.py +16 -21
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pg_dbi.py +57 -62
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pg_file.py +44 -49
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pg_lock.py +6 -12
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pg_log.py +87 -96
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pg_opt.py +83 -89
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pg_sig.py +23 -29
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pg_split.py +14 -21
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pg_util.py +15 -20
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pgpassword.py +10 -15
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/PKG-INFO +1 -1
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/LICENSE +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/README.md +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/setup.cfg +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/PgCMD.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/PgDBI.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/PgFile.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/PgLOG.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/PgLock.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/PgOPT.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/PgSIG.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/PgSplit.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/PgUtil.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/__init__.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common/pg_password.py +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/SOURCES.txt +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/dependency_links.txt +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/entry_points.txt +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/requires.txt +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/src/rda_python_common.egg-info/top_level.txt +0 -0
- {rda_python_common-2.0.12 → rda_python_common-2.0.14}/test/test_common.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rda_python_common
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.14
|
|
4
4
|
Summary: RDA Python common library codes shared by other RDA python packages
|
|
5
5
|
Author-email: Zaihua Ji <zji@ucar.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/NCAR/rda-python-common
|
|
@@ -1,18 +1,13 @@
|
|
|
1
|
-
#
|
|
2
1
|
###############################################################################
|
|
3
|
-
#
|
|
4
|
-
#
|
|
5
|
-
#
|
|
6
|
-
# Date : 08/25/2020
|
|
2
|
+
# Title: pg_cmd.py
|
|
3
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
4
|
+
# Date: 08/25/2020
|
|
7
5
|
# 2025-01-10 transferred to package rda_python_common from
|
|
8
6
|
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
|
-
# Purpose
|
|
7
|
+
# Purpose: python library module for functions to record commands for delayed
|
|
10
8
|
# mode or command recovery
|
|
11
|
-
#
|
|
12
|
-
# Github : https://github.com/NCAR/rda-python-common.git
|
|
13
|
-
#
|
|
9
|
+
# Github: https://github.com/NCAR/rda-python-common.git
|
|
14
10
|
###############################################################################
|
|
15
|
-
#
|
|
16
11
|
import os
|
|
17
12
|
import re
|
|
18
13
|
import sys
|
|
@@ -25,18 +20,18 @@ class PgCMD(PgLock):
|
|
|
25
20
|
super().__init__() # initialize parent class
|
|
26
21
|
# cached dscheck info
|
|
27
22
|
self.DSCHK = {}
|
|
28
|
-
self.BOPTIONS = {"hostname"
|
|
23
|
+
self.BOPTIONS = {"hostname": None, "qoptions": None, "modules": None, "environments": None}
|
|
29
24
|
self.BFIELDS = ', '.join(self.BOPTIONS)
|
|
30
25
|
self.TRYLMTS = {
|
|
31
|
-
'dsquasar'
|
|
32
|
-
'dsarch'
|
|
33
|
-
'default'
|
|
26
|
+
'dsquasar': 3,
|
|
27
|
+
'dsarch': 2,
|
|
28
|
+
'default': 1
|
|
34
29
|
}
|
|
35
30
|
self.DLYPTN = r'(^|\s)-(d|BP|BatchProcess|DelayedMode)(\s|$)'
|
|
36
31
|
self.DLYOPT = {
|
|
37
|
-
'dsarch'
|
|
38
|
-
'dsupdt'
|
|
39
|
-
'dsrqst'
|
|
32
|
+
'dsarch': ' -d',
|
|
33
|
+
'dsupdt': ' -d',
|
|
34
|
+
'dsrqst': ' -d'
|
|
40
35
|
}
|
|
41
36
|
|
|
42
37
|
# params: dict array holding option values
|
|
@@ -381,7 +376,7 @@ class PgCMD(PgLock):
|
|
|
381
376
|
fcnt = pgrec['fcount']
|
|
382
377
|
else:
|
|
383
378
|
fcnt = 0
|
|
384
|
-
pgrec = {'fcount'
|
|
379
|
+
pgrec = {'fcount': 0}
|
|
385
380
|
if not fcnt: fcnt = self.pgget("wfrqst", "", cnd, logact)
|
|
386
381
|
if fcnt and fcount != fcnt: fcount = fcnt
|
|
387
382
|
if fcount:
|
|
@@ -402,14 +397,14 @@ class PgCMD(PgLock):
|
|
|
402
397
|
|
|
403
398
|
# set dscheck fcount
|
|
404
399
|
def set_dscheck_fcount(self, count, logact = 0):
|
|
405
|
-
record = {'fcount'
|
|
400
|
+
record = {'fcount': count, 'chktime': int(time.time())}
|
|
406
401
|
self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
407
402
|
self.DSCHK['fcount'] = count
|
|
408
403
|
return self.DSCHK['dcount'] # return Done count
|
|
409
404
|
|
|
410
405
|
# set dscheck dcount
|
|
411
406
|
def set_dscheck_dcount(self, count, size, logact = 0):
|
|
412
|
-
record = {'dcount'
|
|
407
|
+
record = {'dcount': count, 'size': size, 'chktime': int(time.time())}
|
|
413
408
|
self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
414
409
|
self.DSCHK['dcount'] = count
|
|
415
410
|
self.DSCHK['size'] = size
|
|
@@ -444,7 +439,7 @@ class PgCMD(PgLock):
|
|
|
444
439
|
if pgrec['pid'] != cpid or pgrec['lockhost'] != chost: return 0
|
|
445
440
|
|
|
446
441
|
# update dscheck status only if it is still locked by the current process
|
|
447
|
-
record = {'status'
|
|
442
|
+
record = {'status': stat, 'chktime': int(time.time()), 'pid': 0}
|
|
448
443
|
return self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
|
|
449
444
|
|
|
450
445
|
# get the number of tries to execute for a given cmd under dscheck control
|
|
@@ -1,18 +1,13 @@
|
|
|
1
|
-
#
|
|
2
1
|
###############################################################################
|
|
3
|
-
#
|
|
4
|
-
#
|
|
5
|
-
#
|
|
6
|
-
# Date : 06/07/2022
|
|
2
|
+
# Title: pg_dbi.py -- PostgreSQL DataBase Interface
|
|
3
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
4
|
+
# Date: 06/07/2022
|
|
7
5
|
# 2025-01-10 transferred to package rda_python_common from
|
|
8
6
|
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
7
|
# 2025-11-24 convert to class PgDBI
|
|
10
|
-
# Purpose
|
|
11
|
-
#
|
|
12
|
-
# Github : https://github.com/NCAR/rda-python-common.git
|
|
13
|
-
#
|
|
8
|
+
# Purpose: Python library module to handle query and manipulate PostgreSQL database
|
|
9
|
+
# Github: https://github.com/NCAR/rda-python-common.git
|
|
14
10
|
###############################################################################
|
|
15
|
-
|
|
16
11
|
import os
|
|
17
12
|
import re
|
|
18
13
|
import time
|
|
@@ -49,22 +44,22 @@ class PgDBI(PgLOG):
|
|
|
49
44
|
self.PGSIGNS = ['!', '<', '>', '<>']
|
|
50
45
|
self.CHCODE = 1042
|
|
51
46
|
# hard coded db ports for dbnames
|
|
52
|
-
self.DBPORTS = {'default'
|
|
47
|
+
self.DBPORTS = {'default': 0}
|
|
53
48
|
self.DBPASS = {}
|
|
54
49
|
self.DBBAOS = {}
|
|
55
50
|
# hard coded db names for given schema names
|
|
56
51
|
self.DBNAMES = {
|
|
57
|
-
'ivaddb'
|
|
58
|
-
'cntldb'
|
|
59
|
-
'cdmsdb'
|
|
60
|
-
'ispddb'
|
|
61
|
-
'obsua'
|
|
62
|
-
'default'
|
|
52
|
+
'ivaddb': 'ivaddb',
|
|
53
|
+
'cntldb': 'ivaddb',
|
|
54
|
+
'cdmsdb': 'ivaddb',
|
|
55
|
+
'ispddb': 'ispddb',
|
|
56
|
+
'obsua': 'upadb',
|
|
57
|
+
'default': 'rdadb',
|
|
63
58
|
}
|
|
64
59
|
# hard coded socket paths for machine_dbnames
|
|
65
|
-
self.DBSOCKS = {'default'
|
|
60
|
+
self.DBSOCKS = {'default': ''}
|
|
66
61
|
# home path for check db on alter host
|
|
67
|
-
self.VIEWHOMES = {'default'
|
|
62
|
+
self.VIEWHOMES = {'default': self.PGLOG['DSSDBHM']}
|
|
68
63
|
# add more to the list if used for names
|
|
69
64
|
self.PGRES = ['end', 'window']
|
|
70
65
|
self.SETPGDBI('DEFDB', 'rdadb')
|
|
@@ -397,8 +392,8 @@ class PgDBI(PgLOG):
|
|
|
397
392
|
elif reconnect:
|
|
398
393
|
reconnect = 0 # initial connection
|
|
399
394
|
while True:
|
|
400
|
-
config = {'database'
|
|
401
|
-
'user'
|
|
395
|
+
config = {'database': self.PGDBI['DBNAME'],
|
|
396
|
+
'user': self.PGDBI['LNNAME']}
|
|
402
397
|
if self.PGDBI['DBSHOST'] == self.PGLOG['HOSTNAME']:
|
|
403
398
|
config['host'] = 'localhost'
|
|
404
399
|
else:
|
|
@@ -748,8 +743,8 @@ class PgDBI(PgLOG):
|
|
|
748
743
|
|
|
749
744
|
# tablenames: comma deliminated string of one or more tables
|
|
750
745
|
# fields: comma deliminated string of one or more field names,
|
|
751
|
-
# cnddict: condition dict with field names
|
|
752
|
-
# return a dict(field names
|
|
746
|
+
# cnddict: condition dict with field names: values
|
|
747
|
+
# return a dict(field names: values) upon success
|
|
753
748
|
# retrieve one records from tablenames condition dict
|
|
754
749
|
def pghget(self, tablenames, fields, cnddict, logact = None):
|
|
755
750
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -794,8 +789,8 @@ class PgDBI(PgLOG):
|
|
|
794
789
|
|
|
795
790
|
# tablenames: comma deliminated string of one or more tables
|
|
796
791
|
# fields: comma deliminated string of one or more field names,
|
|
797
|
-
# cnddicts: condition dict with field names
|
|
798
|
-
# return a dict(field names
|
|
792
|
+
# cnddicts: condition dict with field names: value lists
|
|
793
|
+
# return a dict(field names: value lists) upon success
|
|
799
794
|
# retrieve multiple records from tablenames for condition dict
|
|
800
795
|
def pgmhget(self, tablenames, fields, cnddicts, logact = None):
|
|
801
796
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -870,7 +865,7 @@ class PgDBI(PgLOG):
|
|
|
870
865
|
|
|
871
866
|
# update one or multiple rows in tablename
|
|
872
867
|
# tablename: update for one table name each call
|
|
873
|
-
# record: dict with field names
|
|
868
|
+
# record: dict with field names: values
|
|
874
869
|
# condition: update conditions for where clause)
|
|
875
870
|
# return number of rows undated upon success
|
|
876
871
|
def pgupdt(self, tablename, record, condition, logact = None):
|
|
@@ -904,8 +899,8 @@ class PgDBI(PgLOG):
|
|
|
904
899
|
|
|
905
900
|
# update one or multiple records in tablename
|
|
906
901
|
# tablename: update for one table name each call
|
|
907
|
-
# record: update values, dict with field names
|
|
908
|
-
# cnddict: condition dict with field names
|
|
902
|
+
# record: update values, dict with field names: values
|
|
903
|
+
# cnddict: condition dict with field names: values
|
|
909
904
|
# return number of records updated upon success
|
|
910
905
|
def pghupdt(self, tablename, record, cnddict, logact = None):
|
|
911
906
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -939,8 +934,8 @@ class PgDBI(PgLOG):
|
|
|
939
934
|
|
|
940
935
|
# update multiple records in tablename
|
|
941
936
|
# tablename: update for one table name each call
|
|
942
|
-
# records: update values, dict with field names
|
|
943
|
-
# cnddicts: condition dict with field names
|
|
937
|
+
# records: update values, dict with field names: value lists
|
|
938
|
+
# cnddicts: condition dict with field names: value lists
|
|
944
939
|
# return number of records updated upon success
|
|
945
940
|
def pgmupdt(self, tablename, records, cnddicts, logact = None):
|
|
946
941
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -1021,7 +1016,7 @@ class PgDBI(PgLOG):
|
|
|
1021
1016
|
|
|
1022
1017
|
# delete one or mutiple records in tablename according condition
|
|
1023
1018
|
# tablename: delete for one table name each call
|
|
1024
|
-
# cndict: delete condition dict for names
|
|
1019
|
+
# cndict: delete condition dict for names: values
|
|
1025
1020
|
# return number of records deleted upon success
|
|
1026
1021
|
def pghdel(self, tablename, cnddict, logact = None):
|
|
1027
1022
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -1052,7 +1047,7 @@ class PgDBI(PgLOG):
|
|
|
1052
1047
|
|
|
1053
1048
|
# delete mutiple records in tablename according condition
|
|
1054
1049
|
# tablename: delete for one table name each call
|
|
1055
|
-
# cndicts: delete condition dict for names
|
|
1050
|
+
# cndicts: delete condition dict for names: value lists
|
|
1056
1051
|
# return number of records deleted upon success
|
|
1057
1052
|
def pgmdel(self, tablename, cnddicts, logact = None):
|
|
1058
1053
|
if logact is None: logact = self.PGDBI['ERRLOG']
|
|
@@ -1157,7 +1152,7 @@ class PgDBI(PgLOG):
|
|
|
1157
1152
|
pgrec = self.pgget("dssdb.user", "uid", "userno = {}".format(userno), self.PGDBI['ERRLOG'])
|
|
1158
1153
|
if pgrec: return pgrec['uid']
|
|
1159
1154
|
pgrec = self.ucar_user_info(userno)
|
|
1160
|
-
if not pgrec: pgrec = {'userno'
|
|
1155
|
+
if not pgrec: pgrec = {'userno': userno, 'stat_flag': 'M'}
|
|
1161
1156
|
uid = self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID))
|
|
1162
1157
|
if uid: self.pglog("{}: Scientist ID Added as user.uid = {}".format(userno, uid), self.LGWNEM)
|
|
1163
1158
|
return uid
|
|
@@ -1179,7 +1174,7 @@ class PgDBI(PgLOG):
|
|
|
1179
1174
|
pgrec = self.pgget("dssdb.user", "uid", "logname = '{}'".format(logname), self.PGDBI['ERRLOG'])
|
|
1180
1175
|
if pgrec: return pgrec['uid']
|
|
1181
1176
|
pgrec = self.ucar_user_info(0, logname)
|
|
1182
|
-
if not pgrec: pgrec = {'logname'
|
|
1177
|
+
if not pgrec: pgrec = {'logname': logname, 'stat_flag': 'M'}
|
|
1183
1178
|
uid = self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID))
|
|
1184
1179
|
if uid: self.pglog("{}: UCAR Login Name Added as user.uid = {}".format(logname, uid), self.LGWNEM)
|
|
1185
1180
|
return uid
|
|
@@ -1187,18 +1182,18 @@ class PgDBI(PgLOG):
|
|
|
1187
1182
|
# get ucar user info for given userno (scientist number) or logname (Ucar login)
|
|
1188
1183
|
def ucar_user_info(self, userno, logname = None):
|
|
1189
1184
|
matches = {
|
|
1190
|
-
'upid'
|
|
1191
|
-
'uid'
|
|
1192
|
-
'username'
|
|
1193
|
-
'lastName'
|
|
1194
|
-
'firstName'
|
|
1195
|
-
'active'
|
|
1196
|
-
'internalOrg'
|
|
1197
|
-
'externalOrg'
|
|
1198
|
-
'country'
|
|
1199
|
-
'forwardEmail'
|
|
1200
|
-
'email'
|
|
1201
|
-
'phone'
|
|
1185
|
+
'upid': "upid",
|
|
1186
|
+
'uid': "userno",
|
|
1187
|
+
'username': "logname",
|
|
1188
|
+
'lastName': "lstname",
|
|
1189
|
+
'firstName': "fstname",
|
|
1190
|
+
'active': "stat_flag",
|
|
1191
|
+
'internalOrg': "division",
|
|
1192
|
+
'externalOrg': "org_name",
|
|
1193
|
+
'country': "country",
|
|
1194
|
+
'forwardEmail': "email",
|
|
1195
|
+
'email': "ucaremail",
|
|
1196
|
+
'phone': "phoneno"
|
|
1202
1197
|
}
|
|
1203
1198
|
buf = self.pgsystem("pgperson " + ("-uid {}".format(userno) if userno else "-username {}".format(logname)), self.LOGWRN, 20)
|
|
1204
1199
|
if not buf: return None
|
|
@@ -1250,13 +1245,13 @@ class PgDBI(PgLOG):
|
|
|
1250
1245
|
# set country code for given coutry name or email address
|
|
1251
1246
|
def set_country_code(self, email, country = None):
|
|
1252
1247
|
codes = {
|
|
1253
|
-
'CHINA'
|
|
1254
|
-
'ENGLAND'
|
|
1255
|
-
'FR'
|
|
1256
|
-
'KOREA'
|
|
1257
|
-
'USSR'
|
|
1258
|
-
'US'
|
|
1259
|
-
'U.S.A.'
|
|
1248
|
+
'CHINA': "P.R.CHINA",
|
|
1249
|
+
'ENGLAND': "UNITED.KINGDOM",
|
|
1250
|
+
'FR': "FRANCE",
|
|
1251
|
+
'KOREA': "SOUTH.KOREA",
|
|
1252
|
+
'USSR': "RUSSIA",
|
|
1253
|
+
'US': "UNITED.STATES",
|
|
1254
|
+
'U.S.A.': "UNITED.STATES"
|
|
1260
1255
|
}
|
|
1261
1256
|
if country:
|
|
1262
1257
|
country = country.upper()
|
|
@@ -1284,7 +1279,7 @@ class PgDBI(PgLOG):
|
|
|
1284
1279
|
pgrec = self.pgget("wuser", "wuid", emcond, self.LOGERR)
|
|
1285
1280
|
if pgrec: return pgrec['wuid']
|
|
1286
1281
|
# now add one in
|
|
1287
|
-
record = {'email'
|
|
1282
|
+
record = {'email': email}
|
|
1288
1283
|
# check again if a ruser is on file
|
|
1289
1284
|
pgrec = self.pgget("ruser", "*", emcond + " AND end_date IS NULL", self.PGDBI['ERRLOG'])
|
|
1290
1285
|
if not pgrec: pgrec = self.pgget("ruser", "*", emcond, self.PGDBI['ERRLOG'])
|
|
@@ -1530,11 +1525,11 @@ class PgDBI(PgLOG):
|
|
|
1530
1525
|
|
|
1531
1526
|
# email: full user email address
|
|
1532
1527
|
# get user real name from table ruser for a given email address
|
|
1533
|
-
# opts == 1
|
|
1534
|
-
# opts == 2
|
|
1535
|
-
# opts == 4
|
|
1536
|
-
# opts == 8
|
|
1537
|
-
# opts == 16
|
|
1528
|
+
# opts == 1: include email
|
|
1529
|
+
# opts == 2: include org_type
|
|
1530
|
+
# opts == 4: include country
|
|
1531
|
+
# opts == 8: include valid_email
|
|
1532
|
+
# opts == 16: include org
|
|
1538
1533
|
def get_ruser_names(self, email, opts = 0, date = None):
|
|
1539
1534
|
fields = "lname lstname, fname fstname"
|
|
1540
1535
|
if opts&1: fields += ", email"
|
|
@@ -1859,9 +1854,9 @@ class PgDBI(PgLOG):
|
|
|
1859
1854
|
self.DBBAOS[dbname] = {}
|
|
1860
1855
|
url = 'https://bao.k8s.ucar.edu/'
|
|
1861
1856
|
baopath = {
|
|
1862
|
-
'ivaddb'
|
|
1863
|
-
'ispddb'
|
|
1864
|
-
'default'
|
|
1857
|
+
'ivaddb': 'gdex/pgdb03',
|
|
1858
|
+
'ispddb': 'gdex/pgdb03',
|
|
1859
|
+
'default': 'gdex/pgdb01'
|
|
1865
1860
|
}
|
|
1866
1861
|
dbpath = baopath[dbname] if dbname in baopath else baopath['default']
|
|
1867
1862
|
client = hvac.Client(url=self.PGDBI.get('BAOURL'))
|
|
@@ -1,19 +1,14 @@
|
|
|
1
|
-
#
|
|
2
1
|
###############################################################################
|
|
3
|
-
#
|
|
4
|
-
#
|
|
5
|
-
#
|
|
6
|
-
# Date : 08/05/2020
|
|
2
|
+
# Title: pg_file.py
|
|
3
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
4
|
+
# Date: 08/05/2020
|
|
7
5
|
# 2025-01-10 transferred to package rda_python_common from
|
|
8
6
|
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
7
|
# 2025-12-01 convert to class PgFile
|
|
10
|
-
# Purpose
|
|
8
|
+
# Purpose: python library module to copy, move and delete data files locally
|
|
11
9
|
# and remotely
|
|
12
|
-
#
|
|
13
|
-
# Github : https://github.com/NCAR/rda-python-common.git
|
|
14
|
-
#
|
|
10
|
+
# Github: https://github.com/NCAR/rda-python-common.git
|
|
15
11
|
###############################################################################
|
|
16
|
-
#
|
|
17
12
|
import sys
|
|
18
13
|
import os
|
|
19
14
|
from os import path as op
|
|
@@ -39,24 +34,24 @@ class PgFile(PgUtil, PgSIG):
|
|
|
39
34
|
super().__init__() # initialize parent class
|
|
40
35
|
self.PGCMPS = {
|
|
41
36
|
# extension Compress Uncompress ArchiveFormat
|
|
42
|
-
'Z'
|
|
43
|
-
'zip'
|
|
44
|
-
'gz'
|
|
45
|
-
'xz'
|
|
46
|
-
'bz2'
|
|
37
|
+
'Z': ['compress -f', 'uncompress -f', 'Z'],
|
|
38
|
+
'zip': ['zip', 'unzip', 'ZIP'],
|
|
39
|
+
'gz': ['gzip', 'gunzip', 'GZ'],
|
|
40
|
+
'xz': ['xz', 'unxz', 'XZ'],
|
|
41
|
+
'bz2': ['bzip2', 'bunzip2', 'BZ2']
|
|
47
42
|
}
|
|
48
43
|
self.CMPSTR = '|'.join(self.PGCMPS)
|
|
49
44
|
self.PGTARS = {
|
|
50
45
|
# extension Packing Unpacking ArchiveFormat
|
|
51
|
-
'tar'
|
|
52
|
-
'tar.Z'
|
|
53
|
-
'zip'
|
|
54
|
-
'tgz'
|
|
55
|
-
'tar.gz'
|
|
56
|
-
'txz'
|
|
57
|
-
'tar.xz'
|
|
58
|
-
'tbz2'
|
|
59
|
-
'tar.bz2'
|
|
46
|
+
'tar': ['tar -cvf', 'tar -xvf', 'TAR'],
|
|
47
|
+
'tar.Z': ['tar -Zcvf', 'tar -xvf', 'TAR.Z'],
|
|
48
|
+
'zip': ['zip -v', 'unzip -v', 'ZIP'],
|
|
49
|
+
'tgz': ['tar -zcvf', 'tar -xvf', 'TGZ'],
|
|
50
|
+
'tar.gz': ['tar -zcvf', 'tar -xvf', 'TAR.GZ'],
|
|
51
|
+
'txz': ['tar -cvJf', 'tar -xvf', 'TXZ'],
|
|
52
|
+
'tar.xz': ['tar -cvJf', 'tar -xvf', 'TAR.XZ'],
|
|
53
|
+
'tbz2': ['tar -cvjf', 'tar -xvf', 'TBZ2'],
|
|
54
|
+
'tar.bz2': ['tar -cvjf', 'tar -xvf', 'TAR.BZ2']
|
|
60
55
|
}
|
|
61
56
|
self.TARSTR = '|'.join(self.PGTARS)
|
|
62
57
|
self.DELDIRS = {}
|
|
@@ -70,42 +65,42 @@ class PgFile(PgUtil, PgSIG):
|
|
|
70
65
|
self.DIRLVLS = 0
|
|
71
66
|
self.BFILES = {} # cache backup file names and dates for each bid
|
|
72
67
|
# record how many errors happen for working with HPSS, local or remote machines
|
|
73
|
-
self.ECNTS = {'D'
|
|
68
|
+
self.ECNTS = {'D': 0, 'H': 0, 'L': 0, 'R': 0, 'O': 0, 'B': 0}
|
|
74
69
|
# up limits for how many continuing errors allowed
|
|
75
|
-
self.ELMTS = {'D'
|
|
70
|
+
self.ELMTS = {'D': 20, 'H': 20, 'L': 20, 'R': 20, 'O': 10, 'B': 10}
|
|
76
71
|
# down storage hostnames & paths
|
|
77
72
|
self.DHOSTS = {
|
|
78
|
-
'G'
|
|
79
|
-
'O'
|
|
80
|
-
'B'
|
|
81
|
-
'D'
|
|
73
|
+
'G': self.PGLOG['GPFSNAME'],
|
|
74
|
+
'O': self.OHOST,
|
|
75
|
+
'B': self.BHOST,
|
|
76
|
+
'D': self.DHOST
|
|
82
77
|
}
|
|
83
78
|
self.DPATHS = {
|
|
84
|
-
'G'
|
|
85
|
-
'O'
|
|
86
|
-
'B'
|
|
87
|
-
'D'
|
|
79
|
+
'G': self.PGLOG['DSSDATA'],
|
|
80
|
+
'O': self.PGLOG['OBJCTBKT'],
|
|
81
|
+
'B': '/' + self.PGLOG['DEFDSID'], # backup globus endpoint
|
|
82
|
+
'D': '/' + self.PGLOG['DEFDSID'] # disaster recovery globus endpoint
|
|
88
83
|
}
|
|
89
84
|
self.QSTATS = {
|
|
90
|
-
'A'
|
|
91
|
-
'I'
|
|
92
|
-
'S'
|
|
93
|
-
'F'
|
|
85
|
+
'A': 'ACTIVE',
|
|
86
|
+
'I': 'INACTIVE',
|
|
87
|
+
'S': 'SUCCEEDED',
|
|
88
|
+
'F': 'FAILED',
|
|
94
89
|
}
|
|
95
90
|
self.QPOINTS = {
|
|
96
|
-
'L'
|
|
97
|
-
'B'
|
|
98
|
-
'D'
|
|
91
|
+
'L': 'gdex-glade',
|
|
92
|
+
'B': 'gdex-quasar',
|
|
93
|
+
'D': 'gdex-quasar-drdata'
|
|
99
94
|
}
|
|
100
95
|
self.QHOSTS = {
|
|
101
|
-
'gdex-glade'
|
|
102
|
-
'gdex-quasar'
|
|
103
|
-
'gdex-quasar-drdata'
|
|
96
|
+
'gdex-glade': self.LHOST,
|
|
97
|
+
'gdex-quasar': self.BHOST,
|
|
98
|
+
'gdex-quasar-drdata': self.DHOST
|
|
104
99
|
}
|
|
105
100
|
self.ENDPOINTS = {
|
|
106
|
-
'gdex-glade'
|
|
107
|
-
'gdex-quasar'
|
|
108
|
-
'gdex-quasar-drdata'
|
|
101
|
+
'gdex-glade': "NCAR GDEX GLADE",
|
|
102
|
+
'gdex-quasar': "NCAR GDEX Quasar",
|
|
103
|
+
'gdex-quasar-drdata': "NCAR GDEX Quasar DRDATA"
|
|
109
104
|
}
|
|
110
105
|
|
|
111
106
|
# reset the up limit for a specified error type
|
|
@@ -339,7 +334,7 @@ class PgFile(PgUtil, PgSIG):
|
|
|
339
334
|
|
|
340
335
|
# submit a globus task and return a task id
|
|
341
336
|
def submit_globus_task(self, cmd, endpoint, logact = 0, qstr = None):
|
|
342
|
-
task = {'id'
|
|
337
|
+
task = {'id': None, 'stat': 'U'}
|
|
343
338
|
loop = reset = 0
|
|
344
339
|
while (loop-reset) < 2:
|
|
345
340
|
buf = self.pgsystem(cmd, logact, self.CMDGLB, qstr)
|
|
@@ -1465,7 +1460,7 @@ class PgFile(PgUtil, PgSIG):
|
|
|
1465
1460
|
|
|
1466
1461
|
# object store function to get file stat
|
|
1467
1462
|
def object_file_stat(self, hash, uhash, opt):
|
|
1468
|
-
info = {'isfile'
|
|
1463
|
+
info = {'isfile': 1, 'data_size': int(hash['Size']), 'fname': op.basename(hash['Key'])}
|
|
1469
1464
|
if not opt: return info
|
|
1470
1465
|
if opt&17:
|
|
1471
1466
|
ms = re.match(r'^(\d+-\d+-\d+)\s+(\d+:\d+:\d+)', hash['LastModified'])
|
|
@@ -1,18 +1,13 @@
|
|
|
1
|
-
#
|
|
2
1
|
###############################################################################
|
|
3
|
-
#
|
|
4
|
-
#
|
|
5
|
-
#
|
|
6
|
-
# Date : 08/118/2020
|
|
2
|
+
# Title: pg_lock.py
|
|
3
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
4
|
+
# Date: 08/118/2020
|
|
7
5
|
# 2025-01-10 transferred to package rda_python_common from
|
|
8
6
|
# https://github.com/NCAR/rda-shared-libraries.git
|
|
9
7
|
# 2025-12-01 convert to class PgLock
|
|
10
|
-
# Purpose
|
|
11
|
-
#
|
|
12
|
-
# Github : https://github.com/NCAR/rda-python-common.git
|
|
13
|
-
#
|
|
8
|
+
# Purpose: python library module for functions to lock RDADB records
|
|
9
|
+
# Github: https://github.com/NCAR/rda-python-common.git
|
|
14
10
|
###############################################################################
|
|
15
|
-
#
|
|
16
11
|
import re
|
|
17
12
|
import time
|
|
18
13
|
from .pg_file import PgFile
|
|
@@ -20,9 +15,8 @@ from .pg_file import PgFile
|
|
|
20
15
|
class PgLock(PgFile):
|
|
21
16
|
|
|
22
17
|
def __init__(self):
|
|
23
|
-
|
|
24
18
|
super().__init__() # initialize parent class
|
|
25
|
-
self.DOLOCKS = {-2
|
|
19
|
+
self.DOLOCKS = {-2: 'Force Unlock', -1: 'Unlock', 0: 'Unlock', 1: 'Relock', 2: 'Force Relock'}
|
|
26
20
|
|
|
27
21
|
def end_db_transaction(self, idx):
|
|
28
22
|
if idx > 0:
|