rda-python-metrics 3.0.4__py3-none-any.whl → 3.0.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -113,7 +113,8 @@ class FillOSDFUsage(PgIPInfo, PgFile, PgSplit):
113
113
  if entcnt%20000 == 0:
114
114
  dcnt = len(records)
115
115
  self.pglog("{}: {}/{} OSDF log entries processed/records added".format(logfile, entcnt, dcnt), self.WARNLG)
116
- ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/rda\/([a-z]\d{6})\/\S+\].* \[Site:(\S+)\].* \[Host:(\S+)\].* \[AppInfo:(\S*)\].* \[Read:(\d+)\]', line)
116
+ # the OSDF namespace was renamed from /ncar/rda to /ncar/gdex
117
+ ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/(?:rda|gdex)\/([a-z]\d{6})\/\S+\].* \[Site:(\S+)\].* \[Host:(\S+)\].* \[AppInfo:(\S*)\].* \[Read:(\d+)\]', line)
117
118
  if not ms: continue
118
119
  dt = ms.group(1)
119
120
  dsid = ms.group(2)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rda_python_metrics
3
- Version: 3.0.4
3
+ Version: 3.0.6
4
4
  Summary: RDA Python Package to gather and view data usage metrics
5
5
  Author-email: Zaihua Ji <zji@ucar.edu>
6
6
  Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
@@ -54,10 +54,7 @@ The package provides two categories of programs:
54
54
  | `fillosdfusage` | Fill OSDF usage metrics into the database |
55
55
  | `fillrdadb` | Fill RDA database usage metrics |
56
56
  | `filltdsusage` | Fill TDS (THREDDS) usage metrics into the database |
57
- | `filluser` | Fill user information into the database |
58
57
  | `fillzenodousage` | Fill Zenodo usage metrics into the database |
59
- | `pgperson` | Retrieve user info from the UCAR People API by field |
60
- | `pgusername` | Retrieve user info from the UCAR People API by username |
61
58
  | `viewallusage` | View combined usage statistics |
62
59
  | `viewawsusage` | View AWS usage statistics |
63
60
  | `viewcheckusage` | View usage check results |
@@ -12,7 +12,6 @@ rda_python_metrics/fill_oneorder.py,sha256=v3lELNVU35BHiQw_IW-VsFkZiwRaA0Jd-s3eE
12
12
  rda_python_metrics/fill_osdfusage.py,sha256=UkvKKbB_sIhg2ibJrvFYpqdN9fnKo9FGMrtfyUf0tdE,7096
13
13
  rda_python_metrics/fill_rdadb.py,sha256=JCeV3PTu0LLQvn8V19sFWf1jNG94LpjHRexN7MpkvzM,5283
14
14
  rda_python_metrics/fill_tdsusage.py,sha256=T1JH4ohwV4L3OUUcUlIri32jHbHoJj613t26arRjVug,10463
15
- rda_python_metrics/fill_user.py,sha256=CvaMRaUPaR9nxJAExkLTb3Ci4sD7RQMOpWhWJdbyMF0,8907
16
15
  rda_python_metrics/fillawsusage.py,sha256=Ac3JsuT1eRr0ttAJxYdBE7rKCmiKwrLZHIOKVLcfDDo,7956
17
16
  rda_python_metrics/fillawsusage.usg,sha256=pyYYbAoTo4JcrR1-TWN66NLjWBbvIh20nz9O3M__e90,835
18
17
  rda_python_metrics/fillcdgusage.py,sha256=mC-D-xQqUhCSQ3CZOuLMzoBG2wNc7iXkao47bZs2Ezg,18884
@@ -29,26 +28,20 @@ rda_python_metrics/fillipinfo.py,sha256=ZrWD13r55XZo4m9pgkLU0yD1dWv1b9fqE_9i6b8h
29
28
  rda_python_metrics/fillipinfo.usg,sha256=wc89cLG_aJJ0ObPR0KvUr89L_9CgwQiuiyG1FvpHRTE,1025
30
29
  rda_python_metrics/filloneorder.py,sha256=Tf2efxIN_m0A4YKJkO_52UdLvpGYHXAGzvYE0zYK9Qw,6413
31
30
  rda_python_metrics/filloneorder.usg,sha256=bj-pSGKPteU68zBxJhrOjxj4X6856tT6yTmDlky_8c0,1731
32
- rda_python_metrics/fillosdfusage.py,sha256=nIOHefxIXjjZ3_-Qt6GnCNJ-TDOdJIEC4xUqANG8RGc,8134
31
+ rda_python_metrics/fillosdfusage.py,sha256=-CTbdwGbdOpQUd6ons4BYxpAdsS4w0SYyravHRVtvVA,8217
33
32
  rda_python_metrics/fillosdfusage.usg,sha256=9dauiI-horMBflGW6wZGQPFBoi9MwquYJ36Z-mzSmto,820
34
33
  rda_python_metrics/fillrdadb.py,sha256=8ZDTxhP3OcWS-6OHg1r44tBv6y1lODjZWzyt076yMBE,6078
35
34
  rda_python_metrics/fillrdadb.usg,sha256=8BMED3KWJTb0INyVlu9UA4N_ulOR26kN0k27FVel3D0,1064
36
35
  rda_python_metrics/filltdsusage.py,sha256=vunz1Ml2xCOPdXLE71iBhnQJH9pTk9D6mB8pcahhmWU,8801
37
36
  rda_python_metrics/filltdsusage.usg,sha256=IFPkAHfGjGuOqGsQUHASX_ony7bIAQbPQsh5YM5rS6g,826
38
- rda_python_metrics/filluser.py,sha256=TBMCTo1KEJNombjYEaj_O9GMrd7rugvxMPy0AOduxPM,9670
39
- rda_python_metrics/filluser.usg,sha256=DR8TVutdI78ksoSqvvl1nefImO1cOtVluMORdf41gWw,689
40
37
  rda_python_metrics/fillzenodousage.py,sha256=Rb09bmHwAnY5_5pyx6qA-1Jgt2W05IqYeCHdYOZS9r8,14954
41
38
  rda_python_metrics/fillzenodousage.usg,sha256=gHeJK_7tcSu5222-0BN4zGWuHoXWQIF1mEh0XsbQFCM,904
42
39
  rda_python_metrics/log_arch.py,sha256=93-ZhkRE4Mf2XVDG67HDmYDZg4BCIvN-lgyj_BeLe5E,14770
43
40
  rda_python_metrics/logarch.py,sha256=8V9vtBetJzgCyMT3_huWbQRlFd0tpcnXJLtmO_1Jg_U,16258
44
41
  rda_python_metrics/logarch.usg,sha256=rhUiKfpCA-GRO_yWy8PcSBoXrsH9u5PEjkf3Wgdr55Y,1066
45
42
  rda_python_metrics/pg_ipinfo.py,sha256=l8rSvrzQUDhaMFXGnJVEchhp7eQnwDFFVd3hOMZBqdk,11017
46
- rda_python_metrics/pg_person.py,sha256=1Pvvpnsv_qjJM6F5ikOEMTEaErB12lALtARx5U9z6f0,2087
47
- rda_python_metrics/pg_username.py,sha256=OBTG4-4tNIX35Ncpm2Xmbl2KHnm_n_xOwSJ8f92b0dA,1264
48
43
  rda_python_metrics/pg_view.py,sha256=yXcdgNPuhHipgcdR-2QGIJe2g_vg3CAVuGlUnUhG7ws,26134
49
- rda_python_metrics/pgperson.py,sha256=EvU7mVV0r0XAZqGE0h3JqPYON8j2FkQFBubSzhcw1aQ,2585
50
44
  rda_python_metrics/pgsyspath.py,sha256=uBVi8VHyzI_1TA8tq2DYVfMkj2HZXu03fPg9Wv1MB6c,1038
51
- rda_python_metrics/pgusername.py,sha256=dzXRqbjHKhQgvXJFJUtHMm9agdWKsF5DAr4wqKheuEI,1680
52
45
  rda_python_metrics/view_allusage.py,sha256=Nu-c31_x1jEe9XpQahemrFx-ATcKpKEv2ZfQVuvsw28,15756
53
46
  rda_python_metrics/view_awsusage.py,sha256=FBBmbel1l4pfVRt7QmvN7ADlqUN6Ia-6FMAVz6WZz3w,14125
54
47
  rda_python_metrics/view_checkusage.py,sha256=Ko96y9m0xWToU0NpqVC-L-2b7hOAdEo_5eZPBiE_Lm8,12985
@@ -79,9 +72,9 @@ rda_python_metrics/viewwebfile.py,sha256=07Xa9e7NVozZ6KVBFvyGhJRVi-Oil26g-e3PMxd
79
72
  rda_python_metrics/viewwebfile.usg,sha256=b7QE8USiVCzY91q5d5IiCekvdZz_YB24hPbz_fqb_jg,6072
80
73
  rda_python_metrics/viewwebusage.py,sha256=LzCGAVHY_xoIFIZdiKivI8idalMqvZdIGz53bYWda38,18768
81
74
  rda_python_metrics/viewwebusage.usg,sha256=FnGoFqS7O5l7EDoLG1pKxMktibxeu28jxV9h9q80hQk,6077
82
- rda_python_metrics-3.0.4.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
83
- rda_python_metrics-3.0.4.dist-info/METADATA,sha256=k9eF-dFAgtZm7D3MKGrig9YQYPcenlg0OGJtCZ3nHMU,6503
84
- rda_python_metrics-3.0.4.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
85
- rda_python_metrics-3.0.4.dist-info/entry_points.txt,sha256=BnO0_Q6GLKOLeEf8fySfQT67RGHm6vrWq8YmOgrLruk,1407
86
- rda_python_metrics-3.0.4.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
87
- rda_python_metrics-3.0.4.dist-info/RECORD,,
75
+ rda_python_metrics-3.0.6.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
76
+ rda_python_metrics-3.0.6.dist-info/METADATA,sha256=ZySRnciB_ztdnv3GCnqk2BAPWPEOao1klQetfYxnnA8,6301
77
+ rda_python_metrics-3.0.6.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
78
+ rda_python_metrics-3.0.6.dist-info/entry_points.txt,sha256=clq4hmexVqZS0rCt68ZY_9gRS8WzZJKevtmPatu5Vao,1271
79
+ rda_python_metrics-3.0.6.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
80
+ rda_python_metrics-3.0.6.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (84.0.0)
2
+ Generator: setuptools (84.2.1)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -11,10 +11,7 @@ filloneorder = rda_python_metrics.filloneorder:main
11
11
  fillosdfusage = rda_python_metrics.fillosdfusage:main
12
12
  fillrdadb = rda_python_metrics.fillrdadb:main
13
13
  filltdsusage = rda_python_metrics.filltdsusage:main
14
- filluser = rda_python_metrics.filluser:main
15
14
  fillzenodousage = rda_python_metrics.fillzenodousage:main
16
- pgperson = rda_python_metrics.pgperson:main
17
- pgusername = rda_python_metrics.pgusername:main
18
15
  setuid_logarch = rda_python_metrics.logarch:main
19
16
  viewallusage = rda_python_metrics.viewallusage:main
20
17
  viewawsusage = rda_python_metrics.viewawsusage:main
@@ -1,216 +0,0 @@
1
- #!/usr/bin/env python3
2
- #
3
- ###############################################################################
4
- #
5
- # Title : filluser
6
- # Author : Zaihua Ji, zji@ucar.edu
7
- # Date : 02/15/2024
8
- # 2025-03-26 transferred to package rda_python_metrics from
9
- # https://github.com/NCAR/rda-database.git
10
- # Purpose : python functions to retrieve info from Oracle database and fill
11
- # table user in PostgreSQL database.schema rdadb.dssdb.
12
- #
13
- # Github : https://github.com/NCAR/rda-python-metrics.git
14
- #
15
- ###############################################################################
16
- import sys
17
- import re
18
- import time
19
- from os import path as op
20
- from rda_python_common import PgLOG
21
- from rda_python_common import PgUtil
22
- from rda_python_common import PgDBI
23
-
24
- TBNAME = 'dssdb.user'
25
-
26
- #
27
- # main function to run this program
28
- #
29
- def main():
30
-
31
- argv = sys.argv[1:]
32
- missed = False
33
- userno = logname = option = None
34
-
35
- for arg in argv:
36
- if arg == "-b":
37
- PgLOG.PGLOG['BCKGRND'] = 1
38
- elif re.match(r'^-[inu]$', arg):
39
- option = arg[1]
40
- if option == 'i':
41
- missed = True
42
- option = None
43
- elif arg[0] == '-':
44
- PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
45
- elif option == "n":
46
- userno = arg
47
- option = None
48
- elif option == "u":
49
- logname = arg
50
- option = None
51
- else:
52
- PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
53
-
54
- if not (missed or userno or logname): PgLOG.show_usage('filluser')
55
-
56
- if not PgLOG.valid_command("pgperson"):
57
- errmsg = PgLOG.PGLOG['SYSERR'] if PgLOG.PGLOG['SYSERR'] else "Cannot find command"
58
- PgLOG.pglog("filluser: Cannot be executed on '{}'\n{}".format(PgLOG.PGLOG['HOSTNAME'], errmsg), PgLOG.LGWNEX)
59
-
60
- PgDBI.dssdb_scname()
61
- PgLOG.cmdlog("filluser {}".format(' '.join(argv)))
62
- if missed: # checking and fill missed ones in user table
63
- fill_missed_users()
64
- else:
65
- fill_one_user(userno, logname)
66
-
67
- sys.exit(0)
68
-
69
- #
70
- # update users with missed info in table dssdb.user
71
- #
72
- def fill_missed_users():
73
-
74
- PgLOG.pglog("Getting incomplete user info", PgLOG.LOGWRN)
75
- pgusers = PgDBI.pgmget(TBNAME, "*", "stat_flag = 'M'", PgLOG.LOGWRN)
76
- cntall = len(pgusers['logname']) if pgusers else 0
77
- s = 's' if cntall > 1 else ''
78
- PgLOG.pglog("{} record{} retrieved at {}".format(cntall, s, PgLOG.current_datetime()), PgLOG.LOGWRN)
79
- if not cntall: return
80
- modcnt = 0
81
- for i in range(cntall):
82
- pgrec = PgUtil.onerecord(pgusers, i)
83
- record = PgDBI.ucar_user_info(pgrec['userno'], pgrec['logname'])
84
- if record:
85
- modcnt += PgDBI.pgupdt(TBNAME, record, "uid = {}".format(pgrec['uid']), PgLOG.LOGWRN)
86
-
87
- if (i%500) == 499:
88
- PgLOG.pglog("{}/{} Records modifiled/processed".format(modcnt, (i+1)), PgLOG.WARNLG)
89
- s = 's' if modcnt > 1 else ''
90
- PgLOG.pglog("{} User Record{} modified".format(modcnt, s), PgLOG.LOGWRN)
91
- return modcnt
92
-
93
- #
94
- # Fill one user for given condition userno or logname
95
- #
96
- def fill_one_user(userno, logname):
97
-
98
- if userno:
99
- msg = "User ID {}: ".format(userno)
100
- else:
101
- msg = "User Login Name {}: ".format(logname)
102
- modcnt = cntadd = 0
103
- newrec = PgDBI.ucar_user_info(userno, logname)
104
- if not newrec:
105
- PgLOG.pglog(msg + "No User info found from People DB", PgLOG.LOGWRN)
106
- return
107
-
108
- cond = 'userno = {}'.format(userno) if userno else "logname = '{}'".format(logname)
109
- pgrec = PgDBI.pgget(TBNAME, "*", cond + " AND until_date is null", PgLOG.LGWNEX)
110
- record = get_user_record(newrec, pgrec, (newrec['stat_flag'] == 'A'))
111
- if record == None:
112
- PgLOG.pglog(msg + "User record saved already", PgLOG.LOGWRN)
113
- return
114
- if record:
115
- if pgrec:
116
- if PgDBI.pgupdt(TBNAME, record, "uid = {}".format(pgrec['uid']), PgLOG.LOGWRN):
117
- PgLOG.pglog(msg + "Existing User record Modified", PgLOG.LOGWRN)
118
- modcnt += 1
119
- else:
120
- if PgDBI.pgadd(TBNAME, record, PgLOG.LGWNEX):
121
- PgLOG.pglog(msg + "New user record added", PgLOG.LOGWRN)
122
- cntadd += 1
123
- else:
124
- record = {'stat_flag' : 'C'}
125
- record['until_date'] = PgUtil.adddate(newrec['start_date'], 0, 0, -1)
126
- if PgDBI.pgupdt(TBNAME, record, "uid = {}".format(pgrec['uid']), PgLOG.LOGWRN):
127
- PgLOG.pglog(msg + "Existing User record Closed", PgLOG.LOGWRN)
128
- modcnt += 1
129
-
130
- record = get_user_record(newrec)
131
- if record and PgDBI.pgadd(TBNAME, record, PgLOG.LGWNEX):
132
- PgLOG.pglog(msg + "Additional New user record added", PgLOG.LOGWRN)
133
- cntadd += 1
134
-
135
- #
136
- # local function: get_user_record(orarec: refer to oracle hush record
137
- # pgrecs: refer to exist, mysql hush records)
138
-
139
- # return: a reference to a new mysql record for update or add
140
- #
141
- def get_user_record(orarec, pgrec = None, neworg = False):
142
-
143
- if not orarec['email']: return None
144
-
145
- ms = re.match("^(.+\@).+\.ucar\.edu$", orarec['email'], re.I)
146
- if ms: orarec['email'] = ms.group(1) + "ucar.edu"
147
-
148
- newrec = {}
149
- if pgrec:
150
- if neworg and PgUtil.diffdate(orarec['start_date'], pgrec['start_date']) <= 0:
151
- neworg = False
152
- if not pgrec['division'] or pgrec['division'] != orarec['division']:
153
- if neworg and orarec['org_type'] == 'NCAR': return 0
154
- newrec['division'] = orarec['division']
155
- if orarec['org_name'] and (not pgrec['org_name'] or pgrec['org_name'] != orarec['org_name']):
156
- if neworg: return 0
157
- newrec['org_name'] = orarec['org_name']
158
- if orarec['country'] and (not pgrec['country'] or orarec['country'] and pgrec['country'] != orarec['country']):
159
- orarec['country'] = PgDBI.set_country_code(orarec)
160
- if not pgrec['country'] or pgrec['country'] != orarec['country']:
161
- if neworg: return 0
162
- newrec['country'] = orarec['country']
163
- if not pgrec['org_type'] or (orarec['org_type'] and pgrec['org_type'] != orarec['org_type']):
164
- orarec['org_type'] = PgDBI.get_org_type(orarec['org_type'], orarec['email'])
165
- if not pgrec['org_type'] or pgrec['org_type'] != orarec['org_type']:
166
- if neworg: return 0
167
- newrec['org_type'] = orarec['org_type']
168
- if not pgrec['email'] or pgrec['email'] != orarec['email']:
169
- if neworg: return 0
170
- newrec['email'] = orarec['email']
171
- if not pgrec['ucaremail'] or pgrec['ucaremail'] != orarec['ucaremail']:
172
- newrec['ucaremail'] = orarec['ucaremail']
173
- if 'until_date' in orarec and PgUtil.diffdate(pgrec['until_date'], orarec['until_date']):
174
- newrec['until_date'] = orarec['until_date']
175
- if not pgrec['userno'] or pgrec['userno'] != orarec['userno']:
176
- newrec['userno'] = orarec['userno']
177
- if not pgrec['upid'] or pgrec['upid'] != orarec['upid']:
178
- newrec['upid'] = orarec['upid']
179
- if not pgrec['logname'] or pgrec['logname'] != orarec['logname']:
180
- newrec['logname'] = orarec['logname']
181
- if orarec['lstname'] and (not pgrec['lstname'] or pgrec['lstname'] != orarec['lstname']):
182
- newrec['lstname'] = orarec['lstname']
183
- if orarec['fstname'] and (not pgrec['fstname'] or pgrec['fstname'] != orarec['fstname']):
184
- newrec['fstname'] = orarec['fstname']
185
- if pgrec['stat_flag'] != orarec['stat_flag']:
186
- newrec['stat_flag'] = orarec['stat_flag']
187
- if 'phoneno' in orarec and orarec['phoneno'] and (not pgrec['phoneno'] or pgrec['phoneno'] != orarec['phoneno']):
188
- newrec['phoneno'] = orarec['phoneno']
189
- if 'faxno' in orarec and orarec['faxno'] and (not pgrec['faxno'] or pgrec['faxno'] != orarec['faxno']):
190
- newrec['faxno'] = orarec['faxno']
191
- if orarec['start_date'] and (not pgrec['start_date'] or PgUtil.diffdate(pgrec['start_date'], orarec['start_date']) > 0):
192
- newrec['start_date'] = orarec['start_date']
193
- elif orarec['stat_flag'] == 'A':
194
- newrec['upid'] = orarec['upid']
195
- newrec['userno'] = orarec['userno']
196
- newrec['logname'] = orarec['logname']
197
- newrec['lstname'] = orarec['lstname']
198
- newrec['fstname'] = orarec['fstname']
199
- newrec['stat_flag'] = orarec['stat_flag']
200
- if orarec['start_date']: newrec['start_date'] = orarec['start_date']
201
- if 'until_date' in orarec: newrec['until_date'] = orarec['until_date']
202
- newrec['division'] = orarec['division']
203
- newrec['org_name'] = orarec['org_name']
204
- newrec['org_type'] = PgDBI.get_org_type(orarec['org_type'], orarec['email'])
205
- newrec['country'] = PgDBI.set_country_code(orarec['email'], orarec['country'])
206
- newrec['email'] = orarec['email']
207
- newrec['ucaremail'] = orarec['ucaremail']
208
- if 'phoneno' in orarec: newrec['phoneno'] = orarec['phoneno']
209
- if 'faxno' in orarec: newrec['faxno'] = orarec['faxno']
210
-
211
- return newrec if newrec else None
212
-
213
- #
214
- # call main() to start program
215
- #
216
- if __name__ == "__main__": main()
@@ -1,194 +0,0 @@
1
- #!/usr/bin/env python3
2
- ###############################################################################
3
- # Title : filluser
4
- # Author : Zaihua Ji, zji@ucar.edu
5
- # Date : 02/15/2024
6
- # 2025-03-26 transferred to package rda_python_metrics from
7
- # https://github.com/NCAR/rda-database.git
8
- # Purpose : python functions to retrieve info from Oracle database and fill
9
- # table user in PostgreSQL database.schema rdadb.dssdb.
10
- # Github : https://github.com/NCAR/rda-python-metrics.git
11
- ###############################################################################
12
- import sys
13
- import re
14
- from rda_python_common.pg_util import PgUtil
15
- from rda_python_common.pg_dbi import PgDBI
16
-
17
- class FillUser(PgUtil, PgDBI):
18
-
19
- """Retrieve user info from the UCAR People API and fill table user in PostgreSQL database rdadb."""
20
-
21
- def __init__(self):
22
- super().__init__()
23
- self.TBNAME = 'dssdb.user'
24
- self.userno = self.logname = None
25
- self.missed = False
26
-
27
- def read_parameters(self):
28
- """Function to read parameters."""
29
- argv = sys.argv[1:]
30
- option = None
31
- for arg in argv:
32
- if arg == "-b":
33
- self.PGLOG['BCKGRND'] = 1
34
- elif re.match(r'^-[inu]$', arg):
35
- option = arg[1]
36
- if option == 'i':
37
- self.missed = True
38
- option = None
39
- elif arg[0] == '-':
40
- self.pglog(arg + ": Invalid Option", self.LGWNEX)
41
- elif option == "n":
42
- self.userno = arg
43
- option = None
44
- elif option == "u":
45
- self.logname = arg
46
- option = None
47
- else:
48
- self.pglog(arg + ": Invalid Parameter", self.LGWNEX)
49
- if not (self.missed or self.userno or self.logname): self.show_usage('filluser')
50
- if not self.valid_command("pgperson"):
51
- errmsg = self.PGLOG['SYSERR'] if self.PGLOG['SYSERR'] else "Cannot find command"
52
- self.pglog("filluser: Cannot be executed on '{}'\n{}".format(self.PGLOG['HOSTNAME'], errmsg), self.LGWNEX)
53
- self.cmdlog("filluser {}".format(' '.join(argv)))
54
-
55
- def start_actions(self):
56
- """Function to start actions."""
57
- self.dssdb_scname()
58
- if self.missed: # checking and fill missed ones in user table
59
- self.fill_missed_users()
60
- else:
61
- self.fill_one_user()
62
-
63
- def fill_missed_users(self):
64
- """Update users with missed info in table dssdb.user."""
65
- self.pglog("Getting incomplete user info", self.LOGWRN)
66
- pgusers = self.pgmget(self.TBNAME, "*", "stat_flag = 'M'", self.LOGWRN)
67
- cntall = len(pgusers['logname']) if pgusers else 0
68
- s = 's' if cntall > 1 else ''
69
- self.pglog("{} record{} retrieved at {}".format(cntall, s, self.current_datetime()), self.LOGWRN)
70
- if not cntall: return
71
- modcnt = 0
72
- for i in range(cntall):
73
- pgrec = self.onerecord(pgusers, i)
74
- record = self.ucar_user_info(pgrec['userno'], pgrec['logname'])
75
- if record:
76
- modcnt += self.pgupdt(self.TBNAME, record, "uid = {}".format(pgrec['uid']), self.LOGWRN)
77
- if (i%500) == 499:
78
- self.pglog("{}/{} Records modifiled/processed".format(modcnt, (i+1)), self.WARNLG)
79
- s = 's' if modcnt > 1 else ''
80
- self.pglog("{} User Record{} modified".format(modcnt, s), self.LOGWRN)
81
- return modcnt
82
-
83
- def fill_one_user(self):
84
- """Fill one user for given condition userno or logname."""
85
- if self.userno:
86
- msg = "User ID {}: ".format(self.userno)
87
- else:
88
- msg = "User Login Name {}: ".format(self.logname)
89
- newrec = self.ucar_user_info(self.userno, self.logname)
90
- if not newrec:
91
- self.pglog(msg + "No User info found from People DB", self.LOGWRN)
92
- return
93
- cond = 'userno = {}'.format(self.userno) if self.userno else "logname = '{}'".format(self.logname)
94
- pgrec = self.pgget(self.TBNAME, "*", cond + " AND until_date is null", self.LGWNEX)
95
- record = self.get_user_record(newrec, pgrec, (newrec['stat_flag'] == 'A'))
96
- if record == None:
97
- self.pglog(msg + "User record saved already", self.LOGWRN)
98
- return
99
- if record:
100
- if pgrec:
101
- if self.pgupdt(self.TBNAME, record, "uid = {}".format(pgrec['uid']), self.LOGWRN):
102
- self.pglog(msg + "Existing User record Modified", self.LOGWRN)
103
- else:
104
- if self.pgadd(self.TBNAME, record, self.LGWNEX):
105
- self.pglog(msg + "New user record added", self.LOGWRN)
106
- else:
107
- record = {'stat_flag' : 'C'}
108
- record['until_date'] = self.adddate(newrec['start_date'], 0, 0, -1)
109
- if self.pgupdt(self.TBNAME, record, "uid = {}".format(pgrec['uid']), self.LOGWRN):
110
- self.pglog(msg + "Existing User record Closed", self.LOGWRN)
111
- record = self.get_user_record(newrec)
112
- if record and self.pgadd(self.TBNAME, record, self.LGWNEX):
113
- self.pglog(msg + "Additional New user record added", self.LOGWRN)
114
-
115
- # local function: get_user_record(orarec: refer to oracle hush record
116
- # pgrecs: refer to exist, mysql hush records)
117
- def get_user_record(self, orarec, pgrec = None, neworg = False):
118
- """ return: a reference to a new mysql record for update or add."""
119
- if not orarec['email']: return None
120
- ms = re.match(r"^(.+@).+\.ucar\.edu$", orarec['email'], re.I)
121
- if ms: orarec['email'] = ms.group(1) + "ucar.edu"
122
- newrec = {}
123
- if pgrec:
124
- if neworg and self.diffdate(orarec['start_date'], pgrec['start_date']) <= 0:
125
- neworg = False
126
- if not pgrec['division'] or pgrec['division'] != orarec['division']:
127
- if neworg and orarec['org_type'] == 'NCAR': return 0
128
- newrec['division'] = orarec['division']
129
- if orarec['org_name'] and (not pgrec['org_name'] or pgrec['org_name'] != orarec['org_name']):
130
- if neworg: return 0
131
- newrec['org_name'] = orarec['org_name']
132
- if orarec['country'] and (not pgrec['country'] or orarec['country'] and pgrec['country'] != orarec['country']):
133
- orarec['country'] = self.set_country_code(orarec['email'], orarec['country'])
134
- if not pgrec['country'] or pgrec['country'] != orarec['country']:
135
- if neworg: return 0
136
- newrec['country'] = orarec['country']
137
- if not pgrec['org_type'] or (orarec['org_type'] and pgrec['org_type'] != orarec['org_type']):
138
- orarec['org_type'] = self.get_org_type(orarec['org_type'], orarec['email'])
139
- if not pgrec['org_type'] or pgrec['org_type'] != orarec['org_type']:
140
- if neworg: return 0
141
- newrec['org_type'] = orarec['org_type']
142
- if not pgrec['email'] or pgrec['email'] != orarec['email']:
143
- if neworg: return 0
144
- newrec['email'] = orarec['email']
145
- if not pgrec['ucaremail'] or pgrec['ucaremail'] != orarec['ucaremail']:
146
- newrec['ucaremail'] = orarec['ucaremail']
147
- if 'until_date' in orarec and self.diffdate(pgrec['until_date'], orarec['until_date']):
148
- newrec['until_date'] = orarec['until_date']
149
- if not pgrec['userno'] or pgrec['userno'] != orarec['userno']:
150
- newrec['userno'] = orarec['userno']
151
- if not pgrec['upid'] or pgrec['upid'] != orarec['upid']:
152
- newrec['upid'] = orarec['upid']
153
- if not pgrec['logname'] or pgrec['logname'] != orarec['logname']:
154
- newrec['logname'] = orarec['logname']
155
- if orarec['lstname'] and (not pgrec['lstname'] or pgrec['lstname'] != orarec['lstname']):
156
- newrec['lstname'] = orarec['lstname']
157
- if orarec['fstname'] and (not pgrec['fstname'] or pgrec['fstname'] != orarec['fstname']):
158
- newrec['fstname'] = orarec['fstname']
159
- if pgrec['stat_flag'] != orarec['stat_flag']:
160
- newrec['stat_flag'] = orarec['stat_flag']
161
- if 'phoneno' in orarec and orarec['phoneno'] and (not pgrec['phoneno'] or pgrec['phoneno'] != orarec['phoneno']):
162
- newrec['phoneno'] = orarec['phoneno']
163
- if 'faxno' in orarec and orarec['faxno'] and (not pgrec['faxno'] or pgrec['faxno'] != orarec['faxno']):
164
- newrec['faxno'] = orarec['faxno']
165
- if orarec['start_date'] and (not pgrec['start_date'] or self.diffdate(pgrec['start_date'], orarec['start_date']) > 0):
166
- newrec['start_date'] = orarec['start_date']
167
- elif orarec['stat_flag'] == 'A':
168
- newrec['upid'] = orarec['upid']
169
- newrec['userno'] = orarec['userno']
170
- newrec['logname'] = orarec['logname']
171
- newrec['lstname'] = orarec['lstname']
172
- newrec['fstname'] = orarec['fstname']
173
- newrec['stat_flag'] = orarec['stat_flag']
174
- if orarec['start_date']: newrec['start_date'] = orarec['start_date']
175
- if 'until_date' in orarec: newrec['until_date'] = orarec['until_date']
176
- newrec['division'] = orarec['division']
177
- newrec['org_name'] = orarec['org_name']
178
- newrec['org_type'] = self.get_org_type(orarec['org_type'], orarec['email'])
179
- newrec['country'] = self.set_country_code(orarec['email'], orarec['country'])
180
- newrec['email'] = orarec['email']
181
- newrec['ucaremail'] = orarec['ucaremail']
182
- if 'phoneno' in orarec: newrec['phoneno'] = orarec['phoneno']
183
- if 'faxno' in orarec: newrec['faxno'] = orarec['faxno']
184
- return newrec if newrec else None
185
-
186
- # main function to execute this script
187
- def main():
188
- object = FillUser()
189
- object.read_parameters()
190
- object.start_actions()
191
- object.pgexit(0)
192
-
193
- # call main() to start program
194
- if __name__ == "__main__": main()
@@ -1,24 +0,0 @@
1
-
2
- filluser - Fill or refresh records in the 'user' table from an Oracle source.
3
-
4
- Retrieves user information from the NCAR ORACLE server and inserts or
5
- updates records in the 'user' table of PostgreSQL database 'rdadb'.
6
-
7
- USAGE:
8
- filluser [-b] (-i | -n UserNumbers | -u UserLoginNames)
9
-
10
- OPTIONS (choose exactly one of -i, -n, or -u):
11
-
12
- -i Refresh users that currently have incomplete information in the
13
- 'user' table.
14
-
15
- -n UserNumbers
16
- Look up users by their NCAR system user numbers.
17
-
18
- -u UserLoginNames
19
- Look up users by their NCAR system login names.
20
-
21
- -b Run in background mode (log to file only, no terminal output).
22
-
23
- EXAMPLE:
24
- filluser -u zji
@@ -1,73 +0,0 @@
1
- #!/usr/bin/env python3
2
- #*******************************************************************
3
- # Title : pgperson.py
4
- # Author : Zaihua Ji, zji@ucar.edu
5
- # Date : 2025-03-27
6
- # Purpose : utility program to retrieve user info from People DB
7
- #
8
- # Github : https://github.com/NCAR/rda-python-metrics.git
9
- #
10
- #*******************************************************************
11
-
12
- import httplib2
13
- import json
14
- import sys
15
- from rda_python_common import PgLOG
16
-
17
- PERSON = [
18
- "upid", # Unique person id
19
- "username", # UCAR login user name
20
- "email", # Email address
21
- "uid", # Scientist id or Unix id
22
- "firstName", # First name
23
- "lastName", # Last name
24
- "forwardEmail" # Forward Email address
25
- ]
26
-
27
- urlfmt="https://people.api.ucar.edu/persons?{}={}&searchScope=all&includeActive=true&includeInactive=true&searchType=advancedSearch"
28
-
29
- #
30
- # main function to execute this script
31
- #
32
- def main():
33
-
34
- pgname = "pgperson"
35
- argv = sys.argv[1:]
36
- argc = len(sys.argv)
37
- optstr = '|'.join(PERSON)
38
- if argc != 3:
39
- print("Usage: {} -({}) OptopnValue".format(pgname, optstr))
40
- sys.exit(0)
41
-
42
- option = optval = None
43
- for arg in argv:
44
- if option:
45
- optval = arg
46
- elif arg[0] == '-':
47
- option = arg[1:]
48
- if option not in PERSON:
49
- PgLOG.pglog("{}: unknown option, must be -({})".format(arg, optstr), PgLOG.LGEREX)
50
- else:
51
- PgLOG.pglog("{}: Value passed in without leading option -({})".format(arg, optstr), PgLOG.LGEREX)
52
-
53
- headers = {'Content-type': 'application/json'}
54
- http=httplib2.Http()
55
- url = urlfmt.format(option, optval)
56
- response, content = http.request(url, 'GET', headers=headers)
57
- status = response.status
58
- if status == 200:
59
- persons=json.loads(content)
60
- for person in persons:
61
- for key, value in person.items():
62
- print("{}<=>{}".format(key, value))
63
- elif status == 399:
64
- print(content)
65
- elif status == 500:
66
- print('Server error')
67
-
68
- sys.exit(0)
69
-
70
- #
71
- # call main() to start program
72
- #
73
- if __name__ == "__main__": main()
@@ -1,51 +0,0 @@
1
- #!/usr/bin/env python3
2
- #*******************************************************************
3
- # Title : pgusername.py
4
- # Author : Zaihua Ji, zji@ucar.edu
5
- # Date : 2025-03-27
6
- # Purpose : utility program to retrieve user info from People DB
7
- # for a given UCAR user login name
8
- #
9
- # Github : https://github.com/NCAR/rda-python-metrics.git
10
- #
11
- #*******************************************************************
12
-
13
- import httplib2
14
- import json
15
- import sys
16
- from rda_python_common import PgLOG
17
-
18
- url="https://people.api.ucar.edu/usernames/"
19
-
20
- #
21
- # main function to execute this script
22
- #
23
- def main():
24
-
25
- pgname = "pgusername"
26
- argc = len(sys.argv)
27
- if argc != 2:
28
- print("Usage: {} UserName".format(pgname))
29
- sys.exit(0)
30
-
31
- uname = sys.argv[1]
32
-
33
- headers = {'Content-type': 'application/json'}
34
- http=httplib2.Http()
35
- response, content = http.request(url + uname, 'GET', headers=headers)
36
- status = response.status
37
- if status == 200:
38
- person=json.loads(content)
39
- for key, value in person.items():
40
- print("{}<=>{}".format(key, value))
41
- elif status == 399:
42
- print(content)
43
- elif status == 500:
44
- print('Server error')
45
-
46
- sys.exit(0)
47
-
48
- #
49
- # call main() to start program
50
- #
51
- if __name__ == "__main__": main()
@@ -1,72 +0,0 @@
1
- #!/usr/bin/env python3
2
- #*******************************************************************
3
- # Title : pgperson.py
4
- # Author : Zaihua Ji, zji@ucar.edu
5
- # Date : 2025-03-27
6
- # 2025-12-19 convert to class PgPerson
7
- # Purpose : utility program to retrieve user info from People DB
8
- # Github : https://github.com/NCAR/rda-python-metrics.git
9
- #*******************************************************************
10
- import httplib2
11
- import json
12
- import sys
13
-
14
- class PgPerson:
15
-
16
- def __init__(self):
17
- super().__init__()
18
- self.PERSON = [
19
- "upid", # Unique person id
20
- "username", # UCAR login user name
21
- "email", # Email address
22
- "uid", # Scientist id or Unix id
23
- "firstName", # First name
24
- "lastName", # Last name
25
- "forwardEmail" # Forward Email address
26
- ]
27
- self.urlfmt = "https://people.api.ucar.edu/persons?{}={}&searchScope=all&includeActive=true&includeInactive=true&searchType=advancedSearch"
28
- self.option = self.optval = None
29
-
30
- # function to read parameters
31
- def read_parameters(self):
32
- pgname = "pgperson"
33
- argv = sys.argv[1:]
34
- argc = len(sys.argv)
35
- optstr = '|'.join(self.PERSON)
36
- if argc != 3:
37
- print("Usage: {} -({}) OptopnValue".format(pgname, optstr))
38
- sys.exit(0)
39
- for arg in argv:
40
- if self.option:
41
- self.optval = arg
42
- elif arg[0] == '-':
43
- self.option = arg[1:]
44
- if self.option not in self.PERSON:
45
- print("{}: unknown option, must be -({})".format(arg, optstr), file=sys.stderr)
46
- else:
47
- print("{}: Value passed in without leading option -({})".format(arg, optstr), file=sys.stderr)
48
-
49
- # function to start actions
50
- def start_actions(self):
51
- headers = {'Content-type': 'application/json'}
52
- http=httplib2.Http()
53
- url = self.urlfmt.format(self.option, self.optval)
54
- response, content = http.request(url, 'GET', headers=headers)
55
- status = response.status
56
- if status != 200:
57
- print("pgperson: HTTP {} from {}\n{}".format(status, url, content.decode(errors='replace')), file=sys.stderr)
58
- sys.exit(1)
59
- persons=json.loads(content)
60
- for person in persons:
61
- for key, value in person.items():
62
- print("{}<=>{}".format(key, value))
63
-
64
- # main function to execute this script
65
- def main():
66
- object = PgPerson()
67
- object.read_parameters()
68
- object.start_actions()
69
- sys.exit(0)
70
-
71
- # call main() to start program
72
- if __name__ == "__main__": main()
@@ -1,53 +0,0 @@
1
- #!/usr/bin/env python3
2
- #*******************************************************************
3
- # Title : pgusername.py
4
- # Author : Zaihua Ji, zji@ucar.edu
5
- # Date : 2025-03-27
6
- # 2025-12-19 convert to class PgUserName
7
- # Purpose : utility program to retrieve user info from People DB
8
- # for a given UCAR user login name
9
- # Github : https://github.com/NCAR/rda-python-metrics.git
10
- #*******************************************************************
11
- import httplib2
12
- import json
13
- import sys
14
-
15
- class PgUserName:
16
-
17
- def __init__(self):
18
- super().__init__()
19
- self.url="https://people.api.ucar.edu/usernames/"
20
- self.uname = None
21
-
22
- # function to read parameters
23
- def read_parameters(self):
24
- pgname = "pgusername"
25
- argc = len(sys.argv)
26
- if argc != 2:
27
- print("Usage: {} UserName".format(pgname))
28
- sys.exit(0)
29
- self.uname = sys.argv[1]
30
-
31
- # function to start actions
32
- def start_actions(self):
33
- headers = {'Content-type': 'application/json'}
34
- http=httplib2.Http()
35
- url = self.url + self.uname
36
- response, content = http.request(url, 'GET', headers=headers)
37
- status = response.status
38
- if status != 200:
39
- print("pgusername: HTTP {} from {}\n{}".format(status, url, content.decode(errors='replace')), file=sys.stderr)
40
- sys.exit(1)
41
- person=json.loads(content)
42
- for key, value in person.items():
43
- print("{}<=>{}".format(key, value))
44
-
45
- # main function to execute this script
46
- def main():
47
- object = PgUserName()
48
- object.read_parameters()
49
- object.start_actions()
50
- sys.exit(0)
51
-
52
- # call main() to start program
53
- if __name__ == "__main__": main()