rda-python-metrics 1.0.4__py3-none-any.whl

rda_python_metrics/filluser.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : filluser
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 02/15/2024
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python functions to retrieve info from Oracle database and fill
+#             table user in PostgreSQL database.schema rdadb.dssdb.
+#
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+import sys
+import re
+import time
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgDBI
+
+TBNAME = 'dssdb.user'
+
+#
+# main function to run this program
+#
+def main():
+
+   argv = sys.argv[1:]
+   missed = False
+   userno = logname = option = None
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif re.match(r'^-[inu]$', arg):
+         option = arg[1]
+         if option == 'i':
+            missed = True
+            option = None
+      elif arg[0] == '-':
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option == "n":
+         userno = arg
+         option = None
+      elif option == "u":
+         logname = arg
+         option = None
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+
+   if not (missed or userno or logname): PgLOG.show_usage('filluser')
+
+   if not PgLOG.valid_command("pgperson"):
+      errmsg = PgLOG.PGLOG['SYSERR'] if PgLOG.PGLOG['SYSERR'] else "Cannot find command"
+      PgLOG.pglog("filluser: Cannot be executed on '{}'\n{}".format(PgLOG.PGLOG['HOSTNAME'], errmsg), PgLOG.LGWNEX)
+
+   PgDBI.dssdb_scname()
+   PgLOG.cmdlog("filluser {}".format(' '.join(argv)))
+   if missed: # checking and fill missed ones in user table
+      fill_missed_users()
+   else:
+      fill_one_user(userno, logname)
+
+   sys.exit(0)
+
+#
+# update users with missed info in table dssdb.user
+#
+def fill_missed_users():
+
+   PgLOG.pglog("Getting incomplete user info", PgLOG.LOGWRN)
+   pgusers = PgDBI.pgmget(TBNAME, "*", "stat_flag = 'M'", PgLOG.LOGWRN)
+   cntall = len(pgusers['logname']) if pgusers else 0
+   s = 's' if cntall > 1 else ''
+   PgLOG.pglog("{} record{} retrieved at {}".format(cntall, s, PgLOG.current_datetime()), PgLOG.LOGWRN)
+   if not cntall: return
+   modcnt = 0
+   for i in range(cntall):
+      pgrec = PgUtil.onerecord(pgusers, i)
+      record = PgDBI.ucar_user_info(pgrec['userno'], pgrec['logname'])
+      if record:
+         modcnt += PgDBI.pgupdt(TBNAME, record, "uid = {}".format(pgrec['uid']), PgLOG.LOGWRN)
+      
+      if (i%500) == 499:
+         PgLOG.pglog("{}/{} Records modifiled/processed".format(modcnt, (i+1)), PgLOG.WARNLG)
+   s = 's' if modcnt > 1 else ''
+   PgLOG.pglog("{} User Record{} modified".format(modcnt, s), PgLOG.LOGWRN)
+   return modcnt
+
+#
+# Fill one user for given condition userno or logname
+#
+def fill_one_user(userno, logname):
+
+   if userno:
+      msg = "User ID {}: ".format(userno)
+   else:
+      msg = "User Login Name {}: ".format(logname)
+   modcnt = cntadd = 0
+   newrec = PgDBI.ucar_user_info(userno, logname)
+   if not newrec:
+      PgLOG.pglog(msg + "No User info found from People DB", PgLOG.LOGWRN)
+      return
+
+   cond = 'userno = {}'.format(userno) if userno else  "logname = '{}'".format(logname)
+   pgrec = PgDBI.pgget(TBNAME, "*", cond + " AND until_date is null", PgLOG.LGWNEX)
+   record = get_user_record(newrec, pgrec, (newrec['stat_flag'] == 'A'))
+   if record == None:
+      PgLOG.pglog(msg + "User record saved already", PgLOG.LOGWRN)
+      return
+   if record:
+      if pgrec:
+         if PgDBI.pgupdt(TBNAME, record, "uid = {}".format(pgrec['uid']), PgLOG.LOGWRN):
+            PgLOG.pglog(msg + "Existing User record Modified", PgLOG.LOGWRN)
+            modcnt += 1
+      else:
+         if PgDBI.pgadd(TBNAME, record, PgLOG.LGWNEX):
+            PgLOG.pglog(msg + "New user record added", PgLOG.LOGWRN)
+            cntadd += 1               
+   else:
+      record = {'stat_flag' : 'C'}
+      record['until_date'] = PgUtil.adddate(newrec['start_date'], 0, 0, -1)
+      if PgDBI.pgupdt(TBNAME, record, "uid = {}".format(pgrec['uid']), PgLOG.LOGWRN):
+         PgLOG.pglog(msg + "Existing User record Closed", PgLOG.LOGWRN)
+         modcnt += 1
+
+      record = get_user_record(newrec)
+      if record and PgDBI.pgadd(TBNAME, record, PgLOG.LGWNEX):
+         PgLOG.pglog(msg + "Additional New user record added", PgLOG.LOGWRN)
+         cntadd += 1
+
+#
+# local function: get_user_record(orarec: refer to oracle hush record 
+#                                 pgrecs: refer to exist, mysql hush records)
+
+#         return: a reference to a new mysql record for update or add
+#
+def get_user_record(orarec, pgrec = None, neworg = False):
+
+   if not orarec['email']: return None
+
+   ms = re.match("^(.+\@).+\.ucar\.edu$", orarec['email'], re.I)
+   if ms: orarec['email'] = ms.group(1) + "ucar.edu"
+
+   newrec = {}   
+   if pgrec:
+      if neworg and PgUtil.diffdate(orarec['start_date'], pgrec['start_date']) <= 0:
+         neworg = False
+      if not pgrec['division'] or pgrec['division'] != orarec['division']:
+         if neworg and orarec['org_type'] == 'NCAR': return 0
+         newrec['division'] = orarec['division']
+      if orarec['org_name'] and (not pgrec['org_name'] or pgrec['org_name'] != orarec['org_name']):
+         if neworg: return 0
+         newrec['org_name'] = orarec['org_name']
+      if orarec['country'] and (not pgrec['country'] or orarec['country'] and pgrec['country'] != orarec['country']):
+         orarec['country'] = PgDBI.set_country_code(orarec)
+         if not pgrec['country'] or pgrec['country'] != orarec['country']:
+            if neworg: return 0
+            newrec['country'] = orarec['country']
+      if not pgrec['org_type'] or (orarec['org_type'] and pgrec['org_type'] != orarec['org_type']):
+         orarec['org_type'] = PgDBI.get_org_type(orarec['org_type'], orarec['email'])
+         if not pgrec['org_type'] or pgrec['org_type'] != orarec['org_type']:
+            if neworg: return 0
+            newrec['org_type'] = orarec['org_type']
+      if not pgrec['email'] or pgrec['email'] != orarec['email']:
+         if neworg: return 0
+         newrec['email'] = orarec['email']
+      if not pgrec['ucaremail'] or pgrec['ucaremail'] != orarec['ucaremail']:
+         newrec['ucaremail'] = orarec['ucaremail']
+      if 'until_date' in orarec and PgUtil.diffdate(pgrec['until_date'], orarec['until_date']):
+         newrec['until_date'] = orarec['until_date']
+      if not pgrec['userno'] or pgrec['userno'] != orarec['userno']:
+         newrec['userno'] = orarec['userno']
+      if not pgrec['upid'] or pgrec['upid'] != orarec['upid']:
+         newrec['upid'] = orarec['upid']
+      if not pgrec['logname'] or pgrec['logname'] != orarec['logname']:
+         newrec['logname'] = orarec['logname']
+      if orarec['lstname'] and (not pgrec['lstname'] or pgrec['lstname'] != orarec['lstname']):
+         newrec['lstname'] = orarec['lstname']
+      if orarec['fstname'] and (not pgrec['fstname'] or pgrec['fstname'] != orarec['fstname']):
+         newrec['fstname'] = orarec['fstname']
+      if pgrec['stat_flag'] != orarec['stat_flag']:
+         newrec['stat_flag'] = orarec['stat_flag']
+      if 'phoneno' in orarec and orarec['phoneno'] and (not pgrec['phoneno'] or pgrec['phoneno'] != orarec['phoneno']):
+         newrec['phoneno'] = orarec['phoneno']
+      if 'faxno' in orarec and orarec['faxno'] and (not pgrec['faxno'] or pgrec['faxno'] != orarec['faxno']):
+         newrec['faxno'] = orarec['faxno']
+      if orarec['start_date'] and (not pgrec['start_date'] or PgUtil.diffdate(pgrec['start_date'], orarec['start_date']) > 0):
+         newrec['start_date'] = orarec['start_date']      
+   elif orarec['stat_flag'] == 'A':
+      newrec['upid'] = orarec['upid']
+      newrec['userno'] = orarec['userno']
+      newrec['logname'] = orarec['logname']
+      newrec['lstname'] = orarec['lstname']
+      newrec['fstname'] = orarec['fstname']
+      newrec['stat_flag'] = orarec['stat_flag']
+      if orarec['start_date']: newrec['start_date'] = orarec['start_date']
+      if 'until_date' in orarec: newrec['until_date'] = orarec['until_date']
+      newrec['division'] = orarec['division']
+      newrec['org_name'] = orarec['org_name']
+      newrec['org_type'] = PgDBI.get_org_type(orarec['org_type'], orarec['email'])
+      newrec['country'] = PgDBI.set_country_code(orarec['email'], orarec['country'])
+      newrec['email'] = orarec['email']
+      newrec['ucaremail'] = orarec['ucaremail']
+      if 'phoneno' in orarec: newrec['phoneno'] = orarec['phoneno']
+      if 'faxno' in orarec: newrec['faxno'] = orarec['faxno']
+
+   return newrec if newrec else None
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/filluser.usg

@@ -0,0 +1,16 @@
+
+ Retrieves user information from ORACLE SERVER and fill 
+ table 'user' in MySQL database 'dssdb'.
+
+ Usage: filluser [-b] [-i] [-n UserNumber] [-u UserLonginName]
+
+      select one of the options, -i, -n or -u each time to run this application. 
+
+      - Option -b, log process information into logfile only;
+
+      - Option -i, get users with incomplete information in 'dssdb.user';
+
+      - Option -n, get users by their user numbers on NCAR system;
+
+      - Option -u, get users by their login names on NCAR system;
+

rda_python_metrics/logarch.py

@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : logarch
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 11/19/2020
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-utility-programs.git
+#   Purpose : archive log files automatically
+#
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+##################################################################################
+
+import sys
+import re
+from os import path as op
+import glob
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgSIG
+
+# the defined options for archiving different logs
+WLOG = 0x01  # archive web log
+TLOG = 0x02  # archive tds log
+DLOG = 0x04  # archive dssdb logs
+SLOG = 0x08  # append dssdb sub batch logs
+ALOG = 0x10  # archive AWS web log
+
+LOGS = {
+   'OPTION' : 0,
+   'AWSLOG' : PgLOG.PGLOG["TRANSFER"] + "/AWSera5log",
+   'WEBLOG' : PgLOG.PGLOG["DSSDATA"] + "/work/logs/gridftp",
+   'MGTLOG' : "/data/logs",
+   'TDSLOG' : "/data/logs/nginx",
+   'RDALOG' : PgLOG.PGLOG['LOGPATH'],
+   'LOGPATH' : None,
+   'CHKLOG' : 1,
+   'DECSLOGS' : PgLOG.PGLOG['DECSHOME'] + "/DECSLOGS"
+}
+
+BIDS = {}
+
+#
+# main function to excecute this script
+#
+def main():
+
+   pgname = "logarch"
+   argv = sys.argv[1:]
+   smonth = None
+
+   # set different log file
+   PgLOG.PGLOG['LOGFILE'] = pgname + '.log'
+   PgLOG.set_suid(PgLOG.PGLOG['EUID'])
+   
+   option = None
+   for arg in argv:
+      ms = re.match(r'^-([abdmnpstw])', arg)
+      if ms:
+         option = ms.group(1)
+         if option in 'mp': continue
+         if option == "b":
+            PgLOG.PGLOG['BCKGRND'] = 1
+         elif option == "d":
+            LOGS['OPTION'] |= DLOG
+         elif option == "w":
+            LOGS['OPTION'] |= WLOG
+         elif option == "a":
+            LOGS['OPTION'] |= ALOG
+         elif option == "s":
+            LOGS['OPTION'] |= SLOG
+         elif option == "t":
+            LOGS['OPTION'] |= TLOG
+         elif option == "n":
+            LOGS['CHKLOG'] = 0
+         option = None
+      elif option == 'm':
+         smonth = arg
+      elif option == 'p':
+         LOGS['LOGPATH'] = arg
+      else:
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      option = None
+
+   if not LOGS['OPTION']: PgLOG.show_usage(pgname)
+   PgLOG.cmdlog("{} {}".format(pgname, ' '.join(argv)))
+
+   if LOGS['OPTION']&SLOG: append_dssdb_sublog()
+   if LOGS['OPTION']&DLOG: archive_dssdb_log()
+   if LOGS['OPTION']&WLOG: archive_web_log(smonth)
+   if LOGS['OPTION']&ALOG: archive_aws_log(smonth)
+   if LOGS['OPTION']&TLOG: archive_tds_log(smonth)
+
+   PgLOG.cmdlog(None, 0, PgLOG.LOGWRN|PgLOG.SNDEML)   
+   PgLOG.pgexit(0)
+
+def get_year_month(smonth):
+
+   if not smonth: smonth = PgUtil.adddate(PgUtil.curdate('YYYY-MM') + '-01', 0, -1, 0, 'YYYY-MM')
+   ms = re.match(r'^(\d+)-(\d+)', smonth)
+   return [ms.group(1), '{:02}'.format(int(ms.group(2)))]
+
+#
+# Archive globus web log files to LOGS['DECSLOGS']
+#
+def archive_web_log(smonth):
+
+   (yr, mn) = get_year_month(smonth)
+   PgFile.change_local_directory(LOGS['DECSLOGS'], PgLOG.LGEREM)
+   logpath = LOGS['LOGPATH'] if LOGS['LOGPATH'] else LOGS['WEBLOG']
+   afile = "globusweb{}-{}.log.tar".format(yr, mn)
+   dfile = "./WEBLOG/{}.gz".format(afile)
+   if op.exists(dfile):
+      PgLOG.pglog("{}: file exists already under {}, remove it before backup again".format(dfile, LOGS['DECSLOGS']), PgLOG.LGWNEM)
+      return
+
+   if op.exists(afile): PgFile.delete_local_file(afile)
+
+   logfiles = sorted(glob.glob("{}/access_log_gridftp??_{}??{}".format(logpath, mn, yr)))
+   topt = '-cvf'
+   tcnt = 0
+   for logfile in logfiles:
+      if not op.exists(logfile):
+         PgLOG.pglog(logfile + ": file not exists", PgLOG.LGWNEM)
+         continue
+      if  op.getsize(logfile) == 0:
+         PgLOG.pglog(logfile + ": empty file", PgLOG.LGWNEM)
+         continue
+      lfile = op.basename(logfile)
+      tcmd = "tar {} {} -C {} {}".format(topt, afile, logpath, lfile)
+      tcnt += PgLOG.pgsystem(tcmd, PgLOG.LGWNEM, 5)
+      topt = '-uvf'
+
+   
+   if tcnt > 0:
+      PgLOG.pgsystem("gzip " + afile, PgLOG.LGWNEM, 5)
+      afile += '.gz'
+      PgFile.move_local_file(dfile, afile, PgLOG.LGWNEM)
+      s = 's' if tcnt > 1 else ''
+      PgLOG.pglog("{}: {} globus log{} tarred, gzipped and archived at {}".format(afile, tcnt, s, PgLOG.current_datetime()), PgLOG.LGWNEM)
+
+#
+# Archive AWS web log files to LOGS['DECSLOGS']
+#
+def archive_aws_log(smonth):
+
+   (yr, mn) = get_year_month(smonth)
+   PgFile.change_local_directory(LOGS['DECSLOGS'], PgLOG.LGEREM)
+   logpath = LOGS['LOGPATH'] if LOGS['LOGPATH'] else LOGS['AWSLOG']
+   afile = "awsweb{}-{}.log.tar".format(yr, mn)
+   dfile = "./AWSLOG/{}.gz".format(afile)
+   if op.exists(dfile):
+      PgLOG.pglog("{}: file exists already under {}, remove it before backup again".format(dfile, LOGS['DECSLOGS']), PgLOG.LGWNEM)
+      return
+
+   if op.exists(afile): PgFile.delete_local_file(afile)
+
+   lfile = "{}/{}".format(yr, mn)
+   tcmd = "tar -cvf {} -C {} {}".format(afile, logpath, lfile)
+   PgLOG.pgsystem(tcmd, PgLOG.LGWNEM, 5)
+
+   PgLOG.pgsystem("gzip " + afile, PgLOG.LGWNEM, 5)
+   afile += '.gz'
+   PgFile.move_local_file(dfile, afile, PgLOG.LGWNEM)
+   PgLOG.pglog("{}: AWS logs tarred, gzipped and archived at {}".format(afile, PgLOG.current_datetime()), PgLOG.LGWNEM)
+
+#
+# Archive monthly tds logs under DECSLOGS/TDSLOG/
+#
+def archive_tds_log(smonth):
+
+   (yr, mn) = get_year_month(smonth)
+   PgFile.change_local_directory(LOGS['DECSLOGS'], PgLOG.LGEREM)
+   logpath = LOGS['LOGPATH'] if LOGS['LOGPATH'] else LOGS['TDSLOG']
+   afile = "thredds{}-{}.log.tar".format(yr, mn)
+   dfile = "./TDSLOG/{}.gz".format(afile)
+   if op.exists(dfile):
+      PgLOG.pglog("{}: file exists already under {}, remove it before backup again".format(dfile, LOGS['DECSLOGS']), PgLOG.LGWNEM)
+      return
+
+   if op.exists(afile): PgFile.delete_local_file(afile)
+
+   logfiles = sorted(glob.glob("{}/{}-{}-??.access.log".format(logpath, yr, mn)))
+   topt = '-cvf'
+   tcnt = 0
+   for logfile in logfiles:
+      if not op.exists(logfile):
+         PgLOG.pglog(logfile + ": file not exists", PgLOG.LGWNEM)
+         continue
+      if  op.getsize(logfile) == 0:
+         PgLOG.pglog(logfile + ": empty file", PgLOG.LGWNEM)
+         continue
+      lfile = op.basename(logfile)
+      tcmd = "tar {} {} -C {} {}".format(topt, afile, logpath, lfile)
+      tcnt += PgLOG.pgsystem(tcmd, PgLOG.LGWNEM, 5)
+      topt = '-uvf'
+
+   if tcnt > 0:
+      PgLOG.pgsystem("gzip " + afile, PgLOG.LGWNEM, 5)
+      afile += '.gz'
+      PgFile.move_local_file(dfile, afile, PgLOG.LGWNEM)
+      s = 's' if tcnt > 1 else ''
+      PgLOG.pglog("{}: {} thredds log{} tarred, gzipped and archived at {}".format(afile, tcnt, s, PgLOG.current_datetime()), PgLOG.LGWNEM)
+
+#
+# Archive current dssdb logs  onto hpss under /DSS/RDADB/LOG
+#
+def archive_dssdb_log():
+
+   cntall = 0
+   logfile = "{}_dblog{}.tar".format(PgLOG.PGLOG['HOSTNAME'], PgUtil.curdate("YYMMDD"))
+   dfile = "{}/RDADB/LOG/{}.gz".format(LOGS['DECSLOGS'], logfile)
+   if LOGS['CHKLOG'] and check_decs_archive(dfile):
+      return PgLOG.pglog(dfile + ": archived already", PgLOG.LGWNEM)
+
+   PgFile.change_local_directory(LOGS['RDALOG'], PgLOG.LWEMEX)
+
+   # collect all the large log/err files
+   files = sorted(glob.glob("*.log") + glob.glob("*.err"))
+   for file in files:
+      info = PgFile.check_local_file(file, 2)
+      if(not info or info['data_size'] < 10000): continue   # skip log files small than 10KB
+
+      PgLOG.pgsystem("cp -p -f {} backup/{}".format(file, file), PgLOG.LWEMEX, 5)
+      if info['logname'] != PgLOG.PGLOG['RDAUSER']: PgLOG.pgsystem("rm -rf " + file)
+      PgLOG.pgsystem("cat /dev/null > " + file, 0, 1024)
+      if op.exists(logfile):
+         PgLOG.pgsystem("tar -uvf {} -C backup {}".format(logfile, file), PgLOG.LWEMEX, 5)
+      else:
+         PgLOG.pgsystem("tar -cvf {} -C backup {}".format(logfile, file), PgLOG.LWEMEX, 5)
+      cntall += 1
+
+   if cntall > 0:
+      if LOGS['CHKLOG']:
+         if backup_decsdata_file(logfile, logfile, dfile, 1):
+            s = 's' if cntall > 1 else ''
+            PgLOG.pglog("{} dssdb log{} archived at {}".format(cntall, s, PgLOG.current_datetime()), PgLOG.LGWNEM)
+      else:
+         PgLOG.pgsystem("gzip " + logfile, PgLOG.LWEMEX, 5)
+         logfile += ".gz"
+         PgLOG.pgsystem("mv -f {} backup/".format(logfile), PgLOG.LWEMEX, 5)
+
+#
+# append individual batch logs to common stdout/error logs 
+#
+def append_dssdb_sublog():
+
+   logpath = LOGS['LOGPATH'] if LOGS['LOGPATH'] else LOGS['RDALOG']
+   PgFile.change_local_directory(logpath, PgLOG.LGWNEX);   
+
+   if not PgLOG.valid_command(PgLOG.BCHCMDS['PBS']):
+      PgLOG.pglog("Must run on PBS Nodes to append sublogs", PgLOG.LGEREM)
+      return
+
+   add_sublog_files('OU', 'log')
+   add_sublog_files('ER', 'err', 37)
+
+#
+# add individual sublog files to the common ones
+#
+def add_sublog_files(fext, sext, minsize = 0):
+
+   subname = 'rdaqsub'
+   pattern = r'^(\d+)\.'
+   afiles = PgFile.local_glob("{}/*.{}".format(subname, fext), 3, PgLOG.LGWNEX)
+   if not afiles:
+      PgLOG.pglog("{}: NO '{}' file found to collect".format(subname, fext), PgLOG.LOGWRN)
+      return
+   if minsize:
+      tmin = PgLOG.PGLOG['MINSIZE']
+      PgLOG.PGLOG['MINSIZE'] = minsize
+   acnt = fcnt = 0
+   sfiles = {}
+   for afile in afiles:
+      fcnt += 1
+      finfo = afiles[afile]
+      sfiles[op.basename(afile)] = "{} {} {}".format(finfo['date_modified'], finfo['time_modified'], finfo['logname'])
+
+   afiles = sorted(sfiles)
+   logfile = "PBS_sublog.{}".format(sext)
+   for afile in afiles:
+      ms = re.match(pattern, afile)
+      if ms:
+         bid = int(ms.group(1))
+         if bid not in BIDS: BIDS[bid] = PgSIG.check_pbs_process(bid, afile)
+         if BIDS[bid] > 0: continue
+      else:
+         continue
+
+      sfile = "{}/{}".format(subname, afile)
+      if minsize and PgFile.local_file_size(sfile, 1) < 1: continue
+      PgLOG.pgsystem("echo '{}: {}' >> {}".format(bid, sfiles[afile], logfile), PgLOG.LGWNEX, 5+1024)
+      PgLOG.pgsystem("cat {} >> {}".format(sfile, logfile), PgLOG.LGWNEX, 5+1024)
+      PgFile.delete_local_file(sfile, PgLOG.LOGWRN)
+      acnt += 1
+   if fcnt > 0:
+      s = 's' if fcnt > 1 else ''
+      PgLOG.pglog("{}: {} of {} '{}' file{} appended at {}".format(logfile, acnt, fcnt, fext, s, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+   if minsize: PgLOG.PGLOG['MINSIZE'] = tmin
+
+#
+# backup a log file to decsdata area
+#
+def backup_decsdata_file(logfile, locfile, dfile, skipcheck = 0):
+   
+   ret = 0
+   if op.getsize(logfile) == 0:
+      PgLOG.pglog(logfile + ": Empty log file", PgLOG.LGWNEM)
+      return 0
+   if not skipcheck and check_decs_archive(dfile, logfile, 1):
+      return 0   # archived already
+
+   if locfile != logfile:
+      locfile = "{}/{}".format(PgLOG.PGLOG['TMPPATH'], locfile)
+      if not PgFile.local_copy_local(locfile, logfile, PgLOG.LGWNEM): return 0
+   lfile = locfile
+   locfile += ".gz"
+   if PgFile.check_local_file(locfile, 0, PgLOG.LGWNEM): PgFile.delete_local_file(locfile, PgLOG.LGWNEM)
+   PgLOG.pgsystem("gzip " + lfile, PgLOG.LWEMEX, 5)
+
+   PgLOG.pglog("archive {} to {}".format(locfile, dfile), PgLOG.LGWNEM)
+   if PgFile.local_copy_local(dfile, locfile, PgLOG.LGWNEM):
+      size = check_decs_archive(dfile)
+      if size:
+         PgLOG.pglog("{}: archived as {}({})".format(logfile, dfile, size), PgLOG.LGWNEM)
+         ret = 1
+   
+   if op.exists(locfile) and ret: PgLOG.pgsystem("rm -f " + locfile, PgLOG.LGWNEM)
+
+   return ret
+
+#
+# return decs file size if archived already; otherwise 0
+#
+def check_decs_archive(afile, logfile = None, checktime = 0):
+   
+   ainfo =  PgFile.check_local_file(afile, 1)
+   if not ainfo: return 0
+   size = ainfo['data_size']
+   if logfile:
+      linfo = PgFile.check_local_file(logfile, 1, PgLOG.LGWNEM)
+      if linfo:
+         if checktime:
+            if linfo['date_modified'] > ainfo['date_modified']: size = 0
+         elif size < linfo['data_size']:
+            size = 0
+
+      if size > 0: PgLOG.pglog("{}: archived on {} as {}({})".format(logfile, ainfo['date_modified'], afile, size), PgLOG.LGWNEM)
+
+   return size
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/logarch.usg

@@ -0,0 +1,27 @@
+
+ Archive RDA log files onto to DECSDATA area. This application can be run
+   at any time. It checks if the given log files are archived or not and will
+   ignore the ones that are backed up already.
+
+ Usage: logarch [-b] [-a] [-d] [-w] [-s] [-t] [-p LogPath]  \
+                [-m YearMonth] [-n]
+
+      - Option -b, do not display processing info on screen;
+
+      - Option -a, archive aws log files;
+
+      - Option -d, archive dssdb log files;
+      
+      - Option -p, provide a LogPath if not the fefault one;
+
+      - Option -s, append PBS sublog files to common log files;
+
+      - Option -t, archive tds log files;
+
+      - Option -w, archive web log files;
+
+      - Option -m, Provided a specified YearMonth (in form of YYYY-MM) to backup logs;
+
+      - Option -n, do not check if a log file is backed up already;
+
+      Set at least one of the options -a, -d, -s, -t and -w to run this application.

rda_python_metrics/pgperson.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+#*******************************************************************
+#     Title : pgperson.py
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 2025-03-27
+#   Purpose : utility program to retrieve user info from People DB
+#
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+#*******************************************************************
+
+import httplib2
+import json
+import sys
+from rda_python_common import PgLOG
+
+PERSON = [
+   "upid",          # Unique person id
+   "username",      # UCAR login  user name
+   "email",         # Email address
+   "uid",           # Scientist id or Unix id
+   "firstName",     # First name
+   "lastName",      # Last name
+   "forwardEmail"   # Forward Email address
+]
+
+urlfmt="https://people.api.ucar.edu/persons?{}={}&searchScope=all&includeActive=true&includeInactive=true&searchType=advancedSearch"
+
+#
+# main function to excecute this script
+#
+def main():
+
+   pgname = "pgperson"
+   argv = sys.argv[1:]
+   optstr = '|'.join(PERSON)
+   if sys.argc != 3:
+      print("Usage: {} -({}) OptopnValue".format(pgname, optstr))
+      sys.exit(0)
+
+   option = optval = None
+   for arg in argv:
+      if option:
+         optval = arg
+      elif arg[0] == '-':
+         option = arg[1:]
+         if option not in PERSON:
+            PgLOG.pglog("{}: unknown option, must be -({})".format(arg, optstr), PgLOG.LGEREX)
+      else:
+         PgLOG.pglog("{}: Value passed in without leading option -({})".format(arg, optstr), PgLOG.LGEREX)
+
+   headers = {'Content-type': 'application/json'}
+   http=httplib2.Http()
+   url = urlfmt.format(option, optval)
+   response, content = http.request(url, 'GET', headers=headers)
+   status = response.status
+   if status == 200:
+      persons=json.loads(content)
+      for person in persons:
+         for key, value in person.items():
+            print("{}<=>{}".format(key, value))
+   elif status == 399:
+      print(content)
+   elif status == 500:
+      print('Server error')
+
+   sys.exit(0)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()