rda-python-metrics 1.0.4__py3-none-any.whl

rda_python_metrics/fillendtime.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillendtime
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 04/08/2024
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to fill field dlupdt.endtime from enddate/endhour
+# 
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+import glob
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from . import PgIPInfo
+
+# the define options for gathering ipinfo data
+MONTH = 0x02  # fix data usages for given months
+YEARS = 0x04  # fix data usages for given years
+NDAYS = 0x08  # fix data usages in recent number of days
+MULTI = (MONTH|YEARS)
+SINGL = (NDAYS)
+
+IPINFO = {
+   'USGTBL'  : ['ipinfo', 'allusage', 'tdsusage'],
+   'CDATE' : PgUtil.curdate(),
+}
+
+#
+# main function to run this program
+#
+def main():
+
+   dsids = []    # empty for all datasets
+   argv = sys.argv[1:]
+   option = 0
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      else:
+         dsids.append(PgUtil.format_dataset_id(arg))
+
+   PgDBI.dssdb_dbname()
+
+   if dsids:
+      for dsid in dsids:
+         fill_endtime(dsid)
+   else:
+      fill_endtime()
+   sys.exit(0)
+
+
+
+#
+# Fill endtime in table dssdb.dlupdt
+#
+def fill_endtime(dsid = None):
+
+   dsids = []
+   cnd = "dsid = '{}' AND ".format(dsid) if dsid else ''
+   cnd += 'enddate <> NULL ORDER BY dsid, lindex'
+   pgrecs = PgDBI.pgmget('dlupdt', 'lindex, dsid, enddate, endhour', cnd)
+
+   cnt = len(pgrecs['lindex']) if pgrecs else 0
+   for i in range(cnt):
+      lidx = pgrecs['lindex'][i]
+      edate = pgrecs['enddate'][i]
+      ehour = pgrecs['endhour'][i]
+      dsid = pgrecs['dsid'][i]
+      if dsid not in dsids: dsids.append()
+      if ehour is None: ehour = 23
+      etime = "{} {}:59:59".format(edate, ehour)
+      PgDBI.pgexec("UPDATE dlupdt SET endtime = '{}' WHERE lindex = {}".format(etime, lidx), PgLOG.LGEREX)
+
+   s = 's' if cnt > 1 else ''
+   dscnt = len(dsids)
+   dsstr = dsids[0] if dscnt == 1 else '{} datasets'.format(dscnt)
+   PgLOG.pglog("{}: {} records updated for dssdb.dlupdt.endtime".format(dsstr, cnt, s), PgLOG.LOGWRN)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/fillglobususage.py

@@ -0,0 +1,287 @@
+##!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillglobususage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 03/11/2022
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to retrieve info from Globus logs 
+#             and fill table wusages in PgSQL database dssdb.
+# 
+#    Github : https://github.com/NCAR/rda-pythn-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+import glob
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from rda_python_common import PgSplit
+from . import PgIPInfo
+
+USAGE = {
+   'PGTBL'  : "wusage",
+   'GBSDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/logs/gridftp/",
+   'GBSLOG' : "access_log_gridftp0{}_{}",
+}
+
+#
+# main function to run this program
+#
+def main():
+
+   params = []  # array of input values
+   argv = sys.argv[1:]
+   option = None
+   datelimits = [None, None]
+   
+
+   for arg in argv:
+      ms = re.match(r'^-(b|d|p|N)$', arg)
+      if ms:
+         opt = ms.group(1)
+         if opt == 'b':
+            PgLOG.PGLOG['BCKGRND'] = 1
+         elif option:
+            PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
+         else:
+            option = opt
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         params.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+   
+   if not (option and params): PgLOG.show_usage('fillglobususage')
+
+   PgDBI.dssdb_dbname()
+   cmdstr = "fillglobususage {}".format(' '.join(argv))
+   PgLOG.cmdlog(cmdstr)
+   PgFile.change_local_directory(USAGE['GBSDIR'])
+   filenames = get_log_file_names(option, params, datelimits)
+   if filenames:
+      fill_globus_usages(filenames, datelimits)
+   else:
+      PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
+
+   PgLOG.pglog(None, PgLOG.LOGWRN)
+   sys.exit(0)
+
+#
+# get the log file dates 
+#
+def get_log_file_names(option, params, datelimits):
+
+   filenames = []
+   if option == 'd':
+      for pdate in params:
+         fdate = PgUtil.format_date(pdate, 'MMDDYYYY')
+         fname = USAGE['GBSLOG'].format('?', fdate)
+         fnames = glob.glob(fname)
+         if fnames: filenames.extend(sorted(fnames))
+   else:
+      if option == 'N':
+         edate = PgUtil.curdate()
+         pdate = datelimits[0] = PgUtil.adddate(edate, 0, 0, -int(params[0]))
+      else:
+         pdate = datelimits[0] = params[0]
+         if len(params) > 1:
+            edate = datelimits[1] = params[1]
+         else:
+            edate = PgUtil.curdate()
+      while pdate <= edate:
+         fdate = PgUtil.format_date(pdate, 'MMDDYYYY')
+         fname = USAGE['GBSLOG'].format('?', fdate)
+         fnames = glob.glob(fname)
+         if fnames: filenames.extend(sorted(fnames))
+         pdate = PgUtil.adddate(pdate, 0, 0, 1)
+
+   return filenames
+
+#
+# Fill Globus usages into table dssdb.globususage of DSS PgSQL database from globus access logs
+#
+def fill_globus_usages(fnames, datelimits):
+
+   cntall = addall = 0
+
+   fcnt = len(fnames)
+   for logfile in fnames:
+      if not op.isfile(logfile):
+         PgLOG.pglog("{}: Not exists for Gathering Globus usage".format(logfile), PgLOG.LOGWRN)
+         continue
+      PgLOG.pglog("Gathering usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
+      globus = PgFile.open_local_file(logfile)
+      if not globus: continue
+      ptime = ''
+      record = {}
+      cntadd = entcnt = 0
+      pkey = None
+      while True:
+         line = globus.readline()
+         if not line: break
+         entcnt += 1
+         if entcnt%10000 == 0:
+            PgLOG.pglog("{}: {}/{} Globus log entries processed/records added".format(logfile, entcnt, cntadd), PgLOG.WARNLG)
+
+         ms = re.match(r'^([\d\.]+)\s.*\s+\[(\S+).*"GET\s+/(ds\d\d\d\.\d|[a-z]\d{6})/(\S+)\s.*\s(200|206)\s+(\d+)\s+"(\S+)"\s+"(.+)"$', line)
+         if not ms: continue
+         size = int(ms.group(6))
+         if size < 100: continue  # ignore small files
+         ip = ms.group(1)
+         dsid = PgUtil.format_dataset_id(ms.group(3))
+         wfile = ms.group(4)
+         stat = ms.group(5)
+         sline = ms.group(7)
+         engine = ms.group(8)
+         (year, quarter, date, time) = get_record_date_time(ms.group(2))
+         if datelimits[0] and date < datelimits[0]: continue
+         if datelimits[1] and date > datelimits[1]: continue
+         locflag = 'O' if re.match(r'^https://stratus\.', sline) else 'G'
+         idx = wfile.find('?')
+         if idx > -1: wfile = wfile[:idx]
+
+         if re.match(r'^curl', engine, re.I):
+            method = "CURL"
+         elif re.match(r'^wget', engine, re.I):
+            method = "WGET"
+         elif re.match(r'^python', engine, re.I):
+            method = "PYTHN"
+         else:
+            method = "WEB"
+
+         key = "{}:{}:{}".format(ip, dsid, wfile) if stat == '206' else None
+
+         if record:
+            if key == pkey:
+               record['size'] += size
+               continue
+            cntadd += add_file_usage(year, record)
+         record = {'ip' : ip, 'dsid' : dsid, 'wfile' : wfile, 'date' : date,
+                   'time' : time, 'quarter' : quarter, 'size' : size,
+                   'locflag' : locflag, 'method' : method}
+         pkey = key
+         if not pkey:
+            cntadd += add_file_usage(year, record)
+            record = None
+      if record: cntadd += add_file_usage(year, record)
+      globus.close()
+      cntall += entcnt
+      addall += cntadd
+      PgLOG.pglog("{} Globus usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+
+def get_record_date_time(ctime):
+   
+   ms = re.search(r'^(\d+)/(\w+)/(\d+):(\d+:\d+:\d+)$', ctime)
+   if ms:
+      d = int(ms.group(1))
+      m = PgUtil.get_month(ms.group(2))
+      y = ms.group(3)
+      t = ms.group(4)
+      q = 1 + (m-1)/3
+      return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
+   else:
+      PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
+
+#
+# Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
+#
+def add_file_usage(year, logrec):
+
+   pgrec = get_wfile_wid(logrec['dsid'], logrec['wfile'])
+   if not pgrec: return 0
+
+   table = "{}_{}".format(USAGE['PGTBL'], year)
+   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
+   if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
+
+   wurec =  get_wuser_record(logrec['ip'], logrec['date'])
+   if not wurec: return 0
+   record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
+   record['wuid_read'] = wurec['wuid']
+   record['date_read'] = logrec['date']
+   record['time_read'] = logrec['time']
+   record['size_read'] = logrec['size']
+   record['method'] = logrec['method']
+   record['locflag'] = logrec['locflag']
+   record['ip'] = logrec['ip']
+   record['quarter'] = logrec['quarter']
+
+   if add_to_allusage(year, logrec, wurec):
+      return PgDBI.add_yearly_wusage(year, record)
+   else:
+      return 0
+
+def add_to_allusage(year, logrec, wurec):
+
+   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'], 'country' : wurec['country']}
+   pgrec['dsid'] = logrec['dsid']
+   pgrec['date'] = logrec['date']
+   pgrec['quarter'] = logrec['quarter']
+   pgrec['time'] = logrec['time']
+   pgrec['size'] = logrec['size']
+   pgrec['method'] = logrec['method']
+   pgrec['ip'] = logrec['ip']
+   pgrec['source'] = 'W'
+   return PgDBI.add_yearly_allusage(year, pgrec)
+
+#
+# return wfile.wid upon success, 0 otherwise
+#
+def get_wfile_wid(dsid, wfile):
+
+   wfcond = "wfile = '{}'".format(wfile) 
+   pgrec = PgSplit.pgget_wfile(dsid, "*", wfcond)
+   if pgrec:
+      pgrec['dsid'] = dsid
+   else:
+      pgrec = PgDBI.pgget("wfile_delete", "*", "{} AND dsid = '{}'".format(wfcond, dsid))
+      if not pgrec:
+         pgrec = PgDBI.pgget("wmove", "wid, dsid", wfcond)
+         if pgrec:
+            pgrec = PgSplit.pgget_wfile(pgrec['dsid'], "*", "wid = {}".format(pgrec['wid']))
+            if pgrec: pgrec['dsid'] = dsid
+
+   return pgrec
+
+# return wuser record upon success, None otherwise
+def get_wuser_record(ip, date):
+
+   ipinfo = PgIPInfo.set_ipinfo(ip)
+   if not ipinfo: return None
+
+   record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
+   email = 'unknown@' + ipinfo['hostname']
+   emcond = "email = '{}'".format(email)
+   flds = 'wuid, email, org_type, country, start_date'   
+   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
+   if pgrec:
+      if PgUtil.diffdate(pgrec['start_date'], date) > 0:
+         pgrec['start_date'] = record['start_date'] = date
+         PgDBI.pgupdt('wuser', record, emcond)
+      return pgrec
+
+   # now add one in
+   record['email'] = email
+   record['stat_flag'] = 'A'
+   record['start_date'] = date
+   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
+   if wuid:
+      record['wuid'] = wuid
+      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
+      return record
+
+   return None
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/fillglobususage.usg

@@ -0,0 +1,17 @@
+
+ Retrieves usage information from GLlobus Server logs under directory
+   /gpfs/fs1/collections/rda/work/logs/gridftp/ to fill table 'wusage' in
+   database 'dssdb'.
+
+ Usage: fillglobususage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
+
+      select option, -d, -N or -p to run this application.
+
+      - Option -b, log process information into logfile only;
+
+      - Option -d, retrieve usage info from given log file dates; 
+
+      - Option -N, retrieve usage info in recent NumberDay days;
+
+      - Option -p, retrieve usage info between given period. For missing EndDate,
+          it defaults to the current date.

rda_python_metrics/fillipinfo.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillipinfo
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 08/26/2023
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to retrieve ip info and  
+#             and fill table ipinfo
+# 
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+import glob
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from . import PgIPInfo
+
+# the define options for gathering ipinfo data
+MONTH = 0x02  # fix data usages for given months
+YEARS = 0x04  # fix data usages for given years
+NDAYS = 0x08  # fix data usages in recent number of days
+MULTI = (MONTH|YEARS)
+SINGL = (NDAYS)
+
+IPINFO = {
+   'USGTBL'  : ['ipinfo', 'allusage', 'tdsusage'],
+   'CDATE' : PgUtil.curdate(),
+}
+
+#
+# main function to run this program
+#
+def main():
+
+   inputs = []  # array of input values
+   table = None  # table names: ipinfo, allusage, globususage, or tdsusage
+   argv = sys.argv[1:]
+   topt = option = 0
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif re.match(r'^-[mNy]$', arg) and option == 0:
+         if arg == "-m":
+            option = MONTH
+         elif arg == "-y":
+            option = YEARS
+         elif arg == "-N":
+            option = NDAYS
+      elif arg == "-t":
+         topt = 1
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif topt:
+         if arg not in IPINFO['USGTBL']:
+            PgLOG.pglog("{}: Invalid Table Name; must be in ({})".format(arg, ','.join(IPINFO['USGTBL'])), PgLOG.LGWNEX)
+         table = arg
+         topt = 0
+      elif option&MULTI or option&SINGL and not inputs:
+         inputs.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+
+   if not (inputs and table): PgLOG.show_usage('fillipinfo')
+   PgDBI.dssdb_dbname()
+   PgLOG.cmdlog("fillipinfo {}".format(' '.join(argv)))
+
+   if option&NDAYS:
+      curdate = IPINFO['CDATE']
+      datelimit = PgUtil.adddate(curdate, 0, 0, -int(inputs[0]))  
+      option = MONTH
+      inputs = []
+      
+      while curdate >= datelimit:
+         tms = curdate.split('-')
+         inputs.append("{}-{}".format(tms[0], tms[1]))
+         curdate = PgUtil.adddate(curdate, 0, 0, -int(tms[2]))
+
+   fill_ip_info(option, inputs, table)
+
+   sys.exit(0)
+
+#
+# Fill ip info in table dssdb.tdsusage
+#
+def fill_ip_info(option, inputs, table):
+
+   cntall = 0
+   date = None
+   for input in inputs:
+      if option&NDAYS:
+         edate = IPINFO['CDATE']
+         date = PgUtil.adddate(edate, 0, 0, -int(input))  
+      elif option&MONTH:
+         tms = input.split('-')
+         date = "{}-{:02}-01".format(tms[0], int(tms[1]))
+         edate = PgUtil.enddate(date, 0, 'M')
+      elif option&YEARS:
+         date = input + "-01-01"
+         edate = input + "-12-31"
+
+      while date <= edate:
+         func = eval('fix_{}_records'.format(table))
+         cntall += func(date)
+         date = PgUtil.adddate(date, 0, 0, 1)
+   return cntall
+
+def fix_allusage_records(date):
+
+   cnt = 0
+   ms = re.match(r'^(\d+)-', date)
+   year = ms.group(1)
+   table = 'allusage_' + year
+   cond = "date = '{}' and org_type = '-'".format(date)
+   pgrecs = PgDBI.pgmget(table, 'aidx, email, ip', cond, PgLOG.LGEREX)
+   if not pgrecs: return 0
+   cnt = len(pgrecs['ip']) if pgrecs else 0
+   mcnt = 0
+   for i in range(cnt):
+      ip = pgrecs['ip'][i]
+      email = pgrecs['email'][i]
+      ipinfo = PgIPInfo.set_ipinfo(ip)
+      if ipinfo:
+         record = {'org_type' : ipinfo['org_type'],
+                   'country' : ipinfo['country']}
+         if not email or re.search(r'-$', email):
+            record['email'] =  'unknown@' + ipinfo['hostname']
+         mcnt += PgDBI.pgupdt(table, record, "aidx = '{}'".format(pgrecs['aidx'][i]))
+
+   s = 's' if cnt > 1 else ''
+   PgLOG.pglog("{}: {} of {} record{} updated for {}".format(table, mcnt, cnt, s, date), PgLOG.LOGWRN)
+
+   return mcnt
+
+def fix_tdsusage_records(date):
+
+   table = 'tdsusage'
+   cond = "date = '{}' and org_type = '-'".format(date)
+   pgrecs = PgDBI.pgmget(table, 'time, ip', cond, PgLOG.LGEREX)
+   if not pgrecs: return 0
+   cnt = len(pgrecs['ip']) if pgrecs else 0
+   mcnt = 0
+   for i in range(cnt):
+      ipinfo = PgIPInfo.set_ipinfo(pgrecs['ip'][i])
+      if ipinfo:
+         record = {'org_type' : ipinfo['org_type'],
+                   'country' : ipinfo['country']}
+         cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, pgrecs['time'][i], pgrecs['ip'][i])
+         mcnt += PgDBI.pgupdt(table, record, cond)
+
+   s = 's' if cnt > 1 else ''
+   PgLOG.pglog("{}: {} of {} record{} updated for {}".format(table, mcnt, cnt, s, date), PgLOG.LOGWRN)
+
+   return mcnt
+
+def fix_ipinfo_records(date):
+
+   table = 'ipinfo'
+   cond = "stat_flag = 'M' and date = '{}'".format(date)
+   pgrecs = PgDBI.pgmget(table, 'ip', cond, PgLOG.LGEREX)
+   if not pgrecs: return 0
+   cnt = len(pgrecs['ip']) if pgrecs else 0
+   mcnt = 0
+   for i in range(cnt):
+      PgIPInfo.set_ipinfo(pgrecs['ip'][i], True)
+
+   mcnt = PgIPInfo.IPINFO['IPUPDT']
+   s = 's' if cnt > 1 else ''
+   PgLOG.pglog("{}: {} of {} record{} updated".format(table, mcnt, cnt, s), PgLOG.LOGWRN)
+
+   return mcnt
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/fillipinfo.usg

@@ -0,0 +1,18 @@
+
+ Check data usage IP related information to fill in usage tables.
+
+ Usage: fillipinfo [-b] -t TableName [-N NumberDay] [-m YearMonths] [-y Years]
+
+      select option, -t and -m, -N or -y to run this application.
+
+      - Option -b, log process information into logfile only;
+
+      - Option -t, table name, ipinfo, allusage, globususage, or tdsusage,
+                   to fix IP related information, such organization names/types,
+                   emails and country names; 
+
+      - Option -N, fix usage info in recent NumberDay days;
+
+      - Option -m, fix usage info in given months;
+
+      - Option -y, fix usage info in given years.