rda-python-metrics 1.0.4__py3-none-any.whl

rda_python_metrics/__init__.py

@@ -0,0 +1 @@
+

rda_python_metrics/fillawsusage.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillawsusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 03/11/2022
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to retrieve info from AWS logs 
+#             and fill table wusages in PgSQL database dssdb.
+# 
+#    Github : https://github.com/NCAR/rda-pythn-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+import glob
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from . import PgIPInfo
+
+USAGE = {
+   'PGTBL'  : "wusage",
+   'AWSDIR' : PgLOG.PGLOG["TRANSFER"] + "/AWSera5log",
+   'AWSLOG' : "{}/{}-00-00-00-*",
+   'PFMT'   : "YYYY/MM/DD"
+}
+
+DSIDS = {'nsf-ncar-era5' : PgUtil.format_dataset_id('d633000')}
+
+#
+# main function to run this program
+#
+def main():
+
+   params = []  # array of input values
+   argv = sys.argv[1:]
+   option = None
+
+   for arg in argv:
+      ms = re.match(r'^-(b|d|p|N)$', arg)
+      if ms:
+         opt = ms.group(1)
+         if opt == 'b':
+            PgLOG.PGLOG['BCKGRND'] = 1
+         elif option:
+            PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
+         else:
+            option = opt
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         params.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+   
+   if not (option and params): PgLOG.show_usage('fillawsusage')
+
+   PgDBI.dssdb_dbname()
+   cmdstr = "fillawsusage {}".format(' '.join(argv))
+   PgLOG.cmdlog(cmdstr)
+   PgFile.change_local_directory(USAGE['AWSDIR'])
+   filenames = get_log_file_names(option, params)
+   if filenames:
+      fill_aws_usages(filenames)
+   else:
+      PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
+
+   PgLOG.pglog(None, PgLOG.LOGWRN)
+   sys.exit(0)
+
+#
+# get the log file dates 
+#
+def get_log_file_names(option, params):
+
+   filenames = []
+   if option == 'd':
+      for dt in params:
+         pdate = PgUtil.format_date(dt)
+         pd = PgUtil.format_date(pdate, USAGE['PFMT'])
+         fname = USAGE['AWSLOG'].format(pd, pdate)
+         fnames = glob.glob(fname)
+         if fnames: filenames.extend(sorted(fnames))
+   else:
+      if option == 'N':
+         edate = PgUtil.curdate()
+         pdate = PgUtil.adddate(edate, 0, 0, -int(params[0]))
+      else:
+         pdate = PgUtil.format_date(params[0])
+         if len(params) > 1:
+            edate = PgUtil.format_date(params[1])
+         else:
+            edate = PgUtil.curdate()
+      while pdate <= edate:
+         pd = PgUtil.format_date(pdate, USAGE['PFMT'])
+         fname = USAGE['AWSLOG'].format(pd, pdate)
+         fnames = glob.glob(fname)
+         if fnames: filenames.extend(sorted(fnames))
+         pdate = PgUtil.adddate(pdate, 0, 0, 1)
+
+   return filenames
+
+#
+# Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs
+#
+def fill_aws_usages(fnames):
+
+   cntall = addall = 0
+   fcnt = len(fnames)
+   for logfile in fnames:
+      if not op.isfile(logfile):
+         PgLOG.pglog("{}: Not exists for Gathering AWS usage".format(logfile), PgLOG.LOGWRN)
+         continue
+      PgLOG.pglog("Gathering usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
+      aws = PgFile.open_local_file(logfile)
+      if not aws: continue
+      ptime = ''
+      record = {}
+      cntadd = entcnt = 0
+      pkey = None
+      while True:
+         line = aws.readline()
+         if not line: break
+         entcnt += 1
+         if entcnt%10000 == 0:
+            PgLOG.pglog("{}: {}/{} AWS log entries processed/records added".format(logfile, entcnt, cntadd), PgLOG.WARNLG)
+
+         ms = re.match(r'^\w+ ([\w-]+) \[(\S+).*\] ([\d\.]+) .+ REST\.GET\.OBJECT (\S+) "GET.+" (200|206) - (\d+) (\d+) .* ".+" "(.+)" ', line)
+         if not ms: continue
+         values = list(ms.groups())
+         if values[0] not in DSIDS: continue
+         dsid = DSIDS[values[0]]
+         size = int(values[5])
+         fsize = int(values[6])
+         if fsize < 100: continue  # ignore small files
+         ip = values[2]
+         wfile = values[3]
+         stat = values[4]
+         engine = values[7]
+         (year, quarter, date, time) = get_record_date_time(values[1])
+         locflag = 'A'
+
+         if re.match(r'^aiobotocore', engine, re.I):
+            method = "AIOBT"
+         elif re.match(r'^rclone', engine, re.I):
+            method = "RCLON"
+         elif re.match(r'^python', engine, re.I):
+            method = "PYTHN"
+         else:
+            method = "WEB"
+         
+         key = "{}:{}:{}".format(ip, dsid, wfile) if stat == '206' else None
+
+         if record:
+            if key == pkey:
+               record['size'] += size
+               continue
+            cntadd += add_file_usage(year, record)
+         record = {'ip' : ip, 'dsid' : dsid, 'wfile' : wfile, 'date' : date,
+                   'time' : time, 'quarter' : quarter, 'size' : size,
+                   'locflag' : locflag, 'method' : method}
+         pkey = key
+         if not pkey:
+            cntadd += add_file_usage(year, record)
+            record = None
+      if record: cntadd += add_file_usage(year, record)
+      aws.close()
+      cntall += entcnt
+      addall += cntadd
+      PgLOG.pglog("{} AWS usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+
+def get_record_date_time(ctime):
+   
+   ms = re.search(r'^(\d+)/(\w+)/(\d+):(\d+:\d+:\d+)$', ctime)
+   if ms:
+      d = int(ms.group(1))
+      m = PgUtil.get_month(ms.group(2))
+      y = ms.group(3)
+      t = ms.group(4)
+      q = 1 + (m-1)/3
+      return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
+   else:
+      PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
+
+#
+# Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
+#
+def add_file_usage(year, logrec):
+
+   pgrec = get_wfile_wid(logrec['dsid'], logrec['wfile'])
+   if not pgrec: return 0
+
+   table = "{}_{}".format(USAGE['PGTBL'], year)
+   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
+   if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
+
+   wurec =  get_wuser_record(logrec['ip'], logrec['date'])
+   if not wurec: return 0
+   record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
+   record['wuid_read'] = wurec['wuid']
+   record['date_read'] = logrec['date']
+   record['time_read'] = logrec['time']
+   record['size_read'] = logrec['size']
+   record['method'] = logrec['method']
+   record['locflag'] = logrec['locflag']
+   record['ip'] = logrec['ip']
+   record['quarter'] = logrec['quarter']
+
+   if add_to_allusage(year, logrec, wurec):
+      return PgDBI.add_yearly_wusage(year, record)
+   else:
+      return 0
+
+def add_to_allusage(year, logrec, wurec):
+
+   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'], 'country' : wurec['country']}
+   pgrec['dsid'] = logrec['dsid']
+   pgrec['date'] = logrec['date']
+   pgrec['quarter'] = logrec['quarter']
+   pgrec['time'] = logrec['time']
+   pgrec['size'] = logrec['size']
+   pgrec['method'] = logrec['method']
+   pgrec['ip'] = logrec['ip']
+   pgrec['source'] = 'A'
+   return PgDBI.add_yearly_allusage(year, pgrec)
+
+#
+# return wfile.wid upon success, 0 otherwise
+#
+def get_wfile_wid(dsid, wfile):
+
+   dscond = "dsid = '{}' AND wfile = '{}'".format(dsid, wfile) 
+   pgrec = PgDBI.pgget("wfile", "*", dscond)
+
+   if not pgrec:
+      pgrec = PgDBI.pgget("wmove", "wid, dsid", dscond)
+      if pgrec:
+         pgrec = PgDBI.pgget("wfile", "*", "wid = {}".format(pgrec['wid']))
+         if pgrec: pgrec['dsid'] = dsid
+
+   return pgrec
+
+# return wuser record upon success, None otherwise
+def get_wuser_record(ip, date):
+
+   ipinfo = PgIPInfo.set_ipinfo(ip)
+   if not ipinfo: return None
+
+   record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
+   email = 'unknown@' + ipinfo['hostname']
+   emcond = "email = '{}'".format(email)
+   flds = 'wuid, email, org_type, country, start_date'   
+   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
+   if pgrec:
+      if PgUtil.diffdate(pgrec['start_date'], date) > 0:
+         pgrec['start_date'] = record['start_date'] = date
+         PgDBI.pgupdt('wuser', record, emcond)
+      return pgrec
+
+   # now add one in
+   record['email'] = email
+   record['stat_flag'] = 'A'
+   record['start_date'] = date
+   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
+   if wuid:
+      record['wuid'] = wuid
+      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
+      return record
+
+   return None
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/fillawsusage.usg

@@ -0,0 +1,17 @@
+
+ Retrieves usage information from AWS Server logs under directory
+   /gpfs/fs1/collections/rda/transer/AWSera5log/ to fill table 'wusage' in
+   database 'dssdb'.
+
+ Usage: fillawsusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
+
+      select option, -d, -N or -p to run this application.
+
+      - Option -b, log process information into logfile only;
+
+      - Option -d, retrieve usage info from given log file dates; 
+
+      - Option -N, retrieve usage info in recent NumberDay days;
+
+      - Option -p, retrieve usage info between given period. For missing EndDate,
+          it defaults to the current date.

rda_python_metrics/fillcodusage.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillcodusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 03/11/2022
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to retrieve info from web logs 
+#             and fill table codusage in PgSQL database dssdb.
+# 
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+import glob
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+
+# the define options for gathering COD data usage, one at a time
+MONTH = 0x02  # fet COD data usages for given months
+YEARS = 0x04  # get COD data usages for given years
+NDAYS = 0x08  # get COD data usages in recent number of days 
+FILES = 0x10  # get given file names
+GTALL = 0x20  # get all data files of read
+MASKS = (MONTH|YEARS|NDAYS|FILES)
+
+USAGE = {
+   'OPTION' : 0,
+   'PGTBL'  : "codusage",
+   'WEBLOG' : "/var/log/httpd",
+}
+
+USERS = {}  # cache user info for aid
+
+#
+# main function to run this program
+#
+def main():
+
+   params = []  # array of input values
+   argv = sys.argv[1:]
+   datelimit = ''
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif re.match(r'^-[afmNy]$', arg) and USAGE['OPTION'] == 0:
+         if arg == "-a":
+            USAGE['OPTION'] = GTALL
+            params = ['']
+         elif arg == "-f":
+            USAGE['OPTION'] = FILES
+         elif arg == "-m":
+            USAGE['OPTION'] = MONTH
+         elif arg == "-y":
+            USAGE['OPTION'] = YEARS
+         elif arg == "-N":
+            USAGE['OPTION'] = NDAYS
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif USAGE['OPTION']&MASKS:
+         params.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+   
+   if not (USAGE['OPTION'] and params): PgLOG.show_usage('fillcodusage')
+
+   PgDBI.dssdb_dbname()
+   PgLOG.cmdlog("fillcodusage {}".format(' '.join(argv)))
+   
+   if USAGE['OPTION']&NDAYS:
+      curdate = PgUtil.curdate()
+      datelimit = PgUtil.adddate(curdate, 0, 0, -int(params[0]))
+   
+      USAGE['OPTION'] = MONTH
+      params = []
+      
+      while curdate >= datelimit:
+         (year, month, day) = curdate.split('-')
+         params.append("{}-{}".format(year, month))
+         curdate = PgUtil.adddate(curdate, 0, 0, -int(day))
+   
+   fill_cod_usages(USAGE['OPTION'], params, datelimit)
+   
+   PgLOG.pglog(None, PgLOG.LOGWRN|PgLOG.SNDEML)  # send email out if any
+   
+   sys.exit(0)
+
+#
+# Fill COD usages into table dssdb.codusage of DSS PgSQL database from cod access logs
+#
+def fill_cod_usages(option, inputs, datelimit):
+
+   cntall = cntadd = 0
+
+   for input in inputs:
+      # get log file names
+      if option&FILES:
+         logfiles = [input]
+      elif option&MONTH:
+         tms = input.split('-')
+         yrmn = "{}{:02}".format(tms[0], int(tms[1]))
+         logfiles = ["{}/{}/access_log".format(USAGE['WEBLOG'], yrmn)]
+      else: # GTALL | YEARS
+         yrmn = input + "*"
+         logfiles = glob.glob("{}/{}/access_log".format(USAGE['WEBLOG'], yrmn))
+   
+      for logfile in logfiles:
+         if not op.isfile(logfile):
+            PgLOG.pglog("{}: Not exists for Gathering custom OPeNDAP usage".format(logfile), PgLOG.LOGWRN)
+            continue
+         PgLOG.pglog("Gathering custom OPeNDAP usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
+         cod = PgFile.open_local_file(logfile)
+         if not cod: continue
+
+         pdate = ''
+         records = {}
+         while True:
+            line = cod.readline()
+            if not line: break
+            cntall += 1
+            if cntall%20000 == 0:
+               s = 's' if cntadd > 1 else ''
+               PgLOG.pglog("{}/{} COD log entries processed/records added".format(cntall, cntadd), PgLOG.WARNLG)
+         
+            ms = re.search(r'GET /opendap/(\w{10})\.dods.*\s200\s+(\d+).{6}([^"]+)', line)
+            if not ms: continue
+            aid = ms.group(1)
+            size = int(ms.group(2))
+            engine = ms.group(3)
+            if not (aid in USERS or cache_users(aid)): continue
+            ms = re.match(r'^([\d\.]+).+\[(\d+)/(\w+)/(\d+):([\d:]+)', line)
+            if not ms: continue
+            ip = ms.group(1)
+            ctime = ms.group(5)
+            cdate = "{}-{:02}-{:02}".format(ms.group(4), PgUtil.get_month(ms.group(3)), int(ms.group(2)))
+            if pdate != cdate:
+               if records:
+                  cntadd += add_usage_records(records, cdate)
+                  records = {}            
+               pdate = cdate
+         
+            if datelimit and cdate < datelimit: continue
+
+            if aid in records:
+               records[aid]['size'] += size
+               records[aid]['count'] += 1
+               USERS[aid]['etime'] = ctime
+            else:
+               records[aid] = {}
+               records[aid]['ip'] = ip
+               records[aid]['count'] = 1
+               records[aid]['email'] = USERS[aid]['email']
+               records[aid]['dsid'] = USERS[aid]['dsid']
+               records[aid]['size'] = size
+               records[aid]['engine'] = engine
+               USERS[aid]['etime'] = ctime
+               USERS[aid]['btime'] = ctime
+         cod.close()
+         if records: cntadd += add_usage_records(records, cdate)
+   
+
+   s = 's' if cntadd > 1 else ''
+   PgLOG.pglog("{} COD usage records added for {} entries at {}".format(cntadd, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+def add_usage_records(records, date):
+
+   ms = re.match(r'(\d+)-(\d+)-', date)
+   if not ms: return 0
+   year = ms.group(1)
+   quarter = 1 + int((int(ms.group(2)) - 1) / 3)
+   cnt = 0
+
+   for aid in records:
+      if PgDBI.pgget(USAGE['PGTBL'], '', "aid = '{}' AND date = '{}'".format(aid, date), PgLOG.LGEREX): continue
+      record = records[aid]
+      if record['email'] == '-':
+         record['org_type'] = record['country'] = '-'
+      else:
+         wuid = PgDBI.check_wuser_wuid(record['email'], date)
+         if not wuid: next
+         pgrec = PgDBI.pgget("wuser",  "org_type, country", "wuid = {}".format(wuid), PgLOG.LGWNEX)
+         if not pgrec: continue
+         record['org_type'] = pgrec['org_type']
+         record['country'] = pgrec['country']
+   
+      record['date'] = date
+      record['time'] = USERS[aid]['btime']
+      record['quarter'] = quarter
+
+      if add_to_allusage(record, year):
+         record['aid'] = aid
+         record['period'] = access_period(USERS[aid]['etime'], record['time'])
+         cnt += PgDBI.pgadd(USAGE['PGTBL'], record, PgLOG.LOGWRN)
+
+   return cnt
+
+
+def add_to_allusage(pgrec, year):
+
+   record = {'method' : 'COD', 'source' : 'C'}
+   for fld in pgrec:
+      ms = re.match(r'^(engine|count)$', fld)
+      if ms: continue
+      record[fld] = pgrec[fld]
+
+   return PgDBI.add_yearly_allusage(year, record)   # change 1 to 0 to stop checking
+
+def cache_users(aid):
+
+   pgrec = PgDBI.pgget("metautil.custom_dap_history", "*", "ID = '{}'".format(aid), PgLOG.LGEREX)
+
+   if pgrec:
+      ms = re.search(r'dsnum=(\d+\.\d|[a-z]\d{6});', pgrec['rinfo'])
+      if ms:
+         dsid = PgUtil.format_dataset_id(ms.group(1))
+         USERS[aid]= {'dsid' : dsid, 'email' : pgrec['duser']}
+         return 1
+
+   return 0
+
+
+def access_period(etime, btime):
+   
+   period = 86400
+   
+   ms = re.search(r'(\d+):(\d+):(\d+)', etime)
+   if ms:
+      period = int(ms.group(1))*3600+int(ms.group(2))*60+int(ms.group(3))
+
+   ms = re.search(r'(\d+):(\d+):(\d+)', btime)
+   if ms:
+      period -= int(ms.group(1))*3600+int(ms.group(2))*60+int(ms.group(3))
+   
+   return period
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/fillcodusage.usg

@@ -0,0 +1,21 @@
+
+ Retrieves usage information from RDA Web logs under /var/log to
+   fill table 'codusage' in MySQL database 'dssdb'.
+
+ Usage: fillcodusage [-a] [-b] [-f LogFileNames] [-m MonthList] [-N NumberDay] [-y YearList]
+
+      select one of the options, -a, -f, -m, -N or -y each time to run
+        this application. 
+
+      - Option -b, log process information into logfile only;
+
+      - Option -a, retrieve usages for all available logs;
+
+      - Option -f, retrieve usage info from given log file names; 
+
+      - Option -m, retrieve usage info in given months; 
+
+      - Option -N, retrieve usage info in recent NumberDay days;
+
+      - Option -y, retrieve usage info in given years. 
+

rda_python_metrics/fillcountry.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillcountry
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 2022-03-11
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to fill missing country field from email info for
+#             given table name
+#
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+from rda_python_common import PgLOG
+from rda_python_common import PgIMMA
+from rda_python_common import PgUtil
+from rda_python_common import PgDBI
+
+#
+# main function to run this program
+#
+def main():
+
+   argv = sys.argv[1:]
+   tables = ['allusage', 'user', 'wuser']
+   table = None
+   
+   # check command line
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif re.match(r'^-.*', arg):
+         PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGEREX)
+      elif not table:
+         table = arg
+      else:
+         PgLOG.pglog(arg + ": one table name at a time", PgLOG.LGEREX)
+   
+   if not table:
+      print("Usage: fillcountry TableName\n")
+      sys.exit(0)
+   elif table not in tables:
+      PgLOG.pglog("{}: table name must be ({})".format(table, '|'.join(tables)), PgLOG.LGEREX)
+
+   PgDBI.dssdb_dbname()
+   PgLOG.cmdlog("fillcountry {}".format(' '.join(argv)))
+
+   process_countries(table)
+   
+   sys.exit(0)
+
+def process_countries(table):
+
+   pgrecs = PgDBI.pgmget(table, "email", "country IS NULL", PgLOG.LOGWRN)
+
+   cntall = len(pgrecs['email']) if pgrecs else 0
+   PgLOG.pglog("Set {} record(s) for missing country in table {}".format(cntall, table), PgLOG.LOGWRN)
+   if not cntall: return
+
+   cntmod = 0
+   for i in range(cntall):
+      if i and (i % 500) == 0:
+         PgLOG.pglog("{}/{} Records modified/processed".format(cntmod, i), PgLOG.WARNLG)
+
+      email = pgrecs['email'][i]
+      record = {'country' : PgDBI.email_to_country(email)}
+      cntmod += PgDBI.pgupdt(table, record, "email = '{}' AND country IS NULL".format(email), PgLOG.LOGWRN)
+
+   PgLOG.pglog("{} Record(s) modified in table '{}'".format(cntmod, table), PgLOG.LOGWRN)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()