rda-python-metrics 1.0.4__tar.gz → 1.0.5__tar.gz

{rda_python_metrics-1.0.4 → rda_python_metrics-1.0.5}/MANIFEST.in

@@ -3,6 +3,7 @@ include src/rda_python_metrics/fillcodusage.usg
 include src/rda_python_metrics/fillglobususage.usg
 include src/rda_python_metrics/fillipinfo.usg
 include src/rda_python_metrics/filloneorder.usg
+include src/rda_python_metrics/fillosdfusage.usg
 include src/rda_python_metrics/fillrdadb.usg
 include src/rda_python_metrics/filltdsusage.usg
 include src/rda_python_metrics/filluser.usg

{rda_python_metrics-1.0.4/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.4
+Version: 1.0.5
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.4 → rda_python_metrics-1.0.5}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.4"
+version = "1.0.5"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]
@@ -33,9 +33,10 @@ where = ["src"]
 [tool.setuptools.package-data]
 "rda_python_metrics" = [
 "logarch.usg", "fillawsusage.usg", "fillcodusage.usg", "fillglobususage.usg",
-"fillipinfo.usg", "filloneorder.usg", "fillrdadb.usg", "filltdsusage.usg",
-"viewallusage.usg", "viewcheckusage.usg", "viewcodusage.usg", "viewordusage.usg",
-"viewrqstusage.usg", "viewtdsusage.usg", "viewwebfile.usg", "viewwebusage.usg"
+"fillipinfo.usg", "filloneorder.usg", "fillosdfusage.usg", "fillrdadb.usg",
+"filltdsusage.usg", "viewallusage.usg", "viewcheckusage.usg", "viewcodusage.usg",
+"viewordusage.usg", "viewrqstusage.usg", "viewtdsusage.usg", "viewwebfile.usg",
+"viewwebusage.usg"
 ]
 
 [project.urls]
@@ -50,8 +51,9 @@ where = ["src"]
 "fillglobususage" = "rda_python_metrics.fillglobususage:main"
 "fillipinfo" = "rda_python_metrics.fillipinfo:main"
 "filloneorder" = "rda_python_metrics.filloneorder:main"
+"fillosdfusage" = "rda_python_metrics.fillosdfusage:main"
 "fillrdadb" = "rda_python_metrics.fillrdadb:main"
-"filltdsusage" = "rda_python_metrics.tdsusage:main"
+"filltdsusage" = "rda_python_metrics.filltdsusage:main"
 "filluser" = "rda_python_metrics.filluser:main"
 "pgperson" = "rda_python_metrics.pgperson:main"
 "pgusername" = "rda_python_metrics.pgusername:main"

{rda_python_metrics-1.0.4 → rda_python_metrics-1.0.5}/src/rda_python_metrics/fillawsusage.py

@@ -184,7 +184,7 @@ def get_record_date_time(ctime):
       m = PgUtil.get_month(ms.group(2))
       y = ms.group(3)
       t = ms.group(4)
-      q = 1 + (m-1)/3
+      q = 1 + int((m-1)/3)
       return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
    else:
       PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)

{rda_python_metrics-1.0.4 → rda_python_metrics-1.0.5}/src/rda_python_metrics/fillglobususage.py

@@ -186,7 +186,7 @@ def get_record_date_time(ctime):
       m = PgUtil.get_month(ms.group(2))
       y = ms.group(3)
       t = ms.group(4)
-      q = 1 + (m-1)/3
+      q = 1 + int((m-1)/3)
       return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
    else:
       PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)

rda_python_metrics-1.0.5/src/rda_python_metrics/fillosdfusage.py

@@ -0,0 +1,265 @@
+##!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillosdfusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 2025-04-01
+#   Purpose : python program to retrieve info from weekly OSDF logs 
+#             and fill table wusages in PgSQL database dssdb.
+# 
+#    Github : https://github.com/NCAR/rda-pythn-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from rda_python_common import PgSplit
+from . import PgIPInfo
+
+USAGE = {
+   'OSDFTBL'  : "wusage",
+   'OSDFDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/logs/gridftp/",
+   'OSDFGET' : 'wget -m -nH -np -nd https://pelicanplatform.org/pelican-access-logs/ncar-access-log/',
+   'OSDFLOG' : "{}.log",   # YYYY-MM-DD.log
+}
+
+#
+# main function to run this program
+#
+def main():
+
+   params = []  # array of input values
+   argv = sys.argv[1:]
+   option = None
+   datelimits = [None, None]
+   
+
+   for arg in argv:
+      ms = re.match(r'^-(b|d|p|N)$', arg)
+      if ms:
+         opt = ms.group(1)
+         if opt == 'b':
+            PgLOG.PGLOG['BCKGRND'] = 1
+         elif option:
+            PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
+         else:
+            option = opt
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         params.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+   
+   if not (option and params): PgLOG.show_usage('fillosdfusage')
+
+   PgDBI.dssdb_dbname()
+   cmdstr = "fillosdfusage {}".format(' '.join(argv))
+   PgLOG.cmdlog(cmdstr)
+   PgFile.change_local_directory(USAGE['OSDFDIR'])
+   filenames = get_log_file_names(option, params, datelimits)
+   if filenames:
+      fill_osdf_usages(filenames, datelimits)
+   else:
+      PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
+
+   PgLOG.pglog(None, PgLOG.LOGWRN)
+   sys.exit(0)
+
+#
+# get the log file dates 
+#
+def get_log_file_names(option, params, datelimits):
+
+   filenames = []
+   if option == 'd':
+      for pdate in params:
+         pdays = PgUtil.get_weekday(pdate)
+         if pdays > 0:
+            PgLOG.pglog(pdate + ": Skip a Non-Sunday date", PgLOG.LOGWRN)
+            continue
+         filenames.append(USAGE['OSDFLOG'].format(pdate))
+   else:
+      if option == 'N':
+         edate = PgUtil.curdate()
+         pdate = datelimits[0] = PgUtil.adddate(edate, 0, 0, -int(params[0]))
+      else:
+         pdate = datelimits[0] = params[0]
+         if len(params) > 1:
+            edate = datelimits[1] = params[1]
+         else:
+            edate = PgUtil.curdate()
+      pdays = PgUtil.get_weekday(pdate)
+      if pdays > 0: pdate = PgUtil.adddate(edate, 0, 0, 7-pdays)
+      while pdate <= edate:
+         filenames.append(USAGE['OSDFLOG'].format(pdate))
+         pdate = PgUtil.adddate(pdate, 0, 0, 7)
+
+   return filenames
+
+#
+# Fill OSDF usages into table dssdb.osdfusage of DSS PgSQL database from osdf access logs
+#
+def fill_osdf_usages(fnames, datelimits):
+
+   cntall = addall = 0
+
+   fcnt = len(fnames)
+   for logfile in fnames:
+      PgLOG.pgsystem(USAGE['OSDFGET'] + logfile, 5, PgLOG.LOGWRN)
+      linfo = PgFile.check_local_file(logfile)
+      if not linfo:
+         PgLOG.pglog("{}: Not exists for Gathering OSDF usage".format(logfile), PgLOG.LOGWRN)
+         continue
+      if linfo['data_size'] == 0:
+         PgLOG.pglog("{}: Empty log for Gathering OSDF usage".format(logfile), PgLOG.LOGWRN)
+         continue
+      PgLOG.pglog("Gathering usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
+      osdf = PgFile.open_local_file(logfile)
+      if not osdf: continue
+      cntadd = entcnt = 0
+      pkey = None
+      while True:
+         line = osdf.readline()
+         if not line: break
+         entcnt += 1
+         if entcnt%10000 == 0:
+            PgLOG.pglog("{}: {}/{} OSDF log entries processed/records added".format(logfile, entcnt, cntadd), PgLOG.WARNLG)
+
+         ms = re.match(r'^\[(\S+)\] \[Objectname:(\/ncar\/rda\/([a-z]\d{6})\/(\S+)\] \[Host:(\S+)\] \[server:(\S+)\] \[Read:(\d+)\]', line)
+         if not ms: continue
+         size = int(ms.group(6))
+         if size < 100: continue  # ignore small files
+         ip = ms.group(4)
+         dsid = PgUtil.format_dataset_id(ms.group(2))
+         wfile = ms.group(3)
+         engine = ms.group(5)
+         
+         (year, quarter, date, time) = get_record_date_time(ms.group(1))
+         if datelimits[0] and date < datelimits[0]: continue
+         if datelimits[1] and date > datelimits[1]: continue
+         locflag = 'C'
+         method = "OSDF"
+
+         record = {'ip' : ip, 'dsid' : dsid, 'wfile' : wfile, 'date' : date,
+                   'time' : time, 'quarter' : quarter, 'size' : size,
+                   'locflag' : locflag, 'method' : method}
+         cntadd += add_file_usage(year, record)
+      osdf.close()
+      cntall += entcnt
+      addall += cntadd
+      PgLOG.pglog("{} OSDF usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+
+def get_record_date_time(ctime):
+
+   ms = re.search(r'^(\d+)-(\d+)-(\d+)T([\d:]+)\.', ctime)
+   if ms:
+      y = ms.group(1)
+      m = int(ms.group(2))
+      d = int(ms.group(3))
+      t = ms.group(4)
+      q = 1 + int((m-1)/3)
+      return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
+   else:
+      PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
+
+#
+# Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
+#
+def add_file_usage(year, logrec):
+
+   pgrec = get_wfile_wid(logrec['dsid'], logrec['wfile'])
+   if not pgrec: return 0
+
+   table = "{}_{}".format(USAGE['OSDFTBL'], year)
+   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
+   if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
+
+   wurec =  get_wuser_record(logrec['ip'], logrec['date'])
+   if not wurec: return 0
+   record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
+   record['wuid_read'] = wurec['wuid']
+   record['date_read'] = logrec['date']
+   record['time_read'] = logrec['time']
+   record['size_read'] = logrec['size']
+   record['method'] = logrec['method']
+   record['locflag'] = logrec['locflag']
+   record['ip'] = logrec['ip']
+   record['quarter'] = logrec['quarter']
+
+   if add_to_allusage(year, logrec, wurec):
+      return PgDBI.add_yearly_wusage(year, record)
+   else:
+      return 0
+
+def add_to_allusage(year, logrec, wurec):
+
+   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'], 'country' : wurec['country']}
+   pgrec['dsid'] = logrec['dsid']
+   pgrec['date'] = logrec['date']
+   pgrec['quarter'] = logrec['quarter']
+   pgrec['time'] = logrec['time']
+   pgrec['size'] = logrec['size']
+   pgrec['method'] = logrec['method']
+   pgrec['ip'] = logrec['ip']
+   pgrec['source'] = 'W'
+   return PgDBI.add_yearly_allusage(year, pgrec)
+
+#
+# return wfile.wid upon success, 0 otherwise
+#
+def get_wfile_wid(dsid, wfile):
+
+   wfcond = "wfile = '{}'".format(wfile) 
+   pgrec = PgSplit.pgget_wfile(dsid, "*", wfcond)
+   if pgrec:
+      pgrec['dsid'] = dsid
+   else:
+      pgrec = PgDBI.pgget("wfile_delete", "*", "{} AND dsid = '{}'".format(wfcond, dsid))
+      if not pgrec:
+         pgrec = PgDBI.pgget("wmove", "wid, dsid", wfcond)
+         if pgrec:
+            pgrec = PgSplit.pgget_wfile(pgrec['dsid'], "*", "wid = {}".format(pgrec['wid']))
+            if pgrec: pgrec['dsid'] = dsid
+
+   return pgrec
+
+# return wuser record upon success, None otherwise
+def get_wuser_record(ip, date):
+
+   ipinfo = PgIPInfo.set_ipinfo(ip)
+   if not ipinfo: return None
+
+   record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
+   email = 'unknown@' + ipinfo['hostname']
+   emcond = "email = '{}'".format(email)
+   flds = 'wuid, email, org_type, country, start_date'   
+   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
+   if pgrec:
+      if PgUtil.diffdate(pgrec['start_date'], date) > 0:
+         pgrec['start_date'] = record['start_date'] = date
+         PgDBI.pgupdt('wuser', record, emcond)
+      return pgrec
+
+   # now add one in
+   record['email'] = email
+   record['stat_flag'] = 'A'
+   record['start_date'] = date
+   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
+   if wuid:
+      record['wuid'] = wuid
+      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
+      return record
+
+   return None
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics-1.0.5/src/rda_python_metrics/fillosdfusage.usg

@@ -0,0 +1,17 @@
+
+ Retrieves usage information from OSDF Server logs under directory
+   /gpfs/fs1/collections/rda/work/zji/osdflogs/ to fill table 'wusage' in
+   database 'dssdb'.
+
+ Usage: fillosdfusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
+
+      select option, -d, -N or -p to run this application.
+
+      - Option -b, log process information into logfile only;
+
+      - Option -d, retrieve usage info from given log file dates; 
+
+      - Option -N, retrieve usage info in recent NumberDay days;
+
+      - Option -p, retrieve usage info between given period. For missing EndDate,
+          it defaults to the current date.

{rda_python_metrics-1.0.4 → rda_python_metrics-1.0.5}/src/rda_python_metrics/viewcodusage.py

@@ -243,7 +243,7 @@ def check_enough_options():
 def build_query_strings(usgtable):
 
    # initialize query strings
-   global condition, fieldnames, tablenames, gfields, sfields
+   global condition, fieldnames, tablenames
    joins = having = groupnames = ''
    tablenames = usgtable
    cols = params['C'][0]

{rda_python_metrics-1.0.4 → rda_python_metrics-1.0.5/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.4
+Version: 1.0.5
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.4 → rda_python_metrics-1.0.5}/src/rda_python_metrics.egg-info/SOURCES.txt

@@ -17,6 +17,8 @@ src/rda_python_metrics/fillipinfo.py
 src/rda_python_metrics/fillipinfo.usg
 src/rda_python_metrics/filloneorder.py
 src/rda_python_metrics/filloneorder.usg
+src/rda_python_metrics/fillosdfusage.py
+src/rda_python_metrics/fillosdfusage.usg
 src/rda_python_metrics/fillrdadb.py
 src/rda_python_metrics/fillrdadb.usg
 src/rda_python_metrics/filltdsusage.py

{rda_python_metrics-1.0.4 → rda_python_metrics-1.0.5}/src/rda_python_metrics.egg-info/entry_points.txt

@@ -6,8 +6,9 @@ fillendtime = rda_python_metrics.fillendtime:main
 fillglobususage = rda_python_metrics.fillglobususage:main
 fillipinfo = rda_python_metrics.fillipinfo:main
 filloneorder = rda_python_metrics.filloneorder:main
+fillosdfusage = rda_python_metrics.fillosdfusage:main
 fillrdadb = rda_python_metrics.fillrdadb:main
-filltdsusage = rda_python_metrics.tdsusage:main
+filltdsusage = rda_python_metrics.filltdsusage:main
 filluser = rda_python_metrics.filluser:main
 logarch.py = rda_python_metrics.logarch:main
 pgperson = rda_python_metrics.pgperson:main