rda-python-metrics 1.0.4__py3-none-any.whl → 1.0.5__py3-none-any.whl

rda_python_metrics/fillawsusage.py

@@ -184,7 +184,7 @@ def get_record_date_time(ctime):
       m = PgUtil.get_month(ms.group(2))
       y = ms.group(3)
       t = ms.group(4)
-      q = 1 + (m-1)/3
+      q = 1 + int((m-1)/3)
       return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
    else:
       PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)

rda_python_metrics/fillglobususage.py

@@ -186,7 +186,7 @@ def get_record_date_time(ctime):
       m = PgUtil.get_month(ms.group(2))
       y = ms.group(3)
       t = ms.group(4)
-      q = 1 + (m-1)/3
+      q = 1 + int((m-1)/3)
       return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
    else:
       PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)

rda_python_metrics/fillosdfusage.py

@@ -0,0 +1,265 @@
+##!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillosdfusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 2025-04-01
+#   Purpose : python program to retrieve info from weekly OSDF logs 
+#             and fill table wusages in PgSQL database dssdb.
+# 
+#    Github : https://github.com/NCAR/rda-pythn-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from rda_python_common import PgSplit
+from . import PgIPInfo
+
+USAGE = {
+   'OSDFTBL'  : "wusage",
+   'OSDFDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/logs/gridftp/",
+   'OSDFGET' : 'wget -m -nH -np -nd https://pelicanplatform.org/pelican-access-logs/ncar-access-log/',
+   'OSDFLOG' : "{}.log",   # YYYY-MM-DD.log
+}
+
+#
+# main function to run this program
+#
+def main():
+
+   params = []  # array of input values
+   argv = sys.argv[1:]
+   option = None
+   datelimits = [None, None]
+   
+
+   for arg in argv:
+      ms = re.match(r'^-(b|d|p|N)$', arg)
+      if ms:
+         opt = ms.group(1)
+         if opt == 'b':
+            PgLOG.PGLOG['BCKGRND'] = 1
+         elif option:
+            PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
+         else:
+            option = opt
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         params.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+   
+   if not (option and params): PgLOG.show_usage('fillosdfusage')
+
+   PgDBI.dssdb_dbname()
+   cmdstr = "fillosdfusage {}".format(' '.join(argv))
+   PgLOG.cmdlog(cmdstr)
+   PgFile.change_local_directory(USAGE['OSDFDIR'])
+   filenames = get_log_file_names(option, params, datelimits)
+   if filenames:
+      fill_osdf_usages(filenames, datelimits)
+   else:
+      PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
+
+   PgLOG.pglog(None, PgLOG.LOGWRN)
+   sys.exit(0)
+
+#
+# get the log file dates 
+#
+def get_log_file_names(option, params, datelimits):
+
+   filenames = []
+   if option == 'd':
+      for pdate in params:
+         pdays = PgUtil.get_weekday(pdate)
+         if pdays > 0:
+            PgLOG.pglog(pdate + ": Skip a Non-Sunday date", PgLOG.LOGWRN)
+            continue
+         filenames.append(USAGE['OSDFLOG'].format(pdate))
+   else:
+      if option == 'N':
+         edate = PgUtil.curdate()
+         pdate = datelimits[0] = PgUtil.adddate(edate, 0, 0, -int(params[0]))
+      else:
+         pdate = datelimits[0] = params[0]
+         if len(params) > 1:
+            edate = datelimits[1] = params[1]
+         else:
+            edate = PgUtil.curdate()
+      pdays = PgUtil.get_weekday(pdate)
+      if pdays > 0: pdate = PgUtil.adddate(edate, 0, 0, 7-pdays)
+      while pdate <= edate:
+         filenames.append(USAGE['OSDFLOG'].format(pdate))
+         pdate = PgUtil.adddate(pdate, 0, 0, 7)
+
+   return filenames
+
+#
+# Fill OSDF usages into table dssdb.osdfusage of DSS PgSQL database from osdf access logs
+#
+def fill_osdf_usages(fnames, datelimits):
+
+   cntall = addall = 0
+
+   fcnt = len(fnames)
+   for logfile in fnames:
+      PgLOG.pgsystem(USAGE['OSDFGET'] + logfile, 5, PgLOG.LOGWRN)
+      linfo = PgFile.check_local_file(logfile)
+      if not linfo:
+         PgLOG.pglog("{}: Not exists for Gathering OSDF usage".format(logfile), PgLOG.LOGWRN)
+         continue
+      if linfo['data_size'] == 0:
+         PgLOG.pglog("{}: Empty log for Gathering OSDF usage".format(logfile), PgLOG.LOGWRN)
+         continue
+      PgLOG.pglog("Gathering usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
+      osdf = PgFile.open_local_file(logfile)
+      if not osdf: continue
+      cntadd = entcnt = 0
+      pkey = None
+      while True:
+         line = osdf.readline()
+         if not line: break
+         entcnt += 1
+         if entcnt%10000 == 0:
+            PgLOG.pglog("{}: {}/{} OSDF log entries processed/records added".format(logfile, entcnt, cntadd), PgLOG.WARNLG)
+
+         ms = re.match(r'^\[(\S+)\] \[Objectname:(\/ncar\/rda\/([a-z]\d{6})\/(\S+)\] \[Host:(\S+)\] \[server:(\S+)\] \[Read:(\d+)\]', line)
+         if not ms: continue
+         size = int(ms.group(6))
+         if size < 100: continue  # ignore small files
+         ip = ms.group(4)
+         dsid = PgUtil.format_dataset_id(ms.group(2))
+         wfile = ms.group(3)
+         engine = ms.group(5)
+         
+         (year, quarter, date, time) = get_record_date_time(ms.group(1))
+         if datelimits[0] and date < datelimits[0]: continue
+         if datelimits[1] and date > datelimits[1]: continue
+         locflag = 'C'
+         method = "OSDF"
+
+         record = {'ip' : ip, 'dsid' : dsid, 'wfile' : wfile, 'date' : date,
+                   'time' : time, 'quarter' : quarter, 'size' : size,
+                   'locflag' : locflag, 'method' : method}
+         cntadd += add_file_usage(year, record)
+      osdf.close()
+      cntall += entcnt
+      addall += cntadd
+      PgLOG.pglog("{} OSDF usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+
+def get_record_date_time(ctime):
+
+   ms = re.search(r'^(\d+)-(\d+)-(\d+)T([\d:]+)\.', ctime)
+   if ms:
+      y = ms.group(1)
+      m = int(ms.group(2))
+      d = int(ms.group(3))
+      t = ms.group(4)
+      q = 1 + int((m-1)/3)
+      return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
+   else:
+      PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
+
+#
+# Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
+#
+def add_file_usage(year, logrec):
+
+   pgrec = get_wfile_wid(logrec['dsid'], logrec['wfile'])
+   if not pgrec: return 0
+
+   table = "{}_{}".format(USAGE['OSDFTBL'], year)
+   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
+   if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
+
+   wurec =  get_wuser_record(logrec['ip'], logrec['date'])
+   if not wurec: return 0
+   record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
+   record['wuid_read'] = wurec['wuid']
+   record['date_read'] = logrec['date']
+   record['time_read'] = logrec['time']
+   record['size_read'] = logrec['size']
+   record['method'] = logrec['method']
+   record['locflag'] = logrec['locflag']
+   record['ip'] = logrec['ip']
+   record['quarter'] = logrec['quarter']
+
+   if add_to_allusage(year, logrec, wurec):
+      return PgDBI.add_yearly_wusage(year, record)
+   else:
+      return 0
+
+def add_to_allusage(year, logrec, wurec):
+
+   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'], 'country' : wurec['country']}
+   pgrec['dsid'] = logrec['dsid']
+   pgrec['date'] = logrec['date']
+   pgrec['quarter'] = logrec['quarter']
+   pgrec['time'] = logrec['time']
+   pgrec['size'] = logrec['size']
+   pgrec['method'] = logrec['method']
+   pgrec['ip'] = logrec['ip']
+   pgrec['source'] = 'W'
+   return PgDBI.add_yearly_allusage(year, pgrec)
+
+#
+# return wfile.wid upon success, 0 otherwise
+#
+def get_wfile_wid(dsid, wfile):
+
+   wfcond = "wfile = '{}'".format(wfile) 
+   pgrec = PgSplit.pgget_wfile(dsid, "*", wfcond)
+   if pgrec:
+      pgrec['dsid'] = dsid
+   else:
+      pgrec = PgDBI.pgget("wfile_delete", "*", "{} AND dsid = '{}'".format(wfcond, dsid))
+      if not pgrec:
+         pgrec = PgDBI.pgget("wmove", "wid, dsid", wfcond)
+         if pgrec:
+            pgrec = PgSplit.pgget_wfile(pgrec['dsid'], "*", "wid = {}".format(pgrec['wid']))
+            if pgrec: pgrec['dsid'] = dsid
+
+   return pgrec
+
+# return wuser record upon success, None otherwise
+def get_wuser_record(ip, date):
+
+   ipinfo = PgIPInfo.set_ipinfo(ip)
+   if not ipinfo: return None
+
+   record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
+   email = 'unknown@' + ipinfo['hostname']
+   emcond = "email = '{}'".format(email)
+   flds = 'wuid, email, org_type, country, start_date'   
+   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
+   if pgrec:
+      if PgUtil.diffdate(pgrec['start_date'], date) > 0:
+         pgrec['start_date'] = record['start_date'] = date
+         PgDBI.pgupdt('wuser', record, emcond)
+      return pgrec
+
+   # now add one in
+   record['email'] = email
+   record['stat_flag'] = 'A'
+   record['start_date'] = date
+   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
+   if wuid:
+      record['wuid'] = wuid
+      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
+      return record
+
+   return None
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/fillosdfusage.usg

@@ -0,0 +1,17 @@
+
+ Retrieves usage information from OSDF Server logs under directory
+   /gpfs/fs1/collections/rda/work/zji/osdflogs/ to fill table 'wusage' in
+   database 'dssdb'.
+
+ Usage: fillosdfusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
+
+      select option, -d, -N or -p to run this application.
+
+      - Option -b, log process information into logfile only;
+
+      - Option -d, retrieve usage info from given log file dates; 
+
+      - Option -N, retrieve usage info in recent NumberDay days;
+
+      - Option -p, retrieve usage info between given period. For missing EndDate,
+          it defaults to the current date.

rda_python_metrics/viewcodusage.py

@@ -243,7 +243,7 @@ def check_enough_options():
 def build_query_strings(usgtable):
 
    # initialize query strings
-   global condition, fieldnames, tablenames, gfields, sfields
+   global condition, fieldnames, tablenames
    joins = having = groupnames = ''
    tablenames = usgtable
    cols = params['C'][0]

{rda_python_metrics-1.0.4.dist-info → rda_python_metrics-1.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.4
+Version: 1.0.5
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.4.dist-info → rda_python_metrics-1.0.5.dist-info}/RECORD

@@ -1,18 +1,20 @@
 rda_python_metrics/PgIPInfo.py,sha256=NJe5hRwxuflH_CZBZmFCgzYU6XFZXP44PoSbqbpPOwM,5727
 rda_python_metrics/PgView.py,sha256=r6otb3DjfiaQJdg0z8bZQAOlhr4JnrXJzp9wgWh_8qQ,24369
 rda_python_metrics/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-rda_python_metrics/fillawsusage.py,sha256=WDwdfF0bDBB8j49vjiWE-EM5GZ0d3PGAbEIxp_Nl4vs,9050
+rda_python_metrics/fillawsusage.py,sha256=dpI3-EFINJECdCSUOm37A97CJNIxOS2nYwA0fB0IpHE,9055
 rda_python_metrics/fillawsusage.usg,sha256=qHdQGMbXlbsqEAoEmM71_zlE3yw704nyhXZ4sw4FIbI,631
 rda_python_metrics/fillcodusage.py,sha256=Hp3VrlIqHBWRI6Zjbi0nxWZPNvPrKsGBSZ1L9qA9_y0,8006
 rda_python_metrics/fillcodusage.usg,sha256=QAM6wqycyDb-6t5PhlL2niPiGxZ4Gln0QVdKdyN3ShI,684
 rda_python_metrics/fillcountry.py,sha256=YYm0lIIfljA7rIAOFgP1fhyhqkBDOY6vdk7g11WFPLc,2359
 rda_python_metrics/fillendtime.py,sha256=skZttlpoY19g0dGwqGQI8t_1YPPTPEXwg3EfNlfL90I,2533
-rda_python_metrics/fillglobususage.py,sha256=5xUbjGNjX8uhNyHDsYlR4cJU2ARPibElSNPUIdrdknc,9437
+rda_python_metrics/fillglobususage.py,sha256=-cvIipaFju75aw9axHkx6JIe9HWYwQOD8-0blQaxxUM,9442
 rda_python_metrics/fillglobususage.usg,sha256=p-f5hGGDkcM2O8ADEP0Do_lMIIFj8KkiFG1HJ-YgtQM,637
 rda_python_metrics/fillipinfo.py,sha256=xIVJ6nDvVvMOjb7s_6-YDLVRBC09pDFugnjB3Nrmqus,5641
 rda_python_metrics/fillipinfo.usg,sha256=taITqZa6GL0-wxXcMEdMU3ZlQbJ1CsmFclTvXpG5TLg,644
 rda_python_metrics/filloneorder.py,sha256=MhllvtS2PM1SMkf1dHmGTIppIkt__SRMKIUPrv_iRRU,5472
 rda_python_metrics/filloneorder.usg,sha256=mtOySKx6-D4k2bbTcmi6cSYtINiycRyHQkHozi0CQu0,1466
+rda_python_metrics/fillosdfusage.py,sha256=LVJKdHIhBStUHgmrbXVWhou9kl3kiD8zw6vdLQ_Fun0,8759
+rda_python_metrics/fillosdfusage.usg,sha256=Jlyn6K1LS1ZqDbmOQpR3KNVnp43-kfyvqYlcDt4jPOM,632
 rda_python_metrics/fillrdadb.py,sha256=MDcl6oM-A1xek9SBP9Gvor_Ouq5dB7mki2Vf_w8VDNk,5199
 rda_python_metrics/fillrdadb.usg,sha256=E_Bf4G8yVABogjRmIOaIbTGgnII2W6RltaFad2XEV2Q,1228
 rda_python_metrics/filltdsusage.py,sha256=Hg09ogrqaTfULyBy3eLGTudDs_5JrXEpf9_V5uEGiN0,10148
@@ -27,7 +29,7 @@ rda_python_metrics/viewallusage.py,sha256=B-4s3aoAkAkeB1QM_xfZceRe_QI16vwpI81ekb
 rda_python_metrics/viewallusage.usg,sha256=ABtOCqGoE6HKE1IPsk02ppC883vNiJILRPBRrpbnzRM,9296
 rda_python_metrics/viewcheckusage.py,sha256=HougqjDAOVG6pYglFjyHQ-UdLBcYe7v_jzU1-80RqFA,12996
 rda_python_metrics/viewcheckusage.usg,sha256=KuJFycggGiUcSezQ9vywDbituvu63SZ-ZnNTaMpbc-A,8930
-rda_python_metrics/viewcodusage.py,sha256=jYeZz86vARtXlDJqqP_gPdmi1BKWhPMMpNI4RfTunwA,14030
+rda_python_metrics/viewcodusage.py,sha256=ScyZFjMSss1GNZdmXVs9wWRbaPZRahaFXsWG8kIVRP4,14012
 rda_python_metrics/viewcodusage.usg,sha256=_kgF7Tk2_n1JVf9km2MiwO86vtZRCdu4i8hkWN0eETo,8637
 rda_python_metrics/viewordusage.py,sha256=9zIJkThKgSOW58qXyQs2Hq8EeEp645lnpD5bstSzR_0,15370
 rda_python_metrics/viewordusage.usg,sha256=TqZDQk-DzOWC6_uzmFzGyA4F98ojOifANJGv9BCfH1I,10599
@@ -39,9 +41,9 @@ rda_python_metrics/viewwebfile.py,sha256=BqtA_YNhprnrGE6GWEW7n5PDxzNlljfv_MOPezO
 rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
 rda_python_metrics/viewwebusage.py,sha256=jhoHuRPVNtp7Lbjv0l-Jy_vp2p2nWQC7IVZ0P4JUJ4I,16657
 rda_python_metrics/viewwebusage.usg,sha256=IsT72v22xyZf7ng_IodVs0dLTsH1Q4BtFvT-gs0-xJY,9946
-rda_python_metrics-1.0.4.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_metrics-1.0.4.dist-info/METADATA,sha256=omB9dxreEDSDXR4LE90kn3xHXUWBvSxztq2QFtnP6rU,667
-rda_python_metrics-1.0.4.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rda_python_metrics-1.0.4.dist-info/entry_points.txt,sha256=UOZhtPc8-DypWpkOP-nnF7OH8Fdp_wk6cuqh19JsmZA,1075
-rda_python_metrics-1.0.4.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
-rda_python_metrics-1.0.4.dist-info/RECORD,,
+rda_python_metrics-1.0.5.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_metrics-1.0.5.dist-info/METADATA,sha256=ESuOgR-2a5KXfttLYT7v9TSWRRYWwOT5_XlNs4YKV7c,667
+rda_python_metrics-1.0.5.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rda_python_metrics-1.0.5.dist-info/entry_points.txt,sha256=uoCDpwsZJNKXSxkngolWGgmi7arXp7BM-p6tVHVWvgc,1133
+rda_python_metrics-1.0.5.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
+rda_python_metrics-1.0.5.dist-info/RECORD,,

{rda_python_metrics-1.0.4.dist-info → rda_python_metrics-1.0.5.dist-info}/entry_points.txt

@@ -6,8 +6,9 @@ fillendtime = rda_python_metrics.fillendtime:main
 fillglobususage = rda_python_metrics.fillglobususage:main
 fillipinfo = rda_python_metrics.fillipinfo:main
 filloneorder = rda_python_metrics.filloneorder:main
+fillosdfusage = rda_python_metrics.fillosdfusage:main
 fillrdadb = rda_python_metrics.fillrdadb:main
-filltdsusage = rda_python_metrics.tdsusage:main
+filltdsusage = rda_python_metrics.filltdsusage:main
 filluser = rda_python_metrics.filluser:main
 logarch.py = rda_python_metrics.logarch:main
 pgperson = rda_python_metrics.pgperson:main