rda-python-metrics 1.0.27__tar.gz → 1.0.46__tar.gz

{rda_python_metrics-1.0.27/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.27
+Version: 1.0.46
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
@@ -17,6 +17,7 @@ Requires-Dist: geoip2
 Requires-Dist: ipinfo
 Requires-Dist: httplib2
 Requires-Dist: dnspython
+Requires-Dist: unidecode
 Dynamic: license-file
 
 RDA Python Package to gather and view data usage metrics.

{rda_python_metrics-1.0.27 → rda_python_metrics-1.0.46}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.27"
+version = "1.0.46"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]
@@ -25,7 +25,8 @@ dependencies = [
   "geoip2",
   "ipinfo",
   "httplib2",
-  "dnspython"
+  "dnspython",
+  "unidecode"
 ]
 
 [tool.pytest.ini_options]
@@ -38,12 +39,14 @@ pythonpath = [
 
 [project.scripts]
 "logarch.py" = "rda_python_metrics.logarch:main"
-"fillawsusage" = "rda_python_metrics.fillaswusage:main"
+"fillawsusage" = "rda_python_metrics.fillawsusage:main"
 "fillcdgusage" = "rda_python_metrics.fillcdgusage:main"
 "fillcodusage" = "rda_python_metrics.fillcodusage:main"
 "fillcountry" = "rda_python_metrics.fillcountry:main"
 "fillendtime" = "rda_python_metrics.fillendtime:main"
 "fillglobususage" = "rda_python_metrics.fillglobususage:main"
+"fillgdexusage" = "rda_python_metrics.fillgdexusage:main"
+"fillzenodousage" = "rda_python_metrics.fillzenodousage:main"
 "fillipinfo" = "rda_python_metrics.fillipinfo:main"
 "filloneorder" = "rda_python_metrics.filloneorder:main"
 "fillosdfusage" = "rda_python_metrics.fillosdfusage:main"
@@ -53,9 +56,11 @@ pythonpath = [
 "pgperson" = "rda_python_metrics.pgperson:main"
 "pgusername" = "rda_python_metrics.pgusername:main"
 "viewallusage" = "rda_python_metrics.viewallusage:main"
+"viewawsusage" = "rda_python_metrics.viewawsusage:main"
 "viewcheckusage" = "rda_python_metrics.viewcheckusage:main"
 "viewcodusage" = "rda_python_metrics.viewcodusage:main"
 "viewordusage" = "rda_python_metrics.viewordusage:main"
+"viewosdfusage" = "rda_python_metrics.viewosdfusage:main"
 "viewrqstusage" = "rda_python_metrics.viewrqstusage:main"
 "viewtdsusage" = "rda_python_metrics.viewtdsusage:main"
 "viewwebfile" = "rda_python_metrics.viewwebfile:main"

{rda_python_metrics-1.0.27 → rda_python_metrics-1.0.46}/src/rda_python_metrics/PgIPInfo.py

@@ -138,7 +138,10 @@ def get_ipinfo_record(ip):
    except Exception as e:
       PgLOG.pglog("ipinfo: {} - {}".format(ip, str(e)), PgLOG.LOGWRN)
       return None
-      
+
+   if 'bogon' in iprec and iprec['bogon']:
+      PgLOG.pglog(f"ipinfo: {ip} - bogon", PgLOG.LOGWRN)
+      return None
    record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip, 'org_type' : '-'}
    get_ip_hostname(ip, iprec, record)
    record['lat'] = float(iprec['latitude']) if iprec['latitude'] else 0
@@ -234,12 +237,9 @@ def update_ipinfo_record(record, pgrec = None):
 #
 def set_ipinfo(ip, ipopt = True):
 
-   if ip in IPRECS:
-      pgrec = IPRECS[ip]
-      if pgrec or not ipopt: return pgrec
-   else:
-      pgrec = PgDBI.pgget('ipinfo', '*', "ip = '{}'".format(ip))
+   if ip in IPRECS: return IPRECS[ip]
 
+   pgrec = PgDBI.pgget('ipinfo', '*', "ip = '{}'".format(ip))
    if not pgrec or ipopt and pgrec['stat_flag'] == 'M':
       record = get_ipinfo_record(ip) if ipopt else None
       if not record: record = get_geoip2_record(ip)
@@ -303,7 +303,7 @@ def get_wuser_record(ip, date, email = None):
    wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
    if wuid:
       record['wuid'] = wuid
-      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
+      PgLOG.pglog("{} Added as wuid({})".format(record['email'], wuid), PgLOG.LGWNEM)
       return record
 
    return None

rda_python_metrics-1.0.46/src/rda_python_metrics/fillawsusage.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillawsusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 03/11/2022
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to retrieve info from AWS logs 
+#             and fill table wusages in PgSQL database dssdb.
+# 
+#    Github : https://github.com/NCAR/rda-pythn-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+import glob
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from . import PgIPInfo
+
+USAGE = {
+   'PGTBL'  : "awsusage",
+   'AWSDIR' : PgLOG.PGLOG["TRANSFER"] + "/AWSera5log",
+   'AWSLOG' : "{}/{}-00-00-00-*",
+   'PFMT'   : "YYYY/MM/DD"
+}
+
+DSIDS = {'nsf-ncar-era5' : 'd633000'}
+
+#
+# main function to run this program
+#
+def main():
+
+   params = []  # array of input values
+   argv = sys.argv[1:]
+   option = None
+
+   for arg in argv:
+      ms = re.match(r'^-(b|d|p|N)$', arg)
+      if ms:
+         opt = ms.group(1)
+         if opt == 'b':
+            PgLOG.PGLOG['BCKGRND'] = 1
+         elif option:
+            PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
+         else:
+            option = opt
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         params.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+   
+   if not (option and params): PgLOG.show_usage('fillawsusage')
+
+   PgDBI.dssdb_dbname()
+   cmdstr = "fillawsusage {}".format(' '.join(argv))
+   PgLOG.cmdlog(cmdstr)
+   PgFile.change_local_directory(USAGE['AWSDIR'])
+   filenames = get_log_file_names(option, params)
+   if filenames:
+      fill_aws_usages(filenames)
+   else:
+      PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
+
+   PgLOG.pglog(None, PgLOG.LOGWRN)
+   sys.exit(0)
+
+#
+# get the log file dates 
+#
+def get_log_file_names(option, params):
+
+   filenames = {}
+   if option == 'd':
+      for dt in params:
+         pdate = PgUtil.format_date(dt)
+         pd = PgUtil.format_date(pdate, USAGE['PFMT'])
+         fname = USAGE['AWSLOG'].format(pd, pdate)
+         fnames = glob.glob(fname)
+         if fnames: filenames[pdate] = sorted(fnames)
+   else:
+      if option == 'N':
+         edate = PgUtil.curdate()
+         pdate = PgUtil.adddate(edate, 0, 0, -int(params[0]))
+      else:
+         pdate = PgUtil.format_date(params[0])
+         if len(params) > 1:
+            edate = PgUtil.adddate(PgUtil.format_date(params[1]), 0, 0, 1)
+         else:
+            edate = PgUtil.curdate()
+      while pdate < edate:
+         pd = PgUtil.format_date(pdate, USAGE['PFMT'])
+         fname = USAGE['AWSLOG'].format(pd, pdate)
+         fnames = glob.glob(fname)
+         if fnames: filenames[pdate] = sorted(fnames)
+         pdate = PgUtil.adddate(pdate, 0, 0, 1)
+
+   return filenames
+
+#
+# Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs
+#
+def fill_aws_usages(filenames):
+
+   year = cntall = addall = 0
+   for pdate in filenames:
+      fnames = filenames[pdate]
+      fcnt = len(fnames)
+      PgLOG.pglog("{}: Gathering AWS usage info from {} log files at {}".format(pdate, fcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+      records = {}
+      cntadd = entcnt = 0
+      for logfile in fnames:
+         if not op.isfile(logfile):
+            PgLOG.pglog("{}: Not exists for Gathering AWS usage".format(logfile), PgLOG.LOGWRN)
+            continue
+         aws = PgFile.open_local_file(logfile)
+         if not aws: continue
+         while True:
+            line = aws.readline()
+            if not line: break
+            entcnt += 1
+            if entcnt%20000 == 0:
+               dcnt = len(records)
+               PgLOG.pglog("{}: {}/{} AWS log entries processed/records to add".format(pdate, entcnt, dcnt), PgLOG.WARNLG)
+   
+            ms = re.match(r'^\w+ ([\w-]+) \[(\S+).*\] ([\d\.]+) .+ REST\.GET\.OBJECT \S+ "GET.+" \d+ - (\d+) \d+ .* ".+" "(.+)" ', line)
+            if not ms: continue
+            values = list(ms.groups())
+            if values[0] not in DSIDS: continue
+            dsid = DSIDS[values[0]]
+            size = int(values[3])
+            ip = values[2]
+            engine = values[4]
+            moff = engine.find('/')
+            if moff > 0:
+               if moff > 20: moff = 20
+               method = engine[0:moff].upper()
+            else:
+               method = "AWS"
+            key = "{}:{}:{}".format(ip, dsid, method)
+            if key in records:
+               records[key]['size'] += size
+               records[key]['fcount'] += 1
+            else:
+               (year, quarter, date, time) = get_record_date_time(values[1])
+               iprec =  PgIPInfo.get_missing_ipinfo(ip)
+               if not iprec: continue
+               records[key] = {'ip' : ip, 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
+                               'size' : size, 'fcount' : 1, 'method' : method, 'engine' : engine,
+                               'org_type' : iprec['org_type'], 'country' : iprec['country'],
+                               'region' : iprec['region'], 'email' : iprec['email']}
+         aws.close()
+      if records: cntadd = add_usage_records(records, year)
+      PgLOG.pglog("{}: {} AWS usage records added for {} entries at {}".format(pdate, cntadd, entcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+      cntall += entcnt
+      if cntadd:
+         addall += cntadd
+         if addall > cntadd:
+            PgLOG.pglog("{} AWS usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+def get_record_date_time(ctime):
+   
+   ms = re.search(r'^(\d+)/(\w+)/(\d+):(\d+:\d+:\d+)$', ctime)
+   if ms:
+      d = int(ms.group(1))
+      m = PgUtil.get_month(ms.group(2))
+      y = ms.group(3)
+      t = ms.group(4)
+      q = 1 + int((m-1)/3)
+      return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
+   else:
+      PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
+
+def add_usage_records(records, year):
+
+   cnt = 0
+   for key in records:
+      record = records[key]
+      cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(record['date'], record['time'], record['ip'])
+      if PgDBI.pgget(USAGE['PGTBL'], '', cond, PgLOG.LGEREX): continue
+      if add_to_allusage(year, record):
+         cnt += PgDBI.pgadd(USAGE['PGTBL'], record, PgLOG.LOGWRN)
+
+   return cnt
+
+
+def add_to_allusage(year, pgrec):
+
+   record = {'source' : 'A'}
+   flds = ['ip', 'dsid', 'date', 'time', 'quarter', 'size', 'method',
+           'org_type', 'country', 'region', 'email']
+
+   for fld in flds:
+      record[fld] = pgrec[fld]
+
+   return PgDBI.add_yearly_allusage(year, record)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

{rda_python_metrics-1.0.27 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillawsusage.usg

@@ -1,6 +1,6 @@
 
  Retrieves usage information from AWS Server logs under directory
-   ../rda/transer/AWSera5log/ to fill table 'wusage' in database 'rdadb'.
+   ../rda/transer/AWSera5log/ to fill table 'awsusage' in database 'rdadb'.
 
  Usage: fillawsusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
 

{rda_python_metrics-1.0.27 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcdgusage.py

@@ -31,6 +31,8 @@ USAGE = {
 }
 
 DSIDS = {
+   'pi_cesm2_atm_river_analysis' : ['d010073'],
+   'na-cordex' : ['d316009'],
    'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
    'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
    'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
@@ -53,7 +55,6 @@ DSIDS = {
    'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
    'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
    'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
-   # new added
    'ucar.cgd.cesm2-waccm.s2s_hindcasts': ['d651040'],
    'ucar.cgd.CESM1.3_SH_storm_tracks': ['d651044'],
    'ucar.cgd.cesm2.waccm6.ssp245': ['d651045'],
@@ -61,18 +62,36 @@ DSIDS = {
    'ucar.cgd.ccsm4.TC-CESM': ['d651047'],
    'ucar.cgd.cesm2.ISSI_OSSE': ['d651048'],
    'ucar.cgd.ccsm4.SOcean_Eddies_mclong': ['d651049'],
-   'trace': ['d651050'],
+   'ucar.cgd.ccsm.trace': ['d651050'],
    'ucar.cgd.cesm2.waccm.solar': ['d651051'],
    'ucar.cgd.ccsm4.CESM1-CCSM4_mid-Pliocene' : ['d651042'],
    'ucar.cgd.ccsm4.PaleoIF' : ['d651052'],
    'ucar.cgd.ccsm4.b.e11.B20LE_fixedO3' : ['d651053'],
    'ucar.cgd.cesm2.single.forcing.large.ensemble' : ['d651055'],
    'ucar.cgd.cesm2le.output': ['d651056'],
+   'ucar.cgd.ccsm4.ARISE-SAI-1.5' : ['d651059'],
    'ucar.cgd.cesm2.s2s_hindcasts': ['d651060'],
    'ucar.cgd.cesm2.s2s_hindcasts.mjo': ['d651061'],
    'ucar.cgd.cesm2.s2s_hindcasts.tc_tracks': ['d651062'],
    'ucar.cgd.cesm2.s2s_hindcasts.cesm2.climo': ['d651063'],
-   'ucar.cgd.ccsm4.ARISE-SAI-1.5' : ['d651059']
+   'ucar.cgd.ccsm4.cesmLME' : ['d651058'],
+   'ucar.cgd.ccsm4.GLENS' : ['d651064'],
+   'ucar.cgd.ccsm4.CESM2-CISM2-LIGtransient' : ['d651066'],
+   'ucar.cgd.cesm2.pacific.pacemaker' : ['d651068'],
+   'ucar.cgd.cesm2.tuned.sea.ice.albedo' : ['d651070'],
+   'ucar.cgd.cesm2.cmip5.forcing' : ['d651075'],
+   'ucar.cgd.cesm2.ssp245.biomass.burning' : ['d651073'],
+   'ucar.cgd.cesm2.ssp585.biomass.burning' : ['d651067'],
+   'ucar.cgd.cesm1.cldmod': ['d651069'],
+   'ucar.cgd.cesm2.marine.biogeochemistry': ['d651071'],
+   'ucar.cgd.nw2.mom6': ['d651072'],
+   'ucar.cgd.cesm2.cam6.ppe': ['d651076'],
+   'ucar.cgd.cesm2.smyle': ['d651065'],
+# new added
+   'gridded_precip_and_temp' : ['d010078'],
+   '29_newman' : ['d010079'],
+   'waccm-x.ion.asymmetry' : ['d010081'],
+   'NARCCAP' : ['d316015']   
 }
 
 ALLIDS = list(DSIDS.keys())
@@ -225,12 +244,14 @@ def fill_cdg_usages(dsids, dranges):
 
    allcnt = awcnt = atcnt = lcnt = 0
    for dates in dranges:
-      for dsid in dsids:
+      for adsid in dsids:
          lcnt += 1
-         dsname = dsid[0]
-         rdaids = dsid[1]
-         cdgids = dsid[2]
-         strids = dsid[3]
+         dsname = adsid[0]
+         rdaids = adsid[1]
+         getdsid = False if len(rdaids) == 1 else True
+         dsid = rdaids[0]
+         cdgids = adsid[2]
+         strids = adsid[3]
          bt = tm()
          pgrecs = get_dsid_records(cdgids, dates, strids)
          pgcnt = len(pgrecs['dataset_file_name']) if pgrecs else 0
@@ -248,6 +269,10 @@ def fill_cdg_usages(dsids, dranges):
                PgLOG.pglog("{}/{}/{} CDG/TDS/WEB records processed to add".format(i, tcnt, wcnt), PgLOG.WARNLG)
 
             pgrec = PgUtil.onerecord(pgrecs, i)
+            wfile = pgrec['dataset_file_name']
+            if not wfile:
+               wfile = pgrec['logic_file_name']
+               if not wfile: continue
             dsize = pgrec['bytes_sent']
             if not dsize: continue
             (year, quarter, date, time) = get_record_date_time(pgrec['date_completed'])
@@ -255,14 +280,13 @@ def fill_cdg_usages(dsids, dranges):
             if not url: url = pgrec['file_access_point_uri']
             ip = pgrec['remote_address']
             engine = pgrec['user_agent_name']
-            wfile = pgrec['dataset_file_name']
-            if not wfile: wfile = pgrec['logic_file_name']
-            wfrec = get_wfile_record(rdaids, wfile)
-            if not wfrec: continue
-            dsid = wfrec['dsid']
             ms = re.search(r'^https*://tds.ucar.edu/thredds/(\w+)/', url)
             if ms:
                # tds usage
+               if getdsid:
+                  wfrec = get_wfile_record(rdaids, wfile)
+                  if not wfrec: continue
+                  dsid = wfrec['dsid']
                method = ms.group(1)
                if pgrec['subset_file_size']:
                   etype = 'S'
@@ -289,6 +313,9 @@ def fill_cdg_usages(dsids, dranges):
                                  'region' : iprec['region'], 'email' : iprec['email']}
             else:
                # web usage
+               wfrec = get_wfile_record(rdaids, wfile)
+               if not wfrec: continue
+               if getdsid: dsid = wfrec['dsid']
                fsize = pgrec['dataset_file_size']
                if not fsize: fsize = pgrec['logic_file_size']
                method = 'CDG'