rda-python-metrics 1.0.7__tar.gz → 1.0.9__tar.gz

rda_python_metrics-1.0.9/MANIFEST.in

@@ -0,0 +1 @@
+include src/rda_python_metrics/*.usg

{rda_python_metrics-1.0.7/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.7
+Version: 1.0.9
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
@@ -13,6 +13,9 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: rda_python_common
 Requires-Dist: rda_python_setuid
+Requires-Dist: geoip2
+Requires-Dist: ipinfo
+Requires-Dist: httplib2
 Dynamic: license-file
 
 RDA Python Package to gather and view data usage metrics.

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.7"
+version = "1.0.9"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]
@@ -22,21 +22,14 @@ classifiers = [
 dependencies = [
   "rda_python_common",
   "rda_python_setuid",
+  "geoip2",
+  "ipinfo",
+  "httplib2"
 ]
 
-[tool.setuptools]
-include-package-data = true
-
-[tool.setuptools.packages.find]
-where = ["src"]
-
-[tool.setuptools.package-data]
-"rda_python_metrics" = [
-"logarch.usg", "fillawsusage.usg", "fillcodusage.usg", "fillglobususage.usg",
-"fillipinfo.usg", "filloneorder.usg", "fillosdfusage.usg", "fillrdadb.usg",
-"filltdsusage.usg", "viewallusage.usg", "viewcheckusage.usg", "viewcodusage.usg",
-"viewordusage.usg", "viewrqstusage.usg", "viewtdsusage.usg", "viewwebfile.usg",
-"viewwebusage.usg"
+[tool.pytest.ini_options]
+pythonpath = [
+  "src"
 ]
 
 [project.urls]
@@ -45,6 +38,7 @@ where = ["src"]
 [project.scripts]
 "logarch.py" = "rda_python_metrics.logarch:main"
 "fillawsusage" = "rda_python_metrics.fillaswusage:main"
+"fillcdgusage" = "rda_python_metrics.fillcdgusage:main"
 "fillcodusage" = "rda_python_metrics.fillcodusage:main"
 "fillcountry" = "rda_python_metrics.fillcountry:main"
 "fillendtime" = "rda_python_metrics.fillendtime:main"

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/src/rda_python_metrics/fillawsusage.usg

@@ -1,7 +1,6 @@
 
  Retrieves usage information from AWS Server logs under directory
-   /gpfs/fs1/collections/rda/transer/AWSera5log/ to fill table 'wusage' in
-   database 'dssdb'.
+   ../rda/transer/AWSera5log/ to fill table 'wusage' in database 'rdadb'.
 
  Usage: fillawsusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
 

rda_python_metrics-1.0.9/src/rda_python_metrics/fillcdgusage.py

@@ -0,0 +1,408 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillcdgusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 2025-04-14
+#   Purpose : python program to retrieve info from GDEX Postgres database for GDS 
+#             file accesses and backup fill table tdsusage in PostgreSQL database dssdb.
+# 
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+import glob
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from rda_python_common import PgSplit
+from . import PgIPInfo
+
+USAGE = {
+   'TDSTBL'  : "tdsusage",
+   'WEBTBL'  : "webusage",
+   'CDATE' : PgUtil.curdate(),
+}
+
+DSIDS = {
+   'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
+   'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
+   'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
+   'tamip' : ['d651013'],
+   'ucar.cgd.ccsm4.CLIVAR_LE' : ['d651014'],
+   'ucar.cgd.cesm2.Gettelman_CESM2_ECS' : ['d651015'],
+   'ucar.cgd.ccsm4.geomip.ssp5' : ['d651024'],
+   'ucar.cgd.ccsm4.IOD-PACEMAKER' : ['d651021'],
+   'ucar.cgd.ccsm4.past2k_transient' : ['651023'],
+   'ucar.cgd.ccsm4.lowwarming' : ['d651025'],
+   'ucar.cgd.ccsm4.CESM_CAM5_BGC_ME' : ['d651000'],
+   'ucar.cgd.ccsm4.iTRACE' : ['d651022'],
+   'ucar.cgd.ccsm4.so2_geoeng' : ['d651026'],
+   'ucar.cgd.ccsm4.cesmLE' : ['d651027'],
+   'ucar.cgd.ccsm4.CESM1-CAM5-DP' : ['d651028'],
+   'ucar.cgd.ccsm4.amv_lens' : ['d651031'],
+   'ucar.cgd.ccsm4.ATL-PACEMAKER' : ['d651032'],
+   'ucar.cgd.ccsm4.pac-pacemaker' : ['d651033'],
+   'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
+   'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
+   'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
+   'ucar.cgd.ccsm4.pliomip2' : ['d651037']
+}
+
+
+#
+# main function to run this program
+#
+def main():
+
+   params = {}  # array of input values
+   argv = sys.argv[1:]
+   opt = None
+   
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif re.match(r'^-[msNy]$', arg):
+         opt = arg[1]
+         params[opt] = []
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif opt:
+         params[opt].append(arg)
+      else:
+         PgLOG.pglog(arg + ": Value passed in without leading option", PgLOG.LGWNEX)
+
+   if not opt:
+      PgLOG.show_usage('fillcdgusage')
+   elif 's' not in params:
+      PgLOG.pglog("-s: Missing dataset short name to gather CDG metrics", PgLOG.LGWNEX)
+   elif len(params) < 2:
+      PgLOG.pglog("-(m|N|y): Missing Month, NumberDays or Year to gather CDG metrics", PgLOG.LGWNEX)
+      
+   
+   PgLOG.cmdlog("fillcdgusage {}".format(' '.join(argv)))
+   dsids = get_dataset_ids(params['s'])
+   if dsids:
+      del params['s']
+      for o in params:
+         dranges = get_date_ranges(o, params[o])
+         fill_cdg_usages(dsids, dranges)
+
+   PgLOG.pglog(None, PgLOG.LOGWRN|PgLOG.SNDEML)  # send email out if any
+
+   sys.exit(0)
+
+#
+# connect to the gdex database esg-production
+#
+def gdex_dbname():
+   PgDBI.set_scname('esg-production', 'metrics', 'gateway-reader', None, 'sagedbprodalma.ucar.edu')
+
+#
+# get datasets
+#
+def get_dataset_ids(dsnames):
+
+   gdex_dbname()
+   dsids = []
+   tbname = 'metadata.dataset'
+   for dsname in dsnames:
+      if dsname not in DSIDS:
+         PgLOG.pglog(dsname + ": Unknown CDG dataset short name", PgLOG.LOGWRN)
+         continue
+      rdaid = DSIDS[dsname]
+      pgrec = PgDBI.pgget(tbname, 'id', "short_name = '{}'".format(dsname))
+      if not (pgrec and pgrec['id']): continue
+      dsid = pgrec['id']
+      if dsid in dsids: continue
+      dsids.append([dsid, rdaid])
+      recursive_dataset_ids(dsid, rdaid, dsids)
+
+   if not dsids: PgLOG.pglog("No Dataset Id identified to gather CDG metrics", PgLOG.LOGWRN)
+
+   return dsids
+
+#
+# get dsids recursivley
+#
+def recursive_dataset_ids(pdsid, rdaid, dsids):
+
+   tbname = 'metadata.dataset'
+   pgrecs = PgDBI.pgmget(tbname, 'id', "parent_dataset_id = '{}'".format(pdsid))
+   if not pgrecs: return
+
+   for dsid in pgrecs['id']:
+      if dsid in dsids: continue
+      dsids.append([dsid, rdaid])
+      recursive_dataset_ids(dsid, rdaid, dsids)
+
+#
+# get the date ranges for given condition
+#
+def get_date_ranges(option, inputs):
+
+   dranges = []
+   for input in inputs:
+      # get date range
+      dates = []
+      if option == 'N':
+         dates[1] = USAGE['CDATE']
+         dates[0] = PgUtil.adddate(USAGE['CDATE'], 0, 0, -int(input))  
+      elif option == 'm':
+         tms = input.split('-')
+         dates[0] = PgUtil.fmtdate(int(tms[0]), int(tms[1]), 1)
+         dates[1] = PgUtil.enddate(dates[0])
+      else:
+         dates[0] = input + "-01-01"
+         dates[1] = input + "-12-31"
+      dranges.append(dates)
+
+   return dranges
+
+#
+# get file download records for given dsid
+#
+def get_dsid_records(dsid, dates):
+
+   gdex_dbname()
+   tbname = 'metrics.file_download'
+   fields = ('date_completed, remote_address, logical_file_size, logical_file_name, file_access_point_uri, user_agent_name, bytes_sent, '
+             'subset_file_size, range_request, dataset_file_size, dataset_file_name, dataset_file_file_access_point_uri')
+   cond = "dataset_id = '{}' AND completed = True AND date_completed BETWEEN '{}' AND '{}' ORDER BY date_completed".format(dsid, dates[0], dates[1])
+   pgrecs = PgDBI.pgmget(tbname, fields, cond)
+   PgDBI.dssdb_dbname()
+
+   return pgrecs
+
+#
+# Fill TDS usages into table dssdb.tdsusage from cdg access records
+#
+def fill_cdg_usages(dsids, dranges):
+
+   allcnt = awcnt = atcnt = 0
+   for dsid in dsids:
+      cdgid = dsid[0]
+      rdaid = dsid[1]
+      for dates in dranges:
+         pgrecs = get_dsid_records(cdgid, dates)
+         pgcnt = len(pgrecs['dataset_file_name']) if pgrecs else 0
+         if pgcnt == 0:
+            PgLOG.pglog("{}: No record found to gather CDG usage between {} and {}".format(rdaid, dates[0], dates[1]), PgLOG.LOGWRN)
+            continue
+         PgLOG.pglog("{}: gather {} records for CDG usage between {} and {}".format(rdaid, pgcnt, dates[0], dates[1]), PgLOG.LOGWRN)
+         tcnt = wcnt = 0
+         pwkey = wrec = cdate = None
+         trecs = {}
+         for i in range(pgcnt):
+            if (i+1)%20000 == 0:
+               PgLOG.pglog("{}/{}/{} CDG/TDS/WEB records processed to add".format(i, tcnt, wcnt), PgLOG.WARNLG)
+
+            pgrec = PgUtil.onerecord(i, pgrecs)
+            dsize = pgrec['bytes_sent']
+            if not dsize: continue
+            (year, quarter, date, time) = get_record_date_time(pgrec['date_completed'])
+            url = pgrec['dataset_file_file_access_point_uri']
+            if not url: url = pgrec['file_access_point_uri']
+            ip = pgrec['remote_address']
+            engine = pgrec['user_agent_name']
+            ms = re.search(r'^https://tds.ucar.edu/thredds/(\w+)/', url)
+            if ms:
+               # tds usage
+               method = ms.group(1)
+               if pgrec['subset_file_size']:
+                  etype = 'S'
+               elif pgrec['range_request']:
+                  etype = 'R'
+               else:
+                  etype = 'F'
+
+               if date != cdate:
+                  if trecs:
+                     tcnt += add_tdsusage_records(year, trecs, cdate)
+                     trecs = {}
+                  cdate = date
+               tkey = "{}:{}:{}:{}".format(ip, rdaid, method, etype)
+               if tkey in trecs:
+                  trecs[tkey]['size'] += dsize
+                  trecs[tkey]['fcount'] += 1
+               else:
+                  trecs[tkey] = {'ip' : ip, 'dsid' : rdaid, 'date' : cdate, 'time' : time, 'size' : dsize,
+                                 'fcount' : 1, 'method' : method, 'etype' : etype, 'engine' : engine}
+            else:
+               # web usage
+               wfile = pgrec['dataset_file_name']
+               if not wfile: wfile = pgrec['logic_file_name']
+               fsize = pgrec['dataset_file_size']
+               if not fsize: fsize = pgrec['logic_file_size']
+               method = 'CDP'
+               if pgrec['subset_file_size'] or pgrec['range_request'] or dsize < fsize:
+                  wkey = "{}:{}:{}".format(ip, rdaid, wfile)
+               else:
+                  wkey = None
+      
+               if wrec:
+                  if wkey == pwkey:
+                     wrec['size'] += dsize
+                     continue
+                  wcnt += add_webfile_usage(year, wrec)
+               wrec = {'ip' : ip, 'dsid' : rdaid, 'wfile' : wfile, 'date' : date,
+                       'time' : time, 'quarter' : quarter, 'size' : dsize,
+                       'locflag' : 'C', 'method' : method}
+               pwkey = wkey
+               if not pwkey:
+                  wcnt += add_webfile_usage(year, wrec)
+                  wrec = None
+
+         if trecs: tcnt += add_tdsusage_records(year, trecs, cdate)
+         if wrec: wcnt += add_webfile_usage(year, wrec)
+         atcnt += tcnt
+         awcnt += wcnt
+         allcnt += pgcnt
+
+   PgLOG.pglog("{}/{} TDS/WEB usage records added for {} CDG entries at {}".format(atcnt, awcnt, allcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+
+def get_record_date_time(ctime):
+
+   ms = re.search(r'^(\d+)/(\w+)/(\d+) (\d+:\d+:\d+)(\.|$)', str(ctime))
+   if ms:
+      d = int(ms.group(1))
+      m = PgUtil.get_month(ms.group(2))
+      q = 1 + int((m-1)/3)
+      y = ms.group(3)
+      t = ms.group(4)
+      return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
+   else:
+      PgLOG.pglog("time: Invalid date format", PgLOG.LGEREX)
+
+def add_tdsusage_records(year, records, date):
+
+   cnt = 0
+   for key in records:
+      record = records[key]
+      cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, record['time'], record['ip'])
+      if PgDBI.pgget(USAGE['TDSTBL'], '', cond, PgLOG.LGEREX): continue
+      record['org_type'] = record['country'] = '-'
+      ipinfo = PgIPInfo.set_ipinfo(record['ip'])
+      if ipinfo:
+         record['org_type'] = ipinfo['org_type']
+         record['country'] = ipinfo['country']
+         record['email'] = 'unknown@' + ipinfo['hostname']
+
+      if add_tds_allusage(year, record):
+         cnt += PgDBI.pgadd(USAGE['TDSTBL'], record, PgLOG.LOGWRN)
+
+   PgLOG.pglog("{}: {} TDS usage records added at {}".format(date, cnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+   return cnt
+
+
+def add_tds_allusage(year, pgrec):
+
+   record = {'method' : 'CDP', 'source' : 'C'}
+
+   for fld in pgrec:
+      if re.match(r'^(engine|method|etype|fcount)$', fld): continue
+      record[fld] = pgrec[fld]
+
+   return PgDBI.add_yearly_allusage(year, record)
+
+#
+# Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
+#
+def add_webfile_usage(year, logrec):
+
+   pgrec = get_wfile_wid(logrec['dsid'], logrec['wfile'])
+   if not pgrec: return 0
+
+   table = "{}_{}".format(USAGE['WEBTBL'], year)
+   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
+   if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
+
+   wurec =  get_wuser_record(logrec['ip'], logrec['date'])
+   if not wurec: return 0
+   record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
+   record['wuid_read'] = wurec['wuid']
+   record['date_read'] = logrec['date']
+   record['time_read'] = logrec['time']
+   record['size_read'] = logrec['size']
+   record['method'] = logrec['method']
+   record['locflag'] = logrec['locflag']
+   record['ip'] = logrec['ip']
+   record['quarter'] = logrec['quarter']
+
+   if add_web_allusage(year, logrec, wurec):
+      return PgDBI.add_yearly_wusage(year, record)
+   else:
+      return 0
+
+def add_web_allusage(year, logrec, wurec):
+
+   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'], 'country' : wurec['country']}
+   pgrec['dsid'] = logrec['dsid']
+   pgrec['date'] = logrec['date']
+   pgrec['quarter'] = logrec['quarter']
+   pgrec['time'] = logrec['time']
+   pgrec['size'] = logrec['size']
+   pgrec['method'] = logrec['method']
+   pgrec['ip'] = logrec['ip']
+   pgrec['source'] = 'C'
+   return PgDBI.add_yearly_allusage(year, pgrec)
+
+#
+# return wfile.wid upon success, 0 otherwise
+#
+def get_wfile_wid(dsid, wfile):
+
+   wfcond = "wfile = '{}'".format(wfile) 
+   pgrec = PgSplit.pgget_wfile(dsid, "*", wfcond)
+   if pgrec:
+      pgrec['dsid'] = dsid
+   else:
+      pgrec = PgDBI.pgget("wfile_delete", "*", "{} AND dsid = '{}'".format(wfcond, dsid))
+      if not pgrec:
+         pgrec = PgDBI.pgget("wmove", "wid, dsid", wfcond)
+         if pgrec:
+            pgrec = PgSplit.pgget_wfile(pgrec['dsid'], "*", "wid = {}".format(pgrec['wid']))
+            if pgrec: pgrec['dsid'] = dsid
+
+   return pgrec
+
+# return wuser record upon success, None otherwise
+def get_wuser_record(ip, date):
+
+   ipinfo = PgIPInfo.set_ipinfo(ip)
+   if not ipinfo: return None
+
+   record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
+   email = 'unknown@' + ipinfo['hostname']
+   emcond = "email = '{}'".format(email)
+   flds = 'wuid, email, org_type, country, start_date'   
+   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
+   if pgrec:
+      if PgUtil.diffdate(pgrec['start_date'], date) > 0:
+         pgrec['start_date'] = record['start_date'] = date
+         PgDBI.pgupdt('wuser', record, emcond)
+      return pgrec
+
+   # now add one in
+   record['email'] = email
+   record['stat_flag'] = 'A'
+   record['start_date'] = date
+   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
+   if wuid:
+      record['wuid'] = wuid
+      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
+      return record
+
+   return None
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics-1.0.9/src/rda_python_metrics/fillcdgusage.usg

@@ -0,0 +1,18 @@
+
+ Retrieves CDG usage information from GDEX metrics database to
+   fill table 'tdsusage' and 'webusage' in PostgreSQL database 'rdadb'.
+
+ Usage: fillcdgusage [-b] -s DatasetShortNames [-m MonthList] [-N NumberDays] [-y YearList]
+
+      select option -s and one of the options, -m, -N or -y each time to run
+        this application. 
+
+      - Option -b, log process information into logfile only;
+
+      - Option -s, retrieve usage info for given dataset short names; 
+
+      - Option -m, retrieve usage info in given months (YYYY-MM); 
+
+      - Option -N, retrieve usage info in recent Number of days;
+
+      - Option -y, retrieve usage info in given years (YYYY). 

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/src/rda_python_metrics/fillcodusage.usg

@@ -1,6 +1,6 @@
 
  Retrieves usage information from RDA Web logs under /var/log to
-   fill table 'codusage' in MySQL database 'dssdb'.
+   fill table 'codusage' in database 'rdadb'.
 
  Usage: fillcodusage [-a] [-b] [-f LogFileNames] [-m MonthList] [-N NumberDay] [-y YearList]
 

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/src/rda_python_metrics/fillcountry.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python3/
 #
 ###############################################################################
 #
@@ -17,7 +17,6 @@
 import sys
 import re
 from rda_python_common import PgLOG
-from rda_python_common import PgIMMA
 from rda_python_common import PgUtil
 from rda_python_common import PgDBI
 
@@ -29,7 +28,7 @@ def main():
    argv = sys.argv[1:]
    tables = ['allusage', 'user', 'wuser']
    table = None
-   
+
    # check command line
    for arg in argv:
       if arg == "-b":
@@ -40,7 +39,7 @@ def main():
          table = arg
       else:
          PgLOG.pglog(arg + ": one table name at a time", PgLOG.LGEREX)
-   
+
    if not table:
       print("Usage: fillcountry TableName\n")
       sys.exit(0)
@@ -51,7 +50,7 @@ def main():
    PgLOG.cmdlog("fillcountry {}".format(' '.join(argv)))
 
    process_countries(table)
-   
+
    sys.exit(0)
 
 def process_countries(table):

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/src/rda_python_metrics/fillglobususage.usg

@@ -1,7 +1,7 @@
 
  Retrieves usage information from GLlobus Server logs under directory
    /gpfs/fs1/collections/rda/work/logs/gridftp/ to fill table 'wusage' in
-   database 'dssdb'.
+   database 'rdadb'.
 
  Usage: fillglobususage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
 

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/src/rda_python_metrics/filloneorder.py

@@ -17,12 +17,11 @@ import sys
 import re
 from rda_python_common import PgLOG
 from rda_python_common import PgDBI
-from rda_python_common import PgIMMA
 from rda_python_common import PgUtil
 
 # -t dsid, -e email, -v request data volume, -i data input volume,
 # -m delivery method, -a amount charged, -p pay method, -d request date, -x close date,
-# -y close time, -c file count, -s specialist login name, -o order id, 
+# -y close time, -c file count, -s specialist login name, -o order id,
 # mandatory options: -t, -e, -v, and -m
 
 #
@@ -63,7 +62,7 @@ def add_one_order(params):
 
    year = None
    record = {}
-   
+
    record['dsid'] = params['t']
    record['wuid_request'] = params['u']
    record['dss_uname'] = params['s']
@@ -100,7 +99,7 @@ def add_to_allusage(record, year, ctime):
       pgrec['method'] = record['method']
       pgrec['source'] = 'O'
       return PgDBI.add_yearly_allusage(year, pgrec)
-      
+
    return 0
 
 #

rda_python_metrics-1.0.9/src/rda_python_metrics/pgsyspath.py

@@ -0,0 +1,36 @@
+#
+##################################################################################
+#
+#     Title: pgsyspath
+#    Author: Zaihua Ji, zji@ucar.edu
+#      Date: 10/21/2020
+#   Purpose: python script to set sys.path properly to include paths for local
+#            modules
+#
+# Work File: $DSSHOME/bin/python/pgsyspath.py*
+#    Github: https://github.com/NCAR/rda-utility-programs.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+
+#
+# intinialize the sys.path to include paths for local modules
+#
+def include_local_paths():
+
+   rpath = '/glade/u/home/rdadata'
+   if re.match('PG', rpath): rpath = os.getenv('DSSHOME', '/glade/u/home/rdadata')
+
+   # add more path to pgpaths list as needed
+   pgpaths = [rpath + '/lib/python',
+              rpath + '/lib/python/site-packages']
+   for pgpath in pgpaths:
+      if pgpath not in sys.path: sys.path.insert(0, pgpath)
+
+#
+# call to include local paths when this module is imported or run independently
+#
+include_local_paths()

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/src/rda_python_metrics/viewordusage.py

@@ -14,11 +14,11 @@
 import os
 import re
 import sys
-import pgsyspath
-import PgLOG
-import PgUtil
-import PgView
-import PgDBI
+from . import pgsyspath
+from . import PgView
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgDBI
 
 VUSG = {
    'SNMS' : "ABCDEFGHIJKLMNOPQRSTVWYZ",          # all available short field names in FLDS
@@ -38,7 +38,7 @@ VUSG = {
 # column 0   - column title showing in usage view
 # column 1   - field name in format as shown in select clauses
 # column 2   - field name shown in where condition query string
-# column 3   - table name that the field belongs to 
+# column 3   - table name that the field belongs to
 # column 4   - output field length, the longer one of data size and comlun title, determine
 #              dynamically if it is 0. Negative values indicate right justification
 # column 5   - precision for floating point value if positive and show total value if not zero
@@ -76,7 +76,7 @@ FLDS = {
 # column 0   - expand ID for group of fields
 # column 1   - field name shown in where condition query string
 # column 2   - field name in format as shown in select clauses
-# column 3   - table name that the field belongs to 
+# column 3   - table name that the field belongs to
 EXPAND = {
 # SHRTNM EXPID     CNDSTR    FIELDNAME       TBLNAM
    'D' : ["TIME",   "dDmy"],
@@ -105,7 +105,7 @@ EXPAND = {
 #   A -- number or records to return
 #   c -- array of specified country codes
 #   C -- a string of short field names for viewing usages
-#   d -- array of specified dates 
+#   d -- array of specified dates
 #   D -- dates range, array of 1 or 2 dates in format of YYYY-MM-DD
 #   e -- array of specified email addresses
 #   E -- use given date or date range for email notice of data update
@@ -116,13 +116,13 @@ EXPAND = {
 #   j -- array of specified projects
 #   l -- array of specified last names
 #   L -- column delimiter for output
-#   m -- array of specified months 
-#   M -- array of specified download methods 
-#   n -- array of specified order numbers 
+#   m -- array of specified months
+#   M -- array of specified download methods
+#   n -- array of specified order numbers
 #   N -- number request range, arrage of 1 or 2 integers
 #   o -- array of specified orginization types
 #   O -- a string of short field names for sorting on
-#   p -- array of specified payment methods 
+#   p -- array of specified payment methods
 #   s -- output data size range, arrage of 1 or 2 sizes in unit of MByte
 #   S -- array of login names of specialists who processed the orders
 #   t -- array of specified dataset names
@@ -130,12 +130,12 @@ EXPAND = {
 #   U -- use given unit for file or data sizes
 #   v -- aray of specified roder numbers
 #   w -- generate view without totals
-#   y -- array of specified years 
+#   y -- array of specified years
 #   z -- generate view including entries without usage
 params = {}
 
 # relationship between parameter options and short field names, A option is not
-# related to a field name if it is not in keys %SNS 
+# related to a field name if it is not in keys %SNS
 SNS = {
    'c' : 'N', 'd' : 'D', 'D' : 'D', 'e' : 'E', 'g' : 'G', 'i' : 'I', 'j' : 'J',
    'l' : 'L', 'm' : 'M', 'M' : 'W', 'n' : 'V', 'N' : 'H', 'o' : 'O', 'p' : 'K',
@@ -194,18 +194,18 @@ def main():
       params[option] = 1
    elif inputs:
       params[option] = inputs   # record input array
-   
+
    if not params:
       PgLOG.show_usage(pgname)
    else:
       check_enough_options()
-   
+
    if 'o' not in params:
       if 'e' not in params:
          params['o'] = ['!', "'DSS'"]   # default to exclude 'DSS' for organization
    elif params['o'][0] == "'ALL'":
       del params['o']
-   
+
    usgtable = "ousage"
    build_query_strings(usgtable)  # build tablenames, fieldnames, and conditions
    records = PgDBI.pgmget(tablenames, fieldnames, condition, PgLOG.UCLWEX)

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/src/rda_python_metrics/viewwebfile.py

@@ -16,9 +16,9 @@
 import os
 import re
 import sys
-import PgLOG
-import PgUtil
-import PgDBI
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgDBI
 from . import PgView
 
 FILE = {
@@ -38,9 +38,9 @@ FILE = {
 # column 0   - column title showing in mss file view
 # column 1   - field name in format as shown in select clauses
 # column 2   - field name shown in where condition query string
-# column 3   - table name that the field belongs to 
+# column 3   - table name that the field belongs to
 # column 4   - output field length, the longer one of data size and comlun title, determine
-#              dynamically if it is 0. Negative values indicate right justification 
+#              dynamically if it is 0. Negative values indicate right justification
 # column 5   - precision for floating point value if positive and show total value if not zero
 # column 6   - field flag to indicate it is a group, distinct or sum field
 FLDS = {
@@ -77,7 +77,7 @@ FLDS = {
 #   a -- 1 to view all usage info available
 #   A -- number or records to return
 #   C -- a string of short field names for viewing usages
-#   d -- array of specified dates of file last written 
+#   d -- array of specified dates of file last written
 #   D -- last written dates range, array of 1 or 2 dates in format of YYYY-MM-DD
 #   e -- array of specified email addresses
 #   f -- array of specified online file names
@@ -90,16 +90,16 @@ FLDS = {
 #   J -- 1 to include group title for GROUP
 #   l -- array of specified last names
 #   L -- column delimiter for output
-#   m -- array of specified months of file last written 
-#   M -- array of specified months of file created 
-#   n -- array of specified user numbers 
+#   m -- array of specified months of file last written
+#   M -- array of specified months of file created
+#   n -- array of specified user numbers
 #   D -- dates range, datasets created between, array of 1 or 2 dates in format of YYYY-MM-DD
 #   o -- array of specified file status
 #   O -- a string of short field names for sorting on
 #   p -- array of web file types, Data, Document, and etc.
 #   r -- group index range, array of 1 or 2 group indices
 #   s -- file size range, arrage of 1 or 2 sizes in unit of MByte
-#   S -- specialist lognames who handle the datasets 
+#   S -- specialist lognames who handle the datasets
 #   t -- array of specified dataset names
 #   T -- dataset range, array of 1 or 2 dataset names
 #   u -- array of specified specialist user names
@@ -113,7 +113,7 @@ FLDS = {
 params = {}
 
 # relationship between parameter options and short field names, A option is not
-# related to a field name if it is not in keys %SNS 
+# related to a field name if it is not in keys %SNS
 SNS = {
    'd' : 'D', 'D' : 'D', 'e' : 'E', 'f' : 'F', 'F' : 'F', 'g' : 'G', 'i' : 'I',
    'l' : 'L', 'm' : 'M', 'M' : 'H', 'N' : 'N', 'o' : 'O', 'p' : 'P', 'r' : 'G',
@@ -174,7 +174,7 @@ def main():
       params[option] = 1
    elif inputs:
       params[option] = inputs   # record input array
-   
+
    if not params:
       PgLOG.show_usage(pgname)
    else:
@@ -196,7 +196,7 @@ def main():
    PgView.simple_output(params, FLDS, records, totals)
 
    PgLOG.pgexit(0)
-   
+
 #
 # cehck if enough information entered on command line for generate view/report, exit if not
 #
@@ -209,7 +209,7 @@ def check_enough_options():
       if sn == 'X': continue  # do not process INDEX field
       if FILE['SNMS'].find(sn) < 0:
          PgLOG.pglog("{}: Field {} must be in field names '{}X'".format(pgname, sn, FILE['SNMS']), PgLOG.LGWNEX)
-   
+
    if 'g' in params or 'G' in params:
       if 't' not in params:
          PgLOG.pglog("Miss dataset condition via Option -t for processing Group", PgLOG.LGWNEX)

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.7
+Version: 1.0.9
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
@@ -13,6 +13,9 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: rda_python_common
 Requires-Dist: rda_python_setuid
+Requires-Dist: geoip2
+Requires-Dist: ipinfo
+Requires-Dist: httplib2
 Dynamic: license-file
 
 RDA Python Package to gather and view data usage metrics.

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/src/rda_python_metrics.egg-info/SOURCES.txt

@@ -7,6 +7,8 @@ src/rda_python_metrics/PgView.py
 src/rda_python_metrics/__init__.py
 src/rda_python_metrics/fillawsusage.py
 src/rda_python_metrics/fillawsusage.usg
+src/rda_python_metrics/fillcdgusage.py
+src/rda_python_metrics/fillcdgusage.usg
 src/rda_python_metrics/fillcodusage.py
 src/rda_python_metrics/fillcodusage.usg
 src/rda_python_metrics/fillcountry.py
@@ -28,6 +30,7 @@ src/rda_python_metrics/filluser.usg
 src/rda_python_metrics/logarch.py
 src/rda_python_metrics/logarch.usg
 src/rda_python_metrics/pgperson.py
+src/rda_python_metrics/pgsyspath.py
 src/rda_python_metrics/pgusername.py
 src/rda_python_metrics/viewallusage.py
 src/rda_python_metrics/viewallusage.usg

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/src/rda_python_metrics.egg-info/entry_points.txt

@@ -1,5 +1,6 @@
 [console_scripts]
 fillawsusage = rda_python_metrics.fillaswusage:main
+fillcdgusage = rda_python_metrics.fillcdgusage:main
 fillcodusage = rda_python_metrics.fillcodusage:main
 fillcountry = rda_python_metrics.fillcountry:main
 fillendtime = rda_python_metrics.fillendtime:main

{rda_python_metrics-1.0.7 → rda_python_metrics-1.0.9}/src/rda_python_metrics.egg-info/requires.txt

@@ -1,2 +1,5 @@
 rda_python_common
 rda_python_setuid
+geoip2
+ipinfo
+httplib2

rda_python_metrics-1.0.9/tests/test_metrics.py

@@ -0,0 +1,31 @@
+# test_metrics.py
+
+import pytest
+
+
+
+def test_metrics():
+    import rda_python_metrics.fillawsusage
+    import rda_python_metrics.fillcodusage
+    import rda_python_metrics.fillcountry
+    import rda_python_metrics.fillendtime
+    import rda_python_metrics.fillglobususage
+    import rda_python_metrics.fillipinfo
+    import rda_python_metrics.filloneorder
+    import rda_python_metrics.fillosdfusage
+    import rda_python_metrics.fillrdadb
+    import rda_python_metrics.filltdsusage
+    import rda_python_metrics.filluser
+    import rda_python_metrics.logarch
+    import rda_python_metrics.PgIPInfo
+    import rda_python_metrics.pgperson
+    import rda_python_metrics.pgusername
+    import rda_python_metrics.PgView
+    import rda_python_metrics.viewallusage
+    import rda_python_metrics.viewcheckusage
+    import rda_python_metrics.viewcodusage
+    import rda_python_metrics.viewordusage
+    import rda_python_metrics.viewrqstusage
+    import rda_python_metrics.viewtdsusage
+    import rda_python_metrics.viewwebfile
+    import rda_python_metrics.viewwebusage

rda_python_metrics-1.0.7/MANIFEST.in

@@ -1,19 +0,0 @@
-include src/rda_python_metrics/fillawsusage.usg
-include src/rda_python_metrics/fillcodusage.usg
-include src/rda_python_metrics/fillglobususage.usg
-include src/rda_python_metrics/fillipinfo.usg
-include src/rda_python_metrics/filloneorder.usg
-include src/rda_python_metrics/fillosdfusage.usg
-include src/rda_python_metrics/fillrdadb.usg
-include src/rda_python_metrics/filltdsusage.usg
-include src/rda_python_metrics/filluser.usg
-include src/rda_python_metrics/logarch.usg
-include src/rda_python_metrics/viewallusage.usg
-include src/rda_python_metrics/viewcheckusage.usg
-include src/rda_python_metrics/viewcodusage.usg
-include src/rda_python_metrics/viewordusage.usg
-include src/rda_python_metrics/viewrqstusage.usg
-include src/rda_python_metrics/viewtdsusage.usg
-include src/rda_python_metrics/viewwebfile.usg
-include src/rda_python_metrics/viewallusage.usg
-include src/rda_python_metrics/viewwebusage.usg

rda_python_metrics-1.0.7/tests/test_metrics.py

@@ -1,6 +0,0 @@
-# test_hello_world.py
-
-import pytest
-
-def test_metrics():
-   pass