rda-python-metrics 1.0.31__tar.gz → 1.0.33__tar.gz

{rda_python_metrics-1.0.31/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.31
+Version: 1.0.33
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.31 → rda_python_metrics-1.0.33}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.31"
+version = "1.0.33"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]
@@ -38,12 +38,13 @@ pythonpath = [
 
 [project.scripts]
 "logarch.py" = "rda_python_metrics.logarch:main"
-"fillawsusage" = "rda_python_metrics.fillaswusage:main"
+"fillawsusage" = "rda_python_metrics.fillawsusage:main"
 "fillcdgusage" = "rda_python_metrics.fillcdgusage:main"
 "fillcodusage" = "rda_python_metrics.fillcodusage:main"
 "fillcountry" = "rda_python_metrics.fillcountry:main"
 "fillendtime" = "rda_python_metrics.fillendtime:main"
 "fillglobususage" = "rda_python_metrics.fillglobususage:main"
+"fillgdexusage" = "rda_python_metrics.fillgdexusage:main"
 "fillipinfo" = "rda_python_metrics.fillipinfo:main"
 "filloneorder" = "rda_python_metrics.filloneorder:main"
 "fillosdfusage" = "rda_python_metrics.fillosdfusage:main"

{rda_python_metrics-1.0.31 → rda_python_metrics-1.0.33}/src/rda_python_metrics/fillcdgusage.py

@@ -239,12 +239,14 @@ def fill_cdg_usages(dsids, dranges):
 
    allcnt = awcnt = atcnt = lcnt = 0
    for dates in dranges:
-      for dsid in dsids:
+      for adsid in dsids:
          lcnt += 1
-         dsname = dsid[0]
-         rdaids = dsid[1]
-         cdgids = dsid[2]
-         strids = dsid[3]
+         dsname = adsid[0]
+         rdaids = adsid[1]
+         getdsid = False if len(rdaids) == 1 else True
+         dsid = rdaids[0]
+         cdgids = adsid[2]
+         strids = adsid[3]
          bt = tm()
          pgrecs = get_dsid_records(cdgids, dates, strids)
          pgcnt = len(pgrecs['dataset_file_name']) if pgrecs else 0
@@ -262,6 +264,10 @@ def fill_cdg_usages(dsids, dranges):
                PgLOG.pglog("{}/{}/{} CDG/TDS/WEB records processed to add".format(i, tcnt, wcnt), PgLOG.WARNLG)
 
             pgrec = PgUtil.onerecord(pgrecs, i)
+            wfile = pgrec['dataset_file_name']
+            if not wfile:
+               wfile = pgrec['logic_file_name']
+               if not wfile: continue
             dsize = pgrec['bytes_sent']
             if not dsize: continue
             (year, quarter, date, time) = get_record_date_time(pgrec['date_completed'])
@@ -269,14 +275,13 @@ def fill_cdg_usages(dsids, dranges):
             if not url: url = pgrec['file_access_point_uri']
             ip = pgrec['remote_address']
             engine = pgrec['user_agent_name']
-            wfile = pgrec['dataset_file_name']
-            if not wfile: wfile = pgrec['logic_file_name']
-            wfrec = get_wfile_record(rdaids, wfile)
-            if not wfrec: continue
-            dsid = wfrec['dsid']
             ms = re.search(r'^https*://tds.ucar.edu/thredds/(\w+)/', url)
             if ms:
                # tds usage
+               if getdsid:
+                  wfrec = get_wfile_record(rdaids, wfile)
+                  if not wfrec: continue
+                  dsid = wfrec['dsid']
                method = ms.group(1)
                if pgrec['subset_file_size']:
                   etype = 'S'
@@ -303,6 +308,9 @@ def fill_cdg_usages(dsids, dranges):
                                  'region' : iprec['region'], 'email' : iprec['email']}
             else:
                # web usage
+               wfrec = get_wfile_record(rdaids, wfile)
+               if not wfrec: continue
+               if getdsid: dsid = wfrec['dsid']
                fsize = pgrec['dataset_file_size']
                if not fsize: fsize = pgrec['logic_file_size']
                method = 'CDG'

{rda_python_metrics-1.0.31 → rda_python_metrics-1.0.33}/src/rda_python_metrics/fillgdexusage.py

@@ -846,8 +846,8 @@ def get_dsid_records(gdexids, dates, strids):
 
    gdex_dbname()
    tbname = 'metrics.file_download'
-   fields = ('date_completed, remote_address, logical_file_size, logical_file_name, file_access_point_uri, user_agent_name, bytes_sent, '
-             'subset_file_size, range_request, dataset_file_size, dataset_file_name, dataset_file_file_access_point_uri')
+   fields = ('date_completed, remote_address, logical_file_size, logical_file_name, user_agent_name, bytes_sent, '
+             'subset_file_size, range_request, dataset_file_size, dataset_file_name')
    dscnt = len(gdexids)
    dscnd = "dataset_id "
    if dscnt == 1:
@@ -867,7 +867,7 @@ def get_dsid_records(gdexids, dates, strids):
 #
 def fill_gdex_usages(dsids, dranges):
 
-   allcnt = awcnt = atcnt = lcnt = 0
+   allcnt = awcnt = lcnt = 0
    for dates in dranges:
       for dsid in dsids:
          lcnt += 1
@@ -883,20 +883,17 @@ def fill_gdex_usages(dsids, dranges):
             continue
          rmsg = PgLOG.seconds_to_string_time(tm() - bt)
          PgLOG.pglog("{}: Got {} records in {} for processing GDEX usage at {}".format(strids, pgcnt, rmsg, PgLOG.current_datetime()), PgLOG.LOGWRN)
-         tcnt = wcnt = 0
+         wcnt = 0
          pwkey = wrec = cdate = None
-         trecs = {}
          bt = tm()
          for i in range(pgcnt):
             if (i+1)%20000 == 0:
-               PgLOG.pglog("{}/{}/{} GDEX/TDS/WEB records processed to add".format(i, tcnt, wcnt), PgLOG.WARNLG)
+               PgLOG.pglog("{}/{} GDEX/WEB records processed to add".format(i, wcnt), PgLOG.WARNLG)
 
             pgrec = PgUtil.onerecord(pgrecs, i)
             dsize = pgrec['bytes_sent']
             if not dsize: continue
             (year, quarter, date, time) = get_record_date_time(pgrec['date_completed'])
-            url = pgrec['dataset_file_file_access_point_uri']
-            if not url: url = pgrec['file_access_point_uri']
             ip = pgrec['remote_address']
             engine = pgrec['user_agent_name']
             wfile = pgrec['dataset_file_name']
@@ -926,11 +923,10 @@ def fill_gdex_usages(dsids, dranges):
                wrec = None
 
          if wrec: wcnt += add_webfile_usage(year, wrec)
-         atcnt += tcnt
          awcnt += wcnt
          allcnt += pgcnt
          rmsg = PgLOG.seconds_to_string_time(tm() - bt)
-         PgLOG.pglog("{}: {}/{} TDS/WEB usage records added for {} GDEX entries in {}".format(strids, atcnt, awcnt, allcnt, rmsg), PgLOG.LOGWRN)
+         PgLOG.pglog("{}: {} WEB usage records added for {} GDEX entries in {}".format(strids, awcnt, allcnt, rmsg), PgLOG.LOGWRN)
 
 def get_record_date_time(ctime):
 

{rda_python_metrics-1.0.31 → rda_python_metrics-1.0.33}/src/rda_python_metrics/fillosdfusage.py

@@ -129,11 +129,12 @@ def fill_osdf_usages(fnames):
             PgLOG.pglog("{}: {}/{} OSDF log entries processed/records added".format(logfile, entcnt, cntadd), PgLOG.WARNLG)
 
          ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/rda\/([a-z]\d{6})\/(\S+)\].* \[Host:(\S+)\].* \[AppInfo:(\S+)\].* \[Read:(\d+)\]', line)
-         if not ms or ms.group(4) == 'N/A': continue
+         if not ms: continue
          dt = ms.group(1)
          dsid = ms.group(2)
          wfile = ms.group(3)
          ip = ms.group(4)
+         if ip == 'N/A': ip = '0.0.0.0'
          engine = ms.group(5)
          size = int(ms.group(6))
          (year, quarter, date, time) = get_record_date_time(dt)

{rda_python_metrics-1.0.31 → rda_python_metrics-1.0.33/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.31
+Version: 1.0.33
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.31 → rda_python_metrics-1.0.33}/src/rda_python_metrics.egg-info/entry_points.txt

@@ -1,9 +1,10 @@
 [console_scripts]
-fillawsusage = rda_python_metrics.fillaswusage:main
+fillawsusage = rda_python_metrics.fillawsusage:main
 fillcdgusage = rda_python_metrics.fillcdgusage:main
 fillcodusage = rda_python_metrics.fillcodusage:main
 fillcountry = rda_python_metrics.fillcountry:main
 fillendtime = rda_python_metrics.fillendtime:main
+fillgdexusage = rda_python_metrics.fillgdexusage:main
 fillglobususage = rda_python_metrics.fillglobususage:main
 fillipinfo = rda_python_metrics.fillipinfo:main
 filloneorder = rda_python_metrics.filloneorder:main