rda-python-metrics 1.0.11__py3-none-any.whl → 1.0.14__py3-none-any.whl

rda_python_metrics/fillcdgusage.py

@@ -16,6 +16,7 @@ import sys
 import re
 import glob
 from os import path as op
+from time import time as tm
 from rda_python_common import PgLOG
 from rda_python_common import PgUtil
 from rda_python_common import PgFile
@@ -116,13 +117,18 @@ def get_dataset_ids(dsnames):
       if dsname not in DSIDS:
          PgLOG.pglog(dsname + ": Unknown CDG dataset short name", PgLOG.LOGWRN)
          continue
+      bt = tm()
       pgrec = PgDBI.pgget(tbname, 'id', "short_name = '{}'".format(dsname))
       if not (pgrec and pgrec['id']): continue
       rdaids = DSIDS[dsname]
+      strids = "{}-{}".format(dsname, rdaids)
       cdgid = pgrec['id']
       cdgids = [cdgid]
-      recursive_dataset_ids(cdgid, cdgids)
-      dsids.append([dsname, rdaids, cdgids])
+      ccnt = 1
+      ccnt += recursive_dataset_ids(cdgid, cdgids)
+      dsids.append([dsname, rdaids, cdgids, strids])
+      rmsg = PgLOG.seconds_to_string_time(tm() - bt)
+      PgLOG.pglog("{}: Found {} CDG dsid/subdsids in {} at {}".format(strids, ccnt, rmsg, PgLOG.current_datetime()), PgLOG.LOGWRN)
 
    if not dsids: PgLOG.pglog("No Dataset Id identified to gather CDG metrics", PgLOG.LOGWRN)
 
@@ -135,12 +141,16 @@ def recursive_dataset_ids(pcdgid, cdgids):
 
    tbname = 'metadata.dataset'
    pgrecs = PgDBI.pgmget(tbname, 'id', "parent_dataset_id = '{}'".format(pcdgid))
-   if not pgrecs: return
+   if not pgrecs: return 0
 
+   ccnt = 0
    for cdgid in pgrecs['id']:
       if cdgid in cdgids: continue
       cdgids.append(cdgid)
-      recursive_dataset_ids(cdgid, cdgids)
+      ccnt += 1
+      ccnt += recursive_dataset_ids(cdgid, cdgids)
+
+   return ccnt
 
 #
 # get the date ranges for given condition
@@ -159,17 +169,17 @@ def get_date_ranges(inputs):
             tms = input.split('-')
             dates.append(PgUtil.fmtdate(int(tms[0]), int(tms[1]), 1))
             dates.append(PgUtil.enddate(dates[0], 0, 'M'))
-         elif opt == 'Y':
+         elif opt == 'y':
             dates.append(input + "-01-01")
             dates.append(input + "-12-31")
-         dranges.append(dates)
+         if dates: dranges.append(dates)
 
    return dranges
 
 #
 # get file download records for given dsid
 #
-def get_dsid_records(cdgids, dates, srdaid):
+def get_dsid_records(cdgids, dates, strids):
 
    gdex_dbname()
    tbname = 'metrics.file_download'
@@ -181,9 +191,9 @@ def get_dsid_records(cdgids, dates, srdaid):
       dscnd += "= '{}'".format(cdgids[0])
    else:
       dscnd += "IN ('" + "','".join(cdgids) + "')"
-   dtcnd = "date_completed BETWEEN '{}' AND '{}'".format(dates[0], dates[1])
+   dtcnd = "date_completed BETWEEN '{} 00:00:00' AND '{} 23:59:59'".format(dates[0], dates[1])
    cond = "{} AND completed = True AND {} ORDER BY date_completed".format(dscnd, dtcnd)
-   PgLOG.pglog("{}: CDG query for {} at {}".format(srdaid, cond, PgLOG.current_datetime()), PgLOG.LOGWRN)
+   PgLOG.pglog("{}: Query for {} CDG dsid/subdsids Completed between {} and {} at {}".format(strids, dscnt, dates[0], dates[1], PgLOG.current_datetime()), PgLOG.LOGWRN)
    pgrecs = PgDBI.pgmget(tbname, fields, cond)
    PgDBI.dssdb_dbname()
 
@@ -201,21 +211,24 @@ def fill_cdg_usages(dsids, dranges):
          dsname = dsid[0]
          rdaids = dsid[1]
          cdgids = dsid[2]
-         srdaid = '|'.join(rdaids)
-         pgrecs = get_dsid_records(cdgids, dates, srdaid)
+         strids = dsid[3]
+         bt = tm()
+         pgrecs = get_dsid_records(cdgids, dates, strids)
          pgcnt = len(pgrecs['dataset_file_name']) if pgrecs else 0
          if pgcnt == 0:
-            PgLOG.pglog("{}: No record found to gather CDG usage between {} and {}".format(srdaid, dates[0], dates[1]), PgLOG.LOGWRN)
+            PgLOG.pglog("{}: No record found to gather CDG usage between {} and {}".format(strids, dates[0], dates[1]), PgLOG.LOGWRN)
             continue
-         PgLOG.pglog("{}: Process {} records for CDG usage at {}".format(srdaid, pgcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+         rmsg = PgLOG.seconds_to_string_time(tm() - bt)
+         PgLOG.pglog("{}: Got {} records in {} for processing CDG usage at {}".format(strids, pgcnt, rmsg, PgLOG.current_datetime()), PgLOG.LOGWRN)
          tcnt = wcnt = 0
          pwkey = wrec = cdate = None
          trecs = {}
+         bt = tm()
          for i in range(pgcnt):
             if (i+1)%20000 == 0:
                PgLOG.pglog("{}/{}/{} CDG/TDS/WEB records processed to add".format(i, tcnt, wcnt), PgLOG.WARNLG)
 
-            pgrec = PgUtil.onerecord(i, pgrecs)
+            pgrec = PgUtil.onerecord(pgrecs, i)
             dsize = pgrec['bytes_sent']
             if not dsize: continue
             (year, quarter, date, time) = get_record_date_time(pgrec['date_completed'])
@@ -283,20 +296,21 @@ def fill_cdg_usages(dsids, dranges):
          atcnt += tcnt
          awcnt += wcnt
          allcnt += pgcnt
-         PgLOG.pglog("{}/{} TDS/WEB usage records added for {} CDG entries at {}".format(atcnt, awcnt, allcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+         rmsg = PgLOG.seconds_to_string_time(tm() - bt)
+         PgLOG.pglog("{}: {}/{} TDS/WEB usage records added for {} CDG entries in {}".format(strids, atcnt, awcnt, allcnt, rmsg), PgLOG.LOGWRN)
 
 def get_record_date_time(ctime):
 
-   ms = re.search(r'^(\d+)/(\w+)/(\d+) (\d+:\d+:\d+)(\.|$)', str(ctime))
+   ms = re.search(r'^(\d+)-(\d+)-(\d+) (\d\d:\d\d:\d\d)', str(ctime))
    if ms:
-      d = int(ms.group(1))
-      m = PgUtil.get_month(ms.group(2))
+      y = ms.group(1)
+      m = int(ms.group(2))
+      d = ms.group(3)
       q = 1 + int((m-1)/3)
-      y = ms.group(3)
       t = ms.group(4)
-      return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
+      return (y, q, "{}-{:02}-{}".format(y, m, d), t)
    else:
-      PgLOG.pglog("time: Invalid date format", PgLOG.LGEREX)
+      PgLOG.pglog(str(ctime) + ": Invalid time format", PgLOG.LGEREX)
 
 def add_tdsusage_records(year, records, date):
 

{rda_python_metrics-1.0.11.dist-info → rda_python_metrics-1.0.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.11
+Version: 1.0.14
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.11.dist-info → rda_python_metrics-1.0.14.dist-info}/RECORD

@@ -3,7 +3,7 @@ rda_python_metrics/PgView.py,sha256=r6otb3DjfiaQJdg0z8bZQAOlhr4JnrXJzp9wgWh_8qQ,
 rda_python_metrics/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 rda_python_metrics/fillawsusage.py,sha256=dpI3-EFINJECdCSUOm37A97CJNIxOS2nYwA0fB0IpHE,9055
 rda_python_metrics/fillawsusage.usg,sha256=pD_nYTfg1so9nvVEyPSWdgKvb9gWdtfHJAs3RsT2MMU,609
-rda_python_metrics/fillcdgusage.py,sha256=D4PhdVyGoISSr0ykNIpmHVMpEAfRZQB9BvaXdYMcGno,15186
+rda_python_metrics/fillcdgusage.py,sha256=jMAE7qFreYKqVw5XFXdGeBiLGGbdkmVQopPffuLQueg,15773
 rda_python_metrics/fillcdgusage.usg,sha256=5lkd4Zdi72nQcha-JtbaLnxl66V4wBDykKwQtUOtMrw,667
 rda_python_metrics/fillcodusage.py,sha256=Hp3VrlIqHBWRI6Zjbi0nxWZPNvPrKsGBSZ1L9qA9_y0,8006
 rda_python_metrics/fillcodusage.usg,sha256=3B5IkQ4uwylqY8uEfUdnZ_MXqhYudeylMp5ulhUGXH8,678
@@ -44,9 +44,9 @@ rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmj
 rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
 rda_python_metrics/viewwebusage.py,sha256=jhoHuRPVNtp7Lbjv0l-Jy_vp2p2nWQC7IVZ0P4JUJ4I,16657
 rda_python_metrics/viewwebusage.usg,sha256=IsT72v22xyZf7ng_IodVs0dLTsH1Q4BtFvT-gs0-xJY,9946
-rda_python_metrics-1.0.11.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_metrics-1.0.11.dist-info/METADATA,sha256=2D84_3Vy1MxxzM--KjIjJmFitL4QJ0JwkuigW0CErik,736
-rda_python_metrics-1.0.11.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rda_python_metrics-1.0.11.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
-rda_python_metrics-1.0.11.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
-rda_python_metrics-1.0.11.dist-info/RECORD,,
+rda_python_metrics-1.0.14.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_metrics-1.0.14.dist-info/METADATA,sha256=tsmYaJjL-cWMvnpC8_WI-4IndMKtidtW9695gUlprBM,736
+rda_python_metrics-1.0.14.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
+rda_python_metrics-1.0.14.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
+rda_python_metrics-1.0.14.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
+rda_python_metrics-1.0.14.dist-info/RECORD,,

{rda_python_metrics-1.0.11.dist-info → rda_python_metrics-1.0.14.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (78.1.1)
 Root-Is-Purelib: true
 Tag: py3-none-any