rda-python-metrics 1.0.9__py3-none-any.whl → 1.0.11__py3-none-any.whl

rda_python_metrics/fillcdgusage.py

@@ -25,7 +25,7 @@ from . import PgIPInfo
 
 USAGE = {
    'TDSTBL'  : "tdsusage",
-   'WEBTBL'  : "webusage",
+   'WEBTBL'  : "wusage",
    'CDATE' : PgUtil.curdate(),
 }
 
@@ -54,6 +54,10 @@ DSIDS = {
    'ucar.cgd.ccsm4.pliomip2' : ['d651037']
 }
 
+ALLIDS = list(DSIDS.keys())
+
+WFILES = {}
+WUSERS = {}
 
 #
 # main function to run this program
@@ -86,13 +90,9 @@ def main():
       
    
    PgLOG.cmdlog("fillcdgusage {}".format(' '.join(argv)))
+   dranges = get_date_ranges(params)
    dsids = get_dataset_ids(params['s'])
-   if dsids:
-      del params['s']
-      for o in params:
-         dranges = get_date_ranges(o, params[o])
-         fill_cdg_usages(dsids, dranges)
-
+   if dranges and dsids: fill_cdg_usages(dsids, dranges)
    PgLOG.pglog(None, PgLOG.LOGWRN|PgLOG.SNDEML)  # send email out if any
 
    sys.exit(0)
@@ -112,68 +112,78 @@ def get_dataset_ids(dsnames):
    dsids = []
    tbname = 'metadata.dataset'
    for dsname in dsnames:
+      if re.match(r'^all$', dsname, re.I): return get_dataset_ids(ALLIDS)
       if dsname not in DSIDS:
          PgLOG.pglog(dsname + ": Unknown CDG dataset short name", PgLOG.LOGWRN)
          continue
-      rdaid = DSIDS[dsname]
       pgrec = PgDBI.pgget(tbname, 'id', "short_name = '{}'".format(dsname))
       if not (pgrec and pgrec['id']): continue
-      dsid = pgrec['id']
-      if dsid in dsids: continue
-      dsids.append([dsid, rdaid])
-      recursive_dataset_ids(dsid, rdaid, dsids)
+      rdaids = DSIDS[dsname]
+      cdgid = pgrec['id']
+      cdgids = [cdgid]
+      recursive_dataset_ids(cdgid, cdgids)
+      dsids.append([dsname, rdaids, cdgids])
 
    if not dsids: PgLOG.pglog("No Dataset Id identified to gather CDG metrics", PgLOG.LOGWRN)
 
    return dsids
 
 #
-# get dsids recursivley
+# get cdgids recursivley
 #
-def recursive_dataset_ids(pdsid, rdaid, dsids):
+def recursive_dataset_ids(pcdgid, cdgids):
 
    tbname = 'metadata.dataset'
-   pgrecs = PgDBI.pgmget(tbname, 'id', "parent_dataset_id = '{}'".format(pdsid))
+   pgrecs = PgDBI.pgmget(tbname, 'id', "parent_dataset_id = '{}'".format(pcdgid))
    if not pgrecs: return
 
-   for dsid in pgrecs['id']:
-      if dsid in dsids: continue
-      dsids.append([dsid, rdaid])
-      recursive_dataset_ids(dsid, rdaid, dsids)
+   for cdgid in pgrecs['id']:
+      if cdgid in cdgids: continue
+      cdgids.append(cdgid)
+      recursive_dataset_ids(cdgid, cdgids)
 
 #
 # get the date ranges for given condition
 #
-def get_date_ranges(option, inputs):
+def get_date_ranges(inputs):
 
    dranges = []
-   for input in inputs:
-      # get date range
-      dates = []
-      if option == 'N':
-         dates[1] = USAGE['CDATE']
-         dates[0] = PgUtil.adddate(USAGE['CDATE'], 0, 0, -int(input))  
-      elif option == 'm':
-         tms = input.split('-')
-         dates[0] = PgUtil.fmtdate(int(tms[0]), int(tms[1]), 1)
-         dates[1] = PgUtil.enddate(dates[0])
-      else:
-         dates[0] = input + "-01-01"
-         dates[1] = input + "-12-31"
-      dranges.append(dates)
+   for opt in inputs:
+      for input in inputs[opt]:
+         # get date range
+         dates = []
+         if opt == 'N':
+            dates.append(PgUtil.adddate(USAGE['CDATE'], 0, 0, -int(input)))
+            dates.append(USAGE['CDATE'])
+         elif opt == 'm':
+            tms = input.split('-')
+            dates.append(PgUtil.fmtdate(int(tms[0]), int(tms[1]), 1))
+            dates.append(PgUtil.enddate(dates[0], 0, 'M'))
+         elif opt == 'Y':
+            dates.append(input + "-01-01")
+            dates.append(input + "-12-31")
+         dranges.append(dates)
 
    return dranges
 
 #
 # get file download records for given dsid
 #
-def get_dsid_records(dsid, dates):
+def get_dsid_records(cdgids, dates, srdaid):
 
    gdex_dbname()
    tbname = 'metrics.file_download'
    fields = ('date_completed, remote_address, logical_file_size, logical_file_name, file_access_point_uri, user_agent_name, bytes_sent, '
              'subset_file_size, range_request, dataset_file_size, dataset_file_name, dataset_file_file_access_point_uri')
-   cond = "dataset_id = '{}' AND completed = True AND date_completed BETWEEN '{}' AND '{}' ORDER BY date_completed".format(dsid, dates[0], dates[1])
+   dscnt = len(cdgids)
+   dscnd = "dataset_id "
+   if dscnt == 1:
+      dscnd += "= '{}'".format(cdgids[0])
+   else:
+      dscnd += "IN ('" + "','".join(cdgids) + "')"
+   dtcnd = "date_completed BETWEEN '{}' AND '{}'".format(dates[0], dates[1])
+   cond = "{} AND completed = True AND {} ORDER BY date_completed".format(dscnd, dtcnd)
+   PgLOG.pglog("{}: CDG query for {} at {}".format(srdaid, cond, PgLOG.current_datetime()), PgLOG.LOGWRN)
    pgrecs = PgDBI.pgmget(tbname, fields, cond)
    PgDBI.dssdb_dbname()
 
@@ -184,17 +194,20 @@ def get_dsid_records(dsid, dates):
 #
 def fill_cdg_usages(dsids, dranges):
 
-   allcnt = awcnt = atcnt = 0
-   for dsid in dsids:
-      cdgid = dsid[0]
-      rdaid = dsid[1]
-      for dates in dranges:
-         pgrecs = get_dsid_records(cdgid, dates)
+   allcnt = awcnt = atcnt = lcnt = 0
+   for dates in dranges:
+      for dsid in dsids:
+         lcnt += 1
+         dsname = dsid[0]
+         rdaids = dsid[1]
+         cdgids = dsid[2]
+         srdaid = '|'.join(rdaids)
+         pgrecs = get_dsid_records(cdgids, dates, srdaid)
          pgcnt = len(pgrecs['dataset_file_name']) if pgrecs else 0
          if pgcnt == 0:
-            PgLOG.pglog("{}: No record found to gather CDG usage between {} and {}".format(rdaid, dates[0], dates[1]), PgLOG.LOGWRN)
+            PgLOG.pglog("{}: No record found to gather CDG usage between {} and {}".format(srdaid, dates[0], dates[1]), PgLOG.LOGWRN)
             continue
-         PgLOG.pglog("{}: gather {} records for CDG usage between {} and {}".format(rdaid, pgcnt, dates[0], dates[1]), PgLOG.LOGWRN)
+         PgLOG.pglog("{}: Process {} records for CDG usage at {}".format(srdaid, pgcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
          tcnt = wcnt = 0
          pwkey = wrec = cdate = None
          trecs = {}
@@ -210,6 +223,11 @@ def fill_cdg_usages(dsids, dranges):
             if not url: url = pgrec['file_access_point_uri']
             ip = pgrec['remote_address']
             engine = pgrec['user_agent_name']
+            wfile = pgrec['dataset_file_name']
+            if not wfile: wfile = pgrec['logic_file_name']
+            wfrec = get_wfile_record(rdaids, wfile)
+            if not wfrec: continue
+            dsid = wfrec['dsid']
             ms = re.search(r'^https://tds.ucar.edu/thredds/(\w+)/', url)
             if ms:
                # tds usage
@@ -226,22 +244,24 @@ def fill_cdg_usages(dsids, dranges):
                      tcnt += add_tdsusage_records(year, trecs, cdate)
                      trecs = {}
                   cdate = date
-               tkey = "{}:{}:{}:{}".format(ip, rdaid, method, etype)
+               tkey = "{}:{}:{}:{}".format(ip, dsid, method, etype)
                if tkey in trecs:
                   trecs[tkey]['size'] += dsize
                   trecs[tkey]['fcount'] += 1
                else:
-                  trecs[tkey] = {'ip' : ip, 'dsid' : rdaid, 'date' : cdate, 'time' : time, 'size' : dsize,
-                                 'fcount' : 1, 'method' : method, 'etype' : etype, 'engine' : engine}
+                  wurec =  get_wuser_record(ip)
+                  if not wurec: return 0
+                  trecs[tkey] = {'ip' : ip, 'dsid' : dsid, 'date' : cdate, 'time' : time, 'size' : dsize,
+                                 'fcount' : 1, 'method' : method, 'etype' : etype, 'engine' : engine,
+                                 'org_type' : wurec['org_type'], 'country' : wurec['country'],
+                                 'email' : wurec['email']}
             else:
                # web usage
-               wfile = pgrec['dataset_file_name']
-               if not wfile: wfile = pgrec['logic_file_name']
                fsize = pgrec['dataset_file_size']
                if not fsize: fsize = pgrec['logic_file_size']
                method = 'CDP'
                if pgrec['subset_file_size'] or pgrec['range_request'] or dsize < fsize:
-                  wkey = "{}:{}:{}".format(ip, rdaid, wfile)
+                  wkey = "{}:{}:{}".format(ip, dsid, wfile)
                else:
                   wkey = None
       
@@ -250,7 +270,7 @@ def fill_cdg_usages(dsids, dranges):
                      wrec['size'] += dsize
                      continue
                   wcnt += add_webfile_usage(year, wrec)
-               wrec = {'ip' : ip, 'dsid' : rdaid, 'wfile' : wfile, 'date' : date,
+               wrec = {'ip' : ip, 'dsid' : dsid, 'wid' : wfrec['wid'], 'date' : date,
                        'time' : time, 'quarter' : quarter, 'size' : dsize,
                        'locflag' : 'C', 'method' : method}
                pwkey = wkey
@@ -263,9 +283,7 @@ def fill_cdg_usages(dsids, dranges):
          atcnt += tcnt
          awcnt += wcnt
          allcnt += pgcnt
-
-   PgLOG.pglog("{}/{} TDS/WEB usage records added for {} CDG entries at {}".format(atcnt, awcnt, allcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
-
+         PgLOG.pglog("{}/{} TDS/WEB usage records added for {} CDG entries at {}".format(atcnt, awcnt, allcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
 
 def get_record_date_time(ctime):
 
@@ -287,12 +305,6 @@ def add_tdsusage_records(year, records, date):
       record = records[key]
       cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, record['time'], record['ip'])
       if PgDBI.pgget(USAGE['TDSTBL'], '', cond, PgLOG.LGEREX): continue
-      record['org_type'] = record['country'] = '-'
-      ipinfo = PgIPInfo.set_ipinfo(record['ip'])
-      if ipinfo:
-         record['org_type'] = ipinfo['org_type']
-         record['country'] = ipinfo['country']
-         record['email'] = 'unknown@' + ipinfo['hostname']
 
       if add_tds_allusage(year, record):
          cnt += PgDBI.pgadd(USAGE['TDSTBL'], record, PgLOG.LOGWRN)
@@ -301,39 +313,43 @@ def add_tdsusage_records(year, records, date):
 
    return cnt
 
+def add_tds_allusage(year, logrec):
 
-def add_tds_allusage(year, pgrec):
-
-   record = {'method' : 'CDP', 'source' : 'C'}
-
-   for fld in pgrec:
-      if re.match(r'^(engine|method|etype|fcount)$', fld): continue
-      record[fld] = pgrec[fld]
+   pgrec = {'method' : 'CDP', 'source' : 'C'}
+   pgrec['email'] = logrec['email']
+   pgrec['org_type'] = logrec['org_type']
+   pgrec['country'] = logrec['country']
+   pgrec['dsid'] = logrec['dsid']
+   pgrec['date'] = logrec['date']
+   pgrec['quarter'] = logrec['quarter']
+   pgrec['time'] = logrec['time']
+   pgrec['size'] = logrec['size']
+   pgrec['ip'] = logrec['ip']
+   return PgDBI.add_yearly_allusage(year, pgrec)
 
-   return PgDBI.add_yearly_allusage(year, record)
 
 #
 # Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
 #
 def add_webfile_usage(year, logrec):
 
-   pgrec = get_wfile_wid(logrec['dsid'], logrec['wfile'])
-   if not pgrec: return 0
-
    table = "{}_{}".format(USAGE['WEBTBL'], year)
-   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
+   cdate = logrec['date']
+   ip = logrec['ip']
+   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(logrec['wid'], logrec['method'], cdate, logrec['time'])
    if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
 
-   wurec =  get_wuser_record(logrec['ip'], logrec['date'])
+   wurec =  get_wuser_record(ip, cdate, False)
    if not wurec: return 0
-   record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
+
+   record = {'wid' : logrec['wid'], 'dsid' : logrec['dsid']}
    record['wuid_read'] = wurec['wuid']
-   record['date_read'] = logrec['date']
+   record['date_read'] = cdate
    record['time_read'] = logrec['time']
    record['size_read'] = logrec['size']
    record['method'] = logrec['method']
    record['locflag'] = logrec['locflag']
-   record['ip'] = logrec['ip']
+   record['ip'] = ip
    record['quarter'] = logrec['quarter']
 
    if add_web_allusage(year, logrec, wurec):
@@ -343,7 +359,10 @@ def add_webfile_usage(year, logrec):
 
 def add_web_allusage(year, logrec, wurec):
 
-   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'], 'country' : wurec['country']}
+   pgrec = {'source' : 'C'}
+   pgrec['email'] = wurec['email']
+   pgrec['org_type'] = wurec['org_type']
+   pgrec['country'] = wurec['country']
    pgrec['dsid'] = logrec['dsid']
    pgrec['date'] = logrec['date']
    pgrec['quarter'] = logrec['quarter']
@@ -351,36 +370,54 @@ def add_web_allusage(year, logrec, wurec):
    pgrec['size'] = logrec['size']
    pgrec['method'] = logrec['method']
    pgrec['ip'] = logrec['ip']
-   pgrec['source'] = 'C'
    return PgDBI.add_yearly_allusage(year, pgrec)
 
 #
 # return wfile.wid upon success, 0 otherwise
 #
-def get_wfile_wid(dsid, wfile):
+def get_wfile_record(dsids, wfile):
 
-   wfcond = "wfile = '{}'".format(wfile) 
-   pgrec = PgSplit.pgget_wfile(dsid, "*", wfcond)
-   if pgrec:
-      pgrec['dsid'] = dsid
-   else:
-      pgrec = PgDBI.pgget("wfile_delete", "*", "{} AND dsid = '{}'".format(wfcond, dsid))
-      if not pgrec:
-         pgrec = PgDBI.pgget("wmove", "wid, dsid", wfcond)
-         if pgrec:
-            pgrec = PgSplit.pgget_wfile(pgrec['dsid'], "*", "wid = {}".format(pgrec['wid']))
-            if pgrec: pgrec['dsid'] = dsid
+   for dsid in dsids:
+      wkey = "{}{}".format(dsid, wfile)
+      if wkey in WFILES: return WFILES[wkey]
+   wfcond = "wfile LIKE '%{}'".format(wfile)
+   pgrec = None
+   for dsid in dsids:
+      pgrec = PgSplit.pgget_wfile(dsid, "wid", wfcond)
+      if pgrec:
+         pgrec['dsid'] = dsid
+         wkey = "{}{}".format(dsid, wfile)
+         WFILES[wkey] = pgrec
+         return pgrec
 
+   for dsid in dsids:
+      pgrec = PgDBI.pgget("wfile_delete", "wid, dsid", "{} AND dsid = '{}'".format(wfcond, dsid))
+      if not pgrec:
+         mvrec = PgDBI.pgget("wmove", "wid, dsid", wfcond)
+         if mvrec:
+            pgrec = PgSplit.pgget_wfile(mvrec['dsid'], "wid", "wid = {}".format(pgrec['wid']))
+            if pgrec: pgrec['dsid'] = mvrec['dsid']
+
+   if pgrec: 
+      wkey = "{}{}".format(pgrec['dsid'], wfile)
+      WFILES[wkey] = pgrec
    return pgrec
 
 # return wuser record upon success, None otherwise
-def get_wuser_record(ip, date):
+def get_wuser_record(ip, date = None, skipwuid = True):
+
+   if ip in WUSERS: return WUSERS[ip]
 
    ipinfo = PgIPInfo.set_ipinfo(ip)
    if not ipinfo: return None
 
    record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
    email = 'unknown@' + ipinfo['hostname']
+   if skipwuid:
+      record['email'] = email
+      WUSERS[ip] = record
+      return record
+
    emcond = "email = '{}'".format(email)
    flds = 'wuid, email, org_type, country, start_date'   
    pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
@@ -388,6 +425,7 @@ def get_wuser_record(ip, date):
       if PgUtil.diffdate(pgrec['start_date'], date) > 0:
          pgrec['start_date'] = record['start_date'] = date
          PgDBI.pgupdt('wuser', record, emcond)
+      WUSERS[ip] = pgrec
       return pgrec
 
    # now add one in
@@ -398,6 +436,7 @@ def get_wuser_record(ip, date):
    if wuid:
       record['wuid'] = wuid
       PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
+      WUSERS[ip] = record
       return record
 
    return None

{rda_python_metrics-1.0.9.dist-info → rda_python_metrics-1.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.9
+Version: 1.0.11
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.9.dist-info → rda_python_metrics-1.0.11.dist-info}/RECORD

@@ -3,7 +3,7 @@ rda_python_metrics/PgView.py,sha256=r6otb3DjfiaQJdg0z8bZQAOlhr4JnrXJzp9wgWh_8qQ,
 rda_python_metrics/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 rda_python_metrics/fillawsusage.py,sha256=dpI3-EFINJECdCSUOm37A97CJNIxOS2nYwA0fB0IpHE,9055
 rda_python_metrics/fillawsusage.usg,sha256=pD_nYTfg1so9nvVEyPSWdgKvb9gWdtfHJAs3RsT2MMU,609
-rda_python_metrics/fillcdgusage.py,sha256=9sDFLAhzndjAVCahIVHofQ4YeslwW7MGIWEDSAQYdqY,13816
+rda_python_metrics/fillcdgusage.py,sha256=D4PhdVyGoISSr0ykNIpmHVMpEAfRZQB9BvaXdYMcGno,15186
 rda_python_metrics/fillcdgusage.usg,sha256=5lkd4Zdi72nQcha-JtbaLnxl66V4wBDykKwQtUOtMrw,667
 rda_python_metrics/fillcodusage.py,sha256=Hp3VrlIqHBWRI6Zjbi0nxWZPNvPrKsGBSZ1L9qA9_y0,8006
 rda_python_metrics/fillcodusage.usg,sha256=3B5IkQ4uwylqY8uEfUdnZ_MXqhYudeylMp5ulhUGXH8,678
@@ -44,9 +44,9 @@ rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmj
 rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
 rda_python_metrics/viewwebusage.py,sha256=jhoHuRPVNtp7Lbjv0l-Jy_vp2p2nWQC7IVZ0P4JUJ4I,16657
 rda_python_metrics/viewwebusage.usg,sha256=IsT72v22xyZf7ng_IodVs0dLTsH1Q4BtFvT-gs0-xJY,9946
-rda_python_metrics-1.0.9.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_metrics-1.0.9.dist-info/METADATA,sha256=k5xThxsVVIf1-uVtr5NDQ0VdUPKjZxeNlgSNysXLT24,735
-rda_python_metrics-1.0.9.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rda_python_metrics-1.0.9.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
-rda_python_metrics-1.0.9.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
-rda_python_metrics-1.0.9.dist-info/RECORD,,
+rda_python_metrics-1.0.11.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_metrics-1.0.11.dist-info/METADATA,sha256=2D84_3Vy1MxxzM--KjIjJmFitL4QJ0JwkuigW0CErik,736
+rda_python_metrics-1.0.11.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rda_python_metrics-1.0.11.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
+rda_python_metrics-1.0.11.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
+rda_python_metrics-1.0.11.dist-info/RECORD,,