rda-python-metrics 1.0.10__py3-none-any.whl → 1.0.11__py3-none-any.whl

rda_python_metrics/fillcdgusage.py

@@ -25,7 +25,7 @@ from . import PgIPInfo
 
 USAGE = {
    'TDSTBL'  : "tdsusage",
-   'WEBTBL'  : "webusage",
+   'WEBTBL'  : "wusage",
    'CDATE' : PgUtil.curdate(),
 }
 
@@ -112,35 +112,35 @@ def get_dataset_ids(dsnames):
    dsids = []
    tbname = 'metadata.dataset'
    for dsname in dsnames:
-      if re.match(r'^all$', dsname, re.I): return ALLIDS
+      if re.match(r'^all$', dsname, re.I): return get_dataset_ids(ALLIDS)
       if dsname not in DSIDS:
          PgLOG.pglog(dsname + ": Unknown CDG dataset short name", PgLOG.LOGWRN)
          continue
-      rdaid = DSIDS[dsname]
       pgrec = PgDBI.pgget(tbname, 'id', "short_name = '{}'".format(dsname))
       if not (pgrec and pgrec['id']): continue
-      dsid = pgrec['id']
-      if dsid in dsids: continue
-      dsids.append([dsid, rdaid])
-      recursive_dataset_ids(dsid, rdaid, dsids)
+      rdaids = DSIDS[dsname]
+      cdgid = pgrec['id']
+      cdgids = [cdgid]
+      recursive_dataset_ids(cdgid, cdgids)
+      dsids.append([dsname, rdaids, cdgids])
 
    if not dsids: PgLOG.pglog("No Dataset Id identified to gather CDG metrics", PgLOG.LOGWRN)
 
    return dsids
 
 #
-# get dsids recursivley
+# get cdgids recursivley
 #
-def recursive_dataset_ids(pdsid, rdaid, dsids):
+def recursive_dataset_ids(pcdgid, cdgids):
 
    tbname = 'metadata.dataset'
-   pgrecs = PgDBI.pgmget(tbname, 'id', "parent_dataset_id = '{}'".format(pdsid))
+   pgrecs = PgDBI.pgmget(tbname, 'id', "parent_dataset_id = '{}'".format(pcdgid))
    if not pgrecs: return
 
-   for dsid in pgrecs['id']:
-      if dsid in dsids: continue
-      dsids.append([dsid, rdaid])
-      recursive_dataset_ids(dsid, rdaid, dsids)
+   for cdgid in pgrecs['id']:
+      if cdgid in cdgids: continue
+      cdgids.append(cdgid)
+      recursive_dataset_ids(cdgid, cdgids)
 
 #
 # get the date ranges for given condition
@@ -169,14 +169,21 @@ def get_date_ranges(inputs):
 #
 # get file download records for given dsid
 #
-def get_dsid_records(dsid, dates):
+def get_dsid_records(cdgids, dates, srdaid):
 
    gdex_dbname()
    tbname = 'metrics.file_download'
    fields = ('date_completed, remote_address, logical_file_size, logical_file_name, file_access_point_uri, user_agent_name, bytes_sent, '
              'subset_file_size, range_request, dataset_file_size, dataset_file_name, dataset_file_file_access_point_uri')
-   cond = "dataset_id = '{}' AND completed = True AND date_completed BETWEEN '{}' AND '{}' ORDER BY date_completed".format(dsid, dates[0], dates[1])
-   PgLOG.pglog("{}: Query CDG usage between {} and {} at {}".format(dsid, dates[0], dates[1], PgLOG.current_datetime()), PgLOG.LOGWRN)
+   dscnt = len(cdgids)
+   dscnd = "dataset_id "
+   if dscnt == 1:
+      dscnd += "= '{}'".format(cdgids[0])
+   else:
+      dscnd += "IN ('" + "','".join(cdgids) + "')"
+   dtcnd = "date_completed BETWEEN '{}' AND '{}'".format(dates[0], dates[1])
+   cond = "{} AND completed = True AND {} ORDER BY date_completed".format(dscnd, dtcnd)
+   PgLOG.pglog("{}: CDG query for {} at {}".format(srdaid, cond, PgLOG.current_datetime()), PgLOG.LOGWRN)
    pgrecs = PgDBI.pgmget(tbname, fields, cond)
    PgDBI.dssdb_dbname()
 
@@ -191,10 +198,11 @@ def fill_cdg_usages(dsids, dranges):
    for dates in dranges:
       for dsid in dsids:
          lcnt += 1
-         cdgid = dsid[0]
-         rdaid = dsid[1]
-         srdaid = '|'.join(rdaid)
-         pgrecs = get_dsid_records(cdgid, dates)
+         dsname = dsid[0]
+         rdaids = dsid[1]
+         cdgids = dsid[2]
+         srdaid = '|'.join(rdaids)
+         pgrecs = get_dsid_records(cdgids, dates, srdaid)
          pgcnt = len(pgrecs['dataset_file_name']) if pgrecs else 0
          if pgcnt == 0:
             PgLOG.pglog("{}: No record found to gather CDG usage between {} and {}".format(srdaid, dates[0], dates[1]), PgLOG.LOGWRN)
@@ -217,8 +225,9 @@ def fill_cdg_usages(dsids, dranges):
             engine = pgrec['user_agent_name']
             wfile = pgrec['dataset_file_name']
             if not wfile: wfile = pgrec['logic_file_name']
-            wfrec = get_wfile_record(rdaid, wfile)
+            wfrec = get_wfile_record(rdaids, wfile)
             if not wfrec: continue
+            dsid = wfrec['dsid']
             ms = re.search(r'^https://tds.ucar.edu/thredds/(\w+)/', url)
             if ms:
                # tds usage
@@ -235,14 +244,14 @@ def fill_cdg_usages(dsids, dranges):
                      tcnt += add_tdsusage_records(year, trecs, cdate)
                      trecs = {}
                   cdate = date
-               tkey = "{}:{}:{}:{}".format(ip, rdaid, method, etype)
+               tkey = "{}:{}:{}:{}".format(ip, dsid, method, etype)
                if tkey in trecs:
                   trecs[tkey]['size'] += dsize
                   trecs[tkey]['fcount'] += 1
                else:
-                  wurec =  get_wuser_record(ip, cdate, skipwuid = True)
+                  wurec =  get_wuser_record(ip)
                   if not wurec: return 0
-                  trecs[tkey] = {'ip' : ip, 'dsid' : wfrec['dsid'], 'date' : cdate, 'time' : time, 'size' : dsize,
+                  trecs[tkey] = {'ip' : ip, 'dsid' : dsid, 'date' : cdate, 'time' : time, 'size' : dsize,
                                  'fcount' : 1, 'method' : method, 'etype' : etype, 'engine' : engine,
                                  'org_type' : wurec['org_type'], 'country' : wurec['country'],
                                  'email' : wurec['email']}
@@ -252,7 +261,7 @@ def fill_cdg_usages(dsids, dranges):
                if not fsize: fsize = pgrec['logic_file_size']
                method = 'CDP'
                if pgrec['subset_file_size'] or pgrec['range_request'] or dsize < fsize:
-                  wkey = "{}:{}:{}".format(ip, rdaid, wfile)
+                  wkey = "{}:{}:{}".format(ip, dsid, wfile)
                else:
                   wkey = None
       
@@ -261,9 +270,7 @@ def fill_cdg_usages(dsids, dranges):
                      wrec['size'] += dsize
                      continue
                   wcnt += add_webfile_usage(year, wrec)
-               wurec =  get_wuser_record(ip, cdate, skipwuid = False)
-               if not wurec: return 0
-               wrec = {'ip' : ip, 'dsid' : wfrec['dsid'], 'wid' : wfrec['wid'], 'date' : date,
+               wrec = {'ip' : ip, 'dsid' : dsid, 'wid' : wfrec['wid'], 'date' : date,
                        'time' : time, 'quarter' : quarter, 'size' : dsize,
                        'locflag' : 'C', 'method' : method}
                pwkey = wkey
@@ -278,9 +285,6 @@ def fill_cdg_usages(dsids, dranges):
          allcnt += pgcnt
          PgLOG.pglog("{}/{} TDS/WEB usage records added for {} CDG entries at {}".format(atcnt, awcnt, allcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
 
-   if lcnt > 1: PgLOG.pglog("{}/{} TDS/WEB usage records added for {} CDG entries at {}".format(atcnt, awcnt, allcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
-
-
 def get_record_date_time(ctime):
 
    ms = re.search(r'^(\d+)/(\w+)/(\d+) (\d+:\d+:\d+)(\.|$)', str(ctime))
@@ -309,33 +313,43 @@ def add_tdsusage_records(year, records, date):
 
    return cnt
 
-def add_tds_allusage(year, pgrec):
+def add_tds_allusage(year, logrec):
 
-   record = {'method' : 'CDP', 'source' : 'C'}
-
-   for fld in pgrec:
-      if re.match(r'^(engine|method|etype|fcount)$', fld): continue
-      record[fld] = pgrec[fld]
+   pgrec = {'method' : 'CDP', 'source' : 'C'}
+   pgrec['email'] = logrec['email']
+   pgrec['org_type'] = logrec['org_type']
+   pgrec['country'] = logrec['country']
+   pgrec['dsid'] = logrec['dsid']
+   pgrec['date'] = logrec['date']
+   pgrec['quarter'] = logrec['quarter']
+   pgrec['time'] = logrec['time']
+   pgrec['size'] = logrec['size']
+   pgrec['ip'] = logrec['ip']
+   return PgDBI.add_yearly_allusage(year, pgrec)
 
-   return PgDBI.add_yearly_allusage(year, record)
 
 #
 # Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
 #
-def add_webfile_usage(year, logrec, wurec):
+def add_webfile_usage(year, logrec):
 
    table = "{}_{}".format(USAGE['WEBTBL'], year)
-   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(logrec['wid'], logrec['method'], logrec['date'], logrec['time'])
+   cdate = logrec['date']
+   ip = logrec['ip']
+   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(logrec['wid'], logrec['method'], cdate, logrec['time'])
    if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
 
+   wurec =  get_wuser_record(ip, cdate, False)
+   if not wurec: return 0
+
    record = {'wid' : logrec['wid'], 'dsid' : logrec['dsid']}
    record['wuid_read'] = wurec['wuid']
-   record['date_read'] = logrec['date']
+   record['date_read'] = cdate
    record['time_read'] = logrec['time']
    record['size_read'] = logrec['size']
    record['method'] = logrec['method']
    record['locflag'] = logrec['locflag']
-   record['ip'] = logrec['ip']
+   record['ip'] = ip
    record['quarter'] = logrec['quarter']
 
    if add_web_allusage(year, logrec, wurec):
@@ -345,7 +359,10 @@ def add_webfile_usage(year, logrec, wurec):
 
 def add_web_allusage(year, logrec, wurec):
 
-   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'], 'country' : wurec['country']}
+   pgrec = {'source' : 'C'}
+   pgrec['email'] = wurec['email']
+   pgrec['org_type'] = wurec['org_type']
+   pgrec['country'] = wurec['country']
    pgrec['dsid'] = logrec['dsid']
    pgrec['date'] = logrec['date']
    pgrec['quarter'] = logrec['quarter']
@@ -353,7 +370,6 @@ def add_web_allusage(year, logrec, wurec):
    pgrec['size'] = logrec['size']
    pgrec['method'] = logrec['method']
    pgrec['ip'] = logrec['ip']
-   pgrec['source'] = 'C'
    return PgDBI.add_yearly_allusage(year, pgrec)
 
 #
@@ -364,7 +380,7 @@ def get_wfile_record(dsids, wfile):
    for dsid in dsids:
       wkey = "{}{}".format(dsid, wfile)
       if wkey in WFILES: return WFILES[wkey]
-   wfcond = "wfile like '%{}'".format(wfile)
+   wfcond = "wfile LIKE '%{}'".format(wfile)
    pgrec = None
    for dsid in dsids:
       pgrec = PgSplit.pgget_wfile(dsid, "wid", wfcond)
@@ -382,11 +398,13 @@ def get_wfile_record(dsids, wfile):
             pgrec = PgSplit.pgget_wfile(mvrec['dsid'], "wid", "wid = {}".format(pgrec['wid']))
             if pgrec: pgrec['dsid'] = mvrec['dsid']
 
-   if pgrec: WFILES[wkey] = pgrec
+   if pgrec: 
+      wkey = "{}{}".format(pgrec['dsid'], wfile)
+      WFILES[wkey] = pgrec
    return pgrec
 
 # return wuser record upon success, None otherwise
-def get_wuser_record(ip, date, skipwuid = True):
+def get_wuser_record(ip, date = None, skipwuid = True):
 
    if ip in WUSERS: return WUSERS[ip]
 

{rda_python_metrics-1.0.10.dist-info → rda_python_metrics-1.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.10
+Version: 1.0.11
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.10.dist-info → rda_python_metrics-1.0.11.dist-info}/RECORD

@@ -3,7 +3,7 @@ rda_python_metrics/PgView.py,sha256=r6otb3DjfiaQJdg0z8bZQAOlhr4JnrXJzp9wgWh_8qQ,
 rda_python_metrics/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 rda_python_metrics/fillawsusage.py,sha256=dpI3-EFINJECdCSUOm37A97CJNIxOS2nYwA0fB0IpHE,9055
 rda_python_metrics/fillawsusage.usg,sha256=pD_nYTfg1so9nvVEyPSWdgKvb9gWdtfHJAs3RsT2MMU,609
-rda_python_metrics/fillcdgusage.py,sha256=m4zXnx5MaCP-GfNVDRfTOKMlVrcWaFNqwtJeGW7W8Hg,14892
+rda_python_metrics/fillcdgusage.py,sha256=D4PhdVyGoISSr0ykNIpmHVMpEAfRZQB9BvaXdYMcGno,15186
 rda_python_metrics/fillcdgusage.usg,sha256=5lkd4Zdi72nQcha-JtbaLnxl66V4wBDykKwQtUOtMrw,667
 rda_python_metrics/fillcodusage.py,sha256=Hp3VrlIqHBWRI6Zjbi0nxWZPNvPrKsGBSZ1L9qA9_y0,8006
 rda_python_metrics/fillcodusage.usg,sha256=3B5IkQ4uwylqY8uEfUdnZ_MXqhYudeylMp5ulhUGXH8,678
@@ -44,9 +44,9 @@ rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmj
 rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
 rda_python_metrics/viewwebusage.py,sha256=jhoHuRPVNtp7Lbjv0l-Jy_vp2p2nWQC7IVZ0P4JUJ4I,16657
 rda_python_metrics/viewwebusage.usg,sha256=IsT72v22xyZf7ng_IodVs0dLTsH1Q4BtFvT-gs0-xJY,9946
-rda_python_metrics-1.0.10.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_metrics-1.0.10.dist-info/METADATA,sha256=LUru8PysvjXZNqq6kZzFGpMBReGYJtLwXpqFKB5X6hQ,736
-rda_python_metrics-1.0.10.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rda_python_metrics-1.0.10.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
-rda_python_metrics-1.0.10.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
-rda_python_metrics-1.0.10.dist-info/RECORD,,
+rda_python_metrics-1.0.11.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_metrics-1.0.11.dist-info/METADATA,sha256=2D84_3Vy1MxxzM--KjIjJmFitL4QJ0JwkuigW0CErik,736
+rda_python_metrics-1.0.11.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rda_python_metrics-1.0.11.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
+rda_python_metrics-1.0.11.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
+rda_python_metrics-1.0.11.dist-info/RECORD,,