rda-python-metrics 1.0.48__tar.gz → 1.0.50__tar.gz

{rda_python_metrics-1.0.48/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.48
+Version: 1.0.50
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.48 → rda_python_metrics-1.0.50}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.48"
+version = "1.0.50"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]

{rda_python_metrics-1.0.48 → rda_python_metrics-1.0.50}/src/rda_python_metrics/PgIPInfo.py

@@ -31,6 +31,7 @@ IPINFO = {
    'IPADD'  : 0
 }
 
+GIP = '0.0.0.0'
 IPDNS = None
 IPDB = None
 G2DB = None
@@ -140,8 +141,10 @@ def get_ipinfo_record(ip):
       return None
 
    if 'bogon' in iprec and iprec['bogon']:
-      PgLOG.pglog(f"ipinfo: {ip} - bogon", PgLOG.LOGWRN)
-      return None
+      PgLOG.pglog(f"ipinfo: {ip} - bogon, use {GIP}", PgLOG.LOGWRN)
+      IPRECS[ip] = PgDBI.pgget('ipinfo', '*', f"ip = '{GIP}'", PgLOG.LGEREX)
+      return IPRECS[ip]
+
    record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip, 'org_type' : '-'}
    get_ip_hostname(ip, iprec, record)
    record['lat'] = float(iprec['latitude']) if iprec['latitude'] else 0
@@ -243,7 +246,9 @@ def set_ipinfo(ip, ipopt = True):
    if not pgrec or ipopt and pgrec['stat_flag'] == 'M':
       record = get_ipinfo_record(ip) if ipopt else None
       if not record: record = get_geoip2_record(ip)
-      if record and update_ipinfo_record(record, pgrec): pgrec = record
+      if record:
+         update_ipinfo_record(record, pgrec)
+         pgrec = record
    
    IPRECS[ip] = pgrec
    return pgrec
@@ -272,7 +277,9 @@ def get_missing_ipinfo(ip, email = None):
    if ipinfo:
       record = {'org_type' : ipinfo['org_type'],
                 'country' : ipinfo['country'],
-                'region' : ipinfo['region']}
+                'region' : ipinfo['region'],
+                'hostname' : ipinfo['hostname'],
+                'ip' : ipinfo['ip']}
       if not email or re.search(r'-$', email):
          record['email'] =  'unknown@' + ipinfo['hostname']
       else:
@@ -289,21 +296,31 @@ def get_wuser_record(ip, date, email = None):
    if not record: return None
 
    emcond = "email = '{}'".format(record['email'])
-   flds = 'wuid, email, org_type, country, region, start_date'   
+   flds = 'wuid, start_date'   
    pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
    if pgrec:
+      record['wuid'] = pgrec['wuid']
       if PgUtil.diffdate(pgrec['start_date'], date) > 0:
-         pgrec['start_date'] = record['start_date'] = date
-         PgDBI.pgupdt('wuser', record, emcond)
-      return pgrec
+         PgDBI.pgupdt('wuser', new_wuser_record(record, date, False), emcond)
+      return record
 
    # now add one in
-   record['stat_flag'] = 'A'
-   record['start_date'] = date
-   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
+   wuid = PgDBI.pgadd("wuser", new_wuser_record(record, date), PgLOG.LOGERR|PgLOG.AUTOID)
    if wuid:
       record['wuid'] = wuid
       PgLOG.pglog("{} Added as wuid({})".format(record['email'], wuid), PgLOG.LGWNEM)
       return record
 
    return None
+
+def new_wuser_record(iprec, date, nuser = True):
+
+   wurec = {'start_date' : date}
+   wurec['org_type'] = iprec['org_type']
+   wurec['country'] = iprec['country']
+   wurec['region'] = iprec['region']
+   if nuser:
+      wurec['email'] = iprec['email']
+      wurec['stat_flag'] = 'A'
+
+   return wurec

{rda_python_metrics-1.0.48 → rda_python_metrics-1.0.50}/src/rda_python_metrics/fillosdfusage.py

@@ -133,7 +133,7 @@ def fill_osdf_usages(fnames):
          dsid = ms.group(2)
          site = ms.group(3)
          ip = ms.group(4)
-         if ip == 'N/A': ip = '0.0.0.0'
+         if ip == 'N/A': ip = PgIPInfo.GIP
          engine = ms.group(5)
          size = int(ms.group(6))
          if re.match(r'^N/A', engine, re.I):
@@ -153,7 +153,7 @@ def fill_osdf_usages(fnames):
             (year, quarter, date, time) = get_record_date_time(dt)
             iprec =  PgIPInfo.get_missing_ipinfo(ip)
             if not iprec: continue
-            records[key] = {'ip' : ip, 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
+            records[key] = {'ip' : iprec['ip'], 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
                             'size' : size, 'fcount' : 1, 'method' : method, 'engine' : engine,
                             'org_type' : iprec['org_type'], 'country' : iprec['country'],
                             'region' : iprec['region'], 'email' : iprec['email'], 'site' : site}

{rda_python_metrics-1.0.48 → rda_python_metrics-1.0.50}/src/rda_python_metrics/filltdsusage.py

@@ -36,7 +36,8 @@ USAGE = {
    'GITDIR' : PgLOG.PGLOG["GDEXWORK"] + "/zji/tdslogs/tds-logs",
    'GITGET' : 'git pull',
    'TDSDIR' : PgLOG.PGLOG["GDEXWORK"] + "/zji/tdslogs/work",
-   'TDSGET' : 'gunzip -c ../tds-logs/logs/{} > {}',
+   'GZFILE' : '../tds-logs/logs/{}.gz',
+   'TDSGET' : 'gunzip -c {} > {}',
    'TDSLOG' : "localhost_access_log.{}.txt"   # {} = YYYY-MM-DD
 }
 
@@ -123,16 +124,17 @@ def fill_tds_usages(fnames):
 
    year = cntall = addall = 0
    for logfile in fnames:
-      gzfile = logfile + '.gz'
-      PgLOG.pgsystem(USAGE['TDSGET'].format(gzfile, logfile), 5, PgLOG.LOGWRN)
-      linfo = PgFile.check_local_file(gzfile)
-      if not linfo:
-         PgLOG.pglog("{}: Not exists for Gathering TDS usage".format(gzfile), PgLOG.LOGWRN)
-         continue
-      PgFile.compress_local_file(gzfile)
       linfo = PgFile.check_local_file(logfile)
       if not linfo:
-         PgLOG.pglog("{}: Error gunzip TDS usage".format(gzfile), PgLOG.LGEREX)
+         gzfile = USAGE['GZFILE'].format(logfile)
+         linfo = PgFile.check_local_file(gzfile)
+         if not linfo:
+            PgLOG.pglog("{}: Not exists for Gathering TDS usage".format(gzfile), PgLOG.LOGWRN)
+            continue
+         PgLOG.pgsystem(USAGE['TDSGET'].format(gzfile, logfile), 5, PgLOG.LOGWRN)
+         linfo = PgFile.check_local_file(logfile)
+         if not linfo:
+            PgLOG.pglog("{}: Error gunzip TDS usage".format(gzfile), PgLOG.LGEREX)
       PgLOG.pglog("{}: Gathering TDS usage at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
       tds = PgFile.open_local_file(logfile)
       if not tds: continue
@@ -162,6 +164,9 @@ def fill_tds_usages(fnames):
          ebuf = ms.group(9)
          ms = re.search(r' "(\w+.*\S+)" ', ebuf)
          engine = ms.group(1) if ms else 'Unknown'
+         iprec = PgIPInfo.set_ipinfo(ip)
+         if not iprec: continue
+         ip = iprec['ip']
          key = "{}:{}:{}:{}".format(ip, dsid, method, etype)
 
          if key in records:
@@ -199,24 +204,16 @@ def add_usage_records(records, date):
       quarter = 1 + int((int(ms.group(2)) - 1)/3)
    for key in records:
       record = records[key]
-      cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, record['time'], record['ip'])
+      cond = "date = '{}' AND time = '{}' AND ip = '{}' AND dsid = '{}'".format(date, record['time'], record['ip'], record['dsid'])
       if PgDBI.pgget(USAGE['PGTBL'], '', cond, PgLOG.LGEREX): continue
-      if record['email'] == '-':
-         wurec = PgIPInfo.get_wuser_record(record['ip'], date)
-         if not wurec: continue
-         record['org_type'] = wurec['org_type']
-         record['country'] = wurec['country']
-         record['region'] = wurec['region']
-         record['email'] = 'unknown@' + wurec['hostname']
-      else:
-         wuid = PgDBI.check_wuser_wuid(record['email'], date)
-         if not wuid: continue
-         pgrec = PgDBI.pgget("wuser",  "org_type, country, region", "wuid = {}".format(wuid), PgLOG.LGWNEX)
-         if not pgrec: continue
-         record['org_type'] = pgrec['org_type']
-         record['country'] = pgrec['country']
-         record['region'] = pgrec['region']
-
+      email = None if record['email'] == '-' else record['email']
+      wurec = PgIPInfo.get_wuser_record(record['ip'], date)
+      if not wurec: continue
+      record['ip'] = wurec['ip']    # in case generic ip
+      record['org_type'] = wurec['org_type']
+      record['country'] = wurec['country']
+      record['region'] = wurec['region']
+      record['email'] = 'unknown@' + wurec['hostname']
       record['quarter'] = quarter
       record['date'] = date
 

{rda_python_metrics-1.0.48 → rda_python_metrics-1.0.50/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.48
+Version: 1.0.50
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics