rda-python-metrics 1.0.49__tar.gz → 1.0.51__tar.gz

{rda_python_metrics-1.0.49/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.49
+Version: 1.0.51
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.49 → rda_python_metrics-1.0.51}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.49"
+version = "1.0.51"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]

{rda_python_metrics-1.0.49 → rda_python_metrics-1.0.51}/src/rda_python_metrics/PgIPInfo.py

@@ -142,7 +142,8 @@ def get_ipinfo_record(ip):
 
    if 'bogon' in iprec and iprec['bogon']:
       PgLOG.pglog(f"ipinfo: {ip} - bogon, use {GIP}", PgLOG.LOGWRN)
-      return set_ipinfo(GIP, False)
+      IPRECS[ip] = PgDBI.pgget('ipinfo', '*', f"ip = '{GIP}'", PgLOG.LGEREX)
+      return IPRECS[ip]
 
    record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip, 'org_type' : '-'}
    get_ip_hostname(ip, iprec, record)
@@ -245,7 +246,9 @@ def set_ipinfo(ip, ipopt = True):
    if not pgrec or ipopt and pgrec['stat_flag'] == 'M':
       record = get_ipinfo_record(ip) if ipopt else None
       if not record: record = get_geoip2_record(ip)
-      if record and update_ipinfo_record(record, pgrec): pgrec = record
+      if record:
+         update_ipinfo_record(record, pgrec)
+         pgrec = record
    
    IPRECS[ip] = pgrec
    return pgrec

{rda_python_metrics-1.0.49 → rda_python_metrics-1.0.51}/src/rda_python_metrics/filltdsusage.py

@@ -124,15 +124,17 @@ def fill_tds_usages(fnames):
 
    year = cntall = addall = 0
    for logfile in fnames:
-      gzfile = USAGE['GZFILE'].format(logfile)
-      linfo = PgFile.check_local_file(gzfile)
-      if not linfo:
-         PgLOG.pglog("{}: Not exists for Gathering TDS usage".format(gzfile), PgLOG.LOGWRN)
-         continue
-      PgLOG.pgsystem(USAGE['TDSGET'].format(gzfile, logfile), 5, PgLOG.LOGWRN)
       linfo = PgFile.check_local_file(logfile)
       if not linfo:
-         PgLOG.pglog("{}: Error gunzip TDS usage".format(gzfile), PgLOG.LGEREX)
+         gzfile = USAGE['GZFILE'].format(logfile)
+         linfo = PgFile.check_local_file(gzfile)
+         if not linfo:
+            PgLOG.pglog("{}: Not exists for Gathering TDS usage".format(gzfile), PgLOG.LOGWRN)
+            continue
+         PgLOG.pgsystem(USAGE['TDSGET'].format(gzfile, logfile), 5, PgLOG.LOGWRN)
+         linfo = PgFile.check_local_file(logfile)
+         if not linfo:
+            PgLOG.pglog("{}: Error gunzip TDS usage".format(gzfile), PgLOG.LGEREX)
       PgLOG.pglog("{}: Gathering TDS usage at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
       tds = PgFile.open_local_file(logfile)
       if not tds: continue
@@ -155,6 +157,7 @@ def fill_tds_usages(fnames):
          ip = ms.group(1)
          email = ms.group(2)
          (date, time) = get_record_date_time(ms.group(3))
+         if not date: continue
          method = ms.group(4)
          etype = ms.group(6)[0].upper()
          dsid = PgUtil.format_dataset_id(ms.group(7))
@@ -162,6 +165,9 @@ def fill_tds_usages(fnames):
          ebuf = ms.group(9)
          ms = re.search(r' "(\w+.*\S+)" ', ebuf)
          engine = ms.group(1) if ms else 'Unknown'
+         iprec = PgIPInfo.set_ipinfo(ip)
+         if not iprec: continue
+         ip = iprec['ip']
          key = "{}:{}:{}:{}".format(ip, dsid, method, etype)
 
          if key in records:
@@ -173,6 +179,7 @@ def fill_tds_usages(fnames):
       tds.close()
       if records: cntadd += add_usage_records(records, date)
       cntall += entcnt
+      addall += cntadd
 
    PgLOG.pglog("{} TDS usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
 
@@ -187,7 +194,8 @@ def get_record_date_time(ctime):
       t = ms.group(4)
       return ("{}-{:02}-{:02}".format(y, m, d), t)
    else:
-      PgLOG.pglog("time: Invalid date format", PgLOG.LGEREX)
+      PgLOG.pglog(f"{ctime}: Invalid time format", PgLOG.LGEREX)
+      return (None, None)
 
 def add_usage_records(records, date):
 
@@ -199,7 +207,7 @@ def add_usage_records(records, date):
       quarter = 1 + int((int(ms.group(2)) - 1)/3)
    for key in records:
       record = records[key]
-      cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, record['time'], record['ip'])
+      cond = "date = '{}' AND time = '{}' AND ip = '{}' AND dsid = '{}'".format(date, record['time'], record['ip'], record['dsid'])
       if PgDBI.pgget(USAGE['PGTBL'], '', cond, PgLOG.LGEREX): continue
       email = None if record['email'] == '-' else record['email']
       wurec = PgIPInfo.get_wuser_record(record['ip'], date)

{rda_python_metrics-1.0.49 → rda_python_metrics-1.0.51/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.49
+Version: 1.0.51
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics