rda-python-metrics 3.0.7__tar.gz → 3.0.9__tar.gz

{rda_python_metrics-3.0.7/src/rda_python_metrics.egg-info → rda_python_metrics-3.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 3.0.7
+Version: 3.0.9
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-3.0.7 → rda_python_metrics-3.0.9}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "3.0.7"
+version = "3.0.9"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]

{rda_python_metrics-3.0.7 → rda_python_metrics-3.0.9}/src/rda_python_metrics/pg_ipinfo.py

@@ -47,6 +47,7 @@ class PgIPInfo(PgUtil, PgDBI):
       self.DMRECS = {}
       self.IPDMS = {}
       self.IPNRM = {}
+      self.IPSKP = {}
       self.COUNTRIES = {}
 
    def normalize_ip(self, ip):
@@ -101,7 +102,7 @@ class PgIPInfo(PgUtil, PgDBI):
 
    def get_country_record_code(self, cname, kname = None):
       """Get country code from name."""
-      name = cname[kname] if kname else cname
+      name = cname.get(kname) if kname else cname
       name = name.replace(' ', '.').upper() if name else 'UNITED.STATES'
       if name == 'CHINA': name = 'P.R.CHINA'
       return name
@@ -176,11 +177,14 @@ class PgIPInfo(PgUtil, PgDBI):
       log file. Doing the new ones up front, several at a time, leaves the host names
       in self.IPDMS for get_ip_hostname() to pick up without waiting again.
       """
-      ips = [ip for ip in {self.normalize_ip(ip) for ip in ips} if ip and ip not in self.IPDMS]
+      ips = [ip for ip in {self.normalize_ip(ip) for ip in ips}
+             if ip and ip not in self.IPDMS and ip not in self.IPSKP]
       if not ips: return 0
       # ips on file are either reused as is or refreshed by ipinfo, which carries its
-      # own host name, so only the ones not on file are worth resolving ahead of time
+      # own host name, so only the ones not on file are worth resolving ahead of time.
+      # remember which ones those were, or every later log file queries them again
       recorded = self.get_recorded_ips(ips)
+      self.IPSKP.update(recorded)
       newips = [ip for ip in ips if ip not in recorded]
       if not newips: return 0
       self.get_dns_resolver()   # create the shared resolver before starting the threads
@@ -237,16 +241,24 @@ class PgIPInfo(PgUtil, PgDBI):
          self.IPRECS[ip] = self.pgget('ipinfo', '*', f"ip = '{self.GIP}'", self.LGEREX)
          return self.IPRECS[ip]
       record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip, 'org_type' : '-'}
-      self.get_ip_hostname(ip, iprec, record)
-      record['lat'] = float(iprec['latitude']) if iprec['latitude'] else 0
-      record['lon'] = float(iprec['longitude']) if iprec['longitude'] else 0
-      if 'org' in iprec: record['org_name'] = iprec['org']
-      record['country'] = self.get_country_record_code(iprec, 'country_name')
-      record['region'] = self.convert_chars(iprec['region']) if 'region' in iprec else None
-      if 'city' in iprec: record['city'] = self.convert_chars(iprec['city'])
-      if 'postal' in iprec: record['postal'] =  iprec['postal']
-      record['timezone'] = iprec['timezone']
-      record['ipinfo'] = json.dumps(iprec)
+      try:
+         self.get_ip_hostname(ip, iprec, record)
+         # latitude and longitude are derived from 'loc', which a sparse ipinfo
+         # answer leaves out altogether, as it does timezone and country name
+         record['lat'] = float(iprec['latitude']) if iprec.get('latitude') else 0
+         record['lon'] = float(iprec['longitude']) if iprec.get('longitude') else 0
+         if 'org' in iprec: record['org_name'] = iprec['org']
+         record['country'] = self.get_country_record_code(iprec, 'country_name')
+         record['region'] = self.convert_chars(iprec['region']) if 'region' in iprec else None
+         if 'city' in iprec: record['city'] = self.convert_chars(iprec['city'])
+         if 'postal' in iprec: record['postal'] =  iprec['postal']
+         record['timezone'] = iprec.get('timezone')
+         record['ipinfo'] = json.dumps(iprec)
+      except (KeyError, TypeError, ValueError) as e:
+         # an ipinfo answer shaped differently again must not end the whole run;
+         # returning None lets set_ipinfo() fall back to the local geoip2 database
+         self.pglog("ipinfo: {} - unusable record ({}: {}), use geoip2".format(ip, type(e).__name__, e), self.LOGWRN)
+         return None
       return record
 
    def set_geoip2_database(self):

{rda_python_metrics-3.0.7 → rda_python_metrics-3.0.9/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 3.0.7
+Version: 3.0.9
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics