rda-python-metrics 3.0.5__tar.gz → 3.0.7__tar.gz

{rda_python_metrics-3.0.5/src/rda_python_metrics.egg-info → rda_python_metrics-3.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 3.0.5
+Version: 3.0.7
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "3.0.5"
+version = "3.0.7"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/src/rda_python_metrics/fillawsusage.py

@@ -107,6 +107,7 @@ class FillAWSUsage(PgIPInfo, PgFile):
          for logfile in fnames:
             aws = self.open_local_file(logfile)
             if not aws: continue
+            self.prescan_ips(aws, r'^\w+ [\w-]+ \[[^\]]*\] ([\da-fA-F.:]+) ')
             while True:
                line = aws.readline()
                if not line: break
@@ -114,13 +115,15 @@ class FillAWSUsage(PgIPInfo, PgFile):
                if entcnt%20000 == 0:
                   dcnt = len(records)
                   self.pglog("{}: {}/{} AWS log entries processed/records to add".format(pdate, entcnt, dcnt), self.WARNLG)
-               ms = re.match(r'^\w+ ([\w-]+) \[(\S+).*\] ([\d\.]+) .+ REST\.GET\.OBJECT \S+ "GET.+" \d+ - (\d+) \d+ .* ".+" "(.+)" ', line)
+               # the ip field holds an IPv6 address as well as an IPv4 one
+               ms = re.match(r'^\w+ ([\w-]+) \[(\S+).*\] ([\da-fA-F.:]+) .+ REST\.GET\.OBJECT \S+ "GET.+" \d+ - (\d+) \d+ .* ".+" "(.+)" ', line)
                if not ms: continue
                values = list(ms.groups())
                if values[0] not in self.DSIDS: continue
                dsid = self.DSIDS[values[0]]
                size = int(values[3])
-               ip = values[2]
+               ip = self.normalize_ip(values[2])
+               if not ip: continue
                engine = values[4]
                moff = engine.find('/')
                if moff > 0:

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/src/rda_python_metrics/fillcdgusage.py

@@ -204,6 +204,8 @@ class FillCDGUsage(PgSplit, PgIPInfo):
       self.pglog("{}: Query for {} CDG dsid/subdsids between {} and {} at {}".format(strids, dscnt, dates[0], dates[1], self.current_datetime()), self.LOGWRN)
       pgrecs = self.pgmget(tbname, fields, cond)
       self.dssdb_dbname()
+      # back on dssdb, where table ipinfo is, before presolving the host names
+      if pgrecs: self.preset_ip_hostnames(pgrecs['remote_address'])
       return pgrecs
 
    def fill_cdg_usages(self, dsids, dranges):
@@ -243,7 +245,8 @@ class FillCDGUsage(PgSplit, PgIPInfo):
                (year, quarter, date, time) = self.get_record_date_time(pgrec['date_completed'])
                url = pgrec['dataset_file_file_access_point_uri']
                if not url: url = pgrec['file_access_point_uri']
-               ip = pgrec['remote_address']
+               ip = self.normalize_ip(pgrec['remote_address'])
+               if not ip: continue
                engine = pgrec['user_agent_name']
                ms = re.search(r'^https*://tds.ucar.edu/thredds/(\w+)/', url)
                if ms:

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/src/rda_python_metrics/fillcodusage.py

@@ -116,9 +116,11 @@ class FillCODUsage(PgIPInfo):
                size = int(ms.group(2))
                engine = ms.group(3)
                if not (aid in self.USERS or self.cache_users(aid)): continue
-               ms = re.match(r'^([\d\.]+).+\[(\d+)/(\w+)/(\d+):([\d:]+)', line)
+               # the ip field holds an IPv6 address as well as an IPv4 one
+               ms = re.match(r'^([\da-fA-F.:]+).+\[(\d+)/(\w+)/(\d+):([\d:]+)', line)
                if not ms: continue
-               ip = ms.group(1)
+               ip = self.normalize_ip(ms.group(1))
+               if not ip: continue
                ctime = ms.group(5)
                cdate = "{}-{:02}-{:02}".format(ms.group(4), self.get_month(ms.group(3)), int(ms.group(2)))
                if pdate != cdate:
@@ -153,6 +155,7 @@ class FillCODUsage(PgIPInfo):
       year = ms.group(1)
       quarter = 1 + int((int(ms.group(2)) - 1) / 3)
       cnt = 0
+      self.preset_ip_hostnames([records[aid]['ip'] for aid in records])
       for aid in records:
          if self.pgget(self.USAGE['PGTBL'], '', "aid = '{}' AND date = '{}'".format(aid, date), self.LGEREX): continue
          record = records[aid]
@@ -162,7 +165,7 @@ class FillCODUsage(PgIPInfo):
             record['org_type'] = wurec['org_type']
             record['country'] = wurec['country']
             record['region'] = wurec['region']
-            record['email'] = 'unknown@' + wurec['hostname']
+            record['email'] = wurec['email']   # already 'unknown@<hostname>' for a generic ip
          else:
             wuid = self.check_wuser_wuid(record['email'], date)
             if not wuid: continue

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/src/rda_python_metrics/fillgdexusage.py

@@ -857,6 +857,8 @@ class FillGDEXUsage(PgSplit, PgIPInfo):
       self.pglog("{}: Query for {} GDEX dsid/subdsids between {} and {} at {}".format(strids, dscnt, dates[0], dates[1], self.current_datetime()), self.LOGWRN)
       pgrecs = self.pgmget(tbname, fields, cond)
       self.dssdb_dbname()
+      # back on dssdb, where table ipinfo is, before presolving the host names
+      if pgrecs: self.preset_ip_hostnames(pgrecs['remote_address'])
 
       return pgrecs
 
@@ -892,7 +894,8 @@ class FillGDEXUsage(PgSplit, PgIPInfo):
                dsize = pgrec['bytes_sent']
                if not dsize: continue
                (year, quarter, date, time) = self.get_record_date_time(pgrec['date_completed'])
-               ip = pgrec['remote_address']
+               ip = self.normalize_ip(pgrec['remote_address'])
+               if not ip: continue
                engine = pgrec['user_agent_name']
                wfile = pgrec['dataset_file_name']
                if not wfile: wfile = pgrec['logical_file_name']

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/src/rda_python_metrics/fillglobususage.py

@@ -106,6 +106,7 @@ class FillGlobusUsage(PgIPInfo, PgSplit, PgFile):
          self.pglog("Gathering usage info from {} at {}".format(logfile, self.current_datetime()), self.LOGWRN)
          globus = self.open_local_file(logfile)
          if not globus: continue
+         self.prescan_ips(globus, r'^([\da-fA-F.:]+)\s')
          ptime = ''
          record = {}
          cntadd = entcnt = 0
@@ -116,11 +117,13 @@ class FillGlobusUsage(PgIPInfo, PgSplit, PgFile):
             entcnt += 1
             if entcnt%10000 == 0:
                self.pglog("{}: {}/{} Globus log entries processed/records added".format(logfile, entcnt, cntadd), self.WARNLG)
-            ms = re.match(r'^([\d\.]+)\s.*\s+\[(\S+).*"GET\s+/(ds\d\d\d\.\d|[a-z]\d{6})/(\S+)\s.*\s(200|206)\s+(\d+)\s+"(\S+)"\s+"(.+)"$', line)
+            # the ip field holds an IPv6 address as well as an IPv4 one
+            ms = re.match(r'^([\da-fA-F.:]+)\s.*\s+\[(\S+).*"GET\s+/(ds\d\d\d\.\d|[a-z]\d{6})/(\S+)\s.*\s(200|206)\s+(\d+)\s+"(\S+)"\s+"(.+)"$', line)
             if not ms: continue
             size = int(ms.group(6))
             if size < 100: continue  # ignore small files
-            ip = ms.group(1)
+            ip = self.normalize_ip(ms.group(1))
+            if not ip: continue
             dsid = self.format_dataset_id(ms.group(3))
             wfile = ms.group(4)
             stat = ms.group(5)

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/src/rda_python_metrics/fillipinfo.py

@@ -117,6 +117,7 @@ class FillIPInfo(PgIPInfo):
       if not pgrecs: return 0
       cnt = len(pgrecs['ip']) if pgrecs else 0
       mcnt = 0
+      self.preset_ip_hostnames(pgrecs['ip'])
       for i in range(cnt):
          record = self.get_missing_ipinfo(pgrecs['ip'][i], pgrecs['email'][i])
          if record: mcnt += self.pgupdt(table, record, "aidx = '{}'".format(pgrecs['aidx'][i]))
@@ -132,6 +133,7 @@ class FillIPInfo(PgIPInfo):
       if not pgrecs: return 0
       cnt = len(pgrecs['ip']) if pgrecs else 0
       mcnt = 0
+      self.preset_ip_hostnames(pgrecs['ip'])
       for i in range(cnt):
          ip = pgrecs['ip'][i]
          record = self.get_missing_ipinfo(ip, pgrecs['email'][i])
@@ -150,6 +152,7 @@ class FillIPInfo(PgIPInfo):
       if not pgrecs: return 0
       cnt = len(pgrecs['ip']) if pgrecs else 0
       mcnt = 0
+      self.preset_ip_hostnames(pgrecs['ip'])
       for i in range(cnt):
          record = self.get_missing_ipinfo(pgrecs['ip'][i], pgrecs['email'][i])
          if record:
@@ -183,6 +186,7 @@ class FillIPInfo(PgIPInfo):
       if not pgrecs: return 0
       cnt = len(pgrecs['ip']) if pgrecs else 0
       mcnt = 0
+      self.preset_ip_hostnames(pgrecs['ip'])
       for i in range(cnt):
          if self.set_ipinfo(pgrecs['ip'][i]): mcnt +=1
       s = 's' if cnt > 1 else ''

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/src/rda_python_metrics/fillosdfusage.py

@@ -105,6 +105,7 @@ class FillOSDFUsage(PgIPInfo, PgFile, PgSplit):
          osdf = self.open_local_file(logfile)
          if not osdf: continue
          records = {}
+         ips = {}
          cntadd = entcnt = 0
          while True:
             line = osdf.readline()
@@ -113,13 +114,16 @@ class FillOSDFUsage(PgIPInfo, PgFile, PgSplit):
             if entcnt%20000 == 0:
                dcnt = len(records)
                self.pglog("{}: {}/{} OSDF log entries processed/records added".format(logfile, entcnt, dcnt), self.WARNLG)
-            ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/rda\/([a-z]\d{6})\/\S+\].* \[Site:(\S+)\].* \[Host:(\S+)\].* \[AppInfo:(\S*)\].* \[Read:(\d+)\]', line)
+            # the OSDF namespace was renamed from /ncar/rda to /ncar/gdex
+            ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/(?:rda|gdex)\/([a-z]\d{6})\/\S+\].* \[Site:(\S+)\].* \[Host:(\S+)\].* \[AppInfo:(\S*)\].* \[Read:(\d+)\]', line)
             if not ms: continue
             dt = ms.group(1)
             dsid = ms.group(2)
             site = ms.group(3)
-            ip = ms.group(4)
-            if ip == 'N/A': ip = self.GIP
+            # normalize before keying on it, so one address spelled two ways in the
+            # log aggregates into one usage record rather than two
+            ip = self.GIP if ms.group(4) == 'N/A' else self.normalize_ip(ms.group(4))
+            if not ip: continue
             engine = ms.group(5)
             size = int(ms.group(6))
             if re.match(r'^N/A', engine, re.I):
@@ -137,13 +141,13 @@ class FillOSDFUsage(PgIPInfo, PgFile, PgSplit):
                records[key]['fcount'] += 1
             else:
                (year, quarter, date, time) = self.get_record_date_time(dt)
-               iprec =  self.get_missing_ipinfo(ip)
-               if not iprec: continue
-               records[key] = {'ip' : iprec['ip'], 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
-                               'size' : size, 'fcount' : 1, 'method' : method, 'engine' : engine,
-                               'org_type' : iprec['org_type'], 'country' : iprec['country'],
-                               'region' : iprec['region'], 'email' : iprec['email'], 'site' : site}
+               ips[ip] = 1
+               records[key] = {'ip' : ip, 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
+                               'size' : size, 'fcount' : 1, 'method' : method, 'engine' : engine, 'site' : site}
          osdf.close()
+         if records:
+            self.preset_ip_hostnames(list(ips))
+            records = self.fill_ipinfo_records(records)
          if records: cntadd = self.add_usage_records(records, year)
          self.pglog("{}: {} OSDF usage records added for {} entries at {}".format(logfile, cntadd, entcnt, self.current_datetime()), self.LOGWRN)
          cntall += entcnt
@@ -165,6 +169,21 @@ class FillOSDFUsage(PgIPInfo, PgFile, PgSplit):
       else:
          self.pglog(ctime + ": Invalid date/time format", self.LGEREX)
 
+   def fill_ipinfo_records(self, records):
+      """Add the ip info fields to the usage records; drop the ones without ip info."""
+      for key in list(records):
+         record = records[key]
+         iprec = self.get_missing_ipinfo(record['ip'])
+         if not iprec:
+            del records[key]
+            continue
+         record['ip'] = iprec['ip']
+         record['org_type'] = iprec['org_type']
+         record['country'] = iprec['country']
+         record['region'] = iprec['region']
+         record['email'] = iprec['email']
+      return records
+
    def add_usage_records(self, records, year):
       """Add usage to table osdfusage."""
       cnt = 0

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/src/rda_python_metrics/filltdsusage.py

@@ -115,6 +115,7 @@ class FillTDSUsage(PgIPInfo, PgFile):
          self.pglog("{}: Gathering TDS usage at {}".format(logfile, self.current_datetime()), self.LOGWRN)
          tds = self.open_local_file(logfile)
          if not tds: continue
+         self.prescan_ips(tds, r'^([\da-fA-F.:]+)\s')
          records = {}
          cntadd = entcnt = 0
          while True:
@@ -128,7 +129,8 @@ class FillTDSUsage(PgIPInfo, PgFile):
             if ms: continue
             ms = re.search(r'/thredds/\S+\.(png|jpg|gif|css|htm)', line)
             if ms: continue
-            ms = re.match(r'^([\d\.]+)\s.*\s(-|\S+@\S+)\s+\[(\S+).*/thredds/(\w+)(/|/grid/)(aggregations|files).*/(ds\d\d\d.\d|[a-z]\d{6})/.*\s200\s+(\d+)(.*)$', line)
+            # the ip field holds an IPv6 address as well as an IPv4 one
+            ms = re.match(r'^([\da-fA-F.:]+)\s.*\s(-|\S+@\S+)\s+\[(\S+).*/thredds/(\w+)(/|/grid/)(aggregations|files).*/(ds\d\d\d.\d|[a-z]\d{6})/.*\s200\s+(\d+)(.*)$', line)
             if not ms: continue
             ip = ms.group(1)
             email = ms.group(2)
@@ -178,6 +180,7 @@ class FillTDSUsage(PgIPInfo, PgFile):
       if ms:
          year = ms.group(1)
          quarter = 1 + int((int(ms.group(2)) - 1)/3)
+      self.preset_ip_hostnames([records[key]['ip'] for key in records])
       for key in records:
          record = records[key]
          cond = "date = '{}' AND time = '{}' AND ip = '{}' AND dsid = '{}'".format(date, record['time'], record['ip'], record['dsid'])
@@ -189,7 +192,7 @@ class FillTDSUsage(PgIPInfo, PgFile):
          record['org_type'] = wurec['org_type']
          record['country'] = wurec['country']
          record['region'] = wurec['region']
-         record['email'] = 'unknown@' + wurec['hostname']
+         record['email'] = wurec['email']   # already 'unknown@<hostname>' for a generic ip
          record['quarter'] = quarter
          record['date'] = date
          if self.add_to_allusage(year, record):

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/src/rda_python_metrics/fillzenodousage.py

@@ -284,6 +284,8 @@ class FillZenodoUsage(PgSplit, PgIPInfo):
       self.pglog("{}: Query for {} GDEX dsid/subdsids between {} and {} at {}".format(strids, dscnt, dates[0], dates[1], self.current_datetime()), self.LOGWRN)
       pgrecs = self.pgmget(tbname, fields, cond)
       self.dssdb_dbname()
+      # back on dssdb, where table ipinfo is, before presolving the host names
+      if pgrecs: self.preset_ip_hostnames(pgrecs['remote_address'])
 
       return pgrecs
 
@@ -322,7 +324,8 @@ class FillZenodoUsage(PgSplit, PgIPInfo):
                (year, quarter, date, time) = self.get_record_date_time(pgrec['date_completed'])
                url = pgrec['dataset_file_file_access_point_uri']
                if not url: url = pgrec['file_access_point_uri']
-               ip = pgrec['remote_address']
+               ip = self.normalize_ip(pgrec['remote_address'])
+               if not ip: continue
                engine = pgrec['user_agent_name']
                ms = re.search(r'^https*://tds.ucar.edu/thredds/(\w+)/', url)
                if ms:

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7}/src/rda_python_metrics/pg_ipinfo.py

@@ -12,9 +12,12 @@
 import re
 import geoip2.database as geodb
 import ipinfo
-import socket
+from ipinfo.exceptions import RequestQuotaExceededError
 import dns.resolver
+import dns.reversename
 import json
+import ipaddress
+from concurrent.futures import ThreadPoolExecutor
 from rda_python_common.pg_dbi import PgDBI
 from rda_python_common.pg_util import PgUtil
 
@@ -30,7 +33,11 @@ class PgIPInfo(PgUtil, PgDBI):
          'DBFILE' : self.PGLOG['DSSHOME'] + '/dssdb/GeoLite2-City.mmdb',
          'CDATE' : self.curdate(),
          'IPUPDT' : 0,
-         'IPADD'  : 0
+         'IPADD'  : 0,
+         'QUOTA'  : 1,   # unset when the ipinfo monthly request quota is used up
+         'DNSTMO' : 1,   # seconds to wait on a single name server
+         'DNSLTM' : 2,   # seconds to spend on one name resolution in total
+         'DNSTHD' : 8    # number of threads used to presolve a list of ip addresses
       }
       self.GIP = '0.0.0.0'
       self.IPDNS = None
@@ -38,11 +45,34 @@ class PgIPInfo(PgUtil, PgDBI):
       self.G2DB = None
       self.IPRECS = {}
       self.DMRECS = {}
+      self.IPDMS = {}
+      self.IPNRM = {}
       self.COUNTRIES = {}
 
+   def normalize_ip(self, ip):
+      """Return the canonical form of an ip address; empty string if it is not valid.
+
+      One IPv6 address can be written several ways, upper or lower case and with the
+      zero groups spelled out or collapsed, and the IPv4-embedded form is longer than
+      the 39 characters ipinfo.ip holds. Canonicalizing on the way in keeps one row
+      and one set of usage records per address whichever way a log spells it.
+      """
+      if ip in self.IPNRM: return self.IPNRM[ip]
+      try:
+         nip = ipaddress.ip_address(ip).compressed
+      except ValueError:
+         self.pglog("{}: invalid ip address".format(ip), self.LOGWRN)
+         nip = ''
+      self.IPNRM[ip] = nip
+      return nip
+
    def get_dns_resolver(self, forceget = False):
       """Get save a global dns.resolver.Resolver object."""
-      if forceget or not self.IPDNS: self.IPDNS = dns.resolver.Resolver()
+      if forceget or not self.IPDNS:
+         self.IPDNS = dns.resolver.Resolver()
+         self.IPDNS.timeout = self.IPINFO['DNSTMO']
+         self.IPDNS.lifetime = self.IPINFO['DNSLTM']
+         self.IPDNS.cache = dns.resolver.Cache()
       return self.IPDNS
 
    def dns_to_ip(self, dmname, type = 'A'):
@@ -89,32 +119,118 @@ class PgIPInfo(PgUtil, PgDBI):
       if ips: return self.set_ipinfo(ips[0])
       return None
 
+   def reverse_resolve_ip(self, ip):
+      """Reverse resolve one ip address without logging; called from worker threads too.
+
+      Returns tuple (ip, hostname, errmsg); hostname is empty if the ip has no PTR record.
+      """
+      try:
+         answers = self.IPDNS.resolve(dns.reversename.from_address(ip), 'PTR')
+         # an ip may have several PTR records, answered in varying order; always pick
+         # the same one so the host name on file does not flip between runs
+         if answers: return (ip, min([str(a).rstrip('.') for a in answers]), None)
+      except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+         pass   # no PTR record on file for the ip, too common to be worth logging
+      except dns.exception.DNSException as e:
+         return (ip, '', "{}: error reverse domain name request: {}".format(ip, e))
+      return (ip, '', None)
+
+   def ip_to_dns(self, ip):
+      """Reverse resolve an ip address to a host name; empty string if none."""
+      if ip in self.IPDMS: return self.IPDMS[ip]
+      self.get_dns_resolver()
+      (ip, dm, errmsg) = self.reverse_resolve_ip(ip)
+      if errmsg: self.pglog(errmsg, self.LOGWRN)
+      self.IPDMS[ip] = dm
+      return dm
+
+   def prescan_ips(self, fd, pattern, group = 1):
+      """Scan an open log file for ip addresses, presolve them, and rewind the file.
+
+      A fill program reads its log line by line and looks each new ip up as it goes,
+      which serializes one slow reverse DNS lookup per address. Picking the addresses
+      out in a first cheap pass lets preset_ip_hostnames() resolve them several at a
+      time, so the main loop finds the host names already cached.
+      """
+      ips = {}
+      for line in fd:
+         ms = re.match(pattern, line)
+         if ms: ips[ms.group(group)] = 1
+      fd.seek(0)
+      return self.preset_ip_hostnames(list(ips))
+
+   def get_recorded_ips(self, ips):
+      """Return a dict of the given ip addresses that have an ipinfo record on file."""
+      recorded = {}
+      for i in range(0, len(ips), 1000):
+         vals = ', '.join(["'{}'".format(ip) for ip in ips[i:i+1000]])
+         pgrecs = self.pgmget('ipinfo', 'ip', "ip IN ({})".format(vals), self.LOGWRN)
+         if pgrecs:
+            for ip in pgrecs['ip']: recorded[ip] = 1
+      return recorded
+
+   def preset_ip_hostnames(self, ips):
+      """Reverse resolve a list of ip addresses in parallel and cache the host names.
+
+      Resolving one ip at a time is the slowest part of gathering usage from a large
+      log file. Doing the new ones up front, several at a time, leaves the host names
+      in self.IPDMS for get_ip_hostname() to pick up without waiting again.
+      """
+      ips = [ip for ip in {self.normalize_ip(ip) for ip in ips} if ip and ip not in self.IPDMS]
+      if not ips: return 0
+      # ips on file are either reused as is or refreshed by ipinfo, which carries its
+      # own host name, so only the ones not on file are worth resolving ahead of time
+      recorded = self.get_recorded_ips(ips)
+      newips = [ip for ip in ips if ip not in recorded]
+      if not newips: return 0
+      self.get_dns_resolver()   # create the shared resolver before starting the threads
+      cnt = 0
+      with ThreadPoolExecutor(max_workers = self.IPINFO['DNSTHD']) as executor:
+         for (ip, dm, errmsg) in executor.map(self.reverse_resolve_ip, newips):
+            if errmsg: self.pglog(errmsg, self.LOGWRN)
+            self.IPDMS[ip] = dm
+            if dm: cnt += 1
+      self.pglog("{}/{} host names presolved for {} IP addresses".format(cnt, len(newips), len(ips)), self.LOGWRN)
+      return cnt
+
    def get_ip_hostname(self, ip, iprec, record):
-      """Try to get hostname via socket for given ip address."""
+      """Try to get hostname via reverse DNS for given ip address."""
       record['hostname'] = ip
       if iprec:
          if 'hostname' in iprec and iprec['hostname']:
-            record['hostname'] = iprec['hostname']         
+            record['hostname'] = iprec['hostname']
             record['org_type'] = self.get_org_type(None, record['hostname'])
             return
          if 'asn' in iprec and iprec['asn'] and 'domain' in iprec['asn'] and iprec['asn']['domain']:
             record['hostname'] += '.' + iprec['asn']['domain']
             record['org_type'] = self.get_org_type(None, record['hostname'])
             return
-      try:
-         hostrec = socket.gethostbyaddr(ip)
-         record['hostname'] = hostrec[1][0] if hostrec[1] else hostrec[0]
-         record['org_type'] = self.get_org_type(None, record['hostname'])
-      except Exception as e:
-         self.pglog("socket: {} - {}".format(ip, str(e)), self.LOGWRN)
+      dm = self.ip_to_dns(ip)
+      if dm:
+         record['hostname'] = dm
+         record['org_type'] = self.get_org_type(None, dm)
+
+   def unset_ipinfo_quota(self):
+      """Stop calling ipinfo for the rest of the run once its quota is used up."""
+      if self.IPINFO['QUOTA']:
+         self.IPINFO['QUOTA'] = 0
+         self.pglog("ipinfo: monthly request quota used up, use geoip2 for the rest of the run", self.LOGWRN)
 
    def get_ipinfo_record(self, ip):
       """Get a ipinfo record for given ip address."""
+      if not self.IPINFO['QUOTA']: return None
       if not self.IPDB: self.set_ipinfo_database()
       try:
          iprec = self.IPDB.getDetails(ip).all
+      except RequestQuotaExceededError:
+         self.unset_ipinfo_quota()
+         return None
       except Exception as e:
-         self.pglog("ipinfo: {} - {}".format(ip, str(e)), self.LOGWRN)
+         estr = str(e)
+         if re.search(r'(\b429\b|quota)', estr, re.I):
+            self.unset_ipinfo_quota()
+         else:
+            self.pglog("ipinfo: {} - {}".format(ip, estr), self.LOGWRN)
          return None
       if 'bogon' in iprec and iprec['bogon']:
          self.pglog(f"ipinfo: {ip} - bogon, use {self.GIP}", self.LOGWRN)
@@ -195,7 +311,10 @@ class PgIPInfo(PgUtil, PgDBI):
 
    def set_ipinfo(self, ip, ipopt = True):
       """If ipopt is True; otherwise, use module geoip2."""
+      ip = self.normalize_ip(ip)
+      if not ip: return None
       if ip in self.IPRECS: return self.IPRECS[ip]
+      if ipopt and not self.IPINFO['QUOTA']: ipopt = False   # quota used up, geoip2 only
       pgrec = self.pgget('ipinfo', '*', "ip = '{}'".format(ip))
       if not pgrec or ipopt and pgrec['stat_flag'] == 'M':
          record = self.get_ipinfo_record(ip) if ipopt else None

{rda_python_metrics-3.0.5 → rda_python_metrics-3.0.7/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 3.0.5
+Version: 3.0.7
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics