rda-python-metrics 1.0.17__tar.gz → 1.0.20__tar.gz

{rda_python_metrics-1.0.17/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.17
+Version: 1.0.20
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
@@ -16,6 +16,7 @@ Requires-Dist: rda_python_setuid
 Requires-Dist: geoip2
 Requires-Dist: ipinfo
 Requires-Dist: httplib2
+Requires-Dist: dnspython
 Dynamic: license-file
 
 RDA Python Package to gather and view data usage metrics.

{rda_python_metrics-1.0.17 → rda_python_metrics-1.0.20}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.17"
+version = "1.0.20"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]
@@ -24,7 +24,8 @@ dependencies = [
   "rda_python_setuid",
   "geoip2",
   "ipinfo",
-  "httplib2"
+  "httplib2",
+  "dnspython"
 ]
 
 [tool.pytest.ini_options]

{rda_python_metrics-1.0.17 → rda_python_metrics-1.0.20}/src/rda_python_metrics/PgIPInfo.py

@@ -13,9 +13,12 @@
 #
 ###############################################################################
 #
+import re
 import geoip2.database as geodb
 import ipinfo
 import socket
+import dns.resolver
+import json
 from rda_python_common import PgLOG
 from rda_python_common import PgDBI
 from rda_python_common import PgUtil
@@ -28,11 +31,42 @@ IPINFO = {
    'IPADD'  : 0
 }
 
+IPDNS = None
 IPDB = None
 G2DB = None
 IPRECS = {}
 COUNTRIES = {}
 
+#
+# get save a global dns.resolver.Resolver object
+#
+def get_dns_resolver(forceget = False):
+
+   global IPDNS
+
+   if forceget or not IPDNS: IPDNS = dns.resolver.Resolver()
+   
+   return IPDNS
+
+#
+# Resolve a domain name to an IP address (A record)
+#
+def dns_to_ip(dmname, type = 'A'):
+   
+   ipdns = get_dns_resolver()
+
+   try:
+      answers = ipdns.resolve(dmname, type)
+      return [str(rdata) for rdata in answers]
+   except dns.resolver.NXDOMAIN:
+      PgLOG.pglog(f"{dmname}: the domain name does not exist", PgLOG.LOGERR)
+   except dns.resolver.Timeout:
+      PgLOG.pglog(f"{dmname}: the domain name request timed out", PgLOG.LOGERR)
+   except dns.exception.DNSException as e:
+      PgLOG.pglog(f"{dmname}: error domain name request: {e}", PgLOG.LOGERR)
+
+   return None
+
 #
 # Get country token name for given two-character domain id
 #
@@ -59,6 +93,17 @@ def set_ipinfo_database():
    except Exception as e:
       PgLOG.pglog('ipinfo: ' + str(e), PgLOG.LGEREX)
 
+#
+# get a ipinfo record for given domain
+#
+def domain_ipinfo_record(dmname):
+
+   ips = dns_to_ip(dmname)
+   
+   if ips: return set_ipinfo(ips[0])
+
+   return None
+
 #
 # get a ipinfo record for given ip address
 #
@@ -67,14 +112,11 @@ def get_ipinfo_record(ip):
    if not IPDB: set_ipinfo_database()
    try:
       iprec = IPDB.getDetails(ip).all
-      if 'hostname' not in iprec:
-         PgLOG.pglog("ipinfo: {} - ip address is not in the database".format(ip), PgLOG.LOGERR)
-         return None
    except Exception as e:
       PgLOG.pglog("ipinfo: {} - {}".format(ip, str(e)), PgLOG.LOGWRN)
       return None
       
-   record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip}
+   record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip, 'org_type' : '-'}
    if 'hostname' in iprec:
       record['hostname'] = iprec['hostname']
       record['org_type'] = PgDBI.get_org_type(None, record['hostname'])
@@ -82,9 +124,11 @@ def get_ipinfo_record(ip):
    record['lon'] = float(iprec['longitude']) if iprec['longitude'] else 0
    if 'org' in iprec: record['org_name'] = iprec['org']
    record['country'] = get_country_record_code(iprec, 'country_name')
+   record['region'] = PgLOG.convert_chars(iprec['region']) if 'region' in iprec else None
    if 'city' in iprec: record['city'] = PgLOG.convert_chars(iprec['city'])
    if 'postal' in iprec: record['postal'] =  iprec['postal']
    record['timezone'] = iprec['timezone']
+   record['ipinfo'] = json.dumps(iprec)
 
    return record
 
@@ -108,15 +152,16 @@ def get_geoip2_record(ip):
       PgLOG.pglog("geoip2: {} - {}".format(ip, str(e)), PgLOG.LOGWRN)
       return None
 
-   record = {'ip' : ip, 'stat_flag' : 'M'}
+   record = {'ip' : ip, 'stat_flag' : 'M', 'org_type' : '-'}
    record['lat'] = float(city.location.latitude) if city.location.latitude else 0
    record['lon'] = float(city.location.longitude) if city.location.longitude else 0
    record['country'] = get_country_name_code(city.country.name)
    record['city'] = PgLOG.convert_chars(city.city.name)
+   record['region'] = PgLOG.convert_chars(city.subdivisions.most_specific.name) if city.subdivisions.most_specific.name else None
    record['postal'] =  city.postal.code
    record['timezone'] = city.location.time_zone
    record['hostname'] = ip
-   record['org_type'] = '-'
+   record['ipinfo'] = json.dumps(object_to_dict(city))
 
    try:
       hostrec = socket.gethostbyaddr(ip)
@@ -128,6 +173,20 @@ def get_geoip2_record(ip):
 
    return record
 
+#
+# change an object to dict recursively
+#
+def object_to_dict(obj):
+    if hasattr(obj, "__dict__"):
+        result = {}
+        for key, value in obj.__dict__.items():
+            result[key] = object_to_dict(value)
+        return result
+    elif isinstance(obj, list):
+        return [object_to_dict(item) for item in obj]
+    else:
+        return obj
+
 #
 # update wuser.email for hostname changed
 #
@@ -160,7 +219,7 @@ def update_ipinfo_record(record, pgrec = None):
 # set ip info into table ipinfo from python module ipinfo
 # if ipopt is True; otherwise, use module geoip2 
 #
-def set_ipinfo(ip, ipopt = False):
+def set_ipinfo(ip, ipopt = True):
 
    if ip in IPRECS:
       pgrec = IPRECS[ip]
@@ -169,8 +228,8 @@ def set_ipinfo(ip, ipopt = False):
       pgrec = PgDBI.pgget('ipinfo', '*', "ip = '{}'".format(ip))
 
    if not pgrec or ipopt and pgrec['stat_flag'] == 'M':
-      record = None if ipopt else get_geoip2_record(ip)
-      if not (record and 'hostname' in record): record = get_ipinfo_record(ip)
+      record = get_ipinfo_record(ip) if ipopt else None
+      if not record: record = get_geoip2_record(ip)
       if record and update_ipinfo_record(record, pgrec): pgrec = record
    
    IPRECS[ip] = pgrec
@@ -186,3 +245,52 @@ def get_update_record(nrec, orec):
       if nrec[fld] != orec[fld]:
          record[fld] = nrec[fld]
    return record
+
+#
+# fill the missing info for given ip
+#
+def get_missing_ipinfo(ip, email = None):
+
+   if not ip:
+      if email and '@' in email: ip = dns_to_ip(email.split('@')[1])
+      if not ip: return None
+
+   ipinfo = set_ipinfo(ip)
+   if ipinfo:
+      record = {'org_type' : ipinfo['org_type'],
+                'country' : ipinfo['country'],
+                'region' : ipinfo['region']}
+      if not email or re.search(r'-$', email):
+         record['email'] =  'unknown@' + ipinfo['hostname']
+      else:
+         record['email'] = email
+      return record
+   else:
+      return None
+
+
+# return wuser record upon success, None otherwise
+def get_wuser_record(ip, date, email = None):
+
+   record = get_missing_ipinfo(ip, email)
+   if not record: return None
+
+   emcond = "email = '{}'".format(record['email'])
+   flds = 'wuid, email, org_type, country, region, start_date'   
+   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
+   if pgrec:
+      if PgUtil.diffdate(pgrec['start_date'], date) > 0:
+         pgrec['start_date'] = record['start_date'] = date
+         PgDBI.pgupdt('wuser', record, emcond)
+      return pgrec
+
+   # now add one in
+   record['stat_flag'] = 'A'
+   record['start_date'] = date
+   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
+   if wuid:
+      record['wuid'] = wuid
+      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
+      return record
+
+   return None

{rda_python_metrics-1.0.17 → rda_python_metrics-1.0.20}/src/rda_python_metrics/fillawsusage.py

@@ -201,7 +201,7 @@ def add_file_usage(year, logrec):
    cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
    if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
 
-   wurec =  get_wuser_record(logrec['ip'], logrec['date'])
+   wurec =  PgIPInfo.get_wuser_record(logrec['ip'], logrec['date'])
    if not wurec: return 0
    record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
    record['wuid_read'] = wurec['wuid']
@@ -220,7 +220,8 @@ def add_file_usage(year, logrec):
 
 def add_to_allusage(year, logrec, wurec):
 
-   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'], 'country' : wurec['country']}
+   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'],
+            'country' : wurec['country'], 'region' : wurec['region']}
    pgrec['dsid'] = logrec['dsid']
    pgrec['date'] = logrec['date']
    pgrec['quarter'] = logrec['quarter']
@@ -247,35 +248,6 @@ def get_wfile_wid(dsid, wfile):
 
    return pgrec
 
-# return wuser record upon success, None otherwise
-def get_wuser_record(ip, date):
-
-   ipinfo = PgIPInfo.set_ipinfo(ip)
-   if not ipinfo: return None
-
-   record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
-   email = 'unknown@' + ipinfo['hostname']
-   emcond = "email = '{}'".format(email)
-   flds = 'wuid, email, org_type, country, start_date'   
-   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
-   if pgrec:
-      if PgUtil.diffdate(pgrec['start_date'], date) > 0:
-         pgrec['start_date'] = record['start_date'] = date
-         PgDBI.pgupdt('wuser', record, emcond)
-      return pgrec
-
-   # now add one in
-   record['email'] = email
-   record['stat_flag'] = 'A'
-   record['start_date'] = date
-   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
-   if wuid:
-      record['wuid'] = wuid
-      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
-      return record
-
-   return None
-
 #
 # call main() to start program
 #

{rda_python_metrics-1.0.17 → rda_python_metrics-1.0.20}/src/rda_python_metrics/fillcdgusage.py

@@ -52,13 +52,26 @@ DSIDS = {
    'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
    'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
    'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
-   'ucar.cgd.ccsm4.pliomip2' : ['d651037']
+   'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
+   # new added
+   'ucar.cgd.cesm2-waccm.s2s_hindcasts': ['d651040'],
+   'ucar.cgd.CESM1.3_SH_storm_tracks': ['d651044'],
+   'ucar.cgd.cesm2.waccm6.ssp245': ['d651045'],
+   'ucar.cgd.cesm2.CESM21-CISM2-JG-BG': ['d651046'],
+   'ucar.cgd.ccsm4.TC-CESM': ['d651047'],
+   'ucar.cgd.cesm2.ISSI_OSSE': ['d651048'],
+   'ucar.cgd.ccsm4.SOcean_Eddies_mclong': ['d651049'],
+   'trace': ['d651050'],
+   'ucar.cgd.cesm2.waccm.solar': ['d651051'],
+   'ucar.cgd.ccsm4.CESM1-CCSM4_mid-Pliocene' : ['d651042'],
+   'ucar.cgd.ccsm4.PaleoIF' : ['d651052'],
+   'ucar.cgd.ccsm4.b.e11.B20LE_fixedO3' : ['d651053'],
+   'ucar.cgd.cesm2.single.forcing.large.ensemble' : ['d651055']
 }
 
 ALLIDS = list(DSIDS.keys())
 
 WFILES = {}
-WUSERS = {}
 
 #
 # main function to run this program
@@ -262,12 +275,12 @@ def fill_cdg_usages(dsids, dranges):
                   trecs[tkey]['size'] += dsize
                   trecs[tkey]['fcount'] += 1
                else:
-                  wurec =  get_wuser_record(ip, cdate)
-                  if not wurec: continue
+                  iprec =  PgIPInfo.get_missing_ipinfo(ip)
+                  if not iprec: continue
                   trecs[tkey] = {'ip' : ip, 'dsid' : dsid, 'date' : cdate, 'time' : time, 'quarter' : quarter,
                                  'size' : dsize, 'fcount' : 1, 'method' : method, 'etype' : etype,
-                                 'engine' : engine, 'org_type' : wurec['org_type'], 'country' : wurec['country'],
-                                 'email' : wurec['email']}
+                                 'engine' : engine, 'org_type' : iprec['org_type'], 'country' : iprec['country'],
+                                 'region' : iprec['region'], 'email' : iprec['email']}
             else:
                # web usage
                fsize = pgrec['dataset_file_size']
@@ -333,6 +346,7 @@ def add_tds_allusage(year, logrec):
    pgrec['email'] = logrec['email']
    pgrec['org_type'] = logrec['org_type']
    pgrec['country'] = logrec['country']
+   pgrec['region'] = logrec['region']
    pgrec['dsid'] = logrec['dsid']
    pgrec['date'] = logrec['date']
    pgrec['quarter'] = logrec['quarter']
@@ -353,7 +367,7 @@ def add_webfile_usage(year, logrec):
    cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(logrec['wid'], logrec['method'], cdate, logrec['time'])
    if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
 
-   wurec =  get_wuser_record(ip, cdate)
+   wurec =  PgIPInfo.get_wuser_record(ip, cdate)
    if not wurec: return 0
 
    record = {'wid' : logrec['wid'], 'dsid' : logrec['dsid']}
@@ -377,6 +391,7 @@ def add_web_allusage(year, logrec, wurec):
    pgrec['email'] = wurec['email']
    pgrec['org_type'] = wurec['org_type']
    pgrec['country'] = wurec['country']
+   pgrec['region'] = wurec['region']
    pgrec['dsid'] = logrec['dsid']
    pgrec['date'] = logrec['date']
    pgrec['quarter'] = logrec['quarter']
@@ -417,39 +432,6 @@ def get_wfile_record(dsids, wfile):
       WFILES[wkey] = pgrec
    return pgrec
 
-# return wuser record upon success, None otherwise
-def get_wuser_record(ip, date = None):
-
-   if ip in WUSERS: return WUSERS[ip]
-
-   ipinfo = PgIPInfo.set_ipinfo(ip)
-   if not ipinfo: return None
-
-   record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
-   email = 'unknown@' + ipinfo['hostname']
-   emcond = "email = '{}'".format(email)
-   flds = 'wuid, email, org_type, country, start_date'   
-   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
-   if pgrec:
-      if date and PgUtil.diffdate(pgrec['start_date'], date) > 0:
-         pgrec['start_date'] = record['start_date'] = date
-         PgDBI.pgupdt('wuser', record, emcond)
-      WUSERS[ip] = pgrec
-      return pgrec
-
-   # now add one in
-   record['email'] = email
-   record['stat_flag'] = 'A'
-   record['start_date'] = date
-   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
-   if wuid:
-      record['wuid'] = wuid
-      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
-      WUSERS[ip] = record
-      return record
-
-   return None
-
 #
 # call main() to start program
 #

{rda_python_metrics-1.0.17 → rda_python_metrics-1.0.20}/src/rda_python_metrics/fillcodusage.py

@@ -22,6 +22,7 @@ from rda_python_common import PgLOG
 from rda_python_common import PgUtil
 from rda_python_common import PgFile
 from rda_python_common import PgDBI
+from . import PgIPInfo
 
 # the define options for gathering COD data usage, one at a time
 MONTH = 0x02  # fet COD data usages for given months
@@ -182,14 +183,20 @@ def add_usage_records(records, date):
       if PgDBI.pgget(USAGE['PGTBL'], '', "aid = '{}' AND date = '{}'".format(aid, date), PgLOG.LGEREX): continue
       record = records[aid]
       if record['email'] == '-':
-         record['org_type'] = record['country'] = '-'
+         wurec = PgIPInfo.get_wuser_record(record['ip'], date)
+         if not wurec: continue
+         record['org_type'] = wurec['org_type']
+         record['country'] = wurec['country']
+         record['region'] = wurec['region']
+         record['email'] = 'unknown@' + wurec['hostname']
       else:
          wuid = PgDBI.check_wuser_wuid(record['email'], date)
-         if not wuid: next
-         pgrec = PgDBI.pgget("wuser",  "org_type, country", "wuid = {}".format(wuid), PgLOG.LGWNEX)
+         if not wuid: continue
+         pgrec = PgDBI.pgget("wuser",  "org_type, country, region", "wuid = {}".format(wuid), PgLOG.LGWNEX)
          if not pgrec: continue
          record['org_type'] = pgrec['org_type']
          record['country'] = pgrec['country']
+         record['region'] = pgrec['region']
    
       record['date'] = date
       record['time'] = USERS[aid]['btime']

{rda_python_metrics-1.0.17 → rda_python_metrics-1.0.20}/src/rda_python_metrics/fillglobususage.py

@@ -203,7 +203,7 @@ def add_file_usage(year, logrec):
    cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
    if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
 
-   wurec =  get_wuser_record(logrec['ip'], logrec['date'])
+   wurec =  PgIPInfo.get_wuser_record(logrec['ip'], logrec['date'])
    if not wurec: return 0
    record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
    record['wuid_read'] = wurec['wuid']
@@ -222,7 +222,8 @@ def add_file_usage(year, logrec):
 
 def add_to_allusage(year, logrec, wurec):
 
-   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'], 'country' : wurec['country']}
+   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'],
+            'country' : wurec['country'], 'region' : wurec['region']}
    pgrec['dsid'] = logrec['dsid']
    pgrec['date'] = logrec['date']
    pgrec['quarter'] = logrec['quarter']
@@ -252,35 +253,6 @@ def get_wfile_wid(dsid, wfile):
 
    return pgrec
 
-# return wuser record upon success, None otherwise
-def get_wuser_record(ip, date):
-
-   ipinfo = PgIPInfo.set_ipinfo(ip)
-   if not ipinfo: return None
-
-   record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
-   email = 'unknown@' + ipinfo['hostname']
-   emcond = "email = '{}'".format(email)
-   flds = 'wuid, email, org_type, country, start_date'   
-   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
-   if pgrec:
-      if PgUtil.diffdate(pgrec['start_date'], date) > 0:
-         pgrec['start_date'] = record['start_date'] = date
-         PgDBI.pgupdt('wuser', record, emcond)
-      return pgrec
-
-   # now add one in
-   record['email'] = email
-   record['stat_flag'] = 'A'
-   record['start_date'] = date
-   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
-   if wuid:
-      record['wuid'] = wuid
-      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
-      return record
-
-   return None
-
 #
 # call main() to start program
 #

{rda_python_metrics-1.0.17 → rda_python_metrics-1.0.20}/src/rda_python_metrics/fillipinfo.py

@@ -32,7 +32,7 @@ MULTI = (MONTH|YEARS)
 SINGL = (NDAYS)
 
 IPINFO = {
-   'USGTBL'  : ['ipinfo', 'allusage', 'tdsusage'],
+   'USGTBL'  : ['ipinfo', 'wuser', 'allusage', 'codusage', 'tdsusage'],
    'CDATE' : PgUtil.curdate(),
 }
 
@@ -120,20 +120,14 @@ def fix_allusage_records(date):
    ms = re.match(r'^(\d+)-', date)
    year = ms.group(1)
    table = 'allusage_' + year
-   cond = "date = '{}' and org_type = '-'".format(date)
+   cond = "date = '{}' AND region IS NULL".format(date)
    pgrecs = PgDBI.pgmget(table, 'aidx, email, ip', cond, PgLOG.LGEREX)
    if not pgrecs: return 0
    cnt = len(pgrecs['ip']) if pgrecs else 0
    mcnt = 0
    for i in range(cnt):
-      ip = pgrecs['ip'][i]
-      email = pgrecs['email'][i]
-      ipinfo = PgIPInfo.set_ipinfo(ip)
-      if ipinfo:
-         record = {'org_type' : ipinfo['org_type'],
-                   'country' : ipinfo['country']}
-         if not email or re.search(r'-$', email):
-            record['email'] =  'unknown@' + ipinfo['hostname']
+      record = PgIPInfo.get_missing_ipinfo(pgrecs['ip'][i], pgrecs['email'][i])
+      if record:
          mcnt += PgDBI.pgupdt(table, record, "aidx = '{}'".format(pgrecs['aidx'][i]))
 
    s = 's' if cnt > 1 else ''
@@ -144,17 +138,16 @@ def fix_allusage_records(date):
 def fix_tdsusage_records(date):
 
    table = 'tdsusage'
-   cond = "date = '{}' and org_type = '-'".format(date)
-   pgrecs = PgDBI.pgmget(table, 'time, ip', cond, PgLOG.LGEREX)
+   cond = "date = '{}' AND region IS NULL".format(date)
+   pgrecs = PgDBI.pgmget(table, 'time, email, ip', cond, PgLOG.LGEREX)
    if not pgrecs: return 0
    cnt = len(pgrecs['ip']) if pgrecs else 0
    mcnt = 0
    for i in range(cnt):
-      ipinfo = PgIPInfo.set_ipinfo(pgrecs['ip'][i])
-      if ipinfo:
-         record = {'org_type' : ipinfo['org_type'],
-                   'country' : ipinfo['country']}
-         cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, pgrecs['time'][i], pgrecs['ip'][i])
+      ip = pgrecs['ip'][i]
+      record = PgIPInfo.get_missing_ipinfo(ip, pgrecs['email'][i])
+      if record:
+         cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, pgrecs['time'][i], ip)
          mcnt += PgDBI.pgupdt(table, record, cond)
 
    s = 's' if cnt > 1 else ''
@@ -162,16 +155,54 @@ def fix_tdsusage_records(date):
 
    return mcnt
 
+def fix_codusage_records(date):
+
+   table = 'codusage'
+   cond = "date = '{}' AND region IS NULL".format(date)
+   pgrecs = PgDBI.pgmget(table, 'codidx, email, ip', cond, PgLOG.LGEREX)
+   if not pgrecs: return 0
+   cnt = len(pgrecs['ip']) if pgrecs else 0
+   mcnt = 0
+   for i in range(cnt):
+      record = PgIPInfo.get_missing_ipinfo(pgrecs['ip'][i], pgrecs['email'][i])
+      if record:
+         mcnt += PgDBI.pgupdt(table, record, "codidx = '{}'".format(pgrecs['codidx'][i]))
+
+   s = 's' if cnt > 1 else ''
+   PgLOG.pglog("{}: {} of {} record{} updated for {}".format(table, mcnt, cnt, s, date), PgLOG.LOGWRN)
+
+   return mcnt
+
+def fix_wuser_records(date):
+
+   table = 'wuser'
+   cond = "start_date = '{}' AND region IS NULL".format(date)
+   pgrecs = PgDBI.pgmget(table, 'wuid, email, ip', cond, PgLOG.LGEREX)
+   if not pgrecs: return 0
+   cnt = len(pgrecs['ip']) if pgrecs else 0
+   mcnt = 0
+   for i in range(cnt):
+      ip = pgrecs['ip'][i]
+      email = pgrecs['email'][i]
+      record = PgIPInfo.get_missing_ipinfo(ip, email)
+      if record:
+         mcnt += PgDBI.pgupdt(table, record, "wuid = '{}'".format(pgrecs['wuid'][i]))
+
+   s = 's' if cnt > 1 else ''
+   PgLOG.pglog("{}: {} of {} record{} updated for {}".format(table, mcnt, cnt, s, date), PgLOG.LOGWRN)
+
+   return mcnt
+
 def fix_ipinfo_records(date):
 
    table = 'ipinfo'
-   cond = "stat_flag = 'M' and date = '{}'".format(date)
+   cond = "date = '{}' AND region IS NULL".format(date)
    pgrecs = PgDBI.pgmget(table, 'ip', cond, PgLOG.LGEREX)
    if not pgrecs: return 0
    cnt = len(pgrecs['ip']) if pgrecs else 0
    mcnt = 0
    for i in range(cnt):
-      PgIPInfo.set_ipinfo(pgrecs['ip'][i], True)
+      PgIPInfo.set_ipinfo(pgrecs['ip'][i])
 
    mcnt = PgIPInfo.IPINFO['IPUPDT']
    s = 's' if cnt > 1 else ''

{rda_python_metrics-1.0.17 → rda_python_metrics-1.0.20}/src/rda_python_metrics/fillipinfo.usg

@@ -7,7 +7,7 @@
 
       - Option -b, log process information into logfile only;
 
-      - Option -t, table name, ipinfo, allusage, globususage, or tdsusage,
+      - Option -t, table name, ipinfo, wuser, allusage, codusage or tdsusage,
                    to fix IP related information, such organization names/types,
                    emails and country names; 
 

{rda_python_metrics-1.0.17 → rda_python_metrics-1.0.20}/src/rda_python_metrics/filloneorder.py

@@ -87,7 +87,7 @@ def add_one_order(params):
 
 def add_to_allusage(record, year, ctime):
 
-   pgrec = PgDBI.pgget("wuser",  "email, org_type, country",
+   pgrec = PgDBI.pgget("wuser",  "email, org_type, country, region",
                        "wuid = {}".format(record['wuid_request']), PgLOG.LGWNEX)
    if pgrec:
       pgrec['dsid'] = record['dsid']