rda-python-metrics 1.0.18__tar.gz → 1.0.20__tar.gz

{rda_python_metrics-1.0.18/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.18
+Version: 1.0.20
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.18 → rda_python_metrics-1.0.20}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.18"
+version = "1.0.20"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_metrics-1.0.18 → rda_python_metrics-1.0.20}/src/rda_python_metrics/PgIPInfo.py

@@ -13,6 +13,7 @@
 #
 ###############################################################################
 #
+import re
 import geoip2.database as geodb
 import ipinfo
 import socket
@@ -54,8 +55,6 @@ def dns_to_ip(dmname, type = 'A'):
    
    ipdns = get_dns_resolver()
 
-   result = []
-
    try:
       answers = ipdns.resolve(dmname, type)
       return [str(rdata) for rdata in answers]
@@ -113,14 +112,11 @@ def get_ipinfo_record(ip):
    if not IPDB: set_ipinfo_database()
    try:
       iprec = IPDB.getDetails(ip).all
-      if 'hostname' not in iprec:
-         PgLOG.pglog("ipinfo: {} - ip address is not in the database".format(ip), PgLOG.LOGERR)
-         return None
    except Exception as e:
       PgLOG.pglog("ipinfo: {} - {}".format(ip, str(e)), PgLOG.LOGWRN)
       return None
       
-   record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip}
+   record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip, 'org_type' : '-'}
    if 'hostname' in iprec:
       record['hostname'] = iprec['hostname']
       record['org_type'] = PgDBI.get_org_type(None, record['hostname'])
@@ -128,7 +124,7 @@ def get_ipinfo_record(ip):
    record['lon'] = float(iprec['longitude']) if iprec['longitude'] else 0
    if 'org' in iprec: record['org_name'] = iprec['org']
    record['country'] = get_country_record_code(iprec, 'country_name')
-   if 'region' in iprec: record['region'] = PgLOG.convert_chars(iprec['region'])
+   record['region'] = PgLOG.convert_chars(iprec['region']) if 'region' in iprec else None
    if 'city' in iprec: record['city'] = PgLOG.convert_chars(iprec['city'])
    if 'postal' in iprec: record['postal'] =  iprec['postal']
    record['timezone'] = iprec['timezone']
@@ -156,17 +152,16 @@ def get_geoip2_record(ip):
       PgLOG.pglog("geoip2: {} - {}".format(ip, str(e)), PgLOG.LOGWRN)
       return None
 
-   record = {'ip' : ip, 'stat_flag' : 'M'}
+   record = {'ip' : ip, 'stat_flag' : 'M', 'org_type' : '-'}
    record['lat'] = float(city.location.latitude) if city.location.latitude else 0
    record['lon'] = float(city.location.longitude) if city.location.longitude else 0
    record['country'] = get_country_name_code(city.country.name)
    record['city'] = PgLOG.convert_chars(city.city.name)
-   if city.subdivisions.most_specific.name: record['region'] = PgLOG.convert_chars(city.subdivisions.most_specific.name)
+   record['region'] = PgLOG.convert_chars(city.subdivisions.most_specific.name) if city.subdivisions.most_specific.name else None
    record['postal'] =  city.postal.code
    record['timezone'] = city.location.time_zone
    record['hostname'] = ip
-   record['org_type'] = '-'
-   record['ipinfo'] = json.dumps(city.__dict__)
+   record['ipinfo'] = json.dumps(object_to_dict(city))
 
    try:
       hostrec = socket.gethostbyaddr(ip)
@@ -178,6 +173,20 @@ def get_geoip2_record(ip):
 
    return record
 
+#
+# change an object to dict recursively
+#
+def object_to_dict(obj):
+    if hasattr(obj, "__dict__"):
+        result = {}
+        for key, value in obj.__dict__.items():
+            result[key] = object_to_dict(value)
+        return result
+    elif isinstance(obj, list):
+        return [object_to_dict(item) for item in obj]
+    else:
+        return obj
+
 #
 # update wuser.email for hostname changed
 #
@@ -237,17 +246,36 @@ def get_update_record(nrec, orec):
          record[fld] = nrec[fld]
    return record
 
-# return wuser record upon success, None otherwise
-def get_wuser_record(ip, date):
+#
+# fill the missing info for given ip
+#
+def get_missing_ipinfo(ip, email = None):
+
+   if not ip:
+      if email and '@' in email: ip = dns_to_ip(email.split('@')[1])
+      if not ip: return None
 
    ipinfo = set_ipinfo(ip)
-   if not ipinfo: return None
+   if ipinfo:
+      record = {'org_type' : ipinfo['org_type'],
+                'country' : ipinfo['country'],
+                'region' : ipinfo['region']}
+      if not email or re.search(r'-$', email):
+         record['email'] =  'unknown@' + ipinfo['hostname']
+      else:
+         record['email'] = email
+      return record
+   else:
+      return None
+
+
+# return wuser record upon success, None otherwise
+def get_wuser_record(ip, date, email = None):
+
+   record = get_missing_ipinfo(ip, email)
+   if not record: return None
 
-   record = {'org_type' : ipinfo['org_type'],
-             'country' : ipinfo['country'],
-             'region' : ipinfo['region']}
-   email = 'unknown@' + ipinfo['hostname']
-   emcond = "email = '{}'".format(email)
+   emcond = "email = '{}'".format(record['email'])
    flds = 'wuid, email, org_type, country, region, start_date'   
    pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
    if pgrec:
@@ -257,7 +285,6 @@ def get_wuser_record(ip, date):
       return pgrec
 
    # now add one in
-   record['email'] = email
    record['stat_flag'] = 'A'
    record['start_date'] = date
    wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)

{rda_python_metrics-1.0.18 → rda_python_metrics-1.0.20}/src/rda_python_metrics/fillcdgusage.py

@@ -72,7 +72,6 @@ DSIDS = {
 ALLIDS = list(DSIDS.keys())
 
 WFILES = {}
-WUSERS = {}
 
 #
 # main function to run this program
@@ -276,12 +275,12 @@ def fill_cdg_usages(dsids, dranges):
                   trecs[tkey]['size'] += dsize
                   trecs[tkey]['fcount'] += 1
                else:
-                  wurec =  get_wuser_record(ip, cdate)
-                  if not wurec: continue
+                  iprec =  PgIPInfo.get_missing_ipinfo(ip)
+                  if not iprec: continue
                   trecs[tkey] = {'ip' : ip, 'dsid' : dsid, 'date' : cdate, 'time' : time, 'quarter' : quarter,
                                  'size' : dsize, 'fcount' : 1, 'method' : method, 'etype' : etype,
-                                 'engine' : engine, 'org_type' : wurec['org_type'], 'country' : wurec['country'],
-                                 'email' : wurec['email']}
+                                 'engine' : engine, 'org_type' : iprec['org_type'], 'country' : iprec['country'],
+                                 'region' : iprec['region'], 'email' : iprec['email']}
             else:
                # web usage
                fsize = pgrec['dataset_file_size']
@@ -347,6 +346,7 @@ def add_tds_allusage(year, logrec):
    pgrec['email'] = logrec['email']
    pgrec['org_type'] = logrec['org_type']
    pgrec['country'] = logrec['country']
+   pgrec['region'] = logrec['region']
    pgrec['dsid'] = logrec['dsid']
    pgrec['date'] = logrec['date']
    pgrec['quarter'] = logrec['quarter']
@@ -367,7 +367,7 @@ def add_webfile_usage(year, logrec):
    cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(logrec['wid'], logrec['method'], cdate, logrec['time'])
    if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
 
-   wurec =  get_wuser_record(ip, cdate)
+   wurec =  PgIPInfo.get_wuser_record(ip, cdate)
    if not wurec: return 0
 
    record = {'wid' : logrec['wid'], 'dsid' : logrec['dsid']}
@@ -391,6 +391,7 @@ def add_web_allusage(year, logrec, wurec):
    pgrec['email'] = wurec['email']
    pgrec['org_type'] = wurec['org_type']
    pgrec['country'] = wurec['country']
+   pgrec['region'] = wurec['region']
    pgrec['dsid'] = logrec['dsid']
    pgrec['date'] = logrec['date']
    pgrec['quarter'] = logrec['quarter']
@@ -431,39 +432,6 @@ def get_wfile_record(dsids, wfile):
       WFILES[wkey] = pgrec
    return pgrec
 
-# return wuser record upon success, None otherwise
-def get_wuser_record(ip, date = None):
-
-   if ip in WUSERS: return WUSERS[ip]
-
-   ipinfo = PgIPInfo.set_ipinfo(ip)
-   if not ipinfo: return None
-
-   record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
-   email = 'unknown@' + ipinfo['hostname']
-   emcond = "email = '{}'".format(email)
-   flds = 'wuid, email, org_type, country, start_date'   
-   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
-   if pgrec:
-      if date and PgUtil.diffdate(pgrec['start_date'], date) > 0:
-         pgrec['start_date'] = record['start_date'] = date
-         PgDBI.pgupdt('wuser', record, emcond)
-      WUSERS[ip] = pgrec
-      return pgrec
-
-   # now add one in
-   record['email'] = email
-   record['stat_flag'] = 'A'
-   record['start_date'] = date
-   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
-   if wuid:
-      record['wuid'] = wuid
-      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
-      WUSERS[ip] = record
-      return record
-
-   return None
-
 #
 # call main() to start program
 #

{rda_python_metrics-1.0.18 → rda_python_metrics-1.0.20}/src/rda_python_metrics/fillipinfo.py

@@ -126,7 +126,7 @@ def fix_allusage_records(date):
    cnt = len(pgrecs['ip']) if pgrecs else 0
    mcnt = 0
    for i in range(cnt):
-      record = get_missing_info(pgrecs['ip'][i], pgrecs['email'][i])
+      record = PgIPInfo.get_missing_ipinfo(pgrecs['ip'][i], pgrecs['email'][i])
       if record:
          mcnt += PgDBI.pgupdt(table, record, "aidx = '{}'".format(pgrecs['aidx'][i]))
 
@@ -145,7 +145,7 @@ def fix_tdsusage_records(date):
    mcnt = 0
    for i in range(cnt):
       ip = pgrecs['ip'][i]
-      record = get_missing_info(ip, pgrecs['email'][i])
+      record = PgIPInfo.get_missing_ipinfo(ip, pgrecs['email'][i])
       if record:
          cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, pgrecs['time'][i], ip)
          mcnt += PgDBI.pgupdt(table, record, cond)
@@ -164,7 +164,7 @@ def fix_codusage_records(date):
    cnt = len(pgrecs['ip']) if pgrecs else 0
    mcnt = 0
    for i in range(cnt):
-      record = get_missing_info(pgrecs['ip'][i], pgrecs['email'][i])
+      record = PgIPInfo.get_missing_ipinfo(pgrecs['ip'][i], pgrecs['email'][i])
       if record:
          mcnt += PgDBI.pgupdt(table, record, "codidx = '{}'".format(pgrecs['codidx'][i]))
 
@@ -184,10 +184,7 @@ def fix_wuser_records(date):
    for i in range(cnt):
       ip = pgrecs['ip'][i]
       email = pgrecs['email'][i]
-      if not ip:
-         if email and '@' in email: ip = PgIPInfo.dns_to_ip(email.split('@')[1])
-         if not ip: continue
-      record = get_missing_info(ip, email)
+      record = PgIPInfo.get_missing_ipinfo(ip, email)
       if record:
          mcnt += PgDBI.pgupdt(table, record, "wuid = '{}'".format(pgrecs['wuid'][i]))
 
@@ -213,22 +210,6 @@ def fix_ipinfo_records(date):
 
    return mcnt
 
-#
-# fill the missing info for given ip
-#
-def get_missing_info(ip, email):
-
-   ipinfo = PgIPInfo.set_ipinfo(ip)
-   if ipinfo:
-      record = {'org_type' : ipinfo['org_type'],
-                'country' : ipinfo['country'],
-                'region' : ipinfo['region']}
-      if not email or re.search(r'-$', email):
-         record['email'] =  'unknown@' + ipinfo['hostname']
-      return record
-   else:
-      return None
-
 #
 # call main() to start program
 #

{rda_python_metrics-1.0.18 → rda_python_metrics-1.0.20}/src/rda_python_metrics/viewallusage.usg

@@ -6,7 +6,7 @@
                      [-A RowLimit] [-c CountryCodes] [-d DateList] \
                      [-D StartDate [EndDate]] [-e EMailList] -h \
                      [-E StartNoticeDate [EndNoticeDate]] [-f DataFormats] \
-                     [-g SourceGroups] [-I EmailIDList] [-k RegionLost] \
+                     [-g SourceGroups] [-I EmailIDList] [-k RegionList] \
                      [-m MonthList] [-M MethodList] [-N MinNumberRead [MaxNumberRead]] \
                      [-o OrganizationTypes] [-q QuaterList] [-s MinSize [MaxSize]] \
                      [-S SpecialistLoginNames] [-t DatasetList] \

{rda_python_metrics-1.0.18 → rda_python_metrics-1.0.20}/src/rda_python_metrics/viewtdsusage.usg

@@ -7,7 +7,7 @@
                      [-D StartDate [EndDate]] [-e EMailList] -h \
                      [-E StartNoticeDate [EndNoticeDate]] \
                      [-f EndTypeFlags] [-i IPAddresses] [-I EmailIDList] \
-                     [-RegionNamess] [-m MonthList] [-M AccessMethods]  \
+                     [-k RegionNames] [-m MonthList] [-M AccessMethods]  \
                      [-N MinNumberRead [MaxNumberRead]] \
                      [-o OrganizationTypes] \
                      [-q QuaterList] [-s MinSize [MaxSize]] \

{rda_python_metrics-1.0.18 → rda_python_metrics-1.0.20/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.18
+Version: 1.0.20
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics