rda-python-metrics 1.0.18__py3-none-any.whl → 1.0.20__py3-none-any.whl

rda_python_metrics/PgIPInfo.py

@@ -13,6 +13,7 @@
 #
 ###############################################################################
 #
+import re
 import geoip2.database as geodb
 import ipinfo
 import socket
@@ -54,8 +55,6 @@ def dns_to_ip(dmname, type = 'A'):
    
    ipdns = get_dns_resolver()
 
-   result = []
-
    try:
       answers = ipdns.resolve(dmname, type)
       return [str(rdata) for rdata in answers]
@@ -113,14 +112,11 @@ def get_ipinfo_record(ip):
    if not IPDB: set_ipinfo_database()
    try:
       iprec = IPDB.getDetails(ip).all
-      if 'hostname' not in iprec:
-         PgLOG.pglog("ipinfo: {} - ip address is not in the database".format(ip), PgLOG.LOGERR)
-         return None
    except Exception as e:
       PgLOG.pglog("ipinfo: {} - {}".format(ip, str(e)), PgLOG.LOGWRN)
       return None
       
-   record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip}
+   record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip, 'org_type' : '-'}
    if 'hostname' in iprec:
       record['hostname'] = iprec['hostname']
       record['org_type'] = PgDBI.get_org_type(None, record['hostname'])
@@ -128,7 +124,7 @@ def get_ipinfo_record(ip):
    record['lon'] = float(iprec['longitude']) if iprec['longitude'] else 0
    if 'org' in iprec: record['org_name'] = iprec['org']
    record['country'] = get_country_record_code(iprec, 'country_name')
-   if 'region' in iprec: record['region'] = PgLOG.convert_chars(iprec['region'])
+   record['region'] = PgLOG.convert_chars(iprec['region']) if 'region' in iprec else None
    if 'city' in iprec: record['city'] = PgLOG.convert_chars(iprec['city'])
    if 'postal' in iprec: record['postal'] =  iprec['postal']
    record['timezone'] = iprec['timezone']
@@ -156,17 +152,16 @@ def get_geoip2_record(ip):
       PgLOG.pglog("geoip2: {} - {}".format(ip, str(e)), PgLOG.LOGWRN)
       return None
 
-   record = {'ip' : ip, 'stat_flag' : 'M'}
+   record = {'ip' : ip, 'stat_flag' : 'M', 'org_type' : '-'}
    record['lat'] = float(city.location.latitude) if city.location.latitude else 0
    record['lon'] = float(city.location.longitude) if city.location.longitude else 0
    record['country'] = get_country_name_code(city.country.name)
    record['city'] = PgLOG.convert_chars(city.city.name)
-   if city.subdivisions.most_specific.name: record['region'] = PgLOG.convert_chars(city.subdivisions.most_specific.name)
+   record['region'] = PgLOG.convert_chars(city.subdivisions.most_specific.name) if city.subdivisions.most_specific.name else None
    record['postal'] =  city.postal.code
    record['timezone'] = city.location.time_zone
    record['hostname'] = ip
-   record['org_type'] = '-'
-   record['ipinfo'] = json.dumps(city.__dict__)
+   record['ipinfo'] = json.dumps(object_to_dict(city))
 
    try:
       hostrec = socket.gethostbyaddr(ip)
@@ -178,6 +173,20 @@ def get_geoip2_record(ip):
 
    return record
 
+#
+# change an object to dict recursively
+#
+def object_to_dict(obj):
+    if hasattr(obj, "__dict__"):
+        result = {}
+        for key, value in obj.__dict__.items():
+            result[key] = object_to_dict(value)
+        return result
+    elif isinstance(obj, list):
+        return [object_to_dict(item) for item in obj]
+    else:
+        return obj
+
 #
 # update wuser.email for hostname changed
 #
@@ -237,17 +246,36 @@ def get_update_record(nrec, orec):
          record[fld] = nrec[fld]
    return record
 
-# return wuser record upon success, None otherwise
-def get_wuser_record(ip, date):
+#
+# fill the missing info for given ip
+#
+def get_missing_ipinfo(ip, email = None):
+
+   if not ip:
+      if email and '@' in email: ip = dns_to_ip(email.split('@')[1])
+      if not ip: return None
 
    ipinfo = set_ipinfo(ip)
-   if not ipinfo: return None
+   if ipinfo:
+      record = {'org_type' : ipinfo['org_type'],
+                'country' : ipinfo['country'],
+                'region' : ipinfo['region']}
+      if not email or re.search(r'-$', email):
+         record['email'] =  'unknown@' + ipinfo['hostname']
+      else:
+         record['email'] = email
+      return record
+   else:
+      return None
+
+
+# return wuser record upon success, None otherwise
+def get_wuser_record(ip, date, email = None):
+
+   record = get_missing_ipinfo(ip, email)
+   if not record: return None
 
-   record = {'org_type' : ipinfo['org_type'],
-             'country' : ipinfo['country'],
-             'region' : ipinfo['region']}
-   email = 'unknown@' + ipinfo['hostname']
-   emcond = "email = '{}'".format(email)
+   emcond = "email = '{}'".format(record['email'])
    flds = 'wuid, email, org_type, country, region, start_date'   
    pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
    if pgrec:
@@ -257,7 +285,6 @@ def get_wuser_record(ip, date):
       return pgrec
 
    # now add one in
-   record['email'] = email
    record['stat_flag'] = 'A'
    record['start_date'] = date
    wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)

rda_python_metrics/fillcdgusage.py

@@ -72,7 +72,6 @@ DSIDS = {
 ALLIDS = list(DSIDS.keys())
 
 WFILES = {}
-WUSERS = {}
 
 #
 # main function to run this program
@@ -276,12 +275,12 @@ def fill_cdg_usages(dsids, dranges):
                   trecs[tkey]['size'] += dsize
                   trecs[tkey]['fcount'] += 1
                else:
-                  wurec =  get_wuser_record(ip, cdate)
-                  if not wurec: continue
+                  iprec =  PgIPInfo.get_missing_ipinfo(ip)
+                  if not iprec: continue
                   trecs[tkey] = {'ip' : ip, 'dsid' : dsid, 'date' : cdate, 'time' : time, 'quarter' : quarter,
                                  'size' : dsize, 'fcount' : 1, 'method' : method, 'etype' : etype,
-                                 'engine' : engine, 'org_type' : wurec['org_type'], 'country' : wurec['country'],
-                                 'email' : wurec['email']}
+                                 'engine' : engine, 'org_type' : iprec['org_type'], 'country' : iprec['country'],
+                                 'region' : iprec['region'], 'email' : iprec['email']}
             else:
                # web usage
                fsize = pgrec['dataset_file_size']
@@ -347,6 +346,7 @@ def add_tds_allusage(year, logrec):
    pgrec['email'] = logrec['email']
    pgrec['org_type'] = logrec['org_type']
    pgrec['country'] = logrec['country']
+   pgrec['region'] = logrec['region']
    pgrec['dsid'] = logrec['dsid']
    pgrec['date'] = logrec['date']
    pgrec['quarter'] = logrec['quarter']
@@ -367,7 +367,7 @@ def add_webfile_usage(year, logrec):
    cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(logrec['wid'], logrec['method'], cdate, logrec['time'])
    if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
 
-   wurec =  get_wuser_record(ip, cdate)
+   wurec =  PgIPInfo.get_wuser_record(ip, cdate)
    if not wurec: return 0
 
    record = {'wid' : logrec['wid'], 'dsid' : logrec['dsid']}
@@ -391,6 +391,7 @@ def add_web_allusage(year, logrec, wurec):
    pgrec['email'] = wurec['email']
    pgrec['org_type'] = wurec['org_type']
    pgrec['country'] = wurec['country']
+   pgrec['region'] = wurec['region']
    pgrec['dsid'] = logrec['dsid']
    pgrec['date'] = logrec['date']
    pgrec['quarter'] = logrec['quarter']
@@ -431,39 +432,6 @@ def get_wfile_record(dsids, wfile):
       WFILES[wkey] = pgrec
    return pgrec
 
-# return wuser record upon success, None otherwise
-def get_wuser_record(ip, date = None):
-
-   if ip in WUSERS: return WUSERS[ip]
-
-   ipinfo = PgIPInfo.set_ipinfo(ip)
-   if not ipinfo: return None
-
-   record = {'org_type' : ipinfo['org_type'], 'country' : ipinfo['country']}
-   email = 'unknown@' + ipinfo['hostname']
-   emcond = "email = '{}'".format(email)
-   flds = 'wuid, email, org_type, country, start_date'   
-   pgrec = PgDBI.pgget("wuser", flds, emcond, PgLOG.LOGERR)
-   if pgrec:
-      if date and PgUtil.diffdate(pgrec['start_date'], date) > 0:
-         pgrec['start_date'] = record['start_date'] = date
-         PgDBI.pgupdt('wuser', record, emcond)
-      WUSERS[ip] = pgrec
-      return pgrec
-
-   # now add one in
-   record['email'] = email
-   record['stat_flag'] = 'A'
-   record['start_date'] = date
-   wuid = PgDBI.pgadd("wuser", record, PgLOG.LOGERR|PgLOG.AUTOID)
-   if wuid:
-      record['wuid'] = wuid
-      PgLOG.pglog("{} Added as wuid({})".format(email, wuid), PgLOG.LGWNEM)
-      WUSERS[ip] = record
-      return record
-
-   return None
-
 #
 # call main() to start program
 #

rda_python_metrics/fillipinfo.py

@@ -126,7 +126,7 @@ def fix_allusage_records(date):
    cnt = len(pgrecs['ip']) if pgrecs else 0
    mcnt = 0
    for i in range(cnt):
-      record = get_missing_info(pgrecs['ip'][i], pgrecs['email'][i])
+      record = PgIPInfo.get_missing_ipinfo(pgrecs['ip'][i], pgrecs['email'][i])
       if record:
          mcnt += PgDBI.pgupdt(table, record, "aidx = '{}'".format(pgrecs['aidx'][i]))
 
@@ -145,7 +145,7 @@ def fix_tdsusage_records(date):
    mcnt = 0
    for i in range(cnt):
       ip = pgrecs['ip'][i]
-      record = get_missing_info(ip, pgrecs['email'][i])
+      record = PgIPInfo.get_missing_ipinfo(ip, pgrecs['email'][i])
       if record:
          cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, pgrecs['time'][i], ip)
          mcnt += PgDBI.pgupdt(table, record, cond)
@@ -164,7 +164,7 @@ def fix_codusage_records(date):
    cnt = len(pgrecs['ip']) if pgrecs else 0
    mcnt = 0
    for i in range(cnt):
-      record = get_missing_info(pgrecs['ip'][i], pgrecs['email'][i])
+      record = PgIPInfo.get_missing_ipinfo(pgrecs['ip'][i], pgrecs['email'][i])
       if record:
          mcnt += PgDBI.pgupdt(table, record, "codidx = '{}'".format(pgrecs['codidx'][i]))
 
@@ -184,10 +184,7 @@ def fix_wuser_records(date):
    for i in range(cnt):
       ip = pgrecs['ip'][i]
       email = pgrecs['email'][i]
-      if not ip:
-         if email and '@' in email: ip = PgIPInfo.dns_to_ip(email.split('@')[1])
-         if not ip: continue
-      record = get_missing_info(ip, email)
+      record = PgIPInfo.get_missing_ipinfo(ip, email)
       if record:
          mcnt += PgDBI.pgupdt(table, record, "wuid = '{}'".format(pgrecs['wuid'][i]))
 
@@ -213,22 +210,6 @@ def fix_ipinfo_records(date):
 
    return mcnt
 
-#
-# fill the missing info for given ip
-#
-def get_missing_info(ip, email):
-
-   ipinfo = PgIPInfo.set_ipinfo(ip)
-   if ipinfo:
-      record = {'org_type' : ipinfo['org_type'],
-                'country' : ipinfo['country'],
-                'region' : ipinfo['region']}
-      if not email or re.search(r'-$', email):
-         record['email'] =  'unknown@' + ipinfo['hostname']
-      return record
-   else:
-      return None
-
 #
 # call main() to start program
 #

rda_python_metrics/viewallusage.usg

@@ -6,7 +6,7 @@
                      [-A RowLimit] [-c CountryCodes] [-d DateList] \
                      [-D StartDate [EndDate]] [-e EMailList] -h \
                      [-E StartNoticeDate [EndNoticeDate]] [-f DataFormats] \
-                     [-g SourceGroups] [-I EmailIDList] [-k RegionLost] \
+                     [-g SourceGroups] [-I EmailIDList] [-k RegionList] \
                      [-m MonthList] [-M MethodList] [-N MinNumberRead [MaxNumberRead]] \
                      [-o OrganizationTypes] [-q QuaterList] [-s MinSize [MaxSize]] \
                      [-S SpecialistLoginNames] [-t DatasetList] \

rda_python_metrics/viewtdsusage.usg

@@ -7,7 +7,7 @@
                      [-D StartDate [EndDate]] [-e EMailList] -h \
                      [-E StartNoticeDate [EndNoticeDate]] \
                      [-f EndTypeFlags] [-i IPAddresses] [-I EmailIDList] \
-                     [-RegionNamess] [-m MonthList] [-M AccessMethods]  \
+                     [-k RegionNames] [-m MonthList] [-M AccessMethods]  \
                      [-N MinNumberRead [MaxNumberRead]] \
                      [-o OrganizationTypes] \
                      [-q QuaterList] [-s MinSize [MaxSize]] \

{rda_python_metrics-1.0.18.dist-info → rda_python_metrics-1.0.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.18
+Version: 1.0.20
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.18.dist-info → rda_python_metrics-1.0.20.dist-info}/RECORD

@@ -1,9 +1,9 @@
-rda_python_metrics/PgIPInfo.py,sha256=YTptfEWKZ0Fd09ddjER_z8SfqVcXXklG-FvwU_-huio,7996
+rda_python_metrics/PgIPInfo.py,sha256=HkGja5ctMfV4aJ3spcIsTmDljkU1Q6-RiUzXkwxyi8g,8644
 rda_python_metrics/PgView.py,sha256=r6otb3DjfiaQJdg0z8bZQAOlhr4JnrXJzp9wgWh_8qQ,24369
 rda_python_metrics/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 rda_python_metrics/fillawsusage.py,sha256=SZH9G4jruKvavqSr9DiVmHVW929E60tJGUkv8CK3GQI,8152
 rda_python_metrics/fillawsusage.usg,sha256=pD_nYTfg1so9nvVEyPSWdgKvb9gWdtfHJAs3RsT2MMU,609
-rda_python_metrics/fillcdgusage.py,sha256=Wf4pTOMPwLFRmH_TvhlsTyEXwTO4cNfsbghP6RxemEA,16320
+rda_python_metrics/fillcdgusage.py,sha256=1PDlXwwaHH9F5ngyZsd4XYoSj4pfSrET7FVUMc1mbhI,15366
 rda_python_metrics/fillcdgusage.usg,sha256=5lkd4Zdi72nQcha-JtbaLnxl66V4wBDykKwQtUOtMrw,667
 rda_python_metrics/fillcodusage.py,sha256=_RUcgY1Cf17dSjidhQCMOwVaPO1VL26s3bYGT8oSPWU,8322
 rda_python_metrics/fillcodusage.usg,sha256=3B5IkQ4uwylqY8uEfUdnZ_MXqhYudeylMp5ulhUGXH8,678
@@ -11,7 +11,7 @@ rda_python_metrics/fillcountry.py,sha256=7i5LNi3scRoyRCT6t7aeNTGKOpxzJ2mA9tnvUqj
 rda_python_metrics/fillendtime.py,sha256=skZttlpoY19g0dGwqGQI8t_1YPPTPEXwg3EfNlfL90I,2533
 rda_python_metrics/fillglobususage.py,sha256=ahz8XnnJdD_AbSYqJ34lWmDuzws_-SNmCR8QE20aovA,8539
 rda_python_metrics/fillglobususage.usg,sha256=1GgmCP22IQZdADwL5Mmkz3v8Ws-G7U3teQ1AxRJfV_4,637
-rda_python_metrics/fillipinfo.py,sha256=hkCwQp9OK70lmHshx2J41sMA1K_TNJDMEuDrAOOfVIU,7145
+rda_python_metrics/fillipinfo.py,sha256=Ja1BbyiCA7IueM6o9KxfD-BstA4EVSnjzrwj6hkFgoY,6641
 rda_python_metrics/fillipinfo.usg,sha256=6NoOTStxsVXwycN8u8znnwC6AemKftR2ZGrW_RL41og,647
 rda_python_metrics/filloneorder.py,sha256=ADHbcKCDh9bJunnxYbkbjwU2QpC43hvGlLWaURHNxkg,5433
 rda_python_metrics/filloneorder.usg,sha256=mtOySKx6-D4k2bbTcmi6cSYtINiycRyHQkHozi0CQu0,1466
@@ -29,7 +29,7 @@ rda_python_metrics/pgperson.py,sha256=q_0jSlTyqEQh2J8yq5Nrf9TLg1awvREdp3E8XyN7Po
 rda_python_metrics/pgsyspath.py,sha256=DZhFp-r-LzN0qrHtfdCYfnGDnmD_R4ufuEduk0_vRIw,1039
 rda_python_metrics/pgusername.py,sha256=sNtPNKdubZYNPWR89pAXHGTuP6q8kuf71ig7-XJLXrQ,1245
 rda_python_metrics/viewallusage.py,sha256=DlAvY2bieJWrrrYMPhCkKWgqdfSN492_LZCS95BhY0A,15753
-rda_python_metrics/viewallusage.usg,sha256=6DnzY930-MS7Yp2ELVdMgFFcOQ8zTofQf8HqGkVhuTE,10141
+rda_python_metrics/viewallusage.usg,sha256=Ftqdn7YoqaqX8Vblfo28yC0xf6ccyCLWcLTgOxP6MGg,10141
 rda_python_metrics/viewcheckusage.py,sha256=HougqjDAOVG6pYglFjyHQ-UdLBcYe7v_jzU1-80RqFA,12996
 rda_python_metrics/viewcheckusage.usg,sha256=KuJFycggGiUcSezQ9vywDbituvu63SZ-ZnNTaMpbc-A,8930
 rda_python_metrics/viewcodusage.py,sha256=6Shmbzq_DNh0uvT9lPxpB8ic2JnrmQzmR7Bc-9U4gl0,14243
@@ -39,14 +39,14 @@ rda_python_metrics/viewordusage.usg,sha256=19tHhPZB9y247BddfwtXA3_K50BnzEJcWUbBN
 rda_python_metrics/viewrqstusage.py,sha256=wNH5DTEBYrUQKAms10weBH939r-m3tLXXg5PwS6bzlk,16690
 rda_python_metrics/viewrqstusage.usg,sha256=Ii5-7h_RO2rkoE9VLxuLhc9klgkEJSqHoDrsOlQOTKo,10481
 rda_python_metrics/viewtdsusage.py,sha256=nmtH4d7pPqSwLoAlocb5UTk0W38TT57gWmCyuKL4bF8,14505
-rda_python_metrics/viewtdsusage.usg,sha256=FngS342hHG5ZnolXXXv9eByZG2f3jdsrpAHl_elYYaw,9033
+rda_python_metrics/viewtdsusage.usg,sha256=Gmk0Y4Cs8WDtA3zYJt6gfVH5q5U5rlQZcJI40W5EiCE,9034
 rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmjv5dI,13646
 rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
 rda_python_metrics/viewwebusage.py,sha256=ES2lI8NaCeCpTGi94HU-cDRBxHMiUBbplyYsZf2KqF0,16650
 rda_python_metrics/viewwebusage.usg,sha256=OVDZ78p87E3HLW34ZhasNJ7Zmw8XXjmZPPWZfRhPLXo,9936
-rda_python_metrics-1.0.18.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_metrics-1.0.18.dist-info/METADATA,sha256=LohYZdn33fQUxwuqwfPBPHrmPUun6Yne_GpWEOJK3S4,761
-rda_python_metrics-1.0.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rda_python_metrics-1.0.18.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
-rda_python_metrics-1.0.18.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
-rda_python_metrics-1.0.18.dist-info/RECORD,,
+rda_python_metrics-1.0.20.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_metrics-1.0.20.dist-info/METADATA,sha256=zbFFqGetH5z8euwjan-oU--BVDuBsbJnMbggMNdSpDw,761
+rda_python_metrics-1.0.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rda_python_metrics-1.0.20.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
+rda_python_metrics-1.0.20.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
+rda_python_metrics-1.0.20.dist-info/RECORD,,