rda-python-metrics 1.0.22__py3-none-any.whl → 1.0.24__py3-none-any.whl

rda_python_metrics/PgIPInfo.py

@@ -104,6 +104,23 @@ def domain_ipinfo_record(dmname):
 
    return None
 
+#
+# try to get hostname via socket for given ip address
+#
+def get_ip_hostname(ip, iprec, record):
+
+   if iprec and 'hostname' in iprec:
+      record['hostname'] = iprec['hostname']
+      record['org_type'] = PgDBI.get_org_type(None, record['hostname'])
+   else:
+      record['hostname'] = ip
+      try:
+         hostrec = socket.gethostbyaddr(ip)
+         record['hostname'] = hostrec[1][0] if hostrec[1] else hostrec[0]
+         record['org_type'] = PgDBI.get_org_type(None, record['hostname'])
+      except Exception as e:
+         PgLOG.pglog("socket: {} - {}".format(ip, str(e)), PgLOG.LOGWRN)
+
 #
 # get a ipinfo record for given ip address
 #
@@ -117,9 +134,7 @@ def get_ipinfo_record(ip):
       return None
       
    record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip, 'org_type' : '-'}
-   if 'hostname' in iprec:
-      record['hostname'] = iprec['hostname']
-      record['org_type'] = PgDBI.get_org_type(None, record['hostname'])
+   get_ip_hostname(ip, iprec, record)
    record['lat'] = float(iprec['latitude']) if iprec['latitude'] else 0
    record['lon'] = float(iprec['longitude']) if iprec['longitude'] else 0
    if 'org' in iprec: record['org_name'] = iprec['org']
@@ -153,6 +168,7 @@ def get_geoip2_record(ip):
       return None
 
    record = {'ip' : ip, 'stat_flag' : 'M', 'org_type' : '-'}
+   get_ip_hostname(ip, None, record)
    record['lat'] = float(city.location.latitude) if city.location.latitude else 0
    record['lon'] = float(city.location.longitude) if city.location.longitude else 0
    record['country'] = get_country_name_code(city.country.name)
@@ -160,17 +176,8 @@ def get_geoip2_record(ip):
    record['region'] = PgLOG.convert_chars(city.subdivisions.most_specific.name) if city.subdivisions.most_specific.name else None
    record['postal'] =  city.postal.code
    record['timezone'] = city.location.time_zone
-   record['hostname'] = ip
    record['ipinfo'] = json.dumps(object_to_dict(city))
 
-   try:
-      hostrec = socket.gethostbyaddr(ip)
-   except Exception as e:
-      PgLOG.pglog("socket: {} - {}".format(ip, str(e)), PgLOG.LOGWRN)
-      return record
-   record['hostname'] = hostrec[1][0] if hostrec[1] else hostrec[0]
-   record['org_type'] = PgDBI.get_org_type(None, record['hostname'])
-
    return record
 
 #
@@ -223,11 +230,11 @@ def set_ipinfo(ip, ipopt = True):
 
    if ip in IPRECS:
       pgrec = IPRECS[ip]
-      if not pgrec or not ipopt or pgrec['stat_flag'] == 'A': return pgrec
+      if pgrec or not ipopt: return pgrec
    else:
       pgrec = PgDBI.pgget('ipinfo', '*', "ip = '{}'".format(ip))
 
-   if not pgrec or ipopt and pgrec['stat_flag'] == 'M':
+   if not pgrec:
       record = get_ipinfo_record(ip) if ipopt else None
       if not record: record = get_geoip2_record(ip)
       if record and update_ipinfo_record(record, pgrec): pgrec = record

rda_python_metrics/fillipinfo.py

@@ -25,10 +25,11 @@ from rda_python_common import PgDBI
 from . import PgIPInfo
 
 # the define options for gathering ipinfo data
+DATES = 0x01  # fix data usages for given dates
 MONTH = 0x02  # fix data usages for given months
 YEARS = 0x04  # fix data usages for given years
 NDAYS = 0x08  # fix data usages in recent number of days
-MULTI = (MONTH|YEARS)
+MULTI = (DATES|MONTH|YEARS)
 SINGL = (NDAYS)
 
 IPINFO = {
@@ -49,8 +50,10 @@ def main():
    for arg in argv:
       if arg == "-b":
          PgLOG.PGLOG['BCKGRND'] = 1
-      elif re.match(r'^-[mNy]$', arg) and option == 0:
-         if arg == "-m":
+      elif re.match(r'^-[dmNy]$', arg) and option == 0:
+         if arg == "-d":
+            option = DATES
+         elif arg == "-m":
             option = MONTH
          elif arg == "-y":
             option = YEARS
@@ -74,17 +77,6 @@ def main():
    PgDBI.dssdb_dbname()
    PgLOG.cmdlog("fillipinfo {}".format(' '.join(argv)))
 
-   if option&NDAYS:
-      curdate = IPINFO['CDATE']
-      datelimit = PgUtil.adddate(curdate, 0, 0, -int(inputs[0]))  
-      option = MONTH
-      inputs = []
-      
-      while curdate >= datelimit:
-         tms = curdate.split('-')
-         inputs.append("{}-{}".format(tms[0], tms[1]))
-         curdate = PgUtil.adddate(curdate, 0, 0, -int(tms[2]))
-
    fill_ip_info(option, inputs, table)
 
    sys.exit(0)
@@ -95,35 +87,49 @@ def main():
 def fill_ip_info(option, inputs, table):
 
    cntall = 0
-   date = None
+   func = eval('fix_{}_records'.format(table))
    for input in inputs:
       if option&NDAYS:
          edate = IPINFO['CDATE']
-         date = PgUtil.adddate(edate, 0, 0, -int(input))  
+         date = PgUtil.adddate(edate, 0, 0, -int(input))
+      elif option&DATES:
+         edate = date = input
       elif option&MONTH:
          tms = input.split('-')
          date = "{}-{:02}-01".format(tms[0], int(tms[1]))
          edate = PgUtil.enddate(date, 0, 'M')
-      elif option&YEARS:
+      else:
          date = input + "-01-01"
          edate = input + "-12-31"
+      while True:
+         (ndate, cond) = get_next_date(date, edate)
+         cntall += func(date, cond)
+         if ndate >= edate: break
+         date = PgUtil.adddate(ndate, 0, 0, 1)
 
-      while date <= edate:
-         func = eval('fix_{}_records'.format(table))
-         cntall += func(date)
-         date = PgUtil.adddate(date, 0, 0, 1)
-   
    if cntall > 2:
       PgLOG.pglog("{}: Total {} records updated".format(table, cntall), PgLOG.LOGWRN)
 
+def get_next_date(date, edate):
+
+   if date < edate:
+      ndate = PgUtil.enddate(date, 'M')
+      if ndate < edate: edate = ndate
+   if date < edate:
+      cond = f"BETWEEN '{date}' AND '{edate}'"
+   else:
+      cond = f"= '{date}'"
+
+   return (edate, cond)
+
 
-def fix_allusage_records(date):
+def fix_allusage_records(date, cnd):
 
    cnt = 0
    ms = re.match(r'^(\d+)-', date)
    year = ms.group(1)
    table = 'allusage_' + year
-   cond = "date = '{}' AND region IS NULL".format(date)
+   cond = f"date {cnd} AND region IS NULL"
    pgrecs = PgDBI.pgmget(table, 'aidx, email, ip', cond, PgLOG.LGEREX)
    if not pgrecs: return 0
    cnt = len(pgrecs['ip']) if pgrecs else 0
@@ -134,15 +140,15 @@ def fix_allusage_records(date):
          mcnt += PgDBI.pgupdt(table, record, "aidx = '{}'".format(pgrecs['aidx'][i]))
 
    s = 's' if cnt > 1 else ''
-   PgLOG.pglog("{}: {} of {} record{} updated for {}".format(table, mcnt, cnt, s, date), PgLOG.LOGWRN)
+   PgLOG.pglog(f"{table}: {mcnt} of {cnt} record{s} updated for date {cnd}", PgLOG.LOGWRN)
 
    return mcnt
 
-def fix_tdsusage_records(date):
+def fix_tdsusage_records(date, cnd):
 
    table = 'tdsusage'
-   cond = "date = '{}' AND region IS NULL".format(date)
-   pgrecs = PgDBI.pgmget(table, 'time, email, ip', cond, PgLOG.LGEREX)
+   cond = f"date {cnd} AND region IS NULL"
+   pgrecs = PgDBI.pgmget(table, 'date, time, email, ip', cond, PgLOG.LGEREX)
    if not pgrecs: return 0
    cnt = len(pgrecs['ip']) if pgrecs else 0
    mcnt = 0
@@ -150,18 +156,18 @@ def fix_tdsusage_records(date):
       ip = pgrecs['ip'][i]
       record = PgIPInfo.get_missing_ipinfo(ip, pgrecs['email'][i])
       if record:
-         cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, pgrecs['time'][i], ip)
+         cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(pgrecs['date'][i], pgrecs['time'][i], ip)
          mcnt += PgDBI.pgupdt(table, record, cond)
 
    s = 's' if cnt > 1 else ''
-   PgLOG.pglog("{}: {} of {} record{} updated for {}".format(table, mcnt, cnt, s, date), PgLOG.LOGWRN)
+   PgLOG.pglog(f"{table}: {mcnt} of {cnt} record{s} updated for date {cnd}", PgLOG.LOGWRN)
 
    return mcnt
 
-def fix_codusage_records(date):
+def fix_codusage_records(date, cnd):
 
    table = 'codusage'
-   cond = "date = '{}' AND region IS NULL".format(date)
+   cond = f"date {cnd} AND region IS NULL"
    pgrecs = PgDBI.pgmget(table, 'codidx, email, ip', cond, PgLOG.LGEREX)
    if not pgrecs: return 0
    cnt = len(pgrecs['ip']) if pgrecs else 0
@@ -172,14 +178,14 @@ def fix_codusage_records(date):
          mcnt += PgDBI.pgupdt(table, record, "codidx = '{}'".format(pgrecs['codidx'][i]))
 
    s = 's' if cnt > 1 else ''
-   PgLOG.pglog("{}: {} of {} record{} updated for {}".format(table, mcnt, cnt, s, date), PgLOG.LOGWRN)
+   PgLOG.pglog(f"{table}: {mcnt} of {cnt} record{s} updated for date {cnd}", PgLOG.LOGWRN)
 
    return mcnt
 
-def fix_wuser_records(date):
+def fix_wuser_records(date, cnd):
 
    table = 'wuser'
-   cond = "start_date = '{}' AND region IS NULL".format(date)
+   cond = f"start_date {cnd} AND region IS NULL"
    pgrecs = PgDBI.pgmget(table, 'wuid, email, ip', cond, PgLOG.LGEREX)
    if not pgrecs: return 0
    cnt = len(pgrecs['ip']) if pgrecs else 0
@@ -192,14 +198,14 @@ def fix_wuser_records(date):
          mcnt += PgDBI.pgupdt(table, record, "wuid = '{}'".format(pgrecs['wuid'][i]))
 
    s = 's' if cnt > 1 else ''
-   PgLOG.pglog("{}: {} of {} record{} updated for {}".format(table, mcnt, cnt, s, date), PgLOG.LOGWRN)
+   PgLOG.pglog(f"{table}: {mcnt} of {cnt} record{s} updated for start_date {cnd}", PgLOG.LOGWRN)
 
    return mcnt
 
-def fix_ipinfo_records(date):
+def fix_ipinfo_records(date, cnd):
 
    table = 'ipinfo'
-   cond = "adddate = '{}' AND region IS NULL".format(date)
+   cond = f"adddate {cnd} AND region IS NULL"
    pgrecs = PgDBI.pgmget(table, 'ip', cond, PgLOG.LGEREX)
    if not pgrecs: return 0
    cnt = len(pgrecs['ip']) if pgrecs else 0
@@ -208,7 +214,7 @@ def fix_ipinfo_records(date):
       if PgIPInfo.set_ipinfo(pgrecs['ip'][i]): mcnt +=1
 
    s = 's' if cnt > 1 else ''
-   PgLOG.pglog("{}: {} of {} record{} updated for {}".format(table, mcnt, cnt, s, date), PgLOG.LOGWRN)
+   PgLOG.pglog(f"{table}: {mcnt} of {cnt} record{s} updated for adddate {cnd}", PgLOG.LOGWRN)
 
    return mcnt
 

rda_python_metrics/fillipinfo.usg

@@ -3,7 +3,7 @@
 
  Usage: fillipinfo [-b] -t TableName [-N NumberDay] [-m YearMonths] [-y Years]
 
-      select option, -t and -m, -N or -y to run this application.
+      select option, -t and -d, -m, -N or -y to run this application.
 
       - Option -b, log process information into logfile only;
 
@@ -13,6 +13,8 @@
 
       - Option -N, fix usage info in recent NumberDay days;
 
+      - Option -d, fix usage info in given dates;
+
       - Option -m, fix usage info in given months;
 
       - Option -y, fix usage info in given years.

{rda_python_metrics-1.0.22.dist-info → rda_python_metrics-1.0.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.22
+Version: 1.0.24
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.22.dist-info → rda_python_metrics-1.0.24.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rda_python_metrics/PgIPInfo.py,sha256=SP8vTyCV6owV942xwLvS4CqnyPdkMTEVuQzyB-dkeNk,8644
+rda_python_metrics/PgIPInfo.py,sha256=mvNIgHY-2fpGbGkrzM244x02Ab2OyiV2JVq7XqInwW4,8772
 rda_python_metrics/PgView.py,sha256=r6otb3DjfiaQJdg0z8bZQAOlhr4JnrXJzp9wgWh_8qQ,24369
 rda_python_metrics/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 rda_python_metrics/fillawsusage.py,sha256=SZH9G4jruKvavqSr9DiVmHVW929E60tJGUkv8CK3GQI,8152
@@ -11,8 +11,8 @@ rda_python_metrics/fillcountry.py,sha256=7i5LNi3scRoyRCT6t7aeNTGKOpxzJ2mA9tnvUqj
 rda_python_metrics/fillendtime.py,sha256=skZttlpoY19g0dGwqGQI8t_1YPPTPEXwg3EfNlfL90I,2533
 rda_python_metrics/fillglobususage.py,sha256=ahz8XnnJdD_AbSYqJ34lWmDuzws_-SNmCR8QE20aovA,8539
 rda_python_metrics/fillglobususage.usg,sha256=1GgmCP22IQZdADwL5Mmkz3v8Ws-G7U3teQ1AxRJfV_4,637
-rda_python_metrics/fillipinfo.py,sha256=O2uTGCw13ITZn6tfhNT_9DQJ8-RM5TVZfYIJCYm01oY,6726
-rda_python_metrics/fillipinfo.usg,sha256=6NoOTStxsVXwycN8u8znnwC6AemKftR2ZGrW_RL41og,647
+rda_python_metrics/fillipinfo.py,sha256=wFqyGYlD1gvIZ3e5qlIwjAz8LrhMKDfB-WWY5yE97-c,6770
+rda_python_metrics/fillipinfo.usg,sha256=YeCR58xGv0emqHUZ_9R977HrqaeBwbd6j5QRF2Lc7TA,702
 rda_python_metrics/filloneorder.py,sha256=ADHbcKCDh9bJunnxYbkbjwU2QpC43hvGlLWaURHNxkg,5433
 rda_python_metrics/filloneorder.usg,sha256=mtOySKx6-D4k2bbTcmi6cSYtINiycRyHQkHozi0CQu0,1466
 rda_python_metrics/fillosdfusage.py,sha256=z62l7RtbcfnR84xn2ffQXq0cPfcBkvUciP3u6vb09JI,7855
@@ -44,9 +44,9 @@ rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmj
 rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
 rda_python_metrics/viewwebusage.py,sha256=ES2lI8NaCeCpTGi94HU-cDRBxHMiUBbplyYsZf2KqF0,16650
 rda_python_metrics/viewwebusage.usg,sha256=OVDZ78p87E3HLW34ZhasNJ7Zmw8XXjmZPPWZfRhPLXo,9936
-rda_python_metrics-1.0.22.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_metrics-1.0.22.dist-info/METADATA,sha256=1EgQwJ6ezL7DX5PRIPAP6tdEfG_3aIKdDstqjQ7NJN0,761
-rda_python_metrics-1.0.22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rda_python_metrics-1.0.22.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
-rda_python_metrics-1.0.22.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
-rda_python_metrics-1.0.22.dist-info/RECORD,,
+rda_python_metrics-1.0.24.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_metrics-1.0.24.dist-info/METADATA,sha256=ye_l-YywMIqo8DOKhp0tpRAbkDoM-h6ofixlnWcY43U,761
+rda_python_metrics-1.0.24.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rda_python_metrics-1.0.24.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
+rda_python_metrics-1.0.24.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
+rda_python_metrics-1.0.24.dist-info/RECORD,,