rda-python-metrics 1.0.46__py3-none-any.whl → 1.0.47__py3-none-any.whl

rda_python_metrics/filltdsusage.py

@@ -34,7 +34,9 @@ USAGE = {
    'OPTION' : 0,
    'PGTBL'  : "tdsusage",
    'TDSLOG' : "/data/logs/nginx/{}.access.log",
-   'CDATE' : PgUtil.curdate()
+   'TDSDIR' : PgLOG.PGLOG["GDEXWORK"] + "/zji/tdslogs/",
+   'TDSGET' : 'wget -m -nH -np -nd https://github.com/NCAR/tds-logs/blob/3ffb86d54aa8a164bbd60995247dc1a7e50813b6/logs/',
+   'TDSLOG' : "localhost_access_log.{}.txt"   # {} = YYYY-MM-DD
 }
 
 #
@@ -44,120 +46,125 @@ def main():
 
    params = []  # array of input values
    argv = sys.argv[1:]
-   datelimit = ''
-   fixrec = False
-   
+   option = None
+   datelimits = [None, None]
+
    for arg in argv:
-      if arg == "-b":
-         PgLOG.PGLOG['BCKGRND'] = 1
-      elif arg == "-f":
-         fixrec = True
-      elif re.match(r'^-[mNy]$', arg) and USAGE['OPTION'] == 0:
-         if arg == "-m":
-            USAGE['OPTION'] = MONTH
-         elif arg == "-y":
-            USAGE['OPTION'] = YEARS
-         elif arg == "-N":
-            USAGE['OPTION'] = NDAYS
+      ms = re.match(r'^-(b|d|p|N)$', arg)
+      if ms:
+         opt = ms.group(1)
+         if opt == 'b':
+            PgLOG.PGLOG['BCKGRND'] = 1
+         elif option:
+            PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
+         else:
+            option = opt
       elif re.match(r'^-', arg):
          PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
-      elif USAGE['OPTION']&MASKS:
+      elif option:
          params.append(arg)
       else:
          PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
    
-   if not (USAGE['OPTION'] and params): PgLOG.show_usage('filltdsusage')
-   PgDBI.dssdb_dbname()
-   PgLOG.cmdlog("filltdsusage {}".format(' '.join(argv)))
+   if not (option and params): PgLOG.show_usage('filltdsusage')
 
-   if fixrec:
-      fix_tds_usages(USAGE['OPTION'], params)
+   PgDBI.dssdb_dbname()
+   cmdstr = "filltdsusage {}".format(' '.join(argv))
+   PgLOG.cmdlog(cmdstr)
+   PgFile.change_local_directory(USAGE['TDSDIR'])
+   filenames = get_log_file_names(option, params, datelimits)
+   if filenames:
+      fill_tds_usages(filenames)
    else:
-      if USAGE['OPTION']&NDAYS:
-         curdate = USAGE['CDATE']
-         datelimit = PgUtil.adddate(curdate, 0, 0, -int(params[0]))  
-         USAGE['OPTION'] = MONTH
-         params = []
-         
-         while curdate >= datelimit:
-            tms = curdate.split('-')
-            params.append("{}-{}".format(tms[0], tms[1]))
-            curdate = PgUtil.adddate(curdate, 0, 0, -int(tms[2]))
-   
-      fill_tds_usages(USAGE['OPTION'], params, datelimit)
-
-   PgLOG.pglog(None, PgLOG.LOGWRN|PgLOG.SNDEML)  # send email out if any
+      PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
 
+   PgLOG.pglog(None, PgLOG.LOGWRN)
    sys.exit(0)
 
 #
-# Fill TDS usages into table dssdb.tdsusage from tds access logs
+# get the log file dates 
 #
-def fill_tds_usages(option, inputs, datelimit):
-
-   cntall = cntadd = 0
+def get_log_file_names(option, params, datelimits):
 
-   for input in inputs:
-      # get log file names
-      if option&MONTH:
-         tms = input.split('-')
-         yrmn = "{}-{:02}".format(tms[0], int(tms[1]))
+   filenames = []
+   if option == 'd':
+      for pdate in params:
+         filenames.append(USAGE['TDSLOG'].format(pdate))
+   else:
+      if option == 'N':
+         edate = PgUtil.curdate()
+         pdate = datelimits[0] = PgUtil.adddate(edate, 0, 0, -int(params[0]))
       else:
-         yrmn = input
+         pdate = datelimits[0] = params[0]
+         if len(params) > 1:
+            edate = datelimits[1] = params[1]
+         else:
+            edate = PgUtil.curdate()
+      while pdate <= edate:
+         filenames.append(USAGE['TDSLOG'].format(pdate))
+         pdate = PgUtil.adddate(pdate, 0, 0, 1)
 
-      logfiles = glob.glob(USAGE['TDSLOG'].format(yrmn + '*'))
-      if not logfiles: PgLOG.pglog("{}: No file found to gather TDS usage".format(yrmn), PgLOG.LOGWRN)
-      for logfile in logfiles:
-         if not op.isfile(logfile):
-            PgLOG.pglog("{}: Not exists to gather TDS usage".format(logfile), PgLOG.LOGWRN)
-            continue
-         fdate = None
-         ms = re.search(r'(\d+-\d+-\d+).access.log$', logfile)
-         if ms:
-            fdate = ms.group(1)
-            if fdate >= USAGE['CDATE']: continue
-            if datelimit and fdate < datelimit: continue
-         PgLOG.pglog("Gathering usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
-         tds = PgFile.open_local_file(logfile)
-         if not tds: continue
-         ptime = ''
-         records = {}
-         entcnt = 0
-         while True:
-            line = tds.readline()
-            if not line: break
-            entcnt += 1
-            if entcnt%20000 == 0:
-               cnt = len(records)
-               PgLOG.pglog("{}/{} TDS log entries processed/records to add".format(entcnt, cnt), PgLOG.WARNLG)
-
-            ms = re.search(r'(/thredds/catalog|\sGooglebot/)', line)
-            if ms: continue
-            ms = re.search(r'/thredds/\S+\.(png|jpg|gif|css|htm)', line)
-            if ms: continue
-            ms = re.match(r'^([\d\.]+)\s.*\s(-|\S+@\S+)\s+\[(\S+).*/thredds/(\w+)(/|/grid/)(aggregations|files).*/(ds\d\d\d.\d|[a-z]\d{6})/.*\s200\s+(\d+)(.*)$', line)
-            if not ms: continue
-            ip = ms.group(1)
-            email = ms.group(2)
-            (date, time) = get_record_date_time(ms.group(3))
-            method = ms.group(4)
-            etype = ms.group(6)[0].upper()
-            dsid = PgUtil.format_dataset_id(ms.group(7))
-            size = int(ms.group(8))
-            ebuf = ms.group(9)
-            ms = re.search(r' "(\w+.*\S+)" ', ebuf)
-            engine = ms.group(1) if ms else 'Unknown'
-            key = "{}:{}:{}:{}".format(ip, dsid, method, etype)
+   return filenames
 
-            if key in records:
-               records[key]['size'] += size
-               records[key]['fcount'] += 1
-            else:
-               records[key] = {'ip' : ip, 'email' : email, 'dsid' : dsid, 'time' : time, 'size' : size,
-                              'fcount' : 1, 'method' : method, 'etype' : etype, 'engine' : engine}
-         tds.close()
-         if records: cntadd += add_usage_records(records, fdate)
-         cntall += entcnt
+#
+# Fill TDS usages into table dssdb.tdsusage from tds access logs
+#
+def fill_tds_usages(fnames):
+
+   year = cntall = addall = 0
+   for logfile in fnames:
+      linfo = PgFile.check_local_file(logfile)
+      if not linfo:
+         gzfile = logfile + '.gz'
+         PgLOG.pgsystem(USAGE['TDSGET'] + gzfile, 5, PgLOG.LOGWRN)
+         linfo = PgFile.check_local_file(gzfile)
+         if not linfo:
+            PgLOG.pglog("{}: Not exists for Gathering TDS usage".format(gzfile), PgLOG.LOGWRN)
+            continue
+         PgFile.compress_local_file(gzfile)
+         linfo = PgFile.check_local_file(logfile)
+         if not linfo:
+            PgLOG.pglog("{}: Error ungzip TDS usage".format(gzfile), PgLOG.LGEREX)
+      PgLOG.pglog("{}: Gathering TDS usage at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
+      tds = PgFile.open_local_file(logfile)
+      if not tds: continue
+      records = {}
+      cntadd = entcnt = 0
+      while True:
+         line = tds.readline()
+         if not line: break
+         entcnt += 1
+         if entcnt%20000 == 0:
+            cnt = len(records)
+            PgLOG.pglog("{}/{} TDS log entries processed/records to add".format(entcnt, cnt), PgLOG.WARNLG)
+
+         ms = re.search(r'(/thredds/catalog|\sGooglebot/)', line)
+         if ms: continue
+         ms = re.search(r'/thredds/\S+\.(png|jpg|gif|css|htm)', line)
+         if ms: continue
+         ms = re.match(r'^([\d\.]+)\s.*\s(-|\S+@\S+)\s+\[(\S+).*/thredds/(\w+)(/|/grid/)(aggregations|files).*/(ds\d\d\d.\d|[a-z]\d{6})/.*\s200\s+(\d+)(.*)$', line)
+         if not ms: continue
+         ip = ms.group(1)
+         email = ms.group(2)
+         (date, time) = get_record_date_time(ms.group(3))
+         method = ms.group(4)
+         etype = ms.group(6)[0].upper()
+         dsid = PgUtil.format_dataset_id(ms.group(7))
+         size = int(ms.group(8))
+         ebuf = ms.group(9)
+         ms = re.search(r' "(\w+.*\S+)" ', ebuf)
+         engine = ms.group(1) if ms else 'Unknown'
+         key = "{}:{}:{}:{}".format(ip, dsid, method, etype)
+
+         if key in records:
+            records[key]['size'] += size
+            records[key]['fcount'] += 1
+         else:
+            records[key] = {'ip' : ip, 'email' : email, 'dsid' : dsid, 'time' : time, 'size' : size,
+                           'fcount' : 1, 'method' : method, 'etype' : etype, 'engine' : engine}
+      tds.close()
+      if records: cntadd += add_usage_records(records, date)
+      cntall += entcnt
 
    PgLOG.pglog("{} TDS usage records added for {} entries at {}".format(cntadd, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
 

rda_python_metrics/filltdsusage.usg

@@ -1,17 +1,16 @@
 
- Retrieves usage information from RDA TDS Server logs under ../logs/tomcat to
+ Retrieves usage information from RDA TDS Server logs on github to
    fill table 'tdsusage' in MySQL database 'dssdb'.
 
- Usage: filltdsusage [-b] [-m MonthList] [-N NumberDay] [-y YearList]
+ Usage: filltdsusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
 
-      select one of the options, -m, -N or -y each time to run
-        this application. 
+      select option, -d, -N or -p to run this application.
 
       - Option -b, log process information into logfile only;
 
-      - Option -m, retrieve usage info in given months; 
+      - Option -d, retrieve usage info from given log file dates; 
 
       - Option -N, retrieve usage info in recent NumberDay days;
 
-      - Option -y, retrieve usage info in given years. 
-
+      - Option -p, retrieve usage info between given period. For missing EndDate,
+          it defaults to the current date.

{rda_python_metrics-1.0.46.dist-info → rda_python_metrics-1.0.47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.46
+Version: 1.0.47
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.46.dist-info → rda_python_metrics-1.0.47.dist-info}/RECORD

@@ -21,8 +21,8 @@ rda_python_metrics/fillosdfusage.py,sha256=osl2jdbDrh5l2H19Aw8LN-drSTV1NAKPc_cKd
 rda_python_metrics/fillosdfusage.usg,sha256=Qc5YdUuOiAH74FfVdkwkrQRDYXcASVbxMdBsVQj4X1k,635
 rda_python_metrics/fillrdadb.py,sha256=5OIzLUo-MHWVIA3sQE7CRMQSW8tX9PH7DocDy4rvbYA,5282
 rda_python_metrics/fillrdadb.usg,sha256=E_Bf4G8yVABogjRmIOaIbTGgnII2W6RltaFad2XEV2Q,1228
-rda_python_metrics/filltdsusage.py,sha256=vOwVzAtWUHO4O-FCSJMg0GKxw-Xc5AzSbfqVFktUAlA,10201
-rda_python_metrics/filltdsusage.usg,sha256=yqTHRe8WpZWpu3gso_obgt6LO41-JC27rTIDfdZcISo,538
+rda_python_metrics/filltdsusage.py,sha256=fM6wThVrcozWU6HCyPRaRWsNjGobYEsVwQbhzWp_UcI,10392
+rda_python_metrics/filltdsusage.usg,sha256=HhqJnju6iqKrsB7AQfdwDxXnZ_NOykkDA0vsKEso4xY,588
 rda_python_metrics/filluser.py,sha256=CvaMRaUPaR9nxJAExkLTb3Ci4sD7RQMOpWhWJdbyMF0,8907
 rda_python_metrics/filluser.usg,sha256=Xgqi0QwA9-4jpYj3L4Q4TISpVwRlsomt2G7T0oYAFak,520
 rda_python_metrics/fillzenodousage.py,sha256=wZiHs_cD061sUwTM0zujSkddUKtzACR1l2UG7smOioA,13212
@@ -51,9 +51,9 @@ rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmj
 rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
 rda_python_metrics/viewwebusage.py,sha256=ES2lI8NaCeCpTGi94HU-cDRBxHMiUBbplyYsZf2KqF0,16650
 rda_python_metrics/viewwebusage.usg,sha256=OVDZ78p87E3HLW34ZhasNJ7Zmw8XXjmZPPWZfRhPLXo,9936
-rda_python_metrics-1.0.46.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_metrics-1.0.46.dist-info/METADATA,sha256=EKyf_eq-mCqY18vi3MiJHlSh9Tqq2z-mLeCUlfFMjcA,786
-rda_python_metrics-1.0.46.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rda_python_metrics-1.0.46.dist-info/entry_points.txt,sha256=fv4wjyEdiLFYNaJzhmet0SxZcF_J6M42koft2Na1XMs,1403
-rda_python_metrics-1.0.46.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
-rda_python_metrics-1.0.46.dist-info/RECORD,,
+rda_python_metrics-1.0.47.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_metrics-1.0.47.dist-info/METADATA,sha256=XflHm_AHkQiBCuTdbWTlNsJsnS-elQ85Ebyt7niZQaQ,786
+rda_python_metrics-1.0.47.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rda_python_metrics-1.0.47.dist-info/entry_points.txt,sha256=fv4wjyEdiLFYNaJzhmet0SxZcF_J6M42koft2Na1XMs,1403
+rda_python_metrics-1.0.47.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
+rda_python_metrics-1.0.47.dist-info/RECORD,,