rda-python-metrics 1.0.43__tar.gz → 1.0.47__tar.gz

{rda_python_metrics-1.0.43/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.43
+Version: 1.0.47
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
@@ -17,6 +17,7 @@ Requires-Dist: geoip2
 Requires-Dist: ipinfo
 Requires-Dist: httplib2
 Requires-Dist: dnspython
+Requires-Dist: unidecode
 Dynamic: license-file
 
 RDA Python Package to gather and view data usage metrics.

{rda_python_metrics-1.0.43 → rda_python_metrics-1.0.47}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.43"
+version = "1.0.47"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]
@@ -25,7 +25,8 @@ dependencies = [
   "geoip2",
   "ipinfo",
   "httplib2",
-  "dnspython"
+  "dnspython",
+  "unidecode"
 ]
 
 [tool.pytest.ini_options]

{rda_python_metrics-1.0.43 → rda_python_metrics-1.0.47}/src/rda_python_metrics/fillcdgusage.py

@@ -86,7 +86,12 @@ DSIDS = {
    'ucar.cgd.cesm2.marine.biogeochemistry': ['d651071'],
    'ucar.cgd.nw2.mom6': ['d651072'],
    'ucar.cgd.cesm2.cam6.ppe': ['d651076'],
-   'ucar.cgd.cesm2.smyle': ['d651065']
+   'ucar.cgd.cesm2.smyle': ['d651065'],
+# new added
+   'gridded_precip_and_temp' : ['d010078'],
+   '29_newman' : ['d010079'],
+   'waccm-x.ion.asymmetry' : ['d010081'],
+   'NARCCAP' : ['d316015']   
 }
 
 ALLIDS = list(DSIDS.keys())

{rda_python_metrics-1.0.43 → rda_python_metrics-1.0.47}/src/rda_python_metrics/fillglobususage.py

@@ -27,7 +27,7 @@ from . import PgIPInfo
 
 USAGE = {
    'PGTBL'  : "wusage",
-   'GBSDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/logs/gridftp/",
+   'GBSDIR' : PgLOG.PGLOG["GDEXWORK"] + "/logs/gridftp/",
    'GBSLOG' : "access_log_gridftp0{}_{}",
 }
 
@@ -43,7 +43,7 @@ def main():
    
 
    for arg in argv:
-      ms = re.match(r'^-(b|d|p|N)$', arg)
+      ms = re.match(r'^-(b|d|f|p|N)$', arg)
       if ms:
          opt = ms.group(1)
          if opt == 'b':
@@ -80,7 +80,9 @@ def main():
 def get_log_file_names(option, params, datelimits):
 
    filenames = []
-   if option == 'd':
+   if option == 'f':
+      filenames = params
+   elif option == 'd':
       for pdate in params:
          fdate = PgUtil.format_date(pdate, 'MMDDYYYY')
          fname = USAGE['GBSLOG'].format('?', fdate)

{rda_python_metrics-1.0.43 → rda_python_metrics-1.0.47}/src/rda_python_metrics/fillglobususage.usg

@@ -3,14 +3,16 @@
    /gpfs/fs1/collections/rda/work/logs/gridftp/ to fill table 'wusage' in
    database 'rdadb'.
 
- Usage: fillglobususage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
+ Usage: fillglobususage [-b] [-d LogFileDates] [-f LogFileNames] [-N NumberDay] [-p BeginDate [Enddate]]
 
-      select option, -d, -N or -p to run this application.
+      select option, -d, -f, -N or -p to run this application.
 
       - Option -b, log process information into logfile only;
 
       - Option -d, retrieve usage info from given log file dates; 
 
+      - Option -f, retrieve usage info from given log file names; 
+
       - Option -N, retrieve usage info in recent NumberDay days;
 
       - Option -p, retrieve usage info between given period. For missing EndDate,

{rda_python_metrics-1.0.43 → rda_python_metrics-1.0.47}/src/rda_python_metrics/fillosdfusage.py

@@ -23,7 +23,7 @@ from . import PgIPInfo
 
 USAGE = {
    'OSDFTBL'  : "osdfusage",
-   'OSDFDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/zji/osdflogs/",
+   'OSDFDIR' : PgLOG.PGLOG["GDEXWORK"] + "/zji/osdflogs/",
    'OSDFGET' : 'wget -m -nH -np -nd https://pelicanplatform.org/pelican-access-logs/ncar-access-log/',
    'OSDFLOG' : "{}-cache.log",   # YYYY-MM-DD-cache.log
 }

{rda_python_metrics-1.0.43 → rda_python_metrics-1.0.47}/src/rda_python_metrics/filltdsusage.py

@@ -34,7 +34,9 @@ USAGE = {
    'OPTION' : 0,
    'PGTBL'  : "tdsusage",
    'TDSLOG' : "/data/logs/nginx/{}.access.log",
-   'CDATE' : PgUtil.curdate()
+   'TDSDIR' : PgLOG.PGLOG["GDEXWORK"] + "/zji/tdslogs/",
+   'TDSGET' : 'wget -m -nH -np -nd https://github.com/NCAR/tds-logs/blob/3ffb86d54aa8a164bbd60995247dc1a7e50813b6/logs/',
+   'TDSLOG' : "localhost_access_log.{}.txt"   # {} = YYYY-MM-DD
 }
 
 #
@@ -44,120 +46,125 @@ def main():
 
    params = []  # array of input values
    argv = sys.argv[1:]
-   datelimit = ''
-   fixrec = False
-   
+   option = None
+   datelimits = [None, None]
+
    for arg in argv:
-      if arg == "-b":
-         PgLOG.PGLOG['BCKGRND'] = 1
-      elif arg == "-f":
-         fixrec = True
-      elif re.match(r'^-[mNy]$', arg) and USAGE['OPTION'] == 0:
-         if arg == "-m":
-            USAGE['OPTION'] = MONTH
-         elif arg == "-y":
-            USAGE['OPTION'] = YEARS
-         elif arg == "-N":
-            USAGE['OPTION'] = NDAYS
+      ms = re.match(r'^-(b|d|p|N)$', arg)
+      if ms:
+         opt = ms.group(1)
+         if opt == 'b':
+            PgLOG.PGLOG['BCKGRND'] = 1
+         elif option:
+            PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
+         else:
+            option = opt
       elif re.match(r'^-', arg):
          PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
-      elif USAGE['OPTION']&MASKS:
+      elif option:
          params.append(arg)
       else:
          PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
    
-   if not (USAGE['OPTION'] and params): PgLOG.show_usage('filltdsusage')
-   PgDBI.dssdb_dbname()
-   PgLOG.cmdlog("filltdsusage {}".format(' '.join(argv)))
+   if not (option and params): PgLOG.show_usage('filltdsusage')
 
-   if fixrec:
-      fix_tds_usages(USAGE['OPTION'], params)
+   PgDBI.dssdb_dbname()
+   cmdstr = "filltdsusage {}".format(' '.join(argv))
+   PgLOG.cmdlog(cmdstr)
+   PgFile.change_local_directory(USAGE['TDSDIR'])
+   filenames = get_log_file_names(option, params, datelimits)
+   if filenames:
+      fill_tds_usages(filenames)
    else:
-      if USAGE['OPTION']&NDAYS:
-         curdate = USAGE['CDATE']
-         datelimit = PgUtil.adddate(curdate, 0, 0, -int(params[0]))  
-         USAGE['OPTION'] = MONTH
-         params = []
-         
-         while curdate >= datelimit:
-            tms = curdate.split('-')
-            params.append("{}-{}".format(tms[0], tms[1]))
-            curdate = PgUtil.adddate(curdate, 0, 0, -int(tms[2]))
-   
-      fill_tds_usages(USAGE['OPTION'], params, datelimit)
-
-   PgLOG.pglog(None, PgLOG.LOGWRN|PgLOG.SNDEML)  # send email out if any
+      PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
 
+   PgLOG.pglog(None, PgLOG.LOGWRN)
    sys.exit(0)
 
 #
-# Fill TDS usages into table dssdb.tdsusage from tds access logs
+# get the log file dates 
 #
-def fill_tds_usages(option, inputs, datelimit):
-
-   cntall = cntadd = 0
+def get_log_file_names(option, params, datelimits):
 
-   for input in inputs:
-      # get log file names
-      if option&MONTH:
-         tms = input.split('-')
-         yrmn = "{}-{:02}".format(tms[0], int(tms[1]))
+   filenames = []
+   if option == 'd':
+      for pdate in params:
+         filenames.append(USAGE['TDSLOG'].format(pdate))
+   else:
+      if option == 'N':
+         edate = PgUtil.curdate()
+         pdate = datelimits[0] = PgUtil.adddate(edate, 0, 0, -int(params[0]))
       else:
-         yrmn = input
+         pdate = datelimits[0] = params[0]
+         if len(params) > 1:
+            edate = datelimits[1] = params[1]
+         else:
+            edate = PgUtil.curdate()
+      while pdate <= edate:
+         filenames.append(USAGE['TDSLOG'].format(pdate))
+         pdate = PgUtil.adddate(pdate, 0, 0, 1)
 
-      logfiles = glob.glob(USAGE['TDSLOG'].format(yrmn + '*'))
-      if not logfiles: PgLOG.pglog("{}: No file found to gather TDS usage".format(yrmn), PgLOG.LOGWRN)
-      for logfile in logfiles:
-         if not op.isfile(logfile):
-            PgLOG.pglog("{}: Not exists to gather TDS usage".format(logfile), PgLOG.LOGWRN)
-            continue
-         fdate = None
-         ms = re.search(r'(\d+-\d+-\d+).access.log$', logfile)
-         if ms:
-            fdate = ms.group(1)
-            if fdate >= USAGE['CDATE']: continue
-            if datelimit and fdate < datelimit: continue
-         PgLOG.pglog("Gathering usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
-         tds = PgFile.open_local_file(logfile)
-         if not tds: continue
-         ptime = ''
-         records = {}
-         entcnt = 0
-         while True:
-            line = tds.readline()
-            if not line: break
-            entcnt += 1
-            if entcnt%20000 == 0:
-               cnt = len(records)
-               PgLOG.pglog("{}/{} TDS log entries processed/records to add".format(entcnt, cnt), PgLOG.WARNLG)
-
-            ms = re.search(r'(/thredds/catalog|\sGooglebot/)', line)
-            if ms: continue
-            ms = re.search(r'/thredds/\S+\.(png|jpg|gif|css|htm)', line)
-            if ms: continue
-            ms = re.match(r'^([\d\.]+)\s.*\s(-|\S+@\S+)\s+\[(\S+).*/thredds/(\w+)(/|/grid/)(aggregations|files).*/(ds\d\d\d.\d|[a-z]\d{6})/.*\s200\s+(\d+)(.*)$', line)
-            if not ms: continue
-            ip = ms.group(1)
-            email = ms.group(2)
-            (date, time) = get_record_date_time(ms.group(3))
-            method = ms.group(4)
-            etype = ms.group(6)[0].upper()
-            dsid = PgUtil.format_dataset_id(ms.group(7))
-            size = int(ms.group(8))
-            ebuf = ms.group(9)
-            ms = re.search(r' "(\w+.*\S+)" ', ebuf)
-            engine = ms.group(1) if ms else 'Unknown'
-            key = "{}:{}:{}:{}".format(ip, dsid, method, etype)
+   return filenames
 
-            if key in records:
-               records[key]['size'] += size
-               records[key]['fcount'] += 1
-            else:
-               records[key] = {'ip' : ip, 'email' : email, 'dsid' : dsid, 'time' : time, 'size' : size,
-                              'fcount' : 1, 'method' : method, 'etype' : etype, 'engine' : engine}
-         tds.close()
-         if records: cntadd += add_usage_records(records, fdate)
-         cntall += entcnt
+#
+# Fill TDS usages into table dssdb.tdsusage from tds access logs
+#
+def fill_tds_usages(fnames):
+
+   year = cntall = addall = 0
+   for logfile in fnames:
+      linfo = PgFile.check_local_file(logfile)
+      if not linfo:
+         gzfile = logfile + '.gz'
+         PgLOG.pgsystem(USAGE['TDSGET'] + gzfile, 5, PgLOG.LOGWRN)
+         linfo = PgFile.check_local_file(gzfile)
+         if not linfo:
+            PgLOG.pglog("{}: Not exists for Gathering TDS usage".format(gzfile), PgLOG.LOGWRN)
+            continue
+         PgFile.compress_local_file(gzfile)
+         linfo = PgFile.check_local_file(logfile)
+         if not linfo:
+            PgLOG.pglog("{}: Error ungzip TDS usage".format(gzfile), PgLOG.LGEREX)
+      PgLOG.pglog("{}: Gathering TDS usage at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
+      tds = PgFile.open_local_file(logfile)
+      if not tds: continue
+      records = {}
+      cntadd = entcnt = 0
+      while True:
+         line = tds.readline()
+         if not line: break
+         entcnt += 1
+         if entcnt%20000 == 0:
+            cnt = len(records)
+            PgLOG.pglog("{}/{} TDS log entries processed/records to add".format(entcnt, cnt), PgLOG.WARNLG)
+
+         ms = re.search(r'(/thredds/catalog|\sGooglebot/)', line)
+         if ms: continue
+         ms = re.search(r'/thredds/\S+\.(png|jpg|gif|css|htm)', line)
+         if ms: continue
+         ms = re.match(r'^([\d\.]+)\s.*\s(-|\S+@\S+)\s+\[(\S+).*/thredds/(\w+)(/|/grid/)(aggregations|files).*/(ds\d\d\d.\d|[a-z]\d{6})/.*\s200\s+(\d+)(.*)$', line)
+         if not ms: continue
+         ip = ms.group(1)
+         email = ms.group(2)
+         (date, time) = get_record_date_time(ms.group(3))
+         method = ms.group(4)
+         etype = ms.group(6)[0].upper()
+         dsid = PgUtil.format_dataset_id(ms.group(7))
+         size = int(ms.group(8))
+         ebuf = ms.group(9)
+         ms = re.search(r' "(\w+.*\S+)" ', ebuf)
+         engine = ms.group(1) if ms else 'Unknown'
+         key = "{}:{}:{}:{}".format(ip, dsid, method, etype)
+
+         if key in records:
+            records[key]['size'] += size
+            records[key]['fcount'] += 1
+         else:
+            records[key] = {'ip' : ip, 'email' : email, 'dsid' : dsid, 'time' : time, 'size' : size,
+                           'fcount' : 1, 'method' : method, 'etype' : etype, 'engine' : engine}
+      tds.close()
+      if records: cntadd += add_usage_records(records, date)
+      cntall += entcnt
 
    PgLOG.pglog("{} TDS usage records added for {} entries at {}".format(cntadd, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
 

rda_python_metrics-1.0.47/src/rda_python_metrics/filltdsusage.usg

@@ -0,0 +1,16 @@
+
+ Retrieves usage information from RDA TDS Server logs on github to
+   fill table 'tdsusage' in MySQL database 'dssdb'.
+
+ Usage: filltdsusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
+
+      select option, -d, -N or -p to run this application.
+
+      - Option -b, log process information into logfile only;
+
+      - Option -d, retrieve usage info from given log file dates; 
+
+      - Option -N, retrieve usage info in recent NumberDay days;
+
+      - Option -p, retrieve usage info between given period. For missing EndDate,
+          it defaults to the current date.

{rda_python_metrics-1.0.43 → rda_python_metrics-1.0.47}/src/rda_python_metrics/logarch.py

@@ -262,8 +262,9 @@ def archive_dssdb_log():
       if(not info or info['data_size'] < 10000): continue   # skip log files small than 10KB
 
       PgLOG.pgsystem("cp -p -f {} backup/{}".format(file, file), PgLOG.LWEMEX, 5)
-      if info['logname'] != PgLOG.PGLOG['RDAUSER']: PgLOG.pgsystem("rm -rf " + file)
+      if info['logname'] != PgLOG.PGLOG['GDEXUSER']: PgLOG.pgsystem("rm -rf " + file)
       PgLOG.pgsystem("cat /dev/null > " + file, 0, 1024)
+      if file == 'gdexls.log': PgLOG.pgsystem("chmod 666 " + file, 0, 1024)
       if op.exists(logfile):
          PgLOG.pgsystem("tar -uvf {} -C backup {}".format(logfile, file), PgLOG.LWEMEX, 5)
       else:

{rda_python_metrics-1.0.43 → rda_python_metrics-1.0.47/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.43
+Version: 1.0.47
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
@@ -17,6 +17,7 @@ Requires-Dist: geoip2
 Requires-Dist: ipinfo
 Requires-Dist: httplib2
 Requires-Dist: dnspython
+Requires-Dist: unidecode
 Dynamic: license-file
 
 RDA Python Package to gather and view data usage metrics.

{rda_python_metrics-1.0.43 → rda_python_metrics-1.0.47}/src/rda_python_metrics.egg-info/requires.txt

@@ -4,3 +4,4 @@ geoip2
 ipinfo
 httplib2
 dnspython
+unidecode

rda_python_metrics-1.0.43/src/rda_python_metrics/filltdsusage.usg

@@ -1,17 +0,0 @@
-
- Retrieves usage information from RDA TDS Server logs under ../logs/tomcat to
-   fill table 'tdsusage' in MySQL database 'dssdb'.
-
- Usage: filltdsusage [-b] [-m MonthList] [-N NumberDay] [-y YearList]
-
-      select one of the options, -m, -N or -y each time to run
-        this application. 
-
-      - Option -b, log process information into logfile only;
-
-      - Option -m, retrieve usage info in given months; 
-
-      - Option -N, retrieve usage info in recent NumberDay days;
-
-      - Option -y, retrieve usage info in given years. 
-