rda-python-metrics 3.0.2__tar.gz → 3.0.4__tar.gz

{rda_python_metrics-3.0.2/src/rda_python_metrics.egg-info → rda_python_metrics-3.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 3.0.2
+Version: 3.0.4
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "3.0.2"
+version = "3.0.4"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/fill_oneorder.py

@@ -143,7 +143,7 @@ def check_inputs(params):
 #
 def check_specialist(dsid, specialist):
 
-   if specialist and PgDBI.pgget("dssgrp", "", "logname = 'specialist'", PgLOG.LGEREX): return specialist
+   if specialist and PgDBI.pgget("dssgrp", "", "logname = '{}'".format(specialist), PgLOG.LGEREX): return specialist
    scond = "specialist = logname AND dsid = '{}' AND priority = 1".format(dsid)
    pgrec = PgDBI.pgget("dsowner, dssgrp", "specialist", scond, PgLOG.LGEREX)
    return pgrec['specialist'] if pgrec else "datahelp"

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/fillawsusage.usg

@@ -1,7 +1,7 @@
 
  fillawsusage - Fill table 'awsusage' from AWS Server log files.
 
- Reads AWS access logs from /gpfs/fs1/collections/rda/transer/AWSera5log/
+ Reads AWS access logs from /gpfs/fs1/collections/rda/transfer/AWSera5log/
  and inserts usage records into the 'awsusage' table in PostgreSQL
  database 'rdadb'.
 

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/fillcodusage.py

@@ -39,7 +39,7 @@ class FillCODUsage(PgIPInfo):
       self.params = []  # array of input values
 
    def read_parameters(self):
-      """Function to readparameters."""
+      """Function to read parameters."""
       argv = sys.argv[1:]
       for arg in argv:
          if arg == "-b":

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/filloneorder.py

@@ -124,7 +124,7 @@ class FillONEOrder(PgUtil):
 
    def check_specialist(self, dsid, specialist):
       """Return the dataset owner if specialist not given."""
-      if specialist and self.pgget("dssgrp", "", "logname = 'specialist'", self.LGEREX): return specialist
+      if specialist and self.pgget("dssgrp", "", "logname = '{}'".format(specialist), self.LGEREX): return specialist
       scond = "specialist = logname AND dsid = '{}' AND priority = 1".format(dsid)
       pgrec = self.pgget("dsowner, dssgrp", "specialist", scond, self.LGEREX)
       return pgrec['specialist'] if pgrec else "datahelp"

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/filluser.py

@@ -11,11 +11,10 @@
 ###############################################################################
 import sys
 import re
-import time
-from os import path as op
 from rda_python_common.pg_util import PgUtil
+from rda_python_common.pg_dbi import PgDBI
 
-class FillUser(PgUtil):
+class FillUser(PgUtil, PgDBI):
 
    """Retrieve user info from the UCAR People API and fill table user in PostgreSQL database rdadb."""
 
@@ -131,7 +130,7 @@ class FillUser(PgUtil):
             if neworg: return 0
             newrec['org_name'] = orarec['org_name']
          if orarec['country'] and (not pgrec['country'] or orarec['country'] and pgrec['country'] != orarec['country']):
-            orarec['country'] = self.set_country_code(orarec)
+            orarec['country'] = self.set_country_code(orarec['email'], orarec['country'])
             if not pgrec['country'] or pgrec['country'] != orarec['country']:
                if neworg: return 0
                newrec['country'] = orarec['country']

rda_python_metrics-3.0.4/src/rda_python_metrics/fillzenodousage.usg

@@ -0,0 +1,32 @@
+
+ fillzenodousage - Fill table 'gdexzenodo' from Zenodo metrics.
+
+ Retrieves Zenodo usage info for GDEX-published datasets and inserts
+ records into the 'gdexzenodo' table in PostgreSQL database 'rdadb'.
+
+ USAGE:
+   fillzenodousage [-b] -s DatasetShortNames
+                   (-m MonthList | -N NumberDays | -y YearList)
+
+ OPTIONS:
+
+   -s DatasetShortNames   (required)
+        Gather usage info for the given dataset short names.
+
+   One of the following time-range options is required:
+
+   -m MonthList
+        Gather usage info for the given months.
+        Month format: YYYY-MM (e.g. 2024-01).
+
+   -N NumberDays
+        Gather usage info for the most recent NumberDays days.
+
+   -y YearList
+        Gather usage info for the given years.
+        Year format: YYYY (e.g. 2024).
+
+   -b   Run in background mode (log to file only, no terminal output).
+
+ EXAMPLE:
+   fillzenodousage -s ds083.2 -y 2024

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/log_arch.py

@@ -23,7 +23,7 @@ from rda_python_common import PgFile
 from rda_python_common import PgSIG
 
 # the defined options for archiving different logs
-WLOG = 0x21  # archive web log
+WLOG = 0x01  # archive web log
 TLOG = 0x02  # archive tds log
 DLOG = 0x04  # archive dssdb logs
 SLOG = 0x08  # append dssdb sub batch logs
@@ -33,8 +33,8 @@ OLOG = 0x20  # archive OSDF web log
 LOGS = {
    'OPTION' : 0,
    'AWSLOG' : PgLOG.PGLOG["TRANSFER"] + "/AWSera5log",
-   'WEBLOG' : PgLOG.PGLOG["DSSDATA"] + "/work/logs/gridftp",
-   'OSDFLOG' : PgLOG.PGLOG["DSSDATA"] + "/zji/osdflogs",
+   'WEBLOG' : PgLOG.PGLOG["GDEXWORK"] + "/logs/gridftp",
+   'OSDFLOG' : PgLOG.PGLOG["GDEXWORK"] + "/zji/osdflogs",
    'MGTLOG' : "/data/logs",
    'TDSLOG' : "/data/logs/nginx",
    'RDALOG' : PgLOG.PGLOG['LOGPATH'],
@@ -60,10 +60,10 @@ def main():
    
    option = None
    for arg in argv:
-      ms = re.match(r'^-([abdmnpstw])', arg)
+      ms = re.match(r'^-([abcdmnopstw])', arg)
       if ms:
          option = ms.group(1)
-         if option in 'mp': continue
+         if option in 'cmp': continue
          if option == "b":
             PgLOG.PGLOG['BCKGRND'] = 1
          elif option == "d":
@@ -81,6 +81,8 @@ def main():
          elif option == "n":
             LOGS['CHKLOG'] = 0
          option = None
+      elif option == 'c':
+         PgLOG.add_carbon_copy(arg, 1)
       elif option == 'm':
          smonth = arg
       elif option == 'p':
@@ -125,6 +127,9 @@ def archive_web_log(smonth):
    if op.exists(afile): PgFile.delete_local_file(afile)
 
    logfiles = sorted(glob.glob("{}/access_log_gridftp??_{}??{}".format(logpath, mn, yr)))
+   if not logfiles:
+      PgLOG.pglog("{}: NO globus web log file found for {}-{} to archive".format(logpath, yr, mn), PgLOG.LGWNEM)
+      return
    topt = '-cvf'
    tcnt = 0
    for logfile in logfiles:
@@ -164,6 +169,9 @@ def archive_osdf_log(smonth):
    if op.exists(afile): PgFile.delete_local_file(afile)
 
    logfiles = sorted(glob.glob("{}/{}-{}-??.log".format(logpath, yr, mn)))
+   if not logfiles:
+      PgLOG.pglog("{}: NO OSDF web log file found for {}-{} to archive".format(logpath, yr, mn), PgLOG.LGWNEM)
+      return
    topt = '-cvf'
    tcnt = 0
    for logfile in logfiles:
@@ -177,7 +185,7 @@ def archive_osdf_log(smonth):
       afile += '.gz'
       PgFile.move_local_file(dfile, afile, PgLOG.LGWNEM)
       s = 's' if tcnt > 1 else ''
-      PgLOG.pglog("{}: {} globus log{} tarred, gzipped and archived at {}".format(afile, tcnt, s, PgLOG.current_datetime()), PgLOG.LGWNEM)
+      PgLOG.pglog("{}: {} OSDF log{} tarred, gzipped and archived at {}".format(afile, tcnt, s, PgLOG.current_datetime()), PgLOG.LGWNEM)
 
 #
 # Archive AWS web log files to LOGS['DECSLOGS']
@@ -196,6 +204,9 @@ def archive_aws_log(smonth):
    if op.exists(afile): PgFile.delete_local_file(afile)
 
    lfile = "{}/{}".format(yr, mn)
+   if not op.exists("{}/{}".format(logpath, lfile)):
+      PgLOG.pglog("{}/{}: NO AWS web log directory found to archive".format(logpath, lfile), PgLOG.LGWNEM)
+      return
    tcmd = "tar -cvf {} -C {} {}".format(afile, logpath, lfile)
    PgLOG.pgsystem(tcmd, PgLOG.LGWNEM, 5)
 
@@ -221,6 +232,9 @@ def archive_tds_log(smonth):
    if op.exists(afile): PgFile.delete_local_file(afile)
 
    logfiles = sorted(glob.glob("{}/{}-{}-??.access.log".format(logpath, yr, mn)))
+   if not logfiles:
+      PgLOG.pglog("{}: NO thredds log file found for {}-{} to archive".format(logpath, yr, mn), PgLOG.LGWNEM)
+      return
    topt = '-cvf'
    tcnt = 0
    for logfile in logfiles:
@@ -257,6 +271,9 @@ def archive_dssdb_log():
 
    # collect all the large log/err files
    files = sorted(glob.glob("*.log") + glob.glob("*.err"))
+   if not files:
+      PgLOG.pglog("{}: NO dssdb log file found to archive".format(LOGS['RDALOG']), PgLOG.LGWNEM)
+      return
    for file in files:
       info = PgFile.check_local_file(file, 2)
       if(not info or info['data_size'] < 10000): continue   # skip log files small than 10KB

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/logarch.py

@@ -21,7 +21,7 @@ class LogArch(PgFile):
    def __init__(self):
       super().__init__()
       # the defined options for archiving different logs
-      self.WLOG = 0x21  # archive web log
+      self.WLOG = 0x01  # archive web log
       self.TLOG = 0x02  # archive tds log
       self.DLOG = 0x04  # archive dssdb logs
       self.SLOG = 0x08  # append dssdb sub batch logs
@@ -30,8 +30,8 @@ class LogArch(PgFile):
       self.LOGS = {
          'OPTION' : 0,
          'AWSLOG' : self.PGLOG["TRANSFER"] + "/AWSera5log",
-         'WEBLOG' : self.PGLOG["DSSDATA"] + "/work/logs/gridftp",
-         'OSDFLOG' : self.PGLOG["DSSDATA"] + "/zji/osdflogs",
+         'WEBLOG' : self.PGLOG["GDEXWORK"] + "/logs/gridftp",
+         'OSDFLOG' : self.PGLOG["GDEXWORK"] + "/zji/osdflogs",
          'MGTLOG' : "/data/logs",
          'TDSLOG' : "/data/logs/nginx",
          'RDALOG' : self.PGLOG['LOGPATH'],
@@ -51,10 +51,10 @@ class LogArch(PgFile):
       self.set_suid(self.PGLOG['EUID'])
       option = None
       for arg in argv:
-         ms = re.match(r'^-([abdmnpstw])', arg)
+         ms = re.match(r'^-([abcdmnopstw])', arg)
          if ms:
             option = ms.group(1)
-            if option in 'mp': continue
+            if option in 'cmp': continue
             if option == "b":
                self.PGLOG['BCKGRND'] = 1
             elif option == "d":
@@ -72,6 +72,8 @@ class LogArch(PgFile):
             elif option == "n":
                self.LOGS['CHKLOG'] = 0
             option = None
+         elif option == 'c':
+            self.add_carbon_copy(arg, 1)
          elif option == 'm':
             self.smonth = arg
          elif option == 'p':
@@ -110,6 +112,9 @@ class LogArch(PgFile):
          return
       if op.exists(afile): self.delete_local_file(afile)
       logfiles = sorted(glob.glob("{}/access_log_gridftp??_{}??{}".format(logpath, mn, yr)))
+      if not logfiles:
+         self.pglog("{}: NO globus web log file found for {}-{} to archive".format(logpath, yr, mn), self.LGWNEM)
+         return
       topt = '-cvf'
       tcnt = 0
       for logfile in logfiles:
@@ -122,7 +127,7 @@ class LogArch(PgFile):
          lfile = op.basename(logfile)
          tcmd = "tar {} {} -C {} {}".format(topt, afile, logpath, lfile)
          tcnt += self.pgsystem(tcmd, self.LGWNEM, 5)
-         topt = '-uvf'   
+         topt = '-uvf'
       if tcnt > 0:
          self.pgsystem("gzip " + afile, self.LGWNEM, 5)
          afile += '.gz'
@@ -142,6 +147,9 @@ class LogArch(PgFile):
          return
       if op.exists(afile): self.delete_local_file(afile)
       logfiles = sorted(glob.glob("{}/{}-{}-??.log".format(logpath, yr, mn)))
+      if not logfiles:
+         self.pglog("{}: NO OSDF web log file found for {}-{} to archive".format(logpath, yr, mn), self.LGWNEM)
+         return
       topt = '-cvf'
       tcnt = 0
       for logfile in logfiles:
@@ -154,7 +162,7 @@ class LogArch(PgFile):
          afile += '.gz'
          self.move_local_file(dfile, afile, self.LGWNEM)
          s = 's' if tcnt > 1 else ''
-         self.pglog("{}: {} globus log{} tarred, gzipped and archived at {}".format(afile, tcnt, s, self.current_datetime()), self.LGWNEM)
+         self.pglog("{}: {} OSDF log{} tarred, gzipped and archived at {}".format(afile, tcnt, s, self.current_datetime()), self.LGWNEM)
 
    # Archive AWS web log files to self.LOGS['DECSLOGS']
    def archive_aws_log(self):
@@ -168,6 +176,9 @@ class LogArch(PgFile):
          return
       if op.exists(afile): self.delete_local_file(afile)
       lfile = "{}/{}".format(yr, mn)
+      if not op.exists("{}/{}".format(logpath, lfile)):
+         self.pglog("{}/{}: NO AWS web log directory found to archive".format(logpath, lfile), self.LGWNEM)
+         return
       tcmd = "tar -cvf {} -C {} {}".format(afile, logpath, lfile)
       self.pgsystem(tcmd, self.LGWNEM, 5)
       self.pgsystem("gzip " + afile, self.LGWNEM, 5)
@@ -187,6 +198,9 @@ class LogArch(PgFile):
          return
       if op.exists(afile): self.delete_local_file(afile)
       logfiles = sorted(glob.glob("{}/{}-{}-??.access.log".format(logpath, yr, mn)))
+      if not logfiles:
+         self.pglog("{}: NO thredds log file found for {}-{} to archive".format(logpath, yr, mn), self.LGWNEM)
+         return
       topt = '-cvf'
       tcnt = 0
       for logfile in logfiles:
@@ -217,6 +231,9 @@ class LogArch(PgFile):
       self.change_local_directory(self.LOGS['RDALOG'], self.LWEMEX)
       # collect all the large log/err files
       files = sorted(glob.glob("*.log") + glob.glob("*.err"))
+      if not files:
+         self.pglog("{}: NO dssdb log file found to archive".format(self.LOGS['RDALOG']), self.LGWNEM)
+         return
       for file in files:
          info = self.check_local_file(file, 2)
          if(not info or info['data_size'] < 10000): continue   # skip log files small than 10KB

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/logarch.usg

@@ -5,7 +5,7 @@
  detected and skipped (unless -n is given).
 
  USAGE:
-   logarch [-b] [-n] [-s] [-p LogPath] [-m YearMonth]
+   logarch [-b] [-n] [-s] [-c CcAddress] [-p LogPath] [-m YearMonth]
            (-a | -d | -o | -t | -w) [more archive-type flags]
 
  ARCHIVE TYPES (at least one required):
@@ -21,6 +21,10 @@
    -m YearMonth
         Archive logs for the given YYYY-MM (e.g. 2024-01) only.
 
+   -c CcAddress
+        Carbon-copy the email notice to one or more additional specialists
+        (comma/space-separated). May be given more than once.
+
    -p LogPath
         Use LogPath instead of the default log directory.
 

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/pgperson.py

@@ -53,15 +53,13 @@ class PgPerson:
       url = self.urlfmt.format(self.option, self.optval)
       response, content = http.request(url, 'GET', headers=headers)
       status = response.status
-      if status == 200:
-         persons=json.loads(content)
-         for person in persons:
-            for key, value in person.items():
-               print("{}<=>{}".format(key, value))
-      elif status == 399:
-         print(content)
-      elif status == 500:
-         print('Server error')
+      if status != 200:
+         print("pgperson: HTTP {} from {}\n{}".format(status, url, content.decode(errors='replace')), file=sys.stderr)
+         sys.exit(1)
+      persons=json.loads(content)
+      for person in persons:
+         for key, value in person.items():
+            print("{}<=>{}".format(key, value))
 
 # main function to execute this script
 def main():

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/pgusername.py

@@ -32,16 +32,15 @@ class PgUserName:
    def start_actions(self):
       headers = {'Content-type': 'application/json'}
       http=httplib2.Http()
-      response, content = http.request(self.url + self.uname, 'GET', headers=headers)
+      url = self.url + self.uname
+      response, content = http.request(url, 'GET', headers=headers)
       status = response.status
-      if status == 200:
-         person=json.loads(content)
-         for key, value in person.items():
-            print("{}<=>{}".format(key, value))
-      elif status == 399:
-         print(content)
-      elif status == 500:
-         print('Server error')
+      if status != 200:
+         print("pgusername: HTTP {} from {}\n{}".format(status, url, content.decode(errors='replace')), file=sys.stderr)
+         sys.exit(1)
+      person=json.loads(content)
+      for key, value in person.items():
+         print("{}<=>{}".format(key, value))
 
 # main function to execute this script
 def main():

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/view_allusage.py

@@ -247,7 +247,7 @@ def check_enough_options():
       if 'z' not in params or sn in EXPAND: continue
       fld = FLDS[sn]
       if fld[6] != 'G': continue
-      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
+      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(pgname, sn, fld[0]), PgLOG.LGWNEX)
 
    if 'E' in params or 'I' in params:
       if 'z' in params:

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/view_awsusage.py

@@ -224,7 +224,7 @@ def check_enough_options():
       if 'z' not in params or sn in EXPAND: continue
       fld = FLDS[sn]
       if fld[6] != 'G': continue
-      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
+      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(pgname, sn, fld[0]), PgLOG.LGWNEX)
 
    if 'E' in params or 'I' in params:
       if 'z' in params:

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/view_checkusage.py

@@ -23,7 +23,7 @@ from rda_python_common import PgDBI
 from . import PgView
 
 VUSG = {
-   'SNMS' : "ABCDEFGHIJKLMNPQRSTUVWZ",   # all available short field names in FLDS
+   'SNMS' : "ABCDEFGHIJKLMNPQRSTUVWYZ",   # all available short field names in FLDS
    'OPTS' : 'aABcCdDhHilLmnOqrsStTwyz',  # all available options, used for params
    'NOPT' : 'abwz',                      # stand alone option without inputs
    'ACND' : 'cdhilLmnsSty',              # available array condition options
@@ -120,7 +120,7 @@ params = {}
 # related to a field name if it is not in keys SNS 
 SNS = {
    'b' : 'L', 'B' : 'L', 'c' : 'C', 'd' : 'D', 'D' : 'D', 'h' : 'H', 'i' : 'I', 'm' : 'M', 'n' : 'N',
-   'q' : 'Q', 'r' : 'R', 'q' : 'Q', 's' : 'W', 'S' : 'S', 't' : 'T', 'T' : 'T', 'y' : 'Y'
+   'q' : 'Q', 'r' : 'R', 's' : 'W', 'S' : 'S', 't' : 'T', 'T' : 'T', 'y' : 'Y'
 }
 
 tablenames = fieldnames = condition = ''
@@ -203,7 +203,7 @@ def check_enough_options():
    for sn in flds:
       if sn == 'X': continue  # do not process INDEX field
       if VUSG['SNMS'].find(sn) == -1:
-         PgLOG.pglog("{}: Field sn must be in short field names: {}X".format(pgname, VUSG['SNMS']), PgLOG.LGWNEX)
+         PgLOG.pglog("{}: Field {} must be in short field names: {}X".format(pgname, sn, VUSG['SNMS']), PgLOG.LGWNEX)
 
       if 'z' not in params or sn in EXPAND: continue
       fld = FLDS[sn]

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/view_codusage.py

@@ -225,7 +225,7 @@ def check_enough_options():
       if 'z' not in params or sn in EXPAND: continue
       fld = FLDS[sn]
       if fld[6] != 'G': continue
-      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
+      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(pgname, sn, fld[0]), PgLOG.LGWNEX)
 
    if 'E' in params or 'I' in params:
       if 'z' in params:

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/view_ordusage.py

@@ -240,7 +240,7 @@ def check_enough_options():
       if 'z' not in params or sn in EXPAND: continue
       fld = FLDS[sn]
       if fld[6] != 'G': continue
-      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
+      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(pgname, sn, fld[0]), PgLOG.LGWNEX)
 
    if 'E' in params:
       if 'z' in params:

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/view_osdfusage.py

@@ -86,7 +86,7 @@ EXPAND = {
    'T' : ["DSID",   "StT", "dataset.dsid",   "dataset"],
    'P' : ["DSID",   "StT", "specialist",     "dsowner"],
 
-   'W' : ["METHOD", "fM",      "method",       "osdfusage"]
+   'W' : ["METHOD", "M",       "method",       "osdfusage"]
 }
 
 # valid options for params, a hash array of command line parameters
@@ -224,7 +224,7 @@ def check_enough_options():
       if 'z' not in params or sn in EXPAND: continue
       fld = FLDS[sn]
       if fld[6] != 'G': continue
-      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
+      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(pgname, sn, fld[0]), PgLOG.LGWNEX)
 
    if 'E' in params or 'I' in params:
       if 'z' in params:

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/view_rqstusage.py

@@ -248,7 +248,7 @@ def check_enough_options():
       if 'z' not in params or sn in EXPAND: continue
       fld = FLDS[sn]
       if fld[6] != 'G': continue
-      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
+      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(pgname, sn, fld[0]), PgLOG.LGWNEX)
 
    if 'E' in params or 'I' in params:
       if 'z' in params:

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/view_tdsusage.py

@@ -229,7 +229,7 @@ def check_enough_options():
       if 'z' not in params or sn in EXPAND: continue
       fld = FLDS[sn]
       if fld[6] != 'G': continue
-      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
+      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(pgname, sn, fld[0]), PgLOG.LGWNEX)
 
    if 'E' in params or 'I' in params:
       if 'z' in params:

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/view_webfile.py

@@ -22,15 +22,15 @@ from rda_python_common import PgDBI
 from . import PgView
 
 FILE = {
-   'SNMS' : "BCDEFGHIJKLMNOPQRSTUVWYZ",    # all available short field names in FLDS
-   'OPTS' : 'AabCdDefFgGHijJlLmMnNoOprsStTuUvwyYzZ',  # all available options, used for params
+   'SNMS' : "ABCDEFGHIJKLMNOPQRSTUWYZ",    # all available short field names in FLDS
+   'OPTS' : 'AabCdDefFgGHijJlLmMNoOprsStTuUWwyYzZ',  # all available options, used for params
    'NOPT' : 'abjJw',                       # stand alone option without inputs
-   'ACND' : 'defgGilmMnopStuvyYz',         # available array condition options
+   'ACND' : 'defgGilmMopStuWyYz',          # available array condition options
    'RCND' : 'DFNrsTZ',                     # available range condition options
-   'CNDS' : 'adDefFgGilmMnNoprstTuvyYzZ',  # condition options, ACND, RCND and 'a'
+   'CNDS' : 'adDefFgGilmMNoprstTuWyYzZ',   # condition options, ACND, RCND and 'a'
    'ECND' : 'mMyY',                        # condition options need evaluating
-   'SFLD' : 'DEFILNOPQTUVZ',               # string fields, to be quoted in condition
-   'UFLD' : 'ILOPV',                       # string fields must be in upper case
+   'SFLD' : 'DEFILNOPQTUWZ',               # string fields, to be quoted in condition
+   'UFLD' : 'ILOPW',                        # string fields must be in upper case
    'LFLD' : 'EQTU'                         # string fields must be in lower case
 }
 
@@ -91,8 +91,7 @@ FLDS = {
 #   L -- column delimiter for output
 #   m -- array of specified months of file last written
 #   M -- array of specified months of file created
-#   n -- array of specified user numbers
-#   D -- dates range, datasets created between, array of 1 or 2 dates in format of YYYY-MM-DD
+#   N -- dataset created dates range, array of 1 or 2 dates in format of YYYY-MM-DD
 #   o -- array of specified file status
 #   O -- a string of short field names for sorting on
 #   p -- array of web file types, Data, Document, and etc.
@@ -104,6 +103,7 @@ FLDS = {
 #   u -- array of specified specialist user names
 #   U -- use given unit for file or data sizes
 #   w -- generate view without totals
+#   W -- array of specified dataset types
 #   y -- array of specified years of file last written
 #   Y -- array of specified years of file created
 #   z -- array of specified dates when files created

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/view_webusage.py

@@ -270,7 +270,7 @@ def check_enough_options():
       if 'z' not in params or sn in EXPAND: continue
       fld = FLDS[sn]
       if fld[6] != 'G': continue
-      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
+      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(pgname, sn, fld[0]), PgLOG.LGWNEX)
 
    if 'E' in params or 'I' in params:
       if 'z' in params:

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewallusage.py

@@ -230,7 +230,7 @@ class ViewALLUsage(PgView):
          if 'z' not in self.params or sn in self.EXPAND: continue
          fld = self.FLDS[sn]
          if fld[6] != 'G': continue
-         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(self.pgname, sn, fld[0]), self.LGWNEX)
+         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(self.pgname, sn, fld[0]), self.LGWNEX)
       if 'E' in self.params or 'I' in self.params:
          if 'z' in self.params:
             self.pglog(self.pgname + ": option -z and -E/-I can not be present at the same time", self.LGWNEX)

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewawsusage.py

@@ -202,7 +202,7 @@ class ViewAWSUsage(PgView):
          if 'z' not in self.params or sn in self.EXPAND: continue
          fld = self.FLDS[sn]
          if fld[6] != 'G': continue
-         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(self.pgname, sn, fld[0]), self.LGWNEX)
+         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(self.pgname, sn, fld[0]), self.LGWNEX)
       if 'E' in self.params or 'I' in self.params:
          if 'z' in self.params:
             self.pglog(self.pgname + ": option -z and -E/-I can not be present at the same time", self.LGWNEX)

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewcheckusage.py

@@ -22,7 +22,7 @@ class ViewCheckUsage(PgView):
    def __init__(self):
       super().__init__()
       self.VUSG = {
-         'SNMS' : "ABCDEFGHIJKLMNPQRSTUVWZ",   # all available short field names in FLDS
+         'SNMS' : "ABCDEFGHIJKLMNPQRSTUVWYZ",   # all available short field names in FLDS
          'OPTS' : 'aABcCdDhHilLmnOqrsStTwyz',  # all available options, used for params
          'NOPT' : 'abwz',                      # stand alone option without inputs
          'ACND' : 'cdhilLmnsSty',              # available array condition options
@@ -115,7 +115,7 @@ class ViewCheckUsage(PgView):
       # related to a field name if it is not in keys SNS 
       self.SNS = {
          'b' : 'L', 'B' : 'L', 'c' : 'C', 'd' : 'D', 'D' : 'D', 'h' : 'H', 'i' : 'I', 'm' : 'M', 'n' : 'N',
-         'q' : 'Q', 'r' : 'R', 'q' : 'Q', 's' : 'W', 'S' : 'S', 't' : 'T', 'T' : 'T', 'y' : 'Y'
+         'q' : 'Q', 'r' : 'R', 's' : 'W', 'S' : 'S', 't' : 'T', 'T' : 'T', 'y' : 'Y'
       }
       self.tablenames = self.fieldnames = self.condition = ''
       self.sfields = []
@@ -187,7 +187,7 @@ class ViewCheckUsage(PgView):
       for sn in flds:
          if sn == 'X': continue  # do not process INDEX field
          if self.VUSG['SNMS'].find(sn) == -1:
-            self.pglog("{}: Field sn must be in short field names: {}X".format(self.pgname, self.VUSG['SNMS']), self.LGWNEX)
+            self.pglog("{}: Field {} must be in short field names: {}X".format(self.pgname, sn, self.VUSG['SNMS']), self.LGWNEX)
          if 'z' not in self.params or sn in self.EXPAND: continue
          fld = self.FLDS[sn]
          if fld[6] != 'G': continue

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewcodusage.py

@@ -204,7 +204,7 @@ class ViewCODUsage(PgView):
          if 'z' not in self.params or sn in self.EXPAND: continue
          fld = self.FLDS[sn]
          if fld[6] != 'G': continue
-         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(self.pgname, sn, fld[0]), self.LGWNEX)
+         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(self.pgname, sn, fld[0]), self.LGWNEX)
       if 'E' in self.params or 'I' in self.params:
          if 'z' in self.params:
             self.pglog(self.pgname + ": option -z and -E/-I can not be present at the same time", self.LGWNEX)

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewordusage.py

@@ -217,7 +217,7 @@ class ViewORDUsage(PgView):
          if 'z' not in self.params or sn in self.EXPAND: continue
          fld = self.FLDS[sn]
          if fld[6] != 'G': continue
-         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(self.pgname, sn, fld[0]), self.LGWNEX)
+         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(self.pgname, sn, fld[0]), self.LGWNEX)
       if 'E' in self.params:
          if 'z' in self.params:
             self.pglog(self.pgname + ": option -z and -E can not be present at the same time", self.LGWNEX)

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewosdfusage.py

@@ -80,7 +80,7 @@ class ViewOSDFUsage(PgView):
          'R' : ["DSID",   "StT", "search.datasets.title", "search.datasets"],
          'T' : ["DSID",   "StT", "dataset.dsid",   "dataset"],
          'P' : ["DSID",   "StT", "specialist",     "dsowner"],
-         'W' : ["METHOD", "fM",      "method",       "osdfusage"]
+         'W' : ["METHOD", "M",       "method",       "osdfusage"]
       }
       # valid options for params, a hash array of command line parameters
       #   a -- 1 to view all usage info available
@@ -202,7 +202,7 @@ class ViewOSDFUsage(PgView):
          if 'z' not in self.params or sn in self.EXPAND: continue
          fld = self.FLDS[sn]
          if fld[6] != 'G': continue
-         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(self.pgname, sn, fld[0]), self.LGWNEX)
+         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(self.pgname, sn, fld[0]), self.LGWNEX)
       if 'E' in self.params or 'I' in self.params:
          if 'z' in self.params:
             self.pglog(self.pgname + ": option -z and -E/-I can not be present at the same time", self.LGWNEX)

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewrqstusage.py

@@ -224,7 +224,7 @@ class ViewRQSTUsage(PgView):
          if 'z' not in self.params or sn in self.EXPAND: continue
          fld = self.FLDS[sn]
          if fld[6] != 'G': continue
-         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(self.pgname, sn, fld[0]), self.LGWNEX)
+         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(self.pgname, sn, fld[0]), self.LGWNEX)
       if 'E' in self.params or 'I' in self.params:
          if 'z' in self.params:
             self.pglog(self.pgname + ": option -z and -E/-I can not be present at the same time", self.LGWNEX)

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewtdsusage.py

@@ -207,7 +207,7 @@ class ViewTDSUsage(PgView):
          if 'z' not in self.params or sn in self.EXPAND: continue
          fld = self.FLDS[sn]
          if fld[6] != 'G': continue
-         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(self.pgname, sn, fld[0]), self.LGWNEX)
+         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(self.pgname, sn, fld[0]), self.LGWNEX)
       if 'E' in self.params or 'I' in self.params:
          if 'z' in self.params:
             self.pglog(self.pgname + ": option -z and -E/-I can not be present at the same time", self.LGWNEX)

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewwebfile.py

@@ -21,15 +21,15 @@ class ViewWEBFile(PgView):
    def __init__(self):
       super().__init__()
       self.FILE = {
-         'SNMS' : "BCDEFGHIJKLMNOPQRSTUVWYZ",    # all available short field names in FLDS
-         'OPTS' : 'AabCdDefFgGHijJlLmMnNoOprsStTuUvwyYzZ',  # all available options, used for params
+         'SNMS' : "ABCDEFGHIJKLMNOPQRSTUWYZ",    # all available short field names in FLDS
+         'OPTS' : 'AabCdDefFgGHijJlLmMNoOprsStTuUWwyYzZ',  # all available options, used for params
          'NOPT' : 'abjJw',                       # stand alone option without inputs
-         'ACND' : 'defgGilmMnopStuvyYz',         # available array condition options
+         'ACND' : 'defgGilmMopStuWyYz',          # available array condition options
          'RCND' : 'DFNrsTZ',                     # available range condition options
-         'CNDS' : 'adDefFgGilmMnNoprstTuvyYzZ',  # condition options, ACND, RCND and 'a'
+         'CNDS' : 'adDefFgGilmMNoprstTuWyYzZ',   # condition options, ACND, RCND and 'a'
          'ECND' : 'mMyY',                        # condition options need evaluating
-         'SFLD' : 'DEFILNOPQTUVZ',               # string fields, to be quoted in condition
-         'UFLD' : 'ILOPV',                       # string fields must be in upper case
+         'SFLD' : 'DEFILNOPQTUWZ',               # string fields, to be quoted in condition
+         'UFLD' : 'ILOPW',                       # string fields must be in upper case
          'LFLD' : 'EQTU'                         # string fields must be in lower case
       }
       # keys FLDS - short field names
@@ -88,8 +88,7 @@ class ViewWEBFile(PgView):
       #   L -- column delimiter for output
       #   m -- array of specified months of file last written
       #   M -- array of specified months of file created
-      #   n -- array of specified user numbers
-      #   D -- dates range, datasets created between, array of 1 or 2 dates in format of YYYY-MM-DD
+      #   N -- dataset created dates range, array of 1 or 2 dates in format of YYYY-MM-DD
       #   o -- array of specified file status
       #   O -- a string of short field names for sorting on
       #   p -- array of web file types, Data, Document, and etc.
@@ -101,6 +100,7 @@ class ViewWEBFile(PgView):
       #   u -- array of specified specialist user names
       #   U -- use given unit for file or data sizes
       #   w -- generate view without totals
+      #   W -- array of specified dataset types
       #   y -- array of specified years of file last written
       #   Y -- array of specified years of file created
       #   z -- array of specified dates when files created

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewwebfile.usg

@@ -17,14 +17,14 @@
       The '-C' flag may be omitted if ColumnNames is the first argument.
 
    2. At least one condition option must be given:
-        -a -d -D -e -f -F -g -G -i -l -m -M -n -o -p -r -s -t -T
-        -u -v -W -y -Y -z -Z
+        -a -d -D -e -f -F -g -G -i -l -m -M -N -o -p -r -s -t -T
+        -u -W -y -Y -z -Z
 
    3. Negation: for any condition option except -a, prepend '!' between
       the flag and value to invert the filter, e.g. '-o ! DSS'.
 
    4. Wildcards: condition options that take strings (-e, -f, -G, -i, -l,
-      -o, -t, -u, -v) accept SQL wildcards '%' (any chars) and '_' (one
+      -o, -t, -u) accept SQL wildcards '%' (any chars) and '_' (one
       char).
 
    5. By default the output is paginated with a header, footer, and
@@ -46,35 +46,36 @@
    Column letters:
 
      Group Columns:
-       A  - CYEAR       - year the file was created
        D  - DATEWRITE   - date the file was last written
        E  - EMAIL       - email of file-written user
        F  - FILENAME    - WEB data file name
        G  - PRODUCT     - product (top group) information
        H  - CMONTH      - month the file was created
        I  - FIRSTNAME   - first name of file-written user
-       K  - TIMEWRITE   - time the file was last written
+       K  - CYEAR       - year the file was created
        L  - LASTNAME    - last name of file-written user
        M  - WMONTH      - month the file was last written
-       N  - USERNO      - account number of file-written user
+       N  - DSCREATED   - date the dataset was created
        O  - STAT        - file status: Public or Internal
        P  - TYPE        - file type: Data, Software, etc.
-       Q  - TIMECREATE  - time the file was created
+       Q  - DSOWNER     - specialist who owns the dataset
        R* - DSTITLE     - dataset title
        S  - FILESIZE    - individual file size (bytes)
        T  - DATASET     - dataset name
-       U  - USERNAME    - login name of file-written user
-       V  - DIV.        - division of file-written user
-       W  - DSTYPE      - dataset type flag: P, H, I, or W
+       U  - SPECIALIST  - login name of file-written user
+       W  - DSTYPE      - dataset type flag: D, H, I, P, or W
        Y  - WYEAR       - year the file was last written
        Z  - DATECREATE  - date the file was created
 
        (*) = column may be displayed with zero usages when -z is used.
 
      Summary Columns:
-       B - MBYTESFILES  - data size (MB) by the group
-       C - #USER        - number of users in the group
-       J - #FILE        - number of files by the group
+       B - MBYTEDATA    - data size (MB) by the group
+
+     Count Columns:
+       A - DSCOUNT      - number of distinct file names in the group
+       C - #UNIQSPLST   - number of distinct specialists in the group
+       J - #UNIQFILE    - number of distinct files in the group
 
      Index Column:
        X - INDEX        - 1-based row index (use as the first column)
@@ -97,7 +98,7 @@
    -l LastNameList                   Users with these last names.
    -m MonthList                      Last written in these months (YYYY-MM).
    -M MonthList                      Created in these months (YYYY-MM).
-   -n UserNumberList                 These user account numbers.
+   -N StartDate [EndDate]            Datasets created between two dates.
    -o FileStatus                     File status: P (Public) or I (Internal).
    -p TypeList                       File types: D (Data), I (Internal Data),
                                      O (Document), S (Software).
@@ -108,9 +109,7 @@
                                      e.g. '-T d540000 d550009'.
    -u UserNameList                   Last written by users with these
                                      login names.
-   -v DivisionList                   Last written by users from these
-                                     divisions.
-   -W DatasetTypes                   Dataset type flag: P, H, I, or W.
+   -W DatasetTypes                   Dataset type flag: D, H, I, P, or W.
    -y YearList                       Last written in these years (YYYY).
    -Y YearList                       Created in these years (YYYY).
    -z DateList                       Created on these dates.
@@ -129,8 +128,8 @@
 
  EXAMPLES:
 
-   View files last written in 2004 with columns INDEX, USERNAME, STAT,
-   #FILE, MBYTESFILES; sort by STAT ascending and MBYTESFILES descending:
+   View files last written in 2004 with columns INDEX, SPECIALIST, STAT,
+   #UNIQFILE, MBYTEDATA; sort by STAT ascending and MBYTEDATA descending:
 
       viewwebfile XUOJB -y 2004 -O Ob
 

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics/viewwebusage.py

@@ -247,7 +247,7 @@ class ViewWEBUsage(PgView):
          if 'z' not in self.params or sn in self.EXPAND: continue
          fld = self.FLDS[sn]
          if fld[6] != 'G': continue
-         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(self.pgname, sn, fld[0]), self.LGWNEX)
+         self.pglog("{}: cannot show zero usage for unexpandable field {} - {}".format(self.pgname, sn, fld[0]), self.LGWNEX)
       if 'E' in self.params or 'I' in self.params:
          if 'z' in self.params:
             self.pglog(self.pgname + ": option -z and -E/-I can not be present at the same time", self.LGWNEX)

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 3.0.2
+Version: 3.0.4
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-3.0.2 → rda_python_metrics-3.0.4}/src/rda_python_metrics.egg-info/SOURCES.txt

@@ -42,6 +42,7 @@ src/rda_python_metrics/filltdsusage.usg
 src/rda_python_metrics/filluser.py
 src/rda_python_metrics/filluser.usg
 src/rda_python_metrics/fillzenodousage.py
+src/rda_python_metrics/fillzenodousage.usg
 src/rda_python_metrics/log_arch.py
 src/rda_python_metrics/logarch.py
 src/rda_python_metrics/logarch.usg