rda-python-metrics 1.0.34__py3-none-any.whl → 1.0.36__py3-none-any.whl

rda_python_metrics/fillawsusage.py

@@ -94,7 +94,7 @@ def get_log_file_names(option, params):
       else:
          pdate = PgUtil.format_date(params[0])
          if len(params) > 1:
-            edate = PgUtil.format_date(params[1])
+            edate = PgUtil.adddate(PgUtil.format_date(params[1]), 0, 0, 1)
          else:
             edate = PgUtil.curdate()
       while pdate < edate:
@@ -114,13 +114,14 @@ def fill_aws_usages(filenames):
    year = cntall = addall = 0
    for pdate in filenames:
       fnames = filenames[pdate]
+      fcnt = len(fnames)
+      PgLOG.pglog("{}: Gathering AWS usage info from {} log files at {}".format(pdate, fcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
       records = {}
       cntadd = entcnt = 0
       for logfile in fnames:
          if not op.isfile(logfile):
             PgLOG.pglog("{}: Not exists for Gathering AWS usage".format(logfile), PgLOG.LOGWRN)
             continue
-         PgLOG.pglog("Gathering AWS usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
          aws = PgFile.open_local_file(logfile)
          if not aws: continue
          while True:
@@ -184,7 +185,7 @@ def add_usage_records(records, year):
    cnt = 0
    for key in records:
       record = records[key]
-      cond = "date = '{}' AND time = '{}' AND ip = '{}' AND dsid = '{}'".format(record['date'], record['time'], record['ip'], record['dsid'])
+      cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(record['date'], record['time'], record['ip'])
       if PgDBI.pgget(USAGE['PGTBL'], '', cond, PgLOG.LGEREX): continue
       if add_to_allusage(year, record):
          cnt += PgDBI.pgadd(USAGE['PGTBL'], record, PgLOG.LOGWRN)

rda_python_metrics/fillglobususage.py

@@ -147,18 +147,14 @@ def fill_globus_usages(fnames, datelimits):
          locflag = 'O' if re.match(r'^https://stratus\.', sline) else 'G'
          idx = wfile.find('?')
          if idx > -1: wfile = wfile[:idx]
-
-         if re.match(r'^curl', engine, re.I):
-            method = "CURL"
-         elif re.match(r'^wget', engine, re.I):
-            method = "WGET"
-         elif re.match(r'^python', engine, re.I):
-            method = "PYTHN"
+         moff = engine.find('/')
+         if moff > 0:
+            if moff > 20: moff = 20
+            method = engine[0:moff].upper()
          else:
             method = "WEB"
 
          key = "{}:{}:{}".format(ip, dsid, wfile) if stat == '206' else None
-
          if record:
             if key == pkey:
                record['size'] += size

rda_python_metrics/fillosdfusage.py

@@ -185,7 +185,7 @@ def add_usage_records(records, year):
    cnt = 0
    for key in records:
       record = records[key]
-      cond = "date = '{}' AND time = '{}' AND ip = '{}' AND dsid = '{}'".format(record['date'], record['time'], record['ip'], record['dsid'])
+      cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(record['date'], record['time'], record['ip'])
       if PgDBI.pgget(USAGE['OSDFTBL'], '', cond, PgLOG.LGEREX): continue
       if add_to_allusage(year, record):
          cnt += PgDBI.pgadd(USAGE['OSDFTBL'], record, PgLOG.LOGWRN)
@@ -203,69 +203,6 @@ def add_to_allusage(year, pgrec):
 
    return PgDBI.add_yearly_allusage(year, record)
 
-
-#
-# Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
-#
-def add_file_usage(year, logrec):
-
-   pgrec = get_wfile_wid(logrec['dsid'], logrec['wfile'])
-   if not pgrec: return 0
-
-   table = "{}_{}".format(USAGE['OSDFTBL'], year)
-   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
-   if PgDBI.pgget(USAGE['OSDFTBL'], "", cond, PgLOG.LOGWRN): return 0
-
-   wurec =  PgIPInfo.get_wuser_record(logrec['ip'], logrec['date'])
-   if not wurec: return 0
-   record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
-   record['wuid_read'] = wurec['wuid']
-   record['date_read'] = logrec['date']
-   record['time_read'] = logrec['time']
-   record['size_read'] = logrec['size']
-   record['method'] = logrec['method']
-   record['locflag'] = logrec['locflag']
-   record['ip'] = logrec['ip']
-   record['quarter'] = logrec['quarter']
-
-   if add_to_allusage(year, logrec, wurec):
-      return PgDBI.add_yearly_wusage(year, record)
-   else:
-      return 0
-
-def add_to_allusage(year, logrec, wurec):
-
-   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'],
-            'country' : wurec['country'], 'region' : wurec['region']}
-   pgrec['dsid'] = logrec['dsid']
-   pgrec['date'] = logrec['date']
-   pgrec['quarter'] = logrec['quarter']
-   pgrec['time'] = logrec['time']
-   pgrec['size'] = logrec['size']
-   pgrec['method'] = logrec['method']
-   pgrec['ip'] = logrec['ip']
-   pgrec['source'] = 'P'
-   return PgDBI.add_yearly_allusage(year, pgrec)
-
-#
-# return wfile.wid upon success, 0 otherwise
-#
-def get_wfile_wid(dsid, wfile):
-
-   wfcond = "wfile = '{}'".format(wfile) 
-   pgrec = PgSplit.pgget_wfile(dsid, "*", wfcond)
-   if pgrec:
-      pgrec['dsid'] = dsid
-   else:
-      pgrec = PgDBI.pgget("wfile_delete", "*", "{} AND dsid = '{}'".format(wfcond, dsid))
-      if not pgrec:
-         pgrec = PgDBI.pgget("wmove", "wid, dsid", wfcond)
-         if pgrec:
-            pgrec = PgSplit.pgget_wfile(pgrec['dsid'], "*", "wid = {}".format(pgrec['wid']))
-            if pgrec: pgrec['dsid'] = dsid
-
-   return pgrec
-
 #
 # call main() to start program
 #

rda_python_metrics/viewawsusage.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : viewawsusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 2025-08-13
+#   Purpose : python program to view aws usage information
+#
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+#
+import os
+import re
+import sys
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgDBI
+from . import PgView
+
+VUSG = {
+   'SNMS' : "ABCDEHIKMNOPQRSTUWY",           # all available short field names in %FLDS
+   'OPTS' : 'AabcCdDeEhHikLmMnoOqsStTUwyz',  # all available options, used for %params
+   'NOPT' : 'abhnwz',                        # stand alone option without inputs
+   'ACND' : 'cdeiIkmMoqSty',                 # available array condition options
+   'RCND' : 'DEsT',                          # available range condition options
+   'CNDS' : 'acdDeEiIkmMnoqsStTy',           # condition options, ACND, RCND and 'a'
+   'ECND' : 'my',                            # condition options need evaluating
+   'SFLD' : 'DEIKNOTUW',                     # string fields, to be quoted in condition
+   'UFLD' : 'NO',                            # string fields must be in upper case
+   'LFLD' : 'EMPT'                           # string fields must be in lower case
+}
+
+# keys %FLDS - short field names
+# column 0   - column title showing in usage view
+# column 1   - field name in format as shown in select clauses
+# column 2   - field name shown in where condition query string
+# column 3   - table name that the field belongs to 
+# column 4   - output field length, the longer one of data size and comlun title, determine
+#              dynamically if it is 0. Negative values indicate right justification
+# column 5   - precision for floating point value if positive and show total value if not zero
+# column 6   - field flag to indicate it is a group, distinct or sum field
+FLDS = {
+# SHRTNM COLUMNNANE   FIELDNAME                         CNDNAME       TBLNAM        Size Prc Grp/Sum
+   'D' : ['DATE',      "date",                           'date',       'awsusage',  10,   0,  'G'],
+   'E' : ['EMAIL',     "awsusage.email",        'awsusage.email',      'awsusage',   0,   0,  'G'],
+   'I' : ['IP',        "ip",                             'ip',         'awsusage',   0,   0,  'G'],
+   'M' : ['MONTH',     PgDBI.fmtym("date"),              'date',       'awsusage',   7,   0,  'G'],
+   'N' : ['COUNTRY',   "country",                        'country',    'awsusage',   0,   0,  'G'],
+   'K' : ['REGION',    "region",                         'region',     'awsusage',   0,   0,  'G'],
+   'O' : ['ORGTYPE',   "org_type",                       'org_type',   'awsusage',   7,   0,  'G'],
+   'P' : ['DSOWNER',   "specialist",                     'specialist', 'dsowner',    8,   0,  'G'],
+   'Q' : ['QUARTER',   "quarter",                        'quarter',    'awsusage',   7,   0,  'G'],
+   'R' : ['DSTITLE',   "search.datasets.title", 'search.datasets.title',   'search.datasets',   0,   0,  'G'],
+   'S' : ['BYTESIZE',  "size",                           'size',       'awsusage', -14,  -1,  'G'],
+   'T' : ['DATASET',   "awsusage.dsid",         'awsusage.dsid',       'awsusage',   0,   0,  'G'],
+   'W' : ['METHOD',    "method",                         'method',      'awsusage',  0,   0,  'G'],
+   'Y' : ['YEAR',      PgDBI.fmtyr("date"),              'date',       'awsusage',   4,   0,  'G'],
+   'A' : ['DSCOUNT',   "awsusage.dsid",                  'A',          'awsusage',  -7,  -1,  'D'],
+   'B' : ['MBYTEREAD', "round(sum(size)/(1000000), 4)",  'B',          'awsusage', -14,   3,  'S'],
+   'C' : ['#UNIQUSER', "awsusage.email",                 'C',          'awsusage',  -9,  -1,  'D'],
+   'U' : ['#UNIQIP',   "awsusage.ip",                    'U',          'awsusage',  -7,  -1,  'D'],
+   'H' : ['#ACCESS',   "sum(fcount)",                    'H',          'awsusage',  -8,  -1,  'S'],
+   'X' : ['INDEX',     "",                               'X',          '',          -6,   0,  ' ']
+}
+
+# keys %EXPAND - short field names allow zero usage
+# column 0   - expand ID for group of fields
+# column 1   - field name shown in where condition query string
+# column 2   - field name in format as shown in select clauses
+# column 3   - table name that the field belongs to 
+EXPAND = {
+# SHRTNM EXPID     CNDSTR    FIELDNAME       TBLNAM
+   'D' : ["TIME",   "dDmy"],
+   'M' : ["TIME",   "dDmy"],
+   'Q' : ["TIME",   "dDmy"],
+   'Y' : ["TIME",   "dDmy"],
+
+   'E' : ["USER",   "ecko",  "email",        "wuser",  "user"],
+   'O' : ["USER",   "ecko",  "org_type",     "wuser",  "user"],
+   'N' : ["USER",   "ecko",  "country",      "wuser",  "user"],
+   'K' : ["USER",   "ecko",  "region",       "wuser",  "user"],
+
+   'R' : ["DSID",   "StT", "search.datasets.title", "search.datasets"],
+   'T' : ["DSID",   "StT", "dataset.dsid",   "dataset"],
+   'P' : ["DSID",   "StT", "specialist",     "dsowner"],
+
+   'W' : ["METHOD", "M",      "method",       "awsusage"]
+}
+
+# valid options for %params, a hash array of command line parameters
+#   a -- 1 to view all usage info available
+#   A -- number or records to return
+#   c -- array of specified country codes
+#   C -- a string of short field names for viewing usages
+#   d -- array of specified dates 
+#   D -- dates range, array of 1 or 2 dates in format of YYYY-MM-DD
+#   e -- array of specified email addresses
+#   E -- use given date or date range for email notice of data update
+#   h -- for give emails, include their histical emails registered before
+#   H -- a string of report title to replace the default one
+#   i -- array of specified IP addresses
+#   I -- use given email IDs for email notice of data update
+#   k -- array of specified region names
+#   L -- column delimiter for output
+#   m -- array of specified months 
+#   M -- array of specified download methods 
+#   o -- array of specified orginization types
+#   O -- a string of short field names for sorting on
+#   q -- array of the specified quarters, normally combined with years
+#   s -- size range, arrage of 1 or 2 sizes in unit of MByte
+#   S -- array of login names of specialists who owns the datasets
+#   t -- array of specified dataset names
+#   T -- dataset range, array of 1 or 2 dataset names
+#   U -- given unit for file or data sizes
+#   w -- generate view without totals
+#   y -- array of specified years 
+#   z -- generate view including entries without usage
+
+params = {}
+
+# relationship between parameter options and short field names, A option is not
+# related to a field name if it is not in keys %SNS 
+SNS = {
+   'c' : 'N', 'd' : 'D', 'D' : 'D', 'e' : 'E', 'i' : 'I', 'k' : 'K', 'm' : 'M',
+   'M' : 'W', 'o' : 'O', 'q' : 'Q', 's' : 'S', 'S' : 'P', 't' : 'T', 'T' : 'T', 'y' : 'Y'
+}
+
+tablenames = fieldnames = condition = ''
+sfields = []
+gfields = []
+dfields = []
+pgname = 'viewawsusage'
+
+#
+# main function to run this program
+#
+def main():
+
+   PgDBI.view_dbinfo()
+   argv = sys.argv[1:]
+   inputs = []
+   option = 'C'   # default option
+
+   for arg in argv:
+      if re.match(r'^-.*$', arg):
+         curopt = arg[1:2]
+         if curopt and VUSG['OPTS'].find(curopt) > -1:
+            if VUSG['NOPT'].find(option) > -1:
+               params[option] = 1
+            elif inputs:
+               params[option]= inputs   # record input array
+               inputs = []      # empty input array
+            option = curopt     # start a new option
+         else:
+            PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGWNEX)
+      else:
+         val = arg
+         if val != '!':
+            if option == 's':
+               val = int(val)*1000000    # convert MBytes to Bytes
+            elif option in SNS:
+               sfld = SNS[option]
+               if VUSG['SFLD'].find(sfld) > -1:
+                  if VUSG['UFLD'].find(sfld) > -1:
+                     val = arg.upper()     # in case not in upper case
+                  elif VUSG['LFLD'].find(sfld) > -1:
+                     val = arg.lower()     # in case not in lower case
+                  if option == 'c':
+                     val = PgView.get_country_name(val)
+                  elif option == 't' or option == 'T':
+                     val = PgUtil.format_dataset_id(val)   # add 'ds' if only numbers
+                  val = "'{}'".format(val)
+         inputs.append(val)
+
+   # record the last option
+   if VUSG['NOPT'].find(option) > -1:
+      params[option] = 1
+   elif inputs:
+      params[option] = inputs   # record input array
+   
+   if not params:
+      PgLOG.show_usage(pgname)
+   else:
+      check_enough_options()
+   
+   if 'o' not in params:
+      if 'e' not in params:
+         params['o'] = ['!', "'DSS'"]   # default to exclude 'DSS' for organization
+   elif params['o'][0] == "'ALL'":
+      del params['o']
+   
+   usgtable = "awsusage"
+   build_query_strings(usgtable)  # build tablenames, fieldnames, and conditions
+   records = PgDBI.pgmget(tablenames, fieldnames, condition, PgLOG.UCLWEX)
+   if not records: PgLOG.pglog("No Usage Found For Given Conditions", PgLOG.LGWNEX)
+   totals = None if 'w' in params else {}
+   if dfields or totals != None:
+      records = PgView.compact_hash_groups(records, gfields, sfields, dfields, totals)
+   if 'z' in params: records = expand_records(records)
+   ostr = params['O'][0] if 'O' in params else params['C'][0]
+   records = PgView.order_records(records, ostr.replace('X', ''))
+   PgView.simple_output(params, FLDS, records, totals)
+
+   PgLOG.pgexit(0)
+
+#
+# cehck if enough information entered on command line for generate view/report, exit if not
+#
+def check_enough_options():
+
+   cols = params['C'][0] if 'C' in params else 'X'
+   if cols == 'X': PgLOG.pglog("{}: miss field names '{}'".format(pgname, VUSG['SNMS']), PgLOG.LGWNEX)
+
+   if cols.find('Q') > -1 and cols.find('Y') < 0:   # add Y if Q included
+      cols = re.sub('Q', 'YQ', cols)
+      params['C'][0] = cols
+
+   for sn in cols:
+      if sn == 'X': continue  # do not process INDEX field
+      if VUSG['SNMS'].find(sn) < 0:
+         PgLOG.pglog("{}: Field {} must be in field names '{}X'".format(pgname, sn, VUSG['SNMS']), PgLOG.LGWNEX)
+      if 'z' not in params or sn in EXPAND: continue
+      fld = FLDS[sn]
+      if fld[6] != 'G': continue
+      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
+
+   if 'E' in params or 'I' in params:
+      if 'z' in params:
+         PgLOG.pglog(pgname + ": option -z and -E/-I can not be present at the same time", PgLOG.LGWNEX)
+      elif 't' not in params or len(params['t']) > 1:
+         PgLOG.pglog(pgname + ": specify one dataset for viewing usage of notified users", PgLOG.LGWNEX)
+      elif 'E' in params and 'I' in params:
+         PgLOG.pglog(pgname + ": option -E and -I can not be present at the same time", PgLOG.LGWNEX)
+
+   for opt in params:
+      if VUSG['CNDS'].find(opt) > -1: return
+   PgLOG.pglog("{}: miss condition options '{}'".format(pgname, VUSG['CNDS']), PgLOG.LGWNEX)
+
+#
+# process parameter options to build aws query strings
+# global variables are used directly and nothing passes in and returns back
+#
+def build_query_strings(usgtable):
+
+   # initialize query strings
+   global condition, fieldnames, tablenames
+   joins = groupnames = ''
+   tablenames = usgtable
+   cols = params['C'][0]
+
+   if 'U' in params:    # reset units for file and read sizes
+      if cols.find('B') > -1: FLDS['B'] = PgView.set_data_unit(FLDS['B'], params['U'][0], "sum(size)")
+      if cols.find('S') > -1: FLDS['S'] = PgView.set_data_unit(FLDS['S'], params['U'][0], "size")
+
+   if 'e' in params and 'h' in params: params['e'] = PgView.include_historic_emails(params['e'], 3)
+
+   for opt in params:
+      if opt == 'C':   # build field, table and group names
+         for sn in cols:
+            if sn == 'X': continue  # do not process INDEX field
+            fld = FLDS[sn]
+            if fieldnames: fieldnames += ', '
+            fieldnames += "{} {}".format(fld[1], sn)   # add to field name string
+            (tablenames, joins) = PgView.join_query_tables(fld[3], tablenames, joins, usgtable)
+            if fld[6] == 'S':
+               sfields.append(sn)
+            else:
+               if groupnames: groupnames += ', '
+               groupnames += sn     # add to group name string
+               if fld[6] == 'D':
+                  dfields.append(sn)
+               else:
+                  gfields.append(sn)
+      elif opt == 'O':
+         continue   # order records later
+      elif VUSG['CNDS'].find(opt) > -1:
+         if VUSG['NOPT'].find(opt) > -1: continue
+         sn = SNS[opt]
+         fld = FLDS[sn]
+         # build having and where conditon strings
+         cnd = PgView.get_view_condition(opt, sn, fld, params, VUSG)
+         if cnd:
+            if condition: condition += ' AND '
+            condition += cnd
+            (tablenames, joins) = PgView.join_query_tables(fld[3], tablenames, joins, usgtable)
+
+
+   # append joins, group by, order by, and having strings to condition string
+   if 'E' in params or 'I' in params:
+      (tablenames, joins) = PgView.join_query_tables("emreceive", tablenames, joins, usgtable)
+   if joins:
+      if condition:
+         condition = "{} AND {}".format(joins, condition)
+      else:
+         condition = joins
+   if 'E' in params or 'I' in params:
+      condition += PgView.notice_condition(params['E'], None, params['t'][0])
+   if groupnames and sfields: condition += " GROUP BY " + groupnames
+
+
+def expand_records(records):
+
+   recs = PgView.expand_query("TIME", records, params, EXPAND)
+
+   trecs = PgView.expand_query("USER", records, params, EXPAND, VUSG, SNS, FLDS)
+   recs = PgUtil.crosshash(recs, trecs)
+
+   trecs = PgView.expand_query("DSID", records, params, EXPAND, VUSG, SNS, FLDS)
+   recs = PgUtil.crosshash(recs, trecs)
+
+   trecs = PgView.expand_query("METHOD", records, params, EXPAND, VUSG, SNS, FLDS)
+   recs = PgUtil.crosshash(recs, trecs)
+
+   return PgUtil.joinhash(records, recs, 0, 1)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/viewawsusage.usg

@@ -0,0 +1,190 @@
+
+ View usage information of AWS Data Services from information
+ stored in PostgreSQL database 'RDADB'.
+
+ Usage: viewawsusage [-C] ColumnNames [-O OrderColumnNames] [-a] \
+                     [-A RowLimit] [-c CountryCodes] [-d DateList] \
+                     [-D StartDate [EndDate]] [-e EMailList] -h \
+                     [-E StartNoticeDate [EndNoticeDate]] \
+                     [-i IPAddresses] [-I EmailIDList] \
+                     [-k RegionNames] [-m MonthList] [-M AccessMethods]  \
+                     [-N MinNumberRead [MaxNumberRead]] \
+                     [-o OrganizationTypes] \
+                     [-q QuaterList] [-s MinSize [MaxSize]] \
+                     [-S SpecialistLoginNames] [-t DatasetList] \
+                     [-T MinDataset [MaxDataset]] [-y YearList] \
+                     [-H Title] [-L Delimiter] [-U SizeUnit] \
+                     [-w] [-z] [> OutputFileName] [| lp -d PrinterName] 
+
+      Specify [-C] ColumnNames, refer to Option -C section for detail
+      description, and choose at least one of the condition options, -a, -c,
+      -d, -D, -e, -E, -i, -I, -k, -m, -M, -N, -o, -q, -s, -S -t, -T, and -y,
+      to run this application.
+
+      For all condition options, except option -a, an '!' sign can be added
+      between an option flag and its option values to get an excluding
+      condition. For example, choose '-o ! OrganizationTypes' to gather order
+      data usage by users from organization types other than the ones given in
+      OrganizationTypes.  Refer to the example given at the end of this help
+      document for how to select excluding condition.
+
+      String condition options, -c, -e, -g, -i, -k, -M, -o, -S, and -t, allow
+      wildcard inputs. '%' matches any number of characters and '_' matches any one
+      character.  Refer to the example given at the end of this help document
+      for how to use wildcard for string condition options.
+
+      Output of this application is defaulted to page format with a page
+      header on each page. A page header includes main title, sub titles and
+      column titles according to which column names and options are selected,
+      as well as page number and report date. If the output is used directly
+      for input of other applications, add option -w to remove page header
+      and show only the column titles and the usage information.
+
+
+      Column Options:
+      - Option -C, the ColumnNames must be present to run this application.
+           The flag -C can be omitted if it is the first parameter option on
+           the command line. The ColumnNames is a string that includes column
+           names listed below:
+
+         COLUMN - COLUMN       - COLUMN
+           NAME - TITLE        - DESCRIPTION 
+          GroupColumns:
+              D*- DATE         - format as YYYY-MM-DD, for example 2004-04-25
+              E*- EMAIL        - user email address
+              I*- IP           - user IP address
+              M*- MONTH        - format as YYYY-MM, for example 2004-04
+              N*- COUNTRY      - country codes users from
+              K*- REGION       - region names users from
+              O*- ORGTYPE      - organization types (DSS, NCAR, UNIV and OTHER)
+              P*- DSOWNER      - login names of specialists who own the datasets
+              Q*- QUARTER      - quarter of year, 1, 2, 3, or 4
+              R*- DSTITLE      - dataset titles
+              S - BSIZE        - size of data read each time, default to Bytes
+              T*- DATASET      - format as dsnnn.n, for example d540001
+              W*- METHOD       - access methods
+              Y*- YEAR         - format as YYYY, for example 2004
+
+                * - field names can processed with zero usages
+          SummaryColumns:
+              A - DSCOUNT      - number of datasets in given GroupColumns
+              B - MBREAD       - data sizes, default MB, read by given GroupColumns
+              C - #UNIQUSER    - number of unique users in in given GroupColumns
+              U - #UNIQIP      - number of unique users in in given GroupColumns
+              H - #READ        - number of reads by given GroupColumns
+ 
+          IndexColumn:
+              X - INDEX        - index of line, it should be the first column
+
+           The column names are used to build up string of ColumnNames, while
+           their associated column titles are shown in view/report output of
+           this application. The display order of the column titles is
+           determined by the order of the column names in the ColumnNames
+           string. At least one of the group and summary columns must be
+           selected, in the ColumnNames string, to generate all usage
+           view/report;
+
+           For example, choose '-C EMB' to display column titles of EMAIL,
+           MONTH and MBREAD, in the first, second and third columns
+           respectively, for numbers of MBytes of data read by each user
+           in each month;
+
+      - Option -O, sort data usage information in ascending or descending
+           order based on the column names specified in OrderColumnNames
+           string. These column names must be in the selected [-C]
+           ColumnNames string. If an column name is in upper case, its
+           associated column is sorted in ascending order, and a lower
+           case means sorting in descending order;
+
+
+      Condition Options:
+      - Option -a, for all usage in table 'awsusage';
+
+      - Option -A, gives a row limit for querrying; 
+
+      - Option -c, for files read by users from given country codes;
+
+      - Option -d, for data read on given dates, in format YYYY-MM-DD;
+
+      - Option -D, for data read between two given dates, each date
+           is in format YYYY-MM-DD. Omit EndDate for no upper limit;
+
+      - Option -e, for data read by users with given email addresses;
+
+      - Option -E, for data read by users who have been notified 
+           data update of a specified dataset between two given dates,
+           each date is in format YYYY-MM-DD. Omit EndNoticeDate for
+           no upper limit;
+
+      - Option -h, works with Option -e to include historical user emails
+           registered before;
+
+      - Option -i, for data read from machines with given IP addresses;
+ 
+      - Option -k, for files read by users from given region names;
+
+      - Option -m, for data read in given months, in format YYYY-MM;
+
+      - Option -M, for data read via access methods;
+ 
+      - Option -N, for files for numbers of read by each group between
+           MinNumberRead and MaxNumberRead. Omit MaxNumberRead for no
+           upper limit;
+
+      - Option -o, for data read by users from given orgnization types.
+           It defaults to -o ! DSS to exclude usage from DSS specialists;
+           Set it to ALL to include all orgnization types;
+
+      - Option -q, for data read in given quarters;
+
+      - Option -s, for data sizes, unit of MByte, between MinSize and MaxSize.
+           Omit MaxSize for no upper limit;
+
+      - Option -S, for login names of specialsts who own the datasets;
+
+      - Option -t, for data associating to given dataset names;
+
+      - Option -T, for data associating to datasets between 
+           MinDataset and MaxDataset. Omit MaxDataset for no upper limit.
+           For example, -T d540000 d550009, for datasets numbers d540000-d550009;
+
+      - Option -y, for data read in given years in format YYYY;
+
+
+      Miscellaneous Options:
+      - Option -w, view data usage in simple format without totals;
+
+      - Option -z, include datasets without without usage
+
+      - Option -H, use given report title to replace the default one;
+
+      - Option -L, use given delimiter for output, instead of defaulted spaces;
+
+      - Option -U, show data sizes in given unit SizeUnit [BKMG].
+           B - Byte, K - KiloBytes, M  - MegaByte, and G - GigaByte;
+
+      - Option > OutputFilename, redirect output into an output file,
+           for example, ordusage.out, instead of viewing on screen directly;
+
+      - Option | lp -d PrinterName, redirect output to printer of PrinterName.
+           Replace PrinterName with lj100 to print through DSS LaserJet printer.
+
+
+      For example:
+      To view annual data usage in year 2005 with columns, INDEX(X),
+      EMAIL(E), ORGTYPE(O), #READ(H), and MBREAD(B); ordered by ORGTYPE as
+      ascending and MBREAD(B) as descending; the command line should be:
+
+         viewawsusage XEOHB -y 2005 -O Ob
+
+      For usage by users not in Organization 'DDS', out of the file usage
+      gathered above, the command line should be:
+
+         viewawsusage XEOHB -y 2005 -o ! DSS -O Ob
+
+      To redirect the previous output to a file named awsusage.out:
+
+         viewawsusage XEOHB -y 2005 ! DSS -O Ob > awsusage.out
+
+      Then you can view the file or print it as a report.
+

rda_python_metrics/viewosdfusage.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : viewosdfusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 2025-08-13
+#   Purpose : python program to view osdf usage information
+#
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+#
+import os
+import re
+import sys
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgDBI
+from . import PgView
+
+VUSG = {
+   'SNMS' : "ABCDEHIKMNOPQRSTUWY",           # all available short field names in %FLDS
+   'OPTS' : 'AabcCdDeEhHikLmMnoOqsStTUwyz',  # all available options, used for %params
+   'NOPT' : 'abhnwz',                        # stand alone option without inputs
+   'ACND' : 'cdeiIkmMoqSty',                 # available array condition options
+   'RCND' : 'DEsT',                          # available range condition options
+   'CNDS' : 'acdDeEiIkmMnoqsStTy',           # condition options, ACND, RCND and 'a'
+   'ECND' : 'my',                            # condition options need evaluating
+   'SFLD' : 'DEIKNOTUW',                     # string fields, to be quoted in condition
+   'UFLD' : 'NO',                            # string fields must be in upper case
+   'LFLD' : 'EMPT'                           # string fields must be in lower case
+}
+
+# keys %FLDS - short field names
+# column 0   - column title showing in usage view
+# column 1   - field name in format as shown in select clauses
+# column 2   - field name shown in where condition query string
+# column 3   - table name that the field belongs to 
+# column 4   - output field length, the longer one of data size and comlun title, determine
+#              dynamically if it is 0. Negative values indicate right justification
+# column 5   - precision for floating point value if positive and show total value if not zero
+# column 6   - field flag to indicate it is a group, distinct or sum field
+FLDS = {
+# SHRTNM COLUMNNANE   FIELDNAME                         CNDNAME       TBLNAM        Size Prc Grp/Sum
+   'D' : ['DATE',      "date",                           'date',       'osdfusage',  10,   0,  'G'],
+   'E' : ['EMAIL',     "osdfusage.email",        'osdfusage.email',      'osdfusage',   0,   0,  'G'],
+   'I' : ['IP',        "ip",                             'ip',         'osdfusage',   0,   0,  'G'],
+   'M' : ['MONTH',     PgDBI.fmtym("date"),              'date',       'osdfusage',   7,   0,  'G'],
+   'N' : ['COUNTRY',   "country",                        'country',    'osdfusage',   0,   0,  'G'],
+   'K' : ['REGION',    "region",                         'region',     'osdfusage',   0,   0,  'G'],
+   'O' : ['ORGTYPE',   "org_type",                       'org_type',   'osdfusage',   7,   0,  'G'],
+   'P' : ['DSOWNER',   "specialist",                     'specialist', 'dsowner',    8,   0,  'G'],
+   'Q' : ['QUARTER',   "quarter",                        'quarter',    'osdfusage',   7,   0,  'G'],
+   'R' : ['DSTITLE',   "search.datasets.title", 'search.datasets.title',   'search.datasets',   0,   0,  'G'],
+   'S' : ['BYTESIZE',  "size",                           'size',       'osdfusage', -14,  -1,  'G'],
+   'T' : ['DATASET',   "osdfusage.dsid",         'osdfusage.dsid',       'osdfusage',   0,   0,  'G'],
+   'W' : ['METHOD',    "method",                         'method',      'osdfusage',  0,   0,  'G'],
+   'Y' : ['YEAR',      PgDBI.fmtyr("date"),              'date',       'osdfusage',   4,   0,  'G'],
+   'A' : ['DSCOUNT',   "osdfusage.dsid",                  'A',          'osdfusage',  -7,  -1,  'D'],
+   'B' : ['MBYTEREAD', "round(sum(size)/(1000000), 4)",  'B',          'osdfusage', -14,   3,  'S'],
+   'C' : ['#UNIQUSER', "osdfusage.email",                 'C',          'osdfusage',  -9,  -1,  'D'],
+   'U' : ['#UNIQIP',   "osdfusage.ip",                    'U',          'osdfusage',  -7,  -1,  'D'],
+   'H' : ['#ACCESS',   "sum(fcount)",                    'H',          'osdfusage',  -8,  -1,  'S'],
+   'X' : ['INDEX',     "",                               'X',          '',          -6,   0,  ' ']
+}
+
+# keys %EXPAND - short field names allow zero usage
+# column 0   - expand ID for group of fields
+# column 1   - field name shown in where condition query string
+# column 2   - field name in format as shown in select clauses
+# column 3   - table name that the field belongs to 
+EXPAND = {
+# SHRTNM EXPID     CNDSTR    FIELDNAME       TBLNAM
+   'D' : ["TIME",   "dDmy"],
+   'M' : ["TIME",   "dDmy"],
+   'Q' : ["TIME",   "dDmy"],
+   'Y' : ["TIME",   "dDmy"],
+
+   'E' : ["USER",   "ecko",  "email",        "wuser",  "user"],
+   'O' : ["USER",   "ecko",  "org_type",     "wuser",  "user"],
+   'N' : ["USER",   "ecko",  "country",      "wuser",  "user"],
+   'K' : ["USER",   "ecko",  "region",       "wuser",  "user"],
+
+   'R' : ["DSID",   "StT", "search.datasets.title", "search.datasets"],
+   'T' : ["DSID",   "StT", "dataset.dsid",   "dataset"],
+   'P' : ["DSID",   "StT", "specialist",     "dsowner"],
+
+   'W' : ["METHOD", "fM",      "method",       "osdfusage"]
+}
+
+# valid options for %params, a hash array of command line parameters
+#   a -- 1 to view all usage info available
+#   A -- number or records to return
+#   c -- array of specified country codes
+#   C -- a string of short field names for viewing usages
+#   d -- array of specified dates 
+#   D -- dates range, array of 1 or 2 dates in format of YYYY-MM-DD
+#   e -- array of specified email addresses
+#   E -- use given date or date range for email notice of data update
+#   h -- for give emails, include their histical emails registered before
+#   H -- a string of report title to replace the default one
+#   i -- array of specified IP addresses
+#   I -- use given email IDs for email notice of data update
+#   k -- array of specified region names
+#   L -- column delimiter for output
+#   m -- array of specified months 
+#   M -- array of specified download methods 
+#   o -- array of specified orginization types
+#   O -- a string of short field names for sorting on
+#   q -- array of the specified quarters, normally combined with years
+#   s -- size range, arrage of 1 or 2 sizes in unit of MByte
+#   S -- array of login names of specialists who owns the datasets
+#   t -- array of specified dataset names
+#   T -- dataset range, array of 1 or 2 dataset names
+#   U -- given unit for file or data sizes
+#   w -- generate view without totals
+#   y -- array of specified years 
+#   z -- generate view including entries without usage
+
+params = {}
+
+# relationship between parameter options and short field names, A option is not
+# related to a field name if it is not in keys %SNS 
+SNS = {
+   'c' : 'N', 'd' : 'D', 'D' : 'D', 'e' : 'E', 'i' : 'I', 'k' : 'K', 'm' : 'M',
+   'M' : 'W', 'o' : 'O', 'q' : 'Q', 's' : 'S', 'S' : 'P', 't' : 'T', 'T' : 'T', 'y' : 'Y'
+}
+
+tablenames = fieldnames = condition = ''
+sfields = []
+gfields = []
+dfields = []
+pgname = 'viewosdfusage'
+
+#
+# main function to run this program
+#
+def main():
+
+   PgDBI.view_dbinfo()
+   argv = sys.argv[1:]
+   inputs = []
+   option = 'C'   # default option
+
+   for arg in argv:
+      if re.match(r'^-.*$', arg):
+         curopt = arg[1:2]
+         if curopt and VUSG['OPTS'].find(curopt) > -1:
+            if VUSG['NOPT'].find(option) > -1:
+               params[option] = 1
+            elif inputs:
+               params[option]= inputs   # record input array
+               inputs = []      # empty input array
+            option = curopt     # start a new option
+         else:
+            PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGWNEX)
+      else:
+         val = arg
+         if val != '!':
+            if option == 's':
+               val = int(val)*1000000    # convert MBytes to Bytes
+            elif option in SNS:
+               sfld = SNS[option]
+               if VUSG['SFLD'].find(sfld) > -1:
+                  if VUSG['UFLD'].find(sfld) > -1:
+                     val = arg.upper()     # in case not in upper case
+                  elif VUSG['LFLD'].find(sfld) > -1:
+                     val = arg.lower()     # in case not in lower case
+                  if option == 'c':
+                     val = PgView.get_country_name(val)
+                  elif option == 't' or option == 'T':
+                     val = PgUtil.format_dataset_id(val)   # add 'ds' if only numbers
+                  val = "'{}'".format(val)
+         inputs.append(val)
+
+   # record the last option
+   if VUSG['NOPT'].find(option) > -1:
+      params[option] = 1
+   elif inputs:
+      params[option] = inputs   # record input array
+   
+   if not params:
+      PgLOG.show_usage(pgname)
+   else:
+      check_enough_options()
+   
+   if 'o' not in params:
+      if 'e' not in params:
+         params['o'] = ['!', "'DSS'"]   # default to exclude 'DSS' for organization
+   elif params['o'][0] == "'ALL'":
+      del params['o']
+   
+   usgtable = "osdfusage"
+   build_query_strings(usgtable)  # build tablenames, fieldnames, and conditions
+   records = PgDBI.pgmget(tablenames, fieldnames, condition, PgLOG.UCLWEX)
+   if not records: PgLOG.pglog("No Usage Found For Given Conditions", PgLOG.LGWNEX)
+   totals = None if 'w' in params else {}
+   if dfields or totals != None:
+      records = PgView.compact_hash_groups(records, gfields, sfields, dfields, totals)
+   if 'z' in params: records = expand_records(records)
+   ostr = params['O'][0] if 'O' in params else params['C'][0]
+   records = PgView.order_records(records, ostr.replace('X', ''))
+   PgView.simple_output(params, FLDS, records, totals)
+
+   PgLOG.pgexit(0)
+
+#
+# cehck if enough information entered on command line for generate view/report, exit if not
+#
+def check_enough_options():
+
+   cols = params['C'][0] if 'C' in params else 'X'
+   if cols == 'X': PgLOG.pglog("{}: miss field names '{}'".format(pgname, VUSG['SNMS']), PgLOG.LGWNEX)
+
+   if cols.find('Q') > -1 and cols.find('Y') < 0:   # add Y if Q included
+      cols = re.sub('Q', 'YQ', cols)
+      params['C'][0] = cols
+
+   for sn in cols:
+      if sn == 'X': continue  # do not process INDEX field
+      if VUSG['SNMS'].find(sn) < 0:
+         PgLOG.pglog("{}: Field {} must be in field names '{}X'".format(pgname, sn, VUSG['SNMS']), PgLOG.LGWNEX)
+      if 'z' not in params or sn in EXPAND: continue
+      fld = FLDS[sn]
+      if fld[6] != 'G': continue
+      PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
+
+   if 'E' in params or 'I' in params:
+      if 'z' in params:
+         PgLOG.pglog(pgname + ": option -z and -E/-I can not be present at the same time", PgLOG.LGWNEX)
+      elif 't' not in params or len(params['t']) > 1:
+         PgLOG.pglog(pgname + ": specify one dataset for viewing usage of notified users", PgLOG.LGWNEX)
+      elif 'E' in params and 'I' in params:
+         PgLOG.pglog(pgname + ": option -E and -I can not be present at the same time", PgLOG.LGWNEX)
+
+   for opt in params:
+      if VUSG['CNDS'].find(opt) > -1: return
+   PgLOG.pglog("{}: miss condition options '{}'".format(pgname, VUSG['CNDS']), PgLOG.LGWNEX)
+
+#
+# process parameter options to build osdf query strings
+# global variables are used directly and nothing passes in and returns back
+#
+def build_query_strings(usgtable):
+
+   # initialize query strings
+   global condition, fieldnames, tablenames
+   joins = groupnames = ''
+   tablenames = usgtable
+   cols = params['C'][0]
+
+   if 'U' in params:    # reset units for file and read sizes
+      if cols.find('B') > -1: FLDS['B'] = PgView.set_data_unit(FLDS['B'], params['U'][0], "sum(size)")
+      if cols.find('S') > -1: FLDS['S'] = PgView.set_data_unit(FLDS['S'], params['U'][0], "size")
+
+   if 'e' in params and 'h' in params: params['e'] = PgView.include_historic_emails(params['e'], 3)
+
+   for opt in params:
+      if opt == 'C':   # build field, table and group names
+         for sn in cols:
+            if sn == 'X': continue  # do not process INDEX field
+            fld = FLDS[sn]
+            if fieldnames: fieldnames += ', '
+            fieldnames += "{} {}".format(fld[1], sn)   # add to field name string
+            (tablenames, joins) = PgView.join_query_tables(fld[3], tablenames, joins, usgtable)
+            if fld[6] == 'S':
+               sfields.append(sn)
+            else:
+               if groupnames: groupnames += ', '
+               groupnames += sn     # add to group name string
+               if fld[6] == 'D':
+                  dfields.append(sn)
+               else:
+                  gfields.append(sn)
+      elif opt == 'O':
+         continue   # order records later
+      elif VUSG['CNDS'].find(opt) > -1:
+         if VUSG['NOPT'].find(opt) > -1: continue
+         sn = SNS[opt]
+         fld = FLDS[sn]
+         # build having and where conditon strings
+         cnd = PgView.get_view_condition(opt, sn, fld, params, VUSG)
+         if cnd:
+            if condition: condition += ' AND '
+            condition += cnd
+            (tablenames, joins) = PgView.join_query_tables(fld[3], tablenames, joins, usgtable)
+
+
+   # append joins, group by, order by, and having strings to condition string
+   if 'E' in params or 'I' in params:
+      (tablenames, joins) = PgView.join_query_tables("emreceive", tablenames, joins, usgtable)
+   if joins:
+      if condition:
+         condition = "{} AND {}".format(joins, condition)
+      else:
+         condition = joins
+   if 'E' in params or 'I' in params:
+      condition += PgView.notice_condition(params['E'], None, params['t'][0])
+   if groupnames and sfields: condition += " GROUP BY " + groupnames
+
+
+def expand_records(records):
+
+   recs = PgView.expand_query("TIME", records, params, EXPAND)
+
+   trecs = PgView.expand_query("USER", records, params, EXPAND, VUSG, SNS, FLDS)
+   recs = PgUtil.crosshash(recs, trecs)
+
+   trecs = PgView.expand_query("DSID", records, params, EXPAND, VUSG, SNS, FLDS)
+   recs = PgUtil.crosshash(recs, trecs)
+
+   trecs = PgView.expand_query("METHOD", records, params, EXPAND, VUSG, SNS, FLDS)
+   recs = PgUtil.crosshash(recs, trecs)
+
+   return PgUtil.joinhash(records, recs, 0, 1)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/viewosdfusage.usg

@@ -0,0 +1,190 @@
+
+ View usage information of OSDF Data Services from information
+ stored in PostgreSQL database 'RDADB'.
+
+ Usage: viewawsusage [-C] ColumnNames [-O OrderColumnNames] [-a] \
+                     [-A RowLimit] [-c CountryCodes] [-d DateList] \
+                     [-D StartDate [EndDate]] [-e EMailList] -h \
+                     [-E StartNoticeDate [EndNoticeDate]] \
+                     [-i IPAddresses] [-I EmailIDList] \
+                     [-k RegionNames] [-m MonthList] [-M AccessMethods]  \
+                     [-N MinNumberRead [MaxNumberRead]] \
+                     [-o OrganizationTypes] \
+                     [-q QuaterList] [-s MinSize [MaxSize]] \
+                     [-S SpecialistLoginNames] [-t DatasetList] \
+                     [-T MinDataset [MaxDataset]] [-y YearList] \
+                     [-H Title] [-L Delimiter] [-U SizeUnit] \
+                     [-w] [-z] [> OutputFileName] [| lp -d PrinterName] 
+
+      Specify [-C] ColumnNames, refer to Option -C section for detail
+      description, and choose at least one of the condition options, -a, -c,
+      -d, -D, -e, -E, -i, -I, -k, -m, -M, -N, -o, -q, -s, -S -t, -T, and -y,
+      to run this application.
+
+      For all condition options, except option -a, an '!' sign can be added
+      between an option flag and its option values to get an excluding
+      condition. For example, choose '-o ! OrganizationTypes' to gather order
+      data usage by users from organization types other than the ones given in
+      OrganizationTypes.  Refer to the example given at the end of this help
+      document for how to select excluding condition.
+
+      String condition options, -c, -e, -g, -i, -k, -M, -o, -S, and -t, allow
+      wildcard inputs. '%' matches any number of characters and '_' matches any one
+      character.  Refer to the example given at the end of this help document
+      for how to use wildcard for string condition options.
+
+      Output of this application is defaulted to page format with a page
+      header on each page. A page header includes main title, sub titles and
+      column titles according to which column names and options are selected,
+      as well as page number and report date. If the output is used directly
+      for input of other applications, add option -w to remove page header
+      and show only the column titles and the usage information.
+
+
+      Column Options:
+      - Option -C, the ColumnNames must be present to run this application.
+           The flag -C can be omitted if it is the first parameter option on
+           the command line. The ColumnNames is a string that includes column
+           names listed below:
+
+         COLUMN - COLUMN       - COLUMN
+           NAME - TITLE        - DESCRIPTION 
+          GroupColumns:
+              D*- DATE         - format as YYYY-MM-DD, for example 2004-04-25
+              E*- EMAIL        - user email address
+              I*- IP           - user IP address
+              M*- MONTH        - format as YYYY-MM, for example 2004-04
+              N*- COUNTRY      - country codes users from
+              K*- REGION       - region names users from
+              O*- ORGTYPE      - organization types (DSS, NCAR, UNIV and OTHER)
+              P*- DSOWNER      - login names of specialists who own the datasets
+              Q*- QUARTER      - quarter of year, 1, 2, 3, or 4
+              R*- DSTITLE      - dataset titles
+              S - BSIZE        - size of data read each time, default to Bytes
+              T*- DATASET      - format as dsnnn.n, for example d540001
+              W*- METHOD       - access methods
+              Y*- YEAR         - format as YYYY, for example 2004
+
+                * - field names can processed with zero usages
+          SummaryColumns:
+              A - DSCOUNT      - number of datasets in given GroupColumns
+              B - MBREAD       - data sizes, default MB, read by given GroupColumns
+              C - #UNIQUSER    - number of unique users in in given GroupColumns
+              U - #UNIQIP      - number of unique users in in given GroupColumns
+              H - #READ        - number of reads by given GroupColumns
+ 
+          IndexColumn:
+              X - INDEX        - index of line, it should be the first column
+
+           The column names are used to build up string of ColumnNames, while
+           their associated column titles are shown in view/report output of
+           this application. The display order of the column titles is
+           determined by the order of the column names in the ColumnNames
+           string. At least one of the group and summary columns must be
+           selected, in the ColumnNames string, to generate all usage
+           view/report;
+
+           For example, choose '-C EMB' to display column titles of EMAIL,
+           MONTH and MBREAD, in the first, second and third columns
+           respectively, for numbers of MBytes of data read by each user
+           in each month;
+
+      - Option -O, sort data usage information in ascending or descending
+           order based on the column names specified in OrderColumnNames
+           string. These column names must be in the selected [-C]
+           ColumnNames string. If an column name is in upper case, its
+           associated column is sorted in ascending order, and a lower
+           case means sorting in descending order;
+
+
+      Condition Options:
+      - Option -a, for all usage in table 'awsusage';
+
+      - Option -A, gives a row limit for querrying; 
+
+      - Option -c, for files read by users from given country codes;
+
+      - Option -d, for data read on given dates, in format YYYY-MM-DD;
+
+      - Option -D, for data read between two given dates, each date
+           is in format YYYY-MM-DD. Omit EndDate for no upper limit;
+
+      - Option -e, for data read by users with given email addresses;
+
+      - Option -E, for data read by users who have been notified 
+           data update of a specified dataset between two given dates,
+           each date is in format YYYY-MM-DD. Omit EndNoticeDate for
+           no upper limit;
+
+      - Option -h, works with Option -e to include historical user emails
+           registered before;
+
+      - Option -i, for data read from machines with given IP addresses;
+ 
+      - Option -k, for files read by users from given region names;
+
+      - Option -m, for data read in given months, in format YYYY-MM;
+
+      - Option -M, for data read via access methods;
+ 
+      - Option -N, for files for numbers of read by each group between
+           MinNumberRead and MaxNumberRead. Omit MaxNumberRead for no
+           upper limit;
+
+      - Option -o, for data read by users from given orgnization types.
+           It defaults to -o ! DSS to exclude usage from DSS specialists;
+           Set it to ALL to include all orgnization types;
+
+      - Option -q, for data read in given quarters;
+
+      - Option -s, for data sizes, unit of MByte, between MinSize and MaxSize.
+           Omit MaxSize for no upper limit;
+
+      - Option -S, for login names of specialsts who own the datasets;
+
+      - Option -t, for data associating to given dataset names;
+
+      - Option -T, for data associating to datasets between 
+           MinDataset and MaxDataset. Omit MaxDataset for no upper limit.
+           For example, -T d540000 d550009, for datasets numbers d540000-d550009;
+
+      - Option -y, for data read in given years in format YYYY;
+
+
+      Miscellaneous Options:
+      - Option -w, view data usage in simple format without totals;
+
+      - Option -z, include datasets without without usage
+
+      - Option -H, use given report title to replace the default one;
+
+      - Option -L, use given delimiter for output, instead of defaulted spaces;
+
+      - Option -U, show data sizes in given unit SizeUnit [BKMG].
+           B - Byte, K - KiloBytes, M  - MegaByte, and G - GigaByte;
+
+      - Option > OutputFilename, redirect output into an output file,
+           for example, ordusage.out, instead of viewing on screen directly;
+
+      - Option | lp -d PrinterName, redirect output to printer of PrinterName.
+           Replace PrinterName with lj100 to print through DSS LaserJet printer.
+
+
+      For example:
+      To view annual data usage in year 2005 with columns, INDEX(X),
+      EMAIL(E), ORGTYPE(O), #READ(H), and MBREAD(B); ordered by ORGTYPE as
+      ascending and MBREAD(B) as descending; the command line should be:
+
+         viewawsusage XEOHB -y 2005 -O Ob
+
+      For usage by users not in Organization 'DDS', out of the file usage
+      gathered above, the command line should be:
+
+         viewawsusage XEOHB -y 2005 -o ! DSS -O Ob
+
+      To redirect the previous output to a file named awsusage.out:
+
+         viewawsusage XEOHB -y 2005 ! DSS -O Ob > awsusage.out
+
+      Then you can view the file or print it as a report.
+

{rda_python_metrics-1.0.34.dist-info → rda_python_metrics-1.0.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.34
+Version: 1.0.36
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.34.dist-info → rda_python_metrics-1.0.36.dist-info}/RECORD

@@ -1,7 +1,7 @@
 rda_python_metrics/PgIPInfo.py,sha256=CfcnJxD2zHyAhemhTisdnPB72wHwE8MxS0EH4EAKnfE,9151
 rda_python_metrics/PgView.py,sha256=r6otb3DjfiaQJdg0z8bZQAOlhr4JnrXJzp9wgWh_8qQ,24369
 rda_python_metrics/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-rda_python_metrics/fillawsusage.py,sha256=Z4rMXINAA8mM-Wbbj4n5ubp8NJegR-JwZR28BieC5Eg,7040
+rda_python_metrics/fillawsusage.py,sha256=bylk7m8VJyXMZMZwyPkU3rQqIdttjkeo8aTYeneqMDQ,7073
 rda_python_metrics/fillawsusage.usg,sha256=-lLTRLTaEPL2rSQ4mhpIekhyYrLGahy7NWtaq_8PCDM,611
 rda_python_metrics/fillcdgusage.py,sha256=jJVxQ4d5at0lXXKmV7TVf2GrPWSrJv5XBiHGMc3uxEA,16706
 rda_python_metrics/fillcdgusage.usg,sha256=5lkd4Zdi72nQcha-JtbaLnxl66V4wBDykKwQtUOtMrw,667
@@ -11,13 +11,13 @@ rda_python_metrics/fillcountry.py,sha256=7i5LNi3scRoyRCT6t7aeNTGKOpxzJ2mA9tnvUqj
 rda_python_metrics/fillendtime.py,sha256=skZttlpoY19g0dGwqGQI8t_1YPPTPEXwg3EfNlfL90I,2533
 rda_python_metrics/fillgdexusage.py,sha256=8KR5Lt30VCTxWOtc9EStLFzq5qa2di8RRQ3qMuIvSYY,37818
 rda_python_metrics/fillgdexusage.usg,sha256=mVYtK0pIYmvma0skT-wXM-NOEpkN_i3E61UdWgQWLfs,648
-rda_python_metrics/fillglobususage.py,sha256=ahz8XnnJdD_AbSYqJ34lWmDuzws_-SNmCR8QE20aovA,8539
+rda_python_metrics/fillglobususage.py,sha256=zuxzoeV9BAMoVUu_VCYECPddYNrBWpV74kkYpyqIQhg,8443
 rda_python_metrics/fillglobususage.usg,sha256=1GgmCP22IQZdADwL5Mmkz3v8Ws-G7U3teQ1AxRJfV_4,637
 rda_python_metrics/fillipinfo.py,sha256=BrboxAIs8Q5jhz_4GYW_GibMT5GyEbnQkYfFR8hc_yo,6747
 rda_python_metrics/fillipinfo.usg,sha256=YeCR58xGv0emqHUZ_9R977HrqaeBwbd6j5QRF2Lc7TA,702
 rda_python_metrics/filloneorder.py,sha256=ADHbcKCDh9bJunnxYbkbjwU2QpC43hvGlLWaURHNxkg,5433
 rda_python_metrics/filloneorder.usg,sha256=mtOySKx6-D4k2bbTcmi6cSYtINiycRyHQkHozi0CQu0,1466
-rda_python_metrics/fillosdfusage.py,sha256=Mpd2qkLQmOLqfahCI89kXIR9FpOKoQdyfrU86AE_dd0,9257
+rda_python_metrics/fillosdfusage.py,sha256=--c6PzeZ6EZXpb0zxy4lZCDjwwIBb7c6xgP7QCKLoMQ,7087
 rda_python_metrics/fillosdfusage.usg,sha256=Qc5YdUuOiAH74FfVdkwkrQRDYXcASVbxMdBsVQj4X1k,635
 rda_python_metrics/fillrdadb.py,sha256=cb6upPApAZloOouUSzZZmjGvOsYT_Nzh9Lz926WE3ZQ,5333
 rda_python_metrics/fillrdadb.usg,sha256=E_Bf4G8yVABogjRmIOaIbTGgnII2W6RltaFad2XEV2Q,1228
@@ -32,12 +32,16 @@ rda_python_metrics/pgsyspath.py,sha256=DZhFp-r-LzN0qrHtfdCYfnGDnmD_R4ufuEduk0_vR
 rda_python_metrics/pgusername.py,sha256=VoNJfXBsyzdfz49qwCypnUsqcDm4cUaqOYViJ-jzaKI,1265
 rda_python_metrics/viewallusage.py,sha256=DlAvY2bieJWrrrYMPhCkKWgqdfSN492_LZCS95BhY0A,15753
 rda_python_metrics/viewallusage.usg,sha256=Vfwc6aNIkpe1FBdBsV2htyUESgmx6ODhMMwAb1NX8-0,10297
+rda_python_metrics/viewawsusage.py,sha256=jSjudneYBfR5a9565i6Qe0GeKjIxAet9HZfOAfOOjs4,14128
+rda_python_metrics/viewawsusage.usg,sha256=LbolAFqpXYjMdk_dMWKO3SFOTORoGc-LmLv2kI1J_nI,8843
 rda_python_metrics/viewcheckusage.py,sha256=HougqjDAOVG6pYglFjyHQ-UdLBcYe7v_jzU1-80RqFA,12996
 rda_python_metrics/viewcheckusage.usg,sha256=KuJFycggGiUcSezQ9vywDbituvu63SZ-ZnNTaMpbc-A,8930
 rda_python_metrics/viewcodusage.py,sha256=6Shmbzq_DNh0uvT9lPxpB8ic2JnrmQzmR7Bc-9U4gl0,14243
 rda_python_metrics/viewcodusage.usg,sha256=WH3gSml94_jbm20kqAUFUSnla4JQZrseTZe24mmlDEA,8788
 rda_python_metrics/viewordusage.py,sha256=cnZMSfxWlCNbh1Ck3LfC0wQxS6HCv6TBVOnOVzRU65E,15484
 rda_python_metrics/viewordusage.usg,sha256=19tHhPZB9y247BddfwtXA3_K50BnzEJcWUbBNw-3NPU,10568
+rda_python_metrics/viewosdfusage.py,sha256=xpSi1VcjtEmfF9fAwSOeGLYYj3j972bX6kj0fZUL6Q4,14159
+rda_python_metrics/viewosdfusage.usg,sha256=J9dE98j3BUWGej6kIXz46Pl7nntsTBoJvxFcb6mqTb0,8844
 rda_python_metrics/viewrqstusage.py,sha256=wNH5DTEBYrUQKAms10weBH939r-m3tLXXg5PwS6bzlk,16690
 rda_python_metrics/viewrqstusage.usg,sha256=Ii5-7h_RO2rkoE9VLxuLhc9klgkEJSqHoDrsOlQOTKo,10481
 rda_python_metrics/viewtdsusage.py,sha256=nmtH4d7pPqSwLoAlocb5UTk0W38TT57gWmCyuKL4bF8,14505
@@ -46,9 +50,9 @@ rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmj
 rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
 rda_python_metrics/viewwebusage.py,sha256=ES2lI8NaCeCpTGi94HU-cDRBxHMiUBbplyYsZf2KqF0,16650
 rda_python_metrics/viewwebusage.usg,sha256=OVDZ78p87E3HLW34ZhasNJ7Zmw8XXjmZPPWZfRhPLXo,9936
-rda_python_metrics-1.0.34.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_metrics-1.0.34.dist-info/METADATA,sha256=1nqVoglplCc6y7KSpCU4ovnakC7eWfz952a5oZOE0bc,761
-rda_python_metrics-1.0.34.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rda_python_metrics-1.0.34.dist-info/entry_points.txt,sha256=ctvX0Gx9zdnKpHx5sjZdl7_sLSR80LKYhPky9qkEpug,1239
-rda_python_metrics-1.0.34.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
-rda_python_metrics-1.0.34.dist-info/RECORD,,
+rda_python_metrics-1.0.36.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_metrics-1.0.36.dist-info/METADATA,sha256=JyzrAexSHYqltfa28z8N-MH9-6U0Ve2mBVFlwjX05hk,761
+rda_python_metrics-1.0.36.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rda_python_metrics-1.0.36.dist-info/entry_points.txt,sha256=_Cw8KvtYwxLPt3kLJHwQ1r2swsKKbXk-2zYPZAg2czc,1345
+rda_python_metrics-1.0.36.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
+rda_python_metrics-1.0.36.dist-info/RECORD,,

{rda_python_metrics-1.0.34.dist-info → rda_python_metrics-1.0.36.dist-info}/entry_points.txt

@@ -16,9 +16,11 @@ logarch.py = rda_python_metrics.logarch:main
 pgperson = rda_python_metrics.pgperson:main
 pgusername = rda_python_metrics.pgusername:main
 viewallusage = rda_python_metrics.viewallusage:main
+viewawsusage = rda_python_metrics.viewawsusage:main
 viewcheckusage = rda_python_metrics.viewcheckusage:main
 viewcodusage = rda_python_metrics.viewcodusage:main
 viewordusage = rda_python_metrics.viewordusage:main
+viewosdfusage = rda_python_metrics.viewosdfusage:main
 viewrqstusage = rda_python_metrics.viewrqstusage:main
 viewtdsusage = rda_python_metrics.viewtdsusage:main
 viewwebfile = rda_python_metrics.viewwebfile:main