rda-python-metrics 1.0.4__py3-none-any.whl

rda_python_metrics/filloneorder.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : filloneorder
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 03/10/2022
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to fill one order usage on command line
+#
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgIMMA
+from rda_python_common import PgUtil
+
+# -t dsid, -e email, -v request data volume, -i data input volume,
+# -m delivery method, -a amount charged, -p pay method, -d request date, -x close date,
+# -y close time, -c file count, -s specialist login name, -o order id, 
+# mandatory options: -t, -e, -v, and -m
+
+#
+# main function to run this program
+#
+def main():
+
+   option = None
+   params = {}
+   argv = sys.argv[1:]
+
+   for arg in argv:
+      ms = re.match(r'^-(\w)$', arg)
+      if ms:
+         option = ms.group(1)
+         if option == "b":
+            PgLOG.PGLOG['BCKGRND'] = 1
+            option = None
+         elif option not in "acdeimopstvx":
+            PgLOG.pglog("-{}: Invalid Option".format(option), PgLOG.LGWNEX)
+      elif option and option not in params:
+         if option == 't': arg = PgUtil.format_dataset_id(arg)
+         params[option] = arg
+         option = None
+      else:
+         PgLOG.pglog(arg + ": parameter passed in without leading option", PgLOG.LGWNEX)
+
+   if not params: PgLOG.show_usage('filloneorder')
+   PgDBI.dssdb_dbname()
+   PgLOG.cmdlog("filloneorder {}".format(' '.join(argv)))
+
+   check_inputs(params)
+   add_one_order(params)
+
+   sys.exit(0)
+
+def add_one_order(params):
+
+   year = None
+   record = {}
+   
+   record['dsid'] = params['t']
+   record['wuid_request'] = params['u']
+   record['dss_uname'] = params['s']
+   record['date_request'] = params['d']
+   record['date_closed'] = params['x']
+   record['method'] = params['m']
+   record['size_request'] = params['v']
+   record['size_input'] = params['i']
+   if 'a' in params: record['amount'] = params['a']
+   if 'p' in params: record['pay_method'] = params['p']
+   record['count'] = params['c'] if 'c' in params else 0
+   if 'o' in params: record['order_number'] = params['o']
+   ms = re.match(r'(\d+)-(\d+)-', record['date_request'])
+   if ms:
+      year = int(ms.group(1))
+      record['quarter'] = 1 + int((int(ms.group(2)) - 1) / 3)
+
+   if add_to_allusage(record, year, params['y']) and PgDBI.pgadd("ousage", record, PgLOG.LGEREX):
+      PgLOG.pglog("1 order added for " + params['e'], PgLOG.LOGWRN)
+   else:
+      PgLOG.pglog("No order added for " + params['e'], PgLOG.LOGWRN)
+
+def add_to_allusage(record, year, ctime):
+
+   pgrec = PgDBI.pgget("wuser",  "email, org_type, country",
+                       "wuid = {}".format(record['wuid_request']), PgLOG.LGWNEX)
+   if pgrec:
+      pgrec['dsid'] = record['dsid']
+      if pgrec['org_type'] == "UCAR": pgrec['org_type'] = "NCAR"
+      pgrec['date'] = record['date_request']
+      pgrec['time'] = ctime
+      pgrec['quarter'] = record['quarter']
+      pgrec['size'] = record['size_request']
+      pgrec['method'] = record['method']
+      pgrec['source'] = 'O'
+      return PgDBI.add_yearly_allusage(year, pgrec)
+      
+   return 0
+
+#
+# check option inputs and fill up the missing ones for default values
+#
+def check_inputs(params):
+
+   # mandatory inputs
+   if 't' not in params:
+      PgLOG.pglog("Missing Dataset ID per option -t", PgLOG.LGEREX)
+
+   if not PgDBI.pgget("dataset", '', "dsid = '{}'".format(params['t']), PgLOG.LGEREX):
+      PgLOG.pglog(params['t'] + ": dsid not in RDADB", PgLOG.LGEREX)
+
+   if 'v' not in params:
+      PgLOG.pglog("Missing order data value in Bytes per option -v", PgLOG.LGEREX)
+
+   if 'm' not in params:
+      PgLOG.pglog("Missing data delivery method per option -m", PgLOG.LGEREX)
+
+   if 'e' not in params:
+      PgLOG.pglog("Missing user email per option -e", PgLOG.LGEREX)
+
+   (cdate, ctime) = PgUtil.get_date_time()
+   # set default values
+   if 'i' not in params: params['i'] = params['v']
+   if 'x' not in params: params['x'] = cdate
+   if 'y' not in params: params['y'] = ctime
+   if 'd' not in params: params['d'] = params['x']
+
+   params['u'] = PgDBI.check_wuser_wuid(params['e'], params['d'])
+   params['s'] = check_specialist(params['t'], (params['s'] if 's' in params else PgLOG.PGLOG['CURUID']))
+
+   # check if order is recorded already
+   ocond = "dsid = '{}' AND wuid_request = {} AND size_request = {} and date_request = '{}'".format(params['t'], params['u'], params['v'], params['d'])
+   if PgDBI.pgget("ousage", '', ocond, PgLOG.LGEREX):
+      PgLOG.pglog("Order of {} Bytes Data from {} for {} on {} recorded on {} already".format(params['v'], params['t'], params['e'], params['d'], params['x']), PgLOG.LGWNEX)
+
+#
+# return the dataset owner if specialist not given
+#
+def check_specialist(dsid, specialist):
+
+   if specialist and PgDBI.pgget("dssgrp", "", "logname = 'specialist'", PgLOG.LGEREX): return specialist
+   scond = "specialist = logname AND dsid = '{}' AND priority = 1".format(dsid)
+   pgrec = PgDBI.pgget("dsowner, dssgrp", "specialist", scond, PgLOG.LGEREX)
+   return pgrec['specialist'] if pgrec else "datahelp"
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/filloneorder.usg

@@ -0,0 +1,41 @@
+
+ Fill usage information for one special order in table 'ousage' in MySQL
+ database 'dssdb'.
+
+ Usage: filloneorder -e UserEmail -m DeliveryMethod -t DatasetID -v DataVolume \
+                     [-a CostAmount] [-b] [-c FileCount] [-d DateOrdered] \
+                     [-i InputDataVlolume] [-o OerderID] [-p PaymentMethod] \
+                     [-s SpecialistLoginName] [-x DateClosed -x TimeClosed]
+
+   select the Mandatory options, -e, -m, -t and -v, and any other Optional
+   options to run this application. 
+
+   - Option -b, log process information into logfile only;
+
+   - Option -a, the amount of dollars if the order is not free ;
+
+   - Option -c, the number of data files for the order;
+
+   - Option -d, date the order is opened;
+
+   - option -e, user email address who ordered the data;
+
+   - option -i, the data volume involved for processing the order;
+   
+   - Option -m, the delivery method for the ordered data;
+   
+   - option -o, order ID, up to 30 characters;
+   
+   - option -p, the payment method for the amount per option -a;
+
+   - option -s, the specialist who handle the order. The login name is used
+                if it is not specified, and the default dataset owner is 
+                used if given specialist is not a valid DSS specialist;
+
+   - option -t, dataset id the data ordered from;
+
+   - option -v, the final data volume for the order;
+
+   - Option -x, date the order is closed;
+
+   - Option -y, time the order is closed.

rda_python_metrics/fillrdadb.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillrdadb
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 04/07/2022
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to retrieve info from data logs, and fill tables
+#             in PostgreSQL database.schema rdadb.dssdb
+#
+#    Github : https://github.com/NCAR/rda-pythn-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgFile
+from rda_python_common import PgUtil
+from rda_python_common import PgDBI
+
+# the define options for gathering web online file usage, one at a time
+DATES = 0x01  # get web file usages for given dates
+MONTH = 0x02  # fet web file usages for given months
+YEARS = 0x04  # get web file usages for given years
+NDAYS = 0x08  # get web file usages in recent number of days 
+CLNFL = 0x10  # clean unused file only
+MASKS = (MONTH|YEARS|DATES|NDAYS)
+
+RDADB = {
+   'OPTION' : 0,
+   'OPTVAL' : '',
+   'DOMAIL' : 1,
+   'BCKGRND' : ''
+}
+
+#
+# main function to run this program
+#
+def main():
+
+   params = []   # array of input values
+   argv = sys.argv[1:]
+   bckflag = ''
+
+   PgDBI.dssdb_dbname()
+   
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = "-b"
+         RDADB['BCKGRND'] = " -b"
+      elif arg == "-n":
+         RDADB['DOMAIL'] = 0
+         PgLOG.PGLOG['LOGMASK'] &= ~(PgLOG.EMLALL)
+      elif re.match(r'^-[cdmNy]$', arg) and not RDADB['OPTVAL']:
+         RDADB['OPTVAL'] = arg
+         if arg == "-c":
+            RDADB['OPTION'] |= CLNFL
+         elif arg == "-d":
+            RDADB['OPTION'] |= DATES
+         elif arg == "-m":
+            RDADB['OPTION'] |= MONTH
+         elif arg == "-y":
+            RDADB['OPTION'] |= YEARS
+         elif arg == "-N":
+            RDADB['OPTION'] |= NDAYS
+         
+      elif re.match(r'^-.*', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif RDADB['OPTION']&MASKS:
+         params.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+
+   if not RDADB['OPTION'] or (RDADB['OPTION']&MASKS and not params):
+      PgLOG.show_usage('fillrdadb')
+   PgDBI.dssdb_dbname()
+   PgLOG.cmdlog("fillrdadb {}".format(' '.join(argv)))
+
+   if RDADB['OPTION']&CLNFL: # clean unused file only
+      clean_unused_files()
+   elif RDADB['OPTION']&MASKS:
+      fill_rdadb(RDADB['OPTVAL'], params)
+
+   sys.exit(0)
+
+#
+# Fill RDADB info for given condition
+#
+def fill_rdadb(option, params):
+
+   filecond = '{} {}'.format(option, ' '.join(params))
+   PgLOG.pglog("Filling RDADB info for '{}' at {}".format(filecond, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+   # fill available custom OPeNDAP usages
+   PgLOG.pgsystem("fillcodusage {} {}".format(RDADB['BCKGRND'], filecond), PgLOG.LGWNEM, 5)
+   # fill available globus web data usages
+   PgLOG.pgsystem("fillglobususage {} {}".format(RDADB['BCKGRND'], filecond), PgLOG.LGWNEM, 5)
+   # fill available AWS web data usages
+   PgLOG.pgsystem("fillawsusage {} {}".format(RDADB['BCKGRND'], filecond), PgLOG.LGWNEM, 5)
+
+   if RDADB['DOMAIL']: send_email_notice()
+   PgLOG.pglog("End Filling RDADB info at {}".format(PgLOG.current_datetime()), PgLOG.LGWNEM)
+
+#
+# clean unused MSS and Web files
+#
+def clean_unused_files():
+
+   PgLOG.pglog("Check and clean deleted Web files that never been used at {}".format(PgLOG.current_datetime()), PgLOG.LOGWRN)
+   pgrecs = PgDBI.pgmget("wfile", "wid", "status = 'D'", PgLOG.LGWNEX)
+   
+   allcnt = len(pgrecs['wid']) if pgrecs else 0
+   PgLOG.pglog("{} record(s) retrieved from Table 'wfile' at {}".format(allcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+   procnt = delcnt = 0
+   if allcnt:
+      fcond = r"wid = {} AND org_type <> 'DSS' AND wuid_read = wuid"
+      for fid in pgrecs['wid']:
+         procnt += 1
+         if procnt%5000 == 0:
+            PgLOG.pglog("{}/{} record(s) processed/removed from Table 'wfile'".format(procnt, delcnt), PgLOG.WARNLG)
+         if not PgDBI.pgget("wusage, wuser", "", fcond.format(fid), PgLOG.LGWNEX):
+            # deleted web file never been used
+            delcnt += PgDBI.pgdel("wfile", "wid = {}".format(fid), PgLOG.LGWNEX)
+
+   PgLOG.pglog("{} record(s) removed from Table 'wfile' at {}".format(delcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+#
+# email notice of job done
+#
+def send_email_notice():
+
+   msg = ("Hi All,\n\nRDADB weekly data usage gathering is done at {}.\n\n".format(PgLOG.current_datetime()) +
+          "Please Let me know if you notice any problem.\n\nThanks,\n\nHua\n")
+   pgrecs = PgDBI.pgmget("dssgrp", "logname", "email_flag = 'Y'", PgLOG.LGWNEX)
+   if pgrecs:
+      receiver = ""
+      for logname in pgrecs['logname']:
+         if receiver: receiver += ', '
+         receiver += (logname + "@ucar.edu")
+
+      ret = PgLOG.send_email("RDADB Weekly Data Usage Gathered on " + PgUtil.curdate(), receiver, msg)
+      if ret: PgLOG.pglog(ret, PgLOG.LOGWRN)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/fillrdadb.usg

@@ -0,0 +1,32 @@
+
+ Collect MSS information from ORACLE server and online file information
+   from DSS Web Server logs, and fill tables in MySQL database 'RDADB'.
+
+ Backup the records in the tables and compress them, and archive the compressed
+   table files to MSS according to the options chosen on the command line.
+
+
+ Usage: fillrdadb [-b] [-c] [-d DateList] [-m MonthList]
+                  [-n] [-N NumberDay] [-S] [-y YearList]
+
+      Select options, -c, -d, -m, -N, or -y, to run this application. 
+
+      - Option -b, log process information into logfile only;
+
+      - Option -c, clean unused file names only;
+
+      - Option -d, get file info recorded on given dates. Date format is
+           YYYY-MM-DD, for example, 2004-01-01;
+
+      - Option -m, get file info recorded in given months. Month format is
+           YYYY-MM, for example, 2004-1;
+
+      - Option -n, do not send email notification after filling info
+
+      - Option -N, get file info recorded in recent NumberDay days;
+
+      - Option -y, get file info recorded in given years. Year format is YYYY,
+           for example, 2004;
+
+      This is a wrapping funtion. Internally it calls 'fillcodusage', 'fillawsusage',
+      and 'fillglobususage' with selected options.

rda_python_metrics/filltdsusage.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : filltdsusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 03/11/2022
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to retrieve info from TDS logs 
+#             and fill table tdsusage in PostgreSQL database dssdb.
+# 
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+import glob
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from . import PgIPInfo
+
+# the define options for gathering TDS data usage, one at a time
+MONTH = 0x02  # fet TDS data usages for given months
+YEARS = 0x04  # get TDS data usages for given years
+NDAYS = 0x08  # get TDS data usages in recent number of days 
+MASKS = (MONTH|YEARS|NDAYS)
+
+USAGE = {
+   'OPTION' : 0,
+   'PGTBL'  : "tdsusage",
+   'TDSLOG' : "/data/logs/nginx/{}.access.log",
+   'CDATE' : PgUtil.curdate()
+}
+
+#
+# main function to run this program
+#
+def main():
+
+   params = []  # array of input values
+   argv = sys.argv[1:]
+   datelimit = ''
+   fixrec = False
+   
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif arg == "-f":
+         fixrec = True
+      elif re.match(r'^-[mNy]$', arg) and USAGE['OPTION'] == 0:
+         if arg == "-m":
+            USAGE['OPTION'] = MONTH
+         elif arg == "-y":
+            USAGE['OPTION'] = YEARS
+         elif arg == "-N":
+            USAGE['OPTION'] = NDAYS
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif USAGE['OPTION']&MASKS:
+         params.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+   
+   if not (USAGE['OPTION'] and params): PgLOG.show_usage('filltdsusage')
+   PgDBI.dssdb_dbname()
+   PgLOG.cmdlog("filltdsusage {}".format(' '.join(argv)))
+
+   if fixrec:
+      fix_tds_usages(USAGE['OPTION'], params)
+   else:
+      if USAGE['OPTION']&NDAYS:
+         curdate = USAGE['CDATE']
+         datelimit = PgUtil.adddate(curdate, 0, 0, -int(params[0]))  
+         USAGE['OPTION'] = MONTH
+         params = []
+         
+         while curdate >= datelimit:
+            tms = curdate.split('-')
+            params.append("{}-{}".format(tms[0], tms[1]))
+            curdate = PgUtil.adddate(curdate, 0, 0, -int(tms[2]))
+   
+      fill_tds_usages(USAGE['OPTION'], params, datelimit)
+
+   PgLOG.pglog(None, PgLOG.LOGWRN|PgLOG.SNDEML)  # send email out if any
+
+   sys.exit(0)
+
+#
+# Fill TDS usages into table dssdb.tdsusage from tds access logs
+#
+def fill_tds_usages(option, inputs, datelimit):
+
+   cntall = cntadd = 0
+
+   for input in inputs:
+      # get log file names
+      if option&MONTH:
+         tms = input.split('-')
+         yrmn = "{}-{:02}".format(tms[0], int(tms[1]))
+      else:
+         yrmn = input
+
+      logfiles = glob.glob(USAGE['TDSLOG'].format(yrmn + '*'))
+      if not logfiles: PgLOG.pglog("{}: No file found to gather TDS usage".format(yrmn), PgLOG.LOGWRN)
+      for logfile in logfiles:
+         if not op.isfile(logfile):
+            PgLOG.pglog("{}: Not exists to gather TDS usage".format(logfile), PgLOG.LOGWRN)
+            continue
+         fdate = None
+         ms = re.search(r'(\d+-\d+-\d+).access.log$', logfile)
+         if ms:
+            fdate = ms.group(1)
+            if fdate >= USAGE['CDATE']: continue
+            if datelimit and fdate < datelimit: continue
+         PgLOG.pglog("Gathering usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
+         tds = PgFile.open_local_file(logfile)
+         if not tds: continue
+         ptime = ''
+         records = {}
+         entcnt = 0
+         while True:
+            line = tds.readline()
+            if not line: break
+            entcnt += 1
+            if entcnt%20000 == 0:
+               cnt = len(records)
+               PgLOG.pglog("{}/{} TDS log entries processed/records to add".format(entcnt, cnt), PgLOG.WARNLG)
+
+            ms = re.search(r'(/thredds/catalog|\sGooglebot/)', line)
+            if ms: continue
+            ms = re.search(r'/thredds/\S+\.(png|jpg|gif|css|htm)', line)
+            if ms: continue
+            ms = re.match(r'^([\d\.]+)\s.*\s(-|\S+@\S+)\s+\[(\S+).*/thredds/(\w+)(/|/grid/)(aggregations|files).*/(ds\d\d\d.\d|[a-z]\d{6})/.*\s200\s+(\d+)(.*)$', line)
+            if not ms: continue
+            ip = ms.group(1)
+            email = ms.group(2)
+            (date, time) = get_record_date_time(ms.group(3))
+            method = ms.group(4)
+            etype = ms.group(6)[0].upper()
+            dsid = PgUtil.format_dataset_id(ms.group(7))
+            size = int(ms.group(8))
+            ebuf = ms.group(9)
+            ms = re.search(r' "(\w+.*\S+)" ', ebuf)
+            engine = ms.group(1) if ms else 'Unknown'
+            key = "{}:{}:{}:{}".format(ip, dsid, method, etype)
+
+            if key in records:
+               records[key]['size'] += size
+               records[key]['fcount'] += 1
+            else:
+               records[key] = {'ip' : ip, 'email' : email, 'dsid' : dsid, 'time' : time, 'size' : size,
+                              'fcount' : 1, 'method' : method, 'etype' : etype, 'engine' : engine}
+         tds.close()
+         if records: cntadd += add_usage_records(records, fdate)
+         cntall += entcnt
+
+   PgLOG.pglog("{} TDS usage records added for {} entries at {}".format(cntadd, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+
+def get_record_date_time(ctime):
+   
+   ms = re.search(r'^(\d+)/(\w+)/(\d+):(\d+:\d+:\d+)$', ctime)
+   if ms:
+      d = int(ms.group(1))
+      m = PgUtil.get_month(ms.group(2))
+      y = ms.group(3)
+      t = ms.group(4)
+      return ("{}-{:02}-{:02}".format(y, m, d), t)
+   else:
+      PgLOG.pglog("time: Invalid date format", PgLOG.LGEREX)
+
+def add_usage_records(records, date):
+
+   quarter = cnt = 0
+   year = None
+   ms = re.search(r'(\d+)-(\d+)-', date)
+   if ms:
+      year = ms.group(1)
+      quarter = 1 + int((int(ms.group(2)) - 1)/3)
+   for key in records:
+      record = records[key]
+      cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, record['time'], record['ip'])
+      if PgDBI.pgget(USAGE['PGTBL'], '', cond, PgLOG.LGEREX): continue
+      if record['email'] == '-':
+         record['org_type'] = record['country'] = '-'
+         ipinfo = PgIPInfo.set_ipinfo(record['ip'])
+         if ipinfo:
+            record['org_type'] = ipinfo['org_type']
+            record['country'] = ipinfo['country']
+            record['email'] = 'unknown@' + ipinfo['hostname']
+      else:
+         wuid = PgDBI.check_wuser_wuid(record['email'], date)
+         if not wuid: continue
+         pgrec = PgDBI.pgget("wuser",  "org_type, country", "wuid = {}".format(wuid), PgLOG.LGWNEX)
+         if not pgrec: continue
+         record['org_type'] = pgrec['org_type']
+         record['country'] = pgrec['country']
+
+      record['quarter'] = quarter
+      record['date'] = date
+
+      if add_to_allusage(year, record):
+         cnt += PgDBI.pgadd(USAGE['PGTBL'], record, PgLOG.LOGWRN)
+
+   PgLOG.pglog("{}: {} TDS usage records added at {}".format(date, cnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+   return cnt
+
+
+def add_to_allusage(year, pgrec):
+
+   record = {'method' : 'TDS', 'source' : 'T'}
+
+   for fld in pgrec:
+      if re.match(r'^(engine|method|etype|fcount)$', fld): continue
+      record[fld] = pgrec[fld]
+
+   return PgDBI.add_yearly_allusage(year, record)
+
+#
+# Fix TDS usages in table dssdb.tdsusage by combine tds accesses with same ip,dsid,method&etype
+#
+def fix_tds_usages(option, inputs):
+
+   cntall = cntfix = 0
+
+   for input in inputs:
+      if option&NDAYS:
+         edate = USAGE['CDATE']
+         date = PgUtil.adddate(edate, 0, 0, -int(input))  
+      elif option&MONTH:
+         tms = input.split('-')
+         date = "{}-{:02}-01".format(tms[0], int(tms[1]))
+         edate = PgUtil.enddate(date, 0, 'M')
+      else:
+         date = input + "-01-01"
+         edate = input + "-12-31"
+
+      while date <= edate:
+         cond = "date = '{}' and fcount = 0 order by time".format(date)
+         pgrecs = PgDBI.pgmget(USAGE['PGTBL'], '*', cond, PgLOG.LGEREX)
+         cnt = len(pgrecs['ip']) if pgrecs else 0
+         records = {}
+         for i in range(cnt):
+            record = PgUtil.onerecord(pgrecs, i)
+            key = "{}:{}:{}:{}".format(record['ip'], record['dsid'], record['method'], record['etype'])
+            if key in records:
+               records[key]['size'] += record['size']
+               records[key]['fcount'] += 1
+            else:
+               record['fcount'] = 1
+               records[key] = record
+
+         if records: cntfix += fix_usage_records(records, date)
+         cntall += cnt
+         date = PgUtil.adddate(date, 0, 0, 1)
+
+   PgLOG.pglog("{} TDS usage records combined into {} at {}".format(cntall, cntfix, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+def fix_usage_records(records, date):
+
+   cnt = 0
+   ms = re.match(r'^(\d+)-', date)
+   year = ms.group(1)
+   tname = 'allusage_' + year
+   dcnt = PgDBI.pgdel(tname , "date = '{}' AND method = 'TDS'".format(date), PgLOG.LOGWRN)
+   PgLOG.pglog("{} TDS usage records deleted for {} from {}".format(dcnt, date, tname), PgLOG.LOGWRN)
+   for key in records:
+      record = records[key]
+      cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, record['time'], record['ip'])
+      if add_to_allusage(year, record):
+         cnt += PgDBI.pgupdt(USAGE['PGTBL'], record, cond, PgLOG.LOGWRN)
+
+   if cnt:
+      PgLOG.pglog("{} TDS usage records updated for {} in {}".format(cnt, date, USAGE['PGTBL']), PgLOG.LOGWRN)
+      dcnt = PgDBI.pgdel(USAGE['PGTBL'], "date = '{}' and fcount = 0".format(date), PgLOG.LOGWRN) 
+      PgLOG.pglog("{} TDS usage records deleted for {} from {}".format(dcnt, date, USAGE['PGTBL']), PgLOG.LOGWRN)
+
+   return cnt
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_metrics/filltdsusage.usg

@@ -0,0 +1,17 @@
+
+ Retrieves usage information from RDA TDS Server logs under ../logs/tomcat to
+   fill table 'tdsusage' in MySQL database 'dssdb'.
+
+ Usage: filltdsusage [-b] [-m MonthList] [-N NumberDay] [-y YearList]
+
+      select one of the options, -m, -N or -y each time to run
+        this application. 
+
+      - Option -b, log process information into logfile only;
+
+      - Option -m, retrieve usage info in given months; 
+
+      - Option -N, retrieve usage info in recent NumberDay days;
+
+      - Option -y, retrieve usage info in given years. 
+