rda-python-metrics 1.0.32__tar.gz → 1.0.34__tar.gz

{rda_python_metrics-1.0.32/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.32
+Version: 1.0.34
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.34}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.32"
+version = "1.0.34"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]
@@ -38,7 +38,7 @@ pythonpath = [
 
 [project.scripts]
 "logarch.py" = "rda_python_metrics.logarch:main"
-"fillawsusage" = "rda_python_metrics.fillaswusage:main"
+"fillawsusage" = "rda_python_metrics.fillawsusage:main"
 "fillcdgusage" = "rda_python_metrics.fillcdgusage:main"
 "fillcodusage" = "rda_python_metrics.fillcodusage:main"
 "fillcountry" = "rda_python_metrics.fillcountry:main"

rda_python_metrics-1.0.34/src/rda_python_metrics/fillawsusage.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillawsusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 03/11/2022
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : python program to retrieve info from AWS logs 
+#             and fill table wusages in PgSQL database dssdb.
+# 
+#    Github : https://github.com/NCAR/rda-pythn-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+import glob
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from . import PgIPInfo
+
+USAGE = {
+   'PGTBL'  : "awsusage",
+   'AWSDIR' : PgLOG.PGLOG["TRANSFER"] + "/AWSera5log",
+   'AWSLOG' : "{}/{}-00-00-00-*",
+   'PFMT'   : "YYYY/MM/DD"
+}
+
+DSIDS = {'nsf-ncar-era5' : 'd633000'}
+
+#
+# main function to run this program
+#
+def main():
+
+   params = []  # array of input values
+   argv = sys.argv[1:]
+   option = None
+
+   for arg in argv:
+      ms = re.match(r'^-(b|d|p|N)$', arg)
+      if ms:
+         opt = ms.group(1)
+         if opt == 'b':
+            PgLOG.PGLOG['BCKGRND'] = 1
+         elif option:
+            PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
+         else:
+            option = opt
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         params.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
+   
+   if not (option and params): PgLOG.show_usage('fillawsusage')
+
+   PgDBI.dssdb_dbname()
+   cmdstr = "fillawsusage {}".format(' '.join(argv))
+   PgLOG.cmdlog(cmdstr)
+   PgFile.change_local_directory(USAGE['AWSDIR'])
+   filenames = get_log_file_names(option, params)
+   if filenames:
+      fill_aws_usages(filenames)
+   else:
+      PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
+
+   PgLOG.pglog(None, PgLOG.LOGWRN)
+   sys.exit(0)
+
+#
+# get the log file dates 
+#
+def get_log_file_names(option, params):
+
+   filenames = {}
+   if option == 'd':
+      for dt in params:
+         pdate = PgUtil.format_date(dt)
+         pd = PgUtil.format_date(pdate, USAGE['PFMT'])
+         fname = USAGE['AWSLOG'].format(pd, pdate)
+         fnames = glob.glob(fname)
+         if fnames: filenames[pdate] = sorted(fnames)
+   else:
+      if option == 'N':
+         edate = PgUtil.curdate()
+         pdate = PgUtil.adddate(edate, 0, 0, -int(params[0]))
+      else:
+         pdate = PgUtil.format_date(params[0])
+         if len(params) > 1:
+            edate = PgUtil.format_date(params[1])
+         else:
+            edate = PgUtil.curdate()
+      while pdate < edate:
+         pd = PgUtil.format_date(pdate, USAGE['PFMT'])
+         fname = USAGE['AWSLOG'].format(pd, pdate)
+         fnames = glob.glob(fname)
+         if fnames: filenames[pdate] = sorted(fnames)
+         pdate = PgUtil.adddate(pdate, 0, 0, 1)
+
+   return filenames
+
+#
+# Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs
+#
+def fill_aws_usages(filenames):
+
+   year = cntall = addall = 0
+   for pdate in filenames:
+      fnames = filenames[pdate]
+      records = {}
+      cntadd = entcnt = 0
+      for logfile in fnames:
+         if not op.isfile(logfile):
+            PgLOG.pglog("{}: Not exists for Gathering AWS usage".format(logfile), PgLOG.LOGWRN)
+            continue
+         PgLOG.pglog("Gathering AWS usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
+         aws = PgFile.open_local_file(logfile)
+         if not aws: continue
+         while True:
+            line = aws.readline()
+            if not line: break
+            entcnt += 1
+            if entcnt%20000 == 0:
+               dcnt = len(records)
+               PgLOG.pglog("{}: {}/{} AWS log entries processed/records to add".format(pdate, entcnt, dcnt), PgLOG.WARNLG)
+   
+            ms = re.match(r'^\w+ ([\w-]+) \[(\S+).*\] ([\d\.]+) .+ REST\.GET\.OBJECT \S+ "GET.+" \d+ - (\d+) \d+ .* ".+" "(.+)" ', line)
+            if not ms: continue
+            values = list(ms.groups())
+            if values[0] not in DSIDS: continue
+            dsid = DSIDS[values[0]]
+            size = int(values[3])
+            ip = values[2]
+            engine = values[4]
+            moff = engine.find('/')
+            if moff > 0:
+               if moff > 20: moff = 20
+               method = engine[0:moff].upper()
+            else:
+               method = "AWS"
+            key = "{}:{}:{}".format(ip, dsid, method)
+            if key in records:
+               records[key]['size'] += size
+               records[key]['fcount'] += 1
+            else:
+               (year, quarter, date, time) = get_record_date_time(values[1])
+               iprec =  PgIPInfo.get_missing_ipinfo(ip)
+               if not iprec: continue
+               records[key] = {'ip' : ip, 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
+                               'size' : size, 'fcount' : 1, 'method' : method, 'engine' : engine,
+                               'org_type' : iprec['org_type'], 'country' : iprec['country'],
+                               'region' : iprec['region'], 'email' : iprec['email']}
+         aws.close()
+      if records: cntadd = add_usage_records(records, year)
+      PgLOG.pglog("{}: {} AWS usage records added for {} entries at {}".format(pdate, cntadd, entcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+      cntall += entcnt
+      if cntadd:
+         addall += cntadd
+         if addall > cntadd:
+            PgLOG.pglog("{} AWS usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+def get_record_date_time(ctime):
+   
+   ms = re.search(r'^(\d+)/(\w+)/(\d+):(\d+:\d+:\d+)$', ctime)
+   if ms:
+      d = int(ms.group(1))
+      m = PgUtil.get_month(ms.group(2))
+      y = ms.group(3)
+      t = ms.group(4)
+      q = 1 + int((m-1)/3)
+      return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
+   else:
+      PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
+
+def add_usage_records(records, year):
+
+   cnt = 0
+   for key in records:
+      record = records[key]
+      cond = "date = '{}' AND time = '{}' AND ip = '{}' AND dsid = '{}'".format(record['date'], record['time'], record['ip'], record['dsid'])
+      if PgDBI.pgget(USAGE['PGTBL'], '', cond, PgLOG.LGEREX): continue
+      if add_to_allusage(year, record):
+         cnt += PgDBI.pgadd(USAGE['PGTBL'], record, PgLOG.LOGWRN)
+
+   return cnt
+
+
+def add_to_allusage(year, pgrec):
+
+   record = {'source' : 'A'}
+   flds = ['ip', 'dsid', 'date', 'time', 'quarter', 'size', 'method',
+           'org_type', 'country', 'region', 'email']
+
+   for fld in flds:
+      record[fld] = pgrec[fld]
+
+   return PgDBI.add_yearly_allusage(year, record)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.34}/src/rda_python_metrics/fillawsusage.usg

@@ -1,6 +1,6 @@
 
  Retrieves usage information from AWS Server logs under directory
-   ../rda/transer/AWSera5log/ to fill table 'wusage' in database 'rdadb'.
+   ../rda/transer/AWSera5log/ to fill table 'awsusage' in database 'rdadb'.
 
  Usage: fillawsusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
 

{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.34}/src/rda_python_metrics/fillosdfusage.py

@@ -22,7 +22,7 @@ from rda_python_common import PgSplit
 from . import PgIPInfo
 
 USAGE = {
-   'OSDFTBL'  : "wusage",
+   'OSDFTBL'  : "osdfusage",
    'OSDFDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/zji/osdflogs/",
    'OSDFGET' : 'wget -m -nH -np -nd https://pelicanplatform.org/pelican-access-logs/ncar-access-log/',
    'OSDFLOG' : "{}-cache.log",   # YYYY-MM-DD-cache.log
@@ -100,9 +100,7 @@ def get_log_file_names(option, params, datelimits):
 #
 def fill_osdf_usages(fnames):
 
-   cntall = addall = 0
-
-   fcnt = len(fnames)
+   year = cntall = addall = 0
    for logfile in fnames:
       linfo = PgFile.check_local_file(logfile)
       if not linfo:
@@ -119,45 +117,54 @@ def fill_osdf_usages(fnames):
       PgLOG.pglog("{}: Gathering OSDF usage at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
       osdf = PgFile.open_local_file(logfile)
       if not osdf: continue
+      records = {}
       cntadd = entcnt = 0
-      pkey = None
       while True:
          line = osdf.readline()
          if not line: break
          entcnt += 1
-         if entcnt%10000 == 0:
-            PgLOG.pglog("{}: {}/{} OSDF log entries processed/records added".format(logfile, entcnt, cntadd), PgLOG.WARNLG)
+         if entcnt%20000 == 0:
+            dcnt = len(records)
+            PgLOG.pglog("{}: {}/{} OSDF log entries processed/records added".format(logfile, entcnt, dcnt), PgLOG.WARNLG)
 
-         ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/rda\/([a-z]\d{6})\/(\S+)\].* \[Host:(\S+)\].* \[AppInfo:(\S+)\].* \[Read:(\d+)\]', line)
-         if not ms or ms.group(4) == 'N/A': continue
+         ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/rda\/([a-z]\d{6})\/\S+\].* \[Site:(\S+)\].* \[Host:(\S+)\].* \[AppInfo:(\S+)\].* \[Read:(\d+)\]', line)
+         if not ms: continue
          dt = ms.group(1)
          dsid = ms.group(2)
-         wfile = ms.group(3)
+         site = ms.group(3)
          ip = ms.group(4)
+         if ip == 'N/A': ip = '0.0.0.0'
          engine = ms.group(5)
          size = int(ms.group(6))
-         (year, quarter, date, time) = get_record_date_time(dt)
-         locflag = 'C'
-         if re.match(r'^curl', engine, re.I):
-            method = "CURL"
-         elif re.match(r'^wget', engine, re.I):
-            method = "WGET"
-         elif re.match(r'^python', engine, re.I):
-            method = "PYTHN"
-         elif re.match(r'^N/A', engine, re.I):
-            method = "N/A"
+         if re.match(r'^N/A', engine, re.I):
+            method = "OSDF"
          else:
-            method = "WEB"
-         method = "OSDF"
-
-         record = {'ip' : ip, 'dsid' : dsid, 'wfile' : wfile, 'date' : date,
-                   'time' : time, 'quarter' : quarter, 'size' : size,
-                   'locflag' : locflag, 'method' : method}
-         cntadd += add_file_usage(year, record)
+            moff = engine.find('/')
+            if moff > 0:
+               if moff > 20: moff = 20
+               method = engine[0:moff].upper()
+            else:
+               method = "OSDF"
+         key = "{}:{}:{}".format(ip, dsid, method)
+         if key in records:
+            records[key]['size'] += size
+            records[key]['fcount'] += 1
+         else:
+            (year, quarter, date, time) = get_record_date_time(dt)
+            iprec =  PgIPInfo.get_missing_ipinfo(ip)
+            if not iprec: continue
+            records[key] = {'ip' : ip, 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
+                            'size' : size, 'fcount' : 1, 'method' : method, 'engine' : engine,
+                            'org_type' : iprec['org_type'], 'country' : iprec['country'],
+                            'region' : iprec['region'], 'email' : iprec['email'], 'site' : site}
       osdf.close()
+      if records: cntadd = add_usage_records(records, year)
+      PgLOG.pglog("{}: {} OSDF usage records added for {} entries at {}".format(logfile, cntadd, entcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
       cntall += entcnt
-      addall += cntadd
-      PgLOG.pglog("{} OSDF usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
+      if cntadd:
+         addall += cntadd
+         if addall > cntadd:
+            PgLOG.pglog("{} OSDF usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
 
 
 def get_record_date_time(ctime):
@@ -173,6 +180,30 @@ def get_record_date_time(ctime):
    else:
       PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
 
+def add_usage_records(records, year):
+
+   cnt = 0
+   for key in records:
+      record = records[key]
+      cond = "date = '{}' AND time = '{}' AND ip = '{}' AND dsid = '{}'".format(record['date'], record['time'], record['ip'], record['dsid'])
+      if PgDBI.pgget(USAGE['OSDFTBL'], '', cond, PgLOG.LGEREX): continue
+      if add_to_allusage(year, record):
+         cnt += PgDBI.pgadd(USAGE['OSDFTBL'], record, PgLOG.LOGWRN)
+
+   return cnt
+
+def add_to_allusage(year, pgrec):
+
+   record = {'source' : 'P'}
+   flds = ['ip', 'dsid', 'date', 'time', 'quarter', 'size', 'method',
+           'org_type', 'country', 'region', 'email']
+
+   for fld in flds:
+      record[fld] = pgrec[fld]
+
+   return PgDBI.add_yearly_allusage(year, record)
+
+
 #
 # Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
 #
@@ -183,7 +214,7 @@ def add_file_usage(year, logrec):
 
    table = "{}_{}".format(USAGE['OSDFTBL'], year)
    cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
-   if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
+   if PgDBI.pgget(USAGE['OSDFTBL'], "", cond, PgLOG.LOGWRN): return 0
 
    wurec =  PgIPInfo.get_wuser_record(logrec['ip'], logrec['date'])
    if not wurec: return 0

{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.34}/src/rda_python_metrics/fillosdfusage.usg

@@ -1,6 +1,6 @@
 
  Retrieves usage information from OSDF Server logs under directory
-   /gpfs/fs1/collections/rda/work/zji/osdflogs/ to fill table 'wusage' in
+   /gpfs/fs1/collections/rda/work/zji/osdflogs/ to fill table 'osdfusage' in
    database 'dssdb'.
 
  Usage: fillosdfusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]

{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.34/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.32
+Version: 1.0.34
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.34}/src/rda_python_metrics.egg-info/entry_points.txt

@@ -1,5 +1,5 @@
 [console_scripts]
-fillawsusage = rda_python_metrics.fillaswusage:main
+fillawsusage = rda_python_metrics.fillawsusage:main
 fillcdgusage = rda_python_metrics.fillcdgusage:main
 fillcodusage = rda_python_metrics.fillcodusage:main
 fillcountry = rda_python_metrics.fillcountry:main

rda_python_metrics-1.0.32/src/rda_python_metrics/fillawsusage.py

@@ -1,254 +0,0 @@
-#!/usr/bin/env python3
-#
-###############################################################################
-#
-#     Title : fillawsusage
-#    Author : Zaihua Ji,  zji@ucar.edu
-#      Date : 03/11/2022
-#             2025-03-26 transferred to package rda_python_metrics from
-#             https://github.com/NCAR/rda-database.git
-#   Purpose : python program to retrieve info from AWS logs 
-#             and fill table wusages in PgSQL database dssdb.
-# 
-#    Github : https://github.com/NCAR/rda-pythn-metrics.git
-#
-###############################################################################
-#
-import sys
-import re
-import glob
-from os import path as op
-from rda_python_common import PgLOG
-from rda_python_common import PgUtil
-from rda_python_common import PgFile
-from rda_python_common import PgDBI
-from . import PgIPInfo
-
-USAGE = {
-   'PGTBL'  : "wusage",
-   'AWSDIR' : PgLOG.PGLOG["TRANSFER"] + "/AWSera5log",
-   'AWSLOG' : "{}/{}-00-00-00-*",
-   'PFMT'   : "YYYY/MM/DD"
-}
-
-DSIDS = {'nsf-ncar-era5' : PgUtil.format_dataset_id('d633000')}
-
-#
-# main function to run this program
-#
-def main():
-
-   params = []  # array of input values
-   argv = sys.argv[1:]
-   option = None
-
-   for arg in argv:
-      ms = re.match(r'^-(b|d|p|N)$', arg)
-      if ms:
-         opt = ms.group(1)
-         if opt == 'b':
-            PgLOG.PGLOG['BCKGRND'] = 1
-         elif option:
-            PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
-         else:
-            option = opt
-      elif re.match(r'^-', arg):
-         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
-      elif option:
-         params.append(arg)
-      else:
-         PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
-   
-   if not (option and params): PgLOG.show_usage('fillawsusage')
-
-   PgDBI.dssdb_dbname()
-   cmdstr = "fillawsusage {}".format(' '.join(argv))
-   PgLOG.cmdlog(cmdstr)
-   PgFile.change_local_directory(USAGE['AWSDIR'])
-   filenames = get_log_file_names(option, params)
-   if filenames:
-      fill_aws_usages(filenames)
-   else:
-      PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
-
-   PgLOG.pglog(None, PgLOG.LOGWRN)
-   sys.exit(0)
-
-#
-# get the log file dates 
-#
-def get_log_file_names(option, params):
-
-   filenames = []
-   if option == 'd':
-      for dt in params:
-         pdate = PgUtil.format_date(dt)
-         pd = PgUtil.format_date(pdate, USAGE['PFMT'])
-         fname = USAGE['AWSLOG'].format(pd, pdate)
-         fnames = glob.glob(fname)
-         if fnames: filenames.extend(sorted(fnames))
-   else:
-      if option == 'N':
-         edate = PgUtil.curdate()
-         pdate = PgUtil.adddate(edate, 0, 0, -int(params[0]))
-      else:
-         pdate = PgUtil.format_date(params[0])
-         if len(params) > 1:
-            edate = PgUtil.format_date(params[1])
-         else:
-            edate = PgUtil.curdate()
-      while pdate <= edate:
-         pd = PgUtil.format_date(pdate, USAGE['PFMT'])
-         fname = USAGE['AWSLOG'].format(pd, pdate)
-         fnames = glob.glob(fname)
-         if fnames: filenames.extend(sorted(fnames))
-         pdate = PgUtil.adddate(pdate, 0, 0, 1)
-
-   return filenames
-
-#
-# Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs
-#
-def fill_aws_usages(fnames):
-
-   cntall = addall = 0
-   fcnt = len(fnames)
-   for logfile in fnames:
-      if not op.isfile(logfile):
-         PgLOG.pglog("{}: Not exists for Gathering AWS usage".format(logfile), PgLOG.LOGWRN)
-         continue
-      PgLOG.pglog("Gathering usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
-      aws = PgFile.open_local_file(logfile)
-      if not aws: continue
-      ptime = ''
-      record = {}
-      cntadd = entcnt = 0
-      pkey = None
-      while True:
-         line = aws.readline()
-         if not line: break
-         entcnt += 1
-         if entcnt%10000 == 0:
-            PgLOG.pglog("{}: {}/{} AWS log entries processed/records added".format(logfile, entcnt, cntadd), PgLOG.WARNLG)
-
-         ms = re.match(r'^\w+ ([\w-]+) \[(\S+).*\] ([\d\.]+) .+ REST\.GET\.OBJECT (\S+) "GET.+" (200|206) - (\d+) (\d+) .* ".+" "(.+)" ', line)
-         if not ms: continue
-         values = list(ms.groups())
-         if values[0] not in DSIDS: continue
-         dsid = DSIDS[values[0]]
-         size = int(values[5])
-         fsize = int(values[6])
-         if fsize < 100: continue  # ignore small files
-         ip = values[2]
-         wfile = values[3]
-         stat = values[4]
-         engine = values[7]
-         (year, quarter, date, time) = get_record_date_time(values[1])
-         locflag = 'A'
-
-         if re.match(r'^aiobotocore', engine, re.I):
-            method = "AIOBT"
-         elif re.match(r'^rclone', engine, re.I):
-            method = "RCLON"
-         elif re.match(r'^python', engine, re.I):
-            method = "PYTHN"
-         else:
-            method = "WEB"
-         
-         key = "{}:{}:{}".format(ip, dsid, wfile) if stat == '206' else None
-
-         if record:
-            if key == pkey:
-               record['size'] += size
-               continue
-            cntadd += add_file_usage(year, record)
-         record = {'ip' : ip, 'dsid' : dsid, 'wfile' : wfile, 'date' : date,
-                   'time' : time, 'quarter' : quarter, 'size' : size,
-                   'locflag' : locflag, 'method' : method}
-         pkey = key
-         if not pkey:
-            cntadd += add_file_usage(year, record)
-            record = None
-      if record: cntadd += add_file_usage(year, record)
-      aws.close()
-      cntall += entcnt
-      addall += cntadd
-      PgLOG.pglog("{} AWS usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
-
-
-def get_record_date_time(ctime):
-   
-   ms = re.search(r'^(\d+)/(\w+)/(\d+):(\d+:\d+:\d+)$', ctime)
-   if ms:
-      d = int(ms.group(1))
-      m = PgUtil.get_month(ms.group(2))
-      y = ms.group(3)
-      t = ms.group(4)
-      q = 1 + int((m-1)/3)
-      return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
-   else:
-      PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
-
-#
-# Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
-#
-def add_file_usage(year, logrec):
-
-   pgrec = get_wfile_wid(logrec['dsid'], logrec['wfile'])
-   if not pgrec: return 0
-
-   table = "{}_{}".format(USAGE['PGTBL'], year)
-   cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
-   if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
-
-   wurec =  PgIPInfo.get_wuser_record(logrec['ip'], logrec['date'])
-   if not wurec: return 0
-   record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
-   record['wuid_read'] = wurec['wuid']
-   record['date_read'] = logrec['date']
-   record['time_read'] = logrec['time']
-   record['size_read'] = logrec['size']
-   record['method'] = logrec['method']
-   record['locflag'] = logrec['locflag']
-   record['ip'] = logrec['ip']
-   record['quarter'] = logrec['quarter']
-
-   if add_to_allusage(year, logrec, wurec):
-      return PgDBI.add_yearly_wusage(year, record)
-   else:
-      return 0
-
-def add_to_allusage(year, logrec, wurec):
-
-   pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'],
-            'country' : wurec['country'], 'region' : wurec['region']}
-   pgrec['dsid'] = logrec['dsid']
-   pgrec['date'] = logrec['date']
-   pgrec['quarter'] = logrec['quarter']
-   pgrec['time'] = logrec['time']
-   pgrec['size'] = logrec['size']
-   pgrec['method'] = logrec['method']
-   pgrec['ip'] = logrec['ip']
-   pgrec['source'] = 'A'
-   return PgDBI.add_yearly_allusage(year, pgrec)
-
-#
-# return wfile.wid upon success, 0 otherwise
-#
-def get_wfile_wid(dsid, wfile):
-
-   dscond = "dsid = '{}' AND wfile = '{}'".format(dsid, wfile) 
-   pgrec = PgDBI.pgget("wfile", "*", dscond)
-
-   if not pgrec:
-      pgrec = PgDBI.pgget("wmove", "wid, dsid", dscond)
-      if pgrec:
-         pgrec = PgDBI.pgget("wfile", "*", "wid = {}".format(pgrec['wid']))
-         if pgrec: pgrec['dsid'] = dsid
-
-   return pgrec
-
-#
-# call main() to start program
-#
-if __name__ == "__main__": main()