rda-python-metrics 2.0.4__tar.gz → 2.0.6__tar.gz

{rda_python_metrics-2.0.4/src/rda_python_metrics.egg-info → rda_python_metrics-2.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 2.0.4
+Version: 2.0.6
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "2.0.4"
+version = "2.0.6"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/fillawsusage.py

@@ -20,7 +20,7 @@ from .pg_ipinfo import PgIPInfo
 class FillAWSUsage(PgIPInfo, PgFile):
 
    def __init__(self):
-      super().__init()
+      super().__init__()
       self.USAGE = {
          'PGTBL'  : "awsusage",
          'AWSDIR' : self.PGLOG["TRANSFER"] + "/AWSera5log",

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/fillcdgusage.py

@@ -19,7 +19,7 @@ from .pg_ipinfo import PgIPInfo
 class FillCDGUsage(PgSplit, PgIPInfo):
 
    def __init__(self):
-      super().__init()
+      super().__init__()
       self.USAGE = {
          'TDSTBL'  : "tdsusage",
          'WEBTBL'  : "wusage",

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/fillcodusage.py

@@ -19,7 +19,7 @@ from .pg_ipinfo import PgIPInfo
 class FillCODUsage(PgIPInfo):
 
    def __init__(self):
-      super().__init()
+      super().__init__()
       # the define options for gathering COD data usage, one at a time
       self.MONTH = 0x02  # fet COD data usages for given months
       self.YEARS = 0x04  # get COD data usages for given years

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/fillglobususage.py

@@ -21,7 +21,7 @@ from .pg_ipinfo import PgIPInfo
 class FillGlobusUsage(PgIPInfo, PgSplit, PgFile):
 
    def __init__(self):
-      super().__init()
+      super().__init__()
       self.USAGE = {
          'PGTBL'  : "wusage",
          'GBSDIR' : self.PGLOG["GDEXWORK"] + "/logs/gridftp/",

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/fillipinfo.py

@@ -25,12 +25,10 @@ class FillIPInfo(PgIPInfo):
       self.NDAYS = 0x08  # fix data usages in recent number of days
       self.MULTI = (self.DATES|self.MONTH|self.YEARS)
       self.SINGL = (self.NDAYS)
-      self.IPINFO = {
-         'USGTBL'  : ['ipinfo', 'wuser', 'allusage', 'codusage', 'tdsusage'],
-         'CDATE' : self.curdate(),
-      }
+      self.IPINFO['USGTBL'] = ['ipinfo', 'wuser', 'allusage', 'codusage', 'tdsusage']
+      self.IPINFO['CDATE'] = self.curdate()
       self.inputs = []  # array of input values
-      self.table = None  # table names: ipinfo, allusage, globususage, or tdsusage
+      self.table = None
       self.option = 0
 
    # function to read parameters

rda_python_metrics-2.0.6/src/rda_python_metrics/fillrdadb.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+###############################################################################
+#     Title : fillrdadb
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 04/07/2022
+#             2025-03-26 transferred to package rda_python_metrics from
+#             https://github.com/NCAR/rda-database.git
+#             2025-12-17 convert to class FillRDADB
+#   Purpose : python program to retrieve info from data logs, and fill tables
+#             in PostgreSQL database.schema rdadb.dssdb
+#    Github : https://github.com/NCAR/rda-pythn-metrics.git
+###############################################################################
+import sys
+import re
+from rda_python_common.pg_util import PgUtil
+from rda_python_common.pg_dbi import PgDBI
+
+class FillRDADB(PgUtil, PgDBI):
+
+   def __init__(self):
+      super().__init__()
+      # the define options for gathering web online file usage, one at a time
+      self.DATES = 0x01  # get web file usages for given dates
+      self.MONTH = 0x02  # fet web file usages for given months
+      self.YEARS = 0x04  # get web file usages for given years
+      self.NDAYS = 0x08  # get web file usages in recent number of days 
+      self.CLNFL = 0x10  # clean unused file only
+      self.MASKS = (self.MONTH|self.YEARS|self.DATES|self.NDAYS)
+      self.RDADB = {
+         'OPTION' : 0,
+         'OPTVAL' : '',
+         'DOMAIL' : 1,
+         'BCKGRND' : ''
+      }
+      self.params = []   # array of input values
+
+   # function to read parameters
+   def read_parameters(self):
+      argv = sys.argv[1:]
+      for arg in argv:
+         if arg == "-b":
+            self.PGLOG['BCKGRND'] = "-b"
+            self.RDADB['BCKGRND'] = " -b"
+         elif arg == "-n":
+            self.RDADB['DOMAIL'] = 0
+            self.PGLOG['LOGMASK'] &= ~(self.EMLALL)
+         elif re.match(r'^-[cdmNy]$', arg) and not self.RDADB['OPTVAL']:
+            self.RDADB['OPTVAL'] = arg
+            if arg == "-c":
+               self.RDADB['OPTION'] |= self.CLNFL
+            elif arg == "-d":
+               self.RDADB['OPTION'] |= self.DATES
+            elif arg == "-m":
+               self.RDADB['OPTION'] |= self.MONTH
+            elif arg == "-y":
+               self.RDADB['OPTION'] |= self.YEARS
+            elif arg == "-N":
+               self.RDADB['OPTION'] |= self.NDAYS
+         elif re.match(r'^-.*', arg):
+            self.pglog(arg + ": Invalid Option", self.LGWNEX)
+         elif self.RDADB['OPTION']&self.MASKS:
+            self.params.append(arg)
+         else:
+            self.pglog(arg + ": Invalid Parameter", self.LGWNEX)
+      if not self.RDADB['OPTION'] or (self.RDADB['OPTION']&self.MASKS and not self.params):
+         self.show_usage('fillrdadb')
+      self.cmdlog("fillrdadb {}".format(' '.join(argv)))
+   
+   # function to start actions
+   def start_actions(self):
+      self.dssdb_dbname()
+      if self.RDADB['OPTION']&self.CLNFL: # clean unused file only
+         self.clean_unused_files()
+      elif self.RDADB['OPTION']&self.MASKS:
+         self.fill_rdadb(self.RDADB['OPTVAL'])
+   
+   # Fill self.RDADB info for given condition
+   def fill_rdadb(self, option):
+      filecond = '{} {}'.format(option, ' '.join(self.params))
+      self.pglog("Filling RDADB info for '{}' at {}".format(filecond, self.current_datetime()), self.LOGWRN)
+      # fill available custom OPeNDAP usages
+      self.pgsystem("fillcodusage {} {}".format(self.RDADB['BCKGRND'], filecond), self.LGWNEM, 5)
+      # fill available globus web data usages
+      self.pgsystem("fillglobususage {} {}".format(self.RDADB['BCKGRND'], filecond), self.LGWNEM, 5)
+      # fill available AWS web data usages
+      self.pgsystem("fillawsusage {} {}".format(self.RDADB['BCKGRND'], filecond), self.LGWNEM, 5)
+      # fill available OSDF web data usages
+      self.pgsystem("fillosdfusage {} {}".format(self.RDADB['BCKGRND'], filecond), self.LGWNEM, 5)
+      if self.RDADB['DOMAIL']: self.send_email_notice()
+      self.pglog("End Filling RDADB info at {}".format(self.current_datetime()), self.LGWNEM)
+   
+   # clean unused MSS and Web files
+   def clean_unused_files(self):
+      self.pglog("Check and clean deleted Web files that never been used at {}".format(self.current_datetime()), self.LOGWRN)
+      pgrecs = self.pgmget("wfile", "wid", "status = 'D'", self.LGWNEX)
+      allcnt = len(pgrecs['wid']) if pgrecs else 0
+      self.pglog("{} record(s) retrieved from Table 'wfile' at {}".format(allcnt, self.current_datetime()), self.LOGWRN)
+      procnt = delcnt = 0
+      if allcnt:
+         fcond = r"wid = {} AND org_type <> 'DSS' AND wuid_read = wuid"
+         for fid in pgrecs['wid']:
+            procnt += 1
+            if procnt%5000 == 0:
+               self.pglog("{}/{} record(s) processed/removed from Table 'wfile'".format(procnt, delcnt), self.WARNLG)
+            if not self.pgget("wusage, wuser", "", fcond.format(fid), self.LGWNEX):
+               # deleted web file never been used
+               delcnt += self.pgdel("wfile", "wid = {}".format(fid), self.LGWNEX)
+      self.pglog("{} record(s) removed from Table 'wfile' at {}".format(delcnt, self.current_datetime()), self.LOGWRN)
+   
+   # email notice of job done
+   def send_email_notice(self):
+      msg = ("Hi All,\n\nRDADB weekly data usage gathering is done at {}.\n\n".format(self.current_datetime()) +
+             "Please Let me know if you notice any problem.\n\nThanks,\n\nHua\n")
+      pgrecs = self.pgmget("dssgrp", "logname", "email_flag = 'Y'", self.LGWNEX)
+      if pgrecs:
+         receiver = ""
+         for logname in pgrecs['logname']:
+            if receiver: receiver += ', '
+            receiver += (logname + "@ucar.edu")
+         self.send_email("RDADB Weekly Data Usage Gathered on " + self.curdate(), receiver, msg)
+
+# main function to excecute this script
+def main():
+   object = FillRDADB()
+   object.read_parameters()
+   object.start_actions()
+   object.pgexit(0)
+
+# call main() to start program
+if __name__ == "__main__": main()

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/viewallusage.py

@@ -293,11 +293,15 @@ class ViewALLUsage(PgView):
          else:
             self.condition = joins
       if 'E' in self.params or 'I' in self.params:
-         self.condition += self.notice_condition(self.params['E'], self.params['I'], self.params['t'][0])
+         self.condition += self.notice_condition(
+            self.params.get('E'),
+            self.params.get('I'),
+            self.params.get('t', [None])[0]
+         )
       if groupnames and self.sfields: self.condition += " GROUP BY " + groupnames
       if having: self.condition += " HAVING " + having
    
-   # exand records as needed
+   # expand records as needed
    def expand_records(self, records):
       recs = self.expand_query("TIME", records, self.params, self.EXPAND)
       trecs = self.expand_query("USER", records, self.params, self.EXPAND, self.VUSG, self.SNS, self.FLDS)

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/viewawsusage.py

@@ -261,10 +261,14 @@ class ViewAWSUsage(PgView):
          else:
             self.condition = joins
       if 'E' in self.params or 'I' in self.params:
-         self.condition += self.notice_condition(self.params['E'], None, self.params['t'][0])
+         self.condition += self.notice_condition(
+            self.params.get('E'),
+            self.params.get('I'),
+            self.params.get('t', [None])[0]
+         )
       if groupnames and self.sfields: self.condition += " GROUP BY " + groupnames
 
-   # exand records as needed
+   # expand records as needed
    def expand_records(self, records):
       recs = self.expand_query("TIME", records, self.params, self.EXPAND)
       trecs = self.expand_query("USER", records, self.params, self.EXPAND, self.VUSG, self.SNS, self.FLDS)

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/viewcheckusage.py

@@ -245,7 +245,7 @@ class ViewCheckUsage(PgView):
       if groupnames and self.sfields: self.condition += " GROUP BY " + groupnames
       if having: self.condition += " HAVING " + having
 
-   # exand records as needed
+   # expand records as needed
    def expand_records(self, records):
       recs = self.expand_query("TIME", records, self.params, self.EXPAND)
       trecs = self.expand_query("CHECK", records, self.params, self.EXPAND, self.VUSG, self.SNS, self.FLDS)

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/viewcodusage.py

@@ -263,10 +263,14 @@ def build_query_strings(self, usgtable):
       else:
          self.condition = joins
    if 'E' in self.params or 'I' in self.params:
-      self.condition += self.notice_condition(self.params['E'], self.params['I'], self.params['t'][0])
+      self.condition += self.notice_condition(
+         self.params.get('E'),
+         self.params.get('I'),
+         self.params.get('t', [None])[0]
+      )
    if groupnames and self.sfields: self.condition += " GROUP BY " + groupnames
 
-# exand records as needed
+# expand records as needed
 def expand_records(self, records):
    recs = self.expand_query("TIME", records, self.params, self.EXPAND)
    trecs = self.expand_query("USER", records, self.params, self.EXPAND, self.VUSG, self.SNS, self.FLDS)

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/viewordusage.py

@@ -282,7 +282,7 @@ class ViewORDUsage(PgView):
       if groupnames and self.sfields: self.condition += " GROUP BY " + groupnames
       if having: self.condition += " HAVING " + having
 
-   # exand records as needed
+   # expand records as needed
    def expand_records(self, records):
       recs = self.expand_query("TIME", records, self.params, self.EXPAND)
       trecs = self.expand_query("USER", records, self.params, self.EXPAND, self.VUSG, self.SNS, self.FLDS)

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/viewosdfusage.py

@@ -253,18 +253,22 @@ class ViewOSDFUsage(PgView):
                self.condition += cnd
                (self.tablenames, joins) = self.join_query_tables(fld[3], self.tablenames, joins, usgtable)
       # append joins, group by, order by, and having strings to condition string
-      if 'E' in self.params or 'I' in self.self.params:
+      if 'E' in self.params or 'I' in self.params:
          (self.tablenames, joins) = self.join_query_tables("emreceive", self.tablenames, joins, usgtable)
       if joins:
          if self.condition:
             self.condition = "{} AND {}".format(joins, self.condition)
          else:
             self.condition = joins
-      if 'E' in self.self.params or 'I' in self.params:
-         self.condition += self.notice_condition(self.params['E'], None, self.params['t'][0])
+      if 'E' in self.params or 'I' in self.params:
+         self.condition += self.notice_condition(
+            self.params.get('E'),
+            self.params.get('I'),
+            self.params.get('t', [None])[0]
+         )
       if groupnames and self.sfields: self.condition += " GROUP BY " + groupnames
 
-   # exand records as needed
+   # expand records as needed
    def expand_records(self, records):
       recs = self.expand_query("TIME", records, self.params, self.EXPAND)
       trecs = self.expand_query("USER", records, self.params, self.EXPAND, self.VUSG, self.SNS, self.FLDS)

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/viewrqstusage.py

@@ -301,11 +301,15 @@ class ViewRQSTUsage(PgView):
             self.condition = joins
       if 'n' in self.params: self.condition += " AND location > ' '"
       if 'E' in self.params or 'I' in self.params:
-         self.condition += self.notice_condition(self.params['E'], self.params['I'], self.params['t'][0])
+         self.condition += self.notice_condition(
+            self.params.get('E'),
+            self.params.get('I'),
+            self.params.get('t', [None])[0]
+         )
       if groupnames and self.sfields: self.condition += " GROUP BY " + groupnames
       if having: self.condition += " HAVING " + having
    
-   # exand records as needed
+   # expand records as needed
    def expand_records(self, records):
       recs = self.expand_query("TIME", records, self.params, self.EXPAND)
       trecs = self.expand_query("USER", records, self.params, self.EXPAND, self.VUSG, self.SNS, self.FLDS)

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6}/src/rda_python_metrics/viewtdsusage.py

@@ -266,10 +266,14 @@ class ViewTDSUsage(PgView):
          else:
             self.condition = joins
       if 'E' in self.params or 'I' in self.params:
-         self.condition += self.notice_condition(self.params['E'], None, self.params['t'][0])
+         self.condition += self.notice_condition(
+            self.params.get('E'),
+            self.params.get('I'),
+            self.params.get('t', [None])[0]
+         )
       if groupnames and self.sfields: self.condition += " GROUP BY " + groupnames
 
-   # exand records as needed
+   # expand records as needed
    def expand_records(self, records):
       recs = self.expand_query("TIME", records, self.params, self.EXPAND)
       trecs = self.expand_query("USER", records, self.params, self.EXPAND, self.VUSG, self.SNS, self.FLDS)

{rda_python_metrics-2.0.4 → rda_python_metrics-2.0.6/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 2.0.4
+Version: 2.0.6
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

rda_python_metrics-2.0.4/src/rda_python_metrics/fillrdadb.py

@@ -1,129 +0,0 @@
-#!/usr/bin/env python3
-###############################################################################
-#     Title : fillrdadb
-#    Author : Zaihua Ji,  zji@ucar.edu
-#      Date : 04/07/2022
-#             2025-03-26 transferred to package rda_python_metrics from
-#             https://github.com/NCAR/rda-database.git
-#             2025-12-17 convert to class FillRDADB
-#   Purpose : python program to retrieve info from data logs, and fill tables
-#             in PostgreSQL database.schema rdadb.dssdb
-#    Github : https://github.com/NCAR/rda-pythn-metrics.git
-###############################################################################
-import sys
-import re
-from rda_python_common.pg_util import PgUtil
-
-class FillRDADB(PgUtil):
-
-   def __init__(self):
-      super().__init__()
-      # the define options for gathering web online file usage, one at a time
-      self.DATES = 0x01  # get web file usages for given dates
-      self.MONTH = 0x02  # fet web file usages for given months
-      self.YEARS = 0x04  # get web file usages for given years
-      self.NDAYS = 0x08  # get web file usages in recent number of days 
-      self.CLNFL = 0x10  # clean unused file only
-      self.MASKS = (self.MONTH|self.YEARS|self.DATES|self.NDAYS)
-      self.RDADB = {
-         'OPTION' : 0,
-         'OPTVAL' : '',
-         'DOMAIL' : 1,
-         'BCKGRND' : ''
-      }
-      self.params = []   # array of input values
-
-# function to read parameters
-def read_paramers(self):
-   argv = sys.argv[1:]
-   for arg in argv:
-      if arg == "-b":
-         self.PGLOG['BCKGRND'] = "-b"
-         self.RDADB['BCKGRND'] = " -b"
-      elif arg == "-n":
-         self.RDADB['DOMAIL'] = 0
-         self.PGLOG['LOGMASK'] &= ~(self.EMLALL)
-      elif re.match(r'^-[cdmNy]$', arg) and not self.RDADB['OPTVAL']:
-         self.RDADB['OPTVAL'] = arg
-         if arg == "-c":
-            self.RDADB['OPTION'] |= self.CLNFL
-         elif arg == "-d":
-            self.RDADB['OPTION'] |= self.DATES
-         elif arg == "-m":
-            self.RDADB['OPTION'] |= self.MONTH
-         elif arg == "-y":
-            self.RDADB['OPTION'] |= self.YEARS
-         elif arg == "-N":
-            self.RDADB['OPTION'] |= self.NDAYS
-      elif re.match(r'^-.*', arg):
-         self.pglog(arg + ": Invalid Option", self.LGWNEX)
-      elif self.RDADB['OPTION']&self.MASKS:
-         self.params.append(arg)
-      else:
-         self.pglog(arg + ": Invalid Parameter", self.LGWNEX)
-   if not self.RDADB['OPTION'] or (self.RDADB['OPTION']&self.MASKS and not self.params):
-      self.show_usage('fillrdadb')
-   self.cmdlog("fillrdadb {}".format(' '.join(argv)))
-
-# function to start actions
-def start_actions(self):
-   self.dssdb_dbname()
-   if self.RDADB['OPTION']&self.CLNFL: # clean unused file only
-      self.clean_unused_files()
-   elif self.RDADB['OPTION']&self.MASKS:
-      self.fill_rdadb(self.RDADB['OPTVAL'])
-
-# Fill self.RDADB info for given condition
-def fill_rdadb(self, option):
-   filecond = '{} {}'.format(option, ' '.join(self.params))
-   self.pglog("Filling self.RDADB info for '{}' at {}".format(filecond, self.current_datetime()), self.LOGWRN)
-   # fill available custom OPeNDAP usages
-   self.pgsystem("fillcodusage {} {}".format(self.RDADB['BCKGRND'], filecond), self.LGWNEM, 5)
-   # fill available globus web data usages
-   self.pgsystem("fillglobususage {} {}".format(self.RDADB['BCKGRND'], filecond), self.LGWNEM, 5)
-   # fill available AWS web data usages
-   self.pgsystem("fillawsusage {} {}".format(self.RDADB['BCKGRND'], filecond), self.LGWNEM, 5)
-   # fill available OSDF web data usages
-   self.pgsystem("fillosdfusage {} {}".format(self.RDADB['BCKGRND'], filecond), self.LGWNEM, 5)
-   if self.RDADB['DOMAIL']: self.send_email_notice()
-   self.pglog("End Filling self.RDADB info at {}".format(self.current_datetime()), self.LGWNEM)
-
-# clean unused MSS and Web files
-def clean_unused_files(self):
-   self.pglog("Check and clean deleted Web files that never been used at {}".format(self.current_datetime()), self.LOGWRN)
-   pgrecs = self.pgmget("wfile", "wid", "status = 'D'", self.LGWNEX)
-   allcnt = len(pgrecs['wid']) if pgrecs else 0
-   self.pglog("{} record(s) retrieved from Table 'wfile' at {}".format(allcnt, self.current_datetime()), self.LOGWRN)
-   procnt = delcnt = 0
-   if allcnt:
-      fcond = r"wid = {} AND org_type <> 'DSS' AND wuid_read = wuid"
-      for fid in pgrecs['wid']:
-         procnt += 1
-         if procnt%5000 == 0:
-            self.pglog("{}/{} record(s) processed/removed from Table 'wfile'".format(procnt, delcnt), self.WARNLG)
-         if not self.pgget("wusage, wuser", "", fcond.format(fid), self.LGWNEX):
-            # deleted web file never been used
-            delcnt += self.pgdel("wfile", "wid = {}".format(fid), self.LGWNEX)
-   self.pglog("{} record(s) removed from Table 'wfile' at {}".format(delcnt, self.current_datetime()), self.LOGWRN)
-
-# email notice of job done
-def send_email_notice(self):
-   msg = ("Hi All,\n\nself.RDADB weekly data usage gathering is done at {}.\n\n".format(self.current_datetime()) +
-          "Please Let me know if you notice any problem.\n\nThanks,\n\nHua\n")
-   pgrecs = self.pgmget("dssgrp", "logname", "email_flag = 'Y'", self.LGWNEX)
-   if pgrecs:
-      receiver = ""
-      for logname in pgrecs['logname']:
-         if receiver: receiver += ', '
-         receiver += (logname + "@ucar.edu")
-      self.send_email("self.RDADB Weekly Data Usage Gathered on " + self.curdate(), receiver, msg)
-
-# main function to excecute this script
-def main():
-   object = FillRDADB()
-   object.read_parameters()
-   object.start_actions()
-   object.pgexit(0)
-
-# call main() to start program
-if __name__ == "__main__": main()