rda-python-metrics 3.0.8__tar.gz → 3.0.10__tar.gz

{rda_python_metrics-3.0.8/src/rda_python_metrics.egg-info → rda_python_metrics-3.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 3.0.8
+Version: 3.0.10
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-3.0.8 → rda_python_metrics-3.0.10}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "3.0.8"
+version = "3.0.10"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]

{rda_python_metrics-3.0.8 → rda_python_metrics-3.0.10}/src/rda_python_metrics/fillcodusage.py

@@ -28,14 +28,18 @@ class FillCODUsage(PgIPInfo):
       self.NDAYS = 0x08  # get COD data usages in recent number of days 
       self.FILES = 0x10  # get given file names
       self.GTALL = 0x20  # get all data files of read
-      self.MASKS = (self.MONTH|self.YEARS|self.NDAYS|self.FILES)
+      self.DATES = 0x40  # get COD data usages for given dates
+      self.PERID = 0x80  # get COD data usages between two dates
+      self.MASKS = (self.MONTH|self.YEARS|self.NDAYS|self.FILES|self.DATES|self.PERID)
       self.USAGE = {
          'OPTION' : 0,
          'PGTBL'  : "codusage",
          'WEBLOG' : "/var/log/httpd",
       }
       self.USERS = {}  # cache user info for aid
-      self.datelimit = ''
+      self.datelimit = ''   # earliest log entry date to keep
+      self.endlimit = ''    # latest log entry date to keep
+      self.filldates = None # the exact log entry dates to keep, for -d
       self.params = []  # array of input values
 
    def read_parameters(self):
@@ -44,14 +48,18 @@ class FillCODUsage(PgIPInfo):
       for arg in argv:
          if arg == "-b":
             self.PGLOG['BCKGRND'] = 1
-         elif re.match(r'^-[afmNy]$', arg) and self.USAGE['OPTION'] == 0:
+         elif re.match(r'^-[adfmNpy]$', arg) and self.USAGE['OPTION'] == 0:
             if arg == "-a":
                self.USAGE['OPTION'] = self.GTALL
                self.params = ['']
+            elif arg == "-d":
+               self.USAGE['OPTION'] = self.DATES
             elif arg == "-f":
                self.USAGE['OPTION'] = self.FILES
             elif arg == "-m":
                self.USAGE['OPTION'] = self.MONTH
+            elif arg == "-p":
+               self.USAGE['OPTION'] = self.PERID
             elif arg == "-y":
                self.USAGE['OPTION'] = self.YEARS
             elif arg == "-N":
@@ -69,17 +77,40 @@ class FillCODUsage(PgIPInfo):
    def start_actions(self):   
       """Function to start actions."""
       if self.USAGE['OPTION']&self.NDAYS:
-         curdate = self.curdate()
-         self.datelimit = self.adddate(curdate, 0, 0, -int(self.params[0]))
-         self.USAGE['OPTION'] = self.MONTH
-         self.params = []
-         while curdate >= self.datelimit:
-            (year, month, day) = curdate.split('-')
-            self.params.append("{}-{}".format(year, month))
-            curdate = self.adddate(curdate, 0, 0, -int(day))
+         self.datelimit = self.adddate(self.curdate(), 0, 0, -int(self.params[0]))
+         self.set_month_params(self.datelimit, self.curdate())
+      elif self.USAGE['OPTION']&self.DATES:
+         dates = sorted(self.format_date(dt) for dt in self.params)
+         self.filldates = set(dates)
+         self.set_month_params(dates[0], dates[-1])
+      elif self.USAGE['OPTION']&self.PERID:
+         self.datelimit = self.format_date(self.params[0])
+         self.endlimit = self.format_date(self.params[1]) if len(self.params) > 1 else self.curdate()
+         self.set_month_params(self.datelimit, self.endlimit)
       self.fill_cod_usages(self.USAGE['OPTION'], self.params)
       self.pglog(None, self.LOGWRN|self.SNDEML)  # send email out if any
 
+   def set_month_params(self, bdate, edate):
+      """Replace the inputs with the months holding bdate through edate.
+
+      One COD log file holds a whole month, so a request made in dates is served
+      by reading those months and narrowing the entries in skip_log_date().
+      """
+      self.USAGE['OPTION'] = self.MONTH
+      self.params = []
+      curdate = edate
+      while curdate >= bdate:
+         (year, month, day) = curdate.split('-')
+         self.params.append("{}-{}".format(year, month))
+         curdate = self.adddate(curdate, 0, 0, -int(day))
+
+   def skip_log_date(self, cdate):
+      """Tell whether a log entry date falls outside what was asked for."""
+      if self.filldates is not None: return cdate not in self.filldates
+      if self.datelimit and cdate < self.datelimit: return True
+      if self.endlimit and cdate > self.endlimit: return True
+      return False
+
    def fill_cod_usages(self, option, inputs):
       """Fill COD usages into table dssdb.codusage of DSS PostgreSQL database from cod access logs."""
       cntall = cntadd = 0
@@ -128,7 +159,7 @@ class FillCODUsage(PgIPInfo):
                      cntadd += self.add_usage_records(records, cdate)
                      records = {}            
                   pdate = cdate
-               if self.datelimit and cdate < self.datelimit: continue
+               if self.skip_log_date(cdate): continue
                if aid in records:
                   records[aid]['size'] += size
                   records[aid]['count'] += 1

{rda_python_metrics-3.0.8 → rda_python_metrics-3.0.10}/src/rda_python_metrics/fillcodusage.usg

@@ -5,13 +5,18 @@
  into the 'codusage' table in PostgreSQL database 'rdadb'.
 
  USAGE:
-   fillcodusage [-b] (-a | -f LogFileNames | -m MonthList |
-                      -N NumberDays | -y YearList)
+   fillcodusage [-b] (-a | -d LogEntryDates | -f LogFileNames | -m MonthList |
+                      -N NumberDays | -p BeginDate [EndDate] | -y YearList)
 
- OPTIONS (choose exactly one of -a, -f, -m, -N, or -y):
+ OPTIONS (choose exactly one of -a, -d, -f, -m, -N, -p, or -y):
 
    -a   Process all available log files.
 
+   -d LogEntryDates
+        Process the given dates. Date format: YYYY-MM-DD (e.g. 2024-01-15).
+        A log file holds a whole month, so the months holding these dates are
+        read and only entries falling on the given dates are recorded.
+
    -f LogFileNames
         Process the given log file names directly.
 
@@ -22,6 +27,11 @@
    -N NumberDays
         Process log files covering the most recent NumberDays days.
 
+   -p BeginDate [EndDate]
+        Process log entries between BeginDate and EndDate, inclusive.
+        EndDate defaults to today if omitted. As with -d, whole months are
+        read and the entries outside the range are skipped.
+
    -y YearList
         Process log files for the given years.
         Year format: YYYY (e.g. 2024).
@@ -30,3 +40,4 @@
 
  EXAMPLE:
    fillcodusage -m 2024-01 2024-02
+   fillcodusage -d 2024-01-15 2024-02-03

{rda_python_metrics-3.0.8 → rda_python_metrics-3.0.10}/src/rda_python_metrics/fillglobususage.py

@@ -28,9 +28,11 @@ class FillGlobusUsage(PgIPInfo, PgSplit, PgFile):
          'PGTBL'  : "wusage",
          'GBSDIR' : self.PGLOG["GDEXWORK"] + "/logs/gridftp/",
          'GBSLOG' : "access_log_gridftp0{}_{}",
+         'GBSDAYS' : 7,   # a log file is named for its first date and holds a week
       }
       self.logfiles = []
       self.datelimits = [None, None]
+      self.filldates = None   # the exact log entry dates to keep, for -d
       self.params = []  # array of input values
       self.option = self.cmdstr = None
 
@@ -73,11 +75,12 @@ class FillGlobusUsage(PgIPInfo, PgSplit, PgFile):
       if self.option == 'f':
          self.logfiles = self.params
       elif self.option == 'd':
-         for pdate in self.params:
-            fdate = self.format_date(pdate, 'MMDDYYYY')
-            fname = self.USAGE['GBSLOG'].format('?', fdate)
-            fnames = glob.glob(fname)
-            if fnames: self.logfiles.extend(sorted(fnames))
+         # a log file is named for its first date and holds a week, so the file
+         # covering a date is found by looking back, not by its own name
+         self.filldates = {self.format_date(pdate) for pdate in self.params}
+         for pdate in sorted(self.filldates):
+            for fname in self.get_covering_logfiles(pdate):
+               if fname not in self.logfiles: self.logfiles.append(fname)
       else:
          if self.option == 'N':
             edate = self.curdate()
@@ -88,13 +91,33 @@ class FillGlobusUsage(PgIPInfo, PgSplit, PgFile):
                edate = self.datelimits[1] = self.params[1]
             else:
                edate = self.curdate()
-         while pdate <= edate:
-            fdate = self.format_date(pdate, 'MMDDYYYY')
+         # start a week early so the file covering the first dates is picked up;
+         # its entries before datelimits[0] are skipped as they are read
+         gdate = self.adddate(pdate, 0, 0, -self.USAGE['GBSDAYS'])
+         while gdate <= edate:
+            fdate = self.format_date(gdate, 'MMDDYYYY')
             fname = self.USAGE['GBSLOG'].format('?', fdate)
-            fnames = glob.glob(fname)
-            if fnames: self.logfiles.extend(sorted(fnames))
-            pdate = self.adddate(pdate, 0, 0, 1)
+            for gname in sorted(glob.glob(fname)):
+               if gname not in self.logfiles: self.logfiles.append(gname)
+            gdate = self.adddate(gdate, 0, 0, 1)
    
+   def get_covering_logfiles(self, pdate):
+      """Get the log files holding pdate, searching back over a week of names."""
+      gdate = pdate
+      for i in range(self.USAGE['GBSDAYS']):
+         fname = self.USAGE['GBSLOG'].format('?', self.format_date(gdate, 'MMDDYYYY'))
+         fnames = sorted(glob.glob(fname))
+         if fnames: return fnames
+         gdate = self.adddate(gdate, 0, 0, -1)
+      return []
+
+   def skip_log_date(self, date):
+      """Tell whether a log entry date falls outside what was asked for."""
+      if self.filldates is not None: return date not in self.filldates
+      if self.datelimits[0] and date < self.datelimits[0]: return True
+      if self.datelimits[1] and date > self.datelimits[1]: return True
+      return False
+
    def fill_globus_usages(self):
       """Fill Globus usages into table dssdb.globususage of DSS PostgreSQL database from globus access logs."""
       cntall = addall = 0
@@ -130,8 +153,7 @@ class FillGlobusUsage(PgIPInfo, PgSplit, PgFile):
             sline = ms.group(7)
             engine = ms.group(8)
             (year, quarter, date, time) = self.get_record_date_time(ms.group(2))
-            if self.datelimits[0] and date < self.datelimits[0]: continue
-            if self.datelimits[1] and date > self.datelimits[1]: continue
+            if self.skip_log_date(date): continue
             locflag = 'O' if re.match(r'^https://stratus\.', sline) else 'G'
             idx = wfile.find('?')
             if idx > -1: wfile = wfile[:idx]

{rda_python_metrics-3.0.8 → rda_python_metrics-3.0.10}/src/rda_python_metrics/fillrdadb.py

@@ -32,15 +32,26 @@ class FillRDADB(PgUtil, PgDBI):
          'OPTION' : 0,
          'OPTVAL' : '',
          'DOMAIL' : 1,
-         'BCKGRND' : ''
+         'BCKGRND' : '',
+         'BACKDAYS' : 0   # days to reach back for missed dates, set by -B
+      }
+      # the usage table and its date field, per command to reprocess with -d
+      self.BACKFILL = {
+         'fillosdfusage'   : ('osdfusage', 'date'),
+         'fillawsusage'    : ('awsusage', 'date'),
+         'fillglobususage' : ('wusage', 'date_read'),
+         'fillcodusage'    : ('codusage', 'date')
       }
       self.params = []   # array of input values
 
    def read_parameters(self):
       """Function to read parameters."""
       argv = sys.argv[1:]
+      backopt = 0   # the value after -B is the number of days to reach back
       for arg in argv:
-         if arg == "-b":
+         if arg == "-B":
+            backopt = 1
+         elif arg == "-b":
             self.PGLOG['BCKGRND'] = "-b"
             self.RDADB['BCKGRND'] = " -b"
          elif arg == "-n":
@@ -60,12 +71,20 @@ class FillRDADB(PgUtil, PgDBI):
                self.RDADB['OPTION'] |= self.NDAYS
          elif re.match(r'^-.*', arg):
             self.pglog(arg + ": Invalid Option", self.LGWNEX)
+         elif backopt:
+            if not re.match(r'^\d+$', arg):
+               self.pglog(arg + ": Number of days back must be a whole number", self.LGWNEX)
+            self.RDADB['BACKDAYS'] = int(arg)
+            backopt = 0
          elif self.RDADB['OPTION']&self.MASKS:
             self.params.append(arg)
          else:
             self.pglog(arg + ": Invalid Parameter", self.LGWNEX)
+      if backopt: self.pglog("-B: Miss number of days to reach back", self.LGWNEX)
       if not self.RDADB['OPTION'] or (self.RDADB['OPTION']&self.MASKS and not self.params):
          self.show_usage('fillrdadb')
+      if self.RDADB['BACKDAYS'] and not self.RDADB['OPTION']&self.NDAYS:
+         self.pglog("-B: Ignored, reaching back for missed dates needs -N", self.LOGWRN)
       self.cmdlog("fillrdadb {}".format(' '.join(argv)))
    
    def start_actions(self):
@@ -88,9 +107,63 @@ class FillRDADB(PgUtil, PgDBI):
       self.pgsystem("fillawsusage {} {}".format(self.RDADB['BCKGRND'], filecond), self.LGWNEM, 5)
       # fill available OSDF web data usages
       self.pgsystem("fillosdfusage {} {}".format(self.RDADB['BCKGRND'], filecond), self.LGWNEM, 5)
+      # the -N window is the only thing that reaches a log file again, so one
+      # published after its date leaves the window is never asked for; -B catches those
+      if self.RDADB['BACKDAYS'] and self.RDADB['OPTION']&self.NDAYS:
+         self.backfill_missed_dates(int(self.params[0]))
       if self.RDADB['DOMAIL']: self.send_email_notice()
       self.pglog("End Filling RDADB info at {}".format(self.current_datetime()), self.LGWNEM)
    
+   def backfill_missed_dates(self, ndays):
+      """Reprocess the dates before the -N window that still hold no usage record.
+
+      Only runs when -B gave a number of days to reach back. A log file is
+      sometimes published after its date has slid out of the -N window, and
+      nothing fetches it again, so the date stays empty however often the job
+      runs. Asking each usage table which dates it is missing keeps those dates
+      being retried until their log shows up, as far back as -B allows.
+
+      A date whose log holds no usable entry simply keeps being offered until it
+      ages out past BACKDAYS. That is deliberate: the obvious alternative, taking
+      an already processed log as proof of coverage, would silently write off the
+      dates left empty by a run that died partway.
+      """
+      edate = self.adddate(self.curdate(), 0, 0, -ndays)
+      bdate = self.adddate(self.curdate(), 0, 0, -self.RDADB['BACKDAYS'])
+      if bdate >= edate: return
+      for cmd in self.BACKFILL:
+         (tbname, datefld) = self.BACKFILL[cmd]
+         dates = self.get_unfilled_dates(tbname, datefld, bdate, edate)
+         if not dates: continue
+         self.pglog("{}: {} date(s) between {} and {} hold no usage, reprocessing".format(
+                    cmd, len(dates), bdate, edate), self.LOGWRN)
+         self.pgsystem("{} {} -d {}".format(cmd, self.RDADB['BCKGRND'], ' '.join(dates)), self.LGWNEM, 5)
+
+   def get_unfilled_dates(self, tbname, datefld, bdate, edate):
+      """Get the dates in [bdate, edate) holding no record in a usage table."""
+      filled = set()
+      for table in self.get_usage_tables(tbname, bdate, edate):
+         cnd = "{} >= '{}' AND {} < '{}'".format(datefld, bdate, datefld, edate)
+         pgrecs = self.pgmget(table, 'DISTINCT ' + datefld, cnd, self.LOGWRN)
+         if pgrecs and datefld in pgrecs:
+            filled.update(str(d) for d in pgrecs[datefld])
+      dates = []
+      pdate = bdate
+      while pdate < edate:
+         if pdate not in filled: dates.append(pdate)
+         pdate = self.adddate(pdate, 0, 0, 1)
+      return dates
+
+   def get_usage_tables(self, tbname, bdate, edate):
+      """Get the existing table names for a usage table, one per year if split."""
+      tables = []
+      for year in range(int(bdate[0:4]), int(edate[0:4]) + 1):
+         table = "{}_{}".format(tbname, year)
+         if self.pgget('information_schema.tables', 'table_name',
+                       "table_name = '{}'".format(table), self.LOGWRN):
+            tables.append(table)
+      return tables if tables else [tbname]
+
    def clean_unused_files(self):
       """Clean unused MSS and Web files."""
       self.pglog("Check and clean deleted Web files that never been used at {}".format(self.current_datetime()), self.LOGWRN)

{rda_python_metrics-3.0.8 → rda_python_metrics-3.0.10}/src/rda_python_metrics/fillrdadb.usg

@@ -6,7 +6,7 @@
    fillcodusage, fillglobususage, fillawsusage, fillosdfusage.
 
  USAGE:
-   fillrdadb [-b] [-n]
+   fillrdadb [-b] [-n] [-B NumberDaysBack]
              (-c | -d DateList | -m MonthList | -N NumberDays | -y YearList)
 
  OPTIONS (choose exactly one of -c, -d, -m, -N, or -y):
@@ -32,5 +32,23 @@
 
    -n   Do not send email notification when filling is complete.
 
+   -B NumberDaysBack
+        After the -N pass, reprocess any date between NumberDaysBack days ago
+        and the start of the -N window that still holds no usage record.
+
+        A log file is sometimes published after its date has slid out of the
+        -N window, and nothing reaches for it again, so that date stays empty
+        however often the job runs. Each usage table is asked which dates it
+        is missing, and only those dates are reprocessed, so a late log is
+        picked up however late it arrives.
+
+        A date whose log holds no usable entry keeps being offered until it
+        ages past NumberDaysBack. That is deliberate: treating an already
+        processed log as proof of coverage would silently write off the dates
+        left empty by a run that died partway.
+
+        Needs -N, and is ignored with a warning otherwise.
+
  EXAMPLE:
    fillrdadb -N 7
+   fillrdadb -N 14 -B 60     # weekly fill, reaching back two months

{rda_python_metrics-3.0.8 → rda_python_metrics-3.0.10}/src/rda_python_metrics/pg_ipinfo.py

@@ -102,7 +102,7 @@ class PgIPInfo(PgUtil, PgDBI):
 
    def get_country_record_code(self, cname, kname = None):
       """Get country code from name."""
-      name = cname[kname] if kname else cname
+      name = cname.get(kname) if kname else cname
       name = name.replace(' ', '.').upper() if name else 'UNITED.STATES'
       if name == 'CHINA': name = 'P.R.CHINA'
       return name
@@ -241,16 +241,24 @@ class PgIPInfo(PgUtil, PgDBI):
          self.IPRECS[ip] = self.pgget('ipinfo', '*', f"ip = '{self.GIP}'", self.LGEREX)
          return self.IPRECS[ip]
       record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip, 'org_type' : '-'}
-      self.get_ip_hostname(ip, iprec, record)
-      record['lat'] = float(iprec['latitude']) if iprec['latitude'] else 0
-      record['lon'] = float(iprec['longitude']) if iprec['longitude'] else 0
-      if 'org' in iprec: record['org_name'] = iprec['org']
-      record['country'] = self.get_country_record_code(iprec, 'country_name')
-      record['region'] = self.convert_chars(iprec['region']) if 'region' in iprec else None
-      if 'city' in iprec: record['city'] = self.convert_chars(iprec['city'])
-      if 'postal' in iprec: record['postal'] =  iprec['postal']
-      record['timezone'] = iprec['timezone']
-      record['ipinfo'] = json.dumps(iprec)
+      try:
+         self.get_ip_hostname(ip, iprec, record)
+         # latitude and longitude are derived from 'loc', which a sparse ipinfo
+         # answer leaves out altogether, as it does timezone and country name
+         record['lat'] = float(iprec['latitude']) if iprec.get('latitude') else 0
+         record['lon'] = float(iprec['longitude']) if iprec.get('longitude') else 0
+         if 'org' in iprec: record['org_name'] = iprec['org']
+         record['country'] = self.get_country_record_code(iprec, 'country_name')
+         record['region'] = self.convert_chars(iprec['region']) if 'region' in iprec else None
+         if 'city' in iprec: record['city'] = self.convert_chars(iprec['city'])
+         if 'postal' in iprec: record['postal'] =  iprec['postal']
+         record['timezone'] = iprec.get('timezone')
+         record['ipinfo'] = json.dumps(iprec)
+      except (KeyError, TypeError, ValueError) as e:
+         # an ipinfo answer shaped differently again must not end the whole run;
+         # returning None lets set_ipinfo() fall back to the local geoip2 database
+         self.pglog("ipinfo: {} - unusable record ({}: {}), use geoip2".format(ip, type(e).__name__, e), self.LOGWRN)
+         return None
       return record
 
    def set_geoip2_database(self):

{rda_python_metrics-3.0.8 → rda_python_metrics-3.0.10/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 3.0.8
+Version: 3.0.10
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics