rda-python-common 2.0.0__py3-none-any.whl

rda_python_common/pg_util.py

@@ -0,0 +1,1534 @@
+#
+###############################################################################
+#
+#     Title : pg_util.py  -- module for misc utilities.
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 07/27/2020
+#             2025-01-10 transferred to package rda_python_common from
+#             https://github.com/NCAR/rda-shared-libraries.git
+#             2025-11-20 convert to class PgUtil
+#   Purpose : python library module for global misc utilities
+#
+#    Github : https://github.com/NCAR/rda-python-common.git
+#
+###############################################################################
+#
+import os
+import re
+import time
+import datetime
+import calendar
+import glob
+from os import path as op
+from .pg_log import PgLOG
+
+class PgUtil(PgLOG):
+
+   def __init__(self):
+      super().__init__()  # initialize parent class
+      self.DATEFMTS = {
+         'C' : '(CC|C)',                   # century
+         'Y' : '(YYYY|YY00|YYY|YY|YEAR|YR|Y)',  # YYY means decade
+         'Q' : '(QQ|Q)',                   # quarter
+         'M' : '(Month|Mon|MM|M)',         # numeric or string month
+         'W' : '(Week|Www|W)',             # string or numeric weedday
+         'D' : '(DDD|DD|D)',               # days in year or month
+         'H' : '(HHH|HH|H)',               # hours in month or day
+         'N' : '(NNNN|NN|N)',              # minutes in day or hour
+         'S' : '(SSSS|SS|S)'               # seconds in hour or minute
+      }
+      self.MONTHS = [
+         "january", "february", "march",     "april",   "may",      "june",
+         "july",    "august",   "september", "october", "november", "december"
+      ]
+      self.MNS = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"]
+      self.WDAYS = ["sunday", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday"]
+      self.WDS = ["sun", "mon", "tue", "wed", "thu", "fri", "sat"]
+      self.MDAYS = [365, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+
+   # dt: optional given date in format of "YYYY-MM-DD"
+   # return weekday: 0 - Sunday, 1 - Monday, ..., 6 - Saturday
+   def get_weekday(self, date = None):
+      if date is None:
+         ct = time.gmtime() if self.PGLOG['GMTZ'] else time.localtime()
+      else:
+         ct = time.strptime(str(date), "%Y-%m-%d")
+      return (ct[6]+1)%7
+
+   #  mn: given month string like "Jan" or "January", or numeric number 1 to 12
+   # Return: numeric Month if not fmt (default); three-charater or full month names for given fmt
+   def get_month(self, mn, fmt = None):
+      if not isinstance(mn, int):
+         if re.match(r'^\d+$', mn):
+            mn = int(mn)
+         else:
+            for m in range(12):
+               if re.match(mn, self.MONTHS[m], re.I):
+                  mn = m + 1
+                  break
+      if fmt and mn > 0 and mn < 13:
+         slen = len(fmt)
+         if slen == 2:
+            smn = "{:02}".format(mn)
+         elif re.match(r'^mon', fmt, re.I):
+            smn = self.MNS[mn-1] if slen == 3 else self.MONTHS[mn-1]
+            if re.match(r'^Mon', fmt):
+               smn = smn.capitalize()
+            elif re.match(r'^MON', fmt):
+               smn = smn.upper()
+         else:
+            smn = str(mn)
+         return smn
+      else:
+         return mn
+
+   # wday: given weekday string like "Sun" or "Sunday", or numeric number 0 to 6
+   # Return: numeric Weekday if !fmt (default); three-charater or full week name for given fmt
+   def get_wday(self, wday, fmt = None):
+      if not isinstance(wday, int):
+         if re.match(r'^\d+$', wday):
+            wday = int(wday)
+         else:
+            for w in range(7):
+               if re.match(wday, self.WDAYS[w], re.I):
+                  wday = w
+                  break
+      if fmt and wday >= 0 and wday <= 6:
+         slen = len(fmt)
+         if slen == 4:
+            swday = self.WDAYS[w]
+            if re.match(r'^We', fmt):
+               swday = swday.capitalize()
+            elif re.match(r'^WE', fmt):
+               swday = swday.upper()
+         elif slen == 3:
+            swday = self.WDS[wday]
+            if re.match(r'^Ww', fmt):
+               swday = swday.capitalize()
+            elif re.match(r'^WW', fmt):
+               swday = swday.upper()
+         else:
+            swday = str(wday)
+         return swday
+      else:
+         return wday
+
+   #   file: given file name
+   # Return: type if given file name is a valid online file; '' otherwise
+   @staticmethod
+   def valid_online_file(file, type = None, exists = None):
+      if exists is None or exists:
+         if not op.exists(file): return ''    # file does not exist
+      bname = op.basename(file)
+      if re.match(r'^,.*', bname): return ''       # hidden file
+      if re.search(r'index\.(htm|html|shtml)$', bname, re.I): return ''   # index file
+      if  type and type != 'D': return type
+      if re.search(r'\.(doc|php|html|shtml)(\.|$)', bname, re.I): return ''    # file with special extention
+      return 'D'
+
+   # Return: current time string in format of HH:MM:SS
+   def curtime(self, getdate = False):
+      ct = time.gmtime() if self.PGLOG['GMTZ'] else time.localtime()
+      fmt = "%Y-%m-%d %H:%M:%S" if getdate else "%H:%M:%S"
+      return time.strftime(fmt, ct)
+
+   # wrapper function of curtime(True) to get datetime in form of YYYY-MM-DD HH:NN:SS
+   def curdatetime(self):
+      return self.curtime(True)
+
+   #    fmt: optional date format, defaults to YYYY-MM-DD
+   # Return: current (date, hour)
+   def curdatehour(self, fmt = None):
+      ct = time.gmtime() if self.PGLOG['GMTZ'] else time.localtime()
+      dt =  self.fmtdate(ct[0], ct[1], ct[2], fmt) if fmt else time.strftime("%Y-%m-%d", ct)
+      return [dt, ct[3]]
+
+   #     tm: optional time in seconds since the Epoch
+   # Return: current date and time strings
+   def get_date_time(self, tm = None):
+      act = ct = None
+      if tm == None:
+         ct = time.gmtime() if self.PGLOG['GMTZ'] else time.localtime()
+      elif isinstance(tm, str):
+         act = tm.split(' ')
+      elif isinstance(tm, (int, float)):
+         ct = time.localtime(tm)
+      elif isinstance(tm, datetime.datetime):
+         act = str(tm).split(' ')
+      elif isinstance(tm, datetime.date):
+         act = [str(tm), '00:00:00']
+      elif isinstance(tm, datetime.time):
+         act = [None, str(tm)]
+      if ct == None:
+         return act if act else None
+      else:
+         return [time.strftime("%Y-%m-%d", ct), time.strftime("%H:%M:%S", ct)]
+
+   #     tm: optional time in seconds since the Epoch
+   # Return: current datetime strings
+   def get_datetime(self, tm = None):
+      if tm == None:
+         ct = time.gmtime() if self.PGLOG['GMTZ'] else time.localtime()
+         return time.strftime("%Y-%m-%d %H:%M:%S", ct)
+      elif isinstance(tm, str):
+         return tm
+      elif isinstance(tm, (int, float)):
+         ct = time.localtime(tm)
+         return time.strftime("%Y-%m-%d %H:%M:%S", ct)
+      elif isinstance(tm, datetime.datetime):
+         return str(tm)
+      elif isinstance(tm, datetime.date):
+         return (str(tm) + ' 00:00:00')
+      return tm
+
+   #   file: file name, get curent timestamp if missed
+   # Return: timestsmp string in format of 'YYYYMMDDHHMMSS
+   def timestamp(self, file = None):
+      if file is None:
+         ct = time.gmtime() if self.PGLOG['GMTZ'] else time.localtime()
+      else:
+         mt = os.stat(file).st_mtime    # file last modified time
+         ct = time.gmtime(mt) if self.PGLOG['GMTZ'] else time.localtime(mt)
+      return time.strftime("%Y%m%d%H%M%S", ct)
+
+   #  dt: datetime string
+   # check date/time and set to default one if empty date
+   @staticmethod
+   def check_datetime(date, default):
+      if not date: return default
+      if not isinstance(date, str): date = str(date)
+      if re.match(r'^0000', date): return default
+      return date
+
+   #    fmt: date format, default to "YYYY-MM-DD"
+   # Return: new formated current date string
+   def curdate(self, fmt = None):
+      ct = time.gmtime() if self.PGLOG['GMTZ'] else time.localtime()
+      return self.fmtdate(ct[0], ct[1], ct[2], fmt) if fmt else time.strftime("%Y-%m-%d", ct)
+
+   # check given string to identify temporal pattern and their units
+   # defined in (keys self.DATEFMTS)
+   def temporal_pattern_units(self, string, seps):
+      mkeys = ['D', 'Q', 'M', 'C', 'Y', 'H', 'N', 'S']
+      units = {}
+      match = seps[0] + "([^" + seps[1] + "]+)" + seps[1]
+      patterns = re.findall(match, string)
+      for pattern in patterns:
+         # skip generic pattern and current time
+         if re.match(r'^(P\d*|C.+C)$', pattern, re.I): continue
+         for mkey in mkeys:
+            ms = re.findall(self.DATEFMTS[mkey], pattern, re.I)
+            if ms:
+               if mkey == 'Q':
+                  units[mkey] = 3
+               elif mkey == 'C':
+                  units[mkey] = 100
+               else:
+                  units[mkey] = 1
+               for m in ms:
+                  pattern = pattern.replace(m, '', 1)
+      return units
+
+   # format output for given date and hour
+   def format_datehour(self, date, hour, tofmt = None, fromfmt = None):
+      if date:
+         datehour = self.format_date(str(date), tofmt, fromfmt)
+      elif tofmt:
+         datehour = tofmt
+      else:
+         datehour = ''
+      if hour != None:
+         if tofmt:
+            fmts = re.findall(self.DATEFMTS['H'], datehour, re.I)
+            for fmt in fmts:
+               if len(fmt) > 1:
+                  shr = "{:02}".format(int(hour))
+               else:
+                  shr = str(hour)
+               datehour = re.sub(fmt, shr, datehour, 1)
+         else:
+            datehour += " {:02}".format(int(hour))
+      return datehour
+
+   # split a date, time or datetime into an array according to
+   # the sep value; str to int for digital values
+   @staticmethod
+   def split_datetime(sdt, sep = r'\D'):
+      if not isinstance(sdt, str): sdt = str(sdt)
+      adt = re.split(sep, sdt)
+      acnt = len(adt)
+      for i in range(acnt):
+         if re.match(r'^\d+$', adt[i]): adt[i] = int(adt[i])
+      return adt
+
+   #    date: given date in format of fromfmt
+   #   tofmt: date formats; ex. "Month D, YYYY"
+   # fromfmt: date formats, default to YYYY-MM-DD
+   #  Return: new formated date string according to tofmt
+   def format_date(self, cdate, tofmt = None, fromfmt = None):
+      if not cdate: return cdate
+      if not isinstance(cdate, str): cdate = str(cdate)
+      dates = [None, None, None]
+      sep = '|'
+      mns = sep.join(self.MNS)
+      months = sep.join(self.MONTHS)
+      mkeys = ['D', 'M', 'Q', 'Y', 'C', 'H']
+      PATTERNS = [r'(\d\d\d\d)', r'(\d+)', r'(\d\d)',
+                  r'(\d\d\d)', '(' + mns + ')', '(' + months + ')']
+      if not fromfmt:
+         if not tofmt:
+            if re.match(r'^\d\d\d\d-\d\d-\d\d$', cdate): return cdate   # no need formatting
+         ms = re.match(r'^\d+(\W)\d+(\W)\d+', cdate)
+         if ms:
+            fromfmt = "Y" + ms.group(1) + "M" + ms.group(2) + "D"
+         else:
+            self.pglog(cdate + ": Invalid date, should be in format YYYY-MM-DD", self.LGEREX)
+      pattern = fromfmt
+      fmts = {}
+      formats = {}
+      for mkey in mkeys:
+         ms = re.search(self.DATEFMTS[mkey], pattern, re.I)
+         if ms:
+            fmts[mkey] = ms.group(1)
+            pattern = re.sub(fmts[mkey], '', pattern)
+      cnt = 0
+      for mkey in fmts:
+         fmt = fmts[mkey]
+         i = len(fmt)
+         if mkey == 'D':
+            if i == 4: i = 1
+         elif mkey == 'M':
+            if i == 3: i = 4
+         elif mkey == 'Y':
+            if i == 4: i = 0
+         formats[fromfmt.find(fmt)] = fmt
+         fromfmt = fromfmt.replace(fmt, PATTERNS[i])
+         cnt += 1   
+      ms = re.findall(fromfmt, cdate)
+      mcnt = len(ms[0]) if ms else 0
+      i = 0
+      for k in sorted(formats):
+         if i >= mcnt: break
+         fmt = formats[k]
+         val = ms[0][i]
+         if re.match(r'^Y', fmt, re.I):
+            dates[0] = int(val)
+            if len(fmt) == 3: dates[0] *= 10
+         elif re.match(r'^C', fmt, re.I):
+            dates[0] = 100 * int(val)      # year at end of century
+         elif re.match(r'^M', fmt, re.I):
+            if re.match(r'^Mon', fmt, re.I):
+               dates[1] = self.get_month(val)
+            else:
+               dates[1] = int(val)
+         elif re.match(r'^Q', fmt, re.I):
+            dates[1] = 3 * int(val)        # month at end of quarter
+         elif re.match(r'^H', fmt, re.I):  # hour
+            dates.append(int(val))
+         else:    # day
+            dates[2] = int(val)
+         i += 1 
+      if len(dates) > 3:
+         cdate = self.fmtdatehour(dates[0], dates[1], dates[2], dates[3], tofmt)
+      else:
+         cdate = self.fmtdate(dates[0], dates[1], dates[2], tofmt)
+      return cdate
+
+   #     yr: year value
+   #     mn: month value, 1-12
+   #     dy: day of the month
+   #     hr: hour of the day
+   #     nn: minute of the hour
+   #     ss: second of the minute
+   #  tofmt: date format, ex. "Month D, YYYY", default to "YYYY-MM-DD HH:NN:SS"
+   # Return: new formated datehour string
+   def fmtdatetime(self, yr, mn, dy, hr = None, nn = None, ss = None, tofmt = None):
+      if not tofmt: tofmt = "YYYY-MM-DD HH:NN:SS"
+      tms = [ss, nn, hr, dy]
+      fks = ['S', 'N', 'H']
+      ups = [60, 60, 24]
+      # adjust second/minute/hour values out of range
+      for i in range(3):
+         if tms[i] != None and tms[i+1] != None:
+            if tms[i] < 0:
+               while tms[i] < 0:
+                  tms[i] += ups[i]
+                  tms[i+1] -= 1
+            elif tms[i] >= ups[i]:
+               while tms[i] >= ups[i]:
+                  tms[i] -= ups[i]
+                  tms[i+1] += 1
+      sdt = self.fmtdate(yr, mn, dy, tofmt)
+      # format second/minute/hour values
+      for i in range(3):
+         if tms[i] != None:
+            ms = re.search(self.DATEFMTS[fks[i]], sdt, re.I)
+            if ms:
+               fmt = ms.group(1)
+               if len(fmt) == 2:
+                  str = "{:02}".format(tms[i])
+               else:
+                  str = str(tms[i])
+            sdt = re.sub(fmt, str, sdt, 1)   
+      return sdt
+
+   #     yr: year value
+   #     mn: month value, 1-12
+   #     dy: day of the month
+   #     hr: hour of the day
+   #  tofmt: date format, ex. "Month D, YYYY", default to "YYYY-MM-DD:HH"
+   # Return: new formated datehour string
+   def fmtdatehour(self, yr, mn, dy, hr, tofmt = None):
+      if not tofmt: tofmt = "YYYY-MM-DD:HH"
+      if hr != None and dy != None:   # adjust hour value out of range
+         if hr < 0:
+            while hr < 0:
+               hr += 24
+               dy -= 1
+         elif hr > 23:
+            while hr > 23:
+               hr -= 24
+               dy += 1
+      datehour = self.fmtdate(yr, mn, dy, tofmt)
+      if hr != None:
+         ms = re.search(self.DATEFMTS['H'], datehour, re.I)
+         if ms:
+            fmt = ms.group(1)
+            if len(fmt) == 2:
+               shr = "{:02}".format(hr)
+            else:
+               shr = str(hr)
+            datehour = re.sub(fmt, shr, datehour, 1)
+      return datehour
+
+   #     yr: year value
+   #     mn: month value, 1-12
+   #     dy: day of the month
+   #  tofmt: date format, ex. "Month D, YYYY", default to "YYYY-MM-DD"
+   # Return: new formated date string
+   def fmtdate(self, yr, mn, dy, tofmt = None):
+      (y, m, d) = self.adjust_ymd(yr, mn, dy)
+      if not tofmt or tofmt == 'YYYY-MM-DD': return "{}-{:02}-{:02}".format(y, m, d)
+      if dy != None:
+         md = re.search(self.DATEFMTS['D'], tofmt, re.I)
+         if md:
+            fmt = md.group(1)   # day
+            slen = len(fmt)
+            if slen > 2:    # days of the year
+               for i in range(1, m): d += self.MDAYS[i]
+               sdy = "{:03}".format(d)
+            elif slen == 2:
+               sdy = "{:02}".format(d)
+            else:
+               sdy = str(d)
+            tofmt = re.sub(fmt, sdy, tofmt, 1)
+      if mn != None:
+         md = re.search(self.DATEFMTS['M'], tofmt, re.I)
+         if md:
+            fmt = md.group(1)   # month
+            slen = len(fmt)
+            if slen == 2:
+               smn = "{:02}".format(m)
+            elif re.match(r'^mon', fmt, re.I):
+               smn = self.MNS[m-1] if slen == 3 else self.MONTHS[m-1]
+               if re.match(r'^Mo', fmt):
+                  smn = smn.capitalize()
+               elif re.match(r'^MO', fmt):
+                  smn = smn.upper()
+            else:
+               smn = str(m)
+            tofmt = re.sub(fmt, smn, tofmt, 1)
+         else:
+            md = re.search(self.DATEFMTS['Q'], tofmt, re.I)
+            if md:
+               fmt = md.group(1)   # quarter
+               m = int((m+2)/3)
+               smn = "{:02}".format(m) if len(fmt) == 2 else str(m)
+               tofmt = re.sub(fmt, smn, tofmt, 1)
+      if yr != None:
+         md = re.search(self.DATEFMTS['Y'], tofmt, re.I)
+         if md:
+            fmt = md.group(1)   # year
+            slen = len(fmt)
+            if slen == 2:
+               syr = "{:02}".format(y%100)
+            elif slen == 3:      # decade
+               if y > 999: y = int(y/10)
+               syr = "{:03}".format(y)
+            else:
+               if re.search(r'^YY00', fmt, re.I):  y = 100*int(y/100)    # hundred years
+               syr = "{:04}".format(y)
+            tofmt = re.sub(fmt, syr, tofmt, 1)
+         else:
+            md = re.search(self.DATEFMTS['C'], tofmt, re.I)
+            if md:
+               fmt = md.group(1)   # century
+               slen = len(fmt)
+               if y > 999:
+                  y = 1 + int(y/100)
+               elif y > 99:
+                  y = 1 + int(yr/10)
+               syr = "{:02}".format(y)
+               tofmt = re.sub(fmt, syr, tofmt, 1)
+      return tofmt
+
+   # format given date and time into standard timestamp
+   @staticmethod
+   def join_datetime(sdate, stime):
+      if not sdate: return None
+      if not stime: stime = "00:00:00"
+      if not isinstance(sdate, str): sdate = str(sdate)
+      if not isinstance(stime, str): stime = str(stime)
+      if re.match(r'^\d:', stime): stime = '0' + stime
+      return "{} {}".format(sdate, stime)
+   fmttime = join_datetime
+
+   # split a date or datetime into an array of [date, time]
+   @staticmethod
+   def date_and_time(sdt):
+      if not sdt: return [None, None]
+      if not isinstance(sdt, str): sdt = str(sdt)
+      adt = re.split(' ', sdt)
+      acnt = len(adt)
+      if acnt == 1: adt.append('00:00:00')
+      return adt
+
+   # convert given date/time to unix epoch time; -1 if cannot
+   @staticmethod
+   def unixtime(stime):
+      pt = [0]*9
+      if not isinstance(stime, str): stime  = str(stime)
+      ms = re.match(r'^(\d+)-(\d+)-(\d+)', stime)
+      if ms:
+         for i in range(3):
+            pt[i] = int(ms.group(i+1))
+      ms = re.search(r'^(\d+):(\d+):(\d+)$', stime)
+      if ms:
+         for i in range(3):
+            pt[i+3] = int(ms.group(i+1))
+      pt[8] = -1
+      return time.mktime(time.struct_time(pt))
+
+   #  sdate: start date in form of 'YYYY' or 'YYYY-MM' or 'YYYY-MM-DD'
+   #  edate: end date in form of 'YYYY' or 'YYYY-MM' or 'YYYY-MM-DD'
+   # Return: list of start and end dates in format of YYYY-MM-DD
+   def daterange(self, sdate, edate):
+      if sdate:
+         if not isinstance(sdate, str): sdate = str(sdate)
+         if not re.search(r'\d+-\d+-\d+', sdate):
+            ms = re.match(r'^(\W*)(\d+)-(\d+)(\W*)$', sdate)
+            if ms:
+               sdate = "{}{}-{}-01{}".format(ms.group(1), ms.group(2), ms.group(3), ms.group(4))
+            else:
+               ms = re.match(r'^(\W*)(\d+)(\W*)$', sdate)
+               if ms:
+                  sdate = "{}{}-01-01{}".format(ms.group(1), ms.group(2), ms.group(3))
+      if edate:
+         if not isinstance(edate, str): edate = str(edate)
+         if not re.search(r'\d+-\d+-\d+', edate):
+            ms = re.match(r'^(\W*)(\d+)-(\d+)(\W*)$', edate)
+            if ms:
+               edate = "{}{}-{}-01{}".format(ms.group(1), ms.group(2), ms.group(3), ms.group(4))
+               edate = self.adddate(edate, 0, 1, -1)
+            else:
+               ms = re.match(r'^(\W*)(\d+)(\W*)$', edate)
+               if ms:
+                  edate = "{}{}-12-31{}".format(ms.group(1), ms.group(2), ms.group(3))
+      return [sdate, edate]
+
+   # date to datetime range
+   @staticmethod
+   def dtrange(dates):
+      date = dates[0]
+      if date:
+         if not isinstance(date, str): date = str(date)
+         dates[0] = date + ' 00:00:00'
+      date = dates[1]
+      if date:
+         if not isinstance(date, str): date = str(date)
+         dates[1] = date + ' 23:59:59'
+      return dates
+
+   #  sdate: starting date in format of 'YYYY-MM-DD'
+   #  edate: ending date
+   #    fmt: period format, ex. "YYYYMon-YYYMon", default to "YYYYMM-YYYYMM"
+   # Return: a string of formated period
+   def format_period(self, sdate, edate, fmt = None):
+      period = ''
+      if not fmt:
+         sfmt = efmt = "YYYYMM"
+         sep = '-'
+      else:
+         ms = re.match(r'^(.*)(\s*-\s*)(.*)$', fmt)
+         if ms:
+            (sfmt, sep, efmt) = ms.groups()
+         else:
+            sfmt = fmt
+            efmt = None
+            sep  = ''
+      if sdate:
+         if not isinstance(sdate, str): sdate = str(sdate)
+         ms = re.search(r'(\d+)-(\d+)-(\d+)', sdate)
+         if ms:
+            (yr, mn, dy) = ms.groups()
+            period = self.fmtdate(int(yr), int(mn), int(dy), sfmt)
+      if sep: period += sep   
+      if efmt:
+         if re.search(r'current', efmt, re.I):
+            period += efmt
+         elif edate:
+            if not isinstance(edate, str): edate = str(edate)
+            ms = re.search(r'(\d+)-(\d+)-(\d+)', edate)
+            if ms:
+               (yr, mn, dy) = ms.groups()
+               period += self.fmtdate(int(yr), int(mn), int(dy), efmt)
+      return period
+
+   #  dsid: given dataset id in form of dsNNN(.|)N, NNNN.N or [a-z]NNNNNN
+   # newid: True to format a new dsid; defaults to False for now
+   # returns a new or old dsid according to the newid option
+   def format_dataset_id(self, dsid, newid = None, logact = None):
+      if newid is None: newid = self.PGLOG['NEWDSID']
+      if logact is None: logact = self.LGEREX
+      dsid = str(dsid)
+      ms = re.match(r'^([a-z])(\d\d\d)(\d\d\d)$', dsid)
+      if ms:
+         ids = list(ms.groups())
+         if ids[0] not in self.PGLOG['DSIDCHRS']:
+            if logact: self.pglog("{}: dsid leading character must be '{}'".format(dsid, self.PGLOG['DSIDCHRS']), logact)
+            return dsid
+         if newid: return dsid
+         if ids[2][:2] != '00':
+            if logact: self.pglog(dsid + ": Cannot convert new dsid to old format", logact)
+            return dsid
+         return 'ds{}.{}'.format(ids[1], ids[2][2])
+      ms = re.match(r'^ds(\d\d\d)(\.|)(\d)$', dsid, re.I)
+      if not ms: ms = re.match(r'^(\d\d\d)(\.)(\d)$', dsid)
+      if ms:
+         if newid:
+            return "d{}00{}".format(ms.group(1), ms.group(3))
+         else:
+            return 'ds{}.{}'.format(ms.group(1), ms.group(3))
+      if logact: self.pglog(dsid + ": invalid dataset id", logact)
+      return dsid
+
+   #  dsid: given dataset id in form of dsNNN(.|)N, NNNN.N or [a-z]NNNNNN
+   # newid: True to format a new dsid; defaults to False for now
+   # returns a new or old metadata dsid according to the newid option
+   def metadata_dataset_id(self, dsid, newid = None, logact = None):
+      if newid is None: newid = self.PGLOG['NEWDSID']
+      if logact is None: logact = self.LGEREX
+      ms = re.match(r'^([a-z])(\d\d\d)(\d\d\d)$', dsid)
+      if ms:
+         ids = list(ms.groups())
+         if ids[0] not in self.PGLOG['DSIDCHRS']:
+            if logact: self.pglog("{}: dsid leading character must be '{}'".format(dsid, self.PGLOG['DSIDCHRS']), logact)
+            return dsid
+         if newid: return dsid
+         if ids[2][:2] != '00':
+            if logact: self.pglog(dsid + ": Cannot convert new dsid to old format", logact)
+            return dsid
+         return '{}.{}'.format(ids[1], ids[2][2])
+      ms = re.match(r'^ds(\d\d\d)(\.|)(\d)$', dsid)
+      if not ms: ms = re.match(r'^(\d\d\d)(\.)(\d)$', dsid)
+      if ms:
+         if newid:
+            return "d{}00{}".format(ms.group(1), ms.group(3))
+         else:
+            return '{}.{}'.format(ms.group(1), ms.group(3))
+      if logact: self.pglog(dsid + ": invalid dataset id", logact)
+      return dsid
+
+   # idstr: string holding a dsid in form of dsNNN(.|)N, NNNN.N or [a-z]NNNNNN
+   # and find it according to the flag value O (Old), N (New) or B (Both) formats
+   # returns dsid if found in given id string; None otherwise
+   def find_dataset_id(self, idstr, flag = 'B', logact = 0):
+      if flag in 'NB':
+         ms = re.search(r'(^|\W)(([a-z])\d{6})($|\D)', idstr)
+         if ms and ms.group(3) in self.PGLOG['DSIDCHRS']: return ms.group(2)
+      if flag in 'OB':
+         ms = re.search(r'(^|\W)(ds\d\d\d(\.|)\d)($|\D)', idstr)
+         if not ms: ms = re.search(r'(^|\W)(\d\d\d\.\d)($|\D)', idstr)
+         if ms: return ms.group(2)
+      if logact: self.pglog("{} : No valid dsid found for flag {}".format(idstr, flag), logact)
+      return None
+
+   # find and convert all found dsids according to old/new dsids
+   # for newid = False/True
+   def convert_dataset_ids(self, idstr, newid = None, logact = 0):
+      if newid is None: newid = self.PGLOG['NEWDSID']
+      flag = 'O' if newid else 'N'
+      cnt = 0
+      if idstr:
+         while True:
+            dsid = self.find_dataset_id(idstr, flag = flag)
+            if not dsid: break
+            ndsid = self.format_dataset_id(dsid, newid = newid, logact = logact)
+            if ndsid != dsid: idstr = idstr.replace(dsid, ndsid)
+            cnt += 1
+      return (idstr, cnt)
+
+   # records: dict of mutiple records,
+   #     idx: index of the records to return
+   #  Return: a dict to the idx record out of records
+   @staticmethod
+   def onerecord(records, idx):
+      record = {}
+      for fld in records:
+         record[fld] = records[fld][idx]
+      return record
+
+   # records: dict of mutiple records,
+   #  record: record to add
+   #     idx: index of the record to add
+   #  Return: add a record to a dict of lists
+   @staticmethod
+   def addrecord(records, record, idx):
+      if records is None: records = {}   # initialize dist of lists structure
+      if not records:
+         for key in record:
+            records[key] = []
+      for key in record:
+         slen = len(records[key])
+         if idx < slen:
+            records[key][idx] = record[key]
+         else:
+            while idx > slen:
+               records[key].append(None)
+               slen += 1
+            records[key].append(record[key])
+      return records
+
+   # convert a hash with multiple rows from pgmget() to an array of hashes
+   @staticmethod
+   def hash2array(hrecs, hkeys = None):
+      if not hkeys: hkeys = list(hrecs)
+      acnt = len(hrecs[hkeys[0]]) if hrecs and hkeys[0] in hrecs else 0
+      arecs = [None]*acnt
+      for i in range(acnt):
+         arec = {}
+         for hkey in hkeys: arec[hkey] = hrecs[hkey][i]
+         arecs[i] = arec
+      return arecs
+
+   # convert an array of hashes to a hash with multiple rows for pgmget()
+   @staticmethod
+   def array2hash(arecs, hkeys = None):
+      hrecs = {}
+      acnt = len(arecs) if arecs else 0
+      if acnt > 0:
+         if not hkeys: hkeys = list(arecs[0])
+         for hkey in hkeys:
+            hrecs[hkey] = [None]*acnt
+            for i in range(acnt): hrecs[hkey][i] = arecs[i][hkey]
+      return hrecs
+
+   # records: dict of mutiple records,
+   #     opt: 0 - column count,
+   #          1 - row count,
+   #          2 - both
+   #  Return: a single number or list of two dependend on given opt
+   @staticmethod
+   def hashcount(records, opt = 0):
+      ret = [0, 0]
+      if records:
+         clen = len(records)
+         if opt == 0 or opt == 2:
+            ret[0] = clen
+         if opt == 1 or opt == 2:
+            ret[1] = len(next(iter(records.values())))
+      return ret if opt == 2 else ret[opt]
+
+   #   adict: dict a
+   #   bdict: dict b
+   # default: default values if missed
+   #  unique: unique join if set
+   #  Return: the joined dict records with default value for missing ones
+   #          For unique join, a record in bdict must not be contained in adict already
+   @staticmethod
+   def joinhash(adict, bdict, default = None, unique = None):
+      if not bdict: return adict
+      if not adict: return bdict
+      akeys = list(adict.keys())
+      bkeys = list(bdict.keys())
+      acnt = len(adict[akeys[0]])
+      bcnt = len(bdict[bkeys[0]])
+      ckeys = []    # common keys for unique joins
+      # check and assign default value for missing keys in adict
+      for bkey in bkeys:
+         if bkey in akeys:
+            if unique and bkey not in ckeys: ckeys.append(bkey)
+         else:
+            adict[bkey] = [default]*acnt
+      # check and assign default value for missing keys in bdict
+      for akey in akeys:
+         if akey in bkeys:
+            if unique and akey not in ckeys: ckeys.append(akey)
+         else:
+            bdict[akey] = [default]*bcnt
+      if unique:    # append bdict
+         kcnt = len(ckeys)
+         for i in range(bcnt):
+            j = 0
+            while(j < acnt):
+               k = 0
+               for ckey in ckeys:
+                  if PgUtil.pgcmp(adict[ckey][j], bdict[ckey][i]): break
+                  k += 1
+               if k >= kcnt: break
+               j += 1
+   
+            if j >= acnt:
+               for key in adict:
+                  adict[key].append(bdict[key][i])
+      else:
+         for key in adict:
+            adict[key].extend(bdict[key])
+      return adict
+
+   #   lst1: list 1
+   #   lst2: list 2
+   # unique: unique join if set
+   # Return: the joined list
+   @staticmethod
+   def joinarray(lst1, lst2, unique = None):
+      if not lst2: return lst1
+      if not lst1: return lst2
+      cnt1 = len(lst1)
+      cnt2 = len(lst2)
+      if unique:
+         for i in (cnt2):
+            for j in (cnt1):
+               if PgUtil.pgcmp(lst1[j], lst2[i]) != 0: break
+            if j >= cnt1:
+              lst1.append(lst2[i])
+      else:
+         lst1.extend(lst2)
+      return lst1
+
+   # Function: crosshash(ahash, bhash)
+   #   Return: a reference to the cross-joined hash records
+   @staticmethod
+   def crosshash(ahash, bhash):
+      if not bhash: return ahash
+      if not ahash: return bhash
+      akeys = list(ahash.keys())
+      bkeys = list(bhash.keys())
+      acnt = len(ahash[akeys[0]])
+      bcnt = len(bhash[bkeys[0]])
+      rets = {}
+      for key in akeys: rets[key] = []
+      for key in bkeys: rets[key] = []
+      for i in range(acnt):
+         for j in range(bcnt):
+            for key in akeys: rets[key].append(ahash[key][i])
+            for key in bkeys: rets[key].append(bhash[key][j])
+      return rets
+
+   # strip database and table names for a field name
+   @staticmethod
+   def strip_field(field):
+      ms = re.search(r'\.([^\.]+)$', field)
+      if ms: field = ms.group(1)
+      return field
+
+   #   pgrecs: dict obterned from pgmget()
+   #     flds: list of single letter fields to be sorted on
+   #     hash: table dict for pre-defined fields
+   # patterns: optional list of temporal patterns for order fields
+   #   Return: a sorted dict list
+   def sorthash(self, pgrecs, flds, hash, patterns = None):
+      fcnt = len(flds)    # count of fields to be sorted on
+      # set sorting order, descenting (-1) or ascenting (1)
+      # get the full field names to be sorted on
+      desc = [1]*fcnt
+      fields = []
+      nums = [1]*fcnt   # initialize each column as numerical
+      for i in range(fcnt):
+         if flds[i].islower(): desc[i] = -1
+         fld = self.strip_field(hash[flds[i].upper()][1])
+         fields.append(fld)
+      count = len(pgrecs[fields[0]])    # row count of pgrecs
+      if count < 2: return pgrecs       # no need of sording
+      pcnt = len(patterns) if patterns else 0
+      # prepare the dict list for sortting
+      srecs = []
+      for i in range(count):
+         pgrec = self.onerecord(pgrecs, i)
+         rec = []
+         for j in range(fcnt):
+            if j < pcnt and patterns[j]:
+               # get the temporal part of each value matching the pattern
+               val = self.format_date(pgrec[fields[j]], "YYYYMMDDHH", patterns[j])
+            else:
+               # sort on the whole value if no pattern given
+               val = pgrec[fields[j]]
+            if nums[j]: nums[j] = self.pgnum(val)
+            rec.append(val)
+         rec.append(i)   # add column to cache the row index
+         srecs.append(rec)
+      srecs = self.quicksort(srecs, 0, count-1, desc, fcnt, nums)
+      # sort pgrecs according the cached row index column in ordered srecs
+      rets = {}
+      for fld in pgrecs:
+         rets[fld] = []
+      for i in range(count):
+         pgrec = self.onerecord(pgrecs, srecs[i][fcnt])
+         for fld in pgrecs:
+            rets[fld].append(pgrec[fld])
+      return rets
+
+   # Return: the number of days bewteen date1 and date2
+   @staticmethod
+   def diffdate(date1, date2):
+      ut1 = ut2 = 0
+      if date1: ut1 = PgUtil.unixtime(date1)
+      if date2: ut2 = PgUtil.unixtime(date2)
+      return round((ut1 - ut2)/86400)   # 24*60*60
+
+   # Return: the number of seconds bewteen time1 and time2
+   @staticmethod
+   def difftime(time1, time2):
+      ut1 = ut2 = 0
+      if time1: ut1 = PgUtil.unixtime(time1)
+      if time2: ut2 = PgUtil.unixtime(time2)
+      return round(ut1 - ut2)
+   diffdatetime = difftime
+
+   # Return: the number of days between date and '1970-01-01 00:00:00'
+   @staticmethod
+   def get_days(cdate):
+      return PgUtil.diffdate(str(cdate), '1970-01-01')
+
+   # Function: get_month_days(date)
+   #   Return: the number of days in given month
+   @staticmethod
+   def get_month_days(cdate):
+      ms = re.match(r'^(\d+)-(\d+)', str(cdate))
+      if ms:
+         yr = int(ms.group(1))
+         mn = int(ms.group(2))
+         return calendar.monthrange(yr, mn)[1]
+      else:
+         return 0
+
+   # Function: validate_date(date)
+   #   Return: a date in format of YYYY-MM-DD thar all year/month/day are validated
+   @staticmethod
+   def validate_date(cdate):
+      ms = re.match(r'^(\d+)-(\d+)-(\d+)', str(cdate))
+      if ms:
+         (yr, mn, dy) = (int(m) for m in ms.groups())
+         if yr < 1000:
+            yr += 2000
+         elif yr > 9999:
+            yr %= 10000
+         if mn < 1:
+            mn = 1
+         elif mn > 12:
+            mn = 12
+         md = calendar.monthrange(yr, mn)[1]
+         if dy < 1:
+            dy = 1
+         elif dy > md:
+            dy = md
+         cdate = '{}-{:02d}-{:02d}'.format(yr, mn, dy)   
+      return cdate
+
+   # Function: get_date(days)
+   #   Return: the date in format of "YYYY-MM-DD" for given number of days
+   #   from '1970-01-01 00:00:00'
+   def get_date(self, days):
+      return self.adddate('1970-01-01', 0, 0, int(days))
+
+   # compare date/hour and return the different hours
+   @staticmethod
+   def diffdatehour(date1, hour1, date2, hour2):
+      if hour1 is None: hour1 = 23
+      if hour2 is None: hour2 = 23
+      return (hour1 - hour2) + 24*PgUtil.diffdate(date1, date2)
+
+   # hour difference between GMT and local time
+   def diffgmthour(self):
+      tg = time.gmtime()
+      tl = time.localtime()
+      dg = self.fmtdate(tg[0], tg[1], tg[2])
+      dl = self.fmtdate(tl[0], tl[1], tl[2])
+      hg = tg[3]
+      hl = tl[3]
+      return self.diffdatehour(dg, hg, dl, hl)
+
+   # compare date and time (if given) and return 1, 0 and -1
+   @staticmethod
+   def cmptime(date1, time1, date2, time2):
+      stime1 = PgUtil.join_datetime(date1, time1)
+      stime2 = PgUtil.join_datetime(date2, time2)
+      return PgUtil.pgcmp(stime1, stime2)
+
+   #   date: the original date in format of 'YYYY-MM-DD',
+   #     mf: the number of month fractions to add
+   #     nf: number of fractions of a month
+   # Return: new date
+   def addmonth(self, cdate, mf, nf = 1):
+      if not mf: return cdate
+      if not nf or nf < 2: return self.adddate(cdate, 0, mf, 0)
+      ms = re.match(r'^(\d+)-(\d+)-(\d+)$', cdate)
+      if ms:
+         (syr, smn, sdy) = ms.groups()
+         yr = int(syr)
+         mn = int(smn)
+         ody = int(sdy)
+         dy = 0            # set to end of previous month
+         ndy = int(30/nf)  # number of days in each fraction
+         while ody > ndy:
+            dy += ndy
+            ody -= ndy
+         dy += mf * ndy
+         if mf > 0:
+            while dy >= 30:
+               dy -= 30
+               mn += 1
+         else:
+            while dy < 0:
+               dy += 30
+               mn -= 1
+         dy += ody
+         cdate = self.fmtdate(yr, mn, dy)
+      return cdate
+   
+   # add yr years & mn months to yearmonth ym in format YYYYMM
+   @staticmethod
+   def addyearmonth(ym, yr, mn):
+      if yr == None: yr = 0
+      if mn == None: mn = 0   
+      ms =re.match(r'^(\d\d\d\d)(\d\d)$', ym)
+      if ms:
+         (syr, smn) = ms.groups()
+         yr = int(syr)
+         mn = int(smn)
+         if mn < 0:
+            while mn < 0:
+               yr -= 1
+               mn += 12
+         else:
+            while mn > 12:
+               yr += 1
+               mn -= 12
+         ym = "{:04}{:02}".format(yr, mn)
+      return ym
+
+   # set number of days in Beburary for Leap year according PGLOG['NOLEAP']
+   def set_leap_mdays(self, year):
+      if not self.PGLOG['NOLEAP'] and calendar.isleap(year):
+         self.MDAYS[0] = 366
+         self.MDAYS[2] = 29
+         ret = 1
+      else:
+         self.MDAYS[0] = 365
+         self.MDAYS[2] = 28
+         ret = 0
+      return ret
+
+   # wrap on calendar.isleap()
+   is_leapyear = calendar.isleap
+
+   # reutn 1 if is end of month
+   def is_end_month(self, yr, mn, dy):
+      self.set_leap_mdays(yr)
+      return 1 if dy == self.MDAYS[mn] else 0
+
+   # adust the year, month and day values that are out of ranges
+   def adjust_ymd(self, yr, mn, dy):
+      if yr is None: yr = 1970
+      if mn is None: mn = 1
+      if dy is None: dy = 1
+      while True:
+         if mn > 12:
+            yr += 1
+            mn -= 12
+            continue
+         elif mn < 1:
+            yr -= 1
+            mn += 12
+            continue
+         self.set_leap_mdays(yr)
+         if dy < 1:
+            if(dy < -self.MDAYS[0]):
+               yr -= 1
+               dy += self.MDAYS[0]
+            else:
+               mn -= 1
+               if mn < 1:
+                 yr -= 1
+                 mn += 12
+               dy += self.MDAYS[mn]
+            continue
+         elif dy > self.MDAYS[mn]:
+            if(dy > self.MDAYS[0]):
+               dy -= self.MDAYS[0]
+               yr += 1
+            else:
+               dy -= self.MDAYS[mn]
+               mn += 1
+            continue
+         break
+      return [yr, mn, dy]
+
+   #   date: the original date in format of 'YYYY-MM-DD',
+   #     yr: the number of years to add/subtract from the odate for positive/negative value,
+   #     mn: the number of months to add/subtract from the odate for positive/negative value,
+   #     dy: the number of days to add/subtract from the odate for positive/negative value)
+   # Return: new date
+   def adddate(self, cdate, yr, mn = 0, dy = 0, tofmt = None):
+      if not cdate: return cdate
+      if not isinstance(cdate, str): cdate = str(cdate)
+      if yr is None:
+         yr = 0
+      elif isinstance(yr, str):
+         yr = int(yr)
+      if mn is None:
+         mn = 0
+      elif isinstance(mn, str):
+         mn = int(mn)
+      if dy is None:
+         dy = 0
+      elif isinstance(dy, str):
+         dy = int(dy)
+      ms = re.search(r'(\d+)-(\d+)-(\d+)', cdate)
+      if not ms: return cdate    # non-standard date format
+      (nyr, nmn, ndy) = (int(m) for m in ms.groups())
+      mend = 0
+      if mn and ndy > 27: mend = self.is_end_month(nyr, nmn, ndy)
+      if yr: nyr += yr
+      if mn:
+         (nyr, nmn, tdy) = self.adjust_ymd(nyr, nmn+mn+1, 0)
+         if mend: ndy = tdy
+      if dy: ndy += dy
+      return self.fmtdate(nyr, nmn, ndy, tofmt)
+   addNoLeapDate = adddate
+
+   # add given hours to the initial date and time
+   def addhour(self, sdate, stime, nhour):
+      if nhour and isinstance(nhour, str): nhour = int(nhour)
+      if sdate and not isinstance(sdate, str): sdate = str(sdate)
+      if stime and not isinstance(stime, str): stime = str(stime)
+      if not nhour: return [sdate, stime]
+      hr = dy = 0
+      ms = re.match(r'^(\d+)', stime)
+      if ms:
+         shr = ms.group(1)
+         hr = int(shr) + nhour
+         if hr < 0:
+            while hr < 0:
+               dy -= 1
+               hr += 24
+         else:
+            while hr > 23:
+               dy += 1
+               hr -= 24
+      shour = "{:02}".format(hr)
+      if shr != shour: stime = re.sub(shr, shour, stime, 1)
+      if dy: sdate = self.adddate(sdate, 0, 0, dy)
+      return [sdate, stime]
+
+   # add given years, months, days and hours to the initial date and hour
+   def adddatehour(self, sdate, nhour, yr, mn, dy, hr = 0):
+      if sdate and not isinstance(sdate, str): sdate = str(sdate)
+      if hr:
+         if nhour != None:
+            if isinstance(nhour, str): nhour = int(nhour)
+            hr += nhour
+         if hr < 0:
+            while hr < 0:
+               dy -= 1
+               hr += 24
+         else:
+            while hr > 23:
+               dy += 1
+               hr -= 24
+         if nhour != None: nhour = hr
+      if yr or mn or dy: sdate = self.adddate(sdate, yr, mn, dy)
+      return [sdate, nhour]
+
+   # add given yyyy, mm, dd, hh, nn, ss to sdatetime
+   # if nf, add fraction of month only
+   def adddatetime(self, sdatetime, yy, mm, dd, hh, nn, ss, nf = 0):
+      if sdatetime and not isinstance(sdatetime, str): sdatetime = str(sdatetime)
+      (sdate, stime) = re.split(' ', sdatetime)
+      if hh or nn or ss: (sdate, stime) = self.addtime(sdate, stime, hh, nn, ss)
+      if nf:
+         sdate = self.addmonth(sdate, mm, nf)
+         mm = 0
+      if yy or mm or dd: sdate = self.adddate(sdate, yy, mm, dd)
+      return "{} {}".format(sdate, stime)
+
+   # add given yyyy, mm, dd, hh, nn, ss to sdatetime
+   # if nf, add fraction of month only
+   def adddatetime(self, sdatetime, yy, mm, dd, hh, nn, ss, nf = 0):
+      if sdatetime and not isinstance(sdatetime, str): sdatetime = str(sdatetime)
+      (sdate, stime) = re.split(' ', sdatetime)   
+      if hh or nn or ss: (sdate, stime) = self.addtime(sdate, stime, hh, nn, ss)
+      if nf:
+         sdate = self.addmonth(sdate, mm, nf)
+         mm = 0
+      if yy or mm or dd: sdate = self.adddate(sdate, yy, mm, dd)
+      return "{} {}".format(sdate, stime)
+
+   # add given hours, minutes and seconds to the initial date and time
+   def addtime(self, sdate, stime, h, m, s):
+      if sdate and not isinstance(sdate, str): sdate = str(sdate)
+      if stime and not isinstance(stime, str): sdate = str(stime)
+      ups = (60, 60, 24)
+      tms = [0, 0, 0, 0]   # (sec, min, hour, day)
+      if s: tms[0] += s
+      if m: tms[1] += m
+      if h: tms[2] += h
+      if stime:
+         ms = re.match(r'^(\d+):(\d+):(\d+)$', stime)
+         if ms:
+            tms[2] += int(ms.group(1))
+            tms[1] += int(ms.group(2))
+            tms[0] += int(ms.group(3))
+      for i in range(3):
+         if tms[i] < 0:
+            while tms[i] < 0:
+               tms[i] += ups[i]
+               tms[i+1] -= 1
+         elif tms[i] >= ups[i]:
+            while tms[i] >= ups[i]:
+               tms[i] -= ups[i]
+               tms[i+1] += 1
+      stime = "{:02}:{:02}:{:02}".format(tms[2], tms[1], tms[0])
+      if tms[3]: sdate = self.adddate(sdate, 0, 0, tms[3])
+      return [sdate, stime]
+
+   # add time interval array to datetime
+   # opt = -1 - minus, 0 - begin time, 1 - add (default)
+   def addintervals(self, sdatetime, intv, opt = 1):
+      if not isinstance(sdatetime, str): sdatetime = str(sdatetime)
+      if not intv: return sdatetime
+      tv = [0]*7
+      i = 0
+      for v in intv:
+         tv[i] = v
+         i += 1
+      # assume the given datetime is end of the current interval;
+      # add one second to set it to beginning of the next one
+      if opt == 0: sdatetime = self.adddatetime(sdatetime, 0, 0, 0 ,0, 0, 1)
+      if opt < 1: # negative intervals for minus
+         for i in range(6):
+            if tv[i]: tv[i] = -tv[i]
+      return self.adddatetime(sdatetime, tv[0], tv[1], tv[2], tv[3], tv[4], tv[5], tv[6])
+
+   # adjust end date to the specified day days for frequency of year/month/week
+   # end of period if days == 0
+   # nf - number of fractions of a month, for unit of 'M' only
+   def enddate(self, sdate, days, unit, nf = 0):
+      if sdate and not isinstance(sdate, str): sdate = str(sdate)
+      if days and isinstance(days, str): days = int(days)
+      if not (unit and unit in 'YMW'): return sdate
+      if unit == 'Y':
+         ms = re.match(r'^(\d+)', sdate)
+         if ms:
+            yr = int(ms.group(1))
+            if days:
+               mn = 1
+               dy = days
+            else:
+               mn = 12
+               dy = 31
+            sdate = self.fmtdate(yr, mn, dy)
+      elif unit == 'M':
+         ms = re.match(r'^(\d+)-(\d+)-(\d+)', sdate)
+         if ms:
+            (yr, mn, dy) = (int(m) for m in ms.groups())
+         else:
+            ms = re.match(r'^(\d+)-(\d+)', sdate)
+            if ms:
+               (yr, mn) = (int(m) for m in ms.groups())
+               dy = 1
+            else:
+               return sdate
+         if not nf or nf == 1:
+            nd = days if days else calendar.monthrange(yr, mn)[1]
+            if nd != dy: sdate = self.fmtdate(yr, mn, nd)
+         else:
+            val = int(30/nf)
+            if dy >= 28:
+               mf = nf
+            else:
+               mf = int(dy/val)
+               if (mf*val) < dy: mf += 1
+            if days:
+               dy = (mf-1)*val + days
+            elif mf < nf:
+               dy = mf*val
+            else:
+               mn += 1
+               dy = 0
+            sdate = self.fmtdate(yr, mn, dy)
+      elif unit == 'W':
+         val = self.get_weekday(sdate)
+         if days != val: sdate = self.adddate(sdate, 0, 0, days-val)
+      return sdate
+
+   # adjust end time to the specified h/n/s for frequency of hour/mimute/second
+   def endtime(self, stime, unit):
+      if stime and not isinstance(stime, str): stime = str(stime)
+      if not (unit and unit in 'HNS'): return stime  
+      if stime:
+         tm = self.split_datetime(stime, 'T')
+      else:
+         tm = [0, 0, 0]
+      if unit == 'H':
+         tm[1] = tm[2] = 59
+      elif unit == 'N':
+         tm[2] = 59
+      elif unit != 'S':
+         tm[0] = 23
+         tm[1] = tm[2] = 59
+      return "{:02}:{:02}:{:02}".format(tm[0], tm[1]. tm[2])
+
+   # adjust end time to the specified h/n/s for frequency of year/month/week/day/hour/mimute/second
+   def enddatetime(self, sdatetime, unit, days = 0, nf = 0):
+      if sdatetime and not isinstance(sdatetime, str): sdatetime = str(sdatetime)
+      if not (unit and unit in 'YMWDHNS'): return sdatetime
+      (sdate, stime) = re.split(' ', sdatetime)
+      if unit in 'HNS':
+         stime = self.endtime(stime, unit)
+      else:
+         sdate = self.enddate(sdate, days, unit, nf)
+      return "{} {}".format(sdate, stime)
+
+   # get the string length dynamically
+   @staticmethod
+   def get_column_length(colname, values):   
+      clen = len(colname) if colname else 2  # initial column length as the length of column title
+      for val in values:
+         if val is None: continue
+         sval = str(val)
+         if sval and not re.search(r'\n', sval):
+            slen = len(sval)
+            if slen > clen: clen = slen
+      return clen
+
+   # Function: hour2time()
+   #   Return: time string in format of date HH:MM:SS
+   @staticmethod
+   def hour2time(sdate, nhour, endtime = 0):   
+      if sdate and not isinstance(sdate, str): sdate = str(sdate)
+      stime = "{:02}:".format(nhour)
+      if endtime:
+         stime += "59:59"
+      else:
+         stime += "00:00"
+      if sdate:
+         return "{} {}".format(sdate, stime)
+      else:
+         return stime
+
+   # Function: time2hour()
+   #   Return: list of date and hour
+   @staticmethod
+   def time2hour(stime):
+      sdate = nhour = None
+      times = stime.split(' ')
+      if len(times) == 2:
+         sdate = times[0]
+         stime = times[1]
+      ms = re.match(r'^(\d+)', stime)
+      if ms: nhour = int(ms.group(1))
+      return [sdate, nhour]
+
+   # get the all column widths
+   @staticmethod
+   def all_column_widths(pgrecs, flds, tdict):
+      colcnt = len(flds)
+      lens = [0]*colcnt
+      for i in range(colcnt):
+         fld = flds[i]
+         if fld not in tdict: continue
+         field = PgUtil.strip_field(tdict[fld][1])
+         lens[i] = PgUtil.get_column_length(None, pgrecs[field])
+      return lens
+
+   # check a give value, return 1 if numeric, 0 therwise
+   @staticmethod
+   def pgnum(val):
+      if not isinstance(val, str): val = str(val)
+      ms = re.match(r'^\-{0,1}(\d+|\d+\.\d*|d*\.\d+)([eE]\-{0,1}\d+)*$', val)
+      return 1 if ms else 0
+
+   # Function: pgcmp(val1, val2)
+   #   Return: 0 if both empty or two values are identilcal; -1 if val1 < val2; otherwise 1
+   @staticmethod
+   def pgcmp(val1, val2, ignorecase = 0, num = 0):   
+      if val1 is None:
+         if val2 is None:
+            return 0
+         else:
+            return -1
+      elif val2 is None:
+         return 1
+      typ1 = type(val1)
+      typ2 = type(val2)
+      if typ1 != typ2:
+         if num:
+            if typ1 is str:
+               typ1 = int
+               val1 = int(val1)
+            if typ2 is str:
+               typ2 = int
+               val2 = int(val2)
+         else:
+            if typ1 != str:
+               typ1 = str
+               val1 = str(val1)
+            if typ2 != str:
+               typ2 = str
+               val2 = str(val2)
+      if typ1 is str:
+         if num:
+            if typ1 is str and PgUtil.pgnum(val1) and PgUtil.pgnum(val2):
+               val1 = int(val1)
+               val2 = int(val2)
+         elif ignorecase:
+            val1 = val1.lower()
+            val2 = val2.lower()
+      if val1 > val2:
+         return 1
+      elif val1 < val2:
+         return -1
+      else:
+         return 0
+
+   # infiles: initial file list
+   #  Return: final file list with all the subdirectories expanded
+   @staticmethod
+   def recursive_files(infiles):
+      ofiles = []
+      for file in infiles:
+         if op.isdir(file):
+            ofiles.extend(PgUtil.recursive_files(glob.glob(file + "/*")))
+         else:
+            ofiles.append(file)
+      return ofiles
+
+   #   lidx: lower index limit  (including)
+   #   hidx: higher index limit (excluding)
+   #    key: string value to be searched,
+   #   list: reference to a sorted list where the key is searched)
+   # Return: index if found; -1 otherwise
+   @staticmethod
+   def asearch(lidx, hidx, key, list):
+      ret = -1
+      if (hidx - lidx) < 11:   # use linear search for less than 11 items
+         for midx in range(lidx, hidx):
+            if key == list[midx]:
+               ret = midx
+               break
+      else:
+         midx = (lidx + hidx)/2
+         if key == list[midx]:
+            ret = midx
+         elif key < list[midx]:
+            ret = PgUtil.asearch(lidx, midx, key, list)
+         else:
+            ret = PgUtil.asearch(midx + 1, hidx, key, list)
+      return ret
+
+   #   lidx: lower index limit  (including)
+   #   hidx: higher index limit (excluding)
+   #    key: string value to be searched,
+   #   list: reference to a sorted list where the key is searched)
+   # Return: index if found; -1 otherwise
+   @staticmethod
+   def psearch(lidx, hidx, key, list):
+      ret = -1
+      if (hidx - lidx) < 11: # use linear search for less than 11 items
+         for midx in range(lidx, hidx):
+            if re.search(list[midx], key):
+               ret = midx
+               break
+      else:
+         midx = int((lidx + hidx)/2)
+         if re.search(list[midx], key):
+            ret = midx
+         elif key < list[midx]:
+            ret = PgUtil.psearch(lidx, midx, key, list)
+         else:
+            ret = PgUtil.psearch(midx + 1, hidx, key, list)
+      return ret
+
+   # quicksort for pattern
+   @staticmethod
+   def quicksort(srecs, lo, hi, desc, cnt, nums = None):
+      i = lo
+      j = hi
+      mrec = srecs[int((lo+hi)/2)]
+      while True:
+         while PgUtil.cmp_records(srecs[i], mrec, desc, cnt, nums) < 0: i += 1
+         while PgUtil.cmp_records(srecs[j], mrec, desc, cnt, nums) > 0: j -= 1
+         if i <= j:
+            if i < j:
+               tmp = srecs[i]
+               srecs[i] = srecs[j]
+               srecs[j] = tmp
+            i += 1
+            j -= 1
+         if i > j: break   
+      #recursion
+      if lo < j: srecs = PgUtil.quicksort(srecs, lo, j, desc, cnt, nums)
+      if i < hi: srecs = PgUtil.quicksort(srecs, i, hi, desc, cnt, nums)
+      return srecs
+
+   # compare two arrays   
+   @staticmethod
+   def cmp_records(arec, brec, desc, cnt, nums):
+      for i in range(cnt):
+         num = nums[i] if nums else 0
+         ret = PgUtil.pgcmp(arec[i], brec[i], 0, num)
+         if ret != 0:
+            return (ret*desc[i])   
+      return 0   # identical records
+
+   # format one floating point value
+   @staticmethod
+   def format_float_value(val, precision = 2):
+      units = ('B', 'KB', 'MB', 'GB', 'TB', 'PB')
+      if val is None:
+         return ''
+      elif not isinstance(val, int):
+         val = int(val)
+      idx = 0
+      while val >= 1000 and idx < 5:
+         val /= 1000
+         idx += 1
+      return "{:.{}f}{}".format(val, precision, units[idx])
+
+   # check a file is a ASCII text one
+   # return 1 if yes, 0 if not; or -1 if file not checkable
+   @staticmethod
+   def is_text_file(fname, blocksize = 256, threshhold = 0.1):
+      # File doesn't exist or is not a regular file
+      if not op.exists(fname) or not op.isfile(fname): return -1
+      if op.getsize(fname) == 0: return 1  # Empty files are considered text
+      try:
+         buffer = None
+         with open(fname, 'rb') as f:
+            buffer = f.read(blocksize)
+         # Check for null bytes (a strong indicator of a binary file)
+         if not buffer or b'\0' in buffer: return 0
+         text_characters = (
+            b'\t\n\r\f\v' +        # Whitespace characters
+            bytes(range(32, 127))  # Printable ASCII characters
+         )
+         non_text_count = 0
+         for byte in buffer:
+            if byte not in text_characters:
+               non_text_count += 1  # Count non-text characters
+         # If a significant portion of the buffer consists of non-text characters,
+         # it's likely a binary file.
+         return 1 if((non_text_count/len(buffer)) < threshhold) else 0
+      except IOError:
+         return -1   # Handle cases where the file cannot be opened or read