rda-python-common 1.0.17__py3-none-any.whl → 1.0.21__py3-none-any.whl

rda_python_common/PgFile.py

@@ -67,7 +67,7 @@ OHOST = PgLOG.PGLOG['OBJCTSTR']
 BHOST = PgLOG.PGLOG['BACKUPNM']
 DHOST = PgLOG.PGLOG['DRDATANM']
 OBJCTCMD = "isd_s3_cli"
-BACKCMD = "dsglobus" 
+BACKCMD = "dsglobus"
 
 HLIMIT = 0    # HTAR file count limit
 BLIMIT = 2    # minimum back tar file size in DB 
@@ -338,17 +338,28 @@ def local_copy_object(tofile, fromfile, bucket = None, meta = None, logact = 0):
 def quasar_multiple_trasnfer(tofiles, fromfiles, topoint, frompoint, logact = 0):
 
    ret = PgLOG.FAILURE
-   qstr = '{"action":"transfer","label":"%s","verify_checksum":true,' % ENDPOINTS[topoint]
-#   qstr = '{"action":"transfer","label":"%s",' % ENDPOINTS[topoint]
-   qstr += '"source_endpoint":"%s","destination_endpoint":"%s","files":[\n' % (frompoint, topoint)
+
    fcnt = len(fromfiles)
-   bstr = ''
+   transfer_files = {"files": []}
    for i in range(fcnt):
-      qstr += '%s{"source_file":"%s","destination_file":"%s"}' % (bstr, fromfiles[i], tofiles[i])
-      if i == 0: bstr = ',\n'
-   qstr += ']}'
-
-   task = submit_globus_task(BACKCMD, topoint, logact, qstr)
+      transfer_files["files"].append({
+         "source_file": fromfiles[i],
+         "destination_file": tofiles[i]
+      })
+   qstr = json.dumps(transfer_files)
+
+   action = 'transfer'
+   source_endpoint = frompoint
+   destination_endpoint = topoint
+   label = f"{ENDPOINTS[frompoint]} to {ENDPOINTS[topoint]} {action}"
+   verify_checksum = True
+
+   cmd = f'{BACKCMD} {action} -se {source_endpoint} -de {destination_endpoint} --label "{label}"'
+   if verify_checksum:
+      cmd += ' -vc'   
+   cmd += ' --batch -'
+
+   task = submit_globus_task(cmd, topoint, logact, qstr)
    if task['stat'] == 'S':
       ret = PgLOG.SUCCESS
    elif task['stat'] == 'A':
@@ -379,7 +390,9 @@ def endpoint_copy_endpoint(tofile, fromfile, topoint, frompoint, logact = 0):
       if tinfo and tinfo['data_size'] > 0:
          return PgLOG.pglog("{}-{}: file exists already".format(topoint, tofile), logact)
 
-   cmd = "{} -t -vc -se {} -de {} -sf {} -df {}".format(BACKCMD, frompoint, topoint, fromfile, tofile)
+   action = 'transfer'
+   cmd = f'{BACKCMD} {action} -se {frompoint} -de {topoint} -sf {fromfile} -df {tofile} -vc'
+
    task = submit_globus_task(cmd, topoint, logact)
    if task['stat'] == 'S':
       ret = PgLOG.SUCCESS
@@ -435,7 +448,8 @@ def check_globus_status(taskid, endpoint = None, logact = 0):
    if not taskid: return ret
    if not endpoint: endpoint = PgLOG.PGLOG['BACKUPEP']
    mp = r'Status:\s+({})'.format('|'.join(QSTATS.values()))
-   cmd = "{} -gt --task-id {}".format(BACKCMD, taskid)
+
+   cmd = f"{BACKCMD} get-task {taskid}"
    astats = ['OK', 'Queued']
 
    for loop in range(2):
@@ -452,7 +466,7 @@ def check_globus_status(taskid, endpoint = None, logact = 0):
                      if logact&PgLOG.NOWAIT:
                         errmsg = "{}: Cancel Task due to {}:\n{}".format(taskid, detail, buf)
                         errlog(errmsg, 'B', 1, logact)
-                        ccmd = "{} -ct --task-id {}".format(BACKCMD, taskid)
+                        ccmd = f"{BACKCMD} cancel-task {taskid}"
                         PgLOG.pgsystem(ccmd, logact, 7)
                      else:
                         time.sleep(PgSIG.PGSIG['ETIME'])
@@ -759,7 +773,7 @@ def delete_backup_file(file, endpoint = None, logact = 0):
    info = check_backup_file(file, endpoint, 0, logact)
    if not info: return PgLOG.FAILURE
 
-   cmd = "{} -d -ep {} -tf {}".format(BACKCMD, endpoint, file)
+   cmd = f"{BACKCMD} delete -ep {endpoint} -tf {file}"
    task = submit_globus_task(cmd, endpoint, logact)
    if task['stat'] == 'S':
       return PgLOG.SUCCESS
@@ -1028,7 +1042,7 @@ def move_backup_file(tofile, fromfile, endpoint = None, logact = 0):
    elif tinfo != None:
       return ret
 
-   cmd = "{} --rename -ep {} --oldpath {} --newpath {}".format(BACKCMD, endpoint, fromfile, tofile)
+   cmd = f"{BACKCMD} rename -ep {endpoint} --old-path {fromfile} --new-path {tofile}"
    loop = 0
    while loop < 2:
       buf = PgLOG.pgsystem(cmd, logact, CMDRET)
@@ -1158,7 +1172,7 @@ def make_one_backup_directory(dir, odir, endpoint = None, logact = 0):
    if not odir: odir = dir
    if not make_one_backup_directory(op.dirname(dir), odir, endpoint, logact): return PgLOG.FAILURE
 
-   cmd = "{} --mkdir -ep {} -p {}".format(BACKCMD, endpoint, dir)
+   cmd = f"{BACKCMD} mkdir -ep {endpoint} -p {dir}"
    for loop in range(2):
       buf = PgLOG.pgsystem(cmd, logact, CMDRET)
       syserr = PgLOG.PGLOG['SYSERR']
@@ -1843,7 +1857,7 @@ def check_backup_file(file, endpoint = None, opt = 0, logact = 0):
    if not endpoint: endpoint = PgLOG.PGLOG['BACKUPEP']
    bdir = op.dirname(file)
    bfile = op.basename(file)
-   cmd = "{} -ls -ep {} -p {} --filter {}".format(BACKCMD, endpoint, bdir, bfile)
+   cmd = f"{BACKCMD} ls -ep {endpoint} -p {bdir} --filter {bfile}"
    ccnt = loop = 0
    while loop < 2:
       buf = PgLOG.pgsystem(cmd, logact, CMDRET)
@@ -2208,7 +2222,8 @@ def backup_glob(dir, endpoint = None, opt = 0, logact = 0):
 
    if not dir: return None
    if not endpoint: endpoint = PgLOG.PGLOG['BACKUPEP']
-   cmd = "{} -ls -ep {} -p {}".format(BACKCMD, endpoint, dir)
+
+   cmd = f"{BACKCMD} ls -ep {endpoint} -p {dir}"
    flist = {}
    for loop in range(2):
       buf = PgLOG.pgsystem(cmd, logact, CMDRET)

rda_python_common/PgLOG.py

@@ -11,7 +11,7 @@
 #             message on screen and exit script
 #
 #    Github : https://github.com/NCAR/rda-python-common.git
-# 
+#
 ###############################################################################
 
 import sys
@@ -29,8 +29,8 @@ import traceback
 
 # define some constants for logging actions
 MSGLOG = (0x00001)   # logging message
-WARNLG = (0x00002)   # show logging message as warning 
-EXITLG = (0x00004)   # exit after logging 
+WARNLG = (0x00002)   # show logging message as warning
+EXITLG = (0x00004)   # exit after logging
 LOGWRN = (0x00003)   # MSGLOG|WARNLG
 LOGEXT = (0x00005)   # MSGLOG|EXITLG
 WRNEXT = (0x00006)   # WARNLG|EXITLG
@@ -57,7 +57,7 @@ MISLOG = (0x00811)   # cannot access logfile
 EMLSUM = (0x08000)   # record as email summary
 EMEROL = (0x10000)   # record error as email only
 EMLALL = (0x1D208)   # all email acts
-DOSUDO = (0x20000)   # add 'sudo -u PGLOG['RDAUSER']' 
+DOSUDO = (0x20000)   # add 'sudo -u PGLOG['RDAUSER']'
 NOTLOG = (0x40000)   # do not log any thing
 OVRIDE = (0x80000)   # do override existing file or record
 NOWAIT = (0x100000)  # do not wait on globus task to finish
@@ -95,7 +95,7 @@ PGLOG = {   # more defined in untaint_suid() with environment variables
    'ARCHROOT': "/FS/DECS",      # root path for segregated tape on hpss
    'BACKROOT': "/DRDATA/DECS",  # backup path for desaster recovering tape on hpss
    'OLDAROOT': "/FS/DSS",       # old root path on hpss
-   'OLDBROOT': "/DRDATA/DSS",   # old backup tape on hpss 
+   'OLDBROOT': "/DRDATA/DSS",   # old backup tape on hpss
    'RDAUSER' : "rdadata",  # common rda user name
    'RDAEMAIL' : "zji",     # specialist to receipt email intead of common rda user name
    'SUDORDA' : 0,          # 1 to allow sudo to PGLOG['RDAUSER']
@@ -112,7 +112,7 @@ PGLOG = {   # more defined in untaint_suid() with environment variables
    'PUSGDIR' : None,
    'BCHHOSTS' : "PBS",
    'HOSTTYPE' : 'dav',    # default HOSTTYPE
-   'EMLMAX' : 256,       # up limit of email line count 
+   'EMLMAX' : 256,       # up limit of email line count
    'PGBATCH' : '',       # current batch service name, SLURM or PBS
    'PGBINDIR' : '',
    'SLMTIME' : 604800,   # max runtime for SLURM bath job, (7x24x60x60 seconds)
@@ -121,7 +121,7 @@ PGLOG = {   # more defined in untaint_suid() with environment variables
    'RDAGRP'  : "decs",
    'DSCHECK' : None,     # carry some cached dscheck information
    'PGDBBUF' : None,      # reference to a connected database object
-   'HPSSLMT' : 10,       # up limit of HPSS streams 
+   'HPSSLMT' : 10,       # up limit of HPSS streams
    'NOQUIT'  : 0,        # do not quit if this flag is set for daemons
    'DBRETRY' : 2,        # db retry count after error
    'TIMEOUT' : 15,       # default timeout (in seconds) for tosystem()
@@ -174,15 +174,15 @@ def current_datetime(ctime = 0):
 # get an environment variable and untaint it
 #
 def get_environment(name, default = None, logact = 0):
- 
+
    env = os.getenv(name, default)
    if env is None and logact:
       pglog(name + ": Environment variable is not defined", logact)
 
    return env
-   
+
 #
-# cache the msg string to global email entries for later call of send_email()  
+# cache the msg string to global email entries for later call of send_email()
 #
 def set_email(msg, logact = 0):
 
@@ -192,7 +192,7 @@ def set_email(msg, logact = 0):
             msg = PGLOG['PRGMSG'] + "\n" + msg
             PGLOG['PRGMSG'] = ""
          if PGLOG['ERRCNT'] == 0:
-            if not re.search(r'\n$', msg): msg += "!\n" 
+            if not re.search(r'\n$', msg): msg += "!\n"
          else:
             if PGLOG['ERRCNT'] == 1:
                msg += " with 1 Error:\n"
@@ -299,7 +299,7 @@ def send_email(subject = None, receiver = None, msg = None, sender = None, logac
    if receiver == PGLOG['RDAUSER']: receiver = PGLOG['RDAEMAIL']
    if receiver.find('@') == -1: receiver += "@ucar.edu"
 
-   if docc and not re.match(PGLOG['RDAUSER'], sender): add_carbon_copy(sender, 1) 
+   if docc and not re.match(PGLOG['RDAUSER'], sender): add_carbon_copy(sender, 1)
 
    emlmsg = "From: {}\nTo: {}\n".format(sender, receiver)
    logmsg = "Email " + receiver
@@ -336,7 +336,7 @@ def log_email(emlmsg):
 #
 # Function: cmdlog(cmdline)
 # cmdline - program name and all arguments
-# ctime - time (in seconds) when the command starts 
+# ctime - time (in seconds) when the command starts
 #
 def cmdlog(cmdline = None, ctime = 0, logact = None):
 
@@ -418,7 +418,11 @@ def pglog(msg, logact = MSGLOG):
       if not logact&NOTLOG:
          if logact&ERRLOG:
             if not PGLOG['ERRFILE']: PGLOG['ERRFILE'] = re.sub(r'.log$', '.err', PGLOG['LOGFILE'])
-            ERR = open("{}/{}".format(PGLOG['LOGPATH'], PGLOG['ERRFILE']), 'a')
+            try:
+                ERR = open("{}/{}".format(PGLOG['LOGPATH'], PGLOG['ERRFILE']), 'a')
+            except FileNotFoundError:
+                ERR = open("error.log", 'a')
+                ERR.write(f"Error File not found: {PGLOG['LOGPATH']}/{PGLOG['ERRFILE']}")
             ERR.write(msg)
             if not logact&(EMLALL|SKPTRC): ERR.write(get_call_trace())
             ERR.close()
@@ -437,7 +441,7 @@ def pglog(msg, logact = MSGLOG):
       if logact&SEPLIN: OUT.write(PGLOG['SEPLINE'])
       OUT.write(msg)
 
-   
+
    if logact&EXITLG:
       pgexit(1)
    else:
@@ -488,7 +492,7 @@ def get_caller_file(cidx = 0):
    return traceback.extract_stack()[cidx][0]
 
 #
-# log message, msg, for degugging processes according to the debug level 
+# log message, msg, for degugging processes according to the debug level
 #
 def pgdbg(level, msg = None, do_trace = True):
 
@@ -510,7 +514,7 @@ def pgdbg(level, msg = None, do_trace = True):
          if ms:
             levels[0] = int(ms.group(1)) if ms.group(1) else 0
             levels[1] = int(ms.group(2)) if ms.group(2) else 9999
-   
+
    if level > levels[1] or level < levels[0]: return   # debug level is out of range
 
    if 'DBGPATH' in PGLOG:
@@ -553,9 +557,9 @@ def pgtrim(line, rmcmt = 1):
 # set PGLOG['PUSGDIR'] from the program file with full path
 #
 def set_help_path(progfile):
-   
+
    PGLOG['PUSGDIR'] = op.dirname(op.abspath(progfile))
-   
+
 #
 # Function: show_usage(progname: Perl program name to get file "progname.usg")
 #
@@ -597,7 +601,7 @@ def show_usage(progname, opts = None):
          IN.close()
    else:
       os.system("more " + usgname)
-   
+
    pgexit(0)
 
 #
@@ -647,7 +651,7 @@ def pgsystem(pgcmd, logact = LOGWRN, cmdopt = 5, instr = None, seconds = 0):
    if not pgcmd: return ret  # empty command
 
    act = logact&~EXITLG
-   if act&ERRLOG: 
+   if act&ERRLOG:
       act &= ~ERRLOG
       act |= WARNLG
 
@@ -670,7 +674,7 @@ def pgsystem(pgcmd, logact = LOGWRN, cmdopt = 5, instr = None, seconds = 0):
          pglog("> " + cmdstr, cmdact)
       if cmdopt&512 and (instr or seconds):
          msg = ''
-         if seconds: msg = 'Timeout = {} Seconds'.format(seconds) 
+         if seconds: msg = 'Timeout = {} Seconds'.format(seconds)
          if instr: msg += ' With STDIN:\n' + instr
          if msg: pglog(msg, cmdact)
    stdlog = act if cmdopt&2 else 0
@@ -762,7 +766,7 @@ def pgsystem(pgcmd, logact = LOGWRN, cmdopt = 5, instr = None, seconds = 0):
       if ret == SUCCESS or loop >= loops: break
       time.sleep(6)
 
-   if ret == FAILURE and retbuf and cmdopt&272 == 272: 
+   if ret == FAILURE and retbuf and cmdopt&272 == 272:
       if PGLOG['SYSERR']: PGLOG['SYSERR'] += '\n'
       PGLOG['SYSERR'] += retbuf
       retbuf = ''
@@ -783,7 +787,7 @@ def strip_output_line(line):
    return line
 
 #
-# show command running time string formated by seconds_to_string_time() 
+# show command running time string formated by seconds_to_string_time()
 #
 def cmd_execute_time(cmdstr, last, logact = None):
 
@@ -816,7 +820,7 @@ def seconds_to_string_time(seconds, showzero = 0):
             msg += "{}D".format(int(hours/24))   # days
          if h: msg += "{}H".format(h)
       if m: msg += "{}M".format(m)
-      if s: 
+      if s:
          msg += "%dS"%(s) if isinstance(s, int) else "{:.3f}S".format(s)
    elif showzero:
       msg = "0S"
@@ -907,7 +911,7 @@ def break_long_string(lstr, limit = 1024, bsign = "\n", mline = 200, bchars = '
 #        1: remove path1 from path2
 #
 def join_paths(path1, path2, diff = 0):
-   
+
    if not path2: return path1
    if not path1 or not diff and re.match('/', path2): return path2
 
@@ -1017,7 +1021,7 @@ def get_host(getbatch = 0):
       return PGLOG['HOSTNAME']
    else:
       host = socket.gethostname()
-   
+
    return get_short_host(host)
 
 #
@@ -1069,7 +1073,7 @@ def get_pbs_host():
    return None
 
 #
-# set host status, 0 dead & 1 live, for one or all avalaible slurm hosts 
+# set host status, 0 dead & 1 live, for one or all avalaible slurm hosts
 #
 def set_slurm_host(host = None, stat = 0):
 
@@ -1084,7 +1088,7 @@ def set_slurm_host(host = None, stat = 0):
          SLMSTATS[host] = stat
 
 #
-# set host status, 0 dead & 1 live, for one or all avalaible pbs hosts 
+# set host status, 0 dead & 1 live, for one or all avalaible pbs hosts
 #
 def set_pbs_host(host = None, stat = 0):
 
@@ -1164,7 +1168,7 @@ def get_hpss_command(cmd, asuser = None, hcmd = None):
 
    cuser = PGLOG['SETUID'] if PGLOG['SETUID'] else PGLOG['CURUID']
    if not hcmd: hcmd = 'hsi'
-   
+
    if asuser and cuser != asuser:
       if cuser == PGLOG['RDAUSER']:
          return "{} sudo -u {} {}".format(hcmd, asuser, cmd)      # setuid wrapper as user asuser
@@ -1187,8 +1191,8 @@ def get_hpss_command(cmd, asuser = None, hcmd = None):
 def get_sync_command(host, asuser = None):
 
    host = get_short_host(host)
-  
-   if (not (PGLOG['SETUID'] and PGLOG['SETUID'] == PGLOG['RDAUSER']) and 
+
+   if (not (PGLOG['SETUID'] and PGLOG['SETUID'] == PGLOG['RDAUSER']) and
       (not asuser or asuser == PGLOG['RDAUSER'])):
       return "sync" + host
 
@@ -1321,7 +1325,7 @@ def set_common_pglog():
    SETPGLOG("PVIEWHOST", "rda-pgdb-02.ucar.edu")                   # host name for view only postgresql server
    SETPGLOG("FTPUPLD",  PGLOG['TRANSFER']+"/rossby")    # ftp upload path
    PGLOG['GPFSROOTS'] = "{}|{}|{}".format(PGLOG['DSDHOME'], PGLOG['UPDTWKP'], PGLOG['RQSTHOME'])
- 
+
    if 'ECCODES_DEFINITION_PATH' not in os.environ:
       os.environ['ECCODES_DEFINITION_PATH'] = "/usr/local/share/eccodes/definitions"
    os.environ['history'] = '0'
@@ -1336,6 +1340,7 @@ def set_common_pglog():
       os.environ['TMPDIR'] = PGLOG['TMPDIR']
 
    # empty diretory for HOST-sync
+
    PGLOG['TMPSYNC'] = PGLOG['DSSDBHM'] + "/tmp/.syncdir" 
 
    os.umask(2)
@@ -1387,7 +1392,7 @@ def SETPGLOG(name, value = ''):
 # set specialist home and return the default shell
 #
 def set_specialist_home(specialist):
-   
+
    if specialist == PGLOG['CURUID']: return   # no need reset
    if 'MAIL' in os.environ and re.search(PGLOG['CURUID'], os.environ['MAIL']):
       os.environ['MAIL'] = re.sub(PGLOG['CURUID'], specialist, os.environ['MAIL'])
@@ -1508,11 +1513,11 @@ def check_process_host(hosts, chost = None, mflag = None, pinfo = None, logact =
    error = ''
    if not mflag: mflag = 'G'
    if not chost: chost = get_host(1)
- 
+
    if mflag == 'M':    # exact match
       if not hosts or hosts != chost:
          ret = 0
-         if pinfo: error = "not matched exactly"      
+         if pinfo: error = "not matched exactly"
    elif mflag == 'I':   # inclusive match
       if not hosts or hosts.find('!') == 0 or hosts.find(chost) < 0:
          ret = 0
@@ -1521,7 +1526,7 @@ def check_process_host(hosts, chost = None, mflag = None, pinfo = None, logact =
       if hosts.find(chost) >= 0:
          if hosts.find('!') == 0:
             ret = 0
-            if pinfo: error = "matched exclusively" 
+            if pinfo: error = "matched exclusively"
       elif hosts.find('!') != 0:
          ret = 0
          if pinfo: error = "not matched"
@@ -1567,7 +1572,7 @@ def convert_chars(name, default = 'X'):
 #  Retrieve host and process id
 #
 def current_process_info(realpid = 0):
-   
+
    if realpid or PGLOG['CURBID'] < 1:
       return [PGLOG['HOSTNAME'], os.getpid()]
    else:
@@ -1596,7 +1601,7 @@ def argv_to_string(argv = None, quote = 1, action = None):
    return argstr
 
 #
-# convert an integer to non-10 based string  
+# convert an integer to non-10 based string
 #
 def int2base(x, base):
 
@@ -1614,9 +1619,9 @@ def int2base(x, base):
    dgts.reverse()
 
    return ''.join(dgts)
- 
+
 #
-# convert a non-10 based string to an integer 
+# convert a non-10 based string to an integer
 #
 def base2int(x, base):
 
@@ -1640,7 +1645,7 @@ def base2int(x, base):
 
 #
 # convert integer to ordinal string
-# 
+#
 def int2order(num):
 
    ordstr = ['th', 'st', 'nd', 'rd']

rda_python_common/PgUtil.py

@@ -9,7 +9,7 @@
 #   Purpose : python library module for global misc utilities
 #
 #    Github : https://github.com/NCAR/rda-python-common.git
-# 
+#
 ###############################################################################
 #
 import os
@@ -47,7 +47,7 @@ MDAYS = [365, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
 # dt: optional given date in format of "YYYY-MM-DD"
 # return weekday: 0 - Sunday, 1 - Monday, ..., 6 - Saturday
 #
-def get_weekday(date = None): 
+def get_weekday(date = None):
 
    if date is None:
       ct = time.gmtime() if PgLOG.PGLOG['GMTZ'] else time.localtime()
@@ -117,23 +117,23 @@ def get_wday(wday, fmt = None):
          elif re.match(r'^WW', fmt):
             swday = swday.upper()
       else:
-         swday = str(wday)         
+         swday = str(wday)
       return swday
    else:
       return wday
 
 #
-#   file: given file name 
+#   file: given file name
 # Return: type if given file name is a valid online file; '' otherwise
 #
 def valid_online_file(file, type = None, exists = None):
-   
+
    if exists is None or exists:
       if not op.exists(file): return ''    # file does not exist
 
    bname = op.basename(file)
    if re.match(r'^,.*', bname): return ''       # hidden file
-   
+
    if re.search(r'index\.(htm|html|shtml)$', bname, re.I): return ''   # index file
 
    if  type and type != 'D': return type
@@ -166,9 +166,9 @@ def curdatetime():
 def curdatehour(fmt = None):
 
    ct = time.gmtime() if PgLOG.PGLOG['GMTZ'] else time.localtime()
- 
+
    dt =  fmtdate(ct[0], ct[1], ct[2], fmt) if fmt else time.strftime("%Y-%m-%d", ct)
-   
+
    return [dt, ct[3]]
 
 #
@@ -187,9 +187,9 @@ def get_date_time(tm = None):
    elif isinstance(tm, datetime.datetime):
       act = str(tm).split(' ')
    elif isinstance(tm, datetime.date):
-      act = [str(tm), '00:00:00'] 
+      act = [str(tm), '00:00:00']
    elif isinstance(tm, datetime.time):
-      act = [None, str(tm)] 
+      act = [None, str(tm)]
 
    if ct == None:
       return act if act else None
@@ -209,7 +209,7 @@ def get_datetime(tm = None):
       return tm
    elif isinstance(tm, (int, float)):
       ct = time.localtime(tm)
-      return time.strftime("%Y-%m-%d %H:%M:%S", ct)   
+      return time.strftime("%Y-%m-%d %H:%M:%S", ct)
    elif isinstance(tm, datetime.datetime):
       return str(tm)
    elif isinstance(tm, datetime.date):
@@ -223,7 +223,7 @@ def get_datetime(tm = None):
 # Return: timestsmp string in format of 'YYYYMMDDHHMMSS
 #
 def timestamp(file = None):
-   
+
    if file is None:
       ct = time.gmtime() if PgLOG.PGLOG['GMTZ'] else time.localtime()
    else:
@@ -251,7 +251,7 @@ def check_datetime(date, default):
 def curdate(fmt = None):
 
    ct = time.gmtime() if PgLOG.PGLOG['GMTZ'] else time.localtime()
- 
+
    return fmtdate(ct[0], ct[1], ct[2], fmt) if fmt else time.strftime("%Y-%m-%d", ct)
 
 #
@@ -314,19 +314,19 @@ def format_datehour(date, hour, tofmt = None, fromfmt = None):
 # split a date, time or datetime into an array according to
 # the sep value; str to int for digital values
 #
-def split_datetime(sdt, sep = '\D'):
+def split_datetime(sdt, sep = r'\D'):
 
    if not isinstance(sdt, str): sdt = str(sdt)
    adt = re.split(sep, sdt)
    acnt = len(adt)
    for i in range(acnt):
-      if re.match('^\d+$', adt[i]): adt[i] = int(adt[i])
+      if re.match(r'^\d+$', adt[i]): adt[i] = int(adt[i])
    return adt
 
 #
 #    date: given date in format of fromfmt
 #   tofmt: date formats; ex. "Month D, YYYY"
-# fromfmt: date formats, default to YYYY-MM-DD 
+# fromfmt: date formats, default to YYYY-MM-DD
 #  Return: new formated date string according to tofmt
 #
 def format_date(cdate, tofmt = None, fromfmt = None):
@@ -338,8 +338,8 @@ def format_date(cdate, tofmt = None, fromfmt = None):
    mns = sep.join(MNS)
    months = sep.join(MONTHS)
    mkeys = ['D', 'M', 'Q', 'Y', 'C', 'H']
-   PATTERNS = ['(\d\d\d\d)', '(\d+)', '(\d\d)',
-               '(\d\d\d)', '(' + mns + ')', '(' + months + ')']
+   PATTERNS = [r'(\d\d\d\d)', r'(\d+)', r'(\d\d)',
+               r'(\d\d\d)', '(' + mns + ')', '(' + months + ')']
 
    if not fromfmt:
       if not tofmt:
@@ -358,7 +358,7 @@ def format_date(cdate, tofmt = None, fromfmt = None):
       if ms:
          fmts[mkey] = ms.group(1)
          pattern = re.sub(fmts[mkey], '', pattern)
-         
+
    cnt = 0
    for mkey in fmts:
       fmt = fmts[mkey]
@@ -372,7 +372,7 @@ def format_date(cdate, tofmt = None, fromfmt = None):
       formats[fromfmt.find(fmt)] = fmt
       fromfmt = fromfmt.replace(fmt, PATTERNS[i])
       cnt += 1
-         
+
    ms = re.findall(fromfmt, cdate)
    mcnt = len(ms[0]) if ms else 0
    i = 0
@@ -534,7 +534,7 @@ def fmtdate(yr, mn, dy, tofmt = None):
          if md:
             fmt = md.group(1)   # quarter
             m = int((m+2)/3)
-            smn = "{:02}".format(m) if len(fmt) == 2 else str(m) 
+            smn = "{:02}".format(m) if len(fmt) == 2 else str(m)
             tofmt = re.sub(fmt, smn, tofmt, 1)
 
    if yr != None:
@@ -548,7 +548,7 @@ def fmtdate(yr, mn, dy, tofmt = None):
             if y > 999: y = int(y/10)
             syr = "{:03}".format(y)
          else:
-            if re.search(r'^YY00', fmt, re.I):  y = 100*int(y/100)    # hundred years 
+            if re.search(r'^YY00', fmt, re.I):  y = 100*int(y/100)    # hundred years
             syr = "{:04}".format(y)
          tofmt = re.sub(fmt, syr, tofmt, 1)
       else:
@@ -594,7 +594,7 @@ def date_and_time(sdt):
 
 #
 # convert given date/time to unix epoch time; -1 if cannot
-# 
+#
 def unixtime(stime):
 
    pt = [0]*9
@@ -666,7 +666,7 @@ def dtrange(dates):
 # Return: a string of formated period
 #
 def format_period(sdate, edate, fmt = None):
-  
+
    period = ''
 
    if not fmt:
@@ -828,7 +828,7 @@ def addrecord(records, record, idx):
          records[key] = []
 
    for key in record:
-      slen = len(records[key])         
+      slen = len(records[key])
       if idx < slen:
          records[key][idx] = record[key]
       else:
@@ -851,7 +851,7 @@ def hash2array(hrecs, hkeys = None):
       arec = {}
       for hkey in hkeys: arec[hkey] = hrecs[hkey][i]
       arecs[i] = arec
-   
+
    return arecs
 
 #
@@ -886,7 +886,7 @@ def hashcount(records, opt = 0):
          ret[0] = clen
       if opt == 1 or opt == 2:
          ret[1] = len(next(iter(records.values())))
-         
+
    return ret if opt == 2 else ret[opt]
 
 #
@@ -950,7 +950,7 @@ def joinhash(adict, bdict, default = None, unique = None):
 # Return: the joined list
 #
 def joinarray(lst1, lst2, unique = None):
-   
+
    if not lst2: return lst1
    if not lst1: return lst2
 
@@ -970,7 +970,7 @@ def joinarray(lst1, lst2, unique = None):
 
 #
 # Function: crosshash(ahash, bhash)
-#   Return: a reference to the cross-joined hash records 
+#   Return: a reference to the cross-joined hash records
 #
 def crosshash(ahash, bhash):
 
@@ -997,7 +997,7 @@ def crosshash(ahash, bhash):
 def strip_field(field):
    ms = re.search(r'\.([^\.]+)$', field)
    if ms: field = ms.group(1)
-   
+
    return field
 
 #
@@ -1008,7 +1008,7 @@ def strip_field(field):
 #   Return: a sorted dict list
 #
 def sorthash(pgrecs, flds, hash, patterns = None):
-   
+
    fcnt = len(flds)    # count of fields to be sorted on
 
    # set sorting order, descenting (-1) or ascenting (1)
@@ -1066,7 +1066,7 @@ def diffdate(date1, date2):
    ut1 = ut2 = 0
    if date1: ut1 = unixtime(date1)
    if date2: ut2 = unixtime(date2)
-   return round((ut1 - ut2)/86400)   # 24*60*60 
+   return round((ut1 - ut2)/86400)   # 24*60*60
 
 #
 # Return: the number of seconds bewteen time1 and time2
@@ -1089,7 +1089,7 @@ def get_days(cdate):
 
 #
 # Function: get_month_days(date)
-# 
+#
 #   Return: the number of days in given month
 #
 def get_month_days(cdate):
@@ -1104,7 +1104,7 @@ def get_month_days(cdate):
 
 #
 # Function: validate_date(date)
-# 
+#
 #   Return: a date in format of YYYY-MM-DD thar all year/month/day are validated
 #
 def validate_date(cdate):
@@ -1131,7 +1131,7 @@ def validate_date(cdate):
 
 #
 # Function: get_date(days)
-# 
+#
 #   Return: the date in format of "YYYY-MM-DD" for given number of days
 #   from '1970-01-01 00:00:00'
 #
@@ -1148,8 +1148,8 @@ def diffdatehour(date1, hour1, date2, hour2):
    if hour2 is None: hour2 = 23
    return (hour1 - hour2) + 24*diffdate(date1, date2)
 
-# 
-# hour difference between GMT and local time 
+#
+# hour difference between GMT and local time
 #
 def diffgmthour():
 
@@ -1206,7 +1206,7 @@ def addmonth(cdate, mf, nf = 1):
             mn -= 1
 
       dy += ody
-      cdate = fmtdate(yr, mn, dy)      
+      cdate = fmtdate(yr, mn, dy)
 
    return cdate
 
@@ -1230,7 +1230,7 @@ def addyearmonth(ym, yr, mn):
             yr += 1
             mn -= 12
 
-      ym = "{:04}{:02}".format(yr, mn)      
+      ym = "{:04}{:02}".format(yr, mn)
 
    return ym
 
@@ -1258,7 +1258,7 @@ def set_leap_mdays(year):
 # wrap on calendar.isleap()
 #
 def is_leapyear(year): return calendar.isleap(year)
- 
+
 #
 # reutn 1 if is end of month
 #
@@ -1317,7 +1317,7 @@ def adjust_ymd(yr, mn, dy):
 #     yr: the number of years to add/subtract from the odate for positive/negative value,
 #     mn: the number of months to add/subtract from the odate for positive/negative value,
 #     dy: the number of days to add/subtract from the odate for positive/negative value)
-# 
+#
 # Return: new date
 #
 def adddate(cdate, yr, mn = 0, dy = 0, tofmt = None):
@@ -1401,7 +1401,7 @@ def adddatehour(sdate, nhour, yr, mn, dy, hr = 0):
       if nhour != None: nhour = hr
 
    if yr or mn or dy: sdate = adddate(sdate, yr, mn, dy)
-   
+
    return [sdate, nhour]
 
 #
@@ -1486,7 +1486,7 @@ def addintervals(sdatetime, intv, opt = 1):
    for v in intv:
       tv[i] = v
       i += 1
-      
+
    # assume the given datetime is end of the current interval;
    # add one second to set it to beginning of the next one
    if opt == 0: sdatetime = adddatetime(sdatetime, 0, 0, 0 ,0, 0, 1)
@@ -1530,7 +1530,7 @@ def enddate(sdate, days, unit, nf = 0):
             dy = 1
          else:
             return sdate
-         
+
       if not nf or nf == 1:
          nd = days if days else calendar.monthrange(yr, mn)[1]
          if nd != dy: sdate = fmtdate(yr, mn, nd)
@@ -1632,7 +1632,7 @@ def hour2time(sdate, nhour, endtime = 0):
 #   Return: list of date and hour
 #
 def time2hour(stime):
-   
+
    sdate = nhour = None
    times = stime.split(' ')
 
@@ -1691,14 +1691,14 @@ def pgcmp(val1, val2, ignorecase = 0, num = 0):
             val1 = int(val1)
          if typ2 is str:
             typ2 = int
-            val2 = int(val2)  
+            val2 = int(val2)
       else:
          if typ1 != str:
             typ1 = str
-            val1 = str(val1)  
+            val1 = str(val1)
          if typ2 != str:
             typ2 = str
-            val2 = str(val2)  
+            val2 = str(val2)
 
    if typ1 is str:
       if num:
@@ -1788,13 +1788,13 @@ def psearch(lidx, hidx, key, list):
 # quicksort for pattern
 #
 def quicksort(srecs, lo, hi, desc, cnt, nums = None):
-   
+
    i = lo
    j = hi
    mrec = srecs[int((lo+hi)/2)]
 
    while True:
-      while cmp_records(srecs[i], mrec, desc, cnt, nums) < 0: i += 1 
+      while cmp_records(srecs[i], mrec, desc, cnt, nums) < 0: i += 1
       while cmp_records(srecs[j], mrec, desc, cnt, nums) > 0: j -= 1
       if i <= j:
          if i < j:
@@ -1808,11 +1808,11 @@ def quicksort(srecs, lo, hi, desc, cnt, nums = None):
    #recursion
    if lo < j: srecs = quicksort(srecs, lo, j, desc, cnt, nums)
    if i < hi: srecs = quicksort(srecs, i, hi, desc, cnt, nums)
-   
+
    return srecs
 
 def cmp_records(arec, brec, desc, cnt, nums):
-   
+
    for i in range(cnt):
       num = nums[i] if nums else 0
       ret = pgcmp(arec[i], brec[i], 0, num)

{rda_python_common-1.0.17.dist-info → rda_python_common-1.0.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 1.0.17
+Version: 1.0.21
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -11,6 +11,8 @@ Classifier: Development Status :: 5 - Production/Stable
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: psycopg2-binary
+Requires-Dist: rda-python-globus
 Dynamic: license-file
 
 # rda-python-common

{rda_python_common-1.0.17.dist-info → rda_python_common-1.0.21.dist-info}/RECORD

@@ -1,15 +1,15 @@
 rda_python_common/PgCMD.py,sha256=dsFL_hiAHRZ-BwncgZhxQ7p9O10ox9lUSXOIePjZ-PQ,20487
 rda_python_common/PgDBI.py,sha256=iGy0SQZsdVbSz2ewTIUK3e-RQBC5eQM9-_XUcFU1c50,74211
-rda_python_common/PgFile.py,sha256=nCZcSYdKaa1okVVEvj1Wx6OUbYNAbPWYedoCNvqevQU,98405
-rda_python_common/PgLOG.py,sha256=ZbXbOshjGwqYxpUAqTZqkViOBEZ_zVZF_t2SZ_q_AJg,54149
+rda_python_common/PgFile.py,sha256=756onQnWH_MK8llF0-8b6yV6KC1eY_eCPEkygxeJQQU,98460
+rda_python_common/PgLOG.py,sha256=DFCQgcmbU04H6Kz9X9nxuFdLA1VNRBQefRvK8LvO9Pw,54273
 rda_python_common/PgLock.py,sha256=12i84nsGBuifSyPnm8IR63LvHvRuVU573D5QKFlHdOI,22623
 rda_python_common/PgOPT.py,sha256=BAfuNi_1jEK-UyYVfPY_0f4b7KLhpcFM3BDShLRvT7w,55446
 rda_python_common/PgSIG.py,sha256=oDXcSnnzW6wKm_cSpFxQZWAo2hpPPTcuNDYhPSNQcRc,35607
 rda_python_common/PgSplit.py,sha256=QKPbF55m8KCTGmwVwL3uG_nuylCC4FSVfLuXeLjJHbE,8816
-rda_python_common/PgUtil.py,sha256=OpkNDwt694ebttNNUEDrrtDKyITZkx8g3uAI3b_FF4c,48737
+rda_python_common/PgUtil.py,sha256=OqESKCd72b9g8m8jwjPJhXDtPYlW6G8oSOhwChvz2Cg,48600
 rda_python_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rda_python_common-1.0.17.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_common-1.0.17.dist-info/METADATA,sha256=JOGDLYq0Z3xG0w-JTzBzmHg0WJCmIN0PBTd1DD8ZkuI,652
-rda_python_common-1.0.17.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rda_python_common-1.0.17.dist-info/top_level.txt,sha256=KVQmx7D3DD-jsiheqL8HdTrRE14hpRnZY5_ioMArA5k,18
-rda_python_common-1.0.17.dist-info/RECORD,,
+rda_python_common-1.0.21.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_common-1.0.21.dist-info/METADATA,sha256=aDnBKTHLFSlswpAfmoCtJ6aK_rbymOqV1PkG5b1zUnU,716
+rda_python_common-1.0.21.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rda_python_common-1.0.21.dist-info/top_level.txt,sha256=KVQmx7D3DD-jsiheqL8HdTrRE14hpRnZY5_ioMArA5k,18
+rda_python_common-1.0.21.dist-info/RECORD,,