rda-python-miscs 1.0.1__py3-none-any.whl

rda_python_miscs/__init__.py

@@ -0,0 +1 @@
+

rda_python_miscs/bashqsub.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title: bashqsub
+#    Author: Zaihua Ji, zji@ucar.edu
+#      Date: 11/19/2020
+#            2025-03-07 transferred to package rda_python_miscs from
+#            https://github.com/NCAR/rda-utility-programs.git
+#   Purpose: python script to submit a batch job on PBS node via bash script
+#
+#    Github: https://github.com/NCAR/rda-pythn-miscs.git
+#
+##################################################################################
+
+import os
+import sys
+import re
+from os import path as op
+from rda_python_common import PgLOG
+
+DEFMODS = {
+   'default' : "ncarenv,netcdf,ncl,nco,cdo,conda,grib-util,wgrib2",
+}
+
+DEFLIBS = {
+   'default' : "conda activate /glade/work/rdadata/conda-envs/pg-casper",
+}
+
+SWAPMODS = {
+}
+
+RESOURCES = {   # resource list for option -l
+   'walltime' : '6:00:00',   # if this is changed, change defpbstime in PgCheck.py too
+   'select' : '1:ncpus=1:mem=1gb'
+}
+
+SOPTIONS = {   # single-dash option values
+   'o' : None,    # will set to default if not provided
+   'e' : None,
+   'A' : "P43713000",
+   'q' : "rda@casper-pbs",
+#   'm' : 'a',
+   'm' : 'n',
+}
+
+#
+# main function to excecute this script
+#
+def main():
+
+   aname = 'bashqsub'
+   pname = 'rdaqsub'
+   PgLOG.set_help_path(__file__)
+   rdasub = PgLOG.BCHCMDS['PBS']
+   coptions = {'cmd' : None, 'cwd' : None, 'env' : None, 'mod' : None, 'res' : 'default'}       # customized options
+   copts = '|'.join(coptions)
+   option = None
+   dcount = 0
+   argv = sys.argv[1:]
+   if not argv: PgLOG.show_usage(aname)
+   PgLOG.PGLOG['LOGFILE'] = pname + ".log"
+   PgLOG.cmdlog("{} {}".format(aname, ' '.join(argv)))
+   if not PgLOG.valid_command(rdasub): PgLOG.pglog("{}: miss {} command to submit batch job".format(rdasub, PgLOG.PGLOG['PBSNAME']), PgLOG.LGWNEX)
+
+   while argv:
+      arg = argv.pop(0)
+      ms = re.match(r'^-(\w)$', arg)
+      if ms:
+         option = ms.group(1)
+         if option == "b":
+            PgLOG.PGLOG['BCKGRND'] = 1
+            option = None
+         else:
+            SOPTIONS[option] = ''
+         continue
+      ms = re.match(r'^-({})$'.format(copts), arg)
+      if ms:
+         option = ms.group(1)
+         if option == "env": option = 'v'
+         continue
+
+      if not option: PgLOG.pglog("{}: Value passed in without leading option for {}".format(arg, rdasub), PgLOG.LGEREX)
+      if arg.find(' ') > -1 and not re.match(r'^[\'\"].*[\'\"]$', arg):   # quote string with space but not quoted yet
+         if arg.find("'") > -1:
+            arg = '"{}"'.format(arg)
+         else:
+            arg = "'{}'".format(arg)
+
+      if option in coptions:
+         coptions[option] = arg
+         if option == "cmd": break
+      else:
+         SOPTIONS[option] = arg
+      option = None
+
+   if not coptions['cmd']: PgLOG.pglog(aname + ": specify command via option -cmd to run", PgLOG.LGWNEX)
+   args = PgLOG.argv_to_string(argv, 0)   # append command options
+   if not SOPTIONS['o']: SOPTIONS['o'] = "{}/{}/".format(PgLOG.PGLOG['LOGPATH'], pname)
+   if not SOPTIONS['e']: SOPTIONS['e'] = "{}/{}/".format(PgLOG.PGLOG['LOGPATH'], pname)
+   if 'N' not in SOPTIONS: SOPTIONS['N'] = op.basename(coptions['cmd'])
+   msg = "{}-{}{}".format(PgLOG.PGLOG['HOSTNAME'], PgLOG.PGLOG['CURUID'], PgLOG.current_datetime())
+
+   if coptions['cwd']:
+      if coptions['cwd'].find('$'): coptions['cwd'] = PgLOG.replace_environments(coptions['cwd'], '', PgLOG.LGWNEX)
+      msg += "-" + coptions['cwd']
+      os.chdir(coptions['cwd'])
+
+   cmd = PgLOG.valid_command(coptions['cmd'])
+   if not cmd and not re.match(r'^/', coptions['cmd']): cmd = PgLOG.valid_command('./' + coptions['cmd'])
+   if not cmd: PgLOG.pglog(coptions['cmd'] + ": Cannot find given command to run", PgLOG.LGWNEX)
+   if args: cmd += " " + args
+
+   sbuf = build_bash_script(cmd, coptions, rdasub)
+   PgLOG.pglog(sbuf, PgLOG.MSGLOG)
+   PgLOG.PGLOG['ERR2STD'] = ['bind mouting']
+   PgLOG.pgsystem(rdasub, PgLOG.LOGWRN, 6, sbuf)
+   PgLOG.PGLOG['ERR2STD'] = []
+
+   sys.exit(0)
+
+#
+# build bash script to submit a PBS batch job
+#
+def build_bash_script(cmd, coptions, rdasub):
+
+   buf = "#!/usr/bin/bash\n\n"   # qsub starting bash script
+
+   if 'l' in SOPTIONS: add_resources()
+   # add options to bash script for qsub
+   for option in SOPTIONS:
+      buf += "#PBS -" + option
+      if SOPTIONS[option]: buf += " {}".format(SOPTIONS[option])
+      buf += "\n"
+   for option in RESOURCES:
+      buf += "#PBS -l"
+      if RESOURCES[option]: buf += " {}={}".format(option, RESOURCES[option])
+      buf += "\n"
+
+   # always include the login user's bash resource file
+   homedir = "{}/{}".format(PgLOG.PGLOG['USRHOME'], PgLOG.PGLOG['CURUID'])
+   buf += "export HOME={}\n".format(homedir)
+   buf += "source /etc/profile.d/z00_modules.sh\n"
+   buf += "source /glade/u/apps/opt/conda/etc/profile.d/conda.sh\n"
+   buf += "source {}/.bashrc\n".format(homedir)
+   buf += "pwd; hostname; date\n"
+   buf += add_modules(coptions['res'], coptions['mod'])
+   buf += set_vm_libs(coptions['res'])
+   buf += "\necho {}\n{}\n\ndate\n".format(cmd, cmd)
+   
+   return buf
+
+#
+# check and add resource options 
+#
+def add_resources():
+
+   for res in re.split(',', SOPTIONS['l']):
+      ms = re.match(r'^([^=]+)=(.+)$', res)
+      if ms:
+         RESOURCES[ms.group(1)] = ms.group(2)
+      else:
+         PgLOG.pglog(res + ": use '=' to separate resource name & value", PgLOG.LGEREX)
+   del SOPTIONS['l']
+
+#
+# add module loads for modules provided
+#
+def add_modules(res, mods):
+
+   mbuf = "\n"
+   defmods = DEFMODS[res] if res in DEFMODS else DEFMODS['default']
+
+   dmods = re.split(',', defmods)
+   for dmod in dmods:
+      ms = re.match(r'^(.+)/', dmod)
+      smod = ms.group(1) if ms else dmod
+      if smod in SWAPMODS: mbuf += "module unload {}\n".format(SWAPMODS[smod])
+      mbuf += "module load {}\n".format(dmod)
+
+   if mods:
+      amods = re.split(',', mods)
+      for amod in amods:
+         if re.match(r'^/', amod):
+            mbuf += "module use {}\n".format(amod)
+         else:
+            ms = re.match(r'^(.+)/', amod)
+            smod = ms.group(1) if ms else amod
+            if smod in dmods: continue
+            if smod in SWAPMODS: mbuf += "module unload {}\n".format(SWAPMODS[smod])
+            mbuf += "module load {}\n".format(amod)
+
+   return mbuf
+
+#
+# set virtual machine libraries
+#
+def set_vm_libs(res):
+
+   deflibs = DEFLIBS[res] if res in DEFLIBS else DEFLIBS['default']
+   if not deflibs: return ''
+   
+   dlibs = re.split(',', deflibs)
+   libbuf = "\n"
+   for dlib in dlibs:
+      libbuf += dlib + "\n"
+
+   return libbuf
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_miscs/bashqsub.usg

@@ -0,0 +1,64 @@
+
+ To submit a job execution as a batch job in bash script on a PBS node via 'qsub'.
+ 
+ Usage: rdaqsub [qsub-options] [-cwd WorkDir] [-env EnvironmentPairs]   \
+                [-mod Modules] [-res Reservation] -cmd Command [cmd-options]
+
+      - qsub-options, accepts options that can be passed to 'qsub' in a bash script.
+          Check qsub man page for help on the PBS batch options. Here is the list of
+          qsub options that are included at default:
+            -o LOGPATH/rdaqsub/
+            -e LOGPATH/rdaqsub/
+            -A P43713000
+            -m a
+            -q regular
+            -l walltime=6:00:00,select=1:node=1:mem=1gb
+
+      - Option -cwd, set the working directory for the Command to be executed. If
+          it is not specified, it defaults to the current directory where qsub
+          command is executed.  
+ 
+      - Option -env, set environment name/value pairs separated by ',' dynamically
+          for the Command to be executed. This is equivilent to qsub-option -v.
+
+      - Option -mod, set module list separated  by ',' for loading modules to execute
+          the command.
+
+      - Option -res, set Reservation name to load modules and start VM libs;
+
+      - Option -cmd, mandatory option to lead a Command to be executed;
+
+      - cmd-options, specifies options that can be passed to the Command.
+
+A bash script example:
+#!/usr/bin/bash
+
+#PBS -o /gpfs/u/home/rdadata/dssdb/log/rdaqsub/
+#PBS -e /gpfs/u/home/rdadata/dssdb/log/rdaqsub/
+#PBS -A P43713000
+#PBS -q rda@casper-pbs
+#PBS -m n
+#PBS -N dsrqst
+#PBS -l walltime=1:00:00
+#PBS -l select=1:ncpus=1:mem=1gb
+export HOME=/gpfs/u/home/zji
+source /etc/profile.d/z00_modules.sh
+source /glade/u/apps/opt/conda/etc/profile.d/conda.sh
+source /gpfs/u/home/zji/.bashrc
+pwd; hostname; date
+
+module load ncarenv
+module load netcdf
+module load ncl
+module load nco
+module load cdo
+module load conda
+module load grib-util
+module load wgrib2
+
+conda activate /glade/work/rdadata/conda-envs/pg-casper
+
+echo /gpfs/u/home/rdadata/bin/dsrqst d277000 PR -RI 750748 -b -d
+/gpfs/u/home/rdadata/bin/dsrqst d277000 PR -RI 750748 -b -d
+
+date

rda_python_miscs/pgwget.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : pgwget
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 12/02/2020
+#            2025-03-10 transferred to package rda_python_miscs from
+#            https://github.com/NCAR/rda-utility-programs.git
+#   Purpose : wrapper to wget to get a file with wildcard in name
+#
+#    Github: https://github.com/NCAR/rda-python-miscs.git
+#
+##################################################################################
+
+import sys
+import re
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+
+OPTIONS = {
+   'OP' : "-np -nH -nd -m -e robots=off --no-check-certificate",
+   'UL' : None,
+   'RN' : None,
+   'FN' : None,
+   'FC' : 1,
+   'SM' : 0,
+   'MC' : 0,
+   'CN' : 0,
+   'CR' : 0,
+   'EX' : None,
+   'JC' : 'cat'
+}
+
+#
+# main function to excecute this script
+#
+def main():
+
+   option = None
+   JCS = ['cat', 'tar', 'first', 'last']
+   options = '|'.join(OPTIONS)
+   argv = sys.argv[1:] 
+   PgLOG.PGLOG['LOGFILE'] = "pgwget.log"
+   
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+         option = None
+         continue
+      ms = re.match(r'^-({})$'.format(options), arg, re.I)
+      if ms:
+         option = ms.group(1).upper()
+         if re.match(r'^(CN|CR|SM)$', option):
+            OPTIONS[option] = 1
+            option = None
+         continue   
+      if re.match(r'^-.*$', arg): PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGEREX)
+      if not option: PgLOG.pglog(arg + ": Value passed in without leading option", PgLOG.LGEREX)
+
+      if option == 'JC' and arg not in JCS:
+         PgLOG.pglog(arg + ": Joining Command must be one of {}".format(JCS), PgLOG.LGEREX)
+      OPTIONS[option] = int(arg) if re.match(r'^(FC|MC)$', option) else arg
+      option = None
+
+   if not (OPTIONS['UL'] and OPTIONS['RN']):
+      print("Usage: pgwget [-CN] [-CR] [-FC FileCount] [-JC JoinCommand] [-MC MinFileCount] [-FN FileName] -UL WebURL -RN RootFileName [-EX FileNameExtension]")
+      print("   Provide at least WebURL and RootFileName to wget file(s)")
+      print("   Option -CN - check new file if presents")
+      print("   Option -CR - clean the downloaded remote file(s) if presents")
+      print("   Option -FC - number of files to be valid download; defaults to 1")
+      print("   Option -JC - file joining command, it defaults to cat, could be tar, or last/first to choose the last/first one")
+      print("   Option -SM - Show wget dumping message; defaults to False")
+      print("   Option -MC - minimal number of files to be valid download; defaults to -FC")
+      print("   Option -FN - file name to be used if successful download; defaults to RootFileName.FileNameExtension")
+      print("   Option -OP - options used by wget, defaults to '-np -nH -nd -m -e robots=off'")
+      print("   Option -UL - (mandatory) WebURL with path")
+      print("   Option -RN - (mandatory) the root portion of the remote file name to be downloaded")
+      print("   Option -EX - file name extension to be used.")
+      sys.exit(0)
+
+   PgLOG.cmdlog("pgwget " + ' '.join(argv))
+   if not OPTIONS['MC']: OPTIONS['MC'] = OPTIONS['FC']
+   if not OPTIONS['SM']: OPTIONS['OP'] += ' -q'
+   download_wildcard_files()   
+   PgLOG.cmdlog()
+
+   sys.exit(0)
+
+#
+# download one or multiple remote files via wget; concat files to a single one if multiple
+#
+def download_wildcard_files():
+
+   deleted = 0
+   if OPTIONS['FN']:
+      dfile = OPTIONS['FN']
+   else:
+      dfile = OPTIONS['RN']
+      if OPTIONS['EX']: dfile += "." + OPTIONS['EX']
+
+   dinfo = PgFile.check_local_file(dfile, 1)
+   if dinfo and not OPTIONS['CN']:
+      return PgLOG.pglog("{}: file dowloaded already ({} {})".format(dfile, dinfo['date_modified'], dinfo['time_modified']), PgLOG.LOGWRN)
+
+   build = 0 if dinfo else 1
+   wfile = OPTIONS['RN'] + "*"
+   if OPTIONS['EX']: wfile += "." + OPTIONS['EX']
+   dlist = PgFile.local_glob(wfile, 1)
+   if dfile in dlist and dinfo:
+      del dlist[dfile]
+      deleted = 1
+   dcnt = len(dlist)
+
+   if OPTIONS['CN'] or dcnt < OPTIONS['FC']:
+      cmd = "wget {} {} -A '{}'".format(OPTIONS['OP'], OPTIONS['UL'], wfile)
+      PgLOG.pgsystem(cmd, PgLOG.LOGWRN, 7)
+      nlist = PgFile.local_glob(wfile, 1)
+      if dfile in nlist and dinfo:
+         del nlist[dfile]
+         deleted = 1
+      ncnt = len(nlist)
+   else:
+      nlist = dlist
+      ncnt = dcnt
+
+   if ncnt == 0:
+      if deleted:
+         return PgLOG.pglog("{}: File dowloaded on {}".format(dfile, OPTIONS['UL']), PgLOG.LOGWRN)
+      else:
+         return PgLOG.pglog("{}: NO file to dowload on {}".format(dfile, OPTIONS['UL']), PgLOG.LOGWRN)
+   elif ncnt < OPTIONS['MC']:
+      return PgLOG.pglog("{}: NOT ready, only {} of {} files dowloaded".format(dfile, ncnt, OPTIONS['MC']), PgLOG.LOGWRN)
+
+   rfiles = sorted(nlist)
+   size = skip = 0
+   for i in range(ncnt):
+      rfile = rfiles[i]
+      rinfo = nlist[rfile]
+      size += rinfo['data_size']
+      if dinfo and PgUtil.cmptime(dinfo['date_modified'], dinfo['time_modified'], rinfo['date_modified'], rinfo['time_modified']) >= 0:
+         PgLOG.pglog("{}: Not newer than {}".format(rfile, dfile), PgLOG.LOGWRN)
+         skip += 1
+      elif rfile not in dlist:
+         build = 1
+      elif PgFile.compare_file_info(dlist[rfile], rinfo) > 0:
+         PgLOG.pglog("{}: Newer file dowloaded from {}".format(rfile, OPTIONS['UL']), PgLOG.LOGWRN)
+         build = 1
+      else:
+         PgLOG.pglog("{}: No newer file found on ".format(rfile, OPTIONS['UL']), PgLOG.LOGWRN)
+
+   if skip == ncnt: return 0
+
+   if not (build or size == dinfo['data_size']): build = 1
+   if not build: return PgLOG.pglog(dfile + ": Use existing file", PgLOG.LOGWRN)
+
+   if OPTIONS['JC'] == 'cat':
+      for i in range(ncnt):
+         rfile = rfiles[i]
+         if i == 0:
+            if dfile != rfile: PgFile.local_copy_local(dfile, rfile, PgLOG.LOGWRN)
+         else:
+            PgLOG.pgsystem("cat {} >> {}".format(rfile, dfile), PgLOG.LOGWRN, 5)
+         if OPTIONS['CR'] and dfile != rfile: PgLOG.pgsystem("rm -f " + rfile, PgLOG.LOGWRN, 5)
+   elif OPTIONS['JC'] == 'tar':
+      topt = 'c'
+      for i in range(ncnt):
+         rfile = rfiles[i]
+         PgLOG.pgsystem("tar -{}vf {} {}".format(topt, dfile, rfile), PgLOG.LOGWRN, 5)
+         topt = 'u'
+         if OPTIONS['CR']: PgLOG.pgsystem("rm -f " + rfile, PgLOG.LOGWRN, 5)
+   else:
+      didx = 0 if OPTIONS['JC'] == 'first' else (ncnt - 1)
+      PgLOG.pgsystem("mv {} {}".format(rfiles[didx], dfile), PgLOG.LOGWRN, 5)
+      if OPTIONS['CR']:
+         for i in range(ncnt):
+            if i == didx: continue
+            PgLOG.pgsystem("rm -f " + rfiles[i], PgLOG.LOGWRN, 5)
+
+   return 1
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_miscs/rdacp.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title: rdacp
+#    Author: Zaihua Ji, zji@ucar.edu
+#      Date: 10/24/2020
+#            2025-03-10 transferred to package rda_python_miscs from
+#            https://github.com/NCAR/rda-utility-programs.git
+#   Purpose: copy files locally and remotely by 'rdadata'
+#
+#    Github: https://github.com/NCAR/rda-python-miscs.git
+#
+##################################################################################
+#
+import re
+import os
+import sys
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgDBI
+from rda_python_common import PgFile
+
+RDACP = {
+   'fh' : None,   # from host name, default to localhost
+   'th' : None,   # to host name, defaul to localhost
+   'fb' : None,   # from bucket name for a from file in Object Store
+   'tb' : None,   # to bucket name for a to file in Object Store
+   'fp' : None,   # from Globus endpoint
+   'tp' : None,   # to Globus endpoint
+   'f' : [],      # from file names
+   't' : None,    # to file name
+   'r' : 0,       # 1 if recursive all
+   'R' : 0,       # > 0 to set recursive limit
+   'F' : 0o664,   # to file mode, default to 664
+   'D' : 0o775,   # to directory mode, default to 775
+}
+
+CINFO = {
+   'tcnt' : 0,
+   'htcnt' : 0,
+   'cpflag' : 0,    # 1 file only, 2 directory only, 3 both
+   'cpstr' : ['', 'Files', 'Directories', 'Files/Directories'],
+   'fpath' : None,
+   'tpath' : None,
+   'fhost' : '',
+   'thost' : '',
+   'curdir' : os.getcwd()
+}
+
+#
+# main function to run the application
+#
+def main():
+
+   dohelp = 0
+   argv = sys.argv[1:]
+   PgDBI.dssdb_dbname()
+   PgLOG.set_suid(PgLOG.PGLOG['EUID'])
+   PgLOG.set_help_path(__file__)
+   PgLOG.PGLOG['LOGFILE'] = "rdacp.log"   # set different log file
+   PgLOG.cmdlog("rdacp {} ({})".format(' '.join(argv), CINFO['curdir']))
+   defopt = option = 'f'
+   for arg in argv:
+      if re.match(r'-(h|-help)$', arg, re.I):
+         dohelp = 1
+         continue
+      ms = re.match(r'-(\w+)$', arg)
+      if ms:
+         option = ms.group(1)
+         if option not in RDACP: PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGEREX)
+         if option == 'r':
+            RDACP['r'] = 1
+            option = None
+         continue
+      if not option: PgLOG.pglog(arg + ": Value provided without option", PgLOG.LGEREX)
+      if option == "f":
+         RDACP['f'].append(arg)
+         defopt = None
+      else:
+         if option == 'R':
+            RDACP[option] = int(arg)
+         elif 'FD'.find(option) > -1:
+            RDACP[option] = PgLOG.base2int(arg, 8)
+         else:
+            RDACP[option] = arg
+            if option == 'th':
+               CINFO['thost'] = arg + '-'
+            elif option == 'fh':
+               CINFO['fhost'] = arg + '-'
+         option = defopt
+            
+   if dohelp or not RDACP['f']: PgLOG.show_usage("rdacp")
+   PgDBI.validate_decs_group('rdacp', PgLOG.PGLOG['CURUID'], 1)
+   if not RDACP['R'] and RDACP['r']: RDACP['R'] = 1000
+   if not RDACP['t']:
+      CINFO['tpath'] = RDACP['t'] = "."
+   else:
+      ms = re.match(r'^(.+)/$', RDACP['t'])
+      if ms:
+         CINFO['tpath'] = ms.group(1)
+      else:
+         tinfo = PgFile.check_rda_file(RDACP['t'], RDACP['th'], 0, PgLOG.LGWNEX)
+         if tinfo and tinfo['isfile'] == 0: CINFO['tpath'] = RDACP['t']
+   PgLOG.PGLOG['FILEMODE'] = RDACP['F']
+   PgLOG.PGLOG['EXECMODE'] = RDACP['D']
+
+   fcnt = len(RDACP['f'])
+   if not CINFO['tpath'] and fcnt > 1:
+      PgLOG.pglog("{}{}: Cannot copy multiple files to a single file".format(CINFO['thost'], RDACP['t']), PgLOG.LGEREX)
+   if RDACP['th'] and RDACP['fh'] and RDACP['th'] == RDACP['fh'] and RDACP['fh'] != 'HPSS':
+      PgLOG.pglog(RDACP['fh'] + ": Cannot copy file onto the same host", PgLOG.LGEREX)
+   if RDACP['fb']:
+      PgLOG.PGLOG['OBJCTBKT'] = RDACP['fb']
+   elif RDACP['tb']:
+      PgLOG.PGLOG['OBJCTBKT'] = RDACP['tb']
+   if RDACP['fp']:
+      PgLOG.PGLOG['BACKUPEP'] = RDACP['fp']
+   elif RDACP['tp']:
+      PgLOG.PGLOG['BACKUPEP'] = RDACP['tp']
+
+   copy_top_list(RDACP['f'])
+
+   hinfo = ''
+   if RDACP['fh']: hinfo += " From " + RDACP['fh']
+   if RDACP['th']: hinfo += " To " + RDACP['th']
+
+   if CINFO['tcnt'] > 1:
+      PgLOG.pglog("Total {} {} copiled{}".format(CINFO['tcnt'], CINFO['cpstr'][CINFO['cpflag']], hinfo), PgLOG.LOGWRN)
+   elif CINFO['tcnt'] == 0 and not RDACP['fh']:
+      PgLOG.pglog("{}: No File copied{}".format((CINFO['fpath'] if CINFO['fpath'] else CINFO['curdir']), hinfo), PgLOG.LOGWRN)
+   
+   PgLOG.cmdlog()
+   PgLOG.pgexit(0)
+
+#
+# display the top level list
+#
+def copy_top_list(files):
+   
+   for file in files:
+      if RDACP['th'] and not PgUtil.pgcmp(RDACP['th'], PgLOG.PGLOG['BACKUPNM'], 1):
+         info = PgFile.check_globus_file(file, 'rda-glade', 0, PgLOG.LGWNEX)
+      else:
+         info = PgFile.check_rda_file(file, RDACP['fh'], 0, PgLOG.LGWNEX)
+      if not info:
+         PgLOG.pglog("{}{}: {}".format(CINFO['fhost'], file, PgLOG.PGLOG['MISSFILE']), PgLOG.LOGERR)
+         continue
+
+      dosub = 0
+      if info['isfile'] == 0:
+         CINFO['cpflag'] |= 2
+         if not CINFO['tpath']:
+            PgLOG.pglog("{}{}: Cannot copy directory to a single file".format(CINFO['fhost'], file), PgLOG.LGEREX)
+
+         if re.search(r'/$', file):
+            dosub = 1   # copy the file under this directory if it is ended by '/'
+            file = re.sub(r'/$', '', file)
+      else:
+         CINFO['cpflag'] |= 1
+
+      if not re.match(r'^/', file): file = PgLOG.join_paths(CINFO['curdir'], file)
+      CINFO['fpath'] = (file if dosub else op.dirname(file)) + "/"
+      if info['isfile']:
+         CINFO['tcnt'] += copy_file(file, info['isfile'])
+      elif dosub or RDACP['R']:
+         flist = PgFile.rda_glob(file, RDACP['fh'], 0, PgLOG.LGWNEX)
+         if flist: copy_list(flist, 1, file)
+      else:
+         PgLOG.pglog("{}{}: Add option -r to copy directory".format(CINFO['fhost'], file), PgLOG.LGEREX)
+
+#
+# recursively copy directory/file
+#
+def copy_list(tlist, level, cdir):
+
+   fcnt = 0
+
+   for file in tlist:
+      if tlist[file]['isfile']:
+         fcnt += copy_file(file, tlist[file]['isfile'])
+         CINFO['cpflag'] |= (1 if tlist[file]['isfile'] else 2)
+      elif level < RDACP['R']:
+         flist = PgFile.rda_glob(file, RDACP['fh'], 0, PgLOG.LGWNEX)
+         if flist: copy_list(flist, level+1, file)
+
+   if fcnt > 1:   # display sub count if two or more files are copied
+      PgLOG.pglog("{}{}: {} {} copied from directory".format(CINFO['fhost'], cdir, fcnt, CINFO['cpstr'][CINFO['cpflag']]), PgLOG.LOGWRN)
+   CINFO['tcnt'] += fcnt
+
+#
+# copy one file each time
+#
+def copy_file(fromfile, isfile):
+
+   if CINFO['tpath']:
+      fname = re.sub(r'^{}'.format(CINFO['fpath']), '', fromfile)
+      if isfile:
+         tofile = PgLOG.join_paths(CINFO['tpath'], fname)
+      else:
+         tofile = CINFO['tpath'] + '/'
+   else:
+      tofile = RDACP['t']
+ 
+   return (1 if PgFile.copy_rda_file(tofile, fromfile, RDACP['th'], RDACP['fh'], PgLOG.LGWNEX) else 0)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()
+