rda-python-miscs 1.0.7__tar.gz → 2.0.1__tar.gz

{rda_python_miscs-1.0.7/src/rda_python_miscs.egg-info → rda_python_miscs-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 1.0.7
+Version: 2.0.1
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs

{rda_python_miscs-1.0.7 → rda_python_miscs-2.0.1}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_miscs"
-version = "1.0.7"
+version = "2.0.1"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]
@@ -24,16 +24,10 @@ dependencies = [
   "rda_python_setuid",
 ]
 
-[tool.setuptools]
-include-package-data = true
-
-[tool.setuptools.packages.find]
-where = ["src"]
-
-[tool.setuptools.package-data]
-"rda_python_miscs" = ["bashqsub.usg", "tcshqsub.usg", "rdasub.usg", "rdacp.usg",
-                      "rdakill.usg", "rdals.usg", "rdamod.usg", "rdaown.usg",
-                      "rdaps.usg", "rdazip.usg"]
+[tool.pytest.ini_options]
+pythonpath = [
+  "src"
+]
 
 [project.urls]
 "Homepage" = "https://github.com/NCAR/rda-python-miscs"

rda_python_miscs-1.0.7/src/rda_python_miscs/bashqsub.py → rda_python_miscs-2.0.1/src/rda_python_miscs/bash_qsub.py

@@ -24,7 +24,7 @@ DEFMODS = {
 }
 
 DEFLIBS = {
-   'default' : "conda activate /glade/work/gdexdata/conda-envs/pg-casper",
+   'default' : "conda activate /glade/work/gdexdata/conda-envs/pg-gdex",
 }
 
 SWAPMODS = {
@@ -102,7 +102,7 @@ def main():
    msg = "{}-{}{}".format(PgLOG.PGLOG['HOSTNAME'], PgLOG.PGLOG['CURUID'], PgLOG.current_datetime())
 
    if coptions['cwd']:
-      if coptions['cwd'].find('$'): coptions['cwd'] = PgLOG.replace_environments(coptions['cwd'], '', PgLOG.LGWNEX)
+      if 's' in coptions['cwd']: coptions['cwd'] = PgLOG.replace_environments(coptions['cwd'], '', PgLOG.LGWNEX)
       msg += "-" + coptions['cwd']
       os.chdir(coptions['cwd'])
 

rda_python_miscs-2.0.1/src/rda_python_miscs/bashqsub.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+##################################################################################
+#     Title: bashqsub
+#    Author: Zaihua Ji, zji@ucar.edu
+#      Date: 11/19/2020
+#            2025-03-07 transferred to package rda_python_miscs from
+#            https://github.com/NCAR/rda-utility-programs.git
+#            2025-12-29 convert to class BashQsub
+#   Purpose: python script to submit a batch job on PBS node via bash script
+#    Github: https://github.com/NCAR/rda-pythn-miscs.git
+##################################################################################
+import os
+import sys
+import re
+from os import path as op
+from rda_python_common.pg_log import PgLOG
+
+class BashQsub(PgLOG):
+
+   def __init__(self):
+      super().__init__()
+      self.DEFMODS = {
+         'default': "ncarenv,netcdf,ncl,nco,cdo,conda,grib-util,wgrib2"
+      }
+      self.DEFLIBS = {
+         'default': "conda activate /glade/work/gdexdata/conda-envs/pg-gdex"
+      }
+      self.SWAPMODS = {}
+      self.RESOURCES = {   # resource list for option -l
+         'walltime': '6:00:00',   # if this is changed, change defpbstime in PgCheck.py too
+         'select': '1:ncpus=1:mem=1gb'
+      }
+      self.SOPTIONS = {   # single-dash option values
+         'o': None,    # will set to default if not provided
+         'e': None,
+         'A': "P43713000",
+         'q': "gdex@casper-pbs",
+      #   'm': 'a',
+         'm': 'n',
+      }
+      self.coptions = {'cmd': None, 'cwd': None, 'env': None, 'mod': None, 'res': 'default'}       # customized options
+      self.gdexsub = self.BCHCMDS['PBS']
+      self.args = None
+
+   # function to readparameters
+   def read_parameters(self):
+      aname = 'bashqsub'
+      pname = 'gdexqsub'
+      self.set_help_path(__file__)
+      copts = '|'.join(self.coptions)
+      option = None
+      argv = sys.argv[1:]
+      if not argv: self.show_usage(aname)
+      self.PGLOG['LOGFILE'] = pname + ".log"
+      self.cmdlog("{} {}".format(aname, ' '.join(argv)))
+      if not self.valid_command(self.gdexsub): self.pglog("{}: miss {} command to submit batch job".format(self.gdexsub, self.PGLOG['PBSNAME']), self.LGWNEX)
+      while argv:
+         arg = argv.pop(0)
+         ms = re.match(r'^-(\w)$', arg)
+         if ms:
+            option = ms.group(1)
+            if option == "b":
+               self.PGLOG['BCKGRND'] = 1
+               option = None
+            else:
+               self.SOPTIONS[option] = ''
+            continue
+         ms = re.match(r'^-({})$'.format(copts), arg)
+         if ms:
+            option = ms.group(1)
+            if option == "env": option = 'v'
+            continue
+         if not option: self.pglog("{}: Value passed in without leading option for {}".format(arg, self.gdexsub), self.LGEREX)
+         if arg.find(' ') > -1 and not re.match(r'^[\'\"].*[\'\"]$', arg):   # quote string with space but not quoted yet
+            if arg.find("'") > -1:
+               arg = '"{}"'.format(arg)
+            else:
+               arg = "'{}'".format(arg)
+         if option in self.coptions:
+            self.coptions[option] = arg
+            if option == "cmd": break
+         else:
+            self.SOPTIONS[option] = arg
+         option = None
+      self.args = self.argv_to_string(argv, 0)   # append command options
+      if not self.coptions['cmd']: self.pglog(aname + ": specify command via option -cmd to run", self.LGWNEX)
+      if not self.SOPTIONS['o']: self.SOPTIONS['o'] = "{}/{}/".format(self.PGLOG['LOGPATH'], pname)
+      if not self.SOPTIONS['e']: self.SOPTIONS['e'] = "{}/{}/".format(self.PGLOG['LOGPATH'], pname)
+      if 'N' not in self.SOPTIONS: self.SOPTIONS['N'] = op.basename(self.coptions['cmd'])
+      if self.coptions['cwd']:
+         if 's' in self.coptions['cwd']: self.coptions['cwd'] = self.replace_environments(self.coptions['cwd'], '', self.LGWNEX)
+         os.chdir(self.coptions['cwd'])
+
+   # function to start actions
+   def start_actions(self):
+      cmd = self.valid_command(self.coptions['cmd'])
+      if not cmd and not re.match(r'^/', self.coptions['cmd']): cmd = self.valid_command('./' + self.coptions['cmd'])
+      if not cmd: self.pglog(self.coptions['cmd'] + ": Cannot find given command to run", self.LGWNEX)
+      if self.args: cmd += " " + self.args
+      sbuf = self.build_bash_script(cmd)
+      self.pglog(sbuf, self.MSGLOG)
+      self.PGLOG['ERR2STD'] = ['bind mouting']
+      self.pgsystem(self.gdexsub, self.LOGWRN, 6, sbuf)
+      self.PGLOG['ERR2STD'] = []
+   
+   # build bash script to submit a PBS batch job
+   def build_bash_script(self, cmd):
+      buf = "#!/usr/bin/bash\n\n"   # qsub starting bash script
+      if 'l' in self.SOPTIONS: self.add_resources()
+      # add options to bash script for qsub
+      for option in self.SOPTIONS:
+         buf += "#PBS -" + option
+         if self.SOPTIONS[option]: buf += " {}".format(self.SOPTIONS[option])
+         buf += "\n"
+      for option in self.RESOURCES:
+         buf += "#PBS -l"
+         if self.RESOURCES[option]: buf += " {}={}".format(option, self.RESOURCES[option])
+         buf += "\n"
+      # always include the login user's bash resource file
+      homedir = "{}/{}".format(self.PGLOG['USRHOME'], self.PGLOG['CURUID'])
+      buf += "export HOME={}\n".format(homedir)
+      buf += "source /etc/profile.d/z00_modules.sh\n"
+      buf += "source /glade/u/apps/opt/conda/etc/profile.d/conda.sh\n"
+      buf += "source {}/.bashrc\n".format(homedir)
+      buf += "pwd; hostname; date\n"
+      buf += self.add_modules(self.coptions['res'], self.coptions['mod'])
+      buf += self.set_vm_libs(self.coptions['res'])
+      buf += "\necho {}\n{}\n\ndate\n".format(cmd, cmd)
+      return buf
+
+   # check and add resource options 
+   def add_resources(self):
+      for res in re.split(',', self.SOPTIONS['l']):
+         ms = re.match(r'^([^=]+)=(.+)$', res)
+         if ms:
+            self.RESOURCES[ms.group(1)] = ms.group(2)
+         else:
+            self.pglog(res + ": use '=' to separate resource name & value", self.LGEREX)
+      del self.SOPTIONS['l']
+
+   # add module loads for modules provided
+   def add_modules(self, res, mods):
+      mbuf = "\n"
+      defmods = self.DEFMODS[res] if res in self.DEFMODS else self.DEFMODS['default']
+      dmods = re.split(',', defmods)
+      for dmod in dmods:
+         ms = re.match(r'^(.+)/', dmod)
+         smod = ms.group(1) if ms else dmod
+         if smod in self.SWAPMODS: mbuf += "module unload {}\n".format(self.SWAPMODS[smod])
+         mbuf += "module load {}\n".format(dmod)
+      if mods:
+         amods = re.split(',', mods)
+         for amod in amods:
+            if re.match(r'^/', amod):
+               mbuf += "module use {}\n".format(amod)
+            else:
+               ms = re.match(r'^(.+)/', amod)
+               smod = ms.group(1) if ms else amod
+               if smod in dmods: continue
+               if smod in self.SWAPMODS: mbuf += "module unload {}\n".format(self.SWAPMODS[smod])
+               mbuf += "module load {}\n".format(amod)
+      return mbuf
+
+   # set virtual machine libraries
+   def set_vm_libs(self, res):
+      deflibs = self.DEFLIBS[res] if res in self.DEFLIBS else self.DEFLIBS['default']
+      if not deflibs: return ''
+      dlibs = re.split(',', deflibs)
+      libbuf = "\n"
+      for dlib in dlibs:
+         libbuf += dlib + "\n"
+      return libbuf
+
+# main function to excecute this script
+def main():
+   object = BashQsub()
+   object.read_parameters()
+   object.start_actions()
+   object.pgexit(0)
+
+# call main() to start program
+if __name__ == "__main__": main()

rda_python_miscs-2.0.1/src/rda_python_miscs/gdexls.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+##################################################################################
+#     Title: gdexls
+#    Author: Zaihua Ji, zji@ucar.edu
+#      Date: 10/20/2020
+#            2025-03-10 transferred to package rda_python_miscs from
+#            https://github.com/NCAR/rda-utility-programs.git
+#            2025-09-21 copied from rdals to gdexls
+#   Purpose: list files/directories in a local directory and show additional
+#            information recorded in GDEXDB if any
+#    Github: https://github.com/NCAR/rda-python-miscs.git
+##################################################################################
+import re
+import os
+import sys
+import glob
+from os import path as op
+from rda_python_common.pg_split import PgSplit
+
+class GdexLs(PgSplit):
+
+   def __init__(self):
+      super().__init__()
+      # define some constants for gdexls actions
+      self.DIDX = 3   # description column index
+      self.CLMT = 500   # reformat list if count reach this limit
+      self.WIDTHS = [0, 0, 0]   # WIDTHS for formated display
+      self.ALIGNS = [0, 1, 1]   # alignment, 0 - left; 1 - right
+      self.GDEXLS = {
+         'd': 0,     # 1 to list directory information only
+         'f': 0,     # 1 to list file information only
+         'N': 0,     # 1 to list files unformatted
+         'r': 0,     # 1 if recursive all
+         'R': 0,     # > 0 to set recursive limit
+         'D': None,  # specify delimiting symbols, default to '  '
+      }
+      self.LINFO = {
+         'files': [],
+         'curdir': None,
+         'tpath': None,
+         'dhome': None,
+         'dsid': None,
+         'dcnt': 0,
+         'gcnt': 0,
+         'fcnt': 0,
+         'pcnt': 0,
+         'pgrecs': []
+      }
+
+   # function to read parameters
+   def read_parameters(self):
+      self.set_help_path(__file__)
+      self.PGLOG['LOGFILE'] = "gdexls.log"   # set different log file
+      self.LINFO['curdir'] = self.get_real_path(os.getcwd())
+      argv = sys.argv[1:]
+      self.pglog("gdexls {} ({})".format(' '.join(argv), self.LINFO['curdir']))
+      option = defopt = 'l'
+      for arg in argv:
+         if re.match(r'-(h|-*help|\?)$', arg): self.show_usage("gdexls")
+         ms = re.match(r'-(\w)$', arg)
+         if ms:
+            option = ms.group(1)
+            if option not in self.GDEXLS: self.pglog(arg + ": Unknown Option", self.LGEREX)
+            if 'dfNr'.find(option) > -1:
+               self.GDEXLS[option] = 1
+               option = defopt
+            continue
+         if not option: self.pglog(arg + ": Value provided without option", self.LGEREX)
+         if option == 'l':
+            self.LINFO['files'].append(self.get_real_path(arg))
+            defopt = None
+         else:
+            if option == 'R':
+               self.GDEXLS[option] = int(arg)
+            else:
+               self.GDEXLS[option] = arg
+            option = defopt
+
+   # functio to start actions
+   def start_actions(self):   
+      self.view_dbinfo()
+      if not self.LINFO['files']:
+         self.LINFO['files'] = sorted(glob.glob('*'))   # view all files in current directory
+         if not self.LINFO['files']:
+            sys.stderr.write(self.LINFO['curdir'] + ": Empty directory\n")
+            self.pgexit(1)
+   
+      if not (self.GDEXLS['d'] or self.GDEXLS['f']):
+         self.GDEXLS['d'] = self.GDEXLS['f'] = 1   # list both directories and files as default
+      if not self.GDEXLS['D']: self.GDEXLS['D'] = '|' if self.GDEXLS['N'] else "  "    # default delimiter for no format display
+      if not self.GDEXLS['R'] and self.GDEXLS['r']: self.GDEXLS['R'] = 1000
+   
+      self.display_top_list(self.LINFO['files'])   # display or cache file/directory list
+      if self.LINFO['pcnt'] > 0: self.display_format_list()   # if some left over
+      if (self.LINFO['dcnt'] + self.LINFO['gcnt'] + self.LINFO['fcnt']) > 1:
+         msg = ''
+         if self.LINFO['dcnt'] > 0:
+            s = 's' if self.LINFO['dcnt'] > 1 else ''
+            msg += "{} Dataset{}".format(self.LINFO['dcnt'], s)
+         if self.LINFO['gcnt'] > 0:
+            s = 's' if self.LINFO['gcnt'] > 1 else ''
+            if msg: msg += " & "
+            msg += "{} Group{}".format(self.LINFO['gcnt'], s)
+         if self.LINFO['fcnt'] > 0:
+            s = 's' if self.LINFO['fcnt'] > 1 else ''
+            if msg: msg += " & "
+            msg += "{} File{}".format(self.LINFO['fcnt'], s)
+         print("Total {} displayed".format(msg))
+      elif (self.LINFO['dcnt'] + self.LINFO['gcnt'] + self.LINFO['fcnt']) == 0:
+         sys.stderr.write((self.LINFO['tpath'] if self.LINFO['tpath'] else self.LINFO['curdir']) + ": No GDEX data information found\n")
+         self.pgexit(1)
+
+   # display the top level list
+   def display_top_list(self, files):
+      for file in files:
+         if not op.exists(file):
+            sys.stderr.write(file + ": NOT exists\n")
+            continue
+         isdir = 1 if op.isdir(file) else 0
+         display = 1 
+         if isdir and re.search(r'/$', file):
+            display = 0   # do not display the directory info if it is ended by '/'
+            file = re.sub(r'/$', '', file)
+         if not re.match(r'^/', file): file = self.join_paths(self.LINFO['curdir'], file)
+         self.LINFO['tpath'] = (op.dirname(file) if display else file) + "/"
+         if display: self.display_line(file, isdir)
+         if isdir and (self.GDEXLS['R'] or not display or not self.LINFO['dsid']):
+            fs = sorted(glob.glob(file + "/*"))
+            self.display_list(fs, 1)
+            if self.LINFO['pcnt'] > self.CLMT: self.display_format_list()
+
+   # recursively display directory/file info
+   def display_list(self, files, level):
+      for file in files:
+         isdir = 1 if op.isdir(file) else 0
+         self.display_line(file, isdir)
+         if isdir and level < self.GDEXLS['R']:
+            fs = sorted(glob.glob(file + "/*"))
+            self.display_list(fs, level+1)
+            if self.LINFO['pcnt'] > self.CLMT: self.display_format_list()
+
+   # find dataset/group info; display or cache file
+   def display_line(self, file, isdir):
+      getwfile = 1
+      if self.LINFO['dsid'] and self.LINFO['dhome']:
+         ms = re.match(r'^{}/(.*)$'.format(self.LINFO['dhome']), file)
+         if ms:
+            wfile = ms.group(1)
+            getwfile = 0
+      if getwfile:
+         self.LINFO['dsid'] = self.find_dataset_id(file)
+         if self.LINFO['dsid'] is None: return     # skip for missing dsid
+         pgrec = self.pgget("dataset", "title, (dwebcnt + nwebcnt) nc, (dweb_size + nweb_size) ns", "dsid = '{}'".format(self.LINFO['dsid']), self.LGEREX)
+         if not pgrec: return None
+         self.LINFO['dhome'] = "{}/{}".format(self.PGLOG['DSDHOME'], self.LINFO['dsid'])
+         if self.LINFO['dhome'] == file:
+            file = re.sub(r'^{}'.format(self.LINFO['tpath']), '', file, 1)
+            if self.GDEXLS['d']:
+               title = pgrec['title'] if pgrec['title'] else ''
+               self.display_record(["D" + file, pgrec['ns'], str(pgrec['nc']), title])
+            self.LINFO['dcnt'] += 1
+            return
+         ms = re.match(r'^{}/(.*)$'.format(self.LINFO['dhome']), file)
+         if ms:
+            wfile = ms.group(1)
+         else:
+            return
+      if isdir:
+         if self.GDEXLS['d']:   # check and display group info for directory
+            pgrec = self.pgget("dsgroup", "title, (dwebcnt + nwebcnt) nc, (dweb_size + nweb_size) ns",
+                               "dsid = '{}' AND webpath = '{}'".format(self.LINFO['dsid'], wfile), self.LGEREX)
+            if pgrec:
+               file = re.sub(r'^{}'.format(self.LINFO['tpath']), '', file, 1)
+               title = pgrec['title'] if pgrec['title'] else ''
+               self.display_record(["G" + file, pgrec['ns'], str(pgrec['nc']), title])
+               self.LINFO['gcnt'] += 1
+      elif self.GDEXLS['f']:   # check and display file info
+         pgrec = self.pgget_wfile(self.LINFO['dsid'], "data_size, data_format, note",
+                                  "wfile = '{}'".format(wfile), self.LGEREX)
+         if pgrec:
+            note = re.sub(r'\n', ' ', pgrec['note']) if pgrec['note'] else ''
+            file = re.sub(r'^{}'.format(self.LINFO['tpath']), '', file, 1)
+            self.display_record(["F" + file, pgrec['data_size'], pgrec['data_format'], note])
+            self.LINFO['fcnt'] += 1
+
+   # display one file info
+   def display_record(self, disp):
+      disp[1] = self.get_float_string(disp[1])
+      if self.GDEXLS['N']:
+         print(self.GDEXLS['D'].join(disp))
+      else:
+         self.LINFO['pgrecs'].append(disp)
+         self.LINFO['pcnt'] += 1
+         for i in range(self.DIDX):
+            dlen = len(disp[i])
+            if dlen > self.WIDTHS[i]: self.WIDTHS[i] = dlen
+
+   # display cached list with format
+   def display_format_list(self):
+      for j in range(self.LINFO['pcnt']):
+         disp = self.LINFO['pgrecs'][j]
+         for i in range(self.DIDX):
+            if self.ALIGNS[i] == 1:
+               disp[i] = "{:>{}}".format(disp[i], self.WIDTHS[i])
+            else:
+               disp[i] = "{:{}}".format(disp[i], self.WIDTHS[i])
+         print(self.GDEXLS['D'].join(disp))
+      self.LINFO['pcnt'] = 0
+
+   # change size to floating point value with unit
+   @staticmethod
+   def get_float_string(val):
+      units = ['B', 'K', 'M', 'G', 'T', 'P']
+      idx = 0
+      while val > 1000 and idx < 5:
+         val /= 1000
+         idx += 1
+      if idx > 0:
+         return "{:.2f}{}".format(val, units[idx])
+      else:
+         return "{}{}".format(val, units[idx])
+
+   # replace /gpfs to the path /glade
+   @staticmethod
+   def get_real_path(path):
+      if re.match(r'^/gpfs/u', path):
+         path = re.sub(r'^/gpfs', '/glade', path, 1)
+      elif re.match(r'^/gpfs/csfs1/', path):
+         path = re.sub(r'^/gpfs/csfs1', '/glade/campaign', path, 1)
+      return op.realpath(path)
+
+# main function to excecute this script
+def main():
+   object = GdexLs()
+   object.read_parameters()
+   object.start_actions()
+   object.pgexit(0)
+
+# call main() to start program
+if __name__ == "__main__": main()

{rda_python_miscs-1.0.7 → rda_python_miscs-2.0.1}/src/rda_python_miscs/gdexls_standalone.py

@@ -1,9 +1,9 @@
-#!/glade/work/zji/conda-envs/pg-rda/bin/python
+#!/glade/work/zji/conda-envs/pg-gdex/bin/python
 # -*- coding: utf-8 -*-
 #  2025-09-23, zji@ucar.edu, created for a standalone version of gdexls
 import re
 import sys
-pgpath = '/glade/work/zji/conda-envs/pg-rda/lib/python3.10/site-packages'
+pgpath = '/glade/work/zji/conda-envs/pg-gdex/lib/python3.12/site-packages'
 if pgpath not in sys.path: sys.path.insert(0, pgpath)
 
 from rda_python_miscs.gdexls import main

rda_python_miscs-2.0.1/src/rda_python_miscs/pgwget.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+##################################################################################
+#     Title : pgwget
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 12/02/2020
+#             2025-03-10 transferred to package rda_python_miscs from
+#             https://github.com/NCAR/rda-utility-programs.git
+#             2026-01-05 convert to class PgWget
+#   Purpose : wrapper to wget to get a file with wildcard in name
+#    Github: https://github.com/NCAR/rda-python-miscs.git
+##################################################################################
+import sys
+import re
+from rda_python_common.pg_file import PgFile
+
+class PgWget(PgFile):
+
+   def __init__(self):
+      super().__init__()
+      self.OPTIONS = {
+         'OP' : "-np -nH -nd -m -e robots=off --no-check-certificate",
+         'UL' : None,
+         'RN' : None,
+         'FN' : None,
+         'FC' : 1,
+         'SM' : 0,
+         'MC' : 0,
+         'CN' : 0,
+         'CR' : 0,
+         'EX' : None,
+         'JC' : 'cat'
+      }
+
+   # function to read parameters
+   def read_parameters(self):
+      option = None
+      JCS = ['cat', 'tar', 'first', 'last']
+      options = '|'.join(self.OPTIONS)
+      argv = sys.argv[1:] 
+      self.PGLOG['LOGFILE'] = "pgwget.log"
+      for arg in argv:
+         if arg == "-b":
+            self.PGLOG['BCKGRND'] = 1
+            option = None
+            continue
+         ms = re.match(r'^-({})$'.format(options), arg, re.I)
+         if ms:
+            option = ms.group(1).upper()
+            if re.match(r'^(CN|CR|SM)$', option):
+               self.OPTIONS[option] = 1
+               option = None
+            continue   
+         if re.match(r'^-.*$', arg): self.pglog(arg + ": Unknown Option", self.LGEREX)
+         if not option: self.pglog(arg + ": Value passed in without leading option", self.LGEREX)
+         if option == 'JC' and arg not in JCS:
+            self.pglog(arg + ": Joining Command must be one of {}".format(JCS), self.LGEREX)
+         self.OPTIONS[option] = int(arg) if re.match(r'^(FC|MC)$', option) else arg
+         option = None
+      if not (self.OPTIONS['UL'] and self.OPTIONS['RN']):
+         print("Usage: pgwget [-CN] [-CR] [-FC FileCount] [-JC JoinCommand] [-MC MinFileCount] [-FN FileName] -UL WebURL -RN RootFileName [-EX FileNameExtension]")
+         print("   Provide at least WebURL and RootFileName to wget file(s)")
+         print("   Option -CN - check new file if presents")
+         print("   Option -CR - clean the downloaded remote file(s) if presents")
+         print("   Option -FC - number of files to be valid download; defaults to 1")
+         print("   Option -JC - file joining command, it defaults to cat, could be tar, or last/first to choose the last/first one")
+         print("   Option -SM - Show wget dumping message; defaults to False")
+         print("   Option -MC - minimal number of files to be valid download; defaults to -FC")
+         print("   Option -FN - file name to be used if successful download; defaults to RootFileName.FileNameExtension")
+         print("   Option -OP - options used by wget, defaults to '-np -nH -nd -m -e robots=off'")
+         print("   Option -UL - (mandatory) WebURL with path")
+         print("   Option -RN - (mandatory) the root portion of the remote file name to be downloaded")
+         print("   Option -EX - file name extension to be used.")
+         sys.exit(0)
+      self.cmdlog("pgwget " + ' '.join(argv))
+      if not self.OPTIONS['MC']: self.OPTIONS['MC'] = self.OPTIONS['FC']
+      if not self.OPTIONS['SM']: self.OPTIONS['OP'] += ' -q'
+   
+   # function to start actions
+   def start_actions(self):
+      self.download_wildcard_files()   
+      self.cmdlog()
+   
+   # download one or multiple remote files via wget; concat files to a single one if multiple
+   def download_wildcard_files(self):
+      deleted = 0
+      if self.OPTIONS['FN']:
+         dfile = self.OPTIONS['FN']
+      else:
+         dfile = self.OPTIONS['RN']
+         if self.OPTIONS['EX']: dfile += "." + self.OPTIONS['EX']
+      dinfo = self.check_local_file(dfile, 1)
+      if dinfo and not self.OPTIONS['CN']:
+         return self.pglog("{}: file dowloaded already ({} {})".format(dfile, dinfo['date_modified'], dinfo['time_modified']), self.LOGWRN)
+      build = 0 if dinfo else 1
+      wfile = self.OPTIONS['RN'] + "*"
+      if self.OPTIONS['EX']: wfile += "." + self.OPTIONS['EX']
+      dlist = self.local_glob(wfile, 1)
+      if dfile in dlist and dinfo:
+         del dlist[dfile]
+         deleted = 1
+      dcnt = len(dlist)
+      if self.OPTIONS['CN'] or dcnt < self.OPTIONS['FC']:
+         cmd = "wget {} {} -A '{}'".format(self.OPTIONS['OP'], self.OPTIONS['UL'], wfile)
+         self.pgsystem(cmd, self.LOGWRN, 7)
+         nlist = self.local_glob(wfile, 1)
+         if dfile in nlist and dinfo:
+            del nlist[dfile]
+            deleted = 1
+         ncnt = len(nlist)
+      else:
+         nlist = dlist
+         ncnt = dcnt
+      if ncnt == 0:
+         if deleted:
+            return self.pglog("{}: File dowloaded on {}".format(dfile, self.OPTIONS['UL']), self.LOGWRN)
+         else:
+            return self.pglog("{}: NO file to dowload on {}".format(dfile, self.OPTIONS['UL']), self.LOGWRN)
+      elif ncnt < self.OPTIONS['MC']:
+         return self.pglog("{}: NOT ready, only {} of {} files dowloaded".format(dfile, ncnt, self.OPTIONS['MC']), self.LOGWRN)
+      rfiles = sorted(nlist)
+      size = skip = 0
+      for i in range(ncnt):
+         rfile = rfiles[i]
+         rinfo = nlist[rfile]
+         size += rinfo['data_size']
+         if dinfo and self.cmptime(dinfo['date_modified'], dinfo['time_modified'], rinfo['date_modified'], rinfo['time_modified']) >= 0:
+            self.pglog("{}: Not newer than {}".format(rfile, dfile), self.LOGWRN)
+            skip += 1
+         elif rfile not in dlist:
+            build = 1
+         elif self.compare_file_info(dlist[rfile], rinfo) > 0:
+            self.pglog("{}: Newer file dowloaded from {}".format(rfile, self.OPTIONS['UL']), self.LOGWRN)
+            build = 1
+         else:
+            self.pglog("{}: No newer file found on ".format(rfile, self.OPTIONS['UL']), self.LOGWRN)
+      if skip == ncnt: return 0
+      if not (build or size == dinfo['data_size']): build = 1
+      if not build: return self.pglog(dfile + ": Use existing file", self.LOGWRN)
+      if self.OPTIONS['JC'] == 'cat':
+         for i in range(ncnt):
+            rfile = rfiles[i]
+            if i == 0:
+               if dfile != rfile: self.local_copy_local(dfile, rfile, self.LOGWRN)
+            else:
+               self.pgsystem("cat {} >> {}".format(rfile, dfile), self.LOGWRN, 5)
+            if self.OPTIONS['CR'] and dfile != rfile: self.pgsystem("rm -f " + rfile, self.LOGWRN, 5)
+      elif self.OPTIONS['JC'] == 'tar':
+         topt = 'c'
+         for i in range(ncnt):
+            rfile = rfiles[i]
+            self.pgsystem("tar -{}vf {} {}".format(topt, dfile, rfile), self.LOGWRN, 5)
+            topt = 'u'
+            if self.OPTIONS['CR']: self.pgsystem("rm -f " + rfile, self.LOGWRN, 5)
+      else:
+         didx = 0 if self.OPTIONS['JC'] == 'first' else (ncnt - 1)
+         self.pgsystem("mv {} {}".format(rfiles[didx], dfile), self.LOGWRN, 5)
+         if self.OPTIONS['CR']:
+            for i in range(ncnt):
+               if i == didx: continue
+               self.pgsystem("rm -f " + rfiles[i], self.LOGWRN, 5)
+      return 1
+
+# main function to excecute this script
+def main():
+   object = PgWget()
+   object.read_parameters()
+   object.start_actions()
+   object.pgexit(0)
+
+# call main() to start program
+if __name__ == "__main__": main()