rda-python-miscs 3.0.2__py3-none-any.whl → 3.0.4__py3-none-any.whl

rda_python_miscs/gdexcp.py

@@ -33,15 +33,21 @@ class GdexCp(PgFile):
          'fp': None,   # from Globus endpoint
          'tp': None,   # to Globus endpoint
          'f': [],      # from file names
+         'i': None,    # input file holding a list of from file names, one per line
          't': None,    # to file name
          'r': 0,       # 1 if recursive all
          'R': 0,       # > 0 to set recursive limit
+         'o': 0,       # 1 to force a downloaded file owned by COMMONUSER; needs -fp
+         'O': 0,       # 1 to override an existing target file of the same size
+         'm': 1,       # number of multiple processes to copy files in parallel
+         'd': 0,       # 1 to add a dscheck record for delayed PBS batch process
          'F': 0o664,   # to file mode, default to 664
          'D': 0o775,   # to directory mode, default to 775
       }
       self.CINFO = {
          'tcnt': 0,
          'htcnt': 0,
+         'pcnt': 0,      # count of dispatched child processes for option -m
          'cpflag': 0,    # 1 file only, 2 directory only, 3 both
          'cpstr': ['', 'Files', 'Directories', 'Files/Directories'],
          'fpath': None,
@@ -75,8 +81,8 @@ class GdexCp(PgFile):
          if ms:
             option = ms.group(1)
             if option not in self.RDACP: self.pglog(arg + ": Unknown Option", self.LGEREX)
-            if option == 'r':
-               self.RDACP['r'] = 1
+            if option in ('r', 'o', 'd', 'O'):
+               self.RDACP[option] = 1
                option = None
             continue
          if not option: self.pglog(arg + ": Value provided without option", self.LGEREX)
@@ -84,7 +90,7 @@ class GdexCp(PgFile):
             self.RDACP['f'].append(arg)
             defopt = None
          else:
-            if option == 'R':
+            if option in ('R', 'm'):
                self.RDACP[option] = int(arg)
             elif option in 'FD':
                self.RDACP[option] = self.base2int(arg, 8)
@@ -95,8 +101,29 @@ class GdexCp(PgFile):
                elif option == 'fh':
                   self.CINFO['fhost'] = arg + '-'
             option = defopt
+      if self.RDACP['i']: self.add_input_files(self.RDACP['i'])
       if dohelp or not self.RDACP['f']: self.show_usage("gdexcp")
-   
+
+   # read source paths from an input file and append them to the -f list
+   def add_input_files(self, infile):
+      """Append source paths read from an input file to the -f source list.
+
+      Each non-empty line in the input file is treated as one source path;
+      leading/trailing whitespace is stripped and lines starting with '#' are
+      ignored as comments.
+
+      Args:
+         infile (str): Path to the input file holding one source path per line.
+      """
+      finfo = self.check_local_file(infile, 0, self.LGWNEX)
+      if not finfo: self.pglog("{}: Input file of -i not found".format(infile), self.LGEREX)
+      fd = open(infile, 'r')
+      for line in fd:
+         line = line.strip()
+         if not line or line[0] == '#': continue
+         self.RDACP['f'].append(line)
+      fd.close()
+
    # function to start actions
    def start_actions(self):
       """Validate copy targets, configure host/bucket/endpoint context, and dispatch copies.
@@ -132,7 +159,23 @@ class GdexCp(PgFile):
          self.PGLOG['BACKUPEP'] = self.RDACP['fp']
       elif self.RDACP['tp']:
          self.PGLOG['BACKUPEP'] = self.RDACP['tp']
+      if self.RDACP['o']:
+         if not self.RDACP['fp']:
+            self.pglog("-o: works only when source Globus endpoint -fp is provided", self.LGEREX)
+         if self.RDACP['th'] or self.RDACP['tp'] or self.RDACP['tb']:
+            self.pglog("-o: works only for downloading to local files (no -th/-tp/-tb)", self.LGEREX)
+      if self.RDACP['m'] < 1: self.RDACP['m'] = 1
+      if self.RDACP['m'] > 16:
+         self.pglog("-m {}: process count too large, capped at 16".format(self.RDACP['m']), self.LOGWRN)
+         self.RDACP['m'] = 16
+      if self.RDACP['d']:
+         self.add_delayed_check()
+         self.cmdlog()
+         return
+      if self.RDACP['m'] > 1:
+         self.start_none_daemon('gdexcp', '', self.PGLOG['CURUID'], self.RDACP['m'], 120)
       self.copy_top_list(self.RDACP['f'])
+      if self.RDACP['m'] > 1: self.check_child(None, 0, self.LOGWRN, 1)
       hinfo = ''
       if self.RDACP['fh']: hinfo += " From " + self.RDACP['fh']
       if self.RDACP['th']: hinfo += " To " + self.RDACP['th']
@@ -175,7 +218,7 @@ class GdexCp(PgFile):
          if not re.match(r'^/', file): file = self.join_paths(self.CINFO['curdir'], file)
          self.CINFO['fpath'] = (file if dosub else op.dirname(file)) + "/"
          if info['isfile']:
-            self.CINFO['tcnt'] += self.copy_file(file, info['isfile'])
+            self.CINFO['tcnt'] += self.dispatch_copy(file, info)
          elif dosub or self.RDACP['R']:
             flist = self.gdex_glob(file, self.RDACP['fh'], 0, self.LGWNEX)
             if flist: self.copy_list(flist, 1, file)
@@ -197,7 +240,7 @@ class GdexCp(PgFile):
       fcnt = 0
       for file in tlist:
          if tlist[file]['isfile']:
-            fcnt += self.copy_file(file, tlist[file]['isfile'])
+            fcnt += self.dispatch_copy(file, tlist[file])
             self.CINFO['cpflag'] |= (1 if tlist[file]['isfile'] else 2)
          elif level < self.RDACP['R']:
             flist = self.gdex_glob(file, self.RDACP['fh'], 0, self.LGWNEX)
@@ -206,20 +249,50 @@ class GdexCp(PgFile):
          self.pglog("{}{}: {} {} copied from directory".format(self.CINFO['fhost'], cdir, fcnt, self.CINFO['cpstr'][self.CINFO['cpflag']]), self.LOGWRN)
       self.CINFO['tcnt'] += fcnt
    
+   # copy one file, forking a child process when running with option -m
+   def dispatch_copy(self, fromfile, finfo):
+      """Copy one file, dispatching the copy to a child process when -m > 1.
+
+      With a single process (-m 1) the file is copied in line.  With multiple
+      processes the copy is forked to a child via the PgSIG process pool (up to
+      RDACP['m'] children run concurrently); the parent records one dispatched
+      file and continues traversing, while each child performs the copy, logs
+      its own result, and exits.
+
+      Args:
+         fromfile (str): Absolute source file path.
+         finfo (dict): Source file-info dict (with 'isfile' and 'data_size').
+
+      Returns:
+         int: 1 if the file was copied (single process) or dispatched to a
+            child process (-m > 1), 0 otherwise.
+      """
+      if self.RDACP['m'] < 2: return self.copy_file(fromfile, finfo)
+      stat = self.start_child("gdexcp_{}".format(self.CINFO['pcnt']), self.LOGWRN, 1)
+      if stat <= 0: self.pglog("{}: cannot start child process to copy".format(fromfile), self.LGEREX)
+      if self.PGSIG['PPID'] > 1:   # in child process
+         self.copy_file(fromfile, finfo)
+         sys.exit(0)
+      self.CINFO['pcnt'] += 1   # in parent process; child already dropped its DB link
+      return 1
+
    # copy one file
-   def copy_file(self, fromfile, isfile):
+   def copy_file(self, fromfile, finfo):
       """Resolve the destination path for one source file and perform the copy.
 
       When a target directory is set (tpath), strips the source base path prefix
       and joins the remainder to tpath.  Otherwise copies directly to the -t value.
+      Skips the copy when the target already exists with the same size as the
+      source, unless -O is given to override an existing same-size target.
 
       Args:
          fromfile (str): Absolute source file path.
-         isfile (int): Non-zero when the source is a regular file (vs. a symlink type).
+         finfo (dict): Source file-info dict (with 'isfile' and 'data_size').
 
       Returns:
          int: 1 if the file was copied successfully, 0 otherwise.
       """
+      isfile = finfo['isfile']
       if self.CINFO['tpath']:
          fname = re.sub(r'^{}'.format(self.CINFO['fpath']), '', fromfile)
          if isfile:
@@ -228,7 +301,77 @@ class GdexCp(PgFile):
             tofile = self.CINFO['tpath'] + '/'
       else:
          tofile = self.RDACP['t']
-      return (1 if self.copy_gdex_file(tofile, fromfile, self.RDACP['th'], self.RDACP['fh'], self.LGWNEX) else 0)
+      if isfile and not self.RDACP['O']:
+         tinfo = self.check_gdex_file(tofile, self.RDACP['th'], 0, self.LGWNEX)
+         if tinfo and tinfo['data_size'] == finfo['data_size']:
+            self.pglog("{}{}: Target exists with same size, skip copying".format(self.CINFO['thost'], tofile), self.LOGWRN)
+            return 0
+      if self.RDACP['o']: return self.force_owner_copy(tofile, fromfile)
+      logact = self.LGWNEX | (self.OVRIDE if self.RDACP['O'] else 0)
+      return (1 if self.copy_gdex_file(tofile, fromfile, self.RDACP['th'], self.RDACP['fh'], logact) else 0)
+
+   # copy one file from a Globus endpoint and force COMMONUSER ownership
+   def force_owner_copy(self, tofile, fromfile):
+      """Download a Globus file via a tmp file so the final copy is owned by COMMONUSER.
+
+      A Globus endpoint dumps the local file owned by the endpoint's mapped user
+      rather than COMMONUSER ('gdexdata').  This downloads to a tmp file under
+      PGLOG['TMPPATH'], makes it group readable/writable as its owner via the
+      pgstart_<user> setuid wrapper, then copies it locally so the final file is
+      owned by COMMONUSER, and removes the tmp file.
+
+      Args:
+         tofile (str): Final local destination path.
+         fromfile (str): Source file path on the Globus endpoint.
+
+      Returns:
+         int: 1 if the file was copied successfully, 0 otherwise.
+      """
+      tmpfile = self.join_paths(self.PGLOG['TMPPATH'], "{}.{}".format(op.basename(fromfile), os.getpid()))
+      if not self.copy_gdex_file(tmpfile, fromfile, self.RDACP['th'], self.RDACP['fh'], self.LGWNEX): return 0
+      finfo = self.check_local_file(tmpfile, 2, self.LGWNEX)
+      owner = finfo['logname'] if finfo else None
+      if owner and owner != self.PGLOG['COMMONUSER']:
+         self.pgsystem(self.get_local_command("chmod g+rw " + tmpfile, owner), self.LGWNEX)
+      ret = self.copy_gdex_file(tofile, tmpfile, self.RDACP['th'], None, self.LGWNEX)
+      self.delete_local_file(tmpfile, self.LGWNEX)
+      return (1 if ret else 0)
+
+   # add a dscheck record so this gdexcp command runs later as a PBS batch job
+   def add_delayed_check(self):
+      """Queue this gdexcp invocation as a delayed PBS batch job via a dscheck record.
+
+      Records the current command (with the -d flag stripped so the batch run
+      performs the actual copy) into the RDADB dscheck table for the dscheck
+      daemon to later submit to PBS through bashqsub/tcshqsub.  The qsub resource
+      option always sets a 24 hour walltime; when -m > 1 it also reserves a single
+      node with (number of processes) cpus and 1gb of memory per cpu.
+      """
+      argv = [arg for arg in sys.argv[1:] if arg != '-d']
+      argstr = self.argv_to_string(argv, 1)
+      argextra = None
+      if len(argstr) > 100:
+         argextra = argstr[100:]
+         argstr = argstr[0:100]
+      record = {
+         'command': 'gdexcp',
+         'argv': argstr,
+         'specialist': self.PGLOG['CURUID'],
+         'workdir': self.CINFO['curdir'],
+         'oindex': 0,
+         'otype': '',
+         'action': None,
+         'dsid': None,
+         'mcount': 1,
+      }
+      (record['date'], record['time']) = self.get_date_time()
+      if argextra: record['argextra'] = argextra
+      qoptions = "-l walltime=24:00:00"
+      if self.RDACP['m'] > 1:
+         qoptions += ",select=1:ncpus={0}:mem={0}gb".format(self.RDACP['m'])
+      record['qoptions'] = qoptions
+      cidx = self.pgadd("dscheck", record, self.LGWNEX|self.AUTOID)
+      self.pglog("Chk{}: gdexcp {} added for delayed batch process".format(cidx, argstr), self.LOGWRN)
 
 # main function to execute this script
 def main():

rda_python_miscs/gdexcp.usg

@@ -1,77 +1,145 @@
 
- Copy files and directories to a target location.  Source and target may each
- reside on the local host, a remote host, an Object Store bucket, or a Globus
+ Name: gdexcp - copy files and directories as user 'gdexdata'
+
+ Copy files and directories to a target location.  The source and the target may
+ each reside on the local host, a remote host, an Object Store bucket, or a Globus
  endpoint.  Target files are owned by 'gdexdata' and created with configurable
  permission modes.
 
  Usage: gdexcp [-f] FromDirectories/Files [-t ToDirectory/FileName]   \
+              [-i InputFile] [-m ProcessCount] [-d] [-r] [-R RecursiveLevel] \
               [-fh FromHostName] [-th ToHostName]                     \
               [-fb FromBucket] [-tb ToBucket]                         \
               [-fp FromGlobusEndpoint] [-tp ToGlobusEndpoint]         \
-              [-F FileMode] [-D DirectoryMode] [-r] [-R RecursiveLevel] [-h]
+              [-F FileMode] [-D DirectoryMode] [-o] [-O] [-h]
+
+   gdexcp can be run from any directory.  Usage is displayed when no source paths
+   are given.
+
+
+ SOURCE AND TARGET
+
+   -f  FromDirectories/Files
+          Source directories and/or files to copy.  This is the default option,
+          so paths may be given without the -f flag.  Shell wildcards are
+          supported; use './' or '*' to copy everything in the current directory.
+          A trailing '/' on a source directory path copies the contents of that
+          directory (as a file list) rather than the directory entry itself.
+          Source paths must be readable by user 'gdexdata'; gdexcp attempts to
+          fix the mode when they are not.
+
+   -i  InputFile
+          A file holding a list of source paths to copy, one path per line.
+          Blank lines and lines starting with '#' are ignored.  The paths read
+          are appended to the -f source list.
+
+   -t  ToDirectory/FileName
+          Target directory or file name.  Defaults to '.' (the current
+          directory).  A trailing '/' forces the target to be treated as a
+          directory.  Multiple source files cannot be copied to a single target
+          file name.
+
+
+ SOURCE/TARGET LOCATIONS (default to the local host)
+
+   -fh FromHostName     host name where the source files reside.
+   -th ToHostName       host name where the target files are written.
+   -fb FromBucket       Object Store bucket name for the source files.
+   -tb ToBucket         Object Store bucket name for the target files.
+   -fp FromGlobusEndpoint   Globus endpoint for the source files.
+   -tp ToGlobusEndpoint     Globus endpoint for the target files.
+
+
+ COPY BEHAVIOR
+
+   -r     Copy directories and files recursively, with no depth limit.
+
+   -R  RecursiveLevel
+          Copy recursively up to the given depth.  -R 1 copies only the
+          immediate contents of each source directory.
+
+   -m  ProcessCount
+          Number of processes used to copy files in parallel.  Defaults to 1.
+          When greater than 1, the source files are distributed across that many
+          concurrent child processes.  Capped at 16; a larger value is reduced to
+          16 with a warning.
+
+   -d     Add a dscheck record so this gdexcp command runs later as a delayed PBS
+          batch job (submitted by the dscheck daemon via bashqsub / tcshqsub).
+          The qsub resource always sets a 24 hour walltime; when -m is greater
+          than 1 it also reserves a single node with (ProcessCount) cpus and 1gb
+          of memory per cpu.
+
+   -o     Force a downloaded file to be owned by 'gdexdata'.  A Globus endpoint
+          writes the local file owned by the endpoint's mapped user; with -o the
+          file is downloaded to a tmp file, then copied locally so the final file
+          is owned by 'gdexdata'.  Only valid together with -fp and for
+          downloading to local files (not with -th/-tp/-tb).
+
+   -O     Override an existing target.  By default a source file is skipped when
+          the target already exists with the same size; give -O to copy it anyway
+          and overwrite the existing target.
+
+
+ TARGET PERMISSIONS (octal notation)
 
-      - Option -f, source directories and/or files to copy.  This is the
-          default option, so paths may be given without the -f flag.
-          Shell wildcards are supported.  Use './' or '*' to copy everything
-          in the current directory.  Source paths must be readable by user
-          'gdexdata'; gdexcp will attempt to fix the mode if they are not;
+   -F  FileMode          permission mode for target files.       Defaults to 664.
+   -D  DirectoryMode     permission mode for target directories. Defaults to 775.
 
-      - Option -t, target directory or file name.  Defaults to '.' (current
-          directory).  Multiple source files cannot be copied to a single
-          target file name.  A trailing '/' on the target path treats it as
-          a directory;
 
-      - Option -fh, host name where the source files reside.
-          Defaults to the local host;
+ MISCELLANEOUS
 
-      - Option -th, host name where the target files should be written.
-          Defaults to the local host;
+   -h     Display this help document.
 
-      - Option -fb, Object Store bucket name for the source files;
 
-      - Option -tb, Object Store bucket name for the target files;
+ NOTES
 
-      - Option -fp, Globus endpoint for the source files;
+   - A trailing '/' on a source directory path copies the contents of that
+     directory rather than the directory entry itself.
+   - By default an unchanged target (same size) is skipped; use -O to overwrite.
+   - If a Globus endpoint (-fp/-tp) is locally accessible, a direct local copy
+     (omitting -fp/-tp and giving the local path) is faster, avoiding the Globus
+     transfer overhead.
+   - A delayed batch job (-d) is given a 24 hour walltime.  Do not submit a single
+     batch job to copy too many files at once; if the copy cannot finish within 24
+     hours the job is killed.  Split a very large copy into multiple -d jobs (and/or
+     raise -m) so each one completes well within the walltime.
 
-      - Option -tp, Globus endpoint for the target files;
 
-      - Option -F, permission mode for target files in octal notation.
-          Defaults to 664;
+ EXAMPLES
 
-      - Option -D, permission mode for target directories in octal notation.
-          Defaults to 775;
+   1. Copy all files and subdirectories under the current directory to a remote
+      host:
 
-      - Option -r, copy directories and files recursively (no depth limit);
+         gdexcp -r -f * -t /PathTo/d277006/ -th castle
 
-      - Option -R RecursiveLevel, copy recursively up to the specified depth.
-          -R 1 copies only the immediate contents of each source directory;
+   2. Copy the contents of a local directory to a remote location (trailing '/'
+      on the source omits the directory entry itself):
 
-      - Option -h, display this help document;
+         gdexcp -r -f /PathTo/DirectoryName/ -t /PathTo/d277006/ -th castle
 
-  This utility can be run from any directory.  Usage is displayed if no source
-  files are provided.  A trailing '/' on a source directory path copies the
-  contents of that directory rather than the directory entry itself.
+      Without the trailing '/', DirectoryName itself is also copied:
 
-  Examples:
+         gdexcp -r -f /PathTo/DirectoryName -t /PathTo/d277006/ -th castle
 
-    1. Copy all files and subdirectories under the current directory to a
-       remote host:
+   3. Copy a single file to an Object Store bucket:
 
-          gdexcp -r -f * -t /PathTo/d277006/ -th castle
+         gdexcp -f /PathTo/myfile.nc -tb my-bucket -t myfile.nc
 
-    2. Copy the contents of a specific local directory to a remote location
-       (trailing '/' on source omits the directory entry itself):
+   4. Copy files from a remote host to the local current directory:
 
-          gdexcp -r -f /PathTo/DirectoryName/ -t /PathTo/d277006/ -th castle
+         gdexcp -fh castle -f /PathTo/d277006/myfile.nc
 
-       Without the trailing '/', DirectoryName itself is also copied:
+   5. Download a file from a source Globus endpoint and force the local file to
+      be owned by 'gdexdata' (-o requires -fp):
 
-          gdexcp -r -f /PathTo/DirectoryName -t /PathTo/d277006/ -th castle
+         gdexcp -fp gdex-quasar -f /d277006/myfile.nc -t /PathTo/myfile.nc -o
 
-    3. Copy a single file to an Object Store bucket:
+   6. Copy the source paths listed in an input file using 4 parallel processes:
 
-          gdexcp -f /PathTo/myfile.nc -tb my-bucket -t myfile.nc
+         gdexcp -i filelist.txt -t /PathTo/d277006/ -m 4
 
-    4. Copy files from a remote host to the local current directory:
+   7. Queue a delayed PBS batch job to copy a directory in parallel; the dscheck
+      daemon submits it later, reserving one node with 4 cpus and 4gb of memory:
 
-          gdexcp -fh castle -f /PathTo/d277006/myfile.nc
+         gdexcp -r -f /PathTo/DirectoryName/ -t /PathTo/d277006/ -m 4 -d

rda_python_miscs/pg_rst.py

@@ -140,10 +140,10 @@ class PgRST(PgFile, PgUtil):
       """
       self.OPTS = opts
       self.ALIAS = alias
+      self.DOCS['DOCNAM'] = docname
 
       self.parse_docs(docname)
       if not self.sections: self.pglog(docname + ": empty document", self.LGWNEX)
-      self.DOCS['DOCNAM'] = docname
       if docname in self.LINKS: self.LINKS.remove(docname)
       self.DOCS['DOCLNK'] = r"({})".format('|'.join(self.LINKS))
       self.DOCS['DOCTIT'] = docname.upper()
@@ -152,6 +152,7 @@ class PgRST(PgFile, PgUtil):
       self.write_index(self.sections[0])
       for section in self.sections:
          self.write_section(section)
+      self.write_appendix()
 
    #
    # parse the original document and return a array of sections,
@@ -172,7 +173,9 @@ class PgRST(PgFile, PgUtil):
       with open(docfile, 'r') as fh:
          line = fh.readline()
          while line:
-            if re.match(r'\s*#', line):
+            # Skip full-line authoring comments, but keep '#!' shebang lines so
+            # shell scripts shown in example content blocks stay intact.
+            if re.match(r'\s*#(?!!)', line):
                line = fh.readline()
                continue   # skip comment lines
             ms = re.match(r'^(.*\S)\s+#', line)
@@ -191,13 +194,21 @@ class PgRST(PgFile, PgUtil):
                   option = self.record_option(section, option, example, ms.group(1), ms.group(2))
                   example = None
                elif option:
-                  ms = re.match(r'^  For( | another )example, (.*)$', line)
-                  if ms:    # found example
-                     example = self.record_example(option, example, ms.group(2))
-                  elif example:
-                     example['desc'] += line + "\n"
+                  if option.get('inexm'):
+                     # inside an 'Examples:' block: collect raw lines verbatim
+                     # (split into individual examples later in split_examples)
+                     option['exmraw'] += line + "\n"
                   else:
-                     option['desc'] += line + "\n"
+                     ms = re.match(r'^  (?:For(?: | another )example,|Example[ ]*[,\-])\s*(.*)$', line)
+                     if ms:    # found a single labeled example
+                        example = self.record_example(option, example, ms.group(1))
+                     elif re.match(r'^  Examples:\s*$', line):
+                        option['inexm'] = True   # start of a multi-example block
+                        option['exmraw'] = ""
+                     elif example:
+                        example['desc'] += line + "\n"
+                     else:
+                        option['desc'] += line + "\n"
                else:
                   section['desc'] += line + "\n"
 
@@ -261,11 +272,62 @@ class PgRST(PgFile, PgUtil):
       """
       if option:
          if example: self.record_example(option, example)
+         self.split_examples(option)
          self.options[option['opt']] = option     # record option globally
          section['opts'].append(option['opt']) # record option short name in section
 
       if nopt: return self.init_option(section['secid'], nopt, ndesc)
 
+   def split_examples(self, option):
+      """Split an option's pending ``Examples:`` block into individual examples.
+
+      The raw text collected after an ``Examples:`` header is segmented into
+      blank-line-separated blocks.  A block whose last line ends with ``:`` and
+      whose first line is descriptive prose (not a command, option flag, or a
+      ``<<Content ...>>`` header) starts a new example; following blocks
+      (command synopsis, script content, trailing notes) belong to that
+      example until the next title block.
+
+      Args:
+         option (dict): The option whose ``exmraw`` buffer is parsed; the
+                        buffer is removed and one example is recorded per
+                        title block found.
+      """
+      buf = option.pop('exmraw', None)
+      option.pop('inexm', None)
+      if not buf: return
+
+      blocks = []
+      cur = []
+      for ln in buf.split('\n'):
+         if ln.strip() == '':
+            if cur: blocks.append(cur); cur = []
+         else:
+            cur.append(ln)
+      if cur: blocks.append(cur)
+
+      def is_title(block):
+         if not block[-1].rstrip().endswith(':'): return False
+         first = block[0].strip()
+         if first.startswith('<<'): return False
+         if re.match(r'[-*(]', first): return False
+         if re.match(r'{}\b'.format(self.DOCS['DOCNAM']), first): return False
+         return True
+
+      exmtext = None
+      for block in blocks:
+         btext = '\n'.join(block)
+         if is_title(block):
+            if exmtext is not None:
+               self.record_example(option, self.init_example(option['opt'], exmtext))
+            exmtext = btext + "\n"
+         elif exmtext is not None:
+            exmtext += "\n" + btext + "\n"
+         else:
+            option['desc'] += btext + "\n"   # stray text before the first example
+      if exmtext is not None:
+         self.record_example(option, self.init_example(option['opt'], exmtext))
+
    def record_example(self, option, example, ndesc=None):
       """Append the completed *example* to ``self.examples`` and optionally start a new one.
 
@@ -478,18 +540,21 @@ class PgRST(PgFile, PgUtil):
       """Build and return the RST table-of-contents string of a given section.
 
       Produces a nested bullet list of section links (indented by section
-      level) followed by a flat Appendix A list of all example links.
+      level).  For the index (``csection is None``) the example appendix is a
+      standalone page (``appendixA``) added to the toctree; for a section the
+      examples in its subtree are listed inline as a local Appendix A.
 
       Returns:
          str: RST-formatted TOC content ready for ``__TOC__`` substitution.
       """
-      
+
       content = ""
       clevel = csection['level'] if csection else 0
       csecid = csection['secid'] if csection else ""
       depth = self.TLEVEL - clevel
       level = clevel+1
       preid = csecid+'.'
+      is_index = csection is None
 
       # nested bullet list for all sections
       for section in self.sections:
@@ -497,9 +562,14 @@ class PgRST(PgFile, PgUtil):
          if csecid and not secid.startswith(preid): continue
          if section['level'] == level: content += "   section{}\n".format(secid)
 
+      # The full list of examples lives on its own appendix page in the index.
+      if is_index and self.examples: content += "   appendixA\n"
+
       if not content: return ""
 
       content = f".. toctree::\n   :maxdepth: {depth}\n   :caption: Table of Contents\n\n{content}\n"
+      if is_index: return content
+
       # appendix A: list of examples for the parent section and its subsections
       appendix = ""
       idx = 1  # used as example index
@@ -507,7 +577,7 @@ class PgRST(PgFile, PgUtil):
          opt = exm['opt']
          option = self.options[opt]
          secid = option['secid']
-         if not csecid or secid == csecid or secid.startswith(preid):
+         if secid == csecid or secid.startswith(preid):
             appendix += "- :ref:`A.{}. {} Option -{} (-{}) <{}_e{}>`\n".format(
                         idx, option['type'], opt, option['name'], secid, idx)
          idx += 1
@@ -516,6 +586,28 @@ class PgRST(PgFile, PgUtil):
 
       return content
 
+   #
+   # write the appendix page listing all examples in the document
+   #
+   def write_appendix(self):
+      """Write ``appendixA.rst`` listing every example with a link.
+
+      Each entry links to the example anchor on its section page. Does nothing
+      when the document has no examples.
+      """
+      if not self.examples: return
+      content = ""
+      idx = 1
+      for exm in self.examples:
+         option = self.options[exm['opt']]
+         secid = option['secid']
+         title = exm['title'].strip().rstrip(':')
+         content += "- :ref:`A.{}. {} Option -{} (-{}): {} <{}_e{}>`\n".format(
+                     idx, option['type'], exm['opt'], option['name'], title, secid, idx)
+         idx += 1
+
+      self.template_to_rst("appendix", {'CONTENT': content}, "A")
+
    #
    # create a section rst content
    #
@@ -672,7 +764,8 @@ class PgRST(PgFile, PgUtil):
 
       for optary in opts:
          opt = self.get_short_option(optary[1])
-      
+         if opt is None: continue   # not an option of this document; leave as-is
+
          pre = optary[0]
          after = optary[2]
          secid = self.options[opt]['secid']
@@ -826,7 +919,7 @@ class PgRST(PgFile, PgUtil):
       line0 = lines[0]
       normal = 1
       if dtype == 2:
-         ms = re.match(r'^<<(Content .*)>>$', line0)
+         ms = re.match(r'^\s*<<(Content .*)>>$', line0)
          if ms:   # input files for examples
             content += ms.group(1) + ":\n\n.. code-block:: none\n\n"
             normal = 0
@@ -928,13 +1021,23 @@ class PgRST(PgFile, PgUtil):
                rows.append(tuple(prev_vals))
             content = self.build_rst_list_table(rows)
          else:
-            # multi-column table split on 2+ spaces
-            rows = []
-            for i in range(cnt):
-               line = lines[i].strip()
-               vals = re.split(r'\s{2,}', self.replace_option_link(line, secid, 1))
-               rows.append(vals)
-            content = self.build_rst_simple_table(rows) + "\n"
+            raw = [lines[i].strip() for i in range(cnt)]
+            cmdpat = r'(?:[*\d][\d* ]*\s+)?{}(\s|$)'.format(re.escape(self.DOCS['DOCNAM']))
+            if raw and all(re.match(cmdpat, r) for r in raw):
+               # Command line(s) following a label (e.g. a Quick Start entry):
+               # render as a literal block instead of a (degenerate) table.
+               content = ".. code-block:: none\n\n"
+               for r in raw:
+                  content += "   " + r + "\n"
+               content += "\n"
+            else:
+               # multi-column table split on 2+ spaces
+               rows = []
+               for i in range(cnt):
+                  line = lines[i].strip()
+                  vals = re.split(r'\s{2,}', self.replace_option_link(line, secid, 1))
+                  rows.append(vals)
+               content = self.build_rst_simple_table(rows) + "\n"
 
       return content
 
@@ -1046,10 +1149,10 @@ class PgRST(PgFile, PgUtil):
          p (str): Option name to look up (short, long, or alias).
 
       Returns:
-         str: Canonical two-letter option short name.
-
-      Raises:
-         PgLOG error (LGWNEX) if *p* cannot be resolved.
+         str | None: Canonical two-letter option short name, or ``None`` when
+            *p* does not name an option of this document (e.g. an option of a
+            different program referenced in prose).  Callers skip such tokens
+            so unrelated option-like text is left untouched.
       """
       plen = len(p)
       if plen == 2 and p in self.options: return p
@@ -1061,7 +1164,7 @@ class PgRST(PgFile, PgUtil):
          for alias in self.ALIAS[opt]:
             if re.match(r'^{}$'.format(alias), p, re.I): return opt
 
-      self.pglog("{} - unknown option for {}".format(p, self.DOCS['DOCNAM']), self.LGWNEX)
+      return None
 
    #
    # replace with rst link for a given section title

rda_python_miscs/rst_templates/appendix.rst.temp

@@ -0,0 +1,19 @@
+################################################################################
+#
+#     Title : appendix_rst.temp
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 03/17/2026
+#   Purpose : template file for help document appendixA.rst (reStructuredText)
+#    Github : https://github.com/NCAR/rda-python-mics.git
+#
+################################################################################
+
+.. _appendixA:
+
+============================
+Appendix A: List of Examples
+============================
+
+__CONTENT__
+
+| :ref:`Back to Table of Contents <index>`

{rda_python_miscs-3.0.2.dist-info → rda_python_miscs-3.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.2
+Version: 3.0.4
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs

{rda_python_miscs-3.0.2.dist-info → rda_python_miscs-3.0.4.dist-info}/RECORD

@@ -3,8 +3,8 @@ rda_python_miscs/bash_qsub.py,sha256=NsYg1A7zeRy3mR_rAaJEQoWSDaWFxE7cpHNc4mA_JwA
 rda_python_miscs/bashqsub.py,sha256=SadPwz1ZXHx2zGNYb2VityItQjMEYj-Favq3xDrPBK0,10127
 rda_python_miscs/bashqsub.usg,sha256=QDaFr1FiQ-5FC9UzCRQ0geYjsPXBnVMhe3lM8qH1xss,2176
 rda_python_miscs/gdex_ls.py,sha256=fg9jfYajOT8ps6eEhFUUqmQ791BdGwBkrT_pyjmGGvQ,8860
-rda_python_miscs/gdexcp.py,sha256=b8ZJspBKDP_u8nH15eWkwI0CSABLSX0MzPw2CVyFJYY,10933
-rda_python_miscs/gdexcp.usg,sha256=O-qXc6vMvojEmBWQIqns1pntI2AqxPB4d7LJw4AqV6M,3197
+rda_python_miscs/gdexcp.py,sha256=jCkDzNrZhi7vhZqeSXHR7d1XIKtsionSJE5ntWoZE0g,18110
+rda_python_miscs/gdexcp.usg,sha256=6r8XVi9inPuWuFth6CAzcd3EpXX4Y_ZrT6YubmQ82kM,6172
 rda_python_miscs/gdexkill.py,sha256=sWa5MCBXvDpN-5iqVBLNrfO8og1vZMZhu7HlCR9c7bI,11607
 rda_python_miscs/gdexkill.usg,sha256=ptx_1GuFNI9ISi4mjMVTc4aDj9J4JpMG0gvzKbhDJqA,2459
 rda_python_miscs/gdexls.py,sha256=Z-iB3nC0zUm2fOUIkhpZd_N82j-5WPRbjDcmGKTRGhY,14396
@@ -21,7 +21,7 @@ rda_python_miscs/gdexsub.usg,sha256=SChXnDzo6larOdpR9eZ2CMu8ExJYQUdfFHUzxcGoqJ0,
 rda_python_miscs/gdexzip.py,sha256=wzMVTqeahXAUHsKAtqYG6wrM4J2EahnncrpliCqFyNQ,3252
 rda_python_miscs/gdexzip.usg,sha256=cG1Uwa8WZ3KQNSaSkr29edUEaaLb_4cHTU3GRoZSQ84,1918
 rda_python_miscs/pg_docs.py,sha256=_AoqrWroUu7FQMVWaTSTZBQMwi1BH1-RjKb1nDHgxOI,25369
-rda_python_miscs/pg_rst.py,sha256=XldjKneKuciYvZxrHC3h3_OHwyhh51LKYOuKr0s_2Gg,48480
+rda_python_miscs/pg_rst.py,sha256=Wd4qfeetQYvRio9P2YE2x4kUCDI6ZEcX8CQiI0Xqmyk,53063
 rda_python_miscs/pg_rst.usg,sha256=oYvVEqzHeVmDntN7hfpM8vc6eoYRyQTruQQg2mPfOo4,2484
 rda_python_miscs/pg_wget.py,sha256=YK8EJQtTwGXxHav_SveXOwE_oA9KZ9l12lGSf77JQnE,7571
 rda_python_miscs/pgwget.py,sha256=TFHyquKT0tyz-r-RUAV_wpdwktpVRwyZGrHnZF2B3Vk,8115
@@ -38,11 +38,12 @@ rda_python_miscs/rdals.usg,sha256=ChF-nn3Qb2pds3wMXIWubB_tjwZslxHfSha0dTDqOiY,29
 rda_python_miscs/tcsh_qsub.py,sha256=P4Obzbp5Dvy-N0eRR5F51R9yj3ttWJo6g1NqyVDO6-Y,6670
 rda_python_miscs/tcshqsub.py,sha256=QxBq9MdVIUs9t2d6vHhkxM1nrcLwRNqcq1lJiWhXKUM,10124
 rda_python_miscs/tcshqsub.usg,sha256=JYfhrK7cqme-Sij_JfquONOs3HMu-d5dDGI9K_RdudU,2180
+rda_python_miscs/rst_templates/appendix.rst.temp,sha256=HCujxbj-R4_8FgOrLSlMZcBQHUu8f6kfBMWVRygl7jc,560
 rda_python_miscs/rst_templates/index.rst.temp,sha256=YSa1JM6X9x2SC6UiqJu_9xRrVYGLKwNI43DBJEGDX08,523
 rda_python_miscs/rst_templates/section.rst.temp,sha256=-CUtutvctG2tIdkqrkFxVEzmNN3atRJXBDQaTnMJ6Gw,572
-rda_python_miscs-3.0.2.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_miscs-3.0.2.dist-info/METADATA,sha256=2rjjjAkl_aCtq1tCb880gu1r2oHlZCEy5iLKRg2IpRc,5803
-rda_python_miscs-3.0.2.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-rda_python_miscs-3.0.2.dist-info/entry_points.txt,sha256=pBgb-_g4yZhm6YynwDHtNTAzxVrb8SoDMd7Eiys8gv4,806
-rda_python_miscs-3.0.2.dist-info/top_level.txt,sha256=W5rz7DrWb7hXABUbGgWcwe6D644X338LR8_zdgmtLhg,17
-rda_python_miscs-3.0.2.dist-info/RECORD,,
+rda_python_miscs-3.0.4.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_miscs-3.0.4.dist-info/METADATA,sha256=nxRaTHxAU0aTXLaNZj9RChQZ70oCj0qHdWtZLVAnOOc,5803
+rda_python_miscs-3.0.4.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+rda_python_miscs-3.0.4.dist-info/entry_points.txt,sha256=pBgb-_g4yZhm6YynwDHtNTAzxVrb8SoDMd7Eiys8gv4,806
+rda_python_miscs-3.0.4.dist-info/top_level.txt,sha256=W5rz7DrWb7hXABUbGgWcwe6D644X338LR8_zdgmtLhg,17
+rda_python_miscs-3.0.4.dist-info/RECORD,,