rda-python-miscs 3.0.3__py3-none-any.whl → 3.0.5__py3-none-any.whl

rda_python_miscs/decsdata_restore.py

@@ -0,0 +1,456 @@
+#!/usr/bin/env python3
+##################################################################################
+#     Title: decsdata_restore
+#    Author: Zaihua Ji, zji@ucar.edu
+#      Date: 2026-08-31
+#   Purpose: restore decsdata datasets, or parts of them, out of the GLADE HSM
+#            cold storage; the opposite of decsdata_storage
+#    Github: https://github.com/NCAR/rda-python-miscs.git
+##################################################################################
+import re
+import os
+import sys
+from os import path as op
+from rda_python_common.pg_file import PgFile
+
+class DecsRestore(PgFile):
+   """Restore decsdata datasets out of the GLADE HSM cold storage.
+
+   'glade_hsm recall' only submits a request that the HSM batch processes
+   fulfill later on, so restoring is done in three steps:
+
+      -x  submit the recall requests for the given datasets/paths
+      -s  check the recall status, repeat until nothing is left on tape
+      -r  copy the recalled data back under the decsdata directory
+
+   Recalled files stay readable inside cold storage for 7 days only, after
+   which they are migrated onto tape again, so step -r must be done within
+   that window.
+   """
+
+   def __init__(self):
+      """Initialize DecsRestore with default option values and runtime state."""
+      super().__init__()
+      self.HSM = os.environ.get('GLADE_HSM', op.expanduser('~benkirk/glade_hsm'))
+      self.VALOPTS = 'Dwl'   # single-value options
+      self.MULOPTS = 'd'     # multi-value options
+      self.MODOPTS = 'hf'    # mode options
+      self.ACTOPTS = 'xsr'   # action options, one and only one is required
+      self.OPTS = {
+         'D': None,   # cold storage date, in YYYYMMDD or YYYY-MM-DD
+         'w': None,   # decsdata directory, defaults to PGLOG['DECSHOME']
+         'l': None,   # dataset list file
+         'd': [],     # dataset IDs, a sub-path may be appended to each one
+         't': None,   # target directory of -r, defaults to the decsdata directory
+         'h': 0,      # 1 to show help message
+         'f': 0,      # 1 to copy back while files are still on tape
+      }
+      self.ACTION = None     # one of the ACTOPTS letters
+      self.SIZEUNITS = {     # units reported by 'gladequota', in bytes
+         'B': 1, 'KIB': 1024, 'MIB': 1024**2,
+         'GIB': 1024**3, 'TIB': 1024**4, 'PIB': 1024**5,
+      }
+      self.RINFO = {
+         'decsdir': None,   # decsdata directory the data is restored into
+         'roots': [],       # cold storage directories to look the data up in
+         'acnt': 0,         # number of dataset paths acted on successfully
+      }
+
+   # function to read parameters
+   def read_parameters(self):
+      """Parse the command line into the OPTS option values and the action.
+
+      Single-value options -D, -w and -l take one value each, multi-value
+      option -d gathers every following dataset path, mode options -h and -f
+      are simple flags, and action options -x, -s and -r are mutually
+      exclusive; -r optionally takes a target directory.  Exits with usage if
+      -h is given or no action is specified.
+      """
+      self.set_suid(self.PGLOG['EUID'])
+      self.set_help_path(__file__)
+      self.PGLOG['LOGFILE'] = "decsdata_restore.log"   # set different log file
+      argv = sys.argv[1:]
+      self.cmdlog("decsdata_restore {}".format(' '.join(argv)))
+      option = None
+      for arg in argv:
+         ms = re.match(r'^-(\w+)$', arg)
+         if ms:
+            option = ms.group(1)
+            if option in self.ACTOPTS:
+               self.set_action(option)
+               if option != 'r': option = None   # -r may be followed by a target directory
+            elif option in self.MODOPTS:
+               self.OPTS[option] = 1
+               option = None
+            elif option not in self.VALOPTS and option not in self.MULOPTS:
+               self.pglog(arg + ": Unknown Option", self.LGEREX)
+            continue
+         if not option: self.pglog(arg + ": Value provided without option", self.LGEREX)
+         if option in self.MULOPTS:
+            self.OPTS[option].append(arg)   # gather all values until the next option
+         else:
+            if option == 'r': option = 't'   # the value of -r is the target directory
+            self.OPTS[option] = arg
+            option = None
+      if self.OPTS['h'] or not self.ACTION: self.show_usage("decsdata_restore")
+
+   # remember the action option and reject a second one
+   def set_action(self, option):
+      """Record the single action option to perform.
+
+      Args:
+         option (str): One of the ACTOPTS letters.
+      """
+      if self.ACTION and self.ACTION != option:
+         self.pglog("-{}: Cannot combine with Action -{}".format(option, self.ACTION), self.LGEREX)
+      self.ACTION = option
+
+   # function to start actions
+   def start_actions(self):
+      """Validate the caller, resolve the cold storage paths, and act on each dataset path."""
+      self.dssdb_dbname()
+      self.validate_decs_group('decsdata_restore', self.PGLOG['CURUID'], 1)
+      self.set_restore_paths()
+      specs = self.get_dataset_list()
+      if not specs: self.pglog("No dataset found to restore", self.LGWNEX)
+      if self.ACTION == 'x': self.check_restore_space(specs)
+      for spec in specs:
+         self.restore_one_path(spec)
+      acts = {'x': 'requested Recall', 's': 'checked Status', 'r': 'copied back'}
+      s = ('s' if self.RINFO['acnt'] > 1 else '')
+      self.pglog("{} of {} Dataset Path{} {}".format(self.RINFO['acnt'], len(specs),
+                 s, acts[self.ACTION]), self.LOGWRN)
+      self.cmdlog()
+
+   # resolve the decsdata directory and the cold storage directories
+   def set_restore_paths(self):
+      """Fill RINFO with the decsdata directory and the cold storage directories to search.
+
+      For a given -D date only '<decsdata>/cold_storage_<date>/COLD_STORAGE' is
+      searched.  Without -D both '<decsdata>/COLD_STORAGE' and every
+      '<decsdata>/cold_storage_<YYYYMMDD>/COLD_STORAGE' are searched, the most
+      recent dated one first.
+      """
+      decsdir = self.OPTS['w'] if self.OPTS['w'] else self.PGLOG['DECSHOME']
+      if not self.check_local_file(decsdir, 0, self.LOGWRN):
+         self.pglog(decsdir + ": decsdata directory NOT exists", self.LGEREX)
+      self.RINFO['decsdir'] = decsdir
+      if not self.OPTS['t']: self.OPTS['t'] = decsdir
+      roots = []
+      if self.OPTS['D']:
+         date = re.sub('-', '', self.OPTS['D'])
+         if not re.match(r'^\d{8}$', date):
+            self.pglog(date + ": Invalid cold storage date, YYYYMMDD expected", self.LGEREX)
+         roots.append(self.join_paths(decsdir, "cold_storage_{}/COLD_STORAGE".format(date)))
+      else:
+         roots.append(self.join_paths(decsdir, "COLD_STORAGE"))
+         files = self.local_glob(self.join_paths(decsdir, "cold_storage_[0-9]*/COLD_STORAGE"), 0, self.LOGWRN)
+         for file in sorted(files, reverse = True):
+            if not files[file]['isfile']: roots.append(file)
+      for root in roots:
+         if self.check_local_file(root, 0, 0): self.RINFO['roots'].append(root)
+      if not self.RINFO['roots']:
+         self.pglog("{}: No cold storage directory found in {}".format(', '.join(roots), decsdir), self.LGEREX)
+      self.pglog("Cold storage searched: {}".format(', '.join(self.RINFO['roots'])), self.LOGWRN)
+
+   # gather the dataset paths to restore
+   def get_dataset_list(self):
+      """Return the list of dataset paths to restore.
+
+      Uses the -d values if given.  Otherwise reads the -l list file, creating
+      it first from every dNNNNNN directory in the cold storage directories if
+      it does not exist yet.
+
+      Returns:
+         list[str]: Dataset IDs, each optionally followed by a sub-path.
+      """
+      if self.OPTS['d']:
+         self.pglog("Restore {} given Dataset Path(s)".format(len(self.OPTS['d'])), self.LOGWRN)
+         return self.OPTS['d']
+      lstfile = self.OPTS['l'] if self.OPTS['l'] else "dsids_{}.lst".format(re.sub('-', '', self.curdate()))
+      if not op.isfile(lstfile):
+         dsids = self.get_coldstorage_datasets()
+         with open(lstfile, 'w') as OUT:
+            for dsid in dsids: OUT.write(dsid + "\n")
+         self.pglog("{}: Generated with {} Dataset(s)".format(lstfile, len(dsids)), self.LOGWRN)
+      specs = []
+      with open(lstfile, 'r') as IN:
+         for line in IN:
+            line = line.strip()
+            if line: specs.append(line)
+      self.pglog("{}: Read {} Dataset Path(s) to restore".format(lstfile, len(specs)), self.LOGWRN)
+      return specs
+
+   # find all dNNNNNN dataset directories in the cold storage directories
+   def get_coldstorage_datasets(self):
+      """Return the sorted dataset IDs of every dNNNNNN directory in the cold storage directories.
+
+      Returns:
+         list[str]: Unique dataset IDs; plain files matching the pattern are skipped.
+      """
+      dsids = []
+      for root in self.RINFO['roots']:
+         files = self.local_glob(self.join_paths(root, "d" + "[0-9]"*6), 0, self.LOGWRN)
+         for file in files:
+            if files[file]['isfile']: continue
+            dsid = op.basename(file)
+            if dsid not in dsids: dsids.append(dsid)
+      return sorted(dsids)
+
+   # locate a dataset path in the cold storage directories, the first match wins
+   def find_cold_path(self, spec):
+      """Look a dataset path up in each cold storage directory.
+
+      Warns and names the ignored ones if the path is found in more than one
+      cold storage directory.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+
+      Returns:
+         tuple: (path, info dict) of the first match, or (None, None).
+      """
+      hits = {}
+      for root in self.RINFO['roots']:
+         path = self.join_paths(root, spec)
+         info = self.check_local_file(path, 0, 0)
+         if info: hits[path] = info
+      if not hits: return (None, None)
+      paths = list(hits)
+      if len(paths) > 1:
+         self.pglog("{}: Found in {} cold storage directories, use {}".format(spec, len(paths), paths[0]), self.LOGWRN)
+         self.pglog("{}: Ignored {}".format(spec, ', '.join(paths[1:])), self.LOGWRN)
+      return (paths[0], hits[paths[0]])
+
+   # count the files of a cold storage path still on tape
+   def hsm_offline_count(self, path, isfile):
+      """Return the number of files under a cold storage path that are still on tape.
+
+      Args:
+         path (str): Cold storage path of a file or directory.
+         isfile (int): 1 if path is a regular file, 0 for a directory.
+
+      Returns:
+         int | None: Count of offline files, or None if it cannot be determined.
+      """
+      out = self.pgsystem("{} status {}".format(self.HSM, path), self.LOGWRN, 51)
+      if not out: return None
+      if isfile: return (1 if re.search(r'migrated', out) else 0)
+      cnts = re.findall(r'Offline:\s*([\d,]+)', out)
+      if not cnts: return None
+      return int(re.sub(',', '', cnts[-1]))
+
+   # build the sfile condition of one dataset path
+   def sfile_condition(self, spec):
+      """Turn a dataset path into a condition on table dssdb.sfile.
+
+      A saved file lives in '<decsdata>/<dsid>/<type>/<sfile>', so the first
+      component of the path is the dataset ID, the second one the saved file
+      type, and the rest the leading part of the sfile field.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+
+      Returns:
+         str: The WHERE condition of the saved files under the path.
+      """
+      paths = spec.strip('/').split('/')
+      if not re.match(r'^[a-z]\d{6}$', paths[0]):
+         self.pglog(spec + ": Invalid dataset path, dNNNNNN expected", self.LGEREX)
+      cnd = "dsid = '{}'".format(paths[0])
+      if len(paths) > 1:
+         if not re.match(r'^\w$', paths[1]):
+            self.pglog(spec + ": Invalid saved file type, one word character expected", self.LGEREX)
+         cnd += " AND type = '{}'".format(paths[1])
+      if len(paths) > 2:
+         sfile = '/'.join(paths[2:])
+         if re.search(r"['\\]", sfile):
+            self.pglog(spec + ": Invalid saved file path", self.LGEREX)
+         # the path is either a saved file itself or the directory holding them
+         cnd += " AND (sfile = '{}' OR sfile LIKE '{}/%')".format(sfile, re.sub(r'([%_])', r'\\\1', sfile))
+      return cnd
+
+   # get the archived size of one dataset path
+   def restore_data_size(self, spec):
+      """Return the total size of the saved files under one dataset path.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+
+      Returns:
+         int: Number of bytes recorded in dssdb.sfile; 0 if nothing is found.
+      """
+      pgrec = self.pgget('sfile', "sum(data_size) tsize, count(sid) fcnt",
+                         self.sfile_condition(spec), self.LOGWRN)
+      if not pgrec or not pgrec['tsize']:
+         self.pglog(spec + ": No saved file found in RDADB", self.LOGWRN)
+         return 0
+      self.pglog("{}: {} in {} saved file(s)".format(spec,
+                 self.format_float_value(pgrec['tsize']), pgrec['fcnt']), self.LOGWRN)
+      return int(pgrec['tsize'])
+
+   # get the GLADE space left for the decsdata directory
+   def decsdata_free_size(self):
+      """Return the GLADE space left on the quota holding the decsdata directory.
+
+      Parses the 'Used' and 'Quota' columns of 'gladequota' and picks the
+      longest reported path the decsdata directory falls under.
+
+      Returns:
+         int | None: Number of free bytes, or None if it cannot be determined.
+      """
+      cmd = self.get_local_command("gladequota", self.PGLOG['COMMONUSER'])
+      out = self.pgsystem(cmd, self.LOGWRN, 21)   # 1+4+16, log the command and return stdout
+      if not out: return None
+      target = op.realpath(self.RINFO['decsdir'])
+      (fsize, fpath) = (None, None)
+      for line in out.split('\n'):
+         ms = re.match(r'^(/\S+)\s+([\d.]+)\s*(\w+)\s+([\d.]+)\s*(\w+)', line)
+         if not ms: continue   # skips the header and the 'n/a' quota lines
+         path = ms.group(1)
+         if not (target == path or target.startswith(path + '/')): continue
+         if fpath and len(fpath) >= len(path): continue   # keeps the closest path only
+         used = self.quota_size(ms.group(2), ms.group(3))
+         quota = self.quota_size(ms.group(4), ms.group(5))
+         if used is None or quota is None: continue
+         (fsize, fpath) = (max(quota - used, 0), path)
+      return fsize
+
+   # convert one 'gladequota' size into bytes
+   def quota_size(self, value, unit):
+      """Convert one size reported by 'gladequota' into bytes.
+
+      Args:
+         value (str): The numeric part of the size.
+         unit (str): The unit of the size, such as 'TiB'.
+
+      Returns:
+         int | None: Number of bytes, or None for an unknown unit.
+      """
+      unit = unit.upper()
+      if unit not in self.SIZEUNITS: return None
+      return int(float(value)*self.SIZEUNITS[unit])
+
+   # make sure the decsdata directory has room for the whole restore
+   def check_restore_space(self, specs):
+      """Stop the recall if the decsdata directory cannot hold the whole restore.
+
+      The size to restore is added up from table dssdb.sfile and compared to
+      the GLADE space left for the decsdata directory.  Twice the size is
+      required, since the recall brings the data back on disk inside the cold
+      storage first and Action -r copies it back afterwards, so both copies
+      live under the decsdata quota at the same time.  The check is skipped,
+      with a warning, if either size cannot be determined.
+
+      Args:
+         specs (list[str]): Dataset paths to recall.
+      """
+      tsize = 0
+      for spec in specs:
+         tsize += self.restore_data_size(spec)
+      if not tsize:
+         self.pglog("Unknown size to restore, Skip checking the decsdata space", self.LOGWRN)
+         return
+      fsize = self.decsdata_free_size()
+      if fsize is None:
+         self.pglog("{}: Cannot get the space left, Skip checking the decsdata space".format(self.RINFO['decsdir']), self.LOGWRN)
+         return
+      nsize = 2*tsize   # room for the recalled copy and for the copy of Action -r
+      msg = "{}: Restore {}, needs {} of the {} left".format(self.RINFO['decsdir'],
+            self.format_float_value(tsize), self.format_float_value(nsize),
+            self.format_float_value(fsize))
+      if nsize > fsize:
+         self.pglog(msg + ", NOT enough space", self.LGEREX)
+      self.pglog(msg, self.LOGWRN)
+
+   # act on one dataset path in cold storage
+   def restore_one_path(self, spec):
+      """Perform the requested action on one dataset path in cold storage.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+      """
+      (path, info) = self.find_cold_path(spec)
+      if not path:
+         self.pglog(spec + ": NOT found in cold storage", self.LOGERR)
+         return
+      if self.ACTION == 'x':
+         self.recall_cold_path(spec, path)
+      elif self.ACTION == 's':
+         self.status_cold_path(spec, path, info['isfile'])
+      else:
+         self.copy_cold_path(spec, path, info['isfile'])
+
+   # submit the recall request of one cold storage path
+   def recall_cold_path(self, spec, path):
+      """Submit the HSM recall request for one cold storage path.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+         path (str): Cold storage path of the data.
+      """
+      if self.pgsystem("{} recall -f {}".format(self.HSM, path), self.LOGWRN, 7):
+         self.RINFO['acnt'] += 1
+         self.pglog("{}: Recall requested, check the progress via -s".format(spec), self.LOGWRN)
+      else:
+         self.pglog("{}: Error request Recall of {}".format(spec, path), self.LOGERR)
+
+   # report the recall status of one cold storage path
+   def status_cold_path(self, spec, path, isfile):
+      """Report the HSM and recall status of one cold storage path.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+         path (str): Cold storage path of the data.
+         isfile (int): 1 if path is a regular file, 0 for a directory.
+      """
+      offline = self.hsm_offline_count(path, isfile)
+      if offline is None:
+         self.pglog("{}: Cannot get the offline file count of {}".format(spec, path), self.LOGERR)
+         return
+      self.RINFO['acnt'] += 1
+      if offline > 0:
+         s = ('s' if offline > 1 else '')
+         self.pglog("{}: Recall PENDING, {} File{} still on tape".format(spec, offline, s), self.LOGWRN)
+      else:
+         self.pglog("{}: Recall COMPLETE, ready to copy back via -r".format(spec), self.LOGWRN)
+
+   # copy one recalled cold storage path back into the decsdata directory
+   def copy_cold_path(self, spec, path, isfile):
+      """Copy one recalled cold storage path back to its target directory.
+
+      Nothing is copied while files are still on tape unless -f is given.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+         path (str): Cold storage path of the data.
+         isfile (int): 1 if path is a regular file, 0 for a directory.
+      """
+      offline = self.hsm_offline_count(path, isfile)
+      if offline:
+         s = ('s' if offline > 1 else '')
+         if not self.OPTS['f']:
+            self.pglog("{}: {} File{} still on tape, add Mode -f to copy anyway".format(spec, offline, s), self.LOGERR)
+            return
+         self.pglog("{}: {} File{} still on tape, copy it anyway".format(spec, offline, s), self.LOGWRN)
+      tofile = self.join_paths(self.OPTS['t'], spec)
+      # a directory is copied as '<path>/.' to merge into an existing target
+      fromfile = path if isfile else path + "/."
+      if self.local_copy_local(tofile, fromfile, self.LOGWRN):
+         self.RINFO['acnt'] += 1
+         self.pglog("{}: Copied back to {}".format(spec, tofile), self.LOGWRN)
+      else:
+         self.pglog("{}: Error copy {} back to {}".format(spec, path, tofile), self.LOGERR)
+
+# main function to execute this script
+def main():
+   """Entry point: instantiate DecsRestore, parse arguments, run, and exit."""
+   from rda_python_setuid.setup_guide import show_setup_guide
+   object = DecsRestore()
+   show_setup_guide(object, 'rda_python_miscs', ['decsdata_storage', 'decsdata_restore'])
+   object.read_parameters()
+   object.start_actions()
+   object.pgexit(0)
+
+# call main() to start program
+if __name__ == "__main__": main()

rda_python_miscs/decsdata_restore.usg

@@ -0,0 +1,101 @@
+
+ Restore decsdata datasets, or parts of them, out of the GLADE HSM cold storage.
+ This is the opposite of 'decsdata_storage'.  A recall request is only submitted
+ by 'glade_hsm recall' and is fulfilled by the HSM batch processes later on, so
+ restoring is done in three steps:
+
+    1. Option -x submits the recall requests for the given dataset paths;
+    2. Option -s checks the recall status, repeat it until nothing is on tape;
+    3. Option -r copies the recalled data back into the decsdata directory.
+
+ Recalled files stay readable inside the 'COLD_STORAGE/' path for 7 days only,
+ after which they are migrated onto tape again, so step 3 must be done within
+ that window.
+
+ Usage: decsdata_restore [-D Date] [-w DecsdataDirectory] [-l ListFile]  \
+                         [-d DatasetPathList] [-f] -x|-s|-r [TargetDirectory]
+
+      - Option -D Date, the cold storage date, in YYYYMMDD or YYYY-MM-DD.  Only
+          '<decsdata>/cold_storage_<date>/COLD_STORAGE' is searched for the
+          data.  Without it, '<decsdata>/COLD_STORAGE' and every
+          '<decsdata>/cold_storage_<YYYYMMDD>/COLD_STORAGE' are all searched,
+          the most recent dated one first.  A dataset path found in more than
+          one of them is acted on in the first one only, and the ignored ones
+          are logged;
+
+      - Option -w DecsdataDirectory, the decsdata directory holding the cold
+          storage directories, and the directory the data is copied back into.
+          Defaults to the configured decsdata root path, /gdex/decsdata;
+
+      - Option -l ListFile, a file holding one dataset path per line.  Only
+          used if -d is not given.  Defaults to 'dsids_<today>.lst' in the
+          current directory, which is generated from every dNNNNNN directory
+          found in the cold storage directories if it does not exist yet;
+
+      - Option -d DatasetPathList, one or more dataset IDs to restore, such as
+          '-d d612000 d627000'.  Append a sub-path to a dataset ID to restore
+          part of it only, such as '-d d612000/2020/01';
+
+      - Option -f, copy the data back even while some of its files are still on
+          tape.  Without it, Action -r refuses to copy such data;
+
+      - Option -h, display this help document;
+
+      - Action -x, submit the recall requests to bring the data back on disk.
+          The decsdata area is limited, so the size to restore is added up
+          from Table 'sfile' in RDADB and compared to the space left for the
+          decsdata directory, as reported by 'gladequota', before any recall
+          is requested.  TWICE the size is required, since the recalled copy
+          in the cold storage and the copy made by Action -r later on both
+          live under the decsdata quota; the recall is stopped if the space
+          left is not enough;
+
+      - Action -s, report the HSM status of the data, including the number of
+          files still on tape and the log of any outstanding recall request;
+
+      - Action -r [TargetDirectory], copy the recalled data back under
+          '<decsdata>/<dsid>', or under the given TargetDirectory.  A restored
+          sub-path keeps its relative position, so 'd612000/2020/01' is copied
+          back to '<TargetDirectory>/d612000/2020/01'.  The dataset directory,
+          and any sub-directory of it, is created if it does not exist yet.
+
+  One and only one of the Actions -x, -s and -r is required; this help document
+  is displayed without any of them.  This utility can be run from any directory.
+  It is executed under the effective user 'gdexdata' via setuid, so the restored
+  data is owned by 'gdexdata'.  The cold storage copy of the data is left in
+  place by Action -r; move it out of the 'COLD_STORAGE/' path manually to take
+  a dataset out of the HSM permanently.
+
+  Examples:
+
+    1. Restore a whole dataset from the cold storage of the root decsdata
+       directory, /gdex/decsdata/COLD_STORAGE/d612000:
+
+          decsdata_restore -d d612000 -x
+          decsdata_restore -d d612000 -s
+          decsdata_restore -d d612000 -r
+
+    2. Restore it from the cold storage directory of a specific date,
+       /gdex/decsdata/cold_storage_20250529/COLD_STORAGE/d612000:
+
+          decsdata_restore -D 20250529 -d d612000 -x
+
+    3. Restore one year of a dataset only:
+
+          decsdata_restore -d d612000/2020 -x
+          decsdata_restore -d d612000/2020 -s
+          decsdata_restore -d d612000/2020 -r
+
+    4. Copy the recalled data back to a directory other than the decsdata one:
+
+          decsdata_restore -d d612000 -r /PathTo/OtherDirectory
+
+    5. Check the status of every dataset in every cold storage directory,
+       generating the dataset list file automatically:
+
+          decsdata_restore -s
+
+    6. Copy a dataset back even though some of its files are still on tape,
+       which makes the HSM read them off tape while they are being copied:
+
+          decsdata_restore -d d612000 -f -r

rda_python_miscs/decsdata_storage.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+##################################################################################
+#     Title: decsdata_storage
+#    Author: Zaihua Ji, zji@ucar.edu
+#      Date: 2026-08-31
+#   Purpose: move decsdata datasets into a dated cold storage directory and hand
+#            them to the GLADE HSM to be migrated onto tape
+#    Github: https://github.com/NCAR/rda-python-miscs.git
+##################################################################################
+import re
+import os
+import sys
+from os import path as op
+from rda_python_common.pg_file import PgFile
+
+class DecsStorage(PgFile):
+   """Move decsdata datasets into cold storage and migrate them onto tape.
+
+   Each given dataset directory is moved from the decsdata directory into
+   '<decsdata>/cold_storage_<date>/' and then handed to 'glade_hsm migrate',
+   which relocates it once more into a 'COLD_STORAGE/' sub-directory and lets
+   the HSM batch processes migrate every large file onto tape.  Use
+   decsdata_restore to bring the data back.
+   """
+
+   def __init__(self):
+      """Initialize DecsStorage with default option values and runtime state."""
+      super().__init__()
+      self.HSM = os.environ.get('GLADE_HSM', op.expanduser('~benkirk/glade_hsm'))
+      self.VALOPTS = 'Dwl'   # single-value options
+      self.MULOPTS = 'd'     # multi-value options
+      self.MODOPTS = 'hx'    # mode options
+      self.OPTS = {
+         'D': None,   # cold storage date, in YYYYMMDD or YYYY-MM-DD
+         'w': None,   # decsdata directory, defaults to PGLOG['DECSHOME']
+         'l': None,   # dataset list file
+         'd': [],     # dataset IDs
+         'h': 0,      # 1 to show help message
+         'x': 0,      # 1 to execute, mandatory
+      }
+      self.SINFO = {
+         'decsdir': None,    # decsdata directory the datasets are stored from
+         'coldstor': None,   # cold storage directory the datasets are moved to
+         'date': None,       # cold storage date
+         'dcnt': 0,          # number of datasets moved into cold storage
+      }
+
+   # function to read parameters
+   def read_parameters(self):
+      """Parse the command line into the OPTS option values.
+
+      Single-value options -D, -w and -l take one value each, multi-value
+      option -d gathers every following dataset ID, and mode options -h and -x
+      are simple flags.  Exits with usage if -h is given or -x is missing.
+      """
+      self.set_suid(self.PGLOG['EUID'])
+      self.set_help_path(__file__)
+      self.PGLOG['LOGFILE'] = "decsdata_storage.log"   # set different log file
+      argv = sys.argv[1:]
+      self.cmdlog("decsdata_storage {}".format(' '.join(argv)))
+      option = None
+      for arg in argv:
+         ms = re.match(r'^-(\w+)$', arg)
+         if ms:
+            option = ms.group(1)
+            if option in self.MODOPTS:
+               self.OPTS[option] = 1
+               option = None
+            elif option not in self.VALOPTS and option not in self.MULOPTS:
+               self.pglog(arg + ": Unknown Option", self.LGEREX)
+            continue
+         if not option: self.pglog(arg + ": Value provided without option", self.LGEREX)
+         if option in self.MULOPTS:
+            self.OPTS[option].append(arg)   # gather all values until the next option
+         else:
+            self.OPTS[option] = arg
+            option = None
+      if self.OPTS['h'] or not self.OPTS['x']: self.show_usage("decsdata_storage")
+
+   # function to start actions
+   def start_actions(self):
+      """Validate the caller, resolve the cold storage paths, and store each dataset."""
+      self.dssdb_dbname()
+      self.validate_decs_group('decsdata_storage', self.PGLOG['CURUID'], 1)
+      self.set_storage_paths()
+      dsids = self.get_dataset_list()
+      if not dsids: self.pglog("No dataset found for cold storage", self.LGWNEX)
+      for dsid in dsids:
+         self.store_one_dataset(dsid)
+      s = ('s' if self.SINFO['dcnt'] > 1 else '')
+      self.pglog("{} of {} Dataset{} moved into {}".format(self.SINFO['dcnt'],
+                 len(dsids), s, self.SINFO['coldstor']), self.LOGWRN)
+      self.cmdlog()
+
+   # resolve the decsdata and cold storage directories
+   def set_storage_paths(self):
+      """Fill SINFO with the decsdata directory, cold storage date and cold storage path.
+
+      Defaults the decsdata directory to PGLOG['DECSHOME'] and the date to
+      today.  Dashes are stripped from the given date, which must then be 8
+      digits.
+      """
+      decsdir = self.OPTS['w'] if self.OPTS['w'] else self.PGLOG['DECSHOME']
+      if not self.check_local_file(decsdir, 0, self.LOGWRN):
+         self.pglog(decsdir + ": decsdata directory NOT exists", self.LGEREX)
+      date = re.sub('-', '', self.OPTS['D']) if self.OPTS['D'] else re.sub('-', '', self.curdate())
+      if not re.match(r'^\d{8}$', date):
+         self.pglog(date + ": Invalid cold storage date, YYYYMMDD expected", self.LGEREX)
+      self.SINFO['decsdir'] = decsdir
+      self.SINFO['date'] = date
+      self.SINFO['coldstor'] = self.join_paths(decsdir, "cold_storage_" + date)
+
+   # gather the dataset IDs to move into cold storage
+   def get_dataset_list(self):
+      """Return the list of dataset IDs to store.
+
+      Uses the -d values if given.  Otherwise reads the -l list file, creating
+      it first from every dNNNNNN directory in the decsdata directory if it
+      does not exist yet.
+
+      Returns:
+         list[str]: Dataset IDs, empty if none is found.
+      """
+      if self.OPTS['d']:
+         self.pglog("Store {} given Dataset(s) into cold storage".format(len(self.OPTS['d'])), self.LOGWRN)
+         return self.OPTS['d']
+      lstfile = self.OPTS['l'] if self.OPTS['l'] else "dsids_{}.lst".format(self.SINFO['date'])
+      if not op.isfile(lstfile):
+         dsids = self.get_decsdata_datasets()
+         with open(lstfile, 'w') as OUT:
+            for dsid in dsids: OUT.write(dsid + "\n")
+         self.pglog("{}: Generated with {} Dataset(s)".format(lstfile, len(dsids)), self.LOGWRN)
+      dsids = []
+      with open(lstfile, 'r') as IN:
+         for line in IN:
+            line = line.strip()
+            if line: dsids.append(line)
+      self.pglog("{}: Read {} Dataset(s) for cold storage".format(lstfile, len(dsids)), self.LOGWRN)
+      return dsids
+
+   # find all dNNNNNN dataset directories in the decsdata directory
+   def get_decsdata_datasets(self):
+      """Return the sorted dataset IDs of every dNNNNNN directory in the decsdata directory.
+
+      Returns:
+         list[str]: Dataset IDs; plain files matching the pattern are skipped.
+      """
+      pattern = self.join_paths(self.SINFO['decsdir'], "d" + "[0-9]"*6)
+      files = self.local_glob(pattern, 0, self.LOGWRN)
+      dsids = []
+      for file in files:
+         if not files[file]['isfile']: dsids.append(op.basename(file))
+      return sorted(dsids)
+
+   # move one dataset into cold storage and migrate it onto tape
+   def store_one_dataset(self, dsid):
+      """Move one dataset into the cold storage directory and migrate it onto tape.
+
+      Skips the dataset if it is not an existing directory in the decsdata
+      directory or if the move fails.  Increments SINFO['dcnt'] for each
+      dataset successfully migrated.
+
+      Args:
+         dsid (str): Dataset ID, such as 'd612000'.
+      """
+      fromfile = self.join_paths(self.SINFO['decsdir'], dsid)
+      info = self.check_local_file(fromfile, 0, self.LOGWRN)
+      if not info:
+         self.pglog(fromfile + ": Dataset NOT exists", self.LOGERR)
+         return
+      if info['isfile']:
+         self.pglog(fromfile + ": Not a dataset directory", self.LOGERR)
+         return
+      tofile = self.join_paths(self.SINFO['coldstor'], dsid)
+      if not self.move_local_file(tofile, fromfile, self.LOGWRN):
+         self.pglog("{}: Error move {} into cold storage".format(fromfile, dsid), self.LOGERR)
+         return
+      cmd = "{} migrate -f {}".format(self.HSM, tofile)
+      if self.pgsystem(cmd, self.LOGWRN, 7):
+         self.SINFO['dcnt'] += 1
+         self.pglog("{}: Migrated onto tape from {}".format(dsid, tofile), self.LOGWRN)
+      else:
+         self.pglog("{}: Error migrate onto tape from {}".format(dsid, tofile), self.LOGERR)
+
+# main function to execute this script
+def main():
+   """Entry point: instantiate DecsStorage, parse arguments, run, and exit."""
+   from rda_python_setuid.setup_guide import show_setup_guide
+   object = DecsStorage()
+   show_setup_guide(object, 'rda_python_miscs', ['decsdata_storage', 'decsdata_restore'])
+   object.read_parameters()
+   object.start_actions()
+   object.pgexit(0)
+
+# call main() to start program
+if __name__ == "__main__": main()

rda_python_miscs/decsdata_storage.usg

@@ -0,0 +1,66 @@
+
+ Move decsdata datasets into a dated cold storage directory and hand them to the
+ GLADE HSM to be migrated onto tape.  Each dataset directory is moved from the
+ decsdata directory into '<decsdata>/cold_storage_<date>/' and then passed to
+ 'glade_hsm migrate', which relocates it once more into a 'COLD_STORAGE/'
+ sub-directory, ending up as:
+
+    <decsdata>/cold_storage_<date>/COLD_STORAGE/<dsid>
+
+ The HSM batch processes then migrate every large file (>= 100MB) underneath the
+ 'COLD_STORAGE/' path onto tape.  All files inside a 'COLD_STORAGE/' path are
+ marked immutable, which prevents modification of their contents and metadata.
+ Use 'decsdata_restore' to bring the data back.
+
+ Usage: decsdata_storage [-D Date] [-w DecsdataDirectory] [-l ListFile]  \
+                         [-d DatasetList] -x
+
+      - Option -D Date, the cold storage date, in YYYYMMDD or YYYY-MM-DD.  It
+          names the cold storage directory '<decsdata>/cold_storage_<date>'.
+          Defaults to today;
+
+      - Option -w DecsdataDirectory, the decsdata directory the datasets are
+          stored from, and the directory the cold storage directory is created
+          in.  Defaults to the configured decsdata root path, /gdex/decsdata;
+
+      - Option -l ListFile, a file holding one dataset ID per line.  Only used
+          if -d is not given.  Defaults to 'dsids_<date>.lst' in the current
+          directory, which is generated from every dNNNNNN directory found in
+          the decsdata directory if it does not exist yet;
+
+      - Option -d DatasetList, one or more dataset IDs to move into cold
+          storage, such as '-d d612000 d627000';
+
+      - Option -h, display this help document;
+
+      - Option -x, execute the cold storage moves.  This option is mandatory;
+          this help document is displayed without it.  Note that no
+          confirmation is asked for once -x is given.
+
+  This utility can be run from any directory.  It is executed under the
+  effective user 'gdexdata' via setuid, so the moved data keeps its ownership.
+  A dataset is skipped, and an error is logged, if it is not an existing
+  directory in the decsdata directory or if it cannot be moved.
+
+  Examples:
+
+    1. Move two given datasets into today's cold storage:
+
+          decsdata_storage -d d612000 d627000 -x
+
+    2. Move them into the cold storage directory of a specific date:
+
+          decsdata_storage -D 20250529 -d d612000 -x
+
+       The data then ends up in:
+
+          /gdex/decsdata/cold_storage_20250529/COLD_STORAGE/d612000
+
+    3. Move every dataset listed in a list file:
+
+          decsdata_storage -l my_dsids.lst -x
+
+    4. Move every dNNNNNN dataset found in a non-default decsdata directory,
+       generating the dataset list file automatically:
+
+          decsdata_storage -w /PathTo/decsdata -x

rda_python_miscs/pg_rst.py

@@ -140,10 +140,10 @@ class PgRST(PgFile, PgUtil):
       """
       self.OPTS = opts
       self.ALIAS = alias
+      self.DOCS['DOCNAM'] = docname
 
       self.parse_docs(docname)
       if not self.sections: self.pglog(docname + ": empty document", self.LGWNEX)
-      self.DOCS['DOCNAM'] = docname
       if docname in self.LINKS: self.LINKS.remove(docname)
       self.DOCS['DOCLNK'] = r"({})".format('|'.join(self.LINKS))
       self.DOCS['DOCTIT'] = docname.upper()
@@ -152,6 +152,7 @@ class PgRST(PgFile, PgUtil):
       self.write_index(self.sections[0])
       for section in self.sections:
          self.write_section(section)
+      self.write_appendix()
 
    #
    # parse the original document and return a array of sections,
@@ -172,7 +173,9 @@ class PgRST(PgFile, PgUtil):
       with open(docfile, 'r') as fh:
          line = fh.readline()
          while line:
-            if re.match(r'\s*#', line):
+            # Skip full-line authoring comments, but keep '#!' shebang lines so
+            # shell scripts shown in example content blocks stay intact.
+            if re.match(r'\s*#(?!!)', line):
                line = fh.readline()
                continue   # skip comment lines
             ms = re.match(r'^(.*\S)\s+#', line)
@@ -191,13 +194,21 @@ class PgRST(PgFile, PgUtil):
                   option = self.record_option(section, option, example, ms.group(1), ms.group(2))
                   example = None
                elif option:
-                  ms = re.match(r'^  For( | another )example, (.*)$', line)
-                  if ms:    # found example
-                     example = self.record_example(option, example, ms.group(2))
-                  elif example:
-                     example['desc'] += line + "\n"
+                  if option.get('inexm'):
+                     # inside an 'Examples:' block: collect raw lines verbatim
+                     # (split into individual examples later in split_examples)
+                     option['exmraw'] += line + "\n"
                   else:
-                     option['desc'] += line + "\n"
+                     ms = re.match(r'^  (?:For(?: | another )example,|Example[ ]*[,\-])\s*(.*)$', line)
+                     if ms:    # found a single labeled example
+                        example = self.record_example(option, example, ms.group(1))
+                     elif re.match(r'^  Examples:\s*$', line):
+                        option['inexm'] = True   # start of a multi-example block
+                        option['exmraw'] = ""
+                     elif example:
+                        example['desc'] += line + "\n"
+                     else:
+                        option['desc'] += line + "\n"
                else:
                   section['desc'] += line + "\n"
 
@@ -261,11 +272,62 @@ class PgRST(PgFile, PgUtil):
       """
       if option:
          if example: self.record_example(option, example)
+         self.split_examples(option)
          self.options[option['opt']] = option     # record option globally
          section['opts'].append(option['opt']) # record option short name in section
 
       if nopt: return self.init_option(section['secid'], nopt, ndesc)
 
+   def split_examples(self, option):
+      """Split an option's pending ``Examples:`` block into individual examples.
+
+      The raw text collected after an ``Examples:`` header is segmented into
+      blank-line-separated blocks.  A block whose last line ends with ``:`` and
+      whose first line is descriptive prose (not a command, option flag, or a
+      ``<<Content ...>>`` header) starts a new example; following blocks
+      (command synopsis, script content, trailing notes) belong to that
+      example until the next title block.
+
+      Args:
+         option (dict): The option whose ``exmraw`` buffer is parsed; the
+                        buffer is removed and one example is recorded per
+                        title block found.
+      """
+      buf = option.pop('exmraw', None)
+      option.pop('inexm', None)
+      if not buf: return
+
+      blocks = []
+      cur = []
+      for ln in buf.split('\n'):
+         if ln.strip() == '':
+            if cur: blocks.append(cur); cur = []
+         else:
+            cur.append(ln)
+      if cur: blocks.append(cur)
+
+      def is_title(block):
+         if not block[-1].rstrip().endswith(':'): return False
+         first = block[0].strip()
+         if first.startswith('<<'): return False
+         if re.match(r'[-*(]', first): return False
+         if re.match(r'{}\b'.format(self.DOCS['DOCNAM']), first): return False
+         return True
+
+      exmtext = None
+      for block in blocks:
+         btext = '\n'.join(block)
+         if is_title(block):
+            if exmtext is not None:
+               self.record_example(option, self.init_example(option['opt'], exmtext))
+            exmtext = btext + "\n"
+         elif exmtext is not None:
+            exmtext += "\n" + btext + "\n"
+         else:
+            option['desc'] += btext + "\n"   # stray text before the first example
+      if exmtext is not None:
+         self.record_example(option, self.init_example(option['opt'], exmtext))
+
    def record_example(self, option, example, ndesc=None):
       """Append the completed *example* to ``self.examples`` and optionally start a new one.
 
@@ -478,18 +540,21 @@ class PgRST(PgFile, PgUtil):
       """Build and return the RST table-of-contents string of a given section.
 
       Produces a nested bullet list of section links (indented by section
-      level) followed by a flat Appendix A list of all example links.
+      level).  For the index (``csection is None``) the example appendix is a
+      standalone page (``appendixA``) added to the toctree; for a section the
+      examples in its subtree are listed inline as a local Appendix A.
 
       Returns:
          str: RST-formatted TOC content ready for ``__TOC__`` substitution.
       """
-      
+
       content = ""
       clevel = csection['level'] if csection else 0
       csecid = csection['secid'] if csection else ""
       depth = self.TLEVEL - clevel
       level = clevel+1
       preid = csecid+'.'
+      is_index = csection is None
 
       # nested bullet list for all sections
       for section in self.sections:
@@ -497,9 +562,14 @@ class PgRST(PgFile, PgUtil):
          if csecid and not secid.startswith(preid): continue
          if section['level'] == level: content += "   section{}\n".format(secid)
 
+      # The full list of examples lives on its own appendix page in the index.
+      if is_index and self.examples: content += "   appendixA\n"
+
       if not content: return ""
 
       content = f".. toctree::\n   :maxdepth: {depth}\n   :caption: Table of Contents\n\n{content}\n"
+      if is_index: return content
+
       # appendix A: list of examples for the parent section and its subsections
       appendix = ""
       idx = 1  # used as example index
@@ -507,7 +577,7 @@ class PgRST(PgFile, PgUtil):
          opt = exm['opt']
          option = self.options[opt]
          secid = option['secid']
-         if not csecid or secid == csecid or secid.startswith(preid):
+         if secid == csecid or secid.startswith(preid):
             appendix += "- :ref:`A.{}. {} Option -{} (-{}) <{}_e{}>`\n".format(
                         idx, option['type'], opt, option['name'], secid, idx)
          idx += 1
@@ -516,6 +586,28 @@ class PgRST(PgFile, PgUtil):
 
       return content
 
+   #
+   # write the appendix page listing all examples in the document
+   #
+   def write_appendix(self):
+      """Write ``appendixA.rst`` listing every example with a link.
+
+      Each entry links to the example anchor on its section page. Does nothing
+      when the document has no examples.
+      """
+      if not self.examples: return
+      content = ""
+      idx = 1
+      for exm in self.examples:
+         option = self.options[exm['opt']]
+         secid = option['secid']
+         title = exm['title'].strip().rstrip(':')
+         content += "- :ref:`A.{}. {} Option -{} (-{}): {} <{}_e{}>`\n".format(
+                     idx, option['type'], exm['opt'], option['name'], title, secid, idx)
+         idx += 1
+
+      self.template_to_rst("appendix", {'CONTENT': content}, "A")
+
    #
    # create a section rst content
    #
@@ -672,7 +764,8 @@ class PgRST(PgFile, PgUtil):
 
       for optary in opts:
          opt = self.get_short_option(optary[1])
-      
+         if opt is None: continue   # not an option of this document; leave as-is
+
          pre = optary[0]
          after = optary[2]
          secid = self.options[opt]['secid']
@@ -826,7 +919,7 @@ class PgRST(PgFile, PgUtil):
       line0 = lines[0]
       normal = 1
       if dtype == 2:
-         ms = re.match(r'^<<(Content .*)>>$', line0)
+         ms = re.match(r'^\s*<<(Content .*)>>$', line0)
          if ms:   # input files for examples
             content += ms.group(1) + ":\n\n.. code-block:: none\n\n"
             normal = 0
@@ -928,13 +1021,23 @@ class PgRST(PgFile, PgUtil):
                rows.append(tuple(prev_vals))
             content = self.build_rst_list_table(rows)
          else:
-            # multi-column table split on 2+ spaces
-            rows = []
-            for i in range(cnt):
-               line = lines[i].strip()
-               vals = re.split(r'\s{2,}', self.replace_option_link(line, secid, 1))
-               rows.append(vals)
-            content = self.build_rst_simple_table(rows) + "\n"
+            raw = [lines[i].strip() for i in range(cnt)]
+            cmdpat = r'(?:[*\d][\d* ]*\s+)?{}(\s|$)'.format(re.escape(self.DOCS['DOCNAM']))
+            if raw and all(re.match(cmdpat, r) for r in raw):
+               # Command line(s) following a label (e.g. a Quick Start entry):
+               # render as a literal block instead of a (degenerate) table.
+               content = ".. code-block:: none\n\n"
+               for r in raw:
+                  content += "   " + r + "\n"
+               content += "\n"
+            else:
+               # multi-column table split on 2+ spaces
+               rows = []
+               for i in range(cnt):
+                  line = lines[i].strip()
+                  vals = re.split(r'\s{2,}', self.replace_option_link(line, secid, 1))
+                  rows.append(vals)
+               content = self.build_rst_simple_table(rows) + "\n"
 
       return content
 
@@ -1046,10 +1149,10 @@ class PgRST(PgFile, PgUtil):
          p (str): Option name to look up (short, long, or alias).
 
       Returns:
-         str: Canonical two-letter option short name.
-
-      Raises:
-         PgLOG error (LGWNEX) if *p* cannot be resolved.
+         str | None: Canonical two-letter option short name, or ``None`` when
+            *p* does not name an option of this document (e.g. an option of a
+            different program referenced in prose).  Callers skip such tokens
+            so unrelated option-like text is left untouched.
       """
       plen = len(p)
       if plen == 2 and p in self.options: return p
@@ -1061,7 +1164,7 @@ class PgRST(PgFile, PgUtil):
          for alias in self.ALIAS[opt]:
             if re.match(r'^{}$'.format(alias), p, re.I): return opt
 
-      self.pglog("{} - unknown option for {}".format(p, self.DOCS['DOCNAM']), self.LGWNEX)
+      return None
 
    #
    # replace with rst link for a given section title

rda_python_miscs/rst_templates/appendix.rst.temp

@@ -0,0 +1,19 @@
+################################################################################
+#
+#     Title : appendix_rst.temp
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 03/17/2026
+#   Purpose : template file for help document appendixA.rst (reStructuredText)
+#    Github : https://github.com/NCAR/rda-python-mics.git
+#
+################################################################################
+
+.. _appendixA:
+
+============================
+Appendix A: List of Examples
+============================
+
+__CONTENT__
+
+| :ref:`Back to Table of Contents <index>`

{rda_python_miscs-3.0.3.dist-info → rda_python_miscs-3.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.3
+Version: 3.0.5
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
@@ -45,6 +45,8 @@ The package provides two categories of programs:
 | `gdexcp` | `rdacp` | `setuid_gdexcp` / `setuid_rdacp` | Copy files and directories across local, remote, Object Store, or Globus endpoints |
 | `gdexkill` | `rdakill` | `setuid_gdexkill` / `setuid_rdakill` | Kill local processes and their children, or cancel PBS batch jobs |
 | `gdexmod` | `rdamod` | `setuid_gdexmod` / `setuid_rdamod` | Change permission modes for files and directories owned by gdexdata |
+| `decsdata_storage` | | `setuid_decsdata_storage` | Move decsdata datasets into the GLADE HSM cold storage to migrate them onto tape |
+| `decsdata_restore` | | `setuid_decsdata_restore` | Recall decsdata datasets out of the GLADE HSM cold storage and copy them back |
 
 ## Environment setup
 
@@ -103,8 +105,8 @@ pip install rda_python_miscs
 
 ## Setuid Setup
 
-The setuid programs (`gdexcp`, `gdexkill`, `gdexmod` and their `rda*` aliases)
-execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via
+The setuid programs (`gdexcp`, `gdexkill`, `gdexmod`, `decsdata_storage`,
+`decsdata_restore` and the `rda*` aliases) execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via
 the `rda_python_setuid` mechanism, which is pulled in automatically as a
 dependency.  After `pip install` above, choose one of the wiring options
 below.
@@ -159,3 +161,30 @@ pywrapper-install -u|--update
 The shared setuid setup guide is shown automatically if any `setuid_*`
 connector script is invoked directly before the setuid wrapper has been
 configured.
+
+## Cold storage setup
+
+`decsdata_storage` and `decsdata_restore` drive the GLADE HSM through the
+`glade_hsm` script, which is not part of this package.  They look it up at
+`~benkirk/glade_hsm` unless the environment variable `GLADE_HSM` names another
+copy of it:
+
+```bash
+export GLADE_HSM=/path/to/glade_hsm
+```
+
+Both programs work on the decsdata directory `PGLOG['DECSHOME']` (default
+`/glade/campaign/collections/gdex/decsdata`, also reachable as
+`/gdex/decsdata`), which option `-w` overrides.  Only members of the DECS group
+may run them, and `decsdata_restore` reads Table `sfile` in RDADB to size a
+restore, so a working RDADB login is required as well.
+
+Because the decsdata quota is limited, check the space left with `gladequota`
+before a large restore:
+
+```bash
+pgstart_gdexdata gladequota    # or plain 'gladequota' when logged in as gdexdata
+```
+
+`decsdata_restore -x` runs the same check itself and refuses to request a
+recall unless twice the size of the data is still available.

{rda_python_miscs-3.0.3.dist-info → rda_python_miscs-3.0.5.dist-info}/RECORD

@@ -2,6 +2,10 @@ rda_python_miscs/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,
 rda_python_miscs/bash_qsub.py,sha256=NsYg1A7zeRy3mR_rAaJEQoWSDaWFxE7cpHNc4mA_JwA,6664
 rda_python_miscs/bashqsub.py,sha256=SadPwz1ZXHx2zGNYb2VityItQjMEYj-Favq3xDrPBK0,10127
 rda_python_miscs/bashqsub.usg,sha256=QDaFr1FiQ-5FC9UzCRQ0geYjsPXBnVMhe3lM8qH1xss,2176
+rda_python_miscs/decsdata_restore.py,sha256=JmEiYhQyE_Z6PZstrm35-ImYVz4j7cpffhhLiS4UGHA,20400
+rda_python_miscs/decsdata_restore.usg,sha256=jOD_AbNV4C-dOp9mTkV2MVcUgcf2kf5Q9QqhCS7NENk,4945
+rda_python_miscs/decsdata_storage.py,sha256=5n8bOVIpGxnX7beDNGmrgj4z1OwmU4-9qVsDCbSuymY,8637
+rda_python_miscs/decsdata_storage.usg,sha256=5_OXsIxyKjA46xjycaTgVVIE0c39lkkSekcVliFNgaI,2812
 rda_python_miscs/gdex_ls.py,sha256=fg9jfYajOT8ps6eEhFUUqmQ791BdGwBkrT_pyjmGGvQ,8860
 rda_python_miscs/gdexcp.py,sha256=jCkDzNrZhi7vhZqeSXHR7d1XIKtsionSJE5ntWoZE0g,18110
 rda_python_miscs/gdexcp.usg,sha256=6r8XVi9inPuWuFth6CAzcd3EpXX4Y_ZrT6YubmQ82kM,6172
@@ -21,7 +25,7 @@ rda_python_miscs/gdexsub.usg,sha256=SChXnDzo6larOdpR9eZ2CMu8ExJYQUdfFHUzxcGoqJ0,
 rda_python_miscs/gdexzip.py,sha256=wzMVTqeahXAUHsKAtqYG6wrM4J2EahnncrpliCqFyNQ,3252
 rda_python_miscs/gdexzip.usg,sha256=cG1Uwa8WZ3KQNSaSkr29edUEaaLb_4cHTU3GRoZSQ84,1918
 rda_python_miscs/pg_docs.py,sha256=_AoqrWroUu7FQMVWaTSTZBQMwi1BH1-RjKb1nDHgxOI,25369
-rda_python_miscs/pg_rst.py,sha256=XldjKneKuciYvZxrHC3h3_OHwyhh51LKYOuKr0s_2Gg,48480
+rda_python_miscs/pg_rst.py,sha256=Wd4qfeetQYvRio9P2YE2x4kUCDI6ZEcX8CQiI0Xqmyk,53063
 rda_python_miscs/pg_rst.usg,sha256=oYvVEqzHeVmDntN7hfpM8vc6eoYRyQTruQQg2mPfOo4,2484
 rda_python_miscs/pg_wget.py,sha256=YK8EJQtTwGXxHav_SveXOwE_oA9KZ9l12lGSf77JQnE,7571
 rda_python_miscs/pgwget.py,sha256=TFHyquKT0tyz-r-RUAV_wpdwktpVRwyZGrHnZF2B3Vk,8115
@@ -38,11 +42,12 @@ rda_python_miscs/rdals.usg,sha256=ChF-nn3Qb2pds3wMXIWubB_tjwZslxHfSha0dTDqOiY,29
 rda_python_miscs/tcsh_qsub.py,sha256=P4Obzbp5Dvy-N0eRR5F51R9yj3ttWJo6g1NqyVDO6-Y,6670
 rda_python_miscs/tcshqsub.py,sha256=QxBq9MdVIUs9t2d6vHhkxM1nrcLwRNqcq1lJiWhXKUM,10124
 rda_python_miscs/tcshqsub.usg,sha256=JYfhrK7cqme-Sij_JfquONOs3HMu-d5dDGI9K_RdudU,2180
+rda_python_miscs/rst_templates/appendix.rst.temp,sha256=HCujxbj-R4_8FgOrLSlMZcBQHUu8f6kfBMWVRygl7jc,560
 rda_python_miscs/rst_templates/index.rst.temp,sha256=YSa1JM6X9x2SC6UiqJu_9xRrVYGLKwNI43DBJEGDX08,523
 rda_python_miscs/rst_templates/section.rst.temp,sha256=-CUtutvctG2tIdkqrkFxVEzmNN3atRJXBDQaTnMJ6Gw,572
-rda_python_miscs-3.0.3.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_miscs-3.0.3.dist-info/METADATA,sha256=b4yrhKT1F_QySBljOiOujWrWLMStJO0Vymjx8dO47f4,5803
-rda_python_miscs-3.0.3.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-rda_python_miscs-3.0.3.dist-info/entry_points.txt,sha256=pBgb-_g4yZhm6YynwDHtNTAzxVrb8SoDMd7Eiys8gv4,806
-rda_python_miscs-3.0.3.dist-info/top_level.txt,sha256=W5rz7DrWb7hXABUbGgWcwe6D644X338LR8_zdgmtLhg,17
-rda_python_miscs-3.0.3.dist-info/RECORD,,
+rda_python_miscs-3.0.5.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_miscs-3.0.5.dist-info/METADATA,sha256=9OT0Q0nHql2BdJI85ffYfp6EJD4Qm2KmYgn2R2ny9KM,7106
+rda_python_miscs-3.0.5.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+rda_python_miscs-3.0.5.dist-info/entry_points.txt,sha256=vAM87BmdEbJKdLg92NaffFG_xCdb9JxR2PZgsCjB8yw,936
+rda_python_miscs-3.0.5.dist-info/top_level.txt,sha256=W5rz7DrWb7hXABUbGgWcwe6D644X338LR8_zdgmtLhg,17
+rda_python_miscs-3.0.5.dist-info/RECORD,,

{rda_python_miscs-3.0.3.dist-info → rda_python_miscs-3.0.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (82.0.1)
+Generator: setuptools (84.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{rda_python_miscs-3.0.3.dist-info → rda_python_miscs-3.0.5.dist-info}/entry_points.txt

@@ -11,6 +11,8 @@ rdaown = rda_python_miscs.gdexown:main
 rdaps = rda_python_miscs.gdexps:main
 rdasub = rda_python_miscs.gdexsub:main
 rdazip = rda_python_miscs.gdexzip:main
+setuid_decsdata_restore = rda_python_miscs.decsdata_restore:main
+setuid_decsdata_storage = rda_python_miscs.decsdata_storage:main
 setuid_gdexcp = rda_python_miscs.gdexcp:main
 setuid_gdexkill = rda_python_miscs.gdexkill:main
 setuid_gdexmod = rda_python_miscs.gdexmod:main