rda-python-miscs 3.0.4__tar.gz → 3.0.5__tar.gz

{rda_python_miscs-3.0.4/src/rda_python_miscs.egg-info → rda_python_miscs-3.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.4
+Version: 3.0.5
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
@@ -45,6 +45,8 @@ The package provides two categories of programs:
 | `gdexcp` | `rdacp` | `setuid_gdexcp` / `setuid_rdacp` | Copy files and directories across local, remote, Object Store, or Globus endpoints |
 | `gdexkill` | `rdakill` | `setuid_gdexkill` / `setuid_rdakill` | Kill local processes and their children, or cancel PBS batch jobs |
 | `gdexmod` | `rdamod` | `setuid_gdexmod` / `setuid_rdamod` | Change permission modes for files and directories owned by gdexdata |
+| `decsdata_storage` | | `setuid_decsdata_storage` | Move decsdata datasets into the GLADE HSM cold storage to migrate them onto tape |
+| `decsdata_restore` | | `setuid_decsdata_restore` | Recall decsdata datasets out of the GLADE HSM cold storage and copy them back |
 
 ## Environment setup
 
@@ -103,8 +105,8 @@ pip install rda_python_miscs
 
 ## Setuid Setup
 
-The setuid programs (`gdexcp`, `gdexkill`, `gdexmod` and their `rda*` aliases)
-execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via
+The setuid programs (`gdexcp`, `gdexkill`, `gdexmod`, `decsdata_storage`,
+`decsdata_restore` and the `rda*` aliases) execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via
 the `rda_python_setuid` mechanism, which is pulled in automatically as a
 dependency.  After `pip install` above, choose one of the wiring options
 below.
@@ -159,3 +161,30 @@ pywrapper-install -u|--update
 The shared setuid setup guide is shown automatically if any `setuid_*`
 connector script is invoked directly before the setuid wrapper has been
 configured.
+
+## Cold storage setup
+
+`decsdata_storage` and `decsdata_restore` drive the GLADE HSM through the
+`glade_hsm` script, which is not part of this package.  They look it up at
+`~benkirk/glade_hsm` unless the environment variable `GLADE_HSM` names another
+copy of it:
+
+```bash
+export GLADE_HSM=/path/to/glade_hsm
+```
+
+Both programs work on the decsdata directory `PGLOG['DECSHOME']` (default
+`/glade/campaign/collections/gdex/decsdata`, also reachable as
+`/gdex/decsdata`), which option `-w` overrides.  Only members of the DECS group
+may run them, and `decsdata_restore` reads Table `sfile` in RDADB to size a
+restore, so a working RDADB login is required as well.
+
+Because the decsdata quota is limited, check the space left with `gladequota`
+before a large restore:
+
+```bash
+pgstart_gdexdata gladequota    # or plain 'gladequota' when logged in as gdexdata
+```
+
+`decsdata_restore -x` runs the same check itself and refuses to request a
+recall unless twice the size of the data is still available.

{rda_python_miscs-3.0.4 → rda_python_miscs-3.0.5}/README.md

@@ -28,6 +28,8 @@ The package provides two categories of programs:
 | `gdexcp` | `rdacp` | `setuid_gdexcp` / `setuid_rdacp` | Copy files and directories across local, remote, Object Store, or Globus endpoints |
 | `gdexkill` | `rdakill` | `setuid_gdexkill` / `setuid_rdakill` | Kill local processes and their children, or cancel PBS batch jobs |
 | `gdexmod` | `rdamod` | `setuid_gdexmod` / `setuid_rdamod` | Change permission modes for files and directories owned by gdexdata |
+| `decsdata_storage` | | `setuid_decsdata_storage` | Move decsdata datasets into the GLADE HSM cold storage to migrate them onto tape |
+| `decsdata_restore` | | `setuid_decsdata_restore` | Recall decsdata datasets out of the GLADE HSM cold storage and copy them back |
 
 ## Environment setup
 
@@ -86,8 +88,8 @@ pip install rda_python_miscs
 
 ## Setuid Setup
 
-The setuid programs (`gdexcp`, `gdexkill`, `gdexmod` and their `rda*` aliases)
-execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via
+The setuid programs (`gdexcp`, `gdexkill`, `gdexmod`, `decsdata_storage`,
+`decsdata_restore` and the `rda*` aliases) execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via
 the `rda_python_setuid` mechanism, which is pulled in automatically as a
 dependency.  After `pip install` above, choose one of the wiring options
 below.
@@ -142,3 +144,30 @@ pywrapper-install -u|--update
 The shared setuid setup guide is shown automatically if any `setuid_*`
 connector script is invoked directly before the setuid wrapper has been
 configured.
+
+## Cold storage setup
+
+`decsdata_storage` and `decsdata_restore` drive the GLADE HSM through the
+`glade_hsm` script, which is not part of this package.  They look it up at
+`~benkirk/glade_hsm` unless the environment variable `GLADE_HSM` names another
+copy of it:
+
+```bash
+export GLADE_HSM=/path/to/glade_hsm
+```
+
+Both programs work on the decsdata directory `PGLOG['DECSHOME']` (default
+`/glade/campaign/collections/gdex/decsdata`, also reachable as
+`/gdex/decsdata`), which option `-w` overrides.  Only members of the DECS group
+may run them, and `decsdata_restore` reads Table `sfile` in RDADB to size a
+restore, so a working RDADB login is required as well.
+
+Because the decsdata quota is limited, check the space left with `gladequota`
+before a large restore:
+
+```bash
+pgstart_gdexdata gladequota    # or plain 'gladequota' when logged in as gdexdata
+```
+
+`decsdata_restore -x` runs the same check itself and refuses to request a
+recall unless twice the size of the data is still available.

{rda_python_miscs-3.0.4 → rda_python_miscs-3.0.5}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_miscs"
-version = "3.0.4"
+version = "3.0.5"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]
@@ -52,3 +52,5 @@ pythonpath = [
 "setuid_gdexmod" = "rda_python_miscs.gdexmod:main"
 "setuid_rdamod" = "rda_python_miscs.gdexmod:main"
 "pgrst" = "rda_python_miscs.pg_rst:main"
+"setuid_decsdata_storage" = "rda_python_miscs.decsdata_storage:main"
+"setuid_decsdata_restore" = "rda_python_miscs.decsdata_restore:main"

rda_python_miscs-3.0.5/src/rda_python_miscs/decsdata_restore.py

@@ -0,0 +1,456 @@
+#!/usr/bin/env python3
+##################################################################################
+#     Title: decsdata_restore
+#    Author: Zaihua Ji, zji@ucar.edu
+#      Date: 2026-08-31
+#   Purpose: restore decsdata datasets, or parts of them, out of the GLADE HSM
+#            cold storage; the opposite of decsdata_storage
+#    Github: https://github.com/NCAR/rda-python-miscs.git
+##################################################################################
+import re
+import os
+import sys
+from os import path as op
+from rda_python_common.pg_file import PgFile
+
+class DecsRestore(PgFile):
+   """Restore decsdata datasets out of the GLADE HSM cold storage.
+
+   'glade_hsm recall' only submits a request that the HSM batch processes
+   fulfill later on, so restoring is done in three steps:
+
+      -x  submit the recall requests for the given datasets/paths
+      -s  check the recall status, repeat until nothing is left on tape
+      -r  copy the recalled data back under the decsdata directory
+
+   Recalled files stay readable inside cold storage for 7 days only, after
+   which they are migrated onto tape again, so step -r must be done within
+   that window.
+   """
+
+   def __init__(self):
+      """Initialize DecsRestore with default option values and runtime state."""
+      super().__init__()
+      self.HSM = os.environ.get('GLADE_HSM', op.expanduser('~benkirk/glade_hsm'))
+      self.VALOPTS = 'Dwl'   # single-value options
+      self.MULOPTS = 'd'     # multi-value options
+      self.MODOPTS = 'hf'    # mode options
+      self.ACTOPTS = 'xsr'   # action options, one and only one is required
+      self.OPTS = {
+         'D': None,   # cold storage date, in YYYYMMDD or YYYY-MM-DD
+         'w': None,   # decsdata directory, defaults to PGLOG['DECSHOME']
+         'l': None,   # dataset list file
+         'd': [],     # dataset IDs, a sub-path may be appended to each one
+         't': None,   # target directory of -r, defaults to the decsdata directory
+         'h': 0,      # 1 to show help message
+         'f': 0,      # 1 to copy back while files are still on tape
+      }
+      self.ACTION = None     # one of the ACTOPTS letters
+      self.SIZEUNITS = {     # units reported by 'gladequota', in bytes
+         'B': 1, 'KIB': 1024, 'MIB': 1024**2,
+         'GIB': 1024**3, 'TIB': 1024**4, 'PIB': 1024**5,
+      }
+      self.RINFO = {
+         'decsdir': None,   # decsdata directory the data is restored into
+         'roots': [],       # cold storage directories to look the data up in
+         'acnt': 0,         # number of dataset paths acted on successfully
+      }
+
+   # function to read parameters
+   def read_parameters(self):
+      """Parse the command line into the OPTS option values and the action.
+
+      Single-value options -D, -w and -l take one value each, multi-value
+      option -d gathers every following dataset path, mode options -h and -f
+      are simple flags, and action options -x, -s and -r are mutually
+      exclusive; -r optionally takes a target directory.  Exits with usage if
+      -h is given or no action is specified.
+      """
+      self.set_suid(self.PGLOG['EUID'])
+      self.set_help_path(__file__)
+      self.PGLOG['LOGFILE'] = "decsdata_restore.log"   # set different log file
+      argv = sys.argv[1:]
+      self.cmdlog("decsdata_restore {}".format(' '.join(argv)))
+      option = None
+      for arg in argv:
+         ms = re.match(r'^-(\w+)$', arg)
+         if ms:
+            option = ms.group(1)
+            if option in self.ACTOPTS:
+               self.set_action(option)
+               if option != 'r': option = None   # -r may be followed by a target directory
+            elif option in self.MODOPTS:
+               self.OPTS[option] = 1
+               option = None
+            elif option not in self.VALOPTS and option not in self.MULOPTS:
+               self.pglog(arg + ": Unknown Option", self.LGEREX)
+            continue
+         if not option: self.pglog(arg + ": Value provided without option", self.LGEREX)
+         if option in self.MULOPTS:
+            self.OPTS[option].append(arg)   # gather all values until the next option
+         else:
+            if option == 'r': option = 't'   # the value of -r is the target directory
+            self.OPTS[option] = arg
+            option = None
+      if self.OPTS['h'] or not self.ACTION: self.show_usage("decsdata_restore")
+
+   # remember the action option and reject a second one
+   def set_action(self, option):
+      """Record the single action option to perform.
+
+      Args:
+         option (str): One of the ACTOPTS letters.
+      """
+      if self.ACTION and self.ACTION != option:
+         self.pglog("-{}: Cannot combine with Action -{}".format(option, self.ACTION), self.LGEREX)
+      self.ACTION = option
+
+   # function to start actions
+   def start_actions(self):
+      """Validate the caller, resolve the cold storage paths, and act on each dataset path."""
+      self.dssdb_dbname()
+      self.validate_decs_group('decsdata_restore', self.PGLOG['CURUID'], 1)
+      self.set_restore_paths()
+      specs = self.get_dataset_list()
+      if not specs: self.pglog("No dataset found to restore", self.LGWNEX)
+      if self.ACTION == 'x': self.check_restore_space(specs)
+      for spec in specs:
+         self.restore_one_path(spec)
+      acts = {'x': 'requested Recall', 's': 'checked Status', 'r': 'copied back'}
+      s = ('s' if self.RINFO['acnt'] > 1 else '')
+      self.pglog("{} of {} Dataset Path{} {}".format(self.RINFO['acnt'], len(specs),
+                 s, acts[self.ACTION]), self.LOGWRN)
+      self.cmdlog()
+
+   # resolve the decsdata directory and the cold storage directories
+   def set_restore_paths(self):
+      """Fill RINFO with the decsdata directory and the cold storage directories to search.
+
+      For a given -D date only '<decsdata>/cold_storage_<date>/COLD_STORAGE' is
+      searched.  Without -D both '<decsdata>/COLD_STORAGE' and every
+      '<decsdata>/cold_storage_<YYYYMMDD>/COLD_STORAGE' are searched, the most
+      recent dated one first.
+      """
+      decsdir = self.OPTS['w'] if self.OPTS['w'] else self.PGLOG['DECSHOME']
+      if not self.check_local_file(decsdir, 0, self.LOGWRN):
+         self.pglog(decsdir + ": decsdata directory NOT exists", self.LGEREX)
+      self.RINFO['decsdir'] = decsdir
+      if not self.OPTS['t']: self.OPTS['t'] = decsdir
+      roots = []
+      if self.OPTS['D']:
+         date = re.sub('-', '', self.OPTS['D'])
+         if not re.match(r'^\d{8}$', date):
+            self.pglog(date + ": Invalid cold storage date, YYYYMMDD expected", self.LGEREX)
+         roots.append(self.join_paths(decsdir, "cold_storage_{}/COLD_STORAGE".format(date)))
+      else:
+         roots.append(self.join_paths(decsdir, "COLD_STORAGE"))
+         files = self.local_glob(self.join_paths(decsdir, "cold_storage_[0-9]*/COLD_STORAGE"), 0, self.LOGWRN)
+         for file in sorted(files, reverse = True):
+            if not files[file]['isfile']: roots.append(file)
+      for root in roots:
+         if self.check_local_file(root, 0, 0): self.RINFO['roots'].append(root)
+      if not self.RINFO['roots']:
+         self.pglog("{}: No cold storage directory found in {}".format(', '.join(roots), decsdir), self.LGEREX)
+      self.pglog("Cold storage searched: {}".format(', '.join(self.RINFO['roots'])), self.LOGWRN)
+
+   # gather the dataset paths to restore
+   def get_dataset_list(self):
+      """Return the list of dataset paths to restore.
+
+      Uses the -d values if given.  Otherwise reads the -l list file, creating
+      it first from every dNNNNNN directory in the cold storage directories if
+      it does not exist yet.
+
+      Returns:
+         list[str]: Dataset IDs, each optionally followed by a sub-path.
+      """
+      if self.OPTS['d']:
+         self.pglog("Restore {} given Dataset Path(s)".format(len(self.OPTS['d'])), self.LOGWRN)
+         return self.OPTS['d']
+      lstfile = self.OPTS['l'] if self.OPTS['l'] else "dsids_{}.lst".format(re.sub('-', '', self.curdate()))
+      if not op.isfile(lstfile):
+         dsids = self.get_coldstorage_datasets()
+         with open(lstfile, 'w') as OUT:
+            for dsid in dsids: OUT.write(dsid + "\n")
+         self.pglog("{}: Generated with {} Dataset(s)".format(lstfile, len(dsids)), self.LOGWRN)
+      specs = []
+      with open(lstfile, 'r') as IN:
+         for line in IN:
+            line = line.strip()
+            if line: specs.append(line)
+      self.pglog("{}: Read {} Dataset Path(s) to restore".format(lstfile, len(specs)), self.LOGWRN)
+      return specs
+
+   # find all dNNNNNN dataset directories in the cold storage directories
+   def get_coldstorage_datasets(self):
+      """Return the sorted dataset IDs of every dNNNNNN directory in the cold storage directories.
+
+      Returns:
+         list[str]: Unique dataset IDs; plain files matching the pattern are skipped.
+      """
+      dsids = []
+      for root in self.RINFO['roots']:
+         files = self.local_glob(self.join_paths(root, "d" + "[0-9]"*6), 0, self.LOGWRN)
+         for file in files:
+            if files[file]['isfile']: continue
+            dsid = op.basename(file)
+            if dsid not in dsids: dsids.append(dsid)
+      return sorted(dsids)
+
+   # locate a dataset path in the cold storage directories, the first match wins
+   def find_cold_path(self, spec):
+      """Look a dataset path up in each cold storage directory.
+
+      Warns and names the ignored ones if the path is found in more than one
+      cold storage directory.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+
+      Returns:
+         tuple: (path, info dict) of the first match, or (None, None).
+      """
+      hits = {}
+      for root in self.RINFO['roots']:
+         path = self.join_paths(root, spec)
+         info = self.check_local_file(path, 0, 0)
+         if info: hits[path] = info
+      if not hits: return (None, None)
+      paths = list(hits)
+      if len(paths) > 1:
+         self.pglog("{}: Found in {} cold storage directories, use {}".format(spec, len(paths), paths[0]), self.LOGWRN)
+         self.pglog("{}: Ignored {}".format(spec, ', '.join(paths[1:])), self.LOGWRN)
+      return (paths[0], hits[paths[0]])
+
+   # count the files of a cold storage path still on tape
+   def hsm_offline_count(self, path, isfile):
+      """Return the number of files under a cold storage path that are still on tape.
+
+      Args:
+         path (str): Cold storage path of a file or directory.
+         isfile (int): 1 if path is a regular file, 0 for a directory.
+
+      Returns:
+         int | None: Count of offline files, or None if it cannot be determined.
+      """
+      out = self.pgsystem("{} status {}".format(self.HSM, path), self.LOGWRN, 51)
+      if not out: return None
+      if isfile: return (1 if re.search(r'migrated', out) else 0)
+      cnts = re.findall(r'Offline:\s*([\d,]+)', out)
+      if not cnts: return None
+      return int(re.sub(',', '', cnts[-1]))
+
+   # build the sfile condition of one dataset path
+   def sfile_condition(self, spec):
+      """Turn a dataset path into a condition on table dssdb.sfile.
+
+      A saved file lives in '<decsdata>/<dsid>/<type>/<sfile>', so the first
+      component of the path is the dataset ID, the second one the saved file
+      type, and the rest the leading part of the sfile field.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+
+      Returns:
+         str: The WHERE condition of the saved files under the path.
+      """
+      paths = spec.strip('/').split('/')
+      if not re.match(r'^[a-z]\d{6}$', paths[0]):
+         self.pglog(spec + ": Invalid dataset path, dNNNNNN expected", self.LGEREX)
+      cnd = "dsid = '{}'".format(paths[0])
+      if len(paths) > 1:
+         if not re.match(r'^\w$', paths[1]):
+            self.pglog(spec + ": Invalid saved file type, one word character expected", self.LGEREX)
+         cnd += " AND type = '{}'".format(paths[1])
+      if len(paths) > 2:
+         sfile = '/'.join(paths[2:])
+         if re.search(r"['\\]", sfile):
+            self.pglog(spec + ": Invalid saved file path", self.LGEREX)
+         # the path is either a saved file itself or the directory holding them
+         cnd += " AND (sfile = '{}' OR sfile LIKE '{}/%')".format(sfile, re.sub(r'([%_])', r'\\\1', sfile))
+      return cnd
+
+   # get the archived size of one dataset path
+   def restore_data_size(self, spec):
+      """Return the total size of the saved files under one dataset path.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+
+      Returns:
+         int: Number of bytes recorded in dssdb.sfile; 0 if nothing is found.
+      """
+      pgrec = self.pgget('sfile', "sum(data_size) tsize, count(sid) fcnt",
+                         self.sfile_condition(spec), self.LOGWRN)
+      if not pgrec or not pgrec['tsize']:
+         self.pglog(spec + ": No saved file found in RDADB", self.LOGWRN)
+         return 0
+      self.pglog("{}: {} in {} saved file(s)".format(spec,
+                 self.format_float_value(pgrec['tsize']), pgrec['fcnt']), self.LOGWRN)
+      return int(pgrec['tsize'])
+
+   # get the GLADE space left for the decsdata directory
+   def decsdata_free_size(self):
+      """Return the GLADE space left on the quota holding the decsdata directory.
+
+      Parses the 'Used' and 'Quota' columns of 'gladequota' and picks the
+      longest reported path the decsdata directory falls under.
+
+      Returns:
+         int | None: Number of free bytes, or None if it cannot be determined.
+      """
+      cmd = self.get_local_command("gladequota", self.PGLOG['COMMONUSER'])
+      out = self.pgsystem(cmd, self.LOGWRN, 21)   # 1+4+16, log the command and return stdout
+      if not out: return None
+      target = op.realpath(self.RINFO['decsdir'])
+      (fsize, fpath) = (None, None)
+      for line in out.split('\n'):
+         ms = re.match(r'^(/\S+)\s+([\d.]+)\s*(\w+)\s+([\d.]+)\s*(\w+)', line)
+         if not ms: continue   # skips the header and the 'n/a' quota lines
+         path = ms.group(1)
+         if not (target == path or target.startswith(path + '/')): continue
+         if fpath and len(fpath) >= len(path): continue   # keeps the closest path only
+         used = self.quota_size(ms.group(2), ms.group(3))
+         quota = self.quota_size(ms.group(4), ms.group(5))
+         if used is None or quota is None: continue
+         (fsize, fpath) = (max(quota - used, 0), path)
+      return fsize
+
+   # convert one 'gladequota' size into bytes
+   def quota_size(self, value, unit):
+      """Convert one size reported by 'gladequota' into bytes.
+
+      Args:
+         value (str): The numeric part of the size.
+         unit (str): The unit of the size, such as 'TiB'.
+
+      Returns:
+         int | None: Number of bytes, or None for an unknown unit.
+      """
+      unit = unit.upper()
+      if unit not in self.SIZEUNITS: return None
+      return int(float(value)*self.SIZEUNITS[unit])
+
+   # make sure the decsdata directory has room for the whole restore
+   def check_restore_space(self, specs):
+      """Stop the recall if the decsdata directory cannot hold the whole restore.
+
+      The size to restore is added up from table dssdb.sfile and compared to
+      the GLADE space left for the decsdata directory.  Twice the size is
+      required, since the recall brings the data back on disk inside the cold
+      storage first and Action -r copies it back afterwards, so both copies
+      live under the decsdata quota at the same time.  The check is skipped,
+      with a warning, if either size cannot be determined.
+
+      Args:
+         specs (list[str]): Dataset paths to recall.
+      """
+      tsize = 0
+      for spec in specs:
+         tsize += self.restore_data_size(spec)
+      if not tsize:
+         self.pglog("Unknown size to restore, Skip checking the decsdata space", self.LOGWRN)
+         return
+      fsize = self.decsdata_free_size()
+      if fsize is None:
+         self.pglog("{}: Cannot get the space left, Skip checking the decsdata space".format(self.RINFO['decsdir']), self.LOGWRN)
+         return
+      nsize = 2*tsize   # room for the recalled copy and for the copy of Action -r
+      msg = "{}: Restore {}, needs {} of the {} left".format(self.RINFO['decsdir'],
+            self.format_float_value(tsize), self.format_float_value(nsize),
+            self.format_float_value(fsize))
+      if nsize > fsize:
+         self.pglog(msg + ", NOT enough space", self.LGEREX)
+      self.pglog(msg, self.LOGWRN)
+
+   # act on one dataset path in cold storage
+   def restore_one_path(self, spec):
+      """Perform the requested action on one dataset path in cold storage.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+      """
+      (path, info) = self.find_cold_path(spec)
+      if not path:
+         self.pglog(spec + ": NOT found in cold storage", self.LOGERR)
+         return
+      if self.ACTION == 'x':
+         self.recall_cold_path(spec, path)
+      elif self.ACTION == 's':
+         self.status_cold_path(spec, path, info['isfile'])
+      else:
+         self.copy_cold_path(spec, path, info['isfile'])
+
+   # submit the recall request of one cold storage path
+   def recall_cold_path(self, spec, path):
+      """Submit the HSM recall request for one cold storage path.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+         path (str): Cold storage path of the data.
+      """
+      if self.pgsystem("{} recall -f {}".format(self.HSM, path), self.LOGWRN, 7):
+         self.RINFO['acnt'] += 1
+         self.pglog("{}: Recall requested, check the progress via -s".format(spec), self.LOGWRN)
+      else:
+         self.pglog("{}: Error request Recall of {}".format(spec, path), self.LOGERR)
+
+   # report the recall status of one cold storage path
+   def status_cold_path(self, spec, path, isfile):
+      """Report the HSM and recall status of one cold storage path.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+         path (str): Cold storage path of the data.
+         isfile (int): 1 if path is a regular file, 0 for a directory.
+      """
+      offline = self.hsm_offline_count(path, isfile)
+      if offline is None:
+         self.pglog("{}: Cannot get the offline file count of {}".format(spec, path), self.LOGERR)
+         return
+      self.RINFO['acnt'] += 1
+      if offline > 0:
+         s = ('s' if offline > 1 else '')
+         self.pglog("{}: Recall PENDING, {} File{} still on tape".format(spec, offline, s), self.LOGWRN)
+      else:
+         self.pglog("{}: Recall COMPLETE, ready to copy back via -r".format(spec), self.LOGWRN)
+
+   # copy one recalled cold storage path back into the decsdata directory
+   def copy_cold_path(self, spec, path, isfile):
+      """Copy one recalled cold storage path back to its target directory.
+
+      Nothing is copied while files are still on tape unless -f is given.
+
+      Args:
+         spec (str): Dataset ID, optionally followed by a sub-path.
+         path (str): Cold storage path of the data.
+         isfile (int): 1 if path is a regular file, 0 for a directory.
+      """
+      offline = self.hsm_offline_count(path, isfile)
+      if offline:
+         s = ('s' if offline > 1 else '')
+         if not self.OPTS['f']:
+            self.pglog("{}: {} File{} still on tape, add Mode -f to copy anyway".format(spec, offline, s), self.LOGERR)
+            return
+         self.pglog("{}: {} File{} still on tape, copy it anyway".format(spec, offline, s), self.LOGWRN)
+      tofile = self.join_paths(self.OPTS['t'], spec)
+      # a directory is copied as '<path>/.' to merge into an existing target
+      fromfile = path if isfile else path + "/."
+      if self.local_copy_local(tofile, fromfile, self.LOGWRN):
+         self.RINFO['acnt'] += 1
+         self.pglog("{}: Copied back to {}".format(spec, tofile), self.LOGWRN)
+      else:
+         self.pglog("{}: Error copy {} back to {}".format(spec, path, tofile), self.LOGERR)
+
+# main function to execute this script
+def main():
+   """Entry point: instantiate DecsRestore, parse arguments, run, and exit."""
+   from rda_python_setuid.setup_guide import show_setup_guide
+   object = DecsRestore()
+   show_setup_guide(object, 'rda_python_miscs', ['decsdata_storage', 'decsdata_restore'])
+   object.read_parameters()
+   object.start_actions()
+   object.pgexit(0)
+
+# call main() to start program
+if __name__ == "__main__": main()

rda_python_miscs-3.0.5/src/rda_python_miscs/decsdata_restore.usg

@@ -0,0 +1,101 @@
+
+ Restore decsdata datasets, or parts of them, out of the GLADE HSM cold storage.
+ This is the opposite of 'decsdata_storage'.  A recall request is only submitted
+ by 'glade_hsm recall' and is fulfilled by the HSM batch processes later on, so
+ restoring is done in three steps:
+
+    1. Option -x submits the recall requests for the given dataset paths;
+    2. Option -s checks the recall status, repeat it until nothing is on tape;
+    3. Option -r copies the recalled data back into the decsdata directory.
+
+ Recalled files stay readable inside the 'COLD_STORAGE/' path for 7 days only,
+ after which they are migrated onto tape again, so step 3 must be done within
+ that window.
+
+ Usage: decsdata_restore [-D Date] [-w DecsdataDirectory] [-l ListFile]  \
+                         [-d DatasetPathList] [-f] -x|-s|-r [TargetDirectory]
+
+      - Option -D Date, the cold storage date, in YYYYMMDD or YYYY-MM-DD.  Only
+          '<decsdata>/cold_storage_<date>/COLD_STORAGE' is searched for the
+          data.  Without it, '<decsdata>/COLD_STORAGE' and every
+          '<decsdata>/cold_storage_<YYYYMMDD>/COLD_STORAGE' are all searched,
+          the most recent dated one first.  A dataset path found in more than
+          one of them is acted on in the first one only, and the ignored ones
+          are logged;
+
+      - Option -w DecsdataDirectory, the decsdata directory holding the cold
+          storage directories, and the directory the data is copied back into.
+          Defaults to the configured decsdata root path, /gdex/decsdata;
+
+      - Option -l ListFile, a file holding one dataset path per line.  Only
+          used if -d is not given.  Defaults to 'dsids_<today>.lst' in the
+          current directory, which is generated from every dNNNNNN directory
+          found in the cold storage directories if it does not exist yet;
+
+      - Option -d DatasetPathList, one or more dataset IDs to restore, such as
+          '-d d612000 d627000'.  Append a sub-path to a dataset ID to restore
+          part of it only, such as '-d d612000/2020/01';
+
+      - Option -f, copy the data back even while some of its files are still on
+          tape.  Without it, Action -r refuses to copy such data;
+
+      - Option -h, display this help document;
+
+      - Action -x, submit the recall requests to bring the data back on disk.
+          The decsdata area is limited, so the size to restore is added up
+          from Table 'sfile' in RDADB and compared to the space left for the
+          decsdata directory, as reported by 'gladequota', before any recall
+          is requested.  TWICE the size is required, since the recalled copy
+          in the cold storage and the copy made by Action -r later on both
+          live under the decsdata quota; the recall is stopped if the space
+          left is not enough;
+
+      - Action -s, report the HSM status of the data, including the number of
+          files still on tape and the log of any outstanding recall request;
+
+      - Action -r [TargetDirectory], copy the recalled data back under
+          '<decsdata>/<dsid>', or under the given TargetDirectory.  A restored
+          sub-path keeps its relative position, so 'd612000/2020/01' is copied
+          back to '<TargetDirectory>/d612000/2020/01'.  The dataset directory,
+          and any sub-directory of it, is created if it does not exist yet.
+
+  One and only one of the Actions -x, -s and -r is required; this help document
+  is displayed without any of them.  This utility can be run from any directory.
+  It is executed under the effective user 'gdexdata' via setuid, so the restored
+  data is owned by 'gdexdata'.  The cold storage copy of the data is left in
+  place by Action -r; move it out of the 'COLD_STORAGE/' path manually to take
+  a dataset out of the HSM permanently.
+
+  Examples:
+
+    1. Restore a whole dataset from the cold storage of the root decsdata
+       directory, /gdex/decsdata/COLD_STORAGE/d612000:
+
+          decsdata_restore -d d612000 -x
+          decsdata_restore -d d612000 -s
+          decsdata_restore -d d612000 -r
+
+    2. Restore it from the cold storage directory of a specific date,
+       /gdex/decsdata/cold_storage_20250529/COLD_STORAGE/d612000:
+
+          decsdata_restore -D 20250529 -d d612000 -x
+
+    3. Restore one year of a dataset only:
+
+          decsdata_restore -d d612000/2020 -x
+          decsdata_restore -d d612000/2020 -s
+          decsdata_restore -d d612000/2020 -r
+
+    4. Copy the recalled data back to a directory other than the decsdata one:
+
+          decsdata_restore -d d612000 -r /PathTo/OtherDirectory
+
+    5. Check the status of every dataset in every cold storage directory,
+       generating the dataset list file automatically:
+
+          decsdata_restore -s
+
+    6. Copy a dataset back even though some of its files are still on tape,
+       which makes the HSM read them off tape while they are being copied:
+
+          decsdata_restore -d d612000 -f -r

rda_python_miscs-3.0.5/src/rda_python_miscs/decsdata_storage.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+##################################################################################
+#     Title: decsdata_storage
+#    Author: Zaihua Ji, zji@ucar.edu
+#      Date: 2026-08-31
+#   Purpose: move decsdata datasets into a dated cold storage directory and hand
+#            them to the GLADE HSM to be migrated onto tape
+#    Github: https://github.com/NCAR/rda-python-miscs.git
+##################################################################################
+import re
+import os
+import sys
+from os import path as op
+from rda_python_common.pg_file import PgFile
+
+class DecsStorage(PgFile):
+   """Move decsdata datasets into cold storage and migrate them onto tape.
+
+   Each given dataset directory is moved from the decsdata directory into
+   '<decsdata>/cold_storage_<date>/' and then handed to 'glade_hsm migrate',
+   which relocates it once more into a 'COLD_STORAGE/' sub-directory and lets
+   the HSM batch processes migrate every large file onto tape.  Use
+   decsdata_restore to bring the data back.
+   """
+
+   def __init__(self):
+      """Initialize DecsStorage with default option values and runtime state."""
+      super().__init__()
+      self.HSM = os.environ.get('GLADE_HSM', op.expanduser('~benkirk/glade_hsm'))
+      self.VALOPTS = 'Dwl'   # single-value options
+      self.MULOPTS = 'd'     # multi-value options
+      self.MODOPTS = 'hx'    # mode options
+      self.OPTS = {
+         'D': None,   # cold storage date, in YYYYMMDD or YYYY-MM-DD
+         'w': None,   # decsdata directory, defaults to PGLOG['DECSHOME']
+         'l': None,   # dataset list file
+         'd': [],     # dataset IDs
+         'h': 0,      # 1 to show help message
+         'x': 0,      # 1 to execute, mandatory
+      }
+      self.SINFO = {
+         'decsdir': None,    # decsdata directory the datasets are stored from
+         'coldstor': None,   # cold storage directory the datasets are moved to
+         'date': None,       # cold storage date
+         'dcnt': 0,          # number of datasets moved into cold storage
+      }
+
+   # function to read parameters
+   def read_parameters(self):
+      """Parse the command line into the OPTS option values.
+
+      Single-value options -D, -w and -l take one value each, multi-value
+      option -d gathers every following dataset ID, and mode options -h and -x
+      are simple flags.  Exits with usage if -h is given or -x is missing.
+      """
+      self.set_suid(self.PGLOG['EUID'])
+      self.set_help_path(__file__)
+      self.PGLOG['LOGFILE'] = "decsdata_storage.log"   # set different log file
+      argv = sys.argv[1:]
+      self.cmdlog("decsdata_storage {}".format(' '.join(argv)))
+      option = None
+      for arg in argv:
+         ms = re.match(r'^-(\w+)$', arg)
+         if ms:
+            option = ms.group(1)
+            if option in self.MODOPTS:
+               self.OPTS[option] = 1
+               option = None
+            elif option not in self.VALOPTS and option not in self.MULOPTS:
+               self.pglog(arg + ": Unknown Option", self.LGEREX)
+            continue
+         if not option: self.pglog(arg + ": Value provided without option", self.LGEREX)
+         if option in self.MULOPTS:
+            self.OPTS[option].append(arg)   # gather all values until the next option
+         else:
+            self.OPTS[option] = arg
+            option = None
+      if self.OPTS['h'] or not self.OPTS['x']: self.show_usage("decsdata_storage")
+
+   # function to start actions
+   def start_actions(self):
+      """Validate the caller, resolve the cold storage paths, and store each dataset."""
+      self.dssdb_dbname()
+      self.validate_decs_group('decsdata_storage', self.PGLOG['CURUID'], 1)
+      self.set_storage_paths()
+      dsids = self.get_dataset_list()
+      if not dsids: self.pglog("No dataset found for cold storage", self.LGWNEX)
+      for dsid in dsids:
+         self.store_one_dataset(dsid)
+      s = ('s' if self.SINFO['dcnt'] > 1 else '')
+      self.pglog("{} of {} Dataset{} moved into {}".format(self.SINFO['dcnt'],
+                 len(dsids), s, self.SINFO['coldstor']), self.LOGWRN)
+      self.cmdlog()
+
+   # resolve the decsdata and cold storage directories
+   def set_storage_paths(self):
+      """Fill SINFO with the decsdata directory, cold storage date and cold storage path.
+
+      Defaults the decsdata directory to PGLOG['DECSHOME'] and the date to
+      today.  Dashes are stripped from the given date, which must then be 8
+      digits.
+      """
+      decsdir = self.OPTS['w'] if self.OPTS['w'] else self.PGLOG['DECSHOME']
+      if not self.check_local_file(decsdir, 0, self.LOGWRN):
+         self.pglog(decsdir + ": decsdata directory NOT exists", self.LGEREX)
+      date = re.sub('-', '', self.OPTS['D']) if self.OPTS['D'] else re.sub('-', '', self.curdate())
+      if not re.match(r'^\d{8}$', date):
+         self.pglog(date + ": Invalid cold storage date, YYYYMMDD expected", self.LGEREX)
+      self.SINFO['decsdir'] = decsdir
+      self.SINFO['date'] = date
+      self.SINFO['coldstor'] = self.join_paths(decsdir, "cold_storage_" + date)
+
+   # gather the dataset IDs to move into cold storage
+   def get_dataset_list(self):
+      """Return the list of dataset IDs to store.
+
+      Uses the -d values if given.  Otherwise reads the -l list file, creating
+      it first from every dNNNNNN directory in the decsdata directory if it
+      does not exist yet.
+
+      Returns:
+         list[str]: Dataset IDs, empty if none is found.
+      """
+      if self.OPTS['d']:
+         self.pglog("Store {} given Dataset(s) into cold storage".format(len(self.OPTS['d'])), self.LOGWRN)
+         return self.OPTS['d']
+      lstfile = self.OPTS['l'] if self.OPTS['l'] else "dsids_{}.lst".format(self.SINFO['date'])
+      if not op.isfile(lstfile):
+         dsids = self.get_decsdata_datasets()
+         with open(lstfile, 'w') as OUT:
+            for dsid in dsids: OUT.write(dsid + "\n")
+         self.pglog("{}: Generated with {} Dataset(s)".format(lstfile, len(dsids)), self.LOGWRN)
+      dsids = []
+      with open(lstfile, 'r') as IN:
+         for line in IN:
+            line = line.strip()
+            if line: dsids.append(line)
+      self.pglog("{}: Read {} Dataset(s) for cold storage".format(lstfile, len(dsids)), self.LOGWRN)
+      return dsids
+
+   # find all dNNNNNN dataset directories in the decsdata directory
+   def get_decsdata_datasets(self):
+      """Return the sorted dataset IDs of every dNNNNNN directory in the decsdata directory.
+
+      Returns:
+         list[str]: Dataset IDs; plain files matching the pattern are skipped.
+      """
+      pattern = self.join_paths(self.SINFO['decsdir'], "d" + "[0-9]"*6)
+      files = self.local_glob(pattern, 0, self.LOGWRN)
+      dsids = []
+      for file in files:
+         if not files[file]['isfile']: dsids.append(op.basename(file))
+      return sorted(dsids)
+
+   # move one dataset into cold storage and migrate it onto tape
+   def store_one_dataset(self, dsid):
+      """Move one dataset into the cold storage directory and migrate it onto tape.
+
+      Skips the dataset if it is not an existing directory in the decsdata
+      directory or if the move fails.  Increments SINFO['dcnt'] for each
+      dataset successfully migrated.
+
+      Args:
+         dsid (str): Dataset ID, such as 'd612000'.
+      """
+      fromfile = self.join_paths(self.SINFO['decsdir'], dsid)
+      info = self.check_local_file(fromfile, 0, self.LOGWRN)
+      if not info:
+         self.pglog(fromfile + ": Dataset NOT exists", self.LOGERR)
+         return
+      if info['isfile']:
+         self.pglog(fromfile + ": Not a dataset directory", self.LOGERR)
+         return
+      tofile = self.join_paths(self.SINFO['coldstor'], dsid)
+      if not self.move_local_file(tofile, fromfile, self.LOGWRN):
+         self.pglog("{}: Error move {} into cold storage".format(fromfile, dsid), self.LOGERR)
+         return
+      cmd = "{} migrate -f {}".format(self.HSM, tofile)
+      if self.pgsystem(cmd, self.LOGWRN, 7):
+         self.SINFO['dcnt'] += 1
+         self.pglog("{}: Migrated onto tape from {}".format(dsid, tofile), self.LOGWRN)
+      else:
+         self.pglog("{}: Error migrate onto tape from {}".format(dsid, tofile), self.LOGERR)
+
+# main function to execute this script
+def main():
+   """Entry point: instantiate DecsStorage, parse arguments, run, and exit."""
+   from rda_python_setuid.setup_guide import show_setup_guide
+   object = DecsStorage()
+   show_setup_guide(object, 'rda_python_miscs', ['decsdata_storage', 'decsdata_restore'])
+   object.read_parameters()
+   object.start_actions()
+   object.pgexit(0)
+
+# call main() to start program
+if __name__ == "__main__": main()

rda_python_miscs-3.0.5/src/rda_python_miscs/decsdata_storage.usg

@@ -0,0 +1,66 @@
+
+ Move decsdata datasets into a dated cold storage directory and hand them to the
+ GLADE HSM to be migrated onto tape.  Each dataset directory is moved from the
+ decsdata directory into '<decsdata>/cold_storage_<date>/' and then passed to
+ 'glade_hsm migrate', which relocates it once more into a 'COLD_STORAGE/'
+ sub-directory, ending up as:
+
+    <decsdata>/cold_storage_<date>/COLD_STORAGE/<dsid>
+
+ The HSM batch processes then migrate every large file (>= 100MB) underneath the
+ 'COLD_STORAGE/' path onto tape.  All files inside a 'COLD_STORAGE/' path are
+ marked immutable, which prevents modification of their contents and metadata.
+ Use 'decsdata_restore' to bring the data back.
+
+ Usage: decsdata_storage [-D Date] [-w DecsdataDirectory] [-l ListFile]  \
+                         [-d DatasetList] -x
+
+      - Option -D Date, the cold storage date, in YYYYMMDD or YYYY-MM-DD.  It
+          names the cold storage directory '<decsdata>/cold_storage_<date>'.
+          Defaults to today;
+
+      - Option -w DecsdataDirectory, the decsdata directory the datasets are
+          stored from, and the directory the cold storage directory is created
+          in.  Defaults to the configured decsdata root path, /gdex/decsdata;
+
+      - Option -l ListFile, a file holding one dataset ID per line.  Only used
+          if -d is not given.  Defaults to 'dsids_<date>.lst' in the current
+          directory, which is generated from every dNNNNNN directory found in
+          the decsdata directory if it does not exist yet;
+
+      - Option -d DatasetList, one or more dataset IDs to move into cold
+          storage, such as '-d d612000 d627000';
+
+      - Option -h, display this help document;
+
+      - Option -x, execute the cold storage moves.  This option is mandatory;
+          this help document is displayed without it.  Note that no
+          confirmation is asked for once -x is given.
+
+  This utility can be run from any directory.  It is executed under the
+  effective user 'gdexdata' via setuid, so the moved data keeps its ownership.
+  A dataset is skipped, and an error is logged, if it is not an existing
+  directory in the decsdata directory or if it cannot be moved.
+
+  Examples:
+
+    1. Move two given datasets into today's cold storage:
+
+          decsdata_storage -d d612000 d627000 -x
+
+    2. Move them into the cold storage directory of a specific date:
+
+          decsdata_storage -D 20250529 -d d612000 -x
+
+       The data then ends up in:
+
+          /gdex/decsdata/cold_storage_20250529/COLD_STORAGE/d612000
+
+    3. Move every dataset listed in a list file:
+
+          decsdata_storage -l my_dsids.lst -x
+
+    4. Move every dNNNNNN dataset found in a non-default decsdata directory,
+       generating the dataset list file automatically:
+
+          decsdata_storage -w /PathTo/decsdata -x

{rda_python_miscs-3.0.4 → rda_python_miscs-3.0.5/src/rda_python_miscs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.4
+Version: 3.0.5
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
@@ -45,6 +45,8 @@ The package provides two categories of programs:
 | `gdexcp` | `rdacp` | `setuid_gdexcp` / `setuid_rdacp` | Copy files and directories across local, remote, Object Store, or Globus endpoints |
 | `gdexkill` | `rdakill` | `setuid_gdexkill` / `setuid_rdakill` | Kill local processes and their children, or cancel PBS batch jobs |
 | `gdexmod` | `rdamod` | `setuid_gdexmod` / `setuid_rdamod` | Change permission modes for files and directories owned by gdexdata |
+| `decsdata_storage` | | `setuid_decsdata_storage` | Move decsdata datasets into the GLADE HSM cold storage to migrate them onto tape |
+| `decsdata_restore` | | `setuid_decsdata_restore` | Recall decsdata datasets out of the GLADE HSM cold storage and copy them back |
 
 ## Environment setup
 
@@ -103,8 +105,8 @@ pip install rda_python_miscs
 
 ## Setuid Setup
 
-The setuid programs (`gdexcp`, `gdexkill`, `gdexmod` and their `rda*` aliases)
-execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via
+The setuid programs (`gdexcp`, `gdexkill`, `gdexmod`, `decsdata_storage`,
+`decsdata_restore` and the `rda*` aliases) execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via
 the `rda_python_setuid` mechanism, which is pulled in automatically as a
 dependency.  After `pip install` above, choose one of the wiring options
 below.
@@ -159,3 +161,30 @@ pywrapper-install -u|--update
 The shared setuid setup guide is shown automatically if any `setuid_*`
 connector script is invoked directly before the setuid wrapper has been
 configured.
+
+## Cold storage setup
+
+`decsdata_storage` and `decsdata_restore` drive the GLADE HSM through the
+`glade_hsm` script, which is not part of this package.  They look it up at
+`~benkirk/glade_hsm` unless the environment variable `GLADE_HSM` names another
+copy of it:
+
+```bash
+export GLADE_HSM=/path/to/glade_hsm
+```
+
+Both programs work on the decsdata directory `PGLOG['DECSHOME']` (default
+`/glade/campaign/collections/gdex/decsdata`, also reachable as
+`/gdex/decsdata`), which option `-w` overrides.  Only members of the DECS group
+may run them, and `decsdata_restore` reads Table `sfile` in RDADB to size a
+restore, so a working RDADB login is required as well.
+
+Because the decsdata quota is limited, check the space left with `gladequota`
+before a large restore:
+
+```bash
+pgstart_gdexdata gladequota    # or plain 'gladequota' when logged in as gdexdata
+```
+
+`decsdata_restore -x` runs the same check itself and refuses to request a
+recall unless twice the size of the data is still available.

{rda_python_miscs-3.0.4 → rda_python_miscs-3.0.5}/src/rda_python_miscs.egg-info/SOURCES.txt

@@ -6,6 +6,10 @@ src/rda_python_miscs/__init__.py
 src/rda_python_miscs/bash_qsub.py
 src/rda_python_miscs/bashqsub.py
 src/rda_python_miscs/bashqsub.usg
+src/rda_python_miscs/decsdata_restore.py
+src/rda_python_miscs/decsdata_restore.usg
+src/rda_python_miscs/decsdata_storage.py
+src/rda_python_miscs/decsdata_storage.usg
 src/rda_python_miscs/gdex_ls.py
 src/rda_python_miscs/gdexcp.py
 src/rda_python_miscs/gdexcp.usg

{rda_python_miscs-3.0.4 → rda_python_miscs-3.0.5}/src/rda_python_miscs.egg-info/entry_points.txt

@@ -11,6 +11,8 @@ rdaown = rda_python_miscs.gdexown:main
 rdaps = rda_python_miscs.gdexps:main
 rdasub = rda_python_miscs.gdexsub:main
 rdazip = rda_python_miscs.gdexzip:main
+setuid_decsdata_restore = rda_python_miscs.decsdata_restore:main
+setuid_decsdata_storage = rda_python_miscs.decsdata_storage:main
 setuid_gdexcp = rda_python_miscs.gdexcp:main
 setuid_gdexkill = rda_python_miscs.gdexkill:main
 setuid_gdexmod = rda_python_miscs.gdexmod:main

{rda_python_miscs-3.0.4 → rda_python_miscs-3.0.5}/tests/test_miscs.py

@@ -14,3 +14,5 @@ def test_miscs():
    import rda_python_miscs.gdexps
    import rda_python_miscs.gdexsub
    import rda_python_miscs.gdexzip
+   import rda_python_miscs.decsdata_storage
+   import rda_python_miscs.decsdata_restore