rda-python-dsquasar 3.0.8__tar.gz → 3.0.10__tar.gz

{rda_python_dsquasar-3.0.8/src/rda_python_dsquasar.egg-info → rda_python_dsquasar-3.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.8
+Version: 3.0.10
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.8 → rda_python_dsquasar-3.0.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "3.0.8"
+version = "3.0.10"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-3.0.8 → rda_python_dsquasar-3.0.10}/src/rda_python_dsquasar/dsquasar.py

@@ -90,9 +90,10 @@ class DsQuasar(PgCMD, PgSplit):
       self.MPTMAX = 12            # maximum number of batch processes for tarring
       # uploading (BCKACT) forks one child per BCKSIZE transfer batch (~BCKSIZE/TARSIZE
       # tar files each), so far fewer parallel units than tar files: use a large
-      # per-process limit to avoid over-reserving cpus.
-      self.MPBLIMIT = 900         # tar file count per batch process for uploading
-      self.MPBMAX = 8             # maximum number of batch processes for uploading
+      # per-process limit to avoid over-reserving cpus. Globus runs only a few
+      # concurrent transfers, so extra upload processes just wait on it.
+      self.MPBLIMIT = 5000        # tar file count per batch process for uploading
+      self.MPBMAX = 4             # maximum number of batch processes for uploading
       self.MAXRUNTIME = 23*3600   # 23 hours; stop before the 24-hour PBS walltime
       self.ONEHOUR = 3600         # seconds; time headroom needed to finish before walltime
       # a repeat submit normally blocks as a duplicate while the batch job is running. if
@@ -106,6 +107,7 @@ class DsQuasar(PgCMD, PgSplit):
       self.PGBACK = {
          'workdir' : "{}/{}/quasar_backup".format(self.PGLOG['GDEXWORK'], self.PGLOG['COMMONUSER']),
          'mproc' : 1,
+         'mgiven' : 0,      # set when -m is on the command line, to keep an explicit -m 1
          'action' : self.CTACTS,
          'chgdays' : 0,
          'backflag' : None,
@@ -160,6 +162,7 @@ class DsQuasar(PgCMD, PgSplit):
             elif option == 'm':
                self.PGBACK['mproc'] = int(arg)
                if self.PGBACK['mproc'] < 1: self.pglog(arg +": Multiple Processes(-m) must be > 0", self.LGWNEX)
+               self.PGBACK['mgiven'] = 1
                option = None
             elif option == 'A':
                self.PGBACK['action'] = int(arg)
@@ -468,15 +471,16 @@ class DsQuasar(PgCMD, PgSplit):
             if not pgrec:
                qoptions = '-l walltime=24:00:00'
                if acts&self.NBACTS != acts:
-                  if self.PGBACK['mproc'] < 2: self.PGBACK['mproc'] = self.batch_process_count(acts)
+                  if not self.PGBACK['mgiven']: self.PGBACK['mproc'] = self.batch_process_count(acts)
                   if self.PGBACK['mproc'] > 1:
                      qoptions += ",select=1:ncpus={0}:mem={0}gb".format(self.PGBACK['mproc'])
                self.set_one_boption('qoptions', qoptions, 1)
          self.init_dscheck(0, '', "dsquasar", dsid, cact, self.PGBACK['workdir'],
                             self.PGLOG['CURUID'], self.bopts, self.LOGWRN)
          # on the PBS batch run, take the process count from the reserved ncpus in qoptions
-         # instead of recomputing it
-         if self.PGLOG['DSCHECK'] and self.PGBACK['mproc'] < 2 and acts&self.NBACTS != acts:
+         # instead of recomputing it. an explicit -m N on the command line wins over the
+         # reserved count, including -m 1 to force a single process
+         if self.PGLOG['DSCHECK'] and not self.PGBACK['mgiven'] and acts&self.NBACTS != acts:
             pgrec = self.pgget('dscheck', 'qoptions',
                                "cindex = {}".format(self.PGLOG['DSCHECK']['cindex']), self.LGEREX)
             if pgrec and pgrec['qoptions']:

{rda_python_dsquasar-3.0.8 → rda_python_dsquasar-3.0.10}/src/rda_python_dsquasar/dsquasar.usg

@@ -80,9 +80,12 @@
           to Y.
 
    -m  ProcessCount
-          Number of concurrent child processes.  Defaults to 1.  In delay mode
-          the process count travels through the reserved PBS cpus rather than
-          the command line.
+          Number of concurrent child processes.  Defaults to 1.  When -m is not
+          given, a delayed backup run sizes the count from the number of tar
+          files it expects to handle and reserves matching PBS cpus, so in delay
+          mode the process count normally travels through the reserved cpus
+          rather than the command line.  An explicit -m overrides that sizing,
+          and -m 1 forces a single process.
 
    -n     Gather and show the available file counts only; no backup actions are
           performed.

{rda_python_dsquasar-3.0.8 → rda_python_dsquasar-3.0.10/src/rda_python_dsquasar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.8
+Version: 3.0.10
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar