rda-python-dsquasar 3.0.9__tar.gz → 3.0.11__tar.gz

{rda_python_dsquasar-3.0.9/src/rda_python_dsquasar.egg-info → rda_python_dsquasar-3.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.9
+Version: 3.0.11
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.9 → rda_python_dsquasar-3.0.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "3.0.9"
+version = "3.0.11"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-3.0.9 → rda_python_dsquasar-3.0.11}/src/rda_python_dsquasar/dsquasar.py

@@ -90,9 +90,10 @@ class DsQuasar(PgCMD, PgSplit):
       self.MPTMAX = 12            # maximum number of batch processes for tarring
       # uploading (BCKACT) forks one child per BCKSIZE transfer batch (~BCKSIZE/TARSIZE
       # tar files each), so far fewer parallel units than tar files: use a large
-      # per-process limit to avoid over-reserving cpus.
-      self.MPBLIMIT = 900         # tar file count per batch process for uploading
-      self.MPBMAX = 8             # maximum number of batch processes for uploading
+      # per-process limit to avoid over-reserving cpus. Globus runs only a few
+      # concurrent transfers, so extra upload processes just wait on it.
+      self.MPBLIMIT = 5000        # tar file count per batch process for uploading
+      self.MPBMAX = 4             # maximum number of batch processes for uploading
       self.MAXRUNTIME = 23*3600   # 23 hours; stop before the 24-hour PBS walltime
       self.ONEHOUR = 3600         # seconds; time headroom needed to finish before walltime
       # a repeat submit normally blocks as a duplicate while the batch job is running. if
@@ -1040,7 +1041,7 @@ class DsQuasar(PgCMD, PgSplit):
          fname = pgrec[tname]
          fsize = pgrec['data_size']
          ftype = pgrec['type']
-         fcksm = pgrec['checksum']
+         fcksm = pgrec['checksum'] if pgrec['checksum'] else ''   # a directory, such as a zarr store, has none
          qinfo['fcnt'] += 1
          qinfo['size'] += fsize
          instr = "{}<:>{}<:>{}<:>{}<:>\n".format(fname, ftype, fsize, fcksm)

{rda_python_dsquasar-3.0.9 → rda_python_dsquasar-3.0.11/src/rda_python_dsquasar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.9
+Version: 3.0.11
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar