rda-python-dsquasar 3.0.25__tar.gz → 3.0.27__tar.gz

{rda_python_dsquasar-3.0.25/src/rda_python_dsquasar.egg-info → rda_python_dsquasar-3.0.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.25
+Version: 3.0.27
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.25 → rda_python_dsquasar-3.0.27}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "3.0.25"
+version = "3.0.27"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-3.0.25 → rda_python_dsquasar-3.0.27}/src/rda_python_dsquasar/dsquasar.py

@@ -504,6 +504,10 @@ class DsQuasar(PgCMD, PgSplit):
    # the done fraction dcount/fcount is used instead of a raw count because dcount counts
    # GDEX files for -A 3 but tar files for -A 2/-A 4. a job that is not locked, not started
    # yet, or whose process is gone is not stalled: init_dscheck already restarts a dead one.
+   # not started yet means status 'R': the dscheck daemon sets pid to the PBS job id and
+   # stttime to the submit time already at qsub, and rdaps lists a queued job as alive, so
+   # a job waiting in the PBS queue would otherwise read as one running with nothing done.
+   # init_dscheck sets status 'R' and resets stttime once the batch job really starts.
    # WALLTIME is the 24 hours asked for at submit time, since only the submitting process
    # runs this and set_walltime_deadline reads the granted walltime back in the batch job.
    # when the daemon caps it to a shorter queue limit the projection is optimistic, which
@@ -516,6 +520,7 @@ class DsQuasar(PgCMD, PgSplit):
    # only be added by a cron submit, every 4 hours: a 6 hour grace kept it out of the first
    # 6 to 10 hours of every attempt.
    def worker_progress_stalled(self, pgrec):
+      if pgrec['status'] != 'R': return False   # queued, held or finished: not running
       if not (pgrec['pid'] and pgrec['stttime']): return False
       if self.check_host_pid(pgrec['lockhost'], pgrec['pid']) <= 0: return False
       elapsed = tm() - pgrec['stttime']
@@ -1335,6 +1340,21 @@ class DsQuasar(PgCMD, PgSplit):
          s = 's' if mcnt > 1 else ''
          ftype = 'Saved' if fcate == 's' else 'Web'
          self.pglog("{}: {} of {} {} file{} missing from storage, not backed up".format(dsid, mcnt, fcnt, ftype, s), self.LOGACT)
+         if self.PGBACK['pstep']: self.drop_dscheck_files(mcnt)
+
+   # take files found missing from storage out of the dscheck file count. they are counted
+   # in at start, before any file is checked, but are never tarred and so never reach
+   # dcount: left in, they cap the done fraction below 100% (8,308 of 21,620 files once,
+   # 38%), understate the progress report and make the stall test add a worker the run
+   # does not need. with an upload action every file was counted twice, once to tar and
+   # once to upload, so twice as many come out
+   def drop_dscheck_files(self, mcnt):
+      fcnt = self.DSCHK['fcount'] if 'fcount' in self.DSCHK else 0
+      if not fcnt: return
+      if self.PGBACK['action']&self.BCKACT: mcnt *= 2
+      fcnt -= mcnt
+      if fcnt < self.PGBACK['dcnt']: fcnt = self.PGBACK['dcnt']
+      self.set_dscheck_fcount(fcnt, self.LGEREX)
 
    # initialize dataset backup dict structure
    @staticmethod

{rda_python_dsquasar-3.0.25 → rda_python_dsquasar-3.0.27/src/rda_python_dsquasar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.25
+Version: 3.0.27
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar