rda-python-dsquasar 3.0.14__tar.gz → 3.0.16__tar.gz

{rda_python_dsquasar-3.0.14/src/rda_python_dsquasar.egg-info → rda_python_dsquasar-3.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.14
+Version: 3.0.16
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.14 → rda_python_dsquasar-3.0.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "3.0.14"
+version = "3.0.16"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-3.0.14 → rda_python_dsquasar-3.0.16}/src/rda_python_dsquasar/dsquasar.py

@@ -611,7 +611,7 @@ class DsQuasar(PgCMD, PgSplit):
                qinfo['dslocks'].remove(dsid)
             qinfo['dsids'] = []
       if self.PGBACK['mproc'] > 1:
-         self.check_child(None, 0, self.LOGWRN, 1)   # wait all child processes done
+         self.wait_all_children()   # wait all child processes done
          self.confirm_quasar_counts(qinfo, qinfo['abids'], "status = 'T'")   # recount confirmed backups from RDADB
       if qinfo['dslocks']:
          for dsid in qinfo['dslocks']:
@@ -728,7 +728,7 @@ class DsQuasar(PgCMD, PgSplit):
             self.unlock_backup_datasets(qinfo, dslocks)
          if self.PGBACK['dolock']: self.unlock_backup_datasets(qinfo, [dsid])
       if self.PGBACK['mproc'] > 1:
-         self.check_child(None, 0, self.LOGWRN, 1)   # wait all child processes done
+         self.wait_all_children()   # wait all child processes done
          self.confirm_quasar_counts(qinfo, qinfo['abids'], "status = 'T'")   # recount confirmed backups from RDADB
       qcnt = qinfo['qcnt']
       if qcnt > 0:
@@ -769,7 +769,7 @@ class DsQuasar(PgCMD, PgSplit):
          self.transfer_quasar_tarfiles(qinfo)
       if qinfo['size'] > 0: self.transfer_quasar_tarfiles(qinfo)
       if self.PGBACK['mproc'] > 1:
-         self.check_child(None, 0, self.LOGWRN, 1)   # wait all child processes done
+         self.wait_all_children()   # wait all child processes done
          self.confirm_quasar_counts(qinfo, list(bfiles), "status = 'A'")   # recount confirmed backups from RDADB
       qcnt = qinfo['qcnt']
       if qcnt > 0:
@@ -867,15 +867,17 @@ class DsQuasar(PgCMD, PgSplit):
             self.pglog("Cannot clean the progress report of {}; it is emailed again".format(cnd), self.LOGWRN)
       return estat
 
-   # guard the long PBS batch jobs (-A 2/3/4/6) against the walltime: once past MAXRUNTIME
+   # guard the long PBS batch jobs (-A 2/3/4/6/7) against the walltime: once past MAXRUNTIME
    # the run keeps going, but a progress report is parked in dscheck.einfo so that
    # something is reported even if PBS kills the job at the walltime. the final report
    # follows and drops the parked one if the run does finish in time. the live email
    # buffers are saved and put back, so the final report still carries everything logged
    # before the cutoff.
-   # both queue depths are reported rather than the current phase's: -A 3 and -A 6 run the
-   # build and the transfer phase in one run, and -A 3 tars each input file right after
-   # creating it, so its status 'N' count stays near zero while that is the busy phase.
+   # both queue depths are reported rather than the current phase's, because -A 3 tars each
+   # input file right after creating it, so its status 'N' count stays near zero while that
+   # is the busy phase. they are worded as STATES ('left to build', 'tarred') and not as
+   # work this run will do: only -A 4 and -A 6 transfer, so calling the status 'T' count
+   # 'to transfer' in an -A 2 or -A 3 report claims work that run never performs.
    # a no-op for command-line runs, other actions, before the cutoff, or once a progress
    # report is cached.
    # called from the top of every per-item loop, not only where a tar is dispatched: a long
@@ -885,7 +887,7 @@ class DsQuasar(PgCMD, PgSplit):
    def check_batch_deadline(self):
       act = self.PGBACK['action']
       if self.PGBACK['einfo'] or not (self.PGBACK['doemail'] and self.PGLOG['DSCHECK']): return
-      if self.PGLOG['CURBID'] < 1 or act not in (self.TARACT, self.CTACTS, self.BCKACT, self.TBACTS): return
+      if self.PGLOG['CURBID'] < 1 or act not in (self.TARACT, self.CTACTS, self.BCKACT, self.TBACTS, self.CBACTS): return
       elapsed = tm() - self.PGBACK['starttime']
       if elapsed < self.MAXRUNTIME: return
       tcnt = self.batch_tar_count('N')
@@ -894,7 +896,7 @@ class DsQuasar(PgCMD, PgSplit):
       amsg = self.ACTMSG[act]
       bmsg = self.BACKMSG[self.PGBACK['backflag']] if self.PGBACK['backflag'] else 'backup'
       dmsg = self.batch_done_count()
-      rmsg = "{} {} tar file(s) left to build and {} to transfer".format(tcnt, bmsg, bcnt)
+      rmsg = "{} {} tar file(s) left to build and {} tarred".format(tcnt, bmsg, bcnt)
       msg = "{}: Still running after {} of the {} PBS walltime, {}, with {}".format(amsg, etime, self.seconds_to_string_time(self.WALLTIME), dmsg, rmsg)
       self.pglog(self.INDENT + msg, self.LOGACT)
       saved = {key : self.PGLOG[key] for key in ('EMLMSG', 'ERRMSG', 'ERRCNT', 'SUMMSG', 'PRGMSG')}
@@ -904,6 +906,20 @@ class DsQuasar(PgCMD, PgSplit):
       self.report_dscheck_email(title, 1)
       self.PGLOG.update(saved)
 
+   # wait for every child process to finish, checking the walltime deadline between polls.
+   # check_child(..., 1) does the waiting inside its own loop, so a phase that ends with the
+   # last tar children still running would sit there for the rest of the run without ever
+   # reaching a deadline check - the parent has no per-item loop left to guard. polling with
+   # dowait 0 and sleeping here keeps the same cadence and wait message as check_child does.
+   def wait_all_children(self):
+      if self.PGBACK['mproc'] < 2: return
+      i = 0
+      while True:
+         self.check_batch_deadline()
+         if self.check_child(None, 0, self.LOGWRN, 0) < 1: break
+         self.show_wait_message(i, "{}: wait child processes".format(self.PGSIG['DSTR']), self.LOGWRN, 1)
+         i += 1
+
    # recompute the confirmed backup counts from RDADB after all child processes
    # finished, so a multi-process summary reflects succeeded (not just started) work
    def confirm_quasar_counts(self, qinfo, bids, dcnd):
@@ -1387,6 +1403,10 @@ class DsQuasar(PgCMD, PgSplit):
          dcnt = len(mrecs['dsid']) if mrecs else 0
          pgrecs = [self.onerecord(mrecs, i) for i in range(dcnt)]
       for pgrec in pgrecs:
+         # a gather is not free: it queries every dataset, and each one another worker holds
+         # costs an rdaps call to check that worker's pid. a run under dscheck does two of
+         # them before any work, and -A 3 does a third one before tarring
+         self.check_batch_deadline()
          dsid = pgrec['dsid']
          if unlock and pgrec['pid'] and self.lock_dataset(dsid, 0, self.LOGACT) < 1: continue
          fcnt += self.get_dataset_files(dsid, dsfiles, pgrec['backflag'], logact, sizes)
@@ -1411,6 +1431,7 @@ class DsQuasar(PgCMD, PgSplit):
          pgrecs = self.pgmget("bfile", flds, dcnd, self.LGWNEX)
          bcnt = len(pgrecs['bid']) if pgrecs else 0
          for i in range(bcnt):
+            self.check_batch_deadline()
             cnt = self.get_backup_infile(dsfiles, self.onerecord(pgrecs, i))
             if cnt:
                icnt += 1
@@ -1436,6 +1457,7 @@ class DsQuasar(PgCMD, PgSplit):
          pgrecs = self.pgmget("bfile", flds, dcnd, self.LGWNEX)
          bcnt = len(pgrecs['bid']) if pgrecs else 0
          for i in range(bcnt):
+            self.check_batch_deadline()
             cnt = self.get_backup_tarfile(dsfiles, self.onerecord(pgrecs, i))
             if cnt:
                tcnt += 1

{rda_python_dsquasar-3.0.14 → rda_python_dsquasar-3.0.16/src/rda_python_dsquasar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.14
+Version: 3.0.16
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar