rda-python-dsquasar 3.0.12__tar.gz → 3.0.14__tar.gz

{rda_python_dsquasar-3.0.12/src/rda_python_dsquasar.egg-info → rda_python_dsquasar-3.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.12
+Version: 3.0.14
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-3.0.12 → rda_python_dsquasar-3.0.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "3.0.12"
+version = "3.0.14"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-3.0.12 → rda_python_dsquasar-3.0.14}/src/rda_python_dsquasar/dsquasar.py

@@ -104,7 +104,7 @@ class DsQuasar(PgCMD, PgSplit):
       # for -A 3 but tar files for -A 2/-A 4.
       self.MAXWORKERS = 2         # default maximum concurrent workers per command (-W)
       self.WORKGRACE = 6*3600     # 6 hours before a running job is judged on progress
-      self.MINWDONE = 0.01        # done fraction after WORKGRACE that counts as progressing
+      self.MINWPROJ = 0.9         # projected done fraction at the walltime that counts as finishing
       self.PGBACK = {
          'workdir' : "{}/{}/quasar_backup".format(self.PGLOG['GDEXWORK'], self.PGLOG['COMMONUSER']),
          'mproc' : 1,
@@ -114,6 +114,8 @@ class DsQuasar(PgCMD, PgSplit):
          'backflag' : None,
          'actmsg' : None,
          'pstep' : 0,       # record progress step for under dscheck control
+         'dcnt' : 0,        # files done so far, recorded as dscheck.dcount every pstep
+         'dsize' : 0,
          'errcnt' : 0,
          'bckcnt' : 0,
          'maxcnt' : 10,
@@ -228,6 +230,12 @@ class DsQuasar(PgCMD, PgSplit):
          if acts&self.TARACT: self.build_tarfile_action()
          if acts&self.CINACT: self.create_infile_action()
          if self.dstart and acts&self.TBACTS == self.TBACTS: self.globus_transfer_action()
+         # the queues were empty, so nothing was processed and nothing failed. no dscheck
+         # record is registered for such a run either, so drop the report rather than mail
+         # one about 0 files. a parked progress report still forces one, to clean it up
+         if not (self.PGBACK['bckcnt'] or self.PGBACK['errcnt'] or
+                 self.PGLOG['ERRCNT'] or self.PGBACK['einfo']):
+            self.PGBACK['doemail'] = 0
       if self.PGBACK['doemail']:
          amsg = self.ACTMSG[self.PGBACK['action']]
          bmsg = self.BACKMSG[self.PGBACK['backflag']] if self.PGBACK['backflag'] else 'backup'
@@ -343,7 +351,7 @@ class DsQuasar(PgCMD, PgSplit):
          self.backup_dataset_infiles(dsfiles, 'D')
 
    # gather and transfer tar files to Globus Quasar Servers
-   def globus_transfer_action(self):   
+   def globus_transfer_action(self):
       dsfiles = {'B' : {}, 'D' : {}}
       fcnt = self.gather_dataset_tarfiles(dsfiles)
       if fcnt and self.dstart:
@@ -393,6 +401,17 @@ class DsQuasar(PgCMD, PgSplit):
          return tcnt
       return self.pgget('bfile', '', bcnd, self.LGWNEX)
 
+   # describe how much of the registered work this run has finished. the tar counts alone
+   # do not show it: -A 3 tars each input file right after creating it, so the 'N' count
+   # stays near zero exactly while building is the busy phase, which reads as almost done
+   # when the run is barely started. dcount/fcount is the honest measure. the live dcnt is
+   # used rather than the recorded dcount, which is only flushed to RDADB every pstep files
+   def batch_done_count(self):
+      fcnt = self.DSCHK['fcount'] if 'fcount' in self.DSCHK else 0
+      dcnt = self.PGBACK['dcnt']
+      if not fcnt: return "{} file(s) done".format(dcnt)
+      return "{} of {} file(s) done({}%)".format(dcnt, fcnt, int(100*dcnt/fcnt))
+
    # look up the dscheck record registered for a worker's argv, using the same lookup key
    # init_dscheck builds
    def lookup_worker_dscheck(self, argv):
@@ -403,18 +422,29 @@ class DsQuasar(PgCMD, PgSplit):
          dargv = dargv[0:100]
       return self.get_dscheck("dsquasar", dargv, self.PGBACK['workdir'], self.PGLOG['CURUID'], dargx, self.LOGWRN)
 
-   # a running batch job counts as stalled once it has run longer than WORKGRACE and has
-   # still completed no more than MINWDONE of its recorded work; anything further along
-   # than that is progressing and is left alone. the done fraction dcount/fcount is used
-   # instead of a raw count because dcount counts GDEX files for -A 3 but tar files for
-   # -A 2/-A 4. a job that is not locked, not started yet, or whose process is gone is not
-   # stalled: init_dscheck already restarts a dead one.
+   # a running batch job counts as stalled once it has run longer than WORKGRACE and its
+   # progress so far projects to less than MINWPROJ of its recorded work by the walltime:
+   # such a job cannot finish this attempt however long it is left alone, so an extra
+   # worker is what helps it, not more retries.
+   # the test is on the RATE, not on an absolute done fraction as it first was. a fixed
+   # 1% threshold only catches a job that is nearly frozen, and reads a job that needs
+   # weeks as progressing: the real case was 2% done after 13H35M of a 24 hour walltime,
+   # which is above 1% yet projects to 3.5% by the walltime and ~28 days to finish, and so
+   # never got a second worker.
+   # the done fraction dcount/fcount is used instead of a raw count because dcount counts
+   # GDEX files for -A 3 but tar files for -A 2/-A 4. a job that is not locked, not started
+   # yet, or whose process is gone is not stalled: init_dscheck already restarts a dead one.
+   # WALLTIME is the 24 hours asked for at submit time, since only the submitting process
+   # runs this and set_walltime_deadline reads the granted walltime back in the batch job.
+   # when the daemon caps it to a shorter queue limit the projection is optimistic, which
+   # errs towards leaving the job alone.
    def worker_progress_stalled(self, pgrec):
       if not (pgrec['pid'] and pgrec['stttime']): return False
       if self.check_host_pid(pgrec['lockhost'], pgrec['pid']) <= 0: return False
-      if (tm() - pgrec['stttime']) < self.WORKGRACE: return False
+      elapsed = tm() - pgrec['stttime']
+      if elapsed < self.WORKGRACE: return False
       done = pgrec['dcount']/pgrec['fcount'] if pgrec['fcount'] else 0
-      return done <= self.MINWDONE
+      return (done*self.WALLTIME/elapsed) < self.MINWPROJ
 
    # whether more than one worker may run the current action. two workers stay off each
    # other's files only through dataset locking, so the action must lock what it works on:
@@ -560,6 +590,7 @@ class DsQuasar(PgCMD, PgSplit):
                'size' : 0, 'infiles' : [], 'instr' : '', 'qdsids' : [], 'abids' : [],
                'dslocks' : [], 'qfcnt' : 0, 'qsize' : 0, 'qcnt' : 0}
       for dsid in bfiles:
+         self.check_batch_deadline()
          if len(qinfo['dsids']) == self.DSCNT: self.process_one_backup_file(qinfo, True, False)
          if self.PGBACK['dolock'] and dsid not in qinfo['dslocks']:
             if self.lock_dataset(dsid, 1, self.LOGERR) < 1: continue
@@ -676,6 +707,7 @@ class DsQuasar(PgCMD, PgSplit):
             if self.lock_dataset(dsid, 1, self.LOGERR) < 1: continue
             qinfo['dslocks'].append(dsid)
          for bid in bfiles[dsid]:
+            self.check_batch_deadline()
             # the status 'N' records were gathered before this dataset was locked, so they
             # can be hours old; confirm each one is still untarred before spending a tar on
             # it. holding the dataset lock makes this check race free against a second
@@ -723,6 +755,7 @@ class DsQuasar(PgCMD, PgSplit):
                'size' : 0, 'fromfiles' : [], 'tofiles' : [], 'qdsids' : [],
                'qfcnt' : 0, 'qsize' : 0, 'qcnt' : 0}
       for bid in bfiles:
+         self.check_batch_deadline()
          binfo = bfiles[bid]
          for dsid in binfo['dsids']:
             if dsid not in qinfo['dsids']: qinfo['dsids'].append(dsid)
@@ -752,6 +785,11 @@ class DsQuasar(PgCMD, PgSplit):
 
    # wait all child processes finish and then quit the main program
    def quit_dsquasar(self, qinfo, msg = None):
+      # a forked child reaches here through its own errcnt, which is a copy of the parent's
+      # made at the fork. only the parent reports for the run: a child reporting would email
+      # a second report, drop the progress report parked in dscheck.einfo by the parent, and
+      # mark the shared check record failed while the parent is still working
+      if self.PGSIG['PPID'] > 1: sys.exit(1)
       if self.PGBACK['mproc'] > 1: self.check_child(None, 0, self.LOGWRN, 1)
       if qinfo:
          if 'dslocks' in qinfo and qinfo['dslocks']:
@@ -798,35 +836,52 @@ class DsQuasar(PgCMD, PgSplit):
                  self.PGLOG['CURBID'], self.seconds_to_string_time(self.WALLTIME),
                  self.seconds_to_string_time(self.MAXRUNTIME)), self.LOGWRN)
 
-   # email a report for a run under dscheck control. the report is normally sent right
-   # away, but is cached into dscheck.einfo instead if a progress report is cached there
-   # already, so that the final report replaces it; the dscheck daemon sends and clears
-   # einfo once the check record is unlocked.
+   # email a report for a run under dscheck control. the progress report is parked in
+   # dscheck.einfo and the final report is sent right away. the run holds the check lock
+   # while it works, and the dscheck daemon only mails unlocked records, so a parked
+   # report goes out exactly when the check is unlocked - right after PBS kills the job
+   # off its walltime, in the same pass that resubmits it.
+   # a record with a non-empty einfo is skipped by both the start pass and the purge pass
+   # of the daemon, so the final report must drop the parked one: left behind it would be
+   # mailed a second time as a stale progress report, and would hold the finished check
+   # back from being purged.
    def report_dscheck_email(self, title, cache = 0):
       cnd = "cindex = {}".format(self.PGLOG['DSCHECK']['cindex'])
-      if not (cache or self.PGBACK['einfo']):
-         return self.build_customized_email("dscheck", "einfo", cnd, title, self.LOGWRN)
-      msg = self.get_email()
-      if not msg: return self.FAILURE
-      sender = self.PGLOG['CURUID'] + "@ucar.edu"
-      receiver = self.PGLOG['EMLADDR'] if self.PGLOG['EMLADDR'] else sender
-      if receiver.find(sender) < 0: self.add_carbon_copy(sender, 1)
-      ebuf = "From: {}\nTo: {}\n".format(sender, receiver)
-      if self.PGLOG['CCDADDR']: ebuf += "Cc: {}\n".format(self.PGLOG['CCDADDR'])
-      ebuf += "Subject: {}!\n\n{}\n".format(title, msg)
-      self.PGBACK['einfo'] = 1
-      return self.cache_customized_email("dscheck", "einfo", cnd, ebuf, self.LOGWRN)
+      if cache:
+         msg = self.get_email()
+         if not msg: return self.FAILURE
+         sender = self.PGLOG['CURUID'] + "@ucar.edu"
+         receiver = self.PGLOG['EMLADDR'] if self.PGLOG['EMLADDR'] else sender
+         if receiver.find(sender) < 0: self.add_carbon_copy(sender, 1)
+         ebuf = "From: {}\nTo: {}\n".format(sender, receiver)
+         if self.PGLOG['CCDADDR']: ebuf += "Cc: {}\n".format(self.PGLOG['CCDADDR'])
+         ebuf += "Subject: {}!\n\n{}\n".format(title, msg)
+         self.PGBACK['einfo'] = 1   # report the progress only once per run
+         return self.cache_customized_email("dscheck", "einfo", cnd, ebuf, self.LOGWRN)
+      estat = self.build_customized_email("dscheck", "einfo", cnd, title, self.LOGWRN)
+      # a failed send has already replaced the parked report via cache_customized_email()
+      if estat == self.SUCCESS and self.PGBACK['einfo']:
+         if self.pgexec("UPDATE dscheck set einfo = NULL WHERE " + cnd, self.LOGWRN):
+            self.PGBACK['einfo'] = 0
+         else:
+            self.pglog("Cannot clean the progress report of {}; it is emailed again".format(cnd), self.LOGWRN)
+      return estat
 
    # guard the long PBS batch jobs (-A 2/3/4/6) against the walltime: once past MAXRUNTIME
-   # the run keeps going, but a progress report is cached into dscheck.einfo so an email is
-   # still sent, by the dscheck daemon, if PBS kills the job at the walltime. the final
-   # report replaces it if the run does finish in time. the live email buffers are saved
-   # and put back, so the final report still carries everything logged before the cutoff.
+   # the run keeps going, but a progress report is parked in dscheck.einfo so that
+   # something is reported even if PBS kills the job at the walltime. the final report
+   # follows and drops the parked one if the run does finish in time. the live email
+   # buffers are saved and put back, so the final report still carries everything logged
+   # before the cutoff.
    # both queue depths are reported rather than the current phase's: -A 3 and -A 6 run the
    # build and the transfer phase in one run, and -A 3 tars each input file right after
    # creating it, so its status 'N' count stays near zero while that is the busy phase.
    # a no-op for command-line runs, other actions, before the cutoff, or once a progress
    # report is cached.
+   # called from the top of every per-item loop, not only where a tar is dispatched: a long
+   # stretch of files that are skipped (already backed up, or not enough accumulated size to
+   # tar yet) would otherwise walk past the cutoff without ever reaching the check, which is
+   # how a 23h run reported nothing.
    def check_batch_deadline(self):
       act = self.PGBACK['action']
       if self.PGBACK['einfo'] or not (self.PGBACK['doemail'] and self.PGLOG['DSCHECK']): return
@@ -838,12 +893,13 @@ class DsQuasar(PgCMD, PgSplit):
       etime = self.seconds_to_string_time(int(elapsed))
       amsg = self.ACTMSG[act]
       bmsg = self.BACKMSG[self.PGBACK['backflag']] if self.PGBACK['backflag'] else 'backup'
+      dmsg = self.batch_done_count()
       rmsg = "{} {} tar file(s) left to build and {} to transfer".format(tcnt, bmsg, bcnt)
-      msg = "{}: Still running after {} of the {} PBS walltime, with {}".format(amsg, etime, self.seconds_to_string_time(self.WALLTIME), rmsg)
+      msg = "{}: Still running after {} of the {} PBS walltime, {}, with {}".format(amsg, etime, self.seconds_to_string_time(self.WALLTIME), dmsg, rmsg)
       self.pglog(self.INDENT + msg, self.LOGACT)
       saved = {key : self.PGLOG[key] for key in ('EMLMSG', 'ERRMSG', 'ERRCNT', 'SUMMSG', 'PRGMSG')}
-      self.set_email("{}: {} still in progress after {}, with {}!".format(self.PGBACK['cmd'], amsg, etime, rmsg), self.EMLTOP)
-      title = "dsquasar: {} in progress ({} to build, {} to transfer)".format(amsg, tcnt, bcnt)
+      self.set_email("{}: {} still in progress after {}, {}, with {}!".format(self.PGBACK['cmd'], amsg, etime, dmsg, rmsg), self.EMLTOP)
+      title = "dsquasar: {} in progress ({})".format(amsg, dmsg)
       if self.PGBACK['errcnt']: title += " Error({})".format(self.PGBACK['errcnt'])
       self.report_dscheck_email(title, 1)
       self.PGLOG.update(saved)
@@ -1068,6 +1124,7 @@ class DsQuasar(PgCMD, PgSplit):
       fcate = filetype.lower()
       tname = fcate + 'file'
       for i in range(fcnt):
+         self.check_batch_deadline()
          pgrec = recs[i]
          if not self.evaluate_file_stat(dsid, fcate, pgrec): continue
          fname = pgrec[tname]

{rda_python_dsquasar-3.0.12 → rda_python_dsquasar-3.0.14/src/rda_python_dsquasar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 3.0.12
+Version: 3.0.14
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar