rda-python-common 3.0.18__tar.gz → 3.0.20__tar.gz

{rda_python_common-3.0.18/src/rda_python_common.egg-info → rda_python_common-3.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.18
+Version: 3.0.20
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.18`).  If the import
+You should see the installed version (currently `3.0.20`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.18 → rda_python_common-3.0.20}/README.md

@@ -165,7 +165,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.18`).  If the import
+You should see the installed version (currently `3.0.20`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.18 → rda_python_common-3.0.20}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "3.0.18"
+version = "3.0.20"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_common-3.0.18 → rda_python_common-3.0.20}/src/rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "3.0.18"
+__version__ = "3.0.20"
 
 __all__ = [
    "PgLOG",

{rda_python_common-3.0.18 → rda_python_common-3.0.20}/src/rda_python_common/pg_cmd.py

@@ -207,6 +207,14 @@ class PgCMD(PgLock):
             self.pglog(cmsg + "is Running, No restart", self.LOGWRN)
             sys.exit(0)
          if cidx > 0:
+            # a report parked in einfo by the previous run is only mailed by the dscheck
+            # daemon, which skips any record that is already running. restarting the record
+            # here would strand that report until the next run overwrites it, so leave the
+            # record alone and let the daemon mail it first - it clears einfo when it does
+            if pgrec['einfo']:
+               self.pglog(cmsg + "has a report to email, No restart", self.LOGWRN)
+               self.lock_dscheck(cidx, 0, logact)
+               sys.exit(0)
             if not hosts and pgrec['hostname']:
                hosts = pgrec['hostname']
                self.set_one_boption('hostname', hosts, 0)

{rda_python_common-3.0.18 → rda_python_common-3.0.20}/src/rda_python_common/pg_file.py

@@ -105,7 +105,8 @@ class PgFile(PgUtil, PgSIG):
       }
       self.TARSTR = '|'.join(self.PGTARS)
       self.DELDIRS = {}
-      self.TASKIDS = {}   # cache unfinished 
+      self.TASKIDS = {}   # cache unfinished
+      self.QCANCEL = {}   # taskid -> why a task got cancelled, reported by the waiting caller
       self.LHOST = "localhost"
       self.OHOST = self.PGLOG['OBJCTSTR']
       self.BHOST = self.PGLOG['BACKUPNM']
@@ -117,7 +118,9 @@ class PgFile(PgUtil, PgSIG):
       # record how many errors happen for working with HPSS, local or remote machines
       self.ECNTS = {'D': 0, 'H': 0, 'L': 0, 'R': 0, 'O': 0, 'B': 0}
       # up limits for how many continuing errors allowed
-      self.ELMTS = {'D': 20, 'H': 20, 'L': 20, 'R': 20, 'O': 10, 'B': 10}
+      # 'B' is 5 because a failed Globus transfer counts once; it used to be logged
+      # twice per failure, so the old limit of 10 also gave up after 5 failures
+      self.ELMTS = {'D': 20, 'H': 20, 'L': 20, 'R': 20, 'O': 10, 'B': 5}
       # down storage hostnames & paths
       self.DHOSTS = {
          'G': self.PGLOG['GPFSNAME'],
@@ -532,7 +535,11 @@ class PgFile(PgUtil, PgSIG):
                   time.sleep(self.PGSIG['ETIME'])
                   lp += 1
                if task['stat'] == 'S' or task['stat'] == 'A': break
-               if task['stat'] == 'F' and not syserr: break
+               if task['stat'] == 'F' and not syserr:
+                  # nothing waits on this task, so report the cancellation here instead
+                  if task['id'] in self.QCANCEL:
+                     self.errlog("{}: Cancel Task due to {}".format(task['id'], self.QCANCEL.pop(task['id'])), 'B', 1, logact)
+                  break
          errmsg = "Error Execute: " + cmd
          if qstr: errmsg += " with stdin:\n" + qstr
          if syserr:
@@ -581,8 +588,16 @@ class PgFile(PgUtil, PgSIG):
                      detail = ms.group(1)
                      if detail not in astats:
                         if logact&self.NOWAIT:
-                           errmsg = "{}: Cancel Task due to {}:\n{}".format(taskid, detail, buf)
-                           self.errlog(errmsg, 'B', 1, logact)
+                           # record why, and let the caller waiting on this task report it as a
+                           # single error naming the file; dumping the whole get-task output here
+                           # doubled every failure into two error entries, the first 17 lines long
+                           reason = detail
+                           ms = re.search(r'Bytes Transferred:\s+(\d+)', buf)
+                           if ms: reason += " after " + self.format_float_value(ms.group(1))
+                           ms = re.search(r'Files:\s+(\d+)', buf)
+                           if ms: reason += " of {} file(s)".format(ms.group(1))
+                           self.QCANCEL[taskid] = reason
+                           self.pglog("{}: Cancel Task due to {}".format(taskid, reason), self.LOGWRN)
                            ccmd = f"{bcmd} cancel-task {taskid}"
                            self.pgsystem(ccmd, logact, 7)
                         else:
@@ -640,7 +655,10 @@ class PgFile(PgUtil, PgSIG):
                del self.TASKIDS[ckey]
             else:
                status = self.QSTATS[stat] if stat in self.QSTATS else 'UNKNOWN'
-               self.errlog("{}: Status '{}' for Task {}".format(ckey, status, taskid), 'B', 1, logact)
+               errmsg = "{}: Status '{}' for Task {}".format(ckey, status, taskid)
+               if taskid in self.QCANCEL:
+                  errmsg += " - " + self.QCANCEL.pop(taskid)
+               self.errlog(errmsg, 'B', 1, logact)
                ret = self.FAILURE
             break
       return ret

{rda_python_common-3.0.18 → rda_python_common-3.0.20}/src/rda_python_common/pg_log.py

@@ -249,9 +249,13 @@ class PgLOG:
             if self.PGLOG['PRGMSG']:
                msg = self.PGLOG['PRGMSG'] + "\n" + msg
                self.PGLOG['PRGMSG'] = ""
+            # callers usually end the message with their own '!', so do not add a second
+            # one, and drop it before continuing the sentence with the error count
             if self.PGLOG['ERRCNT'] == 0:
-               if not msg.endswith('\n'): msg += "!\n"
+               if not msg.endswith('\n'):
+                  msg += "\n" if msg.endswith('!') else "!\n"
             else:
+               if msg.endswith('!'): msg = msg[:-1]
                if self.PGLOG['ERRCNT'] == 1:
                   msg += " with 1 Error:\n"
                else:

{rda_python_common-3.0.18 → rda_python_common-3.0.20/src/rda_python_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.18
+Version: 3.0.20
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.18`).  If the import
+You should see the installed version (currently `3.0.20`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.