rda-python-common 2.1.9__tar.gz → 2.1.11__tar.gz

{rda_python_common-2.1.9/src/rda_python_common.egg-info → rda_python_common-2.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 2.1.9
+Version: 2.1.11
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -11,8 +11,8 @@ Classifier: Development Status :: 5 - Production/Stable
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hvac
-Requires-Dist: psycopg2==2.9.10
+Requires-Dist: psycopg2-binary
+Requires-Dist: psutil
 Requires-Dist: rda-python-globus
 Requires-Dist: unidecode
 Requires-Dist: hvac
@@ -92,7 +92,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `2.1.9`).  If the import
+You should see the installed version (currently `2.1.11`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-2.1.9 → rda_python_common-2.1.11}/README.md

@@ -72,7 +72,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `2.1.9`).  If the import
+You should see the installed version (currently `2.1.11`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-2.1.9 → rda_python_common-2.1.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "2.1.9"
+version = "2.1.11"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]
@@ -18,8 +18,8 @@ classifiers = [
     "Development Status :: 5 - Production/Stable",
 ]
 dependencies = [
-  "hvac",
-  "psycopg2==2.9.10",
+  "psycopg2-binary",
+  "psutil",
   "rda-python-globus",
   "unidecode",
   "hvac"

{rda_python_common-2.1.9 → rda_python_common-2.1.11}/src/rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "2.1.9"
+__version__ = "2.1.11"
 
 __all__ = [
    "PgLOG",

{rda_python_common-2.1.9 → rda_python_common-2.1.11}/src/rda_python_common/pg_cmd.py

@@ -389,7 +389,7 @@ class PgCMD(PgLock):
          if pgrqst: pgctl = self.get_dsrqst_control(pgrqst, logact)
       return pgctl
 
-   def get_dynamic_options(self, cmd, oindex, otype):
+   def get_dynamic_options(self, cmd, oindex, otype, hostname=None):
       """Runs cmd to retrieve dynamic option strings, retrying on timeout.
 
       Executes the command up to three times, retrying when a connection
@@ -401,6 +401,11 @@ class PgCMD(PgLock):
          oindex (int or None): Object index appended to cmd when truthy.
          otype (str or None): Object type appended to cmd when truthy;
             'R' selects the first option from a slash-separated pair.
+         hostname (str or None): If provided, the command is executed
+            remotely via ``ssh <hostname> ...`` instead of locally; the
+            leading command name is resolved to its absolute path via
+            ``command_path`` so the remote shell does not depend on its
+            own PATH.
 
       Returns:
          str: The parsed option string, or '' if the command produced no
@@ -408,6 +413,7 @@ class PgCMD(PgLock):
       """
       if oindex: cmd += " {}".format(oindex)
       if otype: cmd += ' ' + otype
+      if hostname: cmd = "ssh {} {}".format(hostname, self.command_path(cmd))
       ret = options = ''
       for loop in range(3):
          ret = self.pgsystem(cmd, self.LOGWRN, 1299)  # 1+2+16+256+1024

{rda_python_common-2.1.9 → rda_python_common-2.1.11}/src/rda_python_common/pg_file.py

@@ -19,6 +19,7 @@ import re
 import time
 import glob
 import json
+import hashlib
 from .pg_util import PgUtil
 from .pg_sig import PgSIG
 
@@ -2874,24 +2875,32 @@ class PgFile(PgUtil, PgSIG):
                             (with None for missing files) for multiple files, or None
                             on failure.
       """
-      cmd = 'md5sum '
       if count > 0:
          checksum = [None]*count
          for i in range(count):
             if op.isfile(file[i]):
-               chksm = self.pgsystem(cmd + file[i], logact, 20)
-               if chksm:
-                  ms = re.search(r'(\w{32})', chksm)
-                  if ms: checksum[i] = ms.group(1)
+               checksum[i] = self._file_md5(file[i], logact)
       else:
          checksum = None
          if op.isfile(file):
-            chksm = self.pgsystem(cmd + file, logact, 20)
-            if chksm:
-               ms = re.search(r'(\w{32})', chksm)
-               if ms: checksum = ms.group(1)
+            checksum = self._file_md5(file, logact)
       return checksum
 
+   def _file_md5(self, path, logact=0):
+      """Compute MD5 hex digest of *path*, reading in 1 MiB chunks.
+
+      Returns the hex digest string, or None on read error.
+      """
+      try:
+         h = hashlib.md5()
+         with open(path, 'rb') as fh:
+            for chunk in iter(lambda: fh.read(1048576), b''):
+               h.update(chunk)
+         return h.hexdigest()
+      except OSError as e:
+         self.pglog("Error md5sum {}: {}".format(path, str(e)), logact)
+         return None
+
    # Evaluate md5 checksums and compare them for two given files
    #  file1, file2: file names
    # Return: 0 if same and 1 if not

{rda_python_common-2.1.9 → rda_python_common-2.1.11}/src/rda_python_common/pg_log.py

@@ -18,6 +18,7 @@ import grp
 import shlex
 import smtplib
 from email.message import EmailMessage
+import subprocess
 from subprocess import Popen, PIPE
 from os import path as op
 import time
@@ -180,6 +181,7 @@ class PgLOG:
       self.BCHCMDS = {'PBS': 'qsub'}
       # global dists to cashe information
       self.COMMANDS = {}
+      self.CMDPATHS = {}    # cache of bare command name -> full path (or '' if not found)
       self.PBSHOSTS = []
       self.PBSSTATS = {}
       # set additional common PGLOG values
@@ -768,7 +770,7 @@ class PgLOG:
                      nilcnt = 0
                   if begin: sys.stdout.write(line)
       else:
-         os.system("more " + usgname)
+         subprocess.run(['more', usgname])
       self.pgexit(0)
 
    def err2std(self, line):
@@ -1197,15 +1199,19 @@ class PgLOG:
       Returns:
           String with the leading command resolved to its full path, or the
           original *cmdstr* if the command already contains a path separator
-          or cannot be found via ``shutil.which``.
+          or cannot be found via ``shutil.which``.  Results of the
+          ``shutil.which`` lookup are cached in ``self.CMDPATHS``.
       """
       if not cmdstr: return ''
-      ary = cmdstr.split(' ', 1)
-      cmd = ary[0]
-      if re.search(r'[\\/]', cmd): return cmdstr
-      optstr = (' ' + ary[1]) if len(ary) > 1 else ''
-      pcmd = shutil.which(cmd)
-      return (pcmd+optstr) if pcmd else cmdstr
+      sp = cmdstr.find(' ')
+      cmd = cmdstr if sp < 0 else cmdstr[:sp]
+      if '/' in cmd or '\\' in cmd: return cmdstr
+      pcmd = self.CMDPATHS.get(cmd)
+      if pcmd is None:
+         pcmd = shutil.which(cmd) or ''
+         self.CMDPATHS[cmd] = pcmd
+      if not pcmd: return cmdstr
+      return pcmd if sp < 0 else pcmd + cmdstr[sp:]
 
    def add_carbon_copy(self, cc=None, isstr=None, exclude=0, specialist=None):
       """Update the Cc address list in ``PGLOG['CCDADDR']``.
@@ -1595,15 +1601,12 @@ class PgLOG:
          os.environ['MAIL'] = re.sub(self.PGLOG['CURUID'], specialist, os.environ['MAIL'])   
       home = "{}/{}".format(self.PGLOG['USRHOME'], specialist)
       shell = "tcsh"
-      buf = self.pgsystem("grep ^{}: /etc/passwd".format(specialist), self.LOGWRN, 20)
-      if buf:
-         lines = buf.split('\n')
-         for line in lines:
-            ms = re.search(r':(/.+):(/.+)', line)
-            if ms:
-               home = ms.group(1)
-               shell = op.basename(ms.group(2))
-               break
+      try:
+         pwent = pwd.getpwnam(specialist)
+         home = pwent.pw_dir
+         shell = op.basename(pwent.pw_shell)
+      except KeyError:
+         pass
       if home != os.environ['HOME'] and op.exists(home):
          os.environ['HOME'] = home
       return shell

{rda_python_common-2.1.9 → rda_python_common-2.1.11}/src/rda_python_common/pg_sig.py

@@ -14,6 +14,8 @@ import sys
 import errno
 import signal
 import time
+import subprocess
+import psutil
 from contextlib import contextmanager
 from .pg_dbi import PgDBI
 
@@ -204,6 +206,42 @@ class PgSIG(PgDBI):
       self.PGLOG['LOGMASK'] |= self.MSGLOG    # turn on logging before daemon stops
       self.pglog("{} Started at {}, Stopped gracefully{} by {}".format(self.PGSIG['DSTR'], self.PGSIG['STRTM'], msg, self.current_datetime()), self.LOGWRN)
 
+   # scan running processes via psutil and return a list of dicts matching aname/uname
+   # Mirrors the semantics of the previous "ps -u U -f | grep A" / "ps -C A -f" pipelines:
+   #   - with uname:  substring match of aname anywhere in cmdline (looser, like grep)
+   #   - without uname: aname matches the executable basename (with optional .ext)
+   # Each entry has keys: 'pid', 'ppid', 'args' ('args' is cmdline[1:] joined).
+   def _scan_app_processes(self, aname, uname=None):
+      """Scan running processes matching an application name, optionally by user.
+
+      Args:
+         aname (str): Application name (basename or substring of cmdline).
+         uname (str, optional): Username filter.
+
+      Returns:
+         list[dict]: Each dict has keys ``pid``, ``ppid``, ``args``.
+      """
+      results = []
+      for proc in psutil.process_iter(['pid', 'ppid', 'username', 'cmdline']):
+         try:
+            info = proc.info
+            cmdline = info.get('cmdline') or []
+            if not cmdline: continue
+            if uname is not None and info.get('username') != uname: continue
+            if uname:
+               if not any(aname in arg for arg in cmdline): continue
+            else:
+               exe = os.path.basename(cmdline[0])
+               if exe != aname and re.sub(r'\.\w+$', '', exe) != aname: continue
+            results.append({
+               'pid': info['pid'],
+               'ppid': info['ppid'],
+               'args': ' '.join(cmdline[1:]),
+            })
+         except (psutil.NoSuchProcess, psutil.AccessDenied):
+            continue
+      return results
+
    # check if a daemon is running already
    # aname - application name for the daemon
    # uname - user login name who started the daemon
@@ -218,21 +256,11 @@ class PgSIG(PgDBI):
       Returns:
          int: The process ID of the running daemon, or 0 if not running.
       """
-      if uname:
-         self.check_vuser(uname, aname)
-         pcmd = "ps -u {} -f | grep {} | grep ' 1 '".format(uname, aname)
-         mp = r"^\s*{}\s+(\d+)\s+1\s+".format(uname)
-      else:
-         pcmd = "ps -C {} -f | grep ' 1 '".format(aname)
-         mp = r"^\s*\w+\s+(\d+)\s+1\s+"
-      buf = self.pgsystem(pcmd, self.LOGWRN, 20+1024)
-      if buf:
-         cpid = os.getpid()
-         lines = buf.split('\n')
-         for line in lines:
-            ms = re.match(mp, line)
-            pid = int(ms.group(1)) if ms else 0
-            if pid > 0 and pid != cpid: return pid
+      if uname: self.check_vuser(uname, aname)
+      cpid = os.getpid()
+      for p in self._scan_app_processes(aname, uname):
+         if p['ppid'] != 1: continue
+         if p['pid'] != cpid: return p['pid']
       return 0
 
    # check if an application is running already; other than the current processs
@@ -251,31 +279,21 @@ class PgSIG(PgDBI):
       Returns:
          int: The process ID of the running instance, or 0 if not found.
       """
-      if uname:
-         self.check_vuser(uname, aname)
-         pcmd = "ps -u {} -f | grep {} | grep -v ' grep '".format(uname, aname)
-         mp = r"^\s*{}\s+(\d+)\s+(\d+)\s+.*{}\S*\s+(.*)$".format(uname, aname)
-      else:
-         pcmd = "ps -C {} -f".format(aname)
-         mp = r"^\s*\w+\s+(\d+)\s+(\d+)\s+.*{}\S*\s+(.*)$".format(aname)
-      buf = self.pgsystem(pcmd, self.LOGWRN, 20+1024)
-      if not buf: return 0
+      if uname: self.check_vuser(uname, aname)
+      procs = self._scan_app_processes(aname, uname)
+      if not procs: return 0
       cpids = [os.getpid(), os.getppid()]
       pids = []
       ppids = []
       astrs = []
-      lines = buf.split('\n')
-      for line in lines:
-         ms = re.match(mp, line)
-         if not ms: continue
-         pid = int(ms.group(1))
-         ppid = int(ms.group(2))
+      for p in procs:
+         pid, ppid = p['pid'], p['ppid']
          if pid in cpids:
             if ppid not in cpids: cpids.append(ppid)
             continue
          pids.append(pid)
          ppids.append(ppid)
-         if sargv: astrs.append(ms.group(3))
+         if sargv: astrs.append(p['args'])
       pcnt = len(pids)
       if not pcnt: return 0
       i = 0
@@ -329,25 +347,15 @@ class PgSIG(PgDBI):
       Returns:
          int: Number of running instances (excluding the current process).
       """
-      if uname:
-         self.check_vuser(uname, aname)
-         pcmd = "ps -u {} -f | grep {} | grep -v ' grep '".format(uname, aname)
-         mp = r"^\s*{}\s+(\d+)\s+(\d+)\s+.*{}\S*\s+(.*)$".format(uname, aname)
-      else:
-         pcmd = "ps -C {} -f".format(aname)
-         mp = r"^\s*\w+\s+(\d+)\s+(\d+)\s+.*{}\S*\s+(.*)$".format(aname)
-      buf = self.pgsystem(pcmd, self.LOGWRN, 20+1024)
-      if not buf: return 0
+      if uname: self.check_vuser(uname, aname)
+      procs = self._scan_app_processes(aname, uname)
+      if not procs: return 0
       dpids = [os.getpid(), os.getppid()]
       pids = []
       ppids = []
       astrs = []
-      lines = buf.split('\n')
-      for line in lines:
-         ms = re.match(mp, line)
-         if not ms: continue
-         pid = int(ms.group(1))
-         ppid = int(ms.group(2))
+      for p in procs:
+         pid, ppid = p['pid'], p['ppid']
          if pid in dpids:
             if ppid > 1 and ppid not in dpids: dpids.append(ppid)
             continue
@@ -356,7 +364,7 @@ class PgSIG(PgDBI):
             continue
          pids.append(pid)
          ppids.append(ppid)
-         if sargv: astrs.append(ms.group(3))
+         if sargv: astrs.append(p['args'])
       pcnt = len(pids)
       if not pcnt: return 0
       i = 0
@@ -627,18 +635,20 @@ class PgSIG(PgDBI):
          list: PIDs of processes that were successfully killed.
       """
       if logact is None: logact = self.LOGWRN
-      buf = self.pgsystem("ps --ppid {} -o pid".format(pid), logact, 20)
       pids = []
-      if buf:
-         lines = buf.split('\n')
-         for line in lines:
-            ms = re.match(r'^\s*(\d+)', line)
-            if not ms: continue
-            cid = int(ms.group(1))
-            if not self.check_process(cid): continue
-            cids = self.kill_children(cid, logact)
-            if cids: pids = cids + pids
-            if self.kill_process(cid, signal.SIGKILL, logact) == self.SUCCESS: pids.insert(0, cid)
+      try:
+         children = psutil.Process(pid).children()
+      except psutil.NoSuchProcess:
+         children = []
+      except Exception as e:
+         self.pglog("Error listing children of pid {}: {}".format(pid, str(e)), logact)
+         children = []
+      for child in children:
+         cid = child.pid
+         if not self.check_process(cid): continue
+         cids = self.kill_children(cid, logact)
+         if cids: pids = cids + pids
+         if self.kill_process(cid, signal.SIGKILL, logact) == self.SUCCESS: pids.insert(0, cid)
       if logact and len(pids): self.pglog("Process({}) Killed".format(','.join(map(str, pids))), logact)
       return pids
 
@@ -808,13 +818,11 @@ class PgSIG(PgDBI):
       Returns:
          int: 1 if the process is running, 0 otherwise.
       """
-      buf = self.pgsystem("ps -p {} -o pid".format(pid), self.LGWNEX, 20)
-      if buf:
-         mp = r'^\s*{}$'.format(pid)
-         lines = buf.split('\n')
-         for line in lines:
-            if re.match(mp, line): return 1
-      return 0
+      try:
+         os.kill(pid, 0)
+      except OSError:
+         return 0
+      return 1
 
    # check a process id on give host
    def check_host_pid(self, host, pid, pmsg=None, logact=None):
@@ -1092,7 +1100,10 @@ class PgSIG(PgDBI):
             self.PGLOG['ERRFILE'] = re.sub(r'\.log$', '.err', self.PGLOG['LOGFILE'], 1)
          bckcmd += " 2>> {}/{}".format(self.PGLOG['LOGPATH'], self.PGLOG['ERRFILE'])
       bckcmd += " &"
-      os.system(bckcmd)
+      # shell=True is required for the redirections (>> / 2>>) and trailing '&';
+      # the '&' makes the shell fork the command and exit, so the command gets
+      # reparented to init (ppid=1), matching the lookup logic in record_background().
+      subprocess.Popen(bckcmd, shell=True)
       return self.record_background(cmd, logact)
 
    # get background process id for given bcmd
@@ -1170,27 +1181,31 @@ class PgSIG(PgDBI):
       """
       if logact is None: logact = self.LOGWRN
       ms = re.match(r'^(\S+)', bcmd)
-      if ms:
-         aname = ms.group(1)
-      else:
-         aname = bcmd
-      mp = r"^\s*(\S+)\s+(\d+)\s+1\s+.*{}(.*)$".format(aname)
-      pc = "ps -u {},{} -f | grep ' 1 ' | grep {}".format(self.PGLOG['CURUID'], self.PGLOG['GDEXUSER'], aname)
+      aname = ms.group(1) if ms else bcmd
+      curuid = self.PGLOG['CURUID']
+      gdexuser = self.PGLOG['GDEXUSER']
       for i in range(2):
-         buf = self.pgsystem(pc, logact, 20+1024)
-         if buf:
-            lines = buf.split('\n')
-            for line in lines:
-               ms = re.match(mp, line)
-               if not ms: continue
-               (uid, sbid, acmd) = ms.groups()
-               bid = int(sbid)
+         for proc in psutil.process_iter(['pid', 'ppid', 'username', 'cmdline']):
+            try:
+               info = proc.info
+               if info.get('ppid') != 1: continue
+               uid = info.get('username')
+               if uid != curuid and uid != gdexuser: continue
+               cmdline = info.get('cmdline') or []
+               if not cmdline: continue
+               line = ' '.join(cmdline)
+               idx = line.find(aname)
+               if idx < 0: continue
+               bid = info['pid']
                if bid in self.CBIDS: return -1
-               if uid == self.PGLOG['GDEXUSER']:
+               acmd = line[idx+len(aname):]
+               if uid == gdexuser:
                   acmd = re.sub(r'^\.(pl|py)\s+', '', acmd, 1)
                if re.match(r'^{}{}'.format(aname, acmd), bcmd): continue
                self.CBIDS[bid] = bcmd
                return 1
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+               continue
          time.sleep(2)
       return 0
 

{rda_python_common-2.1.9 → rda_python_common-2.1.11/src/rda_python_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 2.1.9
+Version: 2.1.11
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -11,8 +11,8 @@ Classifier: Development Status :: 5 - Production/Stable
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hvac
-Requires-Dist: psycopg2==2.9.10
+Requires-Dist: psycopg2-binary
+Requires-Dist: psutil
 Requires-Dist: rda-python-globus
 Requires-Dist: unidecode
 Requires-Dist: hvac
@@ -92,7 +92,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `2.1.9`).  If the import
+You should see the installed version (currently `2.1.11`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-2.1.9 → rda_python_common-2.1.11}/src/rda_python_common.egg-info/requires.txt

@@ -1,5 +1,5 @@
-hvac
-psycopg2==2.9.10
+psycopg2-binary
+psutil
 rda-python-globus
 unidecode
 hvac