rda-python-common 2.1.10__py3-none-any.whl → 3.0.0__py3-none-any.whl

rda_python_common/PgDBI.py

@@ -17,12 +17,53 @@ import re
 import time
 import hvac
 from datetime import datetime
-import psycopg2 as PgSQL
-from psycopg2.extras import execute_values
-from psycopg2.extras import execute_batch
 from os import path as op
 from . import PgLOG
 
+# Prefer psycopg (v3); fall back to psycopg2 if v3 is not installed.
+try:
+   import psycopg as PgSQL
+   PG_DRIVER = 'psycopg3'
+
+   def execute_values(cursor, sql, argslist, page_size=100):
+      """Compatibility shim for psycopg2.extras.execute_values on psycopg3.
+
+      Rewrites ``VALUES %s`` placeholder to ``VALUES (%s, %s, ...)`` based on
+      the column count inferred from the first row, then dispatches to
+      psycopg3's ``executemany`` (which already batches efficiently).
+      """
+      if not argslist: return
+      ncol = len(argslist[0])
+      row_ph = '(' + ','.join(['%s'] * ncol) + ')'
+      new_sql = re.sub(r'(?i)\bVALUES\s+%s\b', 'VALUES ' + row_ph, sql, count=1)
+      cursor.executemany(new_sql, argslist)
+
+   def execute_batch(cursor, sql, argslist, page_size=100):
+      """Compatibility shim for psycopg2.extras.execute_batch on psycopg3."""
+      cursor.executemany(sql, argslist)
+
+   def get_pgcode(pgerr):
+      """Return SQLSTATE for a psycopg3 error (via err.diag.sqlstate)."""
+      diag = getattr(pgerr, 'diag', None)
+      return getattr(diag, 'sqlstate', None) if diag is not None else None
+
+   def get_pgerror(pgerr):
+      """Return primary error message for a psycopg3 error (via err.diag.message_primary)."""
+      diag = getattr(pgerr, 'diag', None)
+      return getattr(diag, 'message_primary', None) if diag is not None else None
+except ImportError:
+   import psycopg2 as PgSQL
+   from psycopg2.extras import execute_values, execute_batch
+   PG_DRIVER = 'psycopg2'
+
+   def get_pgcode(pgerr):
+      """Return SQLSTATE for a psycopg2 error (via err.pgcode)."""
+      return getattr(pgerr, 'pgcode', None)
+
+   def get_pgerror(pgerr):
+      """Return primary error message for a psycopg2 error (via err.pgerror)."""
+      return getattr(pgerr, 'pgerror', None)
+
 pgdb = None    # reference to a connected database object
 curtran = 0    # 0 - no transaction, 1 - in transaction
 NMISSES = []   # array of mising userno
@@ -439,8 +480,8 @@ def check_dberror(pgerr, pgcnt, sqlstr, ary, logact = PGDBI['ERRLOG']):
 
    ret = PgLOG.FAILURE
 
-   pgcode = pgerr.pgcode
-   pgerror = pgerr.pgerror
+   pgcode = get_pgcode(pgerr)
+   pgerror = get_pgerror(pgerr)
    dberror = "{} {}".format(pgcode, pgerror) if pgcode and pgerror else str(pgerr)
    if pgcnt < PgLOG.PGLOG['DBRETRY']:
       if not pgcode:
@@ -517,7 +558,7 @@ def pgconnect(reconnect = 0, pgcnt = 0, autocommit = True):
       reconnect = 0   # initial connection
 
    while True:
-      config = {'database' : PGDBI['DBNAME'],
+      config = {'dbname' : PGDBI['DBNAME'],
                     'user' : PGDBI['LNNAME']}
       if PGDBI['DBSHOST'] == PgLOG.PGLOG['HOSTNAME']:
          config['host'] = 'localhost'
@@ -526,7 +567,7 @@ def pgconnect(reconnect = 0, pgcnt = 0, autocommit = True):
          if not PGDBI['DBPORT']: PGDBI['DBPORT'] = get_dbport(PGDBI['DBNAME'])
       if PGDBI['DBPORT']: config['port'] = PGDBI['DBPORT']
       config['password'] = '***'
-      sqlstr = "psycopg2.connect(**{})".format(config)
+      sqlstr = "{}.connect(**{})".format(PG_DRIVER, config)
       config['password'] = get_pgpass_password()
       if PgLOG.PGLOG['DBGLEVEL']: PgLOG.pgdbg(1000, sqlstr)
       try:

rda_python_common/PgFile.py

@@ -788,7 +788,7 @@ def delete_backup_file(file, endpoint = None, logact = 0):
    return PgLOG.FAILURE
 
 #
-# reset local file/directory information to make them writable for PgLOG.PGLOG['GDEXUSER']
+# reset local file/directory information to make them writable for PgLOG.PGLOG['COMMONUSER']
 # file - file name (mandatory)
 # info - gathered file info with option 14, None means file not exists
 #

rda_python_common/PgLOG.py

@@ -60,7 +60,7 @@ MISLOG = (0x00811)   # cannot access logfile
 EMLSUM = (0x08000)   # record as email summary
 EMEROL = (0x10000)   # record error as email only
 EMLALL = (0x1D208)   # all email acts
-DOSUDO = (0x20000)   # add 'sudo -u PGLOG['GDEXUSER']'
+DOSUDO = (0x20000)   # add 'sudo -u PGLOG['COMMONUSER']'
 NOTLOG = (0x40000)   # do not log any thing
 OVRIDE = (0x80000)   # do override existing file or record
 NOWAIT = (0x100000)  # do not wait on globus task to finish
@@ -100,9 +100,8 @@ PGLOG = {   # more defined in untaint_suid() with environment variables
    'BACKROOT': "/DRDATA/DECS",  # backup path for desaster recovering tape on hpss
    'OLDAROOT': "/FS/DSS",       # old root path on hpss
    'OLDBROOT': "/DRDATA/DSS",   # old backup tape on hpss
-   'GDEXUSER' : "gdexdata",  # common gdex user name
-   'GDEXEMAIL' : "zji",    # specialist to receipt email intead of common gdex user name
-   'SUDOGDEX' : 0,          # 1 to allow sudo to PGLOG['GDEXUSER']
+   # COMMONUSER and ADMINUSER are set below via SETPGLOG (env overrides PG<KEY>)
+   'SUDOGDEX' : 0,          # 1 to allow sudo to PGLOG['COMMONUSER']
    'HOSTNAME' : '',        # current host name the process in running on
    'OBJCTSTR' : "object",
    'BACKUPNM' : "quasar",
@@ -140,10 +139,22 @@ PGLOG = {   # more defined in untaint_suid() with environment variables
    'EMLPORT' : 25
 }
 
-PGLOG['RDAUSER'] = PGLOG['GDEXUSER']
+def SETPGLOG(key, default):
+   """Set ``PGLOG[key]`` from environment variable ``PG<key>`` or fall back
+   to ``default`` if the variable is unset.  Used to make per-environment
+   overrides (e.g. PGCOMMONUSER, PGADMINUSER) survive package upgrades."""
+   PGLOG[key] = os.environ.get('PG' + key, default)
+
+SETPGLOG("COMMONUSER", "gdexdata")
+SETPGLOG("ADMINUSER", "zji")
+
+PGLOG['RDAUSER'] = PGLOG['COMMONUSER']
 PGLOG['RDAGRP'] = PGLOG['GDEXGRP']
-PGLOG['RDAEMAIL'] = PGLOG['GDEXEMAIL']
+PGLOG['RDAEMAIL'] = PGLOG['ADMINUSER']
 PGLOG['SUDORDA'] = PGLOG['SUDOGDEX']
+# backwards-compat aliases (deprecated: use COMMONUSER / ADMINUSER)
+PGLOG['GDEXUSER'] = PGLOG['COMMONUSER']
+PGLOG['GDEXEMAIL'] = PGLOG['ADMINUSER']
 
 HOSTTYPES = {
    'rda' : 'dsg_mach',
@@ -317,15 +328,15 @@ def send_python_email(subject = None, receiver = None, msg = None, sender = None
    docc = False if cc else True
    if not sender:
       sender = PGLOG['CURUID']
-      if sender != PGLOG['GDEXUSER']: docc = False
-   if sender == PGLOG['GDEXUSER']: sender = PGLOG['GDEXEMAIL']
+      if sender != PGLOG['COMMONUSER']: docc = False
+   if sender == PGLOG['COMMONUSER']: sender = PGLOG['ADMINUSER']
    if sender.find('@') == -1: sender += "@ucar.edu"
    if not receiver:
       receiver = PGLOG['EMLADDR'] if PGLOG['EMLADDR'] else PGLOG['CURUID']
-   if receiver == PGLOG['GDEXUSER']: receiver = PGLOG['GDEXEMAIL']
+   if receiver == PGLOG['COMMONUSER']: receiver = PGLOG['ADMINUSER']
    if receiver.find('@') == -1: receiver += "@ucar.edu"
 
-   if docc and not re.match(PGLOG['GDEXUSER'], sender): add_carbon_copy(sender, 1)
+   if docc and not re.match(PGLOG['COMMONUSER'], sender): add_carbon_copy(sender, 1)
    emlmsg = EmailMessage()
    emlmsg.set_content(msg)
    emlmsg['From'] = sender
@@ -1167,6 +1178,10 @@ def get_command(cmdstr = None):
 
    if not cmdstr: cmdstr = sys.argv[0]
    cmdstr = op.basename(cmdstr)
+   if cmdstr.startswith('setuid_'):
+      euser = pwd.getpwuid(os.geteuid()).pw_name
+      if euser == PGLOG['COMMONUSER']:
+         cmdstr = cmdstr[len('setuid_'):]
    ms = re.match(r'^(.+)\.(py|pl)$', cmdstr)
    if ms:
       return ms.group(1)
@@ -1182,11 +1197,11 @@ def get_local_command(cmd, asuser = None):
    cuser = PGLOG['SETUID'] if PGLOG['SETUID'] else PGLOG['CURUID']
    if not asuser or cuser == asuser: return cmd
 
-   if cuser == PGLOG['GDEXUSER']:
+   if cuser == PGLOG['COMMONUSER']:
       wrapper = "pgstart_" + asuser
       if valid_command(wrapper): return "{} {}".format(wrapper, cmd)
-   elif PGLOG['SUDOGDEX'] and asuser == PGLOG['GDEXUSER']:
-      return "sudo -u {} {}".format(PGLOG['GDEXUSER'], cmd)    # sudo as user gdexdata
+   elif PGLOG['SUDOGDEX'] and asuser == PGLOG['COMMONUSER']:
+      return "sudo -u {} {}".format(PGLOG['COMMONUSER'], cmd)    # sudo as user gdexdata
 
    return cmd
 
@@ -1209,12 +1224,12 @@ def get_hpss_command(cmd, asuser = None, hcmd = None):
    if not hcmd: hcmd = 'hsi'
 
    if asuser and cuser != asuser:
-      if cuser == PGLOG['GDEXUSER']:
+      if cuser == PGLOG['COMMONUSER']:
          return "{} sudo -u {} {}".format(hcmd, asuser, cmd)      # setuid wrapper as user asuser
-      elif PGLOG['SUDOGDEX'] and asuser == PGLOG['GDEXUSER']:
-         return "sudo -u {} {} {}".format(PGLOG['GDEXUSER'], hcmd, cmd)    # sudo as user gdexdata
+      elif PGLOG['SUDOGDEX'] and asuser == PGLOG['COMMONUSER']:
+         return "sudo -u {} {} {}".format(PGLOG['COMMONUSER'], hcmd, cmd)    # sudo as user gdexdata
 
-   if cuser != PGLOG['GDEXUSER']:
+   if cuser != PGLOG['COMMONUSER']:
       if re.match(r'^ls ', cmd) and hcmd == 'hsi':
          return "hpss" + cmd    # use 'hpssls' instead of 'hsi ls'
       elif re.match(r'^htar -tvf', hcmd):
@@ -1231,8 +1246,8 @@ def get_sync_command(host, asuser = None):
 
    host = get_short_host(host)
 
-   if (not (PGLOG['SETUID'] and PGLOG['SETUID'] == PGLOG['GDEXUSER']) and
-      (not asuser or asuser == PGLOG['GDEXUSER'])):
+   if (not (PGLOG['SETUID'] and PGLOG['SETUID'] == PGLOG['COMMONUSER']) and
+      (not asuser or asuser == PGLOG['COMMONUSER'])):
       return "sync" + host
 
    return host + "-sync"
@@ -1246,7 +1261,7 @@ def set_suid(cuid = 0):
    if cuid != PGLOG['EUID'] or cuid != PGLOG['RUID']:
       os.setreuid(cuid, cuid)
       PGLOG['SETUID'] = pwd.getpwuid(cuid).pw_name
-      if not (PGLOG['SETUID'] == PGLOG['GDEXUSER'] or cuid == PGLOG['RUID']):
+      if not (PGLOG['SETUID'] == PGLOG['COMMONUSER'] or cuid == PGLOG['RUID']):
          set_specialist_environments(PGLOG['SETUID'])
          PGLOG['CURUID'] == PGLOG['SETUID']      # set CURUID to a specific specialist
 
@@ -1262,12 +1277,12 @@ def set_common_pglog():
    PGLOG['EUID'] = os.geteuid()
    PGLOG['CURUID'] = pwd.getpwuid(PGLOG['RUID']).pw_name
    try:
-      PGLOG['RDAUID'] = PGLOG['GDEXUID'] = pwd.getpwnam(PGLOG['GDEXUSER']).pw_uid
+      PGLOG['RDAUID'] = PGLOG['GDEXUID'] = pwd.getpwnam(PGLOG['COMMONUSER']).pw_uid
       PGLOG['RDAGID'] = PGLOG['GDEXGID'] = grp.getgrnam(PGLOG['GDEXGRP']).gr_gid
    except:
       PGLOG['RDAUID'] = PGLOG['GDEXUID'] = 0
       PGLOG['RDAGID'] = PGLOG['GDEXGID'] = 0
-   if PGLOG['CURUID'] == PGLOG['GDEXUSER']: PGLOG['SETUID'] = PGLOG['GDEXUSER']
+   if PGLOG['CURUID'] == PGLOG['COMMONUSER']: PGLOG['SETUID'] = PGLOG['COMMONUSER']
 
    PGLOG['HOSTNAME'] = get_host()
    for htype in HOSTTYPES:

rda_python_common/PgOPT.py

@@ -624,7 +624,7 @@ def set_email_logact():
 def validate_dsowner(aname, dsid = None, logname = None, pgds = 0, logact = 0):
 
    if not logname: logname = (params['LN'] if 'LN' in params else PgLOG.PGLOG['CURUID'])
-   if logname == PgLOG.PGLOG['GDEXUSER']: return 1
+   if logname == PgLOG.PGLOG['COMMONUSER']: return 1
 
    dsids = {}
    if dsid:
@@ -1638,7 +1638,7 @@ def send_request_email_notice(pgrqst, errmsg, fcount, rstat, readyfile = None, p
    exclude = (einfo['SENDER'] if errmsg else einfo['RECEIVER'])
    if not errmsg and pgcntl and pgcntl['ccemail']:
       PgLOG.add_carbon_copy(pgcntl['ccemail'], 1, exclude, pgrqst['specialist'])
-   if PgLOG.PGLOG['CURUID'] != pgrqst['specialist'] and PgLOG.PGLOG['CURUID'] != PgLOG.PGLOG['GDEXUSER']:
+   if PgLOG.PGLOG['CURUID'] != pgrqst['specialist'] and PgLOG.PGLOG['CURUID'] != PgLOG.PGLOG['COMMONUSER']:
       PgLOG.add_carbon_copy(PgLOG.PGLOG['CURUID'], 1, exclude)
    if 'CC' in params: PgLOG.add_carbon_copy(params['CC'], 0, exclude)
    einfo['CCD'] = PgLOG.PGLOG['CCDADDR']

rda_python_common/PgSIG.py

@@ -1089,7 +1089,7 @@ def record_background(bcmd, logact = PgLOG.LOGWRN):
       aname = bcmd
 
    mp = r"^\s*(\S+)\s+(\d+)\s+1\s+.*{}(.*)$".format(aname)
-   pc = "ps -u {},{} -f | grep ' 1 ' | grep {}".format(PgLOG.PGLOG['CURUID'], PgLOG.PGLOG['GDEXUSER'], aname)
+   pc = "ps -u {},{} -f | grep ' 1 ' | grep {}".format(PgLOG.PGLOG['CURUID'], PgLOG.PGLOG['COMMONUSER'], aname)
    for i in range(2):
       buf = PgLOG.pgsystem(pc, logact, 20+1024)
       if buf:
@@ -1100,7 +1100,7 @@ def record_background(bcmd, logact = PgLOG.LOGWRN):
             (uid, sbid, acmd) = ms.groups()
             bid = int(sbid)
             if bid in CBIDS: return -1
-            if uid == PgLOG.PGLOG['GDEXUSER']:
+            if uid == PgLOG.PGLOG['COMMONUSER']:
                acmd = re.sub(r'^\.(pl|py)\s+', '', acmd, 1)
             if re.match(r'^{}{}'.format(aname, acmd), bcmd): continue
             CBIDS[bid] = bcmd

rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "2.1.10"
+__version__ = "3.0.0"
 
 __all__ = [
    "PgLOG",

rda_python_common/pg_dbi.py

@@ -13,24 +13,75 @@ import re
 import time
 import hvac
 from datetime import datetime
-import psycopg2 as PgSQL
-from psycopg2.extras import execute_values
-from psycopg2.extras import execute_batch
 from os import path as op
 from .pg_log import PgLOG
 
+# Driver selection: prefer psycopg (v3) as the default driver; fall back to
+# psycopg2 when psycopg is not installed.  Both drivers share enough surface
+# (connect(**config), Error/OperationalError, cursor.execute/executemany/
+# fetchone/description, connection.commit/rollback/close/autocommit) for this
+# module to use either transparently.
+try:
+   import psycopg as PgSQL
+   PG_DRIVER = 'psycopg3'
+
+   # psycopg3 removed extras.execute_values / extras.execute_batch.
+   # cursor.executemany() in psycopg3 is efficient by default (it uses prepared
+   # statements internally), so the shims below provide matching signatures.
+
+   def execute_values(cursor, sql, argslist, page_size=100):
+      """Driver-neutral shim providing psycopg2.extras.execute_values() on psycopg3.
+
+      Replaces the single ``VALUES %s`` placeholder in ``sql`` (as expected by
+      psycopg2's execute_values) with a per-row ``VALUES (%s, %s, ...)`` tuple
+      built from the first row, then calls ``cursor.executemany()``.  Lets the
+      same call site work under either psycopg (v3) or psycopg2.
+      """
+      if not argslist: return
+      ncol = len(argslist[0])
+      row_ph = '(' + ','.join(['%s']*ncol) + ')'
+      new_sql = re.sub(r'(?i)\bVALUES\s+%s\b', 'VALUES ' + row_ph, sql, count=1)
+      cursor.executemany(new_sql, argslist)
+
+   def execute_batch(cursor, sql, argslist, page_size=100):
+      """Driver-neutral shim providing psycopg2.extras.execute_batch() on psycopg3."""
+      cursor.executemany(sql, argslist)
+
+   def get_pgcode(pgerr):
+      """Return the 5-char SQLSTATE code from a psycopg3 error, or None."""
+      diag = getattr(pgerr, 'diag', None)
+      return getattr(diag, 'sqlstate', None) if diag is not None else None
+
+   def get_pgerror(pgerr):
+      """Return the primary error message from a psycopg3 error, or None."""
+      diag = getattr(pgerr, 'diag', None)
+      return getattr(diag, 'message_primary', None) if diag is not None else None
+except ImportError:
+   import psycopg2 as PgSQL
+   from psycopg2.extras import execute_values, execute_batch
+   PG_DRIVER = 'psycopg2'
+
+   def get_pgcode(pgerr):
+      """Return the 5-char SQLSTATE code from a psycopg2 error, or None."""
+      return getattr(pgerr, 'pgcode', None)
+
+   def get_pgerror(pgerr):
+      """Return the server error message from a psycopg2 error, or None."""
+      return getattr(pgerr, 'pgerror', None)
+
 class PgDBI(PgLOG):
    """PostgreSQL Database Interface layer extending PgLOG.
 
    Provides a high-level API for connecting to and querying PostgreSQL databases
-   using psycopg2. Supports single and batch INSERT, SELECT, UPDATE, and DELETE
-   operations, transaction management, schema introspection, user lookups, usage
-   tracking, and credential retrieval from .pgpass or OpenBao.
+   using psycopg (v3) when available, falling back to psycopg2. Supports single
+   and batch INSERT, SELECT, UPDATE, and DELETE operations, transaction
+   management, schema introspection, user lookups, usage tracking, and
+   credential retrieval from .pgpass or OpenBao.
 
    Inherits all logging and utility helpers from PgLOG.
 
    Instance Attributes:
-      pgdb (connection | None): Active psycopg2 connection, or None when disconnected.
+      pgdb (connection | None): Active psycopg/psycopg2 connection, or None when disconnected.
       curtran (int): Transaction counter: 0 = idle, >0 = inside a transaction.
       NMISSES (list): Cached list of scientist IDs (userno) not found in the DB.
       LMISSES (list): Cached list of login names not found in the DB.
@@ -40,7 +91,8 @@ class PgDBI(PgLOG):
       SYSDOWN (dict): Cache of system-down status records keyed by hostname.
       PGDBI (dict): Active connection and configuration parameters.
       PGSIGNS (list): Special comparison sign tokens recognised by get_field_condition().
-      CHCODE (int): psycopg2 type code for CHAR columns (used to strip trailing spaces).
+      CHCODE (int): Driver type code for CHAR columns (used to strip trailing spaces);
+                    set from psycopg/psycopg2 depending on which driver is in use.
       DBPORTS (dict): Mapping of database names to non-default TCP port numbers.
       DBPASS (dict): Credentials loaded from .pgpass, keyed by (host, port, db, user).
       DBBAOS (dict): Credentials loaded from OpenBao, keyed by database name.
@@ -544,14 +596,15 @@ class PgDBI(PgLOG):
       return tbname
 
    def check_dberror(self, pgerr, pgcnt, sqlstr, ary, logact = None):
-      """Classify a psycopg2 error and decide whether to retry or abort.
+      """Classify a psycopg/psycopg2 error and decide whether to retry or abort.
 
       Handles connection errors (08xxx, 57xxx), lock errors (55xxx), aborted
       transactions (25P02), and missing-table errors (42P01 with ADDTBL flag).
       Retries up to PGLOG['DBRETRY'] times; exits after that threshold.
 
       Args:
-         pgerr (psycopg2.Error): The caught database exception.
+         pgerr (PgSQL.Error): The caught database exception
+                              (psycopg.Error or psycopg2.Error).
          pgcnt (int): Current retry count (0-based).
          sqlstr (str): SQL statement that caused the error, for logging.
          ary: Bound values that were passed to the statement, for logging.
@@ -562,8 +615,8 @@ class PgDBI(PgLOG):
       """
       if logact is None: logact = self.PGDBI['ERRLOG']
       ret = self.FAILURE
-      pgcode = pgerr.pgcode
-      pgerror = pgerr.pgerror
+      pgcode = get_pgcode(pgerr)
+      pgerror = get_pgerror(pgerr)
       dberror = "{} {}".format(pgcode, pgerror) if pgcode and pgerror else str(pgerr)
       if pgcnt < self.PGLOG['DBRETRY']:
          if not pgcode:
@@ -640,15 +693,15 @@ class PgDBI(PgLOG):
          autocommit (bool): Whether to enable autocommit on the new connection.
 
       Returns:
-         connection | int: psycopg2 connection on success, self.FAILURE on error.
+         connection | int: psycopg/psycopg2 connection on success, self.FAILURE on error.
       """
       if self.pgdb:
          if reconnect and not self.pgdb.closed: return self.pgdb    # no need reconnect
       elif reconnect:
          reconnect = 0   # initial connection
       while True:
-         config = {'database': self.PGDBI['DBNAME'],
-                       'user': self.PGDBI['LNNAME']}
+         config = {'dbname': self.PGDBI['DBNAME'],
+                      'user': self.PGDBI['LNNAME']}
          if self.PGDBI['DBSHOST'] == self.PGLOG['HOSTNAME']:
             config['host'] = 'localhost'
          else:
@@ -656,7 +709,7 @@ class PgDBI(PgLOG):
             if not self.PGDBI['DBPORT']: self.PGDBI['DBPORT'] = self.get_dbport(self.PGDBI['DBNAME'])
          if self.PGDBI['DBPORT']: config['port'] = self.PGDBI['DBPORT']
          config['password'] = '***'
-         sqlstr = "psycopg2.connect(**{})".format(config)
+         sqlstr = "{}.connect(**{})".format(PG_DRIVER, config)
          config['password'] = self.get_pgpass_password()
          if self.PGLOG['DBGLEVEL']: self.pgdbg(1000, sqlstr)
          try:
@@ -675,7 +728,7 @@ class PgDBI(PgLOG):
       errors. The search path includes PGDBI['SCPATH'] when it differs from SCNAME.
 
       Returns:
-         cursor | int: psycopg2 cursor on success, self.FAILURE on error.
+         cursor | int: psycopg/psycopg2 cursor on success, self.FAILURE on error.
       """
       pgcur = None
       if not self.pgdb:
@@ -924,7 +977,8 @@ class PgDBI(PgLOG):
       """Insert multiple records into a database table efficiently.
 
       When getid is set, executes individual inserts to capture each returned ID.
-      Otherwise uses psycopg2 execute_values() for a single bulk INSERT.
+      Otherwise uses execute_values() (psycopg2's bulk helper, or the
+      executemany()-based shim on psycopg v3) for a single bulk INSERT.
 
       Args:
          tablename (str): Target table name.
@@ -1365,8 +1419,9 @@ class PgDBI(PgLOG):
    def pgmupdt(self, tablename, records, cnddicts, logact = None):
       """Update multiple rows using parallel value and condition dicts.
 
-      Uses psycopg2 execute_batch() for efficient bulk updates. The number of
-      values in records and cnddicts must match.
+      Uses execute_batch() (psycopg2's bulk helper, or the executemany()-based
+      shim on psycopg v3) for efficient bulk updates. The number of values in
+      records and cnddicts must match.
 
       Args:
          tablename (str): Target table name.
@@ -1508,7 +1563,8 @@ class PgDBI(PgLOG):
    def pgmdel(self, tablename, cnddicts, logact = None):
       """Delete multiple rows using a multi-value condition dict.
 
-      Uses psycopg2 execute_batch() for efficient bulk deletes.
+      Uses execute_batch() (psycopg2's bulk helper, or the executemany()-based
+      shim on psycopg v3) for efficient bulk deletes.
 
       Args:
          tablename (str): Target table name.

rda_python_common/pg_file.py

@@ -19,6 +19,7 @@ import re
 import time
 import glob
 import json
+import hashlib
 from .pg_util import PgUtil
 from .pg_sig import PgSIG
 
@@ -986,7 +987,7 @@ class PgFile(PgUtil, PgSIG):
          return self.FINISH
       return self.FAILURE
 
-   # reset local file/directory information to make them writable for self.PGLOG['GDEXUSER']
+   # reset local file/directory information to make them writable for self.PGLOG['COMMONUSER']
    # file - file name (mandatory)
    # info - gathered file info with option 14, None means file not exists
    def reset_local_info(self, file, info = None, logact = 0):
@@ -2874,24 +2875,32 @@ class PgFile(PgUtil, PgSIG):
                             (with None for missing files) for multiple files, or None
                             on failure.
       """
-      cmd = 'md5sum '
       if count > 0:
          checksum = [None]*count
          for i in range(count):
             if op.isfile(file[i]):
-               chksm = self.pgsystem(cmd + file[i], logact, 20)
-               if chksm:
-                  ms = re.search(r'(\w{32})', chksm)
-                  if ms: checksum[i] = ms.group(1)
+               checksum[i] = self._file_md5(file[i], logact)
       else:
          checksum = None
          if op.isfile(file):
-            chksm = self.pgsystem(cmd + file, logact, 20)
-            if chksm:
-               ms = re.search(r'(\w{32})', chksm)
-               if ms: checksum = ms.group(1)
+            checksum = self._file_md5(file, logact)
       return checksum
 
+   def _file_md5(self, path, logact=0):
+      """Compute MD5 hex digest of *path*, reading in 1 MiB chunks.
+
+      Returns the hex digest string, or None on read error.
+      """
+      try:
+         h = hashlib.md5()
+         with open(path, 'rb') as fh:
+            for chunk in iter(lambda: fh.read(1048576), b''):
+               h.update(chunk)
+         return h.hexdigest()
+      except OSError as e:
+         self.pglog("Error md5sum {}: {}".format(path, str(e)), logact)
+         return None
+
    # Evaluate md5 checksums and compare them for two given files
    #  file1, file2: file names
    # Return: 0 if same and 1 if not