rda-python-common 3.0.14__tar.gz → 3.0.16__tar.gz

{rda_python_common-3.0.14/src/rda_python_common.egg-info → rda_python_common-3.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.14
+Version: 3.0.16
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.14`).  If the import
+You should see the installed version (currently `3.0.16`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.14 → rda_python_common-3.0.16}/README.md

@@ -165,7 +165,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.14`).  If the import
+You should see the installed version (currently `3.0.16`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.14 → rda_python_common-3.0.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "3.0.14"
+version = "3.0.16"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_common-3.0.14 → rda_python_common-3.0.16}/src/rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "3.0.14"
+__version__ = "3.0.16"
 
 __all__ = [
    "PgLOG",

{rda_python_common-3.0.14 → rda_python_common-3.0.16}/src/rda_python_common/pg_dbi.py

@@ -76,7 +76,7 @@ class PgDBI(PgLOG):
    using psycopg (v3) when available, falling back to psycopg2. Supports single
    and batch INSERT, SELECT, UPDATE, and DELETE operations, transaction
    management, schema introspection, user lookups, usage tracking, and
-   credential retrieval from .pgpass or OpenBao.
+   credential retrieval from the environment, .pgpass or OpenBao.
 
    Inherits all logging and utility helpers from PgLOG.
 
@@ -96,7 +96,10 @@ class PgDBI(PgLOG):
                     set from psycopg/psycopg2 depending on which driver is in use.
       DBPORTS (dict): Mapping of database names to non-default TCP port numbers.
       DBPASS (dict): Credentials loaded from .pgpass, keyed by (host, port, db, user).
-      DBBAOS (dict): Credentials loaded from OpenBao, keyed by database name.
+      DBBAOS (dict): Credentials loaded from OpenBao, keyed by db server node.
+      DBNODES (dict): Mapping of database names to their db server nodes, used when the
+                      node cannot be taken from the host being connected to.
+      DBSKEYS (dict): Mapping of login names to secret key names ('<lnname>pass' by default).
       DBNAMES (dict): Mapping of schema names to their parent database names.
       DBSOCKS (dict): Mapping of database names to Unix socket paths.
       VIEWHOMES (dict): Mapping of hostnames to home directories for the view host.
@@ -139,6 +142,19 @@ class PgDBI(PgLOG):
       self.DBPORTS = {'default': 0}
       self.DBPASS = {}
       self.DBBAOS = {}
+      # hard coded db server nodes for dbnames, for hosts not named for their nodes; a node
+      # names the OpenBao path (kv/gdex/<node>) and the password environment variable
+      # (<NODE>_<KEY>) the same secrets are passed in through
+      self.DBNODES = {
+         'ivaddb': 'pgdb03',
+         'ispddb': 'pgdb03',
+         'default': 'pgdb01',
+      }
+      # hard coded secret key names for db login names, defaulting to <lnname>pass
+      self.DBSKEYS = {
+         'postgres': 'password',
+         'metadata': 'metapass',
+      }
       # hard coded db names for given schema names
       self.DBNAMES = {
          'ivaddb': 'ivaddb',
@@ -2617,19 +2633,91 @@ class PgDBI(PgLOG):
    def get_pgpass_password(self):
       """Return the database password for the current connection settings.
 
-      Checks PGDBI['PWNAME'] first, then tries the .pgpass file (get_pgpassword()),
-      and finally falls back to OpenBao (get_baopassword()).
+      Checks PGDBI['PWNAME'] first, then the environment (get_envpassword(), as set
+      for containers on CIRRUS), then the .pgpass file (get_pgpassword()), and
+      finally falls back to OpenBao (get_baopassword()). A password found on local
+      or batch hosts is cached in the environment (set_envpassword()) so that the
+      child processes inherit it instead of looking it up again.
 
       Returns:
          str | None: Password string, or None when no credential is found.
       """
       if self.PGDBI['PWNAME']: return self.PGDBI['PWNAME']
+      pwname = self.get_envpassword()
+      if pwname: return pwname
       pwname = self.get_pgpassword()
       if not pwname: pwname = self.get_baopassword()
-      if not pwname:
-         self.pglog("Unable to find password for {} in .pgpass or OpenBao".format(self.PGDBI['DBNAME']), self.PGDBI['ERRLOG'])
+      if pwname:
+         self.set_envpassword(pwname)
+      else:
+         self.pglog("Unable to find password for {} in environment, .pgpass or OpenBao".format(self.PGDBI['DBNAME']), self.PGDBI['ERRLOG'])
       return pwname
 
+   def get_dbnodes(self):
+      """Return the db server nodes holding the secrets of the current connection.
+
+      The node named by DBSHOST comes first, such as pgdb02 for the view-only host,
+      so that a node with its own passwords is served its own secrets. The DBNODES
+      node of the current DBNAME follows it as a fallback, such as pgdb01 for pgdb02,
+      for a node that shares the passwords of the default one without holding secrets
+      of its own.
+
+      Returns:
+         list: Node names, such as ['pgdb02', 'pgdb01'], the first one preferred.
+      """
+      dbnodes = []
+      ms = re.match(r'^(pgdb\d+)$', self.PGDBI['DBSHOST']) if self.PGDBI['DBSHOST'] else None
+      if ms: dbnodes.append(ms.group(1))
+      dbnode = self.DBNODES.get(self.PGDBI['DBNAME'], self.DBNODES['default'])
+      if dbnode not in dbnodes: dbnodes.append(dbnode)
+
+      return dbnodes
+
+   def get_secret_name(self, dbnode):
+      """Build the name of the secret holding the password of the current connection.
+
+      The key comes from DBSKEYS for the current LNNAME, defaulting to '<lnname>pass'.
+      The pair names the secret in OpenBao (kv/gdex/<node> key <key>) and the
+      environment variable (<NODE>_<KEY>) the same secret is passed in through
+      on CIRRUS.
+
+      Args:
+         dbnode (str): Db server node holding the secret, such as 'pgdb01'.
+
+      Returns:
+         tuple: (node, key) naming the db server node and the secret key.
+      """
+      lnname = self.PGDBI['LNNAME']
+
+      return (dbnode, self.DBSKEYS.get(lnname, lnname + 'pass'))
+
+   def get_envpassword(self):
+      """Look up the password in the environment, as set for containers on CIRRUS.
+
+      The variable name is the uppercased '<node>_<key>' secret name, for example
+      PGDB01_DSSDBPASS for login name dssdb and PGDB03_IVADDBPASS for ivaddb. The
+      fallback nodes of get_dbnodes() are tried in turn.
+
+      Returns:
+         str | None: Password string, or None when no variable is set.
+      """
+      for dbnode in self.get_dbnodes():
+         pwname = os.environ.get("{}_{}".format(*self.get_secret_name(dbnode)).upper())
+         if pwname: return pwname
+
+      return None
+
+   def set_envpassword(self, pwname):
+      """Cache a password in the environment under the name get_envpassword() reads.
+
+      Called for a password found in .pgpass or OpenBao, on local and PBS batch hosts,
+      so that the child processes of the current one inherit it and skip the lookup.
+
+      Args:
+         pwname (str): Password to pass on to the child processes.
+      """
+      os.environ["{}_{}".format(*self.get_secret_name(self.get_dbnodes()[0])).upper()] = pwname
+
    def get_pgpassword(self):
       """Look up the password in the cached .pgpass credentials.
 
@@ -2646,16 +2734,20 @@ class PgDBI(PgLOG):
       return pwname
 
    def get_baopassword(self):
-      """Look up the password from OpenBao for the current database and login name.
+      """Look up the password from OpenBao for the current db server node and login name.
 
-      Loads OpenBao secrets for PGDBI['DBNAME'] on first call (or when not cached).
+      Loads the OpenBao secrets of a node on first call (or when not cached), trying
+      the fallback nodes of get_dbnodes() in turn.
 
       Returns:
          str | None: Password string, or None when not found in OpenBao.
       """
-      dbname = self.PGDBI['DBNAME']
-      if dbname not in self.DBBAOS: self.read_openbao()
-      return self.DBBAOS[dbname].get(self.PGDBI['LNNAME'])
+      for dbnode in self.get_dbnodes():
+         if dbnode not in self.DBBAOS: self.read_openbao(dbnode)
+         pwname = self.DBBAOS[dbnode].get(self.PGDBI['LNNAME'])
+         if pwname: return pwname
+
+      return None
 
    def read_pgpass(self):
       """Read the .pgpass file and populate DBPASS with credentials.
@@ -2675,21 +2767,17 @@ class PgDBI(PgLOG):
       except Exception:
           pass
 
-   def read_openbao(self):
-      """Read OpenBao secrets and populate DBBAOS with credentials for DBNAME.
+   def read_openbao(self, dbnode):
+      """Read OpenBao secrets and populate DBBAOS with the credentials of a db server node.
 
       Uses the hvac client to fetch key-value secrets from the configured BAOURL.
       Parses keys matching 'pass' patterns to extract database usernames and passwords.
+
+      Args:
+         dbnode (str): Db server node naming the secret path, such as 'pgdb01'.
       """
-      dbname = self.PGDBI['DBNAME']
-      self.DBBAOS[dbname] = {}
-      url = 'https://bao.k8s.ucar.edu/'
-      baopath = {
-         'ivaddb': 'gdex/pgdb03',
-         'ispddb': 'gdex/pgdb03',
-         'default': 'gdex/pgdb01'
-      }
-      dbpath = baopath[dbname] if dbname in baopath else baopath['default']
+      self.DBBAOS[dbnode] = {}
+      dbpath = 'gdex/' + dbnode
       client = hvac.Client(url=self.PGDBI.get('BAOURL'))
       client.token = self.PGLOG.get('BAOTOKEN')
       try:
@@ -2711,4 +2799,4 @@ class PgDBI(PgLOG):
             baoname =  'metadata' if pre == 'meta' else pre
          elif suf == 'word':
             baoname = 'postgres'
-         if baoname: self.DBBAOS[dbname][baoname] = baos[key] 
+         if baoname: self.DBBAOS[dbnode][baoname] = baos[key]

{rda_python_common-3.0.14 → rda_python_common-3.0.16}/src/rda_python_common/pg_file.py

@@ -766,7 +766,9 @@ class PgFile(PgUtil, PgSIG):
       """Download a file from the object store to the local filesystem.
 
       Changes to the target directory, downloads using isd_s3_cli, verifies size,
-      sets permissions, and renames if needed. Retries once on failure.
+      sets permissions, and renames if needed. Retries once on failure. A key that
+      is a prefix of multiple objects, such as a zarr store, is downloaded as a
+      directory by object_copy_local_directory().
 
       Args:
          tofile (str): Destination local file path.
@@ -783,6 +785,28 @@ class PgFile(PgUtil, PgSIG):
       if not finfo:
          if finfo != None: return ret
          return self.lmsg(fromfile, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], tofile), logact)
+      if not finfo['isfile']: return self.object_copy_local_directory(tofile, fromfile, bucket, logact)
+      return self.object_get_local(tofile, fromfile, finfo['data_size'], bucket, logact)
+
+   # Download a single object of a known size to a local file
+   #   tofile - target file name
+   # fromfile - source object key name
+   #    fsize - size of the object, to verify the download against
+   #   bucket - bucket name on Object store
+   def object_get_local(self, tofile, fromfile, fsize, bucket, logact = 0):
+      """Download one object key to a local file and verify its size.
+
+      Args:
+         tofile (str): Destination local file path.
+         fromfile (str): Object key (source path in the bucket).
+         fsize (int): Expected size of the object.
+         bucket (str): Source bucket.
+         logact (int): Logging action flags; default 0.
+
+      Returns:
+         int: self.SUCCESS on success, self.FAILURE on error.
+      """
+      ret = self.FAILURE
       ocmd = self.OBJCTCMD
       cmd = "{} go -k {} -b {}".format(ocmd, fromfile, bucket)
       fromname = op.basename(fromfile)
@@ -797,7 +821,7 @@ class PgFile(PgUtil, PgSIG):
          buf = self.pgsystem(cmd, logact, self.CMDBTH)
          info = self.check_local_file(fromname, 143, logact|self.PFSIZE)   # 1+2+4+8+128
          if info:
-            if info['data_size'] == finfo['data_size']:
+            if info['data_size'] == fsize:
                self.set_local_mode(fromfile, info['isfile'], 0, info['mode'], info['logname'], logact)
                if toname == fromname or self.move_local_file(toname, fromname, logact):
                   ret = self.SUCCESS
@@ -811,6 +835,41 @@ class PgFile(PgUtil, PgSIG):
          self.change_local_directory(odir, logact)
       return ret
 
+   # Copy an object directory, a key prefix holding multiple objects, to local
+   #    todir - target local directory name
+   #  fromdir - source object key prefix
+   #   bucket - bucket name on Object store
+   def object_copy_local_directory(self, todir, fromdir, bucket = None, logact = 0):
+      """Download every object under a key prefix into a local directory.
+
+      The object store has no directories; a zarr store and the like is a set of
+      objects sharing a key prefix. Each object is downloaded individually and
+      keeps its position relative to the prefix under todir.
+
+      Args:
+         todir (str): Destination local directory path.
+         fromdir (str): Object key prefix (source path in the bucket).
+         bucket (str | None): Source bucket; defaults to PGLOG['OBJCTBKT'].
+         logact (int): Logging action flags; default 0.
+
+      Returns:
+         int: self.SUCCESS on success, self.FAILURE on error.
+      """
+      if not bucket: bucket = self.PGLOG['OBJCTBKT']
+      ms = re.match(r'^(.+)/$', fromdir)
+      if ms: fromdir = ms.group(1)
+      flist = self.object_glob(fromdir, bucket, 0, logact)
+      if flist == self.FAILURE: return self.FAILURE
+      prefix = fromdir + '/'
+      keys = [key for key in flist if key.startswith(prefix)]
+      if not keys:
+         return self.lmsg(fromdir, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], todir), logact)
+      plen = len(prefix)
+      for key in keys:
+         tofile = "{}/{}".format(todir, key[plen:])
+         if not self.object_get_local(tofile, key, flist[key]['data_size'], bucket, logact): return self.FAILURE
+      return self.SUCCESS
+
    # Copy a remote file to object
    #   tofile - target object file name
    # fromfile - source remote file name

{rda_python_common-3.0.14 → rda_python_common-3.0.16/src/rda_python_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.14
+Version: 3.0.16
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.14`).  If the import
+You should see the installed version (currently `3.0.16`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.