rda-python-common 3.0.15__tar.gz → 3.0.17__tar.gz

{rda_python_common-3.0.15/src/rda_python_common.egg-info → rda_python_common-3.0.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.15
+Version: 3.0.17
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.15`).  If the import
+You should see the installed version (currently `3.0.17`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.15 → rda_python_common-3.0.17}/README.md

@@ -165,7 +165,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.15`).  If the import
+You should see the installed version (currently `3.0.17`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.15 → rda_python_common-3.0.17}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "3.0.15"
+version = "3.0.17"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_common-3.0.15 → rda_python_common-3.0.17}/src/rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "3.0.15"
+__version__ = "3.0.17"
 
 __all__ = [
    "PgLOG",

{rda_python_common-3.0.15 → rda_python_common-3.0.17}/src/rda_python_common/pg_file.py

@@ -766,7 +766,9 @@ class PgFile(PgUtil, PgSIG):
       """Download a file from the object store to the local filesystem.
 
       Changes to the target directory, downloads using isd_s3_cli, verifies size,
-      sets permissions, and renames if needed. Retries once on failure.
+      sets permissions, and renames if needed. Retries once on failure. A key that
+      is a prefix of multiple objects, such as a zarr store, is downloaded as a
+      directory by object_copy_local_directory().
 
       Args:
          tofile (str): Destination local file path.
@@ -783,6 +785,28 @@ class PgFile(PgUtil, PgSIG):
       if not finfo:
          if finfo != None: return ret
          return self.lmsg(fromfile, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], tofile), logact)
+      if not finfo['isfile']: return self.object_copy_local_directory(tofile, fromfile, bucket, logact)
+      return self.object_get_local(tofile, fromfile, finfo['data_size'], bucket, logact)
+
+   # Download a single object of a known size to a local file
+   #   tofile - target file name
+   # fromfile - source object key name
+   #    fsize - size of the object, to verify the download against
+   #   bucket - bucket name on Object store
+   def object_get_local(self, tofile, fromfile, fsize, bucket, logact = 0):
+      """Download one object key to a local file and verify its size.
+
+      Args:
+         tofile (str): Destination local file path.
+         fromfile (str): Object key (source path in the bucket).
+         fsize (int): Expected size of the object.
+         bucket (str): Source bucket.
+         logact (int): Logging action flags; default 0.
+
+      Returns:
+         int: self.SUCCESS on success, self.FAILURE on error.
+      """
+      ret = self.FAILURE
       ocmd = self.OBJCTCMD
       cmd = "{} go -k {} -b {}".format(ocmd, fromfile, bucket)
       fromname = op.basename(fromfile)
@@ -797,7 +821,7 @@ class PgFile(PgUtil, PgSIG):
          buf = self.pgsystem(cmd, logact, self.CMDBTH)
          info = self.check_local_file(fromname, 143, logact|self.PFSIZE)   # 1+2+4+8+128
          if info:
-            if info['data_size'] == finfo['data_size']:
+            if info['data_size'] == fsize:
                self.set_local_mode(fromfile, info['isfile'], 0, info['mode'], info['logname'], logact)
                if toname == fromname or self.move_local_file(toname, fromname, logact):
                   ret = self.SUCCESS
@@ -811,6 +835,41 @@ class PgFile(PgUtil, PgSIG):
          self.change_local_directory(odir, logact)
       return ret
 
+   # Copy an object directory, a key prefix holding multiple objects, to local
+   #    todir - target local directory name
+   #  fromdir - source object key prefix
+   #   bucket - bucket name on Object store
+   def object_copy_local_directory(self, todir, fromdir, bucket = None, logact = 0):
+      """Download every object under a key prefix into a local directory.
+
+      The object store has no directories; a zarr store and the like is a set of
+      objects sharing a key prefix. Each object is downloaded individually and
+      keeps its position relative to the prefix under todir.
+
+      Args:
+         todir (str): Destination local directory path.
+         fromdir (str): Object key prefix (source path in the bucket).
+         bucket (str | None): Source bucket; defaults to PGLOG['OBJCTBKT'].
+         logact (int): Logging action flags; default 0.
+
+      Returns:
+         int: self.SUCCESS on success, self.FAILURE on error.
+      """
+      if not bucket: bucket = self.PGLOG['OBJCTBKT']
+      ms = re.match(r'^(.+)/$', fromdir)
+      if ms: fromdir = ms.group(1)
+      flist = self.object_glob(fromdir, bucket, 0, logact)
+      if flist == self.FAILURE: return self.FAILURE
+      prefix = fromdir + '/'
+      keys = [key for key in flist if key.startswith(prefix)]
+      if not keys:
+         return self.lmsg(fromdir, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], todir), logact)
+      plen = len(prefix)
+      for key in keys:
+         tofile = "{}/{}".format(todir, key[plen:])
+         if not self.object_get_local(tofile, key, flist[key]['data_size'], bucket, logact): return self.FAILURE
+      return self.SUCCESS
+
    # Copy a remote file to object
    #   tofile - target object file name
    # fromfile - source remote file name
@@ -2298,22 +2357,23 @@ class PgFile(PgUtil, PgSIG):
             if re.match(r'^\[\]', buf): break
             if re.match(r'^\[\{', buf):
                ary = json.loads(buf)
-               hash = ary[0]
+               cnt = len(ary)
+               isdir = (cnt > 1 or ary[0]['Key'] != file)
+               # a key prefix has no timestamp of its own; report the newest member's
+               hash = max(ary, key = lambda a: a['LastModified']) if isdir else ary[0]
                uhash = None
                if ucmd:
                   ubuf = self.pgsystem(ucmd, self.LOGWRN, self.CMDRET)
                   if ubuf and re.match(r'^\{', ubuf): uhash = json.loads(ubuf)
                ret = self.object_file_stat(hash, uhash, opt)
-               if ret:
-                  cnt = len(ary)
-                  if cnt > 1 or hash['Key'] != file:
-                     ret['count'] = cnt
-                     ret['fname'] = op.basename(file)
-                     ret['isfile'] = 0
-                     size = 0
-                     for a in ary:
-                        size += int(a['Size'])
-                     ret['data_size'] = size
+               if ret and isdir:
+                  ret['count'] = cnt
+                  ret['fname'] = op.basename(file)
+                  ret['isfile'] = 0
+                  size = 0
+                  for a in ary:
+                     size += int(a['Size'])
+                  ret['data_size'] = size
                uhash = None
                break
          if opt&64: return self.FAILURE

{rda_python_common-3.0.15 → rda_python_common-3.0.17/src/rda_python_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.15
+Version: 3.0.17
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.15`).  If the import
+You should see the installed version (currently `3.0.17`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.