rda-python-common 3.0.23__tar.gz → 3.0.25__tar.gz

{rda_python_common-3.0.23/src/rda_python_common.egg-info → rda_python_common-3.0.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.23
+Version: 3.0.25
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.23`).  If the import
+You should see the installed version (currently `3.0.25`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.23 → rda_python_common-3.0.25}/README.md

@@ -165,7 +165,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.23`).  If the import
+You should see the installed version (currently `3.0.25`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.23 → rda_python_common-3.0.25}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "3.0.23"
+version = "3.0.25"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_common-3.0.23 → rda_python_common-3.0.25}/src/rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "3.0.23"
+__version__ = "3.0.25"
 
 __all__ = [
    "PgLOG",

{rda_python_common-3.0.23 → rda_python_common-3.0.25}/src/rda_python_common/pg_file.py

@@ -875,8 +875,14 @@ class PgFile(PgUtil, PgSIG):
       """Download every object under a key prefix into a local directory.
 
       The object store has no directories; a zarr store and the like is a set of
-      objects sharing a key prefix. Each object is downloaded individually and
-      keeps its position relative to the prefix under todir.
+      objects sharing a key prefix. The whole prefix comes down in one recursive
+      call, which is what makes this usable at all: such a store routinely holds
+      thousands of tiny members, and one process per member costs hours where
+      the recursive call costs seconds.
+
+      Nothing is fetched when every member is already staged under todir at the
+      object's size and no older than the object itself, so the second copy into
+      the same directory (dsarch makes one when it tars) is free.
 
       Args:
          todir (str): Destination local directory path.
@@ -890,17 +896,100 @@ class PgFile(PgUtil, PgSIG):
       if not bucket: bucket = self.PGLOG['OBJCTBKT']
       ms = re.match(r'^(.+)/$', fromdir)
       if ms: fromdir = ms.group(1)
-      flist = self.object_glob(fromdir, bucket, 0, logact)
+      # opt 1 adds the LastModified the 'lo' listing already carries; it is only
+      # bits 2 and 8 that would cost a metadata call per key
+      flist = self.object_glob(fromdir, bucket, 1, logact)
       if flist == self.FAILURE: return self.FAILURE
       prefix = fromdir + '/'
       keys = [key for key in flist if key.startswith(prefix)]
       if not keys:
          return self.lmsg(fromdir, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], todir), logact)
+      # a partly staged directory is downloaded again in full rather than member by
+      # member: one member costs about as much as the whole recursive call, so
+      # fetching even a handful individually is the slower way round
+      if not self.object_local_missing(todir, prefix, keys, flist, 1, logact):
+         # every member is current, but the key list is the authoritative one: a local
+         # file no longer under the prefix means the store was rewritten in place, and
+         # archiving it would keep members the store has dropped. start over instead
+         extra = self.local_extra_files(todir, prefix, keys)
+         if not extra: return self.SUCCESS
+         self.pglog("{}: restage over {} file(s) no longer in {}-{}".format(
+                    todir, len(extra), self.OHOST, fromdir), logact|self.LOGWRN)
+         if not self.delete_local_file(todir, logact): return self.FAILURE
+      self.make_local_directory(todir, logact)
+      cmd = "{} go -k {} -b {} -ld {} -r".format(self.OBJCTCMD, fromdir, bucket, todir)
+      buf = self.pgsystem(cmd, logact, self.CMDBTH)
+      # verify on size alone; a member just downloaded is newer than the object by
+      # definition, and comparing timestamps here would only expose a clock skew
+      miss = self.object_local_missing(todir, prefix, keys, flist, 0, logact)
+      if miss:
+         return self.errlog("{}-{}: Error download {} of {} member(s) to {}\n{}".format(
+                            self.OHOST, fromdir, len(miss), len(keys), todir, buf), 'O', 1, logact)
+      return self.SUCCESS
+
+   # List the members of an object key prefix that are not staged under todir yet
+   #    todir - target local directory name
+   #   prefix - source object key prefix, including the trailing '/'
+   #     keys - the member keys to check
+   #    flist - object_glob() info, keyed by member key
+   #  chktime - also require the local copy to be no older than the object
+   def object_local_missing(self, todir, prefix, keys, flist, chktime = 0, logact = 0):
+      """Return the member keys that still need downloading under a key prefix.
+
+      A local copy counts as present when it matches the object size, and with
+      chktime also when it is no older than the object's LastModified.
+
+      Args:
+         todir (str): Destination local directory path.
+         prefix (str): Object key prefix including the trailing '/'.
+         keys (list): Member keys to check.
+         flist (dict): Mapping of key → object info from object_glob().
+         chktime (int): Also compare modification times; default 0.
+         logact (int): Logging action flags; default 0.
+
+      Returns:
+         list: The member keys not staged under todir.
+      """
+      miss = []
       plen = len(prefix)
       for key in keys:
+         oinfo = flist[key]
          tofile = "{}/{}".format(todir, key[plen:])
-         if not self.object_get_local(tofile, key, flist[key]['data_size'], bucket, logact): return self.FAILURE
-      return self.SUCCESS
+         linfo = self.check_local_file(tofile, 1, logact)
+         if linfo and linfo['data_size'] == oinfo['data_size']:
+            if not chktime: continue
+            # an unparsable LastModified leaves date_modified unset; fetch again then
+            if 'date_modified' in oinfo and \
+               self.cmptime(linfo['date_modified'], linfo['time_modified'],
+                            oinfo['date_modified'], oinfo['time_modified']) >= 0: continue
+         miss.append(key)
+      return miss
+
+   # List the files staged under todir that the object key prefix no longer holds
+   #    todir - target local directory name
+   #   prefix - source object key prefix, including the trailing '/'
+   #     keys - the member keys the object store currently holds
+   def local_extra_files(self, todir, prefix, keys):
+      """Return the paths under todir that are not members of an object key prefix.
+
+      Paths are relative to todir, matching the layout a recursive download creates.
+
+      Args:
+         todir (str): Local directory holding the staged copy.
+         prefix (str): Object key prefix including the trailing '/'.
+         keys (list): The member keys currently under the prefix.
+
+      Returns:
+         list: Relative paths under todir with no matching member key.
+      """
+      plen = len(prefix)
+      members = set(key[plen:] for key in keys)
+      extra = []
+      for path, dirs, files in os.walk(todir):
+         for fname in files:
+            name = op.relpath(op.join(path, fname), todir)
+            if name not in members: extra.append(name)
+      return extra
 
    # Copy a remote file to object
    #   tofile - target object file name

{rda_python_common-3.0.23 → rda_python_common-3.0.25/src/rda_python_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.23
+Version: 3.0.25
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.23`).  If the import
+You should see the installed version (currently `3.0.25`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.