rda-python-common 3.0.24__tar.gz → 3.0.26__tar.gz

{rda_python_common-3.0.24/src/rda_python_common.egg-info → rda_python_common-3.0.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.24
+Version: 3.0.26
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.24`).  If the import
+You should see the installed version (currently `3.0.26`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.24 → rda_python_common-3.0.26}/README.md

@@ -165,7 +165,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.24`).  If the import
+You should see the installed version (currently `3.0.26`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.24 → rda_python_common-3.0.26}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "3.0.24"
+version = "3.0.26"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_common-3.0.24 → rda_python_common-3.0.26}/src/rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "3.0.24"
+__version__ = "3.0.26"
 
 __all__ = [
    "PgLOG",

{rda_python_common-3.0.24 → rda_python_common-3.0.26}/src/rda_python_common/pg_file.py

@@ -794,7 +794,7 @@ class PgFile(PgUtil, PgSIG):
    #   tofile - target file name
    # fromfile - source file name
    #   bucket - bucket name on Object store
-   def object_copy_local(self, tofile, fromfile, bucket = None, logact = 0):
+   def object_copy_local(self, tofile, fromfile, bucket = None, logact = 0, finfo = None):
       """Download a file from the object store to the local filesystem.
 
       Changes to the target directory, downloads using isd_s3_cli, verifies size,
@@ -807,31 +807,42 @@ class PgFile(PgUtil, PgSIG):
          fromfile (str): Object key (source path in the bucket).
          bucket (str | None): Source bucket; defaults to PGLOG['OBJCTBKT'].
          logact (int): Logging action flags; default 0.
+         finfo (dict | None): check_object_file() info for fromfile, from a call
+                              with opt bit 1, when the caller already has it;
+                              saves listing the object a second time.
 
       Returns:
          int: self.SUCCESS on success, self.FAILURE on error.
       """
       ret = self.FAILURE
       if not bucket: bucket = self.PGLOG['OBJCTBKT']
-      finfo = self.check_object_file(fromfile, bucket, 0, logact)
+      # opt 1 adds the LastModified the 'lo' listing already carries, which
+      # object_get_local needs to tell whether tofile is staged already
+      if not finfo: finfo = self.check_object_file(fromfile, bucket, 1, logact)
       if not finfo:
          if finfo != None: return ret
          return self.lmsg(fromfile, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], tofile), logact)
-      if not finfo['isfile']: return self.object_copy_local_directory(tofile, fromfile, bucket, logact)
-      return self.object_get_local(tofile, fromfile, finfo['data_size'], bucket, logact)
+      if not finfo['isfile']:
+         return self.object_copy_local_directory(tofile, fromfile, bucket, logact, finfo.get('flist'))
+      return self.object_get_local(tofile, fromfile, finfo, bucket, logact)
 
    # Download a single object of a known size to a local file
    #   tofile - target file name
    # fromfile - source object key name
-   #    fsize - size of the object, to verify the download against
+   #    finfo - check_object_file() info of the object, to verify the download against
    #   bucket - bucket name on Object store
-   def object_get_local(self, tofile, fromfile, fsize, bucket, logact = 0):
+   def object_get_local(self, tofile, fromfile, finfo, bucket, logact = 0):
       """Download one object key to a local file and verify its size.
 
+      Nothing is fetched when tofile is already staged at the object's size and
+      no older than the object itself, unless logact carries OVRIDE, so the
+      second copy into the same file (dsarch makes one when it tars) is free.
+
       Args:
          tofile (str): Destination local file path.
          fromfile (str): Object key (source path in the bucket).
-         fsize (int): Expected size of the object.
+         finfo (dict): check_object_file() info of the object; 'data_size' is
+                       required, 'date_modified'/'time_modified' enable the skip.
          bucket (str): Source bucket.
          logact (int): Logging action flags; default 0.
 
@@ -839,6 +850,13 @@ class PgFile(PgUtil, PgSIG):
          int: self.SUCCESS on success, self.FAILURE on error.
       """
       ret = self.FAILURE
+      fsize = finfo['data_size']
+      if not logact&self.OVRIDE and 'date_modified' in finfo:
+         linfo = self.check_local_file(tofile, 1, logact)
+         if linfo and linfo['isfile'] and linfo['data_size'] == fsize and \
+            self.cmptime(linfo['date_modified'], linfo['time_modified'],
+                         finfo['date_modified'], finfo['time_modified']) >= 0:
+            return self.SUCCESS
       ocmd = self.OBJCTCMD
       cmd = "{} go -k {} -b {}".format(ocmd, fromfile, bucket)
       fromname = op.basename(fromfile)
@@ -871,23 +889,28 @@ class PgFile(PgUtil, PgSIG):
    #    todir - target local directory name
    #  fromdir - source object key prefix
    #   bucket - bucket name on Object store
-   def object_copy_local_directory(self, todir, fromdir, bucket = None, logact = 0):
+   #    flist - member listing from check_object_file(), if the caller has it
+   def object_copy_local_directory(self, todir, fromdir, bucket = None, logact = 0, flist = None):
       """Download every object under a key prefix into a local directory.
 
       The object store has no directories; a zarr store and the like is a set of
-      objects sharing a key prefix. Each object is downloaded individually and
-      keeps its position relative to the prefix under todir.
+      objects sharing a key prefix. The whole prefix comes down in one recursive
+      call, which is what makes this usable at all: such a store routinely holds
+      thousands of tiny members, and one process per member costs hours where
+      the recursive call costs seconds.
 
-      A member already staged under todir at the object's size and no older than
-      the object itself is left alone, so a store whose download was cut short
-      (by a batch walltime, say) resumes where it stopped instead of starting
-      over, and a second copy into the same directory costs nothing.
+      Nothing is fetched when every member is already staged under todir at the
+      object's size and no older than the object itself, so the second copy into
+      the same directory (dsarch makes one when it tars) is free.
 
       Args:
          todir (str): Destination local directory path.
          fromdir (str): Object key prefix (source path in the bucket).
          bucket (str | None): Source bucket; defaults to PGLOG['OBJCTBKT'].
          logact (int): Logging action flags; default 0.
+         flist (dict | None): Mapping of key → object info with LastModified, as
+                              check_object_file() keeps it for a key prefix;
+                              listed here when not given.
 
       Returns:
          int: self.SUCCESS on success, self.FAILURE on error.
@@ -895,25 +918,101 @@ class PgFile(PgUtil, PgSIG):
       if not bucket: bucket = self.PGLOG['OBJCTBKT']
       ms = re.match(r'^(.+)/$', fromdir)
       if ms: fromdir = ms.group(1)
-      # opt 1 adds the LastModified the 'lo' listing already carries; it is only
-      # bits 2 and 8 that would cost a metadata call per key
-      flist = self.object_glob(fromdir, bucket, 1, logact)
-      if flist == self.FAILURE: return self.FAILURE
+      if flist is None:
+         # opt 1 adds the LastModified the 'lo' listing already carries; it is only
+         # bits 2 and 8 that would cost a metadata call per key
+         flist = self.object_glob(fromdir, bucket, 1, logact)
+         if flist == self.FAILURE: return self.FAILURE
       prefix = fromdir + '/'
       keys = [key for key in flist if key.startswith(prefix)]
       if not keys:
          return self.lmsg(fromdir, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], todir), logact)
+      # a partly staged directory is downloaded again in full rather than member by
+      # member: one member costs about as much as the whole recursive call, so
+      # fetching even a handful individually is the slower way round
+      if not self.object_local_missing(todir, prefix, keys, flist, 1, logact):
+         # every member is current, but the key list is the authoritative one: a local
+         # file no longer under the prefix means the store was rewritten in place, and
+         # archiving it would keep members the store has dropped. start over instead
+         extra = self.local_extra_files(todir, prefix, keys)
+         if not extra: return self.SUCCESS
+         self.pglog("{}: restage over {} file(s) no longer in {}-{}".format(
+                    todir, len(extra), self.OHOST, fromdir), logact|self.LOGWRN)
+         if not self.delete_local_file(todir, logact): return self.FAILURE
+      self.make_local_directory(todir, logact)
+      cmd = "{} go -k {} -b {} -ld {} -r".format(self.OBJCTCMD, fromdir, bucket, todir)
+      buf = self.pgsystem(cmd, logact, self.CMDBTH)
+      # verify on size alone; a member just downloaded is newer than the object by
+      # definition, and comparing timestamps here would only expose a clock skew
+      miss = self.object_local_missing(todir, prefix, keys, flist, 0, logact)
+      if miss:
+         return self.errlog("{}-{}: Error download {} of {} member(s) to {}\n{}".format(
+                            self.OHOST, fromdir, len(miss), len(keys), todir, buf), 'O', 1, logact)
+      return self.SUCCESS
+
+   # List the members of an object key prefix that are not staged under todir yet
+   #    todir - target local directory name
+   #   prefix - source object key prefix, including the trailing '/'
+   #     keys - the member keys to check
+   #    flist - object_glob() info, keyed by member key
+   #  chktime - also require the local copy to be no older than the object
+   def object_local_missing(self, todir, prefix, keys, flist, chktime = 0, logact = 0):
+      """Return the member keys that still need downloading under a key prefix.
+
+      A local copy counts as present when it matches the object size, and with
+      chktime also when it is no older than the object's LastModified.
+
+      Args:
+         todir (str): Destination local directory path.
+         prefix (str): Object key prefix including the trailing '/'.
+         keys (list): Member keys to check.
+         flist (dict): Mapping of key → object info from object_glob().
+         chktime (int): Also compare modification times; default 0.
+         logact (int): Logging action flags; default 0.
+
+      Returns:
+         list: The member keys not staged under todir.
+      """
+      miss = []
       plen = len(prefix)
       for key in keys:
          oinfo = flist[key]
          tofile = "{}/{}".format(todir, key[plen:])
          linfo = self.check_local_file(tofile, 1, logact)
-         # an unparsable LastModified leaves date_modified unset; download again then
-         if linfo and 'date_modified' in oinfo and linfo['data_size'] == oinfo['data_size'] and \
-            self.cmptime(linfo['date_modified'], linfo['time_modified'],
-                         oinfo['date_modified'], oinfo['time_modified']) >= 0: continue
-         if not self.object_get_local(tofile, key, oinfo['data_size'], bucket, logact): return self.FAILURE
-      return self.SUCCESS
+         if linfo and linfo['data_size'] == oinfo['data_size']:
+            if not chktime: continue
+            # an unparsable LastModified leaves date_modified unset; fetch again then
+            if 'date_modified' in oinfo and \
+               self.cmptime(linfo['date_modified'], linfo['time_modified'],
+                            oinfo['date_modified'], oinfo['time_modified']) >= 0: continue
+         miss.append(key)
+      return miss
+
+   # List the files staged under todir that the object key prefix no longer holds
+   #    todir - target local directory name
+   #   prefix - source object key prefix, including the trailing '/'
+   #     keys - the member keys the object store currently holds
+   def local_extra_files(self, todir, prefix, keys):
+      """Return the paths under todir that are not members of an object key prefix.
+
+      Paths are relative to todir, matching the layout a recursive download creates.
+
+      Args:
+         todir (str): Local directory holding the staged copy.
+         prefix (str): Object key prefix including the trailing '/'.
+         keys (list): The member keys currently under the prefix.
+
+      Returns:
+         list: Relative paths under todir with no matching member key.
+      """
+      plen = len(prefix)
+      members = set(key[plen:] for key in keys)
+      extra = []
+      for path, dirs, files in os.walk(todir):
+         for fname in files:
+            name = op.relpath(op.join(path, fname), todir)
+            if name not in members: extra.append(name)
+      return extra
 
    # Copy a remote file to object
    #   tofile - target object file name
@@ -2376,6 +2475,8 @@ class PgFile(PgUtil, PgSIG):
 
       Strips trailing '/'. Uses opt bit 64 to skip the retry. Fetches metadata
       (uhash) when opt includes bits 2, 4, or 8. Resets ECNTS['O'] on success.
+      Only the key itself, or the keys under it as a directory, count as a
+      match; other keys that merely start with the same characters are ignored.
 
       Args:
          file (str): Object key to check.
@@ -2401,7 +2502,11 @@ class PgFile(PgUtil, PgSIG):
          if buf:
             if re.match(r'^\[\]', buf): break
             if re.match(r'^\[\{', buf):
-               ary = json.loads(buf)
+               # 'lo' matches by key prefix, so x.zarr also lists a sibling x.zarr2/...
+               # and a.nc lists a.nc.bak; keep only the key itself and keys under it
+               prefix = file + '/'
+               ary = [a for a in json.loads(buf) if a['Key'] == file or a['Key'].startswith(prefix)]
+               if not ary: break
                cnt = len(ary)
                isdir = (cnt > 1 or ary[0]['Key'] != file)
                # a key prefix has no timestamp of its own; report the newest member's
@@ -2419,6 +2524,9 @@ class PgFile(PgUtil, PgSIG):
                   for a in ary:
                      size += int(a['Size'])
                   ret['data_size'] = size
+                  # keep the member listing, as object_glob(file, bucket, 1) builds it,
+                  # so a download of the prefix does not list it a second time
+                  ret['flist'] = {a['Key']: self.object_file_stat(a, None, 1) for a in ary}
                uhash = None
                break
          if opt&64: return self.FAILURE

{rda_python_common-3.0.24 → rda_python_common-3.0.26/src/rda_python_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.24
+Version: 3.0.26
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.24`).  If the import
+You should see the installed version (currently `3.0.26`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.