rda-python-common 3.0.23__py3-none-any.whl → 3.0.24__py3-none-any.whl

rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "3.0.23"
+__version__ = "3.0.24"
 
 __all__ = [
    "PgLOG",

rda_python_common/pg_file.py

@@ -878,6 +878,11 @@ class PgFile(PgUtil, PgSIG):
       objects sharing a key prefix. Each object is downloaded individually and
       keeps its position relative to the prefix under todir.
 
+      A member already staged under todir at the object's size and no older than
+      the object itself is left alone, so a store whose download was cut short
+      (by a batch walltime, say) resumes where it stopped instead of starting
+      over, and a second copy into the same directory costs nothing.
+
       Args:
          todir (str): Destination local directory path.
          fromdir (str): Object key prefix (source path in the bucket).
@@ -890,7 +895,9 @@ class PgFile(PgUtil, PgSIG):
       if not bucket: bucket = self.PGLOG['OBJCTBKT']
       ms = re.match(r'^(.+)/$', fromdir)
       if ms: fromdir = ms.group(1)
-      flist = self.object_glob(fromdir, bucket, 0, logact)
+      # opt 1 adds the LastModified the 'lo' listing already carries; it is only
+      # bits 2 and 8 that would cost a metadata call per key
+      flist = self.object_glob(fromdir, bucket, 1, logact)
       if flist == self.FAILURE: return self.FAILURE
       prefix = fromdir + '/'
       keys = [key for key in flist if key.startswith(prefix)]
@@ -898,8 +905,14 @@ class PgFile(PgUtil, PgSIG):
          return self.lmsg(fromdir, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], todir), logact)
       plen = len(prefix)
       for key in keys:
+         oinfo = flist[key]
          tofile = "{}/{}".format(todir, key[plen:])
-         if not self.object_get_local(tofile, key, flist[key]['data_size'], bucket, logact): return self.FAILURE
+         linfo = self.check_local_file(tofile, 1, logact)
+         # an unparsable LastModified leaves date_modified unset; download again then
+         if linfo and 'date_modified' in oinfo and linfo['data_size'] == oinfo['data_size'] and \
+            self.cmptime(linfo['date_modified'], linfo['time_modified'],
+                         oinfo['date_modified'], oinfo['time_modified']) >= 0: continue
+         if not self.object_get_local(tofile, key, oinfo['data_size'], bucket, logact): return self.FAILURE
       return self.SUCCESS
 
    # Copy a remote file to object

{rda_python_common-3.0.23.dist-info → rda_python_common-3.0.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.23
+Version: 3.0.24
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.23`).  If the import
+You should see the installed version (currently `3.0.24`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.23.dist-info → rda_python_common-3.0.24.dist-info}/RECORD

@@ -7,10 +7,10 @@ rda_python_common/PgOPT.py,sha256=0nQPF19N-1VWwrjFHEeruVSwVDwvFKd4uowPc8dK7e4,56
 rda_python_common/PgSIG.py,sha256=0MJPH0fTtSshIsvUXkzQjReHPyCWdzHYOY5LbFSh7-A,34318
 rda_python_common/PgSplit.py,sha256=En7xcdkE4YNRcMX88qXnrUgUOnQU3Kqh8uxRIdlJhEA,9156
 rda_python_common/PgUtil.py,sha256=7hXAznAI8tYh-nJRKBIXejNe2ROQ2Wxn28OAndeg0Iw,48433
-rda_python_common/__init__.py,sha256=0ZmsVtLVT0eP_PN_utNR7xqFsfgJaLbwF5Rqezog2es,995
+rda_python_common/__init__.py,sha256=hn0mszEAFqVd11CUNXtZOrPGjG5VzqQRIiwAzaWLOO0,995
 rda_python_common/pg_cmd.py,sha256=MYldUuM8mSUIT34yKQyUd0i4NL4zdSB8C6Jcc28NRxA,33809
 rda_python_common/pg_dbi.py,sha256=h1snZ8fJwHeBQJmNjsjJoDKkfT4FW9avMvdHgcIc-cA,122328
-rda_python_common/pg_file.py,sha256=70Hqy2NiiVA4x5Tl8Emzn2_1bZzPTQ1uI0GTOCQgbbY,169783
+rda_python_common/pg_file.py,sha256=_0JpsDfWYTS-55QkpYv-ecyR-ZgEr8kir1NfVLXrir8,170659
 rda_python_common/pg_lock.py,sha256=31EaVDjCkcx3-n8-KnzG18R8Pz7Z6KyFsEqcml6Iq5c,32702
 rda_python_common/pg_log.py,sha256=erQN4YYPwiORa3OE0Jt8s05rlWsjtyIs0fJ_KLbNAQ8,82959
 rda_python_common/pg_opt.py,sha256=LxVAf47wi2STkVGSXMvV3IZphEJcoO8bvYXy6kQ5ox4,82403
@@ -20,9 +20,9 @@ rda_python_common/pg_split.py,sha256=aAWKUZPmZ-LQ_fJ3DSKzPKxJw0fMDJ2fgP8ZT629M30
 rda_python_common/pg_util.py,sha256=JL9NoW-VoQvsjXsUr09oBUWYJJNltU4GyQIfWlpkf90,86084
 rda_python_common/pgpassword.py,sha256=aTQaO59QokWrv2EKom4TvKhNW2Mr6aFMMBVG4DCbcKI,5023
 rda_python_common/pgpassword.usg,sha256=Wrle6A8sdlMGx3ZEAmE6TNDPOlG84Whe0T3rCNFzPfY,2013
-rda_python_common-3.0.23.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_common-3.0.23.dist-info/METADATA,sha256=XqOpS77KVBESTSX-Lzp-uJ0KSllqKhQVCbyTtm9zPEE,14530
-rda_python_common-3.0.23.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-rda_python_common-3.0.23.dist-info/entry_points.txt,sha256=pZgVNWspcK-F1TbPav7C3C9NdeHDZMm_25fW9weix00,65
-rda_python_common-3.0.23.dist-info/top_level.txt,sha256=KVQmx7D3DD-jsiheqL8HdTrRE14hpRnZY5_ioMArA5k,18
-rda_python_common-3.0.23.dist-info/RECORD,,
+rda_python_common-3.0.24.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_common-3.0.24.dist-info/METADATA,sha256=DyiVDiGacZIGXMzgH3zfUFYZd96yn94GZohxiCPAxYY,14530
+rda_python_common-3.0.24.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+rda_python_common-3.0.24.dist-info/entry_points.txt,sha256=pZgVNWspcK-F1TbPav7C3C9NdeHDZMm_25fW9weix00,65
+rda_python_common-3.0.24.dist-info/top_level.txt,sha256=KVQmx7D3DD-jsiheqL8HdTrRE14hpRnZY5_ioMArA5k,18
+rda_python_common-3.0.24.dist-info/RECORD,,