rda-python-miscs 3.0.9__py3-none-any.whl → 3.0.10__py3-none-any.whl

rda_python_miscs/gdexcp.py

@@ -254,12 +254,47 @@ class GdexCp(PgFile):
          self.CINFO['fpath'] = (file if dosub else op.dirname(file)) + "/"
          if info['isfile']:
             self.CINFO['tcnt'] += self.dispatch_copy(file, info)
+         elif self.RDACP['r'] and self.can_upload_directory():
+            self.CINFO['tcnt'] += self.upload_directory(file, dosub)
          elif dosub or self.RDACP['R']:
             flist = self.gdex_glob(file, self.RDACP['fh'], 0, self.LGWNEX)
             if flist: self.copy_list(flist, 1, file)
          else:
             self.pglog("{}{}: Add option -r to copy directory".format(self.CINFO['fhost'], file), self.LGEREX)
    
+   # check if a local directory can go to the Object Store in one recursive upload
+   def can_upload_directory(self):
+      """Return True when a -r directory copy can be one recursive Object Store upload.
+
+      Needs a local source, an Object Store target and a key prefix from -t; with
+      the default target '.' the keys would start with '/', so such a copy goes
+      file by file instead.
+      """
+      if self.RDACP['fh'] or not self.RDACP['th']: return False
+      if self.pgcmp(self.strip_host_name(self.RDACP['th']), self.OHOST, 1): return False
+      return (self.CINFO['tpath'] != '.')
+
+   # upload a local directory to the Object Store in one recursive call
+   def upload_directory(self, fromdir, dosub):
+      """Upload a whole local directory to the Object Store with one isd_s3_cli call.
+
+      The files keep their layout under the -t key prefix: with the trailing '/'
+      form (dosub) they land at '{prefix}/{relative path}', otherwise the directory
+      name is kept too, at '{prefix}/{dirname}/{relative path}', the same keys a
+      file-by-file copy gives.  -m is not used; isd_s3_cli uploads in parallel
+      threads of its own.
+
+      Args:
+         fromdir (str): Absolute source directory path.
+         dosub (int): 1 to upload the directory contents without the directory name.
+
+      Returns:
+         int: 1 if the directory was uploaded, 0 otherwise.
+      """
+      todir = self.CINFO['tpath'] if dosub else self.join_paths(self.CINFO['tpath'], op.basename(fromdir))
+      logact = self.LGWNEX | (self.OVRIDE if self.RDACP['O'] else 0)
+      return (1 if self.local_copy_object_directory(todir, fromdir, None, None, logact) else 0)
+
    # recursively copy directory/file
    def copy_list(self, tlist, level, cdir):
       """Recursively copy a directory listing up to the configured depth limit.

rda_python_miscs/gdexcp.usg

@@ -57,6 +57,12 @@
  COPY BEHAVIOR
 
    -r     Copy directories and files recursively, with no depth limit.
+          A local directory copied to an Object Store bucket (-tb) with -r and a
+          -t key prefix is uploaded in a single isd_s3_cli call, which uploads
+          in parallel threads of its own (-m is not used for it).  Its files
+          keep their layout under the prefix.  The directory is skipped when
+          all its files are already in the bucket at the same size; otherwise
+          the whole directory is uploaded again.
 
    -R  RecursiveLevel
           Copy recursively up to the given depth.  -R 1 copies only the

{rda_python_miscs-3.0.9.dist-info → rda_python_miscs-3.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.9
+Version: 3.0.10
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
@@ -11,7 +11,7 @@ Classifier: Development Status :: 5 - Production/Stable
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: rda_python_common
+Requires-Dist: rda_python_common>=3.0.28
 Requires-Dist: rda_python_setuid
 Dynamic: license-file
 

{rda_python_miscs-3.0.9.dist-info → rda_python_miscs-3.0.10.dist-info}/RECORD

@@ -7,8 +7,8 @@ rda_python_miscs/decsdata_restore.usg,sha256=jOD_AbNV4C-dOp9mTkV2MVcUgcf2kf5Q9Qq
 rda_python_miscs/decsdata_storage.py,sha256=5n8bOVIpGxnX7beDNGmrgj4z1OwmU4-9qVsDCbSuymY,8637
 rda_python_miscs/decsdata_storage.usg,sha256=5_OXsIxyKjA46xjycaTgVVIE0c39lkkSekcVliFNgaI,2812
 rda_python_miscs/gdex_ls.py,sha256=fg9jfYajOT8ps6eEhFUUqmQ791BdGwBkrT_pyjmGGvQ,8860
-rda_python_miscs/gdexcp.py,sha256=iVVIwfi9wDzfUmrq70SylNXY_Aj_1D8zAJHBq2XpQng,19761
-rda_python_miscs/gdexcp.usg,sha256=c87lceTBS2AeLZqSSB2BhpdprjWnVwgnQ56OnZWUSxo,6516
+rda_python_miscs/gdexcp.py,sha256=r5-BuOCRQCEP-c-JVP2JSOcLh1Yk7qn6a3Bn3WkR0w8,21566
+rda_python_miscs/gdexcp.usg,sha256=lT7ZXES8pzWVCkh2c9XJDmMYEpLBWQTWbDKw3QfURbE,6957
 rda_python_miscs/gdexdrop.py,sha256=xlD218htNrgHpgG3pipZ5xHUPKEaYfmn-U4tuGkCOgY,16474
 rda_python_miscs/gdexdrop.usg,sha256=3S0TT-Vdb9YNwTb5JcNQ4XjIIfTKrUqlokFM7vTEUQI,3883
 rda_python_miscs/gdexdrop_standalone.py,sha256=w5qOhlRqeAb_G5zfE1ORKguvgla-zbEr-cSg-3rg_gw,1047
@@ -48,9 +48,9 @@ rda_python_miscs/tcshqsub.usg,sha256=JYfhrK7cqme-Sij_JfquONOs3HMu-d5dDGI9K_RdudU
 rda_python_miscs/rst_templates/appendix.rst.temp,sha256=HCujxbj-R4_8FgOrLSlMZcBQHUu8f6kfBMWVRygl7jc,560
 rda_python_miscs/rst_templates/index.rst.temp,sha256=YSa1JM6X9x2SC6UiqJu_9xRrVYGLKwNI43DBJEGDX08,523
 rda_python_miscs/rst_templates/section.rst.temp,sha256=-CUtutvctG2tIdkqrkFxVEzmNN3atRJXBDQaTnMJ6Gw,572
-rda_python_miscs-3.0.9.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_miscs-3.0.9.dist-info/METADATA,sha256=ZQBjKyzYKS0bDAw6BvuEOtx1-KT8U7dliIpEvrRDXJ4,9157
-rda_python_miscs-3.0.9.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-rda_python_miscs-3.0.9.dist-info/entry_points.txt,sha256=cjZOhfF2ZcTvSMJqS-vj6jU3PhfM0Q8HEV_p5wdAFRQ,985
-rda_python_miscs-3.0.9.dist-info/top_level.txt,sha256=W5rz7DrWb7hXABUbGgWcwe6D644X338LR8_zdgmtLhg,17
-rda_python_miscs-3.0.9.dist-info/RECORD,,
+rda_python_miscs-3.0.10.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_miscs-3.0.10.dist-info/METADATA,sha256=llSbWhobyloEH7EOjVl-XUsXVllIO1zvyQdRxy1j1rQ,9166
+rda_python_miscs-3.0.10.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+rda_python_miscs-3.0.10.dist-info/entry_points.txt,sha256=cjZOhfF2ZcTvSMJqS-vj6jU3PhfM0Q8HEV_p5wdAFRQ,985
+rda_python_miscs-3.0.10.dist-info/top_level.txt,sha256=W5rz7DrWb7hXABUbGgWcwe6D644X338LR8_zdgmtLhg,17
+rda_python_miscs-3.0.10.dist-info/RECORD,,