rda-python-miscs 3.0.8__tar.gz → 3.0.10__tar.gz

{rda_python_miscs-3.0.8/src/rda_python_miscs.egg-info → rda_python_miscs-3.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.8
+Version: 3.0.10
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
@@ -11,7 +11,7 @@ Classifier: Development Status :: 5 - Production/Stable
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: rda_python_common
+Requires-Dist: rda_python_common>=3.0.28
 Requires-Dist: rda_python_setuid
 Dynamic: license-file
 

{rda_python_miscs-3.0.8 → rda_python_miscs-3.0.10}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_miscs"
-version = "3.0.8"
+version = "3.0.10"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]
@@ -20,7 +20,7 @@ classifiers = [
     "Development Status :: 5 - Production/Stable",
 ]
 dependencies = [
-  "rda_python_common",
+  "rda_python_common>=3.0.28",
   "rda_python_setuid",
 ]
 

{rda_python_miscs-3.0.8 → rda_python_miscs-3.0.10}/src/rda_python_miscs/gdexcp.py

@@ -135,6 +135,28 @@ class GdexCp(PgFile):
       self.dssdb_dbname()
       self.validate_decs_group('gdexcp', self.PGLOG['CURUID'], 1)
       if not self.RDACP['R'] and self.RDACP['r']: self.RDACP['R'] = 1000
+      # a bucket implies the Object Store host, so -tb/-fb need no -th/-fh
+      if self.RDACP['fb']:
+         self.PGLOG['OBJCTBKT'] = self.RDACP['fb']
+         if not self.RDACP['fh']:
+            self.RDACP['fh'] = self.OHOST
+            self.CINFO['fhost'] = self.OHOST + '-'
+      if self.RDACP['tb']:
+         if not self.RDACP['fb']: self.PGLOG['OBJCTBKT'] = self.RDACP['tb']
+         if not self.RDACP['th']:
+            self.RDACP['th'] = self.OHOST
+            self.CINFO['thost'] = self.OHOST + '-'
+      # a Globus endpoint implies its host, so -tp/-fp need no -th/-fh
+      if self.RDACP['fp']:
+         self.PGLOG['BACKUPEP'] = self.RDACP['fp']
+         if not self.RDACP['fh']:
+            self.RDACP['fh'] = self.endpoint_host(self.RDACP['fp'])
+            if self.RDACP['fh']: self.CINFO['fhost'] = self.RDACP['fh'] + '-'
+      if self.RDACP['tp']:
+         if not self.RDACP['fp']: self.PGLOG['BACKUPEP'] = self.RDACP['tp']
+         if not self.RDACP['th']:
+            self.RDACP['th'] = self.endpoint_host(self.RDACP['tp'])
+            if self.RDACP['th']: self.CINFO['thost'] = self.RDACP['th'] + '-'
       if not self.RDACP['t']:
          self.CINFO['tpath'] = self.RDACP['t'] = "."
       else:
@@ -151,14 +173,6 @@ class GdexCp(PgFile):
          self.pglog("{}{}: Cannot copy multiple files to a single file".format(self.CINFO['thost'], self.RDACP['t']), self.LGEREX)
       if self.RDACP['th'] and self.RDACP['fh'] and self.RDACP['th'] == self.RDACP['fh'] and self.RDACP['fh'] != 'HPSS':
          self.pglog(self.RDACP['fh'] + ": Cannot copy file onto the same host", self.LGEREX)
-      if self.RDACP['fb']:
-         self.PGLOG['OBJCTBKT'] = self.RDACP['fb']
-      elif self.RDACP['tb']:
-         self.PGLOG['OBJCTBKT'] = self.RDACP['tb']
-      if self.RDACP['fp']:
-         self.PGLOG['BACKUPEP'] = self.RDACP['fp']
-      elif self.RDACP['tp']:
-         self.PGLOG['BACKUPEP'] = self.RDACP['tp']
       if self.RDACP['o']:
          if not self.RDACP['fp']:
             self.pglog("-o: works only when source Globus endpoint -fp is provided", self.LGEREX)
@@ -185,6 +199,27 @@ class GdexCp(PgFile):
          self.pglog("{}: No File copied{}".format((self.CINFO['fpath'] if self.CINFO['fpath'] else self.CINFO['curdir']), hinfo), self.LOGWRN)
       self.cmdlog()
    
+   # get the host name a Globus endpoint is reached by
+   def endpoint_host(self, endpoint):
+      """Return the host name that copy_gdex_file() routes a Globus endpoint through.
+
+      Known endpoints map through QHOSTS: 'gdex-quasar' to the Quasar backup host,
+      'gdex-quasar-drdata' to the DRDATA host, and 'gdex-glade' to the local host
+      (returned as None, the default).  Any other endpoint is routed as the Quasar
+      backup host with QPOINTS['B'] pointed at it.
+
+      Args:
+         endpoint (str): Globus endpoint name from -fp or -tp.
+
+      Returns:
+         str | None: Host name for -fh/-th, or None for the local host.
+      """
+      host = self.QHOSTS.get(endpoint)
+      if host is None:
+         self.QPOINTS['B'] = endpoint
+         host = self.BHOST
+      return (None if host == self.LHOST else host)
+
    # copy the top level list
    def copy_top_list(self, files):
       """Iterate the top-level source paths and initiate copies or recursive traversal.
@@ -219,12 +254,47 @@ class GdexCp(PgFile):
          self.CINFO['fpath'] = (file if dosub else op.dirname(file)) + "/"
          if info['isfile']:
             self.CINFO['tcnt'] += self.dispatch_copy(file, info)
+         elif self.RDACP['r'] and self.can_upload_directory():
+            self.CINFO['tcnt'] += self.upload_directory(file, dosub)
          elif dosub or self.RDACP['R']:
             flist = self.gdex_glob(file, self.RDACP['fh'], 0, self.LGWNEX)
             if flist: self.copy_list(flist, 1, file)
          else:
             self.pglog("{}{}: Add option -r to copy directory".format(self.CINFO['fhost'], file), self.LGEREX)
    
+   # check if a local directory can go to the Object Store in one recursive upload
+   def can_upload_directory(self):
+      """Return True when a -r directory copy can be one recursive Object Store upload.
+
+      Needs a local source, an Object Store target and a key prefix from -t; with
+      the default target '.' the keys would start with '/', so such a copy goes
+      file by file instead.
+      """
+      if self.RDACP['fh'] or not self.RDACP['th']: return False
+      if self.pgcmp(self.strip_host_name(self.RDACP['th']), self.OHOST, 1): return False
+      return (self.CINFO['tpath'] != '.')
+
+   # upload a local directory to the Object Store in one recursive call
+   def upload_directory(self, fromdir, dosub):
+      """Upload a whole local directory to the Object Store with one isd_s3_cli call.
+
+      The files keep their layout under the -t key prefix: with the trailing '/'
+      form (dosub) they land at '{prefix}/{relative path}', otherwise the directory
+      name is kept too, at '{prefix}/{dirname}/{relative path}', the same keys a
+      file-by-file copy gives.  -m is not used; isd_s3_cli uploads in parallel
+      threads of its own.
+
+      Args:
+         fromdir (str): Absolute source directory path.
+         dosub (int): 1 to upload the directory contents without the directory name.
+
+      Returns:
+         int: 1 if the directory was uploaded, 0 otherwise.
+      """
+      todir = self.CINFO['tpath'] if dosub else self.join_paths(self.CINFO['tpath'], op.basename(fromdir))
+      logact = self.LGWNEX | (self.OVRIDE if self.RDACP['O'] else 0)
+      return (1 if self.local_copy_object_directory(todir, fromdir, None, None, logact) else 0)
+
    # recursively copy directory/file
    def copy_list(self, tlist, level, cdir):
       """Recursively copy a directory listing up to the configured depth limit.

{rda_python_miscs-3.0.8 → rda_python_miscs-3.0.10}/src/rda_python_miscs/gdexcp.usg

@@ -44,15 +44,25 @@
 
    -fh FromHostName     host name where the source files reside.
    -th ToHostName       host name where the target files are written.
-   -fb FromBucket       Object Store bucket name for the source files.
-   -tb ToBucket         Object Store bucket name for the target files.
-   -fp FromGlobusEndpoint   Globus endpoint for the source files.
-   -tp ToGlobusEndpoint     Globus endpoint for the target files.
+   -fb FromBucket       Object Store bucket name for the source files; implies
+                        the Object Store as the source host (no -fh needed).
+   -tb ToBucket         Object Store bucket name for the target files; implies
+                        the Object Store as the target host (no -th needed).
+   -fp FromGlobusEndpoint   Globus endpoint for the source files; implies the
+                            endpoint's host as the source (no -fh needed).
+   -tp ToGlobusEndpoint     Globus endpoint for the target files; implies the
+                            endpoint's host as the target (no -th needed).
 
 
  COPY BEHAVIOR
 
    -r     Copy directories and files recursively, with no depth limit.
+          A local directory copied to an Object Store bucket (-tb) with -r and a
+          -t key prefix is uploaded in a single isd_s3_cli call, which uploads
+          in parallel threads of its own (-m is not used for it).  Its files
+          keep their layout under the prefix.  The directory is skipped when
+          all its files are already in the bucket at the same size; otherwise
+          the whole directory is uploaded again.
 
    -R  RecursiveLevel
           Copy recursively up to the given depth.  -R 1 copies only the

{rda_python_miscs-3.0.8 → rda_python_miscs-3.0.10/src/rda_python_miscs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.8
+Version: 3.0.10
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
@@ -11,7 +11,7 @@ Classifier: Development Status :: 5 - Production/Stable
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: rda_python_common
+Requires-Dist: rda_python_common>=3.0.28
 Requires-Dist: rda_python_setuid
 Dynamic: license-file
 

rda_python_miscs-3.0.10/src/rda_python_miscs.egg-info/requires.txt

@@ -0,0 +1,2 @@
+rda_python_common>=3.0.28
+rda_python_setuid

rda_python_miscs-3.0.8/src/rda_python_miscs.egg-info/requires.txt

@@ -1,2 +0,0 @@
-rda_python_common
-rda_python_setuid