rda-python-miscs 3.0.7__tar.gz → 3.0.9__tar.gz

{rda_python_miscs-3.0.7/src/rda_python_miscs.egg-info → rda_python_miscs-3.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.7
+Version: 3.0.9
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
@@ -172,6 +172,12 @@ deliberately narrow alternative to `gdexcp`: the destination is always
 directory is rejected, and every source must be readable by the calling user
 rather than only by `gdexdata`.
 
+An absolute `-t` under `/tmp/` names any sub-path of `/tmp` instead of a sub-path
+of the dataset directory, so that a drop can be tried out without touching the
+dataset tree.  The dataset of `-ds` is still checked against the access list, and
+because `/tmp` is world writable — unlike a dataset directory — the part of the
+path that already exists must belong to the calling user or to `gdexdata`.
+
 Everything dropped, including sub-directories created for `-t`, is also set to the
 GDEX group `PGLOG['GDEXGRP']`.  The setuid wrapper switches only the user to
 `gdexdata`, not the group, so without that step a dropped file would stay in the

{rda_python_miscs-3.0.7 → rda_python_miscs-3.0.9}/README.md

@@ -155,6 +155,12 @@ deliberately narrow alternative to `gdexcp`: the destination is always
 directory is rejected, and every source must be readable by the calling user
 rather than only by `gdexdata`.
 
+An absolute `-t` under `/tmp/` names any sub-path of `/tmp` instead of a sub-path
+of the dataset directory, so that a drop can be tried out without touching the
+dataset tree.  The dataset of `-ds` is still checked against the access list, and
+because `/tmp` is world writable — unlike a dataset directory — the part of the
+path that already exists must belong to the calling user or to `gdexdata`.
+
 Everything dropped, including sub-directories created for `-t`, is also set to the
 GDEX group `PGLOG['GDEXGRP']`.  The setuid wrapper switches only the user to
 `gdexdata`, not the group, so without that step a dropped file would stay in the

{rda_python_miscs-3.0.7 → rda_python_miscs-3.0.9}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_miscs"
-version = "3.0.7"
+version = "3.0.9"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_miscs-3.0.7 → rda_python_miscs-3.0.9}/src/rda_python_miscs/gdexcp.py

@@ -135,6 +135,28 @@ class GdexCp(PgFile):
       self.dssdb_dbname()
       self.validate_decs_group('gdexcp', self.PGLOG['CURUID'], 1)
       if not self.RDACP['R'] and self.RDACP['r']: self.RDACP['R'] = 1000
+      # a bucket implies the Object Store host, so -tb/-fb need no -th/-fh
+      if self.RDACP['fb']:
+         self.PGLOG['OBJCTBKT'] = self.RDACP['fb']
+         if not self.RDACP['fh']:
+            self.RDACP['fh'] = self.OHOST
+            self.CINFO['fhost'] = self.OHOST + '-'
+      if self.RDACP['tb']:
+         if not self.RDACP['fb']: self.PGLOG['OBJCTBKT'] = self.RDACP['tb']
+         if not self.RDACP['th']:
+            self.RDACP['th'] = self.OHOST
+            self.CINFO['thost'] = self.OHOST + '-'
+      # a Globus endpoint implies its host, so -tp/-fp need no -th/-fh
+      if self.RDACP['fp']:
+         self.PGLOG['BACKUPEP'] = self.RDACP['fp']
+         if not self.RDACP['fh']:
+            self.RDACP['fh'] = self.endpoint_host(self.RDACP['fp'])
+            if self.RDACP['fh']: self.CINFO['fhost'] = self.RDACP['fh'] + '-'
+      if self.RDACP['tp']:
+         if not self.RDACP['fp']: self.PGLOG['BACKUPEP'] = self.RDACP['tp']
+         if not self.RDACP['th']:
+            self.RDACP['th'] = self.endpoint_host(self.RDACP['tp'])
+            if self.RDACP['th']: self.CINFO['thost'] = self.RDACP['th'] + '-'
       if not self.RDACP['t']:
          self.CINFO['tpath'] = self.RDACP['t'] = "."
       else:
@@ -151,14 +173,6 @@ class GdexCp(PgFile):
          self.pglog("{}{}: Cannot copy multiple files to a single file".format(self.CINFO['thost'], self.RDACP['t']), self.LGEREX)
       if self.RDACP['th'] and self.RDACP['fh'] and self.RDACP['th'] == self.RDACP['fh'] and self.RDACP['fh'] != 'HPSS':
          self.pglog(self.RDACP['fh'] + ": Cannot copy file onto the same host", self.LGEREX)
-      if self.RDACP['fb']:
-         self.PGLOG['OBJCTBKT'] = self.RDACP['fb']
-      elif self.RDACP['tb']:
-         self.PGLOG['OBJCTBKT'] = self.RDACP['tb']
-      if self.RDACP['fp']:
-         self.PGLOG['BACKUPEP'] = self.RDACP['fp']
-      elif self.RDACP['tp']:
-         self.PGLOG['BACKUPEP'] = self.RDACP['tp']
       if self.RDACP['o']:
          if not self.RDACP['fp']:
             self.pglog("-o: works only when source Globus endpoint -fp is provided", self.LGEREX)
@@ -185,6 +199,27 @@ class GdexCp(PgFile):
          self.pglog("{}: No File copied{}".format((self.CINFO['fpath'] if self.CINFO['fpath'] else self.CINFO['curdir']), hinfo), self.LOGWRN)
       self.cmdlog()
    
+   # get the host name a Globus endpoint is reached by
+   def endpoint_host(self, endpoint):
+      """Return the host name that copy_gdex_file() routes a Globus endpoint through.
+
+      Known endpoints map through QHOSTS: 'gdex-quasar' to the Quasar backup host,
+      'gdex-quasar-drdata' to the DRDATA host, and 'gdex-glade' to the local host
+      (returned as None, the default).  Any other endpoint is routed as the Quasar
+      backup host with QPOINTS['B'] pointed at it.
+
+      Args:
+         endpoint (str): Globus endpoint name from -fp or -tp.
+
+      Returns:
+         str | None: Host name for -fh/-th, or None for the local host.
+      """
+      host = self.QHOSTS.get(endpoint)
+      if host is None:
+         self.QPOINTS['B'] = endpoint
+         host = self.BHOST
+      return (None if host == self.LHOST else host)
+
    # copy the top level list
    def copy_top_list(self, files):
       """Iterate the top-level source paths and initiate copies or recursive traversal.

{rda_python_miscs-3.0.7 → rda_python_miscs-3.0.9}/src/rda_python_miscs/gdexcp.usg

@@ -44,10 +44,14 @@
 
    -fh FromHostName     host name where the source files reside.
    -th ToHostName       host name where the target files are written.
-   -fb FromBucket       Object Store bucket name for the source files.
-   -tb ToBucket         Object Store bucket name for the target files.
-   -fp FromGlobusEndpoint   Globus endpoint for the source files.
-   -tp ToGlobusEndpoint     Globus endpoint for the target files.
+   -fb FromBucket       Object Store bucket name for the source files; implies
+                        the Object Store as the source host (no -fh needed).
+   -tb ToBucket         Object Store bucket name for the target files; implies
+                        the Object Store as the target host (no -th needed).
+   -fp FromGlobusEndpoint   Globus endpoint for the source files; implies the
+                            endpoint's host as the source (no -fh needed).
+   -tp ToGlobusEndpoint     Globus endpoint for the target files; implies the
+                            endpoint's host as the target (no -th needed).
 
 
  COPY BEHAVIOR

{rda_python_miscs-3.0.7 → rda_python_miscs-3.0.9}/src/rda_python_miscs/gdexdrop.py

@@ -24,6 +24,7 @@ DROPROOT = "/glade/campaign/collections/gdex/data"      # dataset data root path
 DROPCONF = "/glade/u/home/gdexdata/config/gdexdrop.conf"   # access list
 DROPLOG = "/glade/u/home/gdexdata/dssdb/log"            # log path
 DROPUSER = "gdexdata"                                   # owner of dropped files
+DROPTMP = "/tmp"                                        # alternate root, for testing
 
 class GdexDrop(PgFile):
    """Copy local files and directories into a GDEX dataset directory as 'gdexdata'.
@@ -187,18 +188,47 @@ class GdexDrop(PgFile):
       real path of the dataset directory, so that neither a '..' component in -t
       nor an existing symbolic link can move the drop outside the dataset.
 
+      An absolute -t under DROPTMP names any sub-path of that directory instead,
+      for trying a drop out without touching the dataset tree.  The dataset of
+      -ds is still validated against the access list.
+
       Returns:
          str: Resolved absolute target directory.
       """
       dsid = self.DROP['ds']
       if not re.match(r'^[a-z]\d{6}$', dsid):
          self.pglog(dsid + ": Invalid Dataset ID of -ds, expecting the form d123456", self.LGEREX)
+      if self.DROP['t'] and re.match(r'^{}/'.format(DROPTMP), self.DROP['t']):
+         return self.tmp_target()
       dsroot = op.realpath(op.join(DROPROOT, dsid))
       if not op.isdir(dsroot):
          self.pglog("{}: Dataset directory NOT exists under {}".format(dsid, DROPROOT), self.LGEREX)
       target = self.confine_path(op.join(dsroot, self.DROP['t']) if self.DROP['t'] else dsroot, dsroot)
       return target
 
+   # resolve a target of -t that names a sub-path of DROPTMP
+   def tmp_target(self):
+      """Return the resolved -t path under DROPTMP, exiting if it may not be used.
+
+      DROPTMP is world writable, unlike a dataset directory, so a path there can
+      be replaced by another user between this check and the copy, and may well
+      belong to somebody else already.  The existing part of the path is therefore
+      required to belong to the caller or to DROPUSER, so that a drop cannot be
+      aimed at a directory a third user controls.
+
+      Returns:
+         str: Resolved absolute target directory under DROPTMP.
+      """
+      tmproot = op.realpath(DROPTMP)
+      target = self.confine_path(self.DROP['t'], tmproot)
+      dir = target
+      while not op.isdir(dir): dir = op.dirname(dir)
+      if dir == tmproot: return target   # nothing of the path exists yet
+      info = self.check_local_file(dir, 2, self.LOGWRN)
+      if info and info['logname'] not in (self.PGLOG['CURUID'], DROPUSER):
+         self.pglog("{}: Target path under {} is owned by {}".format(dir, DROPTMP, info['logname']), self.LGEREX)
+      return target
+
    # make sure a path stays inside the dataset directory
    def confine_path(self, path, dsroot):
       """Return the resolved path, exiting if it falls outside the dataset directory.

{rda_python_miscs-3.0.7 → rda_python_miscs-3.0.9}/src/rda_python_miscs/gdexdrop.usg

@@ -17,7 +17,12 @@
           files into.  Missing sub-directories are created.  The path must stay
           inside the dataset directory; '..' components and symbolic links
           leading out of it are rejected.  Defaults to the dataset directory
-          itself;
+          itself.
+          An absolute path under /tmp/ names any sub-path of /tmp instead, for
+          trying a drop out without touching the dataset tree.  The dataset of
+          -ds is still checked against the access list, and because /tmp is
+          world writable, the existing part of the path must belong to the
+          calling user or to 'gdexdata';
 
       - Option -f File/Directory List is the default option, so the source paths
           may be given without it and in any position.  Relative paths are
@@ -77,3 +82,7 @@
     4. Drop a long list of files named in a file, with group-writable modes:
 
           gdexdrop -ds d123456 -i filelist.txt -F 664 -D 775
+
+    5. Try a drop out under /tmp, leaving the dataset directory alone:
+
+          gdexdrop -ds d123456 -t /tmp/mydrop hourly.2026*.nc

{rda_python_miscs-3.0.7 → rda_python_miscs-3.0.9/src/rda_python_miscs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.7
+Version: 3.0.9
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
@@ -172,6 +172,12 @@ deliberately narrow alternative to `gdexcp`: the destination is always
 directory is rejected, and every source must be readable by the calling user
 rather than only by `gdexdata`.
 
+An absolute `-t` under `/tmp/` names any sub-path of `/tmp` instead of a sub-path
+of the dataset directory, so that a drop can be tried out without touching the
+dataset tree.  The dataset of `-ds` is still checked against the access list, and
+because `/tmp` is world writable — unlike a dataset directory — the part of the
+path that already exists must belong to the calling user or to `gdexdata`.
+
 Everything dropped, including sub-directories created for `-t`, is also set to the
 GDEX group `PGLOG['GDEXGRP']`.  The setuid wrapper switches only the user to
 `gdexdata`, not the group, so without that step a dropped file would stay in the