rda-python-miscs 3.0.6__tar.gz → 3.0.8__tar.gz

{rda_python_miscs-3.0.6/src/rda_python_miscs.egg-info → rda_python_miscs-3.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.6
+Version: 3.0.8
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
@@ -172,6 +172,17 @@ deliberately narrow alternative to `gdexcp`: the destination is always
 directory is rejected, and every source must be readable by the calling user
 rather than only by `gdexdata`.
 
+An absolute `-t` under `/tmp/` names any sub-path of `/tmp` instead of a sub-path
+of the dataset directory, so that a drop can be tried out without touching the
+dataset tree.  The dataset of `-ds` is still checked against the access list, and
+because `/tmp` is world writable — unlike a dataset directory — the part of the
+path that already exists must belong to the calling user or to `gdexdata`.
+
+Everything dropped, including sub-directories created for `-t`, is also set to the
+GDEX group `PGLOG['GDEXGRP']`.  The setuid wrapper switches only the user to
+`gdexdata`, not the group, so without that step a dropped file would stay in the
+calling user's group unless the directory it landed in carried the setgid bit.
+
 Who may drop into which dataset is read from the access list
 `/glade/u/home/gdexdata/config/gdexdrop.conf`.  It must be owned by `gdexdata`
 and must not be writable by group or others, otherwise `gdexdrop` refuses to

{rda_python_miscs-3.0.6 → rda_python_miscs-3.0.8}/README.md

@@ -155,6 +155,17 @@ deliberately narrow alternative to `gdexcp`: the destination is always
 directory is rejected, and every source must be readable by the calling user
 rather than only by `gdexdata`.
 
+An absolute `-t` under `/tmp/` names any sub-path of `/tmp` instead of a sub-path
+of the dataset directory, so that a drop can be tried out without touching the
+dataset tree.  The dataset of `-ds` is still checked against the access list, and
+because `/tmp` is world writable — unlike a dataset directory — the part of the
+path that already exists must belong to the calling user or to `gdexdata`.
+
+Everything dropped, including sub-directories created for `-t`, is also set to the
+GDEX group `PGLOG['GDEXGRP']`.  The setuid wrapper switches only the user to
+`gdexdata`, not the group, so without that step a dropped file would stay in the
+calling user's group unless the directory it landed in carried the setgid bit.
+
 Who may drop into which dataset is read from the access list
 `/glade/u/home/gdexdata/config/gdexdrop.conf`.  It must be owned by `gdexdata`
 and must not be writable by group or others, otherwise `gdexdrop` refuses to

{rda_python_miscs-3.0.6 → rda_python_miscs-3.0.8}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_miscs"
-version = "3.0.6"
+version = "3.0.8"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_miscs-3.0.6 → rda_python_miscs-3.0.8}/src/rda_python_miscs/gdexdrop.py

@@ -24,6 +24,7 @@ DROPROOT = "/glade/campaign/collections/gdex/data"      # dataset data root path
 DROPCONF = "/glade/u/home/gdexdata/config/gdexdrop.conf"   # access list
 DROPLOG = "/glade/u/home/gdexdata/dssdb/log"            # log path
 DROPUSER = "gdexdata"                                   # owner of dropped files
+DROPTMP = "/tmp"                                        # alternate root, for testing
 
 class GdexDrop(PgFile):
    """Copy local files and directories into a GDEX dataset directory as 'gdexdata'.
@@ -32,7 +33,8 @@ class GdexDrop(PgFile):
    by -t, and is verified to stay inside that dataset directory.  The caller must
    be granted the dataset in the gdexdrop access list, and must be able to read
    every source path as themselves; both checks exist because the copy itself
-   runs with the privileges of 'gdexdata'.
+   runs with the privileges of 'gdexdata'.  Everything dropped is then set to the
+   GDEX group PGLOG['GDEXGRP'], which the copy does not land in on its own.
    """
 
    def __init__(self):
@@ -164,8 +166,11 @@ class GdexDrop(PgFile):
       """
       self.DINFO['target'] = self.resolve_target()
       self.validate_caller()
+      if not self.PGLOG['GDEXGID']:
+         self.pglog(self.PGLOG['GDEXGRP'] + ": Unknown Group to own the dropped files", self.LGEREX)
       self.PGLOG['FILEMODE'] = self.DROP['F']
       self.PGLOG['EXECMODE'] = self.DROP['D']
+      self.make_target_directory()
       for file in self.DROP['f']:
          self.DINFO['tcnt'] += self.drop_one(file)
       if self.DINFO['tcnt'] > 0:
@@ -183,18 +188,47 @@ class GdexDrop(PgFile):
       real path of the dataset directory, so that neither a '..' component in -t
       nor an existing symbolic link can move the drop outside the dataset.
 
+      An absolute -t under DROPTMP names any sub-path of that directory instead,
+      for trying a drop out without touching the dataset tree.  The dataset of
+      -ds is still validated against the access list.
+
       Returns:
          str: Resolved absolute target directory.
       """
       dsid = self.DROP['ds']
       if not re.match(r'^[a-z]\d{6}$', dsid):
          self.pglog(dsid + ": Invalid Dataset ID of -ds, expecting the form d123456", self.LGEREX)
+      if self.DROP['t'] and re.match(r'^{}/'.format(DROPTMP), self.DROP['t']):
+         return self.tmp_target()
       dsroot = op.realpath(op.join(DROPROOT, dsid))
       if not op.isdir(dsroot):
          self.pglog("{}: Dataset directory NOT exists under {}".format(dsid, DROPROOT), self.LGEREX)
       target = self.confine_path(op.join(dsroot, self.DROP['t']) if self.DROP['t'] else dsroot, dsroot)
       return target
 
+   # resolve a target of -t that names a sub-path of DROPTMP
+   def tmp_target(self):
+      """Return the resolved -t path under DROPTMP, exiting if it may not be used.
+
+      DROPTMP is world writable, unlike a dataset directory, so a path there can
+      be replaced by another user between this check and the copy, and may well
+      belong to somebody else already.  The existing part of the path is therefore
+      required to belong to the caller or to DROPUSER, so that a drop cannot be
+      aimed at a directory a third user controls.
+
+      Returns:
+         str: Resolved absolute target directory under DROPTMP.
+      """
+      tmproot = op.realpath(DROPTMP)
+      target = self.confine_path(self.DROP['t'], tmproot)
+      dir = target
+      while not op.isdir(dir): dir = op.dirname(dir)
+      if dir == tmproot: return target   # nothing of the path exists yet
+      info = self.check_local_file(dir, 2, self.LOGWRN)
+      if info and info['logname'] not in (self.PGLOG['CURUID'], DROPUSER):
+         self.pglog("{}: Target path under {} is owned by {}".format(dir, DROPTMP, info['logname']), self.LGEREX)
+      return target
+
    # make sure a path stays inside the dataset directory
    def confine_path(self, path, dsroot):
       """Return the resolved path, exiting if it falls outside the dataset directory.
@@ -245,6 +279,44 @@ class GdexDrop(PgFile):
       if not ('all' in dsids or self.DROP['ds'] in dsids):
          self.pglog("{}: NOT granted Dataset {} in {}".format(logname, self.DROP['ds'], DROPCONF), self.LGEREX)
 
+   # create the target directory, and set the group of each directory created
+   def make_target_directory(self):
+      """Create the sub-directories named by -t that do not exist yet.
+
+      They are created here rather than on the fly by local_copy_local() so that
+      set_drop_group() can be called on each one; a directory left in the calling
+      user's group could not be written into by the rest of the DECS group later.
+      """
+      newdirs = []
+      dir = self.DINFO['target']
+      while not op.isdir(dir):
+         newdirs.insert(0, dir)
+         dir = op.dirname(dir)
+      if not newdirs: return
+      self.make_local_directory(self.DINFO['target'], self.LGWNEX)
+      for dir in newdirs: self.set_drop_group(dir)
+
+   # set the group of a dropped path, and of everything under it for a directory
+   def set_drop_group(self, path):
+      """Set the group of path, and of all of its contents for a directory.
+
+      The copy runs with an effective user of 'gdexdata' but with the EFFECTIVE
+      GROUP of the caller, so a dropped file lands in the caller's group unless
+      the directory it lands in happens to carry the setgid bit.  Symbolic links
+      are skipped, since os.chown() follows them out of the dataset directory.
+
+      Args:
+         path (str): Absolute path of a dropped file or directory.
+      """
+      if op.islink(path): return
+      self.change_local_group(path, None, None, None, self.LOGWRN)
+      if not op.isdir(path): return
+      for root, dirs, files in os.walk(path):
+         for name in dirs + files:
+            subpath = op.join(root, name)
+            if op.islink(subpath): continue
+            self.change_local_group(subpath, None, None, None, self.LOGWRN)
+
    # copy one source path into the target directory
    def drop_one(self, file):
       """Copy one source file or directory into the target directory.
@@ -269,7 +341,9 @@ class GdexDrop(PgFile):
          if tinfo and tinfo['data_size'] == finfo['data_size']:
             self.pglog(tofile + ": Target exists with same size, skip copying", self.LOGWRN)
             return 0
-      return (1 if self.local_copy_local(tofile, path, self.LGWNEX) else 0)
+      if not self.local_copy_local(tofile, path, self.LGWNEX): return 0
+      self.set_drop_group(tofile)
+      return 1
 
 # main function to execute this script
 def main():

{rda_python_miscs-3.0.6 → rda_python_miscs-3.0.8}/src/rda_python_miscs/gdexdrop.usg

@@ -17,7 +17,12 @@
           files into.  Missing sub-directories are created.  The path must stay
           inside the dataset directory; '..' components and symbolic links
           leading out of it are rejected.  Defaults to the dataset directory
-          itself;
+          itself.
+          An absolute path under /tmp/ names any sub-path of /tmp instead, for
+          trying a drop out without touching the dataset tree.  The dataset of
+          -ds is still checked against the access list, and because /tmp is
+          world writable, the existing part of the path must belong to the
+          calling user or to 'gdexdata';
 
       - Option -f File/Directory List is the default option, so the source paths
           may be given without it and in any position.  Relative paths are
@@ -40,7 +45,9 @@
 
   Every source path must be readable by the calling user, not merely by
   'gdexdata'; a path the caller cannot read themselves is refused.  Copied files
-  and directories are owned by 'gdexdata'.
+  and directories are owned by 'gdexdata' and belong to the GDEX group, as do any
+  sub-directories created for -t; without that they would stay in the calling
+  user's group, since only the user is switched to 'gdexdata', not the group.
 
   Access List
   -----------
@@ -75,3 +82,7 @@
     4. Drop a long list of files named in a file, with group-writable modes:
 
           gdexdrop -ds d123456 -i filelist.txt -F 664 -D 775
+
+    5. Try a drop out under /tmp, leaving the dataset directory alone:
+
+          gdexdrop -ds d123456 -t /tmp/mydrop hourly.2026*.nc

{rda_python_miscs-3.0.6 → rda_python_miscs-3.0.8/src/rda_python_miscs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_miscs
-Version: 3.0.6
+Version: 3.0.8
 Summary: RDA Python package to hold RDA miscellaneous utility programs
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
@@ -172,6 +172,17 @@ deliberately narrow alternative to `gdexcp`: the destination is always
 directory is rejected, and every source must be readable by the calling user
 rather than only by `gdexdata`.
 
+An absolute `-t` under `/tmp/` names any sub-path of `/tmp` instead of a sub-path
+of the dataset directory, so that a drop can be tried out without touching the
+dataset tree.  The dataset of `-ds` is still checked against the access list, and
+because `/tmp` is world writable — unlike a dataset directory — the part of the
+path that already exists must belong to the calling user or to `gdexdata`.
+
+Everything dropped, including sub-directories created for `-t`, is also set to the
+GDEX group `PGLOG['GDEXGRP']`.  The setuid wrapper switches only the user to
+`gdexdata`, not the group, so without that step a dropped file would stay in the
+calling user's group unless the directory it landed in carried the setgid bit.
+
 Who may drop into which dataset is read from the access list
 `/glade/u/home/gdexdata/config/gdexdrop.conf`.  It must be owned by `gdexdata`
 and must not be writable by group or others, otherwise `gdexdrop` refuses to