rda-python-common 3.0.26__py3-none-any.whl → 3.0.28__py3-none-any.whl

rda_python_common/__init__.py

@@ -22,7 +22,7 @@ object that existing callers expect.
 
 from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
 
-__version__ = "3.0.26"
+__version__ = "3.0.28"
 
 __all__ = [
    "PgLOG",

rda_python_common/pg_dbi.py

@@ -2176,6 +2176,7 @@ class PgDBI(PgLOG):
    
       pgrec = self.pgget("dsowner, dssgrp", "specialist, lstname, fstname",
                     "specialist = logname AND dsid = '{}' AND priority = 1".format(dsid), logact)
+      if pgrec == self.FAILURE: return None   # query error; do not cache the fallback
       if pgrec:
          if pgrec['specialist'] == "datahelp" or pgrec['specialist'] == "dss":
             pgrec['lstname'] = "Help"

rda_python_common/pg_file.py

@@ -410,6 +410,92 @@ class PgFile(PgUtil, PgSIG):
          self.errlog("Error Execute: {}\n{}".format(cmd, buf), 'O', loop, logact)
       return self.FAILURE
 
+   # Upload a local directory to object under a key prefix
+   #    todir - target object key prefix, without the trailing '/'
+   #  fromdir - source local directory name
+   def local_copy_object_directory(self, todir, fromdir, bucket = None, meta = None, logact = 0):
+      """Upload every file under a local directory to an object key prefix.
+
+      The whole directory goes up in one recursive call, the upload twin of
+      object_copy_local_directory(): one process per file costs several seconds
+      of start-up each, which dominates for a directory of many files.  Each file
+      lands at '{todir}/{path relative to fromdir}'.
+
+      Nothing is uploaded when every file is already under the prefix at its local
+      size (unless OVRIDE is set); otherwise the whole directory is uploaded again,
+      since the recursive call cannot be limited to the missing files.
+
+      Args:
+         todir (str): Object key prefix (destination path in the bucket); not empty.
+         fromdir (str): Source local directory path.
+         bucket (str | None): Target bucket; defaults to PGLOG['OBJCTBKT'].
+         meta (dict | None): Extra metadata key/value pairs to attach to each object.
+         logact (int): Logging action flags; default 0.
+
+      Returns:
+         int: self.SUCCESS on success, self.FAILURE on error.
+      """
+      if not bucket: bucket = self.PGLOG['OBJCTBKT']
+      if meta is None: meta = {}
+      if 'user' not in meta: meta['user'] = self.PGLOG['CURUID']
+      if 'group' not in meta: meta['group'] = self.PGLOG['GDEXGRP']
+      uinfo = json.dumps(meta)
+      ms = re.match(r'^(.+)/$', fromdir)
+      if ms: fromdir = ms.group(1)
+      ms = re.match(r'^(.+)/$', todir)
+      if ms: todir = ms.group(1)
+      # the same walk isd_s3_cli makes to pick the files: no symlinked directories
+      lfiles = {}
+      for path, dirs, files in os.walk(fromdir):
+         for fname in files:
+            lfile = op.join(path, fname)
+            lfiles[op.relpath(lfile, fromdir)] = op.getsize(lfile)
+      if not lfiles:
+         return self.lmsg(fromdir, "{} to copy to {}-{}".format(self.PGLOG['MISSFILE'], self.OHOST, todir), logact)
+      prefix = todir + '/'
+      if not logact&self.OVRIDE:
+         flist = self.object_glob(todir, bucket, 0, logact)
+         if flist == self.FAILURE: return self.FAILURE
+         miss = self.object_upload_missing(prefix, lfiles, flist)
+         if not miss:
+            self.pglog("{}-{}-{}: all {} file(s) exist already, skip uploading".format(
+                       self.OHOST, bucket, todir, len(lfiles)), logact&~self.EXITLG)
+            return self.SUCCESS
+      # -db too: an upload goes to the default bucket whatever -b says
+      cmd = "{} -db {} um -ld {} -b {} -kp {} -r -md '{}'".format(
+            self.OBJCTCMD, bucket, fromdir, bucket, todir, uinfo)
+      buf = self.pgsystem(cmd, logact, self.CMDBTH)
+      flist = self.object_glob(todir, bucket, 0, logact)
+      if flist == self.FAILURE: return self.FAILURE
+      miss = self.object_upload_missing(prefix, lfiles, flist)
+      if miss:
+         return self.errlog("{}-{}: Error upload {} of {} file(s) to {}-{}\n{}".format(
+                            fromdir, miss[0], len(miss), len(lfiles), self.OHOST, todir, buf), 'O', 1, logact)
+      self.pglog("{}: {} file(s) uploaded to {}-{}-{}".format(
+                 fromdir, len(lfiles), self.OHOST, bucket, todir), logact&~self.EXITLG)
+      return self.SUCCESS
+
+   # List the local files not uploaded under an object key prefix yet
+   #   prefix - target object key prefix, including the trailing '/'
+   #   lfiles - local file sizes, keyed by path relative to the source directory
+   #    flist - object_glob() info, keyed by object key
+   def object_upload_missing(self, prefix, lfiles, flist):
+      """Return the local files that have no object of the same size under a key prefix.
+
+      Args:
+         prefix (str): Object key prefix including the trailing '/'.
+         lfiles (dict): Mapping of relative path → local file size.
+         flist (dict): Mapping of key → object info from object_glob().
+
+      Returns:
+         list: The relative paths still to upload.
+      """
+      miss = []
+      for name in lfiles:
+         oinfo = flist.get(prefix + name)
+         if not (oinfo and oinfo['data_size'] == lfiles[name]): miss.append(name)
+      return miss
+
    # Copy multiple files from a Globus endpoint to another
    #   tofiles - target file name list, echo name leading with /dsnnn.n/ on Quasar and
    #             leading with /data/ or /decsdata/ on local glade disk

{rda_python_common-3.0.26.dist-info → rda_python_common-3.0.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.26
+Version: 3.0.28
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common
@@ -189,7 +189,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN)
 python -c "import rda_python_common; print(rda_python_common.__version__)"
 ```
 
-You should see the installed version (currently `3.0.26`).  If the import
+You should see the installed version (currently `3.0.28`).  If the import
 fails, double-check that the active Python environment is the one where you
 ran `pip install`.
 

{rda_python_common-3.0.26.dist-info → rda_python_common-3.0.28.dist-info}/RECORD

@@ -7,10 +7,10 @@ rda_python_common/PgOPT.py,sha256=0nQPF19N-1VWwrjFHEeruVSwVDwvFKd4uowPc8dK7e4,56
 rda_python_common/PgSIG.py,sha256=0MJPH0fTtSshIsvUXkzQjReHPyCWdzHYOY5LbFSh7-A,34318
 rda_python_common/PgSplit.py,sha256=En7xcdkE4YNRcMX88qXnrUgUOnQU3Kqh8uxRIdlJhEA,9156
 rda_python_common/PgUtil.py,sha256=7hXAznAI8tYh-nJRKBIXejNe2ROQ2Wxn28OAndeg0Iw,48433
-rda_python_common/__init__.py,sha256=8voFQx6KSN8f2L7Nm1zw3EJ4WVK1_Tou-vqKEOyCy44,995
+rda_python_common/__init__.py,sha256=nuSiih-pLPf8s7ct003v0QDJJM3813MwbRld2QrOn80,995
 rda_python_common/pg_cmd.py,sha256=MYldUuM8mSUIT34yKQyUd0i4NL4zdSB8C6Jcc28NRxA,33809
-rda_python_common/pg_dbi.py,sha256=h1snZ8fJwHeBQJmNjsjJoDKkfT4FW9avMvdHgcIc-cA,122328
-rda_python_common/pg_file.py,sha256=h7dXgPumZuAI183TV5eSvdONV-OpOGNwkwy1veutorg,176768
+rda_python_common/pg_dbi.py,sha256=ASIW2vsIjMab2TMfhqJiXS7LQhKKGpKjzNMqo7RCePw,122415
+rda_python_common/pg_file.py,sha256=7NipcgMy4t67zbH2MHtRL3vxyY1FHdIgMTQ-kE0H2fA,181122
 rda_python_common/pg_lock.py,sha256=31EaVDjCkcx3-n8-KnzG18R8Pz7Z6KyFsEqcml6Iq5c,32702
 rda_python_common/pg_log.py,sha256=erQN4YYPwiORa3OE0Jt8s05rlWsjtyIs0fJ_KLbNAQ8,82959
 rda_python_common/pg_opt.py,sha256=LxVAf47wi2STkVGSXMvV3IZphEJcoO8bvYXy6kQ5ox4,82403
@@ -20,9 +20,9 @@ rda_python_common/pg_split.py,sha256=aAWKUZPmZ-LQ_fJ3DSKzPKxJw0fMDJ2fgP8ZT629M30
 rda_python_common/pg_util.py,sha256=JL9NoW-VoQvsjXsUr09oBUWYJJNltU4GyQIfWlpkf90,86084
 rda_python_common/pgpassword.py,sha256=aTQaO59QokWrv2EKom4TvKhNW2Mr6aFMMBVG4DCbcKI,5023
 rda_python_common/pgpassword.usg,sha256=Wrle6A8sdlMGx3ZEAmE6TNDPOlG84Whe0T3rCNFzPfY,2013
-rda_python_common-3.0.26.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_common-3.0.26.dist-info/METADATA,sha256=Vx8i62NJjW2iR0Bnq-flAaXZaIqVLoNsYmVBQWlSp_s,14530
-rda_python_common-3.0.26.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-rda_python_common-3.0.26.dist-info/entry_points.txt,sha256=pZgVNWspcK-F1TbPav7C3C9NdeHDZMm_25fW9weix00,65
-rda_python_common-3.0.26.dist-info/top_level.txt,sha256=KVQmx7D3DD-jsiheqL8HdTrRE14hpRnZY5_ioMArA5k,18
-rda_python_common-3.0.26.dist-info/RECORD,,
+rda_python_common-3.0.28.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_common-3.0.28.dist-info/METADATA,sha256=YFxuS4nRBjgJ0euFjSKU-GNswRGmFLgj7txr1zOeQUY,14530
+rda_python_common-3.0.28.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+rda_python_common-3.0.28.dist-info/entry_points.txt,sha256=pZgVNWspcK-F1TbPav7C3C9NdeHDZMm_25fW9weix00,65
+rda_python_common-3.0.28.dist-info/top_level.txt,sha256=KVQmx7D3DD-jsiheqL8HdTrRE14hpRnZY5_ioMArA5k,18
+rda_python_common-3.0.28.dist-info/RECORD,,

{rda_python_common-3.0.26.dist-info → rda_python_common-3.0.28.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (84.0.0)
+Generator: setuptools (84.2.1)
 Root-Is-Purelib: true
 Tag: py3-none-any