rcsb.utils.io 1.53__py3-none-any.whl

rcsb/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

rcsb/utils/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

rcsb/utils/io/CacheUtils.py

@@ -0,0 +1,66 @@
+##
+# File: CacheUtils.py
+# Date: 18-Jul-2019 jdw
+#
+##
+
+import logging
+from collections import OrderedDict
+
+logger = logging.getLogger(__name__)
+
+
+class CacheUtils:
+    """LRU style cache.
+    """
+
+    def __init__(self, size=10, label="cache"):
+        """LRU style cache
+
+        Args:
+            size (int, optional): maximum number of elements in the cache. Defaults to 10.
+            label (str, optional): A label an instance of the cache.  Defaults to "cache".
+        """
+        self.__size = size
+        self.__cache = OrderedDict()
+        self.__label = label
+
+    def get(self, key):
+        """Return the cached value associated with the input key.  The key:value
+        combination are moved to top of the cache.
+
+        Args:
+            key (hashable): identifier for cached object
+
+        Returns:
+            (any): Cached object associated with the input key or None
+        """
+        try:
+            value = self.__cache.pop(key)
+            self.__cache[key] = value
+            return value
+        except KeyError:
+            return None
+
+    def set(self, key, value):
+        """Store the input value in the cache. The cache contents of the cache
+        are adjusted if the size limit is exceeded.
+
+        Args:
+            key (hashable): identifier for cached object
+            value (any): value to be cached
+
+        Returns:
+            bool: True for success or false otherwise
+        """
+        try:
+            try:
+                self.__cache.pop(key)
+            except KeyError:
+                if len(self.__cache) >= self.__size:
+                    self.__cache.popitem(last=False)
+            self.__cache[key] = value
+            return True
+        except Exception as e:
+            logger.exception("Failing for %s with %s", self.__label, str(e))
+        return False

rcsb/utils/io/CryptUtils.py

@@ -0,0 +1,193 @@
+##
+# File: CryptUtils.py
+# Date: 12-Aug-2019 Jdw
+#
+# Updates:
+#
+##
+"""
+ Utilities supporting encrypting and decrypting flat files and messages.
+"""
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "john.westbrook@rcsb.org"
+__license__ = "Apache 2.0"
+
+import base64
+import binascii
+import hashlib
+import logging
+
+from nacl.encoding import HexEncoder, RawEncoder
+from nacl.secret import SecretBox
+from nacl.utils import random
+
+logger = logging.getLogger(__name__)
+
+
+class CryptUtils(object):
+    """Utility class for encrypting and decrypting data files and messages
+
+    ** This class does not yet support any stream operations so resources must be
+    available to load data sets in memory -
+
+    """
+
+    def __init__(self, *args, **kwargs):
+        _ = args
+        _ = kwargs
+
+    def newKey(self):
+        """Generate a new encryption key.
+
+        Returns:
+            (str): encryption key
+        """
+        key = random(SecretBox.KEY_SIZE)
+        return binascii.hexlify(key).decode("utf-8")
+
+    def encryptMessage(self, msg, hexKey):
+        """Encrypt the input message.
+
+        Args:
+            msg (str): input message
+            hexKey (str): encryption key
+
+        Returns:
+            (str):  encrypted message text
+        """
+        txt = None
+        try:
+            box = SecretBox(hexKey.encode("utf-8"), encoder=HexEncoder)
+            bMsg = msg.encode("utf-8")
+            encMsg = box.encrypt(bMsg, encoder=RawEncoder)
+            txt = base64.b64encode(encMsg).decode("ascii")
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+
+        return txt
+
+    def decryptMessage(self, msg, hexKey):
+        """Decrypt the input message.
+
+        Args:
+            msg (str): input message
+            hexKey (str): encryption key
+
+        Returns:
+            (str):  decrypted message text
+        """
+        txt = None
+        try:
+            box = SecretBox(hexKey.encode("utf-8"), encoder=HexEncoder)
+            bMsg = base64.b64decode(msg)
+            dcrMsg = box.decrypt(bMsg)
+            logger.debug("type %r text %r", type(dcrMsg), dcrMsg)
+            txt = dcrMsg.decode("utf-8")
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+
+        return txt
+
+    def encryptFile(self, inpPath, outPath, hexKey, chunkSize=65536):
+        """Encrypt the data in the input file store this in the output file.
+
+        Args:
+            inpPath (str): input file path
+            outPath (str): output file path (encrypted)
+            hexKey (str): encryption key
+            chunkSize (int, optional): the size of incremental copy operations. Defaults to 65536.
+
+        Returns:
+            (bool): True for success or False otherwise
+        """
+        try:
+            box = SecretBox(hexKey.encode("utf-8"), encoder=HexEncoder)
+            tL = []
+            with open(inpPath, "rb") as ifh:
+                while True:
+                    chunk = ifh.read(chunkSize)
+                    if not chunk:
+                        break
+                    tL.append(chunk)
+            #
+            txt = b"".join(tL)
+            logger.debug("Read %s size %d", inpPath, len(txt))
+            enctxt = box.encrypt(txt, encoder=RawEncoder)
+            tL = [enctxt[i : i + chunkSize] for i in range(0, len(enctxt), chunkSize)]
+            logger.debug("Chunks %d size %d", len(tL), len(enctxt))
+            #
+            with open(outPath, "wb") as ofh:
+                for tV in tL:
+                    ofh.write(tV)
+            return True
+        except Exception as e:
+            logger.exception("Failing with %r", str(e))
+        return False
+
+    def decryptFile(self, inpPath, outPath, hexKey, chunkSize=65536):
+        """Decrypt the data in the input file store the result in the output file.
+
+        Args:
+            inpPath ([type]): input file path (encrypted)
+            outPath ([type]): output file path
+            hexKey ([type]): encryption key
+            chunkSize (int, optional): the size of incremental copy operations. Defaults to 65536.
+
+        Returns:
+            (bool): True for success or False otherwise
+        """
+        try:
+            box = SecretBox(hexKey.encode("utf-8"), encoder=HexEncoder)
+            tL = []
+            with open(inpPath, "rb") as ifh:
+                while True:
+                    chunk = ifh.read(chunkSize)
+                    if not chunk:
+                        break
+                    tL.append(chunk)
+            #
+            enctxt = b"".join(tL)
+            #
+            txt = box.decrypt(enctxt)
+            tL = [txt[i : i + chunkSize] for i in range(0, len(txt), chunkSize)]
+            logger.debug("Chunks %d size %d", len(tL), len(txt))
+            #
+            with open(outPath, "wb") as ofh:
+                for tV in tL:
+                    ofh.write(tV)
+            return True
+        except Exception as e:
+            logger.exception("Failing with %r", str(e))
+        return False
+
+    def getFileHash(self, filePath, hashType="SHA1", blockSize=65536):
+        """Return the hash (hashType) for the input file.
+
+        Args:
+            filePath (str): for input file path
+            hashType (str, optional): one of MD5, SHA1, or SHA256. Defaults to "SHA1".
+            blockSize (int, optional): the size of incremental read operations. Defaults to 65536.
+
+        Returns:
+            (dict): {"hashDigest": xxxx , "hashType": 'SHA1|MD5|SHA256'}
+        """
+        rD = {}
+        hashObj = None
+        if hashType not in ["MD5", "SHA1", "SHA256"]:
+            return rD
+        try:
+            if hashType == "SHA1":
+                hashObj = hashlib.sha1()
+            elif hashType == "SHA256":
+                hashObj = hashlib.sha256()
+            elif hashType == "MD5":
+                hashObj = hashlib.md5()
+
+            with open(filePath, "rb") as ifh:
+                for block in iter(lambda: ifh.read(blockSize), b""):
+                    hashObj.update(block)
+            rD = {"hashDigest": hashObj.hexdigest(), "hashType": hashType}
+        except Exception as e:
+            logger.exception("Failing with file %s %r", filePath, str(e))
+        return rD

rcsb/utils/io/ExecUtils.py

@@ -0,0 +1,111 @@
+##
+# File: ExeceUtils.py
+# Date: 24-Jan-2020 jdw
+#
+##
+import logging
+import os
+import subprocess
+import sys
+
+logger = logging.getLogger(__name__)
+
+
+class ExecUtils(object):
+    """Wrapper for subprocess execution."""
+
+    def __init__(self):
+        """Wrapper for subprocess execution"""
+
+    def run(self, execPath, execArgList=None, outPath=None, outAppend=False, timeOut=None, inpPath=None, suppressStderr=False):
+        """Execute the input program as a blocking subprocess with optional timeout.
+
+        Args:
+            execPath (str): path to executable program or script
+            execArgList (list, optional): argument list. Defaults to None.
+            outPath (str, optional): redirect stdout and stderr to this file handle. Defaults to None.
+            inpPath (str, optional): redirect stdin to this file handle. Defaults to None.
+            outAppend (bool, optional): append output. Defaults to False.
+            timeOut (float, optional): timeout (seconds). Defaults to None.
+            suppressStderr (bool, optional): suppress stderr output (default: combined with stdout)
+
+
+        Returns:
+            bool: true for sucess or False otherwise
+        """
+        retCode = False
+        subProcResult = None
+        kwD = {}
+        if suppressStderr:
+            myStderr = subprocess.DEVNULL
+        else:
+            myStderr = subprocess.STDOUT
+        try:
+            if not os.path.isfile(execPath) and os.access(execPath, os.X_OK):
+                return retCode
+            if sys.version_info[0] > 2 and timeOut:
+                kwD = {"timeout": timeOut}
+            cmdL = [execPath]
+            if execArgList:
+                cmdL.extend(execArgList)
+            if outPath and inpPath:
+                myMode = "a" if outAppend else "w"
+                with open(outPath, myMode) as ofh, open(inpPath, "r") as ifh:
+                    subProcResult = subprocess.call(cmdL, stdout=ofh, stdin=ifh, stderr=myStderr, **kwD)
+            elif outPath:
+                myMode = "a" if outAppend else "w"
+                with open(outPath, myMode) as ofh:
+                    subProcResult = subprocess.call(cmdL, stdout=ofh, stderr=myStderr, **kwD)
+            else:
+                subProcResult = subprocess.call(cmdL, **kwD)
+            retCode = not subProcResult
+        except Exception as e:
+            logger.exception("Failing execution of %s (%s) with %s", execPath, execArgList, str(e))
+        #
+        if subProcResult != 0:
+            logger.error("return code is %r", subProcResult)
+        return retCode
+
+    def runShell(self, script, outPath=None, outAppend=False, timeOut=None, inpPath=None, suppressStderr=False):
+        """Execute the input program as a blocking subprocess with optional timeout.
+
+        Args:
+            script (str): script
+            outPath (str, optional): redirect stdout and stderr to this file handle. Defaults to None.
+            inpPath (str, optional): redirect stdin to this file handle. Defaults to None.
+            outAppend (bool, optional): append output. Defaults to False.
+            timeOut (float, optional): timeout (seconds). Defaults to None.
+            suppressStderr (bool, optional): suppress stderr output (default: combined with stdout)
+
+
+        Returns:
+            bool: true for sucess or False otherwise
+        """
+        retCode = 0
+        kwD = {}
+        subProcResult = None
+        if suppressStderr:
+            myStderr = subprocess.DEVNULL
+        else:
+            myStderr = subprocess.STDOUT
+        try:
+            if sys.version_info[0] > 2 and timeOut:
+                kwD = {"timeout": timeOut}
+            #
+            if outPath and inpPath:
+                myMode = "a" if outAppend else "w"
+                with open(outPath, myMode) as ofh, open(inpPath, "r") as ifh:
+                    subProcResult = subprocess.run(script, stdout=ofh, stdin=ifh, stderr=myStderr, shell=True, check=False, **kwD)
+            elif outPath:
+                myMode = "a" if outAppend else "w"
+                with open(outPath, myMode) as ofh:
+                    subProcResult = subprocess.run(script, stdout=ofh, stderr=myStderr, shell=True, check=False, **kwD)
+            else:
+                subProcResult = subprocess.run(script, shell=True, check=False, **kwD)
+            retCode = not subProcResult.returncode
+        except Exception as e:
+            logger.exception("Failing execution of %r with %s", subProcResult, str(e))
+        #
+        if subProcResult and subProcResult.returncode != 0:
+            logger.warning("return code is %r", subProcResult.returncode)
+        return retCode