dcicutils 6.9.9.0b2__py3-none-any.whl → 6.9.9.0b6__py3-none-any.whl

dcicutils/ff_mocks.py

@@ -190,6 +190,23 @@ def mocked_s3utils(environments=None, require_sse=False, other_access_key_names=
                                         yield mock_boto3
 
 
+DEFAULT_SSE_ENVIRONMENTS_TO_MOCK = ['fourfront-foo', 'fourfront-bar']
+
+
+@contextlib.contextmanager
+def mocked_s3utils_with_sse(beanstalks=None, environments=None, require_sse=True, files=None):
+    # beanstalks= is deprecated. Use environments= instead.
+    environments = (DEFAULT_SSE_ENVIRONMENTS_TO_MOCK
+                    if beanstalks is None and environments is None
+                    else (beanstalks or []) + (environments or []))
+    with mocked_s3utils(environments=environments, require_sse=require_sse) as mock_boto3:
+        s3 = mock_boto3.client('s3')
+        assert isinstance(s3, MockBotoS3Client)
+        for filename, string in (files or {}).items():
+            s3.s3_files.files[filename] = string.encode('utf-8')
+        yield mock_boto3
+
+
 # Here we set up some variables, auxiliary functions, and mocks containing common values needed for testing
 # of the next several functions so that the functions don't have to set them up over
 # and over again in each test.

dcicutils/ff_utils.py

@@ -6,13 +6,11 @@ import requests
 import time
 
 from collections import namedtuple
+from dcicutils.lang_utils import disjoined_list
 from elasticsearch.exceptions import AuthorizationException
 from typing import Dict, List
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
-from . import (
-    s3_utils,
-    es_utils,
-)
+from . import s3_utils, es_utils
 from .misc_utils import PRINT
 
 
@@ -107,8 +105,6 @@ def search_request_with_retries(request_fxn, url, auth, verb, **kwargs):
         try:
             res_json = res.json()
         except ValueError:
-            # PyCharm notes this is unused. -kmp 17-Jul-2020
-            # res_json = {}
             try:
                 res.raise_for_status()
             except Exception as e:
@@ -238,7 +234,7 @@ def internal_compute_authorized_request(url, *, auth, ff_env, verb, retry_fxn, *
     authorized_request('https://data.4dnucleome.org/<some path>', ff_env='fourfront-webprod')
     """
     # Save to uncomment if debugging unit tests...
-    # print("authorized_request\n URL=%s\n auth=%s\n ff_env=%s\n verb=%s\n" % (url, auth, ff_env, verb))
+    # print(f"authorized_request\n URL={url}\n auth={auth}\n ff_env={ff_env}\n verb={verb}\n")
     use_auth = unified_authentication(auth, ff_env)
     headers = kwargs.get('headers')
     if not headers:
@@ -246,11 +242,12 @@ def internal_compute_authorized_request(url, *, auth, ff_env, verb, retry_fxn, *
     if 'timeout' not in kwargs:
         kwargs['timeout'] = 60  # default timeout
 
+    verb_upper = verb
     try:
-        the_verb = REQUESTS_VERBS[verb.upper()]
+        verb_upper = verb.upper()
+        the_verb = REQUESTS_VERBS[verb_upper]
     except KeyError:
-        raise ValueError("Provided verb %s is not valid. Must be one of: %s"
-                         % (verb.upper(), ', '.join(REQUESTS_VERBS.keys())))
+        raise ValueError(f"Provided verb {verb} is not valid. Must be one of {disjoined_list(REQUESTS_VERBS)}.")
     # automatically detect a search and overwrite the retry if it is standard
     if '/search/' in url and retry_fxn == standard_request_with_retries:
         retry_fxn = search_request_with_retries
@@ -262,7 +259,15 @@ def internal_compute_authorized_request(url, *, auth, ff_env, verb, retry_fxn, *
 
 
 def _sls(val):
-    """general helper to check for and strip leading slashes on ids in API fxns
+    """
+    Helper to strip any leading slashes on ids in API functions.
+    Examples:
+        >>> _sls('foo')
+        foo
+        >>> _sls('/foo')
+        foo
+        >>> _sls('/foo/bar/baz/')
+        foo/bar/baz/
     """
     return val.lstrip('/')
 

dcicutils/misc_utils.py

@@ -12,6 +12,7 @@ import io
 import os
 import logging
 import pytz
+import re
 import rfc3986.validators
 import rfc3986.exceptions
 import time
@@ -1316,6 +1317,17 @@ def string_list(s):
     return [p for p in [part.strip() for part in s.split(",")] if p]
 
 
+def is_c4_arn(arn: str) -> bool:
+    """
+    Returns True iff the given (presumed) AWS ARN string value looks like it
+    refers to a CGAP or Fourfront Cloudformation entity, otherwise returns False.
+    :param arn: String representing an AWS ARN.
+    :return: True if the given ARN refers to a CGAP or Fourfront Cloudformation entity, else False.
+    """
+    pattern = r"(fourfront|cgap|[:/]c4-)"
+    return True if re.search(pattern, arn) else False
+
+
 def string_md5(unicode_string):
     """
     Returns the md5 signature for the given u unicode string.

dcicutils/redis_tools.py

@@ -8,6 +8,9 @@ from dcicutils.redis_utils import RedisBase, RedisException
 log = structlog.getLogger(__name__)
 
 
+SESSION_TOKEN_COOKIE = 'c4_st'
+
+
 def make_session_token(n_bytes: int = 32) -> str:
     """ Uses the secrets module to create a cryptographically secure and URL safe string
     :param n_bytes: number of bytes to use, default 32
@@ -21,29 +24,16 @@ class RedisSessionToken:
     Model used by Redis to store session tokens
     Keystore structure:
         <env_namespace>:session:<token>
-            -> Redis hset containing the associated JWT and expiration time (3 hours)
+            -> Redis str of JWT with automated expiration
     """
-    JWT = 'jwt'
-    EXPIRATION = 'expiration'
 
     @staticmethod
-    def _build_session_expiration(n_hours: int = 3) -> str:
+    def _build_session_expiration(n_hours: int = 3) -> datetime.timedelta:
         """ Builds a session expiration date 3 hours after generation
         :param n_hours: timestamp after which the session token is invalid
         :return: datetime string 3 hours from creation time
         """
-        return str(datetime.datetime.utcnow() + datetime.timedelta(hours=n_hours))
-
-    def _build_session_hset(self, jwt: str, expiration=None) -> dict:
-        """ Builds Redis hset record for the session token
-        :param jwt: encoded jwt value
-        :param expiration: expiration if using an existing one
-        :return: dictionary to be stored as a hash in Redis
-        """
-        return {
-            self.JWT: jwt,
-            self.EXPIRATION: self._build_session_expiration() if not expiration else expiration
-        }
+        return datetime.timedelta(hours=n_hours)
 
     @staticmethod
     def _build_redis_key(namespace: str, token: str) -> str:
@@ -69,20 +59,24 @@ class RedisSessionToken:
         self.namespace = namespace
         self.redis_key = self._build_redis_key(self.namespace, self.session_token)
         self.jwt = jwt
-        self.session_hset = self._build_session_hset(self.jwt, expiration=expiration)
+        self.expiration = expiration or self._build_session_expiration()
 
     def __eq__(self, other):
         """ Evaluates equality of two session objects based on the value of the session hset """
-        return self.session_hset == other.session_hset
+        return (self.redis_key == other.redis_key) and (self.jwt == other.jwt)
 
     def get_session_token(self) -> str:
         """ Extracts the session token stored on this object """
         return self.session_token
 
     def get_redis_key(self) -> str:
-        """ Returns the key under which the Redis hset is stored - note that this contains the token! """
+        """ Returns the key under which the Redis set is stored - note that this contains the token! """
         return self.redis_key
 
+    def get_expiration(self) -> datetime:
+        """ Returns the expiration timedelta """
+        return self.expiration
+
     @classmethod
     def from_redis(cls, *, redis_handler: RedisBase, namespace: str, token: str):
         """ Builds a RedisSessionToken from an existing record - allows extracting JWT
@@ -93,9 +87,11 @@ class RedisSessionToken:
         :return: A RedisSessionToken object built from an existing record in Redis
         """
         redis_key = f'{namespace}:session:{token}'
-        redis_entry = redis_handler.hgetall(redis_key)
-        return cls(namespace=namespace, jwt=redis_entry[cls.JWT],
-                   token=token, expiration=redis_entry[cls.EXPIRATION])
+        redis_entry = redis_handler.get(redis_key)
+        if redis_entry:
+            expiration = redis_handler.ttl(redis_key)
+            return cls(namespace=namespace, jwt=redis_entry,
+                       token=token, expiration=expiration)
 
     def decode_jwt(self, audience: str, secret: str, leeway: int = 30) -> dict:
         """ Decodes JWT to grab info such as the email
@@ -113,7 +109,7 @@ class RedisSessionToken:
         :return: True if successful, raise Exception otherwise
         """
         try:
-            redis_handler.hset_multiple(self.redis_key, self.session_hset)
+            redis_handler.set(self.redis_key, self.jwt, exp=self.expiration)
         except Exception as e:
             log.error(str(e))
             raise RedisException()
@@ -124,15 +120,12 @@ class RedisSessionToken:
         :param redis_handler: handle to Redis API
         :return: True if token matches that in Redis and is not expired
         """
-        redis_token = redis_handler.hgetall(self.redis_key)
+        redis_token = redis_handler.get(self.redis_key)
         if not redis_token:
             return False  # if it doesn't exist it's not valid
-        timestamp_is_valid = (datetime.datetime.fromisoformat(
-            redis_token[self.EXPIRATION]) > datetime.datetime.utcnow()
-        )
-        return timestamp_is_valid
+        return True  # if it does exist it must be valid since we always send with TTL
 
-    def update_session_token(self, *, redis_handler: RedisBase, jwt) -> bool:
+    def update_session_token(self, *, redis_handler: RedisBase, jwt: str) -> bool:
         """ Refreshes the session token, jwt (if different) and expiration stored in Redis
         :param redis_handler: handle to Redis API
         :param jwt: jwt of user
@@ -143,13 +136,13 @@ class RedisSessionToken:
         # build new one
         self.session_token = make_session_token()
         self.redis_key = self._build_redis_key(self.namespace, self.session_token)
+        self.expiration = self._build_session_expiration()
         self.jwt = jwt
-        self.session_hset = self._build_session_hset(jwt)
         return self.store_session_token(redis_handler=redis_handler)
 
-    def delete_session_token(self, *, redis_handler) -> bool:
+    def delete_session_token(self, *, redis_handler: RedisBase) -> bool:
         """ Deletes the session token from redis, effectively logging out
         :param redis_handler: handle to Redis API
         :return: True if successful, False otherwise
         """
-        return redis_handler.delete(self.redis_key)
+        return 1 == redis_handler.delete(self.redis_key)

dcicutils/redis_utils.py

@@ -1,4 +1,5 @@
 import redis
+import datetime
 from typing import Union
 # Low level utilities for working with Redis
 
@@ -59,13 +60,17 @@ class RedisBase(object):
         """
         return self.redis.info()
 
-    def set(self, key: str, value: Union[str, int, float]) -> str:
+    def set(self, key: str, value: Union[str, int, float], exp: Union[int, datetime.timedelta] = None) -> str:
         """ Sets the given key to the given value https://redis.io/commands/set/
         :param key: string to store value under
         :param value: value mapped to from key
+        :param exp: expiration time in seconds as int or datetime.timedelta (optional)
         :return: a "true" string <OK>
         """
-        return self.redis.set(self._encode_value(key), self._encode_value(value))
+        kwargs = {}
+        if exp:
+            kwargs['ex'] = exp
+        return self.redis.set(self._encode_value(key), self._encode_value(value), **kwargs)
 
     def get(self, key: str) -> str:
         """ Gets the given key from Redis https://redis.io/commands/get/
@@ -77,6 +82,21 @@ class RedisBase(object):
             val = self._decode_value(val)
         return val
 
+    def set_expiration(self, key: str, t: Union[int, datetime.time]) -> bool:
+        """ Sets the TTL of the given key manually
+        :param key: key to set TTL
+        :param t: time in seconds until expiration (datetime.timedelta or int)
+        :returns: True if successful
+        """
+        return self.redis.expire(key, t, gt=True)
+
+    def ttl(self, key: str) -> datetime.time:
+        """ Gets the TTL of the given key
+        :param key: key to get TTL for
+        :return: datetime value in seconds
+        """
+        return self.redis.ttl(key)
+
     def delete(self, key: str) -> int:
         """ Deletes the given key from Redis https://redis.io/commands/del/
         :param key: key to delete

dcicutils/s3_utils.py

@@ -52,6 +52,7 @@ class HealthPageKey:  # This is moving here from cgap-portal.
     PROCESSED_FILE_BUCKET = 'processed_file_bucket'          # = s3Utils.OUTFILE_BUCKET_HEALTH_PAGE_KEY
     PROJECT_VERSION = 'project_version'
     PYTHON_VERSION = 'python_version'
+    REDIS = 'redis'
     S3_ENCRYPT_KEY_ID = 's3_encrypt_key_id'
     SNOVAULT_VERSION = 'snovault_version'
     SYSTEM_BUCKET = 'system_bucket'                          # = s3Utils.SYS_BUCKET_HEALTH_PAGE_KEY
@@ -235,10 +236,13 @@ class s3Utils(s3Base):  # NOQA - This class name violates style rules, but a lot
         if not es_url.startswith('http'):  # skips on both http:// and https://
             es_url = ('https://' if es_url.endswith(":443") else 'http://') + es_url
 
+        redis_url = s3u.health_page_get(HealthPageKey.REDIS)
+
         env_info = {
             'ff_env': env_full_name,
             'fourfront': portal_url,
-            'es': es_url
+            'es': es_url,
+            'redis': redis_url  # default to None
         }
 
         return env_info

dcicutils/secrets_utils.py

@@ -117,7 +117,8 @@ def apply_overrides(*, secrets: dict, rename_keys: Optional[dict] = None,
                 raise ValueError(f"Cannot rename {key} to {new_name}"
                                  f" because {a_or_an(new_name)} attribute already exists.")
             if key not in secrets:
-                raise ValueError(f"Cannot rename {key} to {new_name} because the secrets has no {key} attribute.")
+                logger.warning(f"Cannot rename {key} to {new_name} because the secrets has no {key} attribute.")
+                continue
             secrets[new_name] = secrets.pop(key)
     if override_values:
         secrets = dict(secrets, **override_values)

{dcicutils-6.9.9.0b2.dist-info → dcicutils-6.9.9.0b6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 6.9.9.0b2
+Version: 6.9.9.0b6
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-6.9.9.0b2.dist-info → dcicutils-6.9.9.0b6.dist-info}/RECORD

@@ -20,23 +20,23 @@ dcicutils/env_utils.py,sha256=MQ3zcXcbxJV9dO5y1nN2HNswtiUw2VDK4KrROrZOUtA,46285
 dcicutils/env_utils_legacy.py,sha256=J81OAtJHN69o1beHO6q1j7_J6TeblSjnAHlS8VA5KSM,29032
 dcicutils/es_utils.py,sha256=ZksLh5ei7kRUfiFltk8sd2ZSfh15twbstrMzBr8HNw4,7541
 dcicutils/exceptions.py,sha256=fO-zIwcT0kdy-BL1qczva3QLY9ZEkhAFVGfBS9w-Row,9257
-dcicutils/ff_mocks.py,sha256=eUdIJQNaNf5UCT7_-ruH3HXP5rlKQNHW3OMKpHymlGQ,36260
-dcicutils/ff_utils.py,sha256=I25EmbKzPm-V7zEaMbx6fZPSD3OKh5sLmKrnsopHtus,64240
+dcicutils/ff_mocks.py,sha256=wmf7-_C0cq9bpZZGUyKQO6eC-1p2ltKPVCSi-ln5y68,37025
+dcicutils/ff_utils.py,sha256=nHrVyf252HgP1mnerFmqJzUo6_Cm2hRk-Rnj5KN1RUQ,64324
 dcicutils/jh_utils.py,sha256=Gpsxb9XEzggF_-Eq3ukjKvTnuyb9V1SCSUXkXsES4Kg,11502
 dcicutils/kibana/dashboards.json,sha256=wHMB_mpJ8OaYhRRgvpZuihaB2lmSF64ADt_8hkBWgQg,16225
 dcicutils/kibana/readme.md,sha256=3KmHF9FH6A6xwYsNxRFLw27q0XzHYnjZOlYUnn3VkQQ,2164
 dcicutils/lang_utils.py,sha256=MaCJG-4KB2oOd3wcdd0e048eRBLkoxCKFkqR1KlrjkI,27302
 dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
-dcicutils/misc_utils.py,sha256=iz-MAy4OlmQJUY2yD5dMRqfcEtNhfBHVkLdCi8QXdsg,86187
+dcicutils/misc_utils.py,sha256=jRazVlj2gJ_DZsI2JjPIEfTmFLkcxDwPQStqGtaGMRc,86651
 dcicutils/obfuscation_utils.py,sha256=xUCQYAe-nQY3vCKMLjT_3tAFhIeYkJuhMaZnxunF0Hc,5510
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
 dcicutils/qa_checkers.py,sha256=hdduo-6ZITEurrmZYjll5N-nKQeyKaFLbERE9d4gRdU,20232
 dcicutils/qa_utils.py,sha256=g0nuAigZUWCXd4ZBqOKHxLq1H3AGI2X2wD57rvBdAOY,120561
-dcicutils/redis_tools.py,sha256=mo3GV6yqmRFN508x7q1aUACB3lpu1uNRuxojkqw3Ilc,6784
-dcicutils/redis_utils.py,sha256=vV-7NwRXVrQZhJMFnU-0qAZTfZX9tWEvJjDojogsmd0,5677
-dcicutils/s3_utils.py,sha256=sl2golIuUCCuO9CsXVGI3ZgRqHuSigkoe0Wy0e_R-3E,25526
-dcicutils/secrets_utils.py,sha256=d-oVHmyJL6Bcslxth73Ab31GeNoYtygWoVtOuzZ0xVQ,19722
+dcicutils/redis_tools.py,sha256=VQ5cTnDnKQ8AX0n9iuikCqtEUtIlE0ZOEqVZR3iNGTI,6389
+dcicutils/redis_utils.py,sha256=VJ-7g8pOZqR1ZCtdcjKz3-6as2DMUcs1b1zG6wSprH4,6462
+dcicutils/s3_utils.py,sha256=PAIUXpv7l7NNK0IzVLS6URJhLxg-SxLbZc0fJTpjI8E,25659
+dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
-dcicutils-6.9.9.0b2.dist-info/WHEEL,sha256=DA86_h4QwwzGeRoz62o1svYt5kGEXpoUTuTtwzoTb30,83
-dcicutils-6.9.9.0b2.dist-info/METADATA,sha256=w0nIhFQw9aesViWOCVBy4I2M3xvqMVGXsEJLY7GAf54,2722
-dcicutils-6.9.9.0b2.dist-info/RECORD,,
+dcicutils-6.9.9.0b6.dist-info/WHEEL,sha256=DA86_h4QwwzGeRoz62o1svYt5kGEXpoUTuTtwzoTb30,83
+dcicutils-6.9.9.0b6.dist-info/METADATA,sha256=oWx6nepGIzhI4H0gi13-CV8KXVaYXtaSRmi8R4F1GUM,2722
+dcicutils-6.9.9.0b6.dist-info/RECORD,,