dcicutils 6.9.9.0b4__tar.gz → 6.9.9.0b6__tar.gz

{dcicutils-6.9.9.0b4 → dcicutils-6.9.9.0b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 6.9.9.0b4
+Version: 6.9.9.0b6
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-6.9.9.0b4 → dcicutils-6.9.9.0b6}/dcicutils/ff_mocks.py

@@ -190,6 +190,23 @@ def mocked_s3utils(environments=None, require_sse=False, other_access_key_names=
                                         yield mock_boto3
 
 
+DEFAULT_SSE_ENVIRONMENTS_TO_MOCK = ['fourfront-foo', 'fourfront-bar']
+
+
+@contextlib.contextmanager
+def mocked_s3utils_with_sse(beanstalks=None, environments=None, require_sse=True, files=None):
+    # beanstalks= is deprecated. Use environments= instead.
+    environments = (DEFAULT_SSE_ENVIRONMENTS_TO_MOCK
+                    if beanstalks is None and environments is None
+                    else (beanstalks or []) + (environments or []))
+    with mocked_s3utils(environments=environments, require_sse=require_sse) as mock_boto3:
+        s3 = mock_boto3.client('s3')
+        assert isinstance(s3, MockBotoS3Client)
+        for filename, string in (files or {}).items():
+            s3.s3_files.files[filename] = string.encode('utf-8')
+        yield mock_boto3
+
+
 # Here we set up some variables, auxiliary functions, and mocks containing common values needed for testing
 # of the next several functions so that the functions don't have to set them up over
 # and over again in each test.

{dcicutils-6.9.9.0b4 → dcicutils-6.9.9.0b6}/dcicutils/ff_utils.py

@@ -6,13 +6,11 @@ import requests
 import time
 
 from collections import namedtuple
+from dcicutils.lang_utils import disjoined_list
 from elasticsearch.exceptions import AuthorizationException
 from typing import Dict, List
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
-from . import (
-    s3_utils,
-    es_utils,
-)
+from . import s3_utils, es_utils
 from .misc_utils import PRINT
 
 
@@ -107,8 +105,6 @@ def search_request_with_retries(request_fxn, url, auth, verb, **kwargs):
         try:
             res_json = res.json()
         except ValueError:
-            # PyCharm notes this is unused. -kmp 17-Jul-2020
-            # res_json = {}
             try:
                 res.raise_for_status()
             except Exception as e:
@@ -238,7 +234,7 @@ def internal_compute_authorized_request(url, *, auth, ff_env, verb, retry_fxn, *
     authorized_request('https://data.4dnucleome.org/<some path>', ff_env='fourfront-webprod')
     """
     # Save to uncomment if debugging unit tests...
-    # print("authorized_request\n URL=%s\n auth=%s\n ff_env=%s\n verb=%s\n" % (url, auth, ff_env, verb))
+    # print(f"authorized_request\n URL={url}\n auth={auth}\n ff_env={ff_env}\n verb={verb}\n")
     use_auth = unified_authentication(auth, ff_env)
     headers = kwargs.get('headers')
     if not headers:
@@ -246,11 +242,12 @@ def internal_compute_authorized_request(url, *, auth, ff_env, verb, retry_fxn, *
     if 'timeout' not in kwargs:
         kwargs['timeout'] = 60  # default timeout
 
+    verb_upper = verb
     try:
-        the_verb = REQUESTS_VERBS[verb.upper()]
+        verb_upper = verb.upper()
+        the_verb = REQUESTS_VERBS[verb_upper]
     except KeyError:
-        raise ValueError("Provided verb %s is not valid. Must be one of: %s"
-                         % (verb.upper(), ', '.join(REQUESTS_VERBS.keys())))
+        raise ValueError(f"Provided verb {verb} is not valid. Must be one of {disjoined_list(REQUESTS_VERBS)}.")
     # automatically detect a search and overwrite the retry if it is standard
     if '/search/' in url and retry_fxn == standard_request_with_retries:
         retry_fxn = search_request_with_retries
@@ -262,7 +259,15 @@ def internal_compute_authorized_request(url, *, auth, ff_env, verb, retry_fxn, *
 
 
 def _sls(val):
-    """general helper to check for and strip leading slashes on ids in API fxns
+    """
+    Helper to strip any leading slashes on ids in API functions.
+    Examples:
+        >>> _sls('foo')
+        foo
+        >>> _sls('/foo')
+        foo
+        >>> _sls('/foo/bar/baz/')
+        foo/bar/baz/
     """
     return val.lstrip('/')
 

{dcicutils-6.9.9.0b4 → dcicutils-6.9.9.0b6}/dcicutils/misc_utils.py

@@ -12,6 +12,7 @@ import io
 import os
 import logging
 import pytz
+import re
 import rfc3986.validators
 import rfc3986.exceptions
 import time
@@ -1316,6 +1317,17 @@ def string_list(s):
     return [p for p in [part.strip() for part in s.split(",")] if p]
 
 
+def is_c4_arn(arn: str) -> bool:
+    """
+    Returns True iff the given (presumed) AWS ARN string value looks like it
+    refers to a CGAP or Fourfront Cloudformation entity, otherwise returns False.
+    :param arn: String representing an AWS ARN.
+    :return: True if the given ARN refers to a CGAP or Fourfront Cloudformation entity, else False.
+    """
+    pattern = r"(fourfront|cgap|[:/]c4-)"
+    return True if re.search(pattern, arn) else False
+
+
 def string_md5(unicode_string):
     """
     Returns the md5 signature for the given u unicode string.

{dcicutils-6.9.9.0b4 → dcicutils-6.9.9.0b6}/dcicutils/redis_tools.py

@@ -24,29 +24,16 @@ class RedisSessionToken:
     Model used by Redis to store session tokens
     Keystore structure:
         <env_namespace>:session:<token>
-            -> Redis hset containing the associated JWT and expiration time (3 hours)
+            -> Redis str of JWT with automated expiration
     """
-    JWT = 'jwt'
-    EXPIRATION = 'expiration'
 
     @staticmethod
-    def _build_session_expiration(n_hours: int = 3) -> str:
+    def _build_session_expiration(n_hours: int = 3) -> datetime.timedelta:
         """ Builds a session expiration date 3 hours after generation
         :param n_hours: timestamp after which the session token is invalid
         :return: datetime string 3 hours from creation time
         """
-        return str(datetime.datetime.utcnow() + datetime.timedelta(hours=n_hours))
-
-    def _build_session_hset(self, jwt: str, expiration=None) -> dict:
-        """ Builds Redis hset record for the session token
-        :param jwt: encoded jwt value
-        :param expiration: expiration if using an existing one
-        :return: dictionary to be stored as a hash in Redis
-        """
-        return {
-            self.JWT: jwt,
-            self.EXPIRATION: self._build_session_expiration() if not expiration else expiration
-        }
+        return datetime.timedelta(hours=n_hours)
 
     @staticmethod
     def _build_redis_key(namespace: str, token: str) -> str:
@@ -72,23 +59,23 @@ class RedisSessionToken:
         self.namespace = namespace
         self.redis_key = self._build_redis_key(self.namespace, self.session_token)
         self.jwt = jwt
-        self.session_hset = self._build_session_hset(self.jwt, expiration=expiration)
+        self.expiration = expiration or self._build_session_expiration()
 
     def __eq__(self, other):
         """ Evaluates equality of two session objects based on the value of the session hset """
-        return self.session_hset == other.session_hset
+        return (self.redis_key == other.redis_key) and (self.jwt == other.jwt)
 
     def get_session_token(self) -> str:
         """ Extracts the session token stored on this object """
         return self.session_token
 
     def get_redis_key(self) -> str:
-        """ Returns the key under which the Redis hset is stored - note that this contains the token! """
+        """ Returns the key under which the Redis set is stored - note that this contains the token! """
         return self.redis_key
 
-    def get_expiration(self) -> str:
-        """ Returns the expiration date stored in the session hset locally """
-        return self.session_hset[self.EXPIRATION]
+    def get_expiration(self) -> datetime:
+        """ Returns the expiration timedelta """
+        return self.expiration
 
     @classmethod
     def from_redis(cls, *, redis_handler: RedisBase, namespace: str, token: str):
@@ -100,10 +87,11 @@ class RedisSessionToken:
         :return: A RedisSessionToken object built from an existing record in Redis
         """
         redis_key = f'{namespace}:session:{token}'
-        redis_entry = redis_handler.hgetall(redis_key)
+        redis_entry = redis_handler.get(redis_key)
         if redis_entry:
-            return cls(namespace=namespace, jwt=redis_entry[cls.JWT],
-                       token=token, expiration=redis_entry[cls.EXPIRATION])
+            expiration = redis_handler.ttl(redis_key)
+            return cls(namespace=namespace, jwt=redis_entry,
+                       token=token, expiration=expiration)
 
     def decode_jwt(self, audience: str, secret: str, leeway: int = 30) -> dict:
         """ Decodes JWT to grab info such as the email
@@ -121,7 +109,7 @@ class RedisSessionToken:
         :return: True if successful, raise Exception otherwise
         """
         try:
-            redis_handler.hset_multiple(self.redis_key, self.session_hset)
+            redis_handler.set(self.redis_key, self.jwt, exp=self.expiration)
         except Exception as e:
             log.error(str(e))
             raise RedisException()
@@ -132,15 +120,12 @@ class RedisSessionToken:
         :param redis_handler: handle to Redis API
         :return: True if token matches that in Redis and is not expired
         """
-        redis_token = redis_handler.hgetall(self.redis_key)
+        redis_token = redis_handler.get(self.redis_key)
         if not redis_token:
             return False  # if it doesn't exist it's not valid
-        timestamp_is_valid = (datetime.datetime.fromisoformat(
-            redis_token[self.EXPIRATION]) > datetime.datetime.utcnow()
-        )
-        return timestamp_is_valid
+        return True  # if it does exist it must be valid since we always send with TTL
 
-    def update_session_token(self, *, redis_handler: RedisBase, jwt) -> bool:
+    def update_session_token(self, *, redis_handler: RedisBase, jwt: str) -> bool:
         """ Refreshes the session token, jwt (if different) and expiration stored in Redis
         :param redis_handler: handle to Redis API
         :param jwt: jwt of user
@@ -151,13 +136,13 @@ class RedisSessionToken:
         # build new one
         self.session_token = make_session_token()
         self.redis_key = self._build_redis_key(self.namespace, self.session_token)
+        self.expiration = self._build_session_expiration()
         self.jwt = jwt
-        self.session_hset = self._build_session_hset(jwt)
         return self.store_session_token(redis_handler=redis_handler)
 
-    def delete_session_token(self, *, redis_handler) -> bool:
+    def delete_session_token(self, *, redis_handler: RedisBase) -> bool:
         """ Deletes the session token from redis, effectively logging out
         :param redis_handler: handle to Redis API
         :return: True if successful, False otherwise
         """
-        return redis_handler.delete(self.redis_key)
+        return 1 == redis_handler.delete(self.redis_key)

{dcicutils-6.9.9.0b4 → dcicutils-6.9.9.0b6}/dcicutils/redis_utils.py

@@ -1,4 +1,5 @@
 import redis
+import datetime
 from typing import Union
 # Low level utilities for working with Redis
 
@@ -59,13 +60,17 @@ class RedisBase(object):
         """
         return self.redis.info()
 
-    def set(self, key: str, value: Union[str, int, float]) -> str:
+    def set(self, key: str, value: Union[str, int, float], exp: Union[int, datetime.timedelta] = None) -> str:
         """ Sets the given key to the given value https://redis.io/commands/set/
         :param key: string to store value under
         :param value: value mapped to from key
+        :param exp: expiration time in seconds as int or datetime.timedelta (optional)
         :return: a "true" string <OK>
         """
-        return self.redis.set(self._encode_value(key), self._encode_value(value))
+        kwargs = {}
+        if exp:
+            kwargs['ex'] = exp
+        return self.redis.set(self._encode_value(key), self._encode_value(value), **kwargs)
 
     def get(self, key: str) -> str:
         """ Gets the given key from Redis https://redis.io/commands/get/
@@ -77,6 +82,21 @@ class RedisBase(object):
             val = self._decode_value(val)
         return val
 
+    def set_expiration(self, key: str, t: Union[int, datetime.time]) -> bool:
+        """ Sets the TTL of the given key manually
+        :param key: key to set TTL
+        :param t: time in seconds until expiration (datetime.timedelta or int)
+        :returns: True if successful
+        """
+        return self.redis.expire(key, t, gt=True)
+
+    def ttl(self, key: str) -> datetime.time:
+        """ Gets the TTL of the given key
+        :param key: key to get TTL for
+        :return: datetime value in seconds
+        """
+        return self.redis.ttl(key)
+
     def delete(self, key: str) -> int:
         """ Deletes the given key from Redis https://redis.io/commands/del/
         :param key: key to delete

{dcicutils-6.9.9.0b4 → dcicutils-6.9.9.0b6}/dcicutils/secrets_utils.py

@@ -117,7 +117,8 @@ def apply_overrides(*, secrets: dict, rename_keys: Optional[dict] = None,
                 raise ValueError(f"Cannot rename {key} to {new_name}"
                                  f" because {a_or_an(new_name)} attribute already exists.")
             if key not in secrets:
-                raise ValueError(f"Cannot rename {key} to {new_name} because the secrets has no {key} attribute.")
+                logger.warning(f"Cannot rename {key} to {new_name} because the secrets has no {key} attribute.")
+                continue
             secrets[new_name] = secrets.pop(key)
     if override_values:
         secrets = dict(secrets, **override_values)

{dcicutils-6.9.9.0b4 → dcicutils-6.9.9.0b6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "6.9.9.0b4"  # to become 7.0.0
+version = "6.9.9.0b6"  # to become 7.0.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

{dcicutils-6.9.9.0b4 → dcicutils-6.9.9.0b6}/setup.py

@@ -32,7 +32,7 @@ install_requires = \
 
 setup_kwargs = {
     'name': 'dcicutils',
-    'version': '6.9.9.0b4',
+    'version': '6.9.9.0b6',
     'description': 'Utility package for interacting with the 4DN Data Portal and other 4DN resources',
     'long_description': '=====\nutils\n=====\n\nCheck out our full documentation `here <https://dcic-utils.readthedocs.io/en/latest/>`_\n\nThis repository contains various utility modules shared amongst several projects in the 4DN-DCIC. It is meant to be used internally by the DCIC team and externally as a Python API to `Fourfront <https://data.4dnucleome.org>`_\\ , the 4DN data portal.\n\npip installable as the ``dcicutils`` package with: ``pip install dcicutils``\n\nSee `this document <https://dcic-utils.readthedocs.io/en/latest/getting_started.html>`_ for tips on getting started. `Go here <https://dcic-utils.readthedocs.io/en/latest/examples.html>`_ for examples of some of the most useful functions.\n\n\n.. image:: https://travis-ci.org/4dn-dcic/utils.svg?branch=master\n   :target: https://travis-ci.org/4dn-dcic/utils\n   :alt: Build Status\n\n\n.. image:: https://coveralls.io/repos/github/4dn-dcic/utils/badge.svg?branch=master\n   :target: https://coveralls.io/github/4dn-dcic/utils?branch=master\n   :alt: Coverage\n\n.. image:: https://readthedocs.org/projects/dcic-utils/badge/?version=latest\n   :target: https://dcic-utils.readthedocs.io/en/latest/?badge=latest\n   :alt: Documentation Status\n',
     'author': '4DN-DCIC Team',