diaspora-event-sdk 0.2.5__tar.gz → 0.2.7__tar.gz

{diaspora-event-sdk-0.2.5/diaspora_event_sdk.egg-info → diaspora-event-sdk-0.2.7}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: diaspora-event-sdk
-Version: 0.2.5
+Version: 0.2.7
 Summary: SDK of Diaspora Event Fabric: Resilience-enabling services for science from HPC to edge
 Home-page: https://github.com/globus-labs/diaspora-event-sdk
-License: LICENSE
+License: Apache 2.0
 Platform: UNKNOWN
 Description-Content-Type: text/markdown
 Provides-Extra: kafka-python

diaspora-event-sdk-0.2.7/diaspora_event_sdk/sdk/aws_iam_msk.py

@@ -0,0 +1,119 @@
+#  Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#  SPDX-License-Identifier: Apache-2.0
+
+import base64
+# import logging
+from datetime import datetime, timezone
+from urllib.parse import parse_qs, urlparse
+
+# import boto3
+# import botocore.session
+# import pkg_resources
+from .botocore.auth import SigV4QueryAuth
+from .botocore.awsrequest import AWSRequest
+# from botocore.config import Config
+from .botocore.credentials import Credentials
+
+ENDPOINT_URL_TEMPLATE = "https://kafka.{}.amazonaws.com/"
+DEFAULT_TOKEN_EXPIRY_SECONDS = 900
+DEFAULT_STS_SESSION_NAME = "MSKSASLDefaultSession"
+ACTION_TYPE = "Action"
+ACTION_NAME = "kafka-cluster:Connect"
+SIGNING_NAME = "kafka-cluster"
+USER_AGENT_KEY = "User-Agent"
+LIB_NAME = "aws-msk-iam-sasl-signer-python"
+
+
+def __get_user_agent__():
+    return (f"{LIB_NAME}/1.0.1")
+
+
+def __get_expiration_time_ms(request):
+    """
+    Private function that parses the url and gets the expiration time
+
+    Args: request (AWSRequest): The signed aws request object
+    """
+    # Parse the signed request
+    parsed_url = urlparse(request.url)
+    parsed_ul_params = parse_qs(parsed_url.query)
+    parsed_signing_time = datetime.strptime(parsed_ul_params['X-Amz-Date'][0],
+                                            "%Y%m%dT%H%M%SZ")
+
+    # Make the datetime object timezone-aware
+    signing_time = parsed_signing_time.replace(tzinfo=timezone.utc)
+
+    # Convert the Unix timestamp to milliseconds
+    expiration_timestamp_seconds = int(
+        signing_time.timestamp()) + DEFAULT_TOKEN_EXPIRY_SECONDS
+
+    # Get lifetime of token
+    expiration_timestamp_ms = expiration_timestamp_seconds * 1000
+
+    return expiration_timestamp_ms
+
+
+def __construct_auth_token(region, aws_credentials):
+    """
+    Private function that constructs the authorization token using IAM
+    Credentials.
+
+    Args: region (str): The AWS region where the cluster is located.
+    aws_credentials (dict): The credentials to be used to generate signed
+    url. Returns: str: A base64-encoded authorization token.
+    """
+    # Validate credentials are not empty
+    if not aws_credentials.access_key or not aws_credentials.secret_key:
+        raise ValueError("AWS Credentials can not be empty")
+
+    # Extract endpoint URL
+    endpoint_url = ENDPOINT_URL_TEMPLATE.format(region)
+
+    # Set up resource path and query parameters
+    query_params = {ACTION_TYPE: ACTION_NAME}
+
+    # Create SigV4 instance
+    sig_v4 = SigV4QueryAuth(
+        aws_credentials, SIGNING_NAME, region,
+        expires=DEFAULT_TOKEN_EXPIRY_SECONDS
+    )
+
+    # Create request with url and parameters
+    request = AWSRequest(method="GET", url=endpoint_url, params=query_params)
+
+    # Add auth to the request and prepare the request
+    sig_v4.add_auth(request)
+    query_params = {USER_AGENT_KEY: __get_user_agent__()}
+    request.params = query_params
+    prepped = request.prepare()
+
+    # Get the signed url
+    signed_url = prepped.url
+
+    # Base 64 encode and remove the padding from the end
+    signed_url_bytes = signed_url.encode("utf-8")
+    base64_bytes = base64.urlsafe_b64encode(signed_url_bytes)
+    base64_encoded_signed_url = base64_bytes.decode("utf-8").rstrip("=")
+    return base64_encoded_signed_url, __get_expiration_time_ms(request)
+
+
+def generate_auth_token(region, aws_debug_creds=False):
+    """
+    Generates an base64-encoded signed url as auth token to authenticate
+    with an Amazon MSK cluster using default IAM credentials.
+
+    Args:
+        region (str): The AWS region where the cluster is located.
+    Returns:
+        str: A base64-encoded authorization token.
+    """
+
+    # Load credentials
+    import os
+    assert os.environ["AWS_ACCESS_KEY_ID"]
+    assert os.environ["AWS_SECRET_ACCESS_KEY"]
+
+    aws_credentials = Credentials(os.environ["AWS_ACCESS_KEY_ID"],
+                                  os.environ["AWS_SECRET_ACCESS_KEY"])
+
+    return __construct_auth_token(region, aws_credentials)

diaspora-event-sdk-0.2.7/diaspora_event_sdk/sdk/botocore/auth.py

@@ -0,0 +1,472 @@
+# Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
+# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import calendar
+import datetime
+import functools
+import hmac
+import json
+import logging
+from collections.abc import Mapping
+from email.utils import formatdate
+from hashlib import sha1, sha256
+
+from .compat import (
+    HTTPHeaders,
+    ensure_unicode,
+    parse_qs,
+    quote,
+    urlsplit,
+    urlunsplit,
+)
+from .exceptions import NoCredentialsError
+from .utils import (
+    is_valid_ipv6_endpoint_url,
+    normalize_url_path,
+    percent_encode_sequence,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+EMPTY_SHA256_HASH = (
+    'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
+)
+# This is the buffer size used when calculating sha256 checksums.
+# Experimenting with various buffer sizes showed that this value generally
+# gave the best result (in terms of performance).
+PAYLOAD_BUFFER = 1024 * 1024
+ISO8601 = '%Y-%m-%dT%H:%M:%SZ'
+SIGV4_TIMESTAMP = '%Y%m%dT%H%M%SZ'
+SIGNED_HEADERS_BLACKLIST = [
+    'expect',
+    'user-agent',
+    'x-amzn-trace-id',
+]
+UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD'
+STREAMING_UNSIGNED_PAYLOAD_TRAILER = 'STREAMING-UNSIGNED-PAYLOAD-TRAILER'
+
+
+def _host_from_url(url):
+    # Given URL, derive value for host header. Ensure that value:
+    # 1) is lowercase
+    # 2) excludes port, if it was the default port
+    # 3) excludes userinfo
+    url_parts = urlsplit(url)
+    host = url_parts.hostname  # urlsplit's hostname is always lowercase
+    if is_valid_ipv6_endpoint_url(url):
+        host = f'[{host}]'
+    default_ports = {
+        'http': 80,
+        'https': 443,
+    }
+    if url_parts.port is not None:
+        if url_parts.port != default_ports.get(url_parts.scheme):
+            host = '%s:%d' % (host, url_parts.port)
+    return host
+
+
+def _get_body_as_dict(request):
+    # For query services, request.data is form-encoded and is already a
+    # dict, but for other services such as rest-json it could be a json
+    # string or bytes. In those cases we attempt to load the data as a
+    # dict.
+    data = request.data
+    if isinstance(data, bytes):
+        data = json.loads(data.decode('utf-8'))
+    elif isinstance(data, str):
+        data = json.loads(data)
+    return data
+
+
+class BaseSigner:
+    REQUIRES_REGION = False
+    REQUIRES_TOKEN = False
+
+    def add_auth(self, request):
+        raise NotImplementedError("add_auth")
+
+
+class SigV4Auth(BaseSigner):
+    """
+    Sign a request with Signature V4.
+    """
+
+    REQUIRES_REGION = True
+
+    def __init__(self, credentials, service_name, region_name):
+        self.credentials = credentials
+        # We initialize these value here so the unit tests can have
+        # valid values.  But these will get overriden in ``add_auth``
+        # later for real requests.
+        self._region_name = region_name
+        self._service_name = service_name
+
+    def _sign(self, key, msg, hex=False):
+        if hex:
+            sig = hmac.new(key, msg.encode('utf-8'), sha256).hexdigest()
+        else:
+            sig = hmac.new(key, msg.encode('utf-8'), sha256).digest()
+        return sig
+
+    def headers_to_sign(self, request):
+        """
+        Select the headers from the request that need to be included
+        in the StringToSign.
+        """
+        header_map = HTTPHeaders()
+        for name, value in request.headers.items():
+            lname = name.lower()
+            if lname not in SIGNED_HEADERS_BLACKLIST:
+                header_map[lname] = value
+        if 'host' not in header_map:
+            # TODO: We should set the host ourselves, instead of relying on our
+            # HTTP client to set it for us.
+            header_map['host'] = _host_from_url(request.url)
+        return header_map
+
+    def canonical_query_string(self, request):
+        # The query string can come from two parts.  One is the
+        # params attribute of the request.  The other is from the request
+        # url (in which case we have to re-split the url into its components
+        # and parse out the query string component).
+        if request.params:
+            return self._canonical_query_string_params(request.params)
+        else:
+            return self._canonical_query_string_url(urlsplit(request.url))
+
+    def _canonical_query_string_params(self, params):
+        # [(key, value), (key2, value2)]
+        key_val_pairs = []
+        if isinstance(params, Mapping):
+            params = params.items()
+        for key, value in params:
+            key_val_pairs.append(
+                (quote(key, safe='-_.~'), quote(str(value), safe='-_.~'))
+            )
+        sorted_key_vals = []
+        # Sort by the URI-encoded key names, and in the case of
+        # repeated keys, then sort by the value.
+        for key, value in sorted(key_val_pairs):
+            sorted_key_vals.append(f'{key}={value}')
+        canonical_query_string = '&'.join(sorted_key_vals)
+        return canonical_query_string
+
+    def _canonical_query_string_url(self, parts):
+        canonical_query_string = ''
+        if parts.query:
+            # [(key, value), (key2, value2)]
+            key_val_pairs = []
+            for pair in parts.query.split('&'):
+                key, _, value = pair.partition('=')
+                key_val_pairs.append((key, value))
+            sorted_key_vals = []
+            # Sort by the URI-encoded key names, and in the case of
+            # repeated keys, then sort by the value.
+            for key, value in sorted(key_val_pairs):
+                sorted_key_vals.append(f'{key}={value}')
+            canonical_query_string = '&'.join(sorted_key_vals)
+        return canonical_query_string
+
+    def canonical_headers(self, headers_to_sign):
+        """
+        Return the headers that need to be included in the StringToSign
+        in their canonical form by converting all header keys to lower
+        case, sorting them in alphabetical order and then joining
+        them into a string, separated by newlines.
+        """
+        headers = []
+        sorted_header_names = sorted(set(headers_to_sign))
+        for key in sorted_header_names:
+            value = ','.join(
+                self._header_value(v) for v in headers_to_sign.get_all(key)
+            )
+            headers.append(f'{key}:{ensure_unicode(value)}')
+        return '\n'.join(headers)
+
+    def _header_value(self, value):
+        # From the sigv4 docs:
+        # Lowercase(HeaderName) + ':' + Trimall(HeaderValue)
+        #
+        # The Trimall function removes excess white space before and after
+        # values, and converts sequential spaces to a single space.
+        return ' '.join(value.split())
+
+    def signed_headers(self, headers_to_sign):
+        headers = sorted(n.lower().strip() for n in set(headers_to_sign))
+        return ';'.join(headers)
+
+    def _is_streaming_checksum_payload(self, request):
+        checksum_context = request.context.get('checksum', {})
+        algorithm = checksum_context.get('request_algorithm')
+        return isinstance(algorithm, dict) and algorithm.get('in') == 'trailer'
+
+    def payload(self, request):
+        if self._is_streaming_checksum_payload(request):
+            return STREAMING_UNSIGNED_PAYLOAD_TRAILER
+        elif not self._should_sha256_sign_payload(request):
+            # When payload signing is disabled, we use this static string in
+            # place of the payload checksum.
+            return UNSIGNED_PAYLOAD
+        request_body = request.body
+        if request_body and hasattr(request_body, 'seek'):
+            position = request_body.tell()
+            read_chunksize = functools.partial(
+                request_body.read, PAYLOAD_BUFFER
+            )
+            checksum = sha256()
+            for chunk in iter(read_chunksize, b''):
+                checksum.update(chunk)
+            hex_checksum = checksum.hexdigest()
+            request_body.seek(position)
+            return hex_checksum
+        elif request_body:
+            # The request serialization has ensured that
+            # request.body is a bytes() type.
+            return sha256(request_body).hexdigest()
+        else:
+            return EMPTY_SHA256_HASH
+
+    def _should_sha256_sign_payload(self, request):
+        # Payloads will always be signed over insecure connections.
+        if not request.url.startswith('https'):
+            return True
+
+        # Certain operations may have payload signing disabled by default.
+        # Since we don't have access to the operation model, we pass in this
+        # bit of metadata through the request context.
+        return request.context.get('payload_signing_enabled', True)
+
+    def canonical_request(self, request):
+        cr = [request.method.upper()]
+        path = self._normalize_url_path(urlsplit(request.url).path)
+        cr.append(path)
+        cr.append(self.canonical_query_string(request))
+        headers_to_sign = self.headers_to_sign(request)
+        cr.append(self.canonical_headers(headers_to_sign) + '\n')
+        cr.append(self.signed_headers(headers_to_sign))
+        if 'X-Amz-Content-SHA256' in request.headers:
+            body_checksum = request.headers['X-Amz-Content-SHA256']
+        else:
+            body_checksum = self.payload(request)
+        cr.append(body_checksum)
+        return '\n'.join(cr)
+
+    def _normalize_url_path(self, path):
+        normalized_path = quote(normalize_url_path(path), safe='/~')
+        return normalized_path
+
+    def scope(self, request):
+        scope = [self.credentials.access_key]
+        scope.append(request.context['timestamp'][0:8])
+        scope.append(self._region_name)
+        scope.append(self._service_name)
+        scope.append('aws4_request')
+        return '/'.join(scope)
+
+    def credential_scope(self, request):
+        scope = []
+        scope.append(request.context['timestamp'][0:8])
+        scope.append(self._region_name)
+        scope.append(self._service_name)
+        scope.append('aws4_request')
+        return '/'.join(scope)
+
+    def string_to_sign(self, request, canonical_request):
+        """
+        Return the canonical StringToSign as well as a dict
+        containing the original version of all headers that
+        were included in the StringToSign.
+        """
+        sts = ['AWS4-HMAC-SHA256']
+        sts.append(request.context['timestamp'])
+        sts.append(self.credential_scope(request))
+        sts.append(sha256(canonical_request.encode('utf-8')).hexdigest())
+        return '\n'.join(sts)
+
+    def signature(self, string_to_sign, request):
+        key = self.credentials.secret_key
+        k_date = self._sign(
+            (f"AWS4{key}").encode(), request.context["timestamp"][0:8]
+        )
+        k_region = self._sign(k_date, self._region_name)
+        k_service = self._sign(k_region, self._service_name)
+        k_signing = self._sign(k_service, 'aws4_request')
+        return self._sign(k_signing, string_to_sign, hex=True)
+
+    def add_auth(self, request):
+        if self.credentials is None:
+            raise NoCredentialsError()
+        datetime_now = datetime.datetime.utcnow()
+        request.context['timestamp'] = datetime_now.strftime(SIGV4_TIMESTAMP)
+        # This could be a retry.  Make sure the previous
+        # authorization header is removed first.
+        self._modify_request_before_signing(request)
+        canonical_request = self.canonical_request(request)
+        logger.debug("Calculating signature using v4 auth.")
+        logger.debug('CanonicalRequest:\n%s', canonical_request)
+        string_to_sign = self.string_to_sign(request, canonical_request)
+        logger.debug('StringToSign:\n%s', string_to_sign)
+        signature = self.signature(string_to_sign, request)
+        logger.debug('Signature:\n%s', signature)
+
+        self._inject_signature_to_request(request, signature)
+
+    def _inject_signature_to_request(self, request, signature):
+        auth_str = ['AWS4-HMAC-SHA256 Credential=%s' % self.scope(request)]
+        headers_to_sign = self.headers_to_sign(request)
+        auth_str.append(
+            f"SignedHeaders={self.signed_headers(headers_to_sign)}"
+        )
+        auth_str.append('Signature=%s' % signature)
+        request.headers['Authorization'] = ', '.join(auth_str)
+        return request
+
+    def _modify_request_before_signing(self, request):
+        if 'Authorization' in request.headers:
+            del request.headers['Authorization']
+        self._set_necessary_date_headers(request)
+        if self.credentials.token:
+            if 'X-Amz-Security-Token' in request.headers:
+                del request.headers['X-Amz-Security-Token']
+            request.headers['X-Amz-Security-Token'] = self.credentials.token
+
+        if not request.context.get('payload_signing_enabled', True):
+            if 'X-Amz-Content-SHA256' in request.headers:
+                del request.headers['X-Amz-Content-SHA256']
+            request.headers['X-Amz-Content-SHA256'] = UNSIGNED_PAYLOAD
+
+    def _set_necessary_date_headers(self, request):
+        # The spec allows for either the Date _or_ the X-Amz-Date value to be
+        # used so we check both.  If there's a Date header, we use the date
+        # header.  Otherwise we use the X-Amz-Date header.
+        if 'Date' in request.headers:
+            del request.headers['Date']
+            datetime_timestamp = datetime.datetime.strptime(
+                request.context['timestamp'], SIGV4_TIMESTAMP
+            )
+            request.headers['Date'] = formatdate(
+                int(calendar.timegm(datetime_timestamp.timetuple()))
+            )
+            if 'X-Amz-Date' in request.headers:
+                del request.headers['X-Amz-Date']
+        else:
+            if 'X-Amz-Date' in request.headers:
+                del request.headers['X-Amz-Date']
+            request.headers['X-Amz-Date'] = request.context['timestamp']
+
+
+class SigV4QueryAuth(SigV4Auth):
+    DEFAULT_EXPIRES = 3600
+
+    def __init__(
+        self, credentials, service_name, region_name, expires=DEFAULT_EXPIRES
+    ):
+        super().__init__(credentials, service_name, region_name)
+        self._expires = expires
+
+    def _modify_request_before_signing(self, request):
+        # We automatically set this header, so if it's the auto-set value we
+        # want to get rid of it since it doesn't make sense for presigned urls.
+        content_type = request.headers.get('content-type')
+        blacklisted_content_type = (
+            'application/x-www-form-urlencoded; charset=utf-8'
+        )
+        if content_type == blacklisted_content_type:
+            del request.headers['content-type']
+
+        # Note that we're not including X-Amz-Signature.
+        # From the docs: "The Canonical Query String must include all the query
+        # parameters from the preceding table except for X-Amz-Signature.
+        signed_headers = self.signed_headers(self.headers_to_sign(request))
+
+        auth_params = {
+            'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
+            'X-Amz-Credential': self.scope(request),
+            'X-Amz-Date': request.context['timestamp'],
+            'X-Amz-Expires': self._expires,
+            'X-Amz-SignedHeaders': signed_headers,
+        }
+        if self.credentials.token is not None:
+            auth_params['X-Amz-Security-Token'] = self.credentials.token
+        # Now parse the original query string to a dict, inject our new query
+        # params, and serialize back to a query string.
+        url_parts = urlsplit(request.url)
+        # parse_qs makes each value a list, but in our case we know we won't
+        # have repeated keys so we know we have single element lists which we
+        # can convert back to scalar values.
+        query_string_parts = parse_qs(url_parts.query, keep_blank_values=True)
+        query_dict = {k: v[0] for k, v in query_string_parts.items()}
+
+        if request.params:
+            query_dict.update(request.params)
+            request.params = {}
+        # The spec is particular about this.  It *has* to be:
+        # https://<endpoint>?<operation params>&<auth params>
+        # You can't mix the two types of params together, i.e just keep doing
+        # new_query_params.update(op_params)
+        # new_query_params.update(auth_params)
+        # percent_encode_sequence(new_query_params)
+        operation_params = ''
+        if request.data:
+            # We also need to move the body params into the query string. To
+            # do this, we first have to convert it to a dict.
+            query_dict.update(_get_body_as_dict(request))
+            request.data = ''
+        if query_dict:
+            operation_params = percent_encode_sequence(query_dict) + '&'
+        new_query_string = (
+            f"{operation_params}{percent_encode_sequence(auth_params)}"
+        )
+        # url_parts is a tuple (and therefore immutable) so we need to create
+        # a new url_parts with the new query string.
+        # <part>   - <index>
+        # scheme   - 0
+        # netloc   - 1
+        # path     - 2
+        # query    - 3  <-- we're replacing this.
+        # fragment - 4
+        p = url_parts
+        new_url_parts = (p[0], p[1], p[2], new_query_string, p[4])
+        request.url = urlunsplit(new_url_parts)
+
+    def _inject_signature_to_request(self, request, signature):
+        # Rather than calculating an "Authorization" header, for the query
+        # param quth, we just append an 'X-Amz-Signature' param to the end
+        # of the query string.
+        request.url += '&X-Amz-Signature=%s' % signature
+
+
+class S3SigV4QueryAuth(SigV4QueryAuth):
+    """S3 SigV4 auth using query parameters.
+
+    This signer will sign a request using query parameters and signature
+    version 4, i.e a "presigned url" signer.
+
+    Based off of:
+
+    http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+
+    """
+
+    def _normalize_url_path(self, path):
+        # For S3, we do not normalize the path.
+        return path
+
+    def payload(self, request):
+        # From the doc link above:
+        # "You don't include a payload hash in the Canonical Request, because
+        # when you create a presigned URL, you don't know anything about the
+        # payload. Instead, you use a constant string "UNSIGNED-PAYLOAD".
+        return UNSIGNED_PAYLOAD