hexkit 0.11.0__tar.gz → 1.0.0__tar.gz

hexkit-1.0.0/MANIFEST.in

@@ -0,0 +1 @@
+include src/hexkit/providers/s3/test_files/*.yaml

{hexkit-0.11.0/src/hexkit.egg-info → hexkit-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 0.11.0
+Version: 1.0.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0

{hexkit-0.11.0 → hexkit-1.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hexkit"
-version = "0.11.0"
+version = "1.0.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 readme = "README.md"
 authors = [

{hexkit-0.11.0 → hexkit-1.0.0}/src/hexkit/providers/akafka/provider.py

@@ -23,10 +23,12 @@ Require dependencies of the `akafka` extra. See the `setup.cfg`.
 
 import json
 import logging
+import ssl
 from contextlib import asynccontextmanager
-from typing import Any, Callable, Protocol, TypeVar
+from typing import Any, Callable, Optional, Protocol, TypeVar
 
 from aiokafka import AIOKafkaConsumer, AIOKafkaProducer
+from aiokafka.helpers import create_ssl_context
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -70,6 +72,29 @@ class KafkaConfig(BaseSettings):
         examples=[["localhost:9092"]],
         description="A list of connection strings to connect to Kafka bootstrap servers.",
     )
+    kafka_security_protocol: Literal["PLAINTEXT", "SSL"] = Field(
+        "PLAINTEXT",
+        description="Protocol used to communicate with brokers. "
+        + "Valid values are: PLAINTEXT, SSL.",
+    )
+    kafka_ssl_cafile: str = Field(
+        "",
+        description="Certificate Authority file path containing certificates"
+        + " used to sign broker certificates. If a CA not specified, the default"
+        + " system CA will be used if found by OpenSSL.",
+    )
+    kafka_ssl_certfile: str = Field(
+        "",
+        description="Optional filename of client certificate, as well as any"
+        + " CA certificates needed to establish the certificate's authenticity.",
+    )
+    kafka_ssl_keyfile: str = Field(
+        "", description="Optional filename containing the client private key."
+    )
+    kafka_ssl_password: str = Field(
+        "",
+        description="Optional password to be used for the client private key.",
+    )
 
 
 class EventTypeNotFoundError(RuntimeError):
@@ -84,13 +109,29 @@ def generate_client_id(*, service_name: str, instance_id: str) -> str:
     return f"{service_name}.{instance_id}"
 
 
+def generate_ssl_context(config: KafkaConfig) -> Optional[ssl.SSLContext]:
+    """Generate SSL context for an encrypted SSL connection to Kafka broker."""
+    return (
+        create_ssl_context(
+            cafile=config.kafka_ssl_cafile,
+            certfile=config.kafka_ssl_certfile,
+            keyfile=config.kafka_ssl_keyfile,
+            password=config.kafka_ssl_password,
+        )
+        if config.kafka_security_protocol == "SSL"
+        else None
+    )
+
+
 class KafkaProducerCompatible(Protocol):
     """A python duck type protocol describing an AIOKafkaProducer or equivalent."""
 
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         *,
         bootstrap_servers: str,
+        security_protocol: str,
+        ssl_context: Optional[ssl.SSLContext],
         client_id: str,
         key_serializer: Callable[[Any], bytes],
         value_serializer: Callable[[Any], bytes],
@@ -149,6 +190,8 @@ class KafkaEventPublisher(EventPublisherProtocol):
 
         producer = kafka_producer_cls(
             bootstrap_servers=",".join(config.kafka_servers),
+            security_protocol=config.kafka_security_protocol,
+            ssl_context=generate_ssl_context(config),
             client_id=client_id,
             key_serializer=lambda key: key.encode("ascii"),
             value_serializer=lambda event_value: json.dumps(event_value).encode(
@@ -220,6 +263,8 @@ class KafkaConsumerCompatible(Protocol):
         self,
         *topics: Ascii,
         bootstrap_servers: str,
+        security_protocol: str,
+        ssl_context: Optional[ssl.SSLContext],
         client_id: str,
         group_id: str,
         auto_offset_reset: Literal["earliest"],
@@ -297,6 +342,8 @@ class KafkaEventSubscriber(InboundProviderBase):
         consumer = kafka_consumer_cls(
             *topics,
             bootstrap_servers=",".join(config.kafka_servers),
+            security_protocol=config.kafka_security_protocol,
+            ssl_context=generate_ssl_context(config),
             client_id=client_id,
             group_id=config.service_name,
             auto_offset_reset="earliest",

hexkit-1.0.0/src/hexkit/providers/akafka/testcontainer.py

@@ -0,0 +1,177 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Improved Kafka test containers."""
+
+import tarfile
+import time
+from io import BytesIO
+from ssl import SSLError
+from textwrap import dedent
+from typing import Literal, Optional
+
+from kafka import KafkaConsumer
+from kafka.errors import KafkaError, NoBrokersAvailable, UnrecognizedBrokerVersion
+from testcontainers.core.container import DockerContainer
+from testcontainers.core.waiting_utils import wait_container_is_ready
+
+__all__ = ["KafkaSSLContainer"]
+
+DEFAULT_IMAGE = "confluentinc/cp-kafka:7.5.1"
+
+DEFAULT_PORT = 9093  # default port for the Kafka container
+BROKER_PORT = 9092  # auxiliary port for inter broker listener
+
+
+class KafkaSSLContainer(DockerContainer):
+    """Kafka container that supports SSL (or actually TLS)."""
+
+    TC_START_SCRIPT = "/tc-start.sh"
+    SECRETS_PATH = "/etc/kafka/secrets"
+
+    def __init__(  # noqa: C901, PLR0912, PLR0913
+        self,
+        image: str = DEFAULT_IMAGE,
+        port: int = DEFAULT_PORT,
+        cert: Optional[str] = None,
+        key: Optional[str] = None,
+        trusted: Optional[str] = None,
+        password: Optional[str] = None,
+        client_auth: Optional[Literal["requested", "required", "none"]] = None,
+        **kwargs,
+    ) -> None:
+        """Initialize the Kafka SSL container with the given parameters.
+
+        "cert" must contain the certificate of the broker and if needed also
+        intermediate certificates. "key" must contain the private key of the
+        broker. If it password  protected, "password" must be specified as well.
+        "trusted" must contain the trusted certificates. In "client_auth" you can
+        specify whether authentication is requested, required or not needed at all.
+        """
+        super().__init__(image, **kwargs)
+        env = self.with_env
+        self.port = port
+        ssl = bool(cert or trusted or client_auth)
+        protocol = "SSL" if ssl else "PLAINTEXT"
+        self.protocol = protocol
+        self.with_exposed_ports(port)
+        self.broker_port = DEFAULT_PORT if port == BROKER_PORT else BROKER_PORT
+        listeners = f"{protocol}://0.0.0.0:{port},BROKER://0.0.0.0:{self.broker_port}"
+        protocol_map = f"BROKER:PLAINTEXT,{protocol}:{protocol}"
+        env("KAFKA_LISTENERS", listeners)
+        env("KAFKA_INTER_BROKER_LISTENER_NAME", "BROKER")
+        env("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", protocol_map)
+        env("KAFKA_BROKER_ID", "1")
+        env("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1")
+        env("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", "1")
+        env("KAFKA_LOG_FLUSH_INTERVAL_MESSAGES", "10000000")
+        env("KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS", "0")
+        if ssl:
+            if cert:
+                cert = cert.strip().replace("\n", "\\n")
+            if key:
+                key = key.strip().replace("\n", "\\n")
+            if password:
+                password = password.strip().replace("\n", "\\n")
+            if trusted:
+                trusted = trusted.strip().replace("\n", "\\n")
+            if cert:
+                if not cert.startswith("-----BEGIN") or "CERTIFICATE" not in cert:
+                    raise ValueError("Certificate chain must be in PEM format")
+                env("KAFKA_SSL_KEYSTORE_CERTIFICATE_CHAIN", cert)
+            if key:
+                if not key.startswith("-----BEGIN") or "PRIVATE KEY" not in key:
+                    raise ValueError("Private key must be in PEM format")
+                env("KAFKA_SSL_KEYSTORE_KEY", key)
+            if cert or key:
+                env("KAFKA_SSL_KEYSTORE_TYPE", "PEM")
+            if key and password:
+                env("KAFKA_SSL_KEY_PASSWORD", password)
+            if trusted:
+                if not trusted.startswith("-----BEGIN") or "CERTIFICATE" not in trusted:
+                    raise ValueError("Trusted certificates must be in PEM format")
+                env("KAFKA_SSL_TRUSTSTORE_CERTIFICATES", trusted)
+                env("KAFKA_SSL_TRUSTSTORE_TYPE", "PEM")
+            if client_auth:
+                env("KAFKA_SSL_CLIENT_AUTH", client_auth)
+            env("KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM", " ")
+
+    def get_bootstrap_server(self) -> str:
+        """Get the Kafka bootstrap server."""
+        host = self.get_container_host_ip()
+        port = self.get_exposed_port(self.port)
+        return f"{host}:{port}"
+
+    def start(self) -> "KafkaSSLContainer":
+        """Start the Docker container."""
+        script = self.TC_START_SCRIPT
+        command = f'sh -c "while [ ! -f {script} ]; do sleep 0.1; done; sh {script}"'
+        self.with_command(command)
+        super().start()
+        self.tc_start()
+        self._connect()
+        return self
+
+    @wait_container_is_ready(
+        UnrecognizedBrokerVersion, NoBrokersAvailable, KafkaError, ValueError
+    )  # pyright: ignore
+    def _connect(self) -> None:
+        bootstrap_server = self.get_bootstrap_server()
+        try:
+            consumer = KafkaConsumer(
+                group_id="test",
+                bootstrap_servers=[bootstrap_server],
+                security_protocol=self.protocol,
+            )
+        except SSLError:
+            pass  # count this as connected
+        else:
+            if not consumer.bootstrap_connected():
+                raise KafkaError("Unable to connect with Kafka container!")
+
+    def tc_start(self) -> None:
+        """Start the test container."""
+        protocol = self.protocol
+        host = self.get_container_host_ip()
+        port = self.get_exposed_port(self.port)
+        listeners = f"{protocol}://{host}:{port},BROKER://127.0.0.1:{self.broker_port}"
+        script = f"""
+        #!/bin/bash
+        c=/etc/confluent/docker
+        . $c/bash-config
+        export KAFKA_ADVERTISED_LISTENERS={listeners}
+        export KAFKA_ZOOKEEPER_CONNECT=localhost:2181
+        p=zookeeper.properties
+        echo "clientPort=2181" > $p
+        echo "dataDir=/var/lib/zookeeper/data" >> $p
+        echo "dataLogDir=/var/lib/zookeeper/log" >> $p
+        zookeeper-server-start $p &
+        # workaround for https://github.com/confluentinc/kafka-images/issues/244
+        sed -i -E '/^if .*LISTENERS.*SSL:/,/^fi/d' $c/configure
+        $c/configure && $c/launch
+        """
+        self.create_file(dedent(script).strip().encode("utf-8"), self.TC_START_SCRIPT)
+
+    def create_file(self, content: bytes, path: str) -> None:
+        """Create a file inside the container."""
+        with BytesIO() as archive:
+            with tarfile.TarFile(fileobj=archive, mode="w") as tar:
+                tarinfo = tarfile.TarInfo(name=path)
+                tarinfo.size = len(content)
+                tarinfo.mtime = time.time()  # type: ignore
+                tar.addfile(tarinfo, BytesIO(content))
+            archive.seek(0)
+            self.get_wrapped_container().put_archive("/", archive)

{hexkit-0.11.0 → hexkit-1.0.0}/src/hexkit/providers/akafka/testutils.py

@@ -38,6 +38,7 @@ from hexkit.providers.akafka.provider import (
     KafkaEventPublisher,
     get_event_type,
 )
+from hexkit.providers.akafka.testcontainer import DEFAULT_IMAGE as KAFKA_IMAGE
 
 
 @dataclass(frozen=True)
@@ -61,7 +62,7 @@ class ExpectedEvent(EventBase):
 
 @dataclass(frozen=True)
 class RecordedEvent(EventBase):
-    """Used by the EventyRecorder class to describe events recorded in a specific topic."""
+    """Used by the EventRecorder class to describe events recorded in a specific topic."""
 
     key: Ascii
 
@@ -324,7 +325,7 @@ class EventRecorder:
         return self
 
     async def __aexit__(self, error_type, error_val, error_tb):
-        """Stop recording and check the recorded events agains the expectation when
+        """Stop recording and check the recorded events against the expectation when
         exiting the context block.
         """
         await self.stop_recording()
@@ -403,9 +404,9 @@ async def kafka_fixture_function() -> AsyncGenerator[KafkaFixture, None]:
 
     **Do not call directly** Instead, use get_kafka_fixture()
     """
-    with KafkaContainer(image="confluentinc/cp-kafka:5.4.9-1-deb8") as kafka:
+    with KafkaContainer(image=KAFKA_IMAGE) as kafka:
         kafka_servers = [kafka.get_bootstrap_server()]
-        config = KafkaConfig(
+        config = KafkaConfig(  # type: ignore
             service_name="test_publisher",
             service_instance_id="001",
             kafka_servers=kafka_servers,

hexkit-1.0.0/src/hexkit/providers/s3/test_files/test_file1.yaml

@@ -0,0 +1,3 @@
+test: |
+  A test file with some random content used for testing up-
+  and download functionality of the ObjectStorageS3 DAO implementation.

hexkit-1.0.0/src/hexkit/providers/s3/test_files/test_file2.yaml

@@ -0,0 +1,3 @@
+test: |
+  Another test file with some random content used for testing up-
+  and download functionality of the ObjectStorageS3 DAO implementation.

hexkit-1.0.0/src/hexkit/providers/s3/test_files/test_file3.yaml

@@ -0,0 +1,3 @@
+test: |
+  A third test file with some random content used for testing up-
+  and download functionality of the ObjectStorageS3 DAO implementation.

hexkit-1.0.0/src/hexkit/providers/s3/test_files/test_file4.yaml

@@ -0,0 +1,3 @@
+test: |
+  A fourth test file with some random content used for testing up-
+  and download functionality of the ObjectStorageS3 DAO implementation.

{hexkit-0.11.0 → hexkit-1.0.0/src/hexkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 0.11.0
+Version: 1.0.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0

{hexkit-0.11.0 → hexkit-1.0.0}/src/hexkit.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 LICENSE
+MANIFEST.in
 README.md
 pyproject.toml
 src/hexkit/__init__.py
@@ -21,6 +22,7 @@ src/hexkit/protocols/objstorage.py
 src/hexkit/providers/__init__.py
 src/hexkit/providers/akafka/__init__.py
 src/hexkit/providers/akafka/provider.py
+src/hexkit/providers/akafka/testcontainer.py
 src/hexkit/providers/akafka/testutils.py
 src/hexkit/providers/mongodb/__init__.py
 src/hexkit/providers/mongodb/provider.py
@@ -29,6 +31,10 @@ src/hexkit/providers/s3/__init__.py
 src/hexkit/providers/s3/provider.py
 src/hexkit/providers/s3/testutils.py
 src/hexkit/providers/s3/test_files/__init__.py
+src/hexkit/providers/s3/test_files/test_file1.yaml
+src/hexkit/providers/s3/test_files/test_file2.yaml
+src/hexkit/providers/s3/test_files/test_file3.yaml
+src/hexkit/providers/s3/test_files/test_file4.yaml
 src/hexkit/providers/testing/__init__.py
 src/hexkit/providers/testing/eventpub.py
 src/hexkit/providers/testing/utils.py