hexkit 0.11.1__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. hexkit-1.1.0/MANIFEST.in +1 -0
  2. {hexkit-0.11.1/src/hexkit.egg-info → hexkit-1.1.0}/PKG-INFO +4 -4
  3. {hexkit-0.11.1 → hexkit-1.1.0}/pyproject.toml +4 -7
  4. hexkit-1.1.0/src/hexkit/correlation.py +97 -0
  5. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/inject.py +0 -5
  6. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/protocols/dao.py +1 -1
  7. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/protocols/objstorage.py +0 -2
  8. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/akafka/provider.py +49 -9
  9. hexkit-1.1.0/src/hexkit/providers/akafka/testcontainer.py +177 -0
  10. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/akafka/testutils.py +5 -4
  11. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/mongodb/provider.py +3 -3
  12. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/s3/provider.py +2 -8
  13. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/s3/testutils.py +0 -3
  14. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/utils.py +14 -1
  15. {hexkit-0.11.1 → hexkit-1.1.0/src/hexkit.egg-info}/PKG-INFO +4 -4
  16. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit.egg-info/SOURCES.txt +3 -0
  17. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit.egg-info/requires.txt +3 -3
  18. {hexkit-0.11.1 → hexkit-1.1.0}/LICENSE +0 -0
  19. {hexkit-0.11.1 → hexkit-1.1.0}/README.md +0 -0
  20. {hexkit-0.11.1 → hexkit-1.1.0}/setup.cfg +0 -0
  21. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/__init__.py +0 -0
  22. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/__main__.py +0 -0
  23. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/base.py +0 -0
  24. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/config.py +0 -0
  25. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/custom_types.py +0 -0
  26. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/protocols/__init__.py +0 -0
  27. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/protocols/eventpub.py +0 -0
  28. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/protocols/eventsub.py +0 -0
  29. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/__init__.py +0 -0
  30. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/akafka/__init__.py +0 -0
  31. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/mongodb/__init__.py +0 -0
  32. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/mongodb/testutils.py +0 -0
  33. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/s3/__init__.py +0 -0
  34. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/s3/test_files/__init__.py +0 -0
  35. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/s3/test_files/test_file1.yaml +0 -0
  36. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/s3/test_files/test_file2.yaml +0 -0
  37. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/s3/test_files/test_file3.yaml +0 -0
  38. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/s3/test_files/test_file4.yaml +0 -0
  39. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/testing/__init__.py +0 -0
  40. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/testing/eventpub.py +0 -0
  41. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit/providers/testing/utils.py +0 -0
  42. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit.egg-info/dependency_links.txt +0 -0
  43. {hexkit-0.11.1 → hexkit-1.1.0}/src/hexkit.egg-info/top_level.txt +0 -0
@@ -0,0 +1 @@
1
+ include src/hexkit/providers/s3/test_files/*.yaml
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: hexkit
3
- Version: 0.11.1
3
+ Version: 1.1.0
4
4
  Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
5
5
  Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
6
6
  License: Apache 2.0
@@ -31,13 +31,13 @@ Provides-Extra: mongodb
31
31
  Requires-Dist: motor<4,>=3.1.1; extra == "mongodb"
32
32
  Provides-Extra: test-akafka
33
33
  Requires-Dist: hexkit[akafka]; extra == "test-akafka"
34
- Requires-Dist: testcontainers[kafka]<4,>=3.4.1; extra == "test-akafka"
34
+ Requires-Dist: testcontainers[kafka]<4,>=3.7.1; extra == "test-akafka"
35
35
  Provides-Extra: test-s3
36
36
  Requires-Dist: hexkit[s3]; extra == "test-s3"
37
- Requires-Dist: testcontainers<4,>=3.4.1; extra == "test-s3"
37
+ Requires-Dist: testcontainers<4,>=3.7.1; extra == "test-s3"
38
38
  Provides-Extra: test-mongodb
39
39
  Requires-Dist: hexkit[mongodb]; extra == "test-mongodb"
40
- Requires-Dist: testcontainers[mongo]<4,>=3.4.1; extra == "test-mongodb"
40
+ Requires-Dist: testcontainers[mongo]<4,>=3.7.1; extra == "test-mongodb"
41
41
  Provides-Extra: test
42
42
  Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
43
43
  Provides-Extra: all
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "hexkit"
7
- version = "0.11.1"
7
+ version = "1.1.0"
8
8
  description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
9
9
  readme = "README.md"
10
10
  authors = [
@@ -35,9 +35,9 @@ akafka = ["aiokafka~=0.8.0", "jsonschema >=4.17.3, <5"]
35
35
  s3 = ["boto3 >=1.26.50, <2", "botocore >=1.29.50, <2"]
36
36
  mongodb = ["motor >=3.1.1, <4"]
37
37
 
38
- test-akafka = ["hexkit[akafka]", "testcontainers[kafka] >=3.4.1, <4"]
39
- test-s3 = ["hexkit[s3]", "testcontainers >=3.4.1, <4"]
40
- test-mongodb = ["hexkit[mongodb]", "testcontainers[mongo] >=3.4.1, <4"]
38
+ test-akafka = ["hexkit[akafka]", "testcontainers[kafka] >=3.7.1, <4"]
39
+ test-s3 = ["hexkit[s3]", "testcontainers >=3.7.1, <4"]
40
+ test-mongodb = ["hexkit[mongodb]", "testcontainers[mongo] >=3.7.1, <4"]
41
41
  test = ["hexkit[test-akafka,test-s3,test-mongodb]"]
42
42
 
43
43
  all = ["hexkit[test]"]
@@ -47,6 +47,3 @@ Repository = "https://github.com/ghga-de/hexkit"
47
47
 
48
48
  [tool.setuptools.packages.find]
49
49
  where = ["src"]
50
-
51
- [tool.setuptools.package-data]
52
- "hexkit.providers.s3.test_files" = ["*.yaml"]
@@ -0,0 +1,97 @@
1
+ # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
+ # for the German Human Genome-Phenome Archive (GHGA)
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ """Utilities related to correlation IDs"""
18
+
19
+ import logging
20
+ from contextlib import asynccontextmanager
21
+ from contextvars import ContextVar
22
+ from uuid import UUID, uuid4
23
+
24
+ from hexkit.utils import set_context_var
25
+
26
+ log = logging.getLogger()
27
+
28
+ correlation_id_var: ContextVar[str] = ContextVar("correlation_id", default="")
29
+
30
+ __all__ = [
31
+ "set_correlation_id",
32
+ "get_correlation_id",
33
+ "new_correlation_id",
34
+ "validate_correlation_id",
35
+ "CorrelationIdContextError",
36
+ "InvalidCorrelationIdError",
37
+ ]
38
+
39
+
40
+ class CorrelationIdContextError(RuntimeError):
41
+ """Raised when the correlation ID ContextVar is unexpectedly not set."""
42
+
43
+
44
+ class InvalidCorrelationIdError(RuntimeError):
45
+ """Raised when a correlation ID fails validation."""
46
+
47
+ def __init__(self, *, correlation_id: str):
48
+ message = f"Invalid correlation ID found: '{correlation_id}'"
49
+ super().__init__(message)
50
+
51
+
52
+ def new_correlation_id() -> str:
53
+ """Generates a new correlation ID."""
54
+ return str(uuid4())
55
+
56
+
57
+ def validate_correlation_id(correlation_id: str):
58
+ """Raises an error if the correlation ID is invalid.
59
+
60
+ Raises:
61
+ InvalidCorrelationIdError: If the correlation ID is empty or invalid.
62
+ """
63
+ try:
64
+ UUID(correlation_id)
65
+ except ValueError as err:
66
+ raise InvalidCorrelationIdError(correlation_id=correlation_id) from err
67
+
68
+
69
+ @asynccontextmanager
70
+ async def set_correlation_id(correlation_id: str):
71
+ """Set the correlation ID for the life of the context.
72
+
73
+ Raises:
74
+ InvalidCorrelationIdError: when the correlation ID is empty or invalid.
75
+ """
76
+ validate_correlation_id(correlation_id)
77
+
78
+ async with set_context_var(correlation_id_var, correlation_id):
79
+ log.info("Set context correlation ID to %s", correlation_id)
80
+ yield
81
+
82
+
83
+ def get_correlation_id() -> str:
84
+ """Get the correlation ID.
85
+
86
+ This should only be called when the correlation ID ContextVar is expected to be set.
87
+
88
+ Raises:
89
+ CorrelationIdContextError: when the correlation ID ContextVar is not set.
90
+ InvalidCorrelationIdError: when the correlation ID is invalid.
91
+ """
92
+ if not (correlation_id := correlation_id_var.get()):
93
+ raise CorrelationIdContextError()
94
+
95
+ validate_correlation_id(correlation_id)
96
+
97
+ return correlation_id
@@ -46,8 +46,6 @@ __all__ = [
46
46
  "Configurator",
47
47
  ]
48
48
 
49
- # pylint: disable=c-extension-no-member
50
-
51
49
 
52
50
  class NotConstructableError(TypeError):
53
51
  """Thrown when an AsyncContextConstructable was expected but not obtained."""
@@ -108,9 +106,6 @@ class AsyncConstructor(dependency_injector.providers.Resource):
108
106
 
109
107
  return resource
110
108
 
111
- # This pylint error is inherited from the dependency_injector framework, other
112
- # DI providers use the same basic signature:
113
- # pylint: disable=keyword-arg-before-vararg
114
109
  def __init__(
115
110
  self,
116
111
  provides: Optional[
@@ -450,7 +450,7 @@ class DaoFactoryProtocol(ABC):
450
450
 
451
451
  @overload
452
452
  @abstractmethod
453
- async def _get_dao( # pylint: disable=arguments-differ
453
+ async def _get_dao(
454
454
  self,
455
455
  *,
456
456
  name: str,
@@ -155,7 +155,6 @@ class ObjectStorageProtocol(ABC):
155
155
  upload_id=upload_id, bucket_id=bucket_id, object_id=object_id
156
156
  )
157
157
 
158
- # pylint: disable=too-many-arguments
159
158
  async def complete_multipart_upload(
160
159
  self,
161
160
  *,
@@ -367,7 +366,6 @@ class ObjectStorageProtocol(ABC):
367
366
  """
368
367
  ...
369
368
 
370
- # pylint: disable=too-many-arguments
371
369
  @abstractmethod
372
370
  async def _complete_multipart_upload(
373
371
  self,
@@ -23,10 +23,12 @@ Require dependencies of the `akafka` extra. See the `setup.cfg`.
23
23
 
24
24
  import json
25
25
  import logging
26
+ import ssl
26
27
  from contextlib import asynccontextmanager
27
- from typing import Any, Callable, Protocol, TypeVar
28
+ from typing import Any, Callable, Literal, Optional, Protocol, TypeVar
28
29
 
29
30
  from aiokafka import AIOKafkaConsumer, AIOKafkaProducer
31
+ from aiokafka.helpers import create_ssl_context
30
32
  from pydantic import Field
31
33
  from pydantic_settings import BaseSettings
32
34
 
@@ -35,11 +37,6 @@ from hexkit.custom_types import Ascii, JsonObject
35
37
  from hexkit.protocols.eventpub import EventPublisherProtocol
36
38
  from hexkit.protocols.eventsub import EventSubscriberProtocol
37
39
 
38
- try: # workaround for https://github.com/pydantic/pydantic/issues/5821
39
- from typing_extensions import Literal
40
- except ImportError:
41
- from typing import Literal # type: ignore
42
-
43
40
  __all__ = [
44
41
  "KafkaConfig",
45
42
  "KafkaEventPublisher",
@@ -70,6 +67,29 @@ class KafkaConfig(BaseSettings):
70
67
  examples=[["localhost:9092"]],
71
68
  description="A list of connection strings to connect to Kafka bootstrap servers.",
72
69
  )
70
+ kafka_security_protocol: Literal["PLAINTEXT", "SSL"] = Field(
71
+ "PLAINTEXT",
72
+ description="Protocol used to communicate with brokers. "
73
+ + "Valid values are: PLAINTEXT, SSL.",
74
+ )
75
+ kafka_ssl_cafile: str = Field(
76
+ "",
77
+ description="Certificate Authority file path containing certificates"
78
+ + " used to sign broker certificates. If a CA not specified, the default"
79
+ + " system CA will be used if found by OpenSSL.",
80
+ )
81
+ kafka_ssl_certfile: str = Field(
82
+ "",
83
+ description="Optional filename of client certificate, as well as any"
84
+ + " CA certificates needed to establish the certificate's authenticity.",
85
+ )
86
+ kafka_ssl_keyfile: str = Field(
87
+ "", description="Optional filename containing the client private key."
88
+ )
89
+ kafka_ssl_password: str = Field(
90
+ "",
91
+ description="Optional password to be used for the client private key.",
92
+ )
73
93
 
74
94
 
75
95
  class EventTypeNotFoundError(RuntimeError):
@@ -84,13 +104,29 @@ def generate_client_id(*, service_name: str, instance_id: str) -> str:
84
104
  return f"{service_name}.{instance_id}"
85
105
 
86
106
 
107
+ def generate_ssl_context(config: KafkaConfig) -> Optional[ssl.SSLContext]:
108
+ """Generate SSL context for an encrypted SSL connection to Kafka broker."""
109
+ return (
110
+ create_ssl_context(
111
+ cafile=config.kafka_ssl_cafile,
112
+ certfile=config.kafka_ssl_certfile,
113
+ keyfile=config.kafka_ssl_keyfile,
114
+ password=config.kafka_ssl_password,
115
+ )
116
+ if config.kafka_security_protocol == "SSL"
117
+ else None
118
+ )
119
+
120
+
87
121
  class KafkaProducerCompatible(Protocol):
88
122
  """A python duck type protocol describing an AIOKafkaProducer or equivalent."""
89
123
 
90
- def __init__(
124
+ def __init__( # noqa: PLR0913
91
125
  self,
92
126
  *,
93
127
  bootstrap_servers: str,
128
+ security_protocol: str,
129
+ ssl_context: Optional[ssl.SSLContext],
94
130
  client_id: str,
95
131
  key_serializer: Callable[[Any], bytes],
96
132
  value_serializer: Callable[[Any], bytes],
@@ -149,6 +185,8 @@ class KafkaEventPublisher(EventPublisherProtocol):
149
185
 
150
186
  producer = kafka_producer_cls(
151
187
  bootstrap_servers=",".join(config.kafka_servers),
188
+ security_protocol=config.kafka_security_protocol,
189
+ ssl_context=generate_ssl_context(config),
152
190
  client_id=client_id,
153
191
  key_serializer=lambda key: key.encode("ascii"),
154
192
  value_serializer=lambda event_value: json.dumps(event_value).encode(
@@ -220,6 +258,8 @@ class KafkaConsumerCompatible(Protocol):
220
258
  self,
221
259
  *topics: Ascii,
222
260
  bootstrap_servers: str,
261
+ security_protocol: str,
262
+ ssl_context: Optional[ssl.SSLContext],
223
263
  client_id: str,
224
264
  group_id: str,
225
265
  auto_offset_reset: Literal["earliest"],
@@ -297,6 +337,8 @@ class KafkaEventSubscriber(InboundProviderBase):
297
337
  consumer = kafka_consumer_cls(
298
338
  *topics,
299
339
  bootstrap_servers=",".join(config.kafka_servers),
340
+ security_protocol=config.kafka_security_protocol,
341
+ ssl_context=generate_ssl_context(config),
300
342
  client_id=client_id,
301
343
  group_id=config.service_name,
302
344
  auto_offset_reset="earliest",
@@ -311,8 +353,6 @@ class KafkaEventSubscriber(InboundProviderBase):
311
353
  finally:
312
354
  await consumer.stop()
313
355
 
314
- # pylint: disable=too-many-arguments
315
- # (some arguments are only used for testing)
316
356
  def __init__(
317
357
  self, *, consumer: KafkaConsumerCompatible, translator: EventSubscriberProtocol
318
358
  ):
@@ -0,0 +1,177 @@
1
+ # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
+ # for the German Human Genome-Phenome Archive (GHGA)
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ """Improved Kafka test containers."""
18
+
19
+ import tarfile
20
+ import time
21
+ from io import BytesIO
22
+ from ssl import SSLError
23
+ from textwrap import dedent
24
+ from typing import Literal, Optional
25
+
26
+ from kafka import KafkaConsumer
27
+ from kafka.errors import KafkaError, NoBrokersAvailable, UnrecognizedBrokerVersion
28
+ from testcontainers.core.container import DockerContainer
29
+ from testcontainers.core.waiting_utils import wait_container_is_ready
30
+
31
+ __all__ = ["KafkaSSLContainer"]
32
+
33
+ DEFAULT_IMAGE = "confluentinc/cp-kafka:7.5.1"
34
+
35
+ DEFAULT_PORT = 9093 # default port for the Kafka container
36
+ BROKER_PORT = 9092 # auxiliary port for inter broker listener
37
+
38
+
39
+ class KafkaSSLContainer(DockerContainer):
40
+ """Kafka container that supports SSL (or actually TLS)."""
41
+
42
+ TC_START_SCRIPT = "/tc-start.sh"
43
+ SECRETS_PATH = "/etc/kafka/secrets"
44
+
45
+ def __init__( # noqa: C901, PLR0912, PLR0913
46
+ self,
47
+ image: str = DEFAULT_IMAGE,
48
+ port: int = DEFAULT_PORT,
49
+ cert: Optional[str] = None,
50
+ key: Optional[str] = None,
51
+ trusted: Optional[str] = None,
52
+ password: Optional[str] = None,
53
+ client_auth: Optional[Literal["requested", "required", "none"]] = None,
54
+ **kwargs,
55
+ ) -> None:
56
+ """Initialize the Kafka SSL container with the given parameters.
57
+
58
+ "cert" must contain the certificate of the broker and if needed also
59
+ intermediate certificates. "key" must contain the private key of the
60
+ broker. If it password protected, "password" must be specified as well.
61
+ "trusted" must contain the trusted certificates. In "client_auth" you can
62
+ specify whether authentication is requested, required or not needed at all.
63
+ """
64
+ super().__init__(image, **kwargs)
65
+ env = self.with_env
66
+ self.port = port
67
+ ssl = bool(cert or trusted or client_auth)
68
+ protocol = "SSL" if ssl else "PLAINTEXT"
69
+ self.protocol = protocol
70
+ self.with_exposed_ports(port)
71
+ self.broker_port = DEFAULT_PORT if port == BROKER_PORT else BROKER_PORT
72
+ listeners = f"{protocol}://0.0.0.0:{port},BROKER://0.0.0.0:{self.broker_port}"
73
+ protocol_map = f"BROKER:PLAINTEXT,{protocol}:{protocol}"
74
+ env("KAFKA_LISTENERS", listeners)
75
+ env("KAFKA_INTER_BROKER_LISTENER_NAME", "BROKER")
76
+ env("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", protocol_map)
77
+ env("KAFKA_BROKER_ID", "1")
78
+ env("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1")
79
+ env("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", "1")
80
+ env("KAFKA_LOG_FLUSH_INTERVAL_MESSAGES", "10000000")
81
+ env("KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS", "0")
82
+ if ssl:
83
+ if cert:
84
+ cert = cert.strip().replace("\n", "\\n")
85
+ if key:
86
+ key = key.strip().replace("\n", "\\n")
87
+ if password:
88
+ password = password.strip().replace("\n", "\\n")
89
+ if trusted:
90
+ trusted = trusted.strip().replace("\n", "\\n")
91
+ if cert:
92
+ if not cert.startswith("-----BEGIN") or "CERTIFICATE" not in cert:
93
+ raise ValueError("Certificate chain must be in PEM format")
94
+ env("KAFKA_SSL_KEYSTORE_CERTIFICATE_CHAIN", cert)
95
+ if key:
96
+ if not key.startswith("-----BEGIN") or "PRIVATE KEY" not in key:
97
+ raise ValueError("Private key must be in PEM format")
98
+ env("KAFKA_SSL_KEYSTORE_KEY", key)
99
+ if cert or key:
100
+ env("KAFKA_SSL_KEYSTORE_TYPE", "PEM")
101
+ if key and password:
102
+ env("KAFKA_SSL_KEY_PASSWORD", password)
103
+ if trusted:
104
+ if not trusted.startswith("-----BEGIN") or "CERTIFICATE" not in trusted:
105
+ raise ValueError("Trusted certificates must be in PEM format")
106
+ env("KAFKA_SSL_TRUSTSTORE_CERTIFICATES", trusted)
107
+ env("KAFKA_SSL_TRUSTSTORE_TYPE", "PEM")
108
+ if client_auth:
109
+ env("KAFKA_SSL_CLIENT_AUTH", client_auth)
110
+ env("KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM", " ")
111
+
112
+ def get_bootstrap_server(self) -> str:
113
+ """Get the Kafka bootstrap server."""
114
+ host = self.get_container_host_ip()
115
+ port = self.get_exposed_port(self.port)
116
+ return f"{host}:{port}"
117
+
118
+ def start(self) -> "KafkaSSLContainer":
119
+ """Start the Docker container."""
120
+ script = self.TC_START_SCRIPT
121
+ command = f'sh -c "while [ ! -f {script} ]; do sleep 0.1; done; sh {script}"'
122
+ self.with_command(command)
123
+ super().start()
124
+ self.tc_start()
125
+ self._connect()
126
+ return self
127
+
128
+ @wait_container_is_ready(
129
+ UnrecognizedBrokerVersion, NoBrokersAvailable, KafkaError, ValueError
130
+ ) # pyright: ignore
131
+ def _connect(self) -> None:
132
+ bootstrap_server = self.get_bootstrap_server()
133
+ try:
134
+ consumer = KafkaConsumer(
135
+ group_id="test",
136
+ bootstrap_servers=[bootstrap_server],
137
+ security_protocol=self.protocol,
138
+ )
139
+ except SSLError:
140
+ pass # count this as connected
141
+ else:
142
+ if not consumer.bootstrap_connected():
143
+ raise KafkaError("Unable to connect with Kafka container!")
144
+
145
+ def tc_start(self) -> None:
146
+ """Start the test container."""
147
+ protocol = self.protocol
148
+ host = self.get_container_host_ip()
149
+ port = self.get_exposed_port(self.port)
150
+ listeners = f"{protocol}://{host}:{port},BROKER://127.0.0.1:{self.broker_port}"
151
+ script = f"""
152
+ #!/bin/bash
153
+ c=/etc/confluent/docker
154
+ . $c/bash-config
155
+ export KAFKA_ADVERTISED_LISTENERS={listeners}
156
+ export KAFKA_ZOOKEEPER_CONNECT=localhost:2181
157
+ p=zookeeper.properties
158
+ echo "clientPort=2181" > $p
159
+ echo "dataDir=/var/lib/zookeeper/data" >> $p
160
+ echo "dataLogDir=/var/lib/zookeeper/log" >> $p
161
+ zookeeper-server-start $p &
162
+ # workaround for https://github.com/confluentinc/kafka-images/issues/244
163
+ sed -i -E '/^if .*LISTENERS.*SSL:/,/^fi/d' $c/configure
164
+ $c/configure && $c/launch
165
+ """
166
+ self.create_file(dedent(script).strip().encode("utf-8"), self.TC_START_SCRIPT)
167
+
168
+ def create_file(self, content: bytes, path: str) -> None:
169
+ """Create a file inside the container."""
170
+ with BytesIO() as archive:
171
+ with tarfile.TarFile(fileobj=archive, mode="w") as tar:
172
+ tarinfo = tarfile.TarInfo(name=path)
173
+ tarinfo.size = len(content)
174
+ tarinfo.mtime = time.time() # type: ignore
175
+ tar.addfile(tarinfo, BytesIO(content))
176
+ archive.seek(0)
177
+ self.get_wrapped_container().put_archive("/", archive)
@@ -38,6 +38,7 @@ from hexkit.providers.akafka.provider import (
38
38
  KafkaEventPublisher,
39
39
  get_event_type,
40
40
  )
41
+ from hexkit.providers.akafka.testcontainer import DEFAULT_IMAGE as KAFKA_IMAGE
41
42
 
42
43
 
43
44
  @dataclass(frozen=True)
@@ -61,7 +62,7 @@ class ExpectedEvent(EventBase):
61
62
 
62
63
  @dataclass(frozen=True)
63
64
  class RecordedEvent(EventBase):
64
- """Used by the EventyRecorder class to describe events recorded in a specific topic."""
65
+ """Used by the EventRecorder class to describe events recorded in a specific topic."""
65
66
 
66
67
  key: Ascii
67
68
 
@@ -324,7 +325,7 @@ class EventRecorder:
324
325
  return self
325
326
 
326
327
  async def __aexit__(self, error_type, error_val, error_tb):
327
- """Stop recording and check the recorded events agains the expectation when
328
+ """Stop recording and check the recorded events against the expectation when
328
329
  exiting the context block.
329
330
  """
330
331
  await self.stop_recording()
@@ -403,9 +404,9 @@ async def kafka_fixture_function() -> AsyncGenerator[KafkaFixture, None]:
403
404
 
404
405
  **Do not call directly** Instead, use get_kafka_fixture()
405
406
  """
406
- with KafkaContainer(image="confluentinc/cp-kafka:5.4.9-1-deb8") as kafka:
407
+ with KafkaContainer(image=KAFKA_IMAGE) as kafka:
407
408
  kafka_servers = [kafka.get_bootstrap_server()]
408
- config = KafkaConfig(
409
+ config = KafkaConfig( # type: ignore
409
410
  service_name="test_publisher",
410
411
  service_instance_id="001",
411
412
  kafka_servers=kafka_servers,
@@ -111,7 +111,7 @@ class MongoDbDaoBase(ABC, Generic[Dto]):
111
111
  Raises:
112
112
  ResourceNotFoundError: when resource with the specified id_ was not found
113
113
  """
114
- document = await self._collection.find_one( # type: ignore
114
+ document = await self._collection.find_one(
115
115
  {"_id": id_},
116
116
  session=self._session,
117
117
  )
@@ -406,7 +406,7 @@ class MongoDbDaoFactory(DaoFactoryProtocol):
406
406
  self._config = config
407
407
 
408
408
  # get a database-specific client:
409
- self._client = AsyncIOMotorClient(
409
+ self._client = AsyncIOMotorClient( # type: ignore
410
410
  self._config.db_connection_str.get_secret_value()
411
411
  )
412
412
  self._db = self._client[self._config.db_name]
@@ -428,7 +428,7 @@ class MongoDbDaoFactory(DaoFactoryProtocol):
428
428
  ...
429
429
 
430
430
  @overload
431
- async def _get_dao( # pylint: disable=arguments-differ
431
+ async def _get_dao(
432
432
  self,
433
433
  *,
434
434
  name: str,
@@ -97,7 +97,6 @@ class S3Config(BaseSettings):
97
97
  description=(
98
98
  "Path to a config file for specifying more advanced S3 parameters."
99
99
  + " This should follow the format described here:"
100
- # pylint: disable=line-too-long
101
100
  + " https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#using-a-configuration-file"
102
101
  ),
103
102
  )
@@ -114,12 +113,10 @@ def read_aws_config_ini(aws_config_ini: Path) -> botocore.config.Config:
114
113
  return botocore.config.Config(**config_profile)
115
114
 
116
115
 
117
- class S3ObjectStorage(
118
- ObjectStorageProtocol
119
- ): # pylint: disable=too-many-instance-attributes
116
+ class S3ObjectStorage(ObjectStorageProtocol):
120
117
  """S3-based provider implementing the ObjectStorageProtocol."""
121
118
 
122
- def __init__( # pylint: disable=too-many-arguments
119
+ def __init__(
123
120
  self,
124
121
  *,
125
122
  config: S3Config,
@@ -548,7 +545,6 @@ class S3ObjectStorage(
548
545
  object_id=object_id,
549
546
  ) from error
550
547
 
551
- # pylint: disable=too-many-arguments
552
548
  async def _check_uploaded_parts(
553
549
  self,
554
550
  *,
@@ -629,7 +625,6 @@ class S3ObjectStorage(
629
625
  + f" first part which had only {first_part_size}.",
630
626
  )
631
627
 
632
- # pylint: disable=too-many-arguments
633
628
  async def _abort_multipart_upload(
634
629
  self,
635
630
  *,
@@ -686,7 +681,6 @@ class S3ObjectStorage(
686
681
  upload_id=upload_id, bucket_id=bucket_id, object_id=object_id
687
682
  )
688
683
 
689
- # pylint: disable=too-many-arguments
690
684
  async def _complete_multipart_upload(
691
685
  self,
692
686
  *,
@@ -200,7 +200,6 @@ def check_part_size(file_path: Path, anticipated_size: int) -> None:
200
200
  )
201
201
 
202
202
 
203
- # pylint: disable=too-many-arguments
204
203
  async def upload_part(
205
204
  storage_dao: ObjectStorageProtocol,
206
205
  upload_id: str,
@@ -220,7 +219,6 @@ async def upload_part(
220
219
  response.raise_for_status()
221
220
 
222
221
 
223
- # pylint: disable=too-many-arguments
224
222
  async def upload_part_of_size(
225
223
  storage_dao: ObjectStorageProtocol,
226
224
  upload_id: str,
@@ -372,7 +370,6 @@ async def prepare_non_completed_upload(s3_fixture_: S3Fixture):
372
370
  # This workflow is defined as a seperate function so that it can also be used
373
371
  # outside of the `tests` package e.g. to test the compliance of an S3-compatible
374
372
  # object storage implementation:
375
- # pylint: disable=too-many-arguments
376
373
  async def typical_workflow(
377
374
  storage_client: ObjectStorageProtocol,
378
375
  test_file_path: Path,
@@ -16,7 +16,9 @@
16
16
  """General utilities that don't require heavy dependencies."""
17
17
 
18
18
  from collections.abc import Collection
19
- from typing import Optional
19
+ from contextlib import asynccontextmanager
20
+ from contextvars import ContextVar
21
+ from typing import Any, Optional
20
22
 
21
23
  from pydantic import BaseModel
22
24
 
@@ -111,3 +113,14 @@ def validate_fields_in_model(
111
113
  if not fields_set.issubset(existing_fields_set):
112
114
  unexpected_fields = fields_set.difference(existing_fields_set)
113
115
  raise FieldNotInModelError(model=model, unexpected_field=unexpected_fields)
116
+
117
+
118
+ @asynccontextmanager
119
+ async def set_context_var(context_var: ContextVar, value: Any):
120
+ """An async context manager to simplify the use of ContextVars.
121
+
122
+ The value will be reset upon exiting the context.
123
+ """
124
+ token = context_var.set(value)
125
+ yield
126
+ context_var.reset(token)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: hexkit
3
- Version: 0.11.1
3
+ Version: 1.1.0
4
4
  Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
5
5
  Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
6
6
  License: Apache 2.0
@@ -31,13 +31,13 @@ Provides-Extra: mongodb
31
31
  Requires-Dist: motor<4,>=3.1.1; extra == "mongodb"
32
32
  Provides-Extra: test-akafka
33
33
  Requires-Dist: hexkit[akafka]; extra == "test-akafka"
34
- Requires-Dist: testcontainers[kafka]<4,>=3.4.1; extra == "test-akafka"
34
+ Requires-Dist: testcontainers[kafka]<4,>=3.7.1; extra == "test-akafka"
35
35
  Provides-Extra: test-s3
36
36
  Requires-Dist: hexkit[s3]; extra == "test-s3"
37
- Requires-Dist: testcontainers<4,>=3.4.1; extra == "test-s3"
37
+ Requires-Dist: testcontainers<4,>=3.7.1; extra == "test-s3"
38
38
  Provides-Extra: test-mongodb
39
39
  Requires-Dist: hexkit[mongodb]; extra == "test-mongodb"
40
- Requires-Dist: testcontainers[mongo]<4,>=3.4.1; extra == "test-mongodb"
40
+ Requires-Dist: testcontainers[mongo]<4,>=3.7.1; extra == "test-mongodb"
41
41
  Provides-Extra: test
42
42
  Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
43
43
  Provides-Extra: all
@@ -1,10 +1,12 @@
1
1
  LICENSE
2
+ MANIFEST.in
2
3
  README.md
3
4
  pyproject.toml
4
5
  src/hexkit/__init__.py
5
6
  src/hexkit/__main__.py
6
7
  src/hexkit/base.py
7
8
  src/hexkit/config.py
9
+ src/hexkit/correlation.py
8
10
  src/hexkit/custom_types.py
9
11
  src/hexkit/inject.py
10
12
  src/hexkit/utils.py
@@ -21,6 +23,7 @@ src/hexkit/protocols/objstorage.py
21
23
  src/hexkit/providers/__init__.py
22
24
  src/hexkit/providers/akafka/__init__.py
23
25
  src/hexkit/providers/akafka/provider.py
26
+ src/hexkit/providers/akafka/testcontainer.py
24
27
  src/hexkit/providers/akafka/testutils.py
25
28
  src/hexkit/providers/mongodb/__init__.py
26
29
  src/hexkit/providers/mongodb/provider.py
@@ -22,12 +22,12 @@ hexkit[test-akafka,test-mongodb,test-s3]
22
22
 
23
23
  [test-akafka]
24
24
  hexkit[akafka]
25
- testcontainers[kafka]<4,>=3.4.1
25
+ testcontainers[kafka]<4,>=3.7.1
26
26
 
27
27
  [test-mongodb]
28
28
  hexkit[mongodb]
29
- testcontainers[mongo]<4,>=3.4.1
29
+ testcontainers[mongo]<4,>=3.7.1
30
30
 
31
31
  [test-s3]
32
32
  hexkit[s3]
33
- testcontainers<4,>=3.4.1
33
+ testcontainers<4,>=3.7.1
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes