hexkit 2.2.0__tar.gz → 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {hexkit-2.2.0 → hexkit-3.0.0}/LICENSE +1 -1
  2. {hexkit-2.2.0 → hexkit-3.0.0}/MANIFEST.in +1 -0
  3. {hexkit-2.2.0/src/hexkit.egg-info → hexkit-3.0.0}/PKG-INFO +4 -3
  4. {hexkit-2.2.0 → hexkit-3.0.0}/pyproject.toml +10 -8
  5. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/__init__.py +1 -1
  6. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/__main__.py +1 -1
  7. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/base.py +1 -1
  8. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/config.py +1 -1
  9. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/correlation.py +1 -1
  10. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/custom_types.py +1 -1
  11. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/log.py +1 -1
  12. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/protocols/__init__.py +1 -1
  13. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/protocols/dao.py +1 -1
  14. hexkit-2.2.0/src/hexkit/protocols/dao_outbox.py → hexkit-3.0.0/src/hexkit/protocols/daopub.py +6 -6
  15. hexkit-3.0.0/src/hexkit/protocols/daosub.py +60 -0
  16. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/protocols/eventpub.py +1 -1
  17. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/protocols/eventsub.py +31 -11
  18. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/protocols/objstorage.py +1 -1
  19. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/__init__.py +1 -1
  20. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/akafka/__init__.py +9 -3
  21. hexkit-3.0.0/src/hexkit/providers/akafka/config.py +78 -0
  22. hexkit-3.0.0/src/hexkit/providers/akafka/provider/__init__.py +38 -0
  23. hexkit-3.0.0/src/hexkit/providers/akafka/provider/daosub.py +142 -0
  24. hexkit-3.0.0/src/hexkit/providers/akafka/provider/eventpub.py +176 -0
  25. hexkit-2.2.0/src/hexkit/providers/akafka/provider.py → hexkit-3.0.0/src/hexkit/providers/akafka/provider/eventsub.py +13 -236
  26. hexkit-3.0.0/src/hexkit/providers/akafka/provider/utils.py +46 -0
  27. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/akafka/testcontainer.py +5 -25
  28. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/akafka/testutils.py +37 -59
  29. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/mongodb/__init__.py +1 -1
  30. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/mongodb/provider.py +1 -1
  31. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/mongodb/testutils.py +1 -1
  32. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/mongokafka/__init__.py +3 -3
  33. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/mongokafka/provider.py +20 -14
  34. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/__init__.py +1 -1
  35. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/provider.py +13 -1
  36. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/__init__.py +1 -1
  37. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/testutils.py +32 -25
  38. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/testing/__init__.py +1 -1
  39. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/testing/eventpub.py +1 -1
  40. hexkit-3.0.0/src/hexkit/py.typed +1 -0
  41. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/utils.py +1 -1
  42. {hexkit-2.2.0 → hexkit-3.0.0/src/hexkit.egg-info}/PKG-INFO +4 -3
  43. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit.egg-info/SOURCES.txt +9 -2
  44. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit.egg-info/requires.txt +2 -2
  45. {hexkit-2.2.0 → hexkit-3.0.0}/README.md +0 -0
  46. {hexkit-2.2.0 → hexkit-3.0.0}/setup.cfg +0 -0
  47. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file1.yaml +0 -0
  48. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file2.yaml +0 -0
  49. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file3.yaml +0 -0
  50. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file4.yaml +0 -0
  51. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit.egg-info/dependency_links.txt +0 -0
  52. {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit.egg-info/top_level.txt +0 -0
@@ -186,7 +186,7 @@
186
186
  same "printed page" as the copyright notice for easier
187
187
  identification within third-party archives.
188
188
 
189
- Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
189
+ Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
190
190
  for the German Human Genome-Phenome Archive (GHGA)
191
191
 
192
192
  Licensed under the Apache License, Version 2.0 (the "License");
@@ -1 +1,2 @@
1
+ include src/hexkit/py.typed
1
2
  include src/hexkit/providers/s3/test_files/*.yaml
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: hexkit
3
- Version: 2.2.0
3
+ Version: 3.0.0
4
4
  Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
5
5
  Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
6
6
  License: Apache 2.0
@@ -10,6 +10,7 @@ Classifier: Operating System :: POSIX :: Linux
10
10
  Classifier: Programming Language :: Python :: 3.9
11
11
  Classifier: Programming Language :: Python :: 3.10
12
12
  Classifier: Programming Language :: Python :: 3.11
13
+ Classifier: Programming Language :: Python :: 3.12
13
14
  Classifier: License :: OSI Approved :: Apache Software License
14
15
  Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
15
16
  Classifier: Topic :: Software Development :: Libraries
@@ -21,8 +22,8 @@ Requires-Dist: pydantic<3,>=2
21
22
  Requires-Dist: pydantic_settings<3,>=2
22
23
  Requires-Dist: PyYAML<7,>=6.0
23
24
  Provides-Extra: akafka
24
- Requires-Dist: aiokafka~=0.8.0; extra == "akafka"
25
- Requires-Dist: jsonschema<5,>=4.17.3; extra == "akafka"
25
+ Requires-Dist: aiokafka~=0.10.0; extra == "akafka"
26
+ Requires-Dist: jsonschema<5,>=4.21; extra == "akafka"
26
27
  Provides-Extra: s3
27
28
  Requires-Dist: boto3<2,>=1.26.50; extra == "s3"
28
29
  Requires-Dist: botocore<2,>=1.29.50; extra == "s3"
@@ -1,6 +1,6 @@
1
1
  [build-system]
2
2
  requires = [
3
- "setuptools>=67.7.2",
3
+ "setuptools>=69",
4
4
  ]
5
5
  build-backend = "setuptools.build_meta"
6
6
 
@@ -16,13 +16,14 @@ classifiers = [
16
16
  "Programming Language :: Python :: 3.9",
17
17
  "Programming Language :: Python :: 3.10",
18
18
  "Programming Language :: Python :: 3.11",
19
+ "Programming Language :: Python :: 3.12",
19
20
  "License :: OSI Approved :: Apache Software License",
20
21
  "Topic :: Internet :: WWW/HTTP :: HTTP Servers",
21
22
  "Topic :: Software Development :: Libraries",
22
23
  "Intended Audience :: Developers",
23
24
  ]
24
25
  name = "hexkit"
25
- version = "2.2.0"
26
+ version = "3.0.0"
26
27
  description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
27
28
  dependencies = [
28
29
  "pydantic >=2, <3",
@@ -35,8 +36,8 @@ text = "Apache 2.0"
35
36
 
36
37
  [project.optional-dependencies]
37
38
  akafka = [
38
- "aiokafka~=0.8.0",
39
- "jsonschema >=4.17.3, <5",
39
+ "aiokafka~=0.10.0",
40
+ "jsonschema >=4.21, <5",
40
41
  ]
41
42
  s3 = [
42
43
  "boto3 >=1.26.50, <2",
@@ -96,8 +97,9 @@ fixable = [
96
97
  "D",
97
98
  ]
98
99
  ignore = [
99
- "E",
100
- "W",
100
+ "E111",
101
+ "E114",
102
+ "E116",
101
103
  "PLW",
102
104
  "RUF001",
103
105
  "RUF010",
@@ -170,7 +172,7 @@ check_untyped_defs = true
170
172
  no_site_packages = false
171
173
 
172
174
  [tool.pytest.ini_options]
173
- minversion = "7.1"
175
+ minversion = "8.0"
174
176
  asyncio_mode = "strict"
175
177
 
176
178
  [tool.coverage.paths]
@@ -181,4 +183,4 @@ source = [
181
183
  ]
182
184
 
183
185
  [tool.tox]
184
- legacy_tox_ini = " [tox]\n min_version = 4.0\n env_list =\n py39\n py311\n\n [testenv]\n pass_env=\n TC_HOST\n DOCKER_HOST\n deps =\n --no-deps -r ./lock/requirements-dev.txt\n commands = pytest ./tests {posargs}\n"
186
+ legacy_tox_ini = " [tox]\n env_list = py3{9,12}\n\n [gh-actions]\n python =\n 3.9: py39\n 3.10: py310\n 3.11: py311\n 3.12: py312\n\n [testenv]\n pass_env =\n TC_HOST\n DOCKER_HOST\n deps =\n --no-deps -r ./lock/requirements-dev.txt\n commands = pytest {posargs}\n"
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -34,7 +34,7 @@ from hexkit.protocols.dao import (
34
34
  )
35
35
 
36
36
 
37
- class DaoOutbox(DaoNaturalId[Dto], typing.Protocol[Dto]):
37
+ class DaoPublisher(DaoNaturalId[Dto], typing.Protocol[Dto]):
38
38
  """A Data Access Object (DAO) that automatically publishes changes according to the
39
39
  outbox pattern.
40
40
  """
@@ -50,7 +50,7 @@ class DaoOutbox(DaoNaturalId[Dto], typing.Protocol[Dto]):
50
50
  ...
51
51
 
52
52
 
53
- class DaoOutboxFactoryProtocol(DaoFactoryBase, ABC):
53
+ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
54
54
  """A protocol describing a factory to produce Data Access Objects (DAO) objects
55
55
  which automatically publish changes according to the outbox pattern.
56
56
  """
@@ -65,7 +65,7 @@ class DaoOutboxFactoryProtocol(DaoFactoryBase, ABC):
65
65
  dto_to_event: Callable[[Dto], JsonObject],
66
66
  event_topic: str,
67
67
  autopublish: bool = True,
68
- ) -> DaoOutbox[Dto]:
68
+ ) -> DaoPublisher[Dto]:
69
69
  """Constructs an Outbox DAO for interacting with resources in a database.
70
70
 
71
71
  Args:
@@ -88,7 +88,7 @@ class DaoOutboxFactoryProtocol(DaoFactoryBase, ABC):
88
88
  Whether to automatically publish changes. Defaults to True.
89
89
 
90
90
  Returns:
91
- A DAO of type DaoOutbox, which requires ID specification upon resource
91
+ A DAO of type DaoPublisher, which requires ID specification upon resource
92
92
  creation.
93
93
 
94
94
  Raises:
@@ -123,7 +123,7 @@ class DaoOutboxFactoryProtocol(DaoFactoryBase, ABC):
123
123
  dto_to_event: Callable[[Dto], JsonObject],
124
124
  event_topic: str,
125
125
  autopublish: bool,
126
- ) -> DaoOutbox[Dto]:
126
+ ) -> DaoPublisher[Dto]:
127
127
  """*To be implemented by the provider. Input validation is done outside of this
128
128
  method.*
129
129
  """
@@ -0,0 +1,60 @@
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
+ # for the German Human Genome-Phenome Archive (GHGA)
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ """A protocol for consuming events published through the DaoPublisherFactoryProtocol."""
18
+
19
+ from abc import ABC, abstractmethod
20
+ from typing import Generic, TypeVar
21
+
22
+ from pydantic import BaseModel
23
+
24
+ Dto = TypeVar("Dto", bound=BaseModel)
25
+
26
+
27
+ class DtoValidationError(ValueError):
28
+ """Raised when the payload of a received event was not formatted as expected."""
29
+
30
+
31
+ class DaoSubscriberProtocol(ABC, Generic[Dto]):
32
+ """A protocol for consuming events published through the DaoPublisherFactoryProtocol.
33
+
34
+ In addition to the methods described below, implementations shall expose the
35
+ the following attributes:
36
+ event_topic:
37
+ The name of the topic from which updates for the given resource are
38
+ consumed.
39
+ dto_model:
40
+ A pydantic model representing the data transfer object (DTO) for the
41
+ payload of changed events.
42
+
43
+ If the upstream provider using this protocol fails to convert the payload of a
44
+ change event to the expected DTO model, it should raise a DtoValidationError.
45
+ """
46
+
47
+ event_topic: str
48
+ dto_model: type[Dto]
49
+
50
+ @abstractmethod
51
+ async def changed(self, resource_id: str, update: Dto) -> None:
52
+ """Consume a change event (created or updated) for the resource with the given
53
+ ID.
54
+ """
55
+ ...
56
+
57
+ @abstractmethod
58
+ async def deleted(self, resource_id: str) -> None:
59
+ """Consume an event indicating the deletion of the resource with the given ID."""
60
+ ...
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -45,27 +45,47 @@ class EventSubscriberProtocol(ABC):
45
45
  topics_of_interest: list[Ascii]
46
46
  types_of_interest: list[Ascii]
47
47
 
48
- async def consume(self, *, payload: JsonObject, type_: Ascii, topic: Ascii) -> None:
48
+ async def consume(
49
+ self,
50
+ *,
51
+ payload: JsonObject,
52
+ type_: Ascii,
53
+ topic: Ascii,
54
+ key: Ascii,
55
+ ) -> None:
49
56
  """Receive an event of interest and process it according to its type.
50
57
 
51
58
  Args:
52
- payload (JsonObject): The data/payload to send with the event.
53
- type_ (str): The type of the event.
54
- topic (str): Name of the topic the event was published to.
59
+ payload: The data/payload to send with the event.
60
+ type_: The type of the event.
61
+ topic: Name of the topic the event was published to.
62
+ key: A key used for routing the event.
55
63
  """
56
64
  check_ascii(type_, topic)
57
- await self._consume_validated(payload=payload, type_=type_, topic=topic)
65
+
66
+ if key:
67
+ check_ascii(key)
68
+
69
+ await self._consume_validated(
70
+ payload=payload, type_=type_, topic=topic, key=key
71
+ )
58
72
 
59
73
  @abstractmethod
60
74
  async def _consume_validated(
61
- self, *, payload: JsonObject, type_: Ascii, topic: Ascii
75
+ self,
76
+ *,
77
+ payload: JsonObject,
78
+ type_: Ascii,
79
+ topic: Ascii,
80
+ key: Ascii,
62
81
  ) -> None:
63
82
  """
64
- Receive and process an event with already validated topic and type.
83
+ Receive and process an event with already validated topic, type, and key.
65
84
 
66
85
  Args:
67
- payload (JsonObject): The data/payload to send with the event.
68
- type_ (str): The type of the event.
69
- topic (str): Name of the topic the event was published to.
86
+ payload: The data/payload to send with the event.
87
+ type_: The type of the event.
88
+ topic: Name of the topic the event was published to.
89
+ key: A key used for routing the event.
70
90
  """
71
91
  ...
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,4 @@
1
- # Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
2
  # for the German Human Genome-Phenome Archive (GHGA)
3
3
  #
4
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,10 +19,16 @@ They correspond to the `EventPublisherProtocol` and `EventSubscriberProtocol`,
19
19
  respectively.
20
20
  """
21
21
 
22
+ from .config import KafkaConfig
22
23
  from .provider import (
23
- KafkaConfig,
24
24
  KafkaEventPublisher,
25
25
  KafkaEventSubscriber,
26
+ KafkaOutboxSubscriber,
26
27
  )
27
28
 
28
- __all__ = ["KafkaEventPublisher", "KafkaEventSubscriber", "KafkaConfig"]
29
+ __all__ = [
30
+ "KafkaEventPublisher",
31
+ "KafkaEventSubscriber",
32
+ "KafkaConfig",
33
+ "KafkaOutboxSubscriber",
34
+ ]
@@ -0,0 +1,78 @@
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
+ # for the German Human Genome-Phenome Archive (GHGA)
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ """Apache Kafka specific configuration."""
18
+
19
+ from typing import Literal
20
+
21
+ from pydantic import Field, SecretStr
22
+ from pydantic_settings import BaseSettings
23
+
24
+
25
+ class KafkaConfig(BaseSettings):
26
+ """Config parameters needed for connecting to Apache Kafka."""
27
+
28
+ service_name: str = Field(
29
+ ...,
30
+ examples=["my-cool-special-service"],
31
+ description="The name of the (micro-)service from which messages are published.",
32
+ )
33
+ service_instance_id: str = Field(
34
+ ...,
35
+ examples=["germany-bw-instance-001"],
36
+ description=(
37
+ "A string that uniquely identifies this instance across all instances of"
38
+ + " this service. A globally unique Kafka client ID will be created by"
39
+ + " concatenating the service_name and the service_instance_id."
40
+ ),
41
+ )
42
+ kafka_servers: list[str] = Field(
43
+ ...,
44
+ examples=[["localhost:9092"]],
45
+ description="A list of connection strings to connect to Kafka bootstrap servers.",
46
+ )
47
+ kafka_security_protocol: Literal["PLAINTEXT", "SSL"] = Field(
48
+ default="PLAINTEXT",
49
+ description="Protocol used to communicate with brokers. "
50
+ + "Valid values are: PLAINTEXT, SSL.",
51
+ )
52
+ kafka_ssl_cafile: str = Field(
53
+ default="",
54
+ description="Certificate Authority file path containing certificates"
55
+ + " used to sign broker certificates. If a CA is not specified, the default"
56
+ + " system CA will be used if found by OpenSSL.",
57
+ )
58
+ kafka_ssl_certfile: str = Field(
59
+ default="",
60
+ description="Optional filename of client certificate, as well as any"
61
+ + " CA certificates needed to establish the certificate's authenticity.",
62
+ )
63
+ kafka_ssl_keyfile: str = Field(
64
+ default="", description="Optional filename containing the client private key."
65
+ )
66
+ kafka_ssl_password: SecretStr = Field(
67
+ default="",
68
+ description="Optional password to be used for the client private key.",
69
+ )
70
+ generate_correlation_id: bool = Field(
71
+ default=True,
72
+ examples=[True, False],
73
+ description=(
74
+ "A flag, which, if False, will result in an error when trying to publish an"
75
+ + " event without a valid correlation ID set for the context. If True, the a"
76
+ + " newly correlation ID will be generated and used in the event header."
77
+ ),
78
+ )
@@ -0,0 +1,38 @@
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
+ # for the German Human Genome-Phenome Archive (GHGA)
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ """A subpackage containing all Apache Kafka-specific providers.
18
+
19
+ Require dependencies of the `akafka` extra.
20
+ """
21
+
22
+ from .daosub import KafkaOutboxSubscriber
23
+ from .eventpub import KafkaEventPublisher
24
+ from .eventsub import (
25
+ ConsumerEvent,
26
+ KafkaEventSubscriber,
27
+ get_header_value,
28
+ headers_as_dict,
29
+ )
30
+
31
+ __all__ = [
32
+ "KafkaEventPublisher",
33
+ "KafkaEventSubscriber",
34
+ "ConsumerEvent",
35
+ "get_header_value",
36
+ "headers_as_dict",
37
+ "KafkaOutboxSubscriber",
38
+ ]
@@ -0,0 +1,142 @@
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
+ # for the German Human Genome-Phenome Archive (GHGA)
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ """Apache Kafka-specific provider using implementations of the
18
+ `DaoSubscriberProtocol`.
19
+ """
20
+
21
+ import logging
22
+ from collections.abc import Sequence
23
+ from contextlib import asynccontextmanager
24
+
25
+ from aiokafka import AIOKafkaConsumer
26
+ from pydantic import ValidationError
27
+
28
+ from hexkit.base import InboundProviderBase
29
+ from hexkit.custom_types import Ascii, JsonObject
30
+ from hexkit.protocols.daosub import (
31
+ DaoSubscriberProtocol,
32
+ DtoValidationError,
33
+ )
34
+ from hexkit.protocols.eventsub import EventSubscriberProtocol
35
+ from hexkit.providers.akafka.config import KafkaConfig
36
+ from hexkit.providers.akafka.provider.eventsub import (
37
+ KafkaConsumerCompatible,
38
+ KafkaEventSubscriber,
39
+ )
40
+
41
+ CHANGE_EVENT_TYPE = "upserted"
42
+ DELETE_EVENT_TYPE = "deleted"
43
+
44
+
45
+ class TranslatorConverter(EventSubscriberProtocol):
46
+ """Takes a list of translators implementing the `DaoSubscriberProtocol` to
47
+ create a single translator implementing the `EventSubscriberProtocol`.
48
+ """
49
+
50
+ types_of_interest = [CHANGE_EVENT_TYPE, DELETE_EVENT_TYPE]
51
+
52
+ def __init__(self, *, translators: Sequence[DaoSubscriberProtocol]):
53
+ self.topics_of_interest = [translator.event_topic for translator in translators]
54
+
55
+ if len(set(self.topics_of_interest)) != len(self.topics_of_interest):
56
+ raise ValueError(
57
+ "Got multiple DaoSubscriberProtocol-compliant translators trying to"
58
+ + " consume from the same event topic."
59
+ )
60
+
61
+ self._translator_by_topic = {
62
+ translator.event_topic: translator for translator in translators
63
+ }
64
+
65
+ async def _consume_validated(
66
+ self, *, payload: JsonObject, type_: Ascii, topic: Ascii, key: Ascii
67
+ ) -> None:
68
+ """
69
+ Receive and process an event with already validated topic, type, and key.
70
+
71
+ Args:
72
+ payload: The data/payload to send with the event.
73
+ type_: The type of the event.
74
+ topic: Name of the topic the event was published to.
75
+ key: A key used for routing the event.
76
+ """
77
+ translator = self._translator_by_topic.get(topic)
78
+
79
+ if translator is None:
80
+ # This should never happen, as the topic should have been filtered out:
81
+ raise RuntimeError
82
+
83
+ if type_ == CHANGE_EVENT_TYPE:
84
+ try:
85
+ dto = translator.dto_model.model_validate(payload)
86
+ except ValidationError as error:
87
+ message = (
88
+ f"The event of type {type_} on topic {topic} was not valid wrt. the"
89
+ + " DTO model."
90
+ )
91
+ logging.error(message)
92
+ raise DtoValidationError(message) from error
93
+
94
+ await translator.changed(resource_id=key, update=dto)
95
+
96
+ else:
97
+ # a deletion event:
98
+ await translator.deleted(resource_id=key)
99
+
100
+
101
+ class KafkaOutboxSubscriber(InboundProviderBase):
102
+ """Apache Kafka-specific provider using translators that implement the
103
+ `DaoSubscriberProtocol`.
104
+ """
105
+
106
+ @classmethod
107
+ @asynccontextmanager
108
+ async def construct(
109
+ cls,
110
+ *,
111
+ config: KafkaConfig,
112
+ translators: Sequence[DaoSubscriberProtocol],
113
+ kafka_consumer_cls: type[KafkaConsumerCompatible] = AIOKafkaConsumer,
114
+ ):
115
+ """Setup and teardown an instance of the provider.
116
+
117
+ Args:
118
+ config: MongoDB-specific config parameters.
119
+
120
+ Returns:
121
+ An instance of the provider.
122
+ """
123
+ translator_converter = TranslatorConverter(translators=translators)
124
+
125
+ async with KafkaEventSubscriber.construct(
126
+ config=config,
127
+ translator=translator_converter,
128
+ kafka_consumer_cls=kafka_consumer_cls,
129
+ ) as event_subscriber:
130
+ yield cls(event_subscriber=event_subscriber)
131
+
132
+ def __init__(self, *, event_subscriber: KafkaEventSubscriber):
133
+ """Please do not call directly! Should be called by the `construct` method."""
134
+ self._event_subscriber = event_subscriber
135
+
136
+ async def run(self, forever: bool = True) -> None:
137
+ """Start consuming events from the Kafka outbox.
138
+
139
+ Args:
140
+ forever: Whether to run the consumer indefinitely. Defaults to True.
141
+ """
142
+ await self._event_subscriber.run(forever=forever)