hexkit 2.2.0__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hexkit-2.2.0 → hexkit-3.0.0}/LICENSE +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/MANIFEST.in +1 -0
- {hexkit-2.2.0/src/hexkit.egg-info → hexkit-3.0.0}/PKG-INFO +4 -3
- {hexkit-2.2.0 → hexkit-3.0.0}/pyproject.toml +10 -8
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/__init__.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/__main__.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/base.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/config.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/correlation.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/custom_types.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/log.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/protocols/__init__.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/protocols/dao.py +1 -1
- hexkit-2.2.0/src/hexkit/protocols/dao_outbox.py → hexkit-3.0.0/src/hexkit/protocols/daopub.py +6 -6
- hexkit-3.0.0/src/hexkit/protocols/daosub.py +60 -0
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/protocols/eventpub.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/protocols/eventsub.py +31 -11
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/protocols/objstorage.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/__init__.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/akafka/__init__.py +9 -3
- hexkit-3.0.0/src/hexkit/providers/akafka/config.py +78 -0
- hexkit-3.0.0/src/hexkit/providers/akafka/provider/__init__.py +38 -0
- hexkit-3.0.0/src/hexkit/providers/akafka/provider/daosub.py +142 -0
- hexkit-3.0.0/src/hexkit/providers/akafka/provider/eventpub.py +176 -0
- hexkit-2.2.0/src/hexkit/providers/akafka/provider.py → hexkit-3.0.0/src/hexkit/providers/akafka/provider/eventsub.py +13 -236
- hexkit-3.0.0/src/hexkit/providers/akafka/provider/utils.py +46 -0
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/akafka/testcontainer.py +5 -25
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/akafka/testutils.py +37 -59
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/mongodb/__init__.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/mongodb/provider.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/mongodb/testutils.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/mongokafka/__init__.py +3 -3
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/mongokafka/provider.py +20 -14
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/__init__.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/provider.py +13 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/__init__.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/testutils.py +32 -25
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/testing/__init__.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/testing/eventpub.py +1 -1
- hexkit-3.0.0/src/hexkit/py.typed +1 -0
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/utils.py +1 -1
- {hexkit-2.2.0 → hexkit-3.0.0/src/hexkit.egg-info}/PKG-INFO +4 -3
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit.egg-info/SOURCES.txt +9 -2
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit.egg-info/requires.txt +2 -2
- {hexkit-2.2.0 → hexkit-3.0.0}/README.md +0 -0
- {hexkit-2.2.0 → hexkit-3.0.0}/setup.cfg +0 -0
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file1.yaml +0 -0
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file2.yaml +0 -0
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file3.yaml +0 -0
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file4.yaml +0 -0
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit.egg-info/dependency_links.txt +0 -0
- {hexkit-2.2.0 → hexkit-3.0.0}/src/hexkit.egg-info/top_level.txt +0 -0
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright 2021 -
|
|
189
|
+
Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
190
190
|
for the German Human Genome-Phenome Archive (GHGA)
|
|
191
191
|
|
|
192
192
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: hexkit
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
|
|
5
5
|
Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
|
|
6
6
|
License: Apache 2.0
|
|
@@ -10,6 +10,7 @@ Classifier: Operating System :: POSIX :: Linux
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
14
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
15
|
Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
|
|
15
16
|
Classifier: Topic :: Software Development :: Libraries
|
|
@@ -21,8 +22,8 @@ Requires-Dist: pydantic<3,>=2
|
|
|
21
22
|
Requires-Dist: pydantic_settings<3,>=2
|
|
22
23
|
Requires-Dist: PyYAML<7,>=6.0
|
|
23
24
|
Provides-Extra: akafka
|
|
24
|
-
Requires-Dist: aiokafka~=0.
|
|
25
|
-
Requires-Dist: jsonschema<5,>=4.
|
|
25
|
+
Requires-Dist: aiokafka~=0.10.0; extra == "akafka"
|
|
26
|
+
Requires-Dist: jsonschema<5,>=4.21; extra == "akafka"
|
|
26
27
|
Provides-Extra: s3
|
|
27
28
|
Requires-Dist: boto3<2,>=1.26.50; extra == "s3"
|
|
28
29
|
Requires-Dist: botocore<2,>=1.29.50; extra == "s3"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[build-system]
|
|
2
2
|
requires = [
|
|
3
|
-
"setuptools>=
|
|
3
|
+
"setuptools>=69",
|
|
4
4
|
]
|
|
5
5
|
build-backend = "setuptools.build_meta"
|
|
6
6
|
|
|
@@ -16,13 +16,14 @@ classifiers = [
|
|
|
16
16
|
"Programming Language :: Python :: 3.9",
|
|
17
17
|
"Programming Language :: Python :: 3.10",
|
|
18
18
|
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
19
20
|
"License :: OSI Approved :: Apache Software License",
|
|
20
21
|
"Topic :: Internet :: WWW/HTTP :: HTTP Servers",
|
|
21
22
|
"Topic :: Software Development :: Libraries",
|
|
22
23
|
"Intended Audience :: Developers",
|
|
23
24
|
]
|
|
24
25
|
name = "hexkit"
|
|
25
|
-
version = "
|
|
26
|
+
version = "3.0.0"
|
|
26
27
|
description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
|
|
27
28
|
dependencies = [
|
|
28
29
|
"pydantic >=2, <3",
|
|
@@ -35,8 +36,8 @@ text = "Apache 2.0"
|
|
|
35
36
|
|
|
36
37
|
[project.optional-dependencies]
|
|
37
38
|
akafka = [
|
|
38
|
-
"aiokafka~=0.
|
|
39
|
-
"jsonschema >=4.
|
|
39
|
+
"aiokafka~=0.10.0",
|
|
40
|
+
"jsonschema >=4.21, <5",
|
|
40
41
|
]
|
|
41
42
|
s3 = [
|
|
42
43
|
"boto3 >=1.26.50, <2",
|
|
@@ -96,8 +97,9 @@ fixable = [
|
|
|
96
97
|
"D",
|
|
97
98
|
]
|
|
98
99
|
ignore = [
|
|
99
|
-
"
|
|
100
|
-
"
|
|
100
|
+
"E111",
|
|
101
|
+
"E114",
|
|
102
|
+
"E116",
|
|
101
103
|
"PLW",
|
|
102
104
|
"RUF001",
|
|
103
105
|
"RUF010",
|
|
@@ -170,7 +172,7 @@ check_untyped_defs = true
|
|
|
170
172
|
no_site_packages = false
|
|
171
173
|
|
|
172
174
|
[tool.pytest.ini_options]
|
|
173
|
-
minversion = "
|
|
175
|
+
minversion = "8.0"
|
|
174
176
|
asyncio_mode = "strict"
|
|
175
177
|
|
|
176
178
|
[tool.coverage.paths]
|
|
@@ -181,4 +183,4 @@ source = [
|
|
|
181
183
|
]
|
|
182
184
|
|
|
183
185
|
[tool.tox]
|
|
184
|
-
legacy_tox_ini = " [tox]\n
|
|
186
|
+
legacy_tox_ini = " [tox]\n env_list = py3{9,12}\n\n [gh-actions]\n python =\n 3.9: py39\n 3.10: py310\n 3.11: py311\n 3.12: py312\n\n [testenv]\n pass_env =\n TC_HOST\n DOCKER_HOST\n deps =\n --no-deps -r ./lock/requirements-dev.txt\n commands = pytest {posargs}\n"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
hexkit-2.2.0/src/hexkit/protocols/dao_outbox.py → hexkit-3.0.0/src/hexkit/protocols/daopub.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -34,7 +34,7 @@ from hexkit.protocols.dao import (
|
|
|
34
34
|
)
|
|
35
35
|
|
|
36
36
|
|
|
37
|
-
class
|
|
37
|
+
class DaoPublisher(DaoNaturalId[Dto], typing.Protocol[Dto]):
|
|
38
38
|
"""A Data Access Object (DAO) that automatically publishes changes according to the
|
|
39
39
|
outbox pattern.
|
|
40
40
|
"""
|
|
@@ -50,7 +50,7 @@ class DaoOutbox(DaoNaturalId[Dto], typing.Protocol[Dto]):
|
|
|
50
50
|
...
|
|
51
51
|
|
|
52
52
|
|
|
53
|
-
class
|
|
53
|
+
class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
|
|
54
54
|
"""A protocol describing a factory to produce Data Access Objects (DAO) objects
|
|
55
55
|
which automatically publish changes according to the outbox pattern.
|
|
56
56
|
"""
|
|
@@ -65,7 +65,7 @@ class DaoOutboxFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
65
65
|
dto_to_event: Callable[[Dto], JsonObject],
|
|
66
66
|
event_topic: str,
|
|
67
67
|
autopublish: bool = True,
|
|
68
|
-
) ->
|
|
68
|
+
) -> DaoPublisher[Dto]:
|
|
69
69
|
"""Constructs an Outbox DAO for interacting with resources in a database.
|
|
70
70
|
|
|
71
71
|
Args:
|
|
@@ -88,7 +88,7 @@ class DaoOutboxFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
88
88
|
Whether to automatically publish changes. Defaults to True.
|
|
89
89
|
|
|
90
90
|
Returns:
|
|
91
|
-
A DAO of type
|
|
91
|
+
A DAO of type DaoPublisher, which requires ID specification upon resource
|
|
92
92
|
creation.
|
|
93
93
|
|
|
94
94
|
Raises:
|
|
@@ -123,7 +123,7 @@ class DaoOutboxFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
123
123
|
dto_to_event: Callable[[Dto], JsonObject],
|
|
124
124
|
event_topic: str,
|
|
125
125
|
autopublish: bool,
|
|
126
|
-
) ->
|
|
126
|
+
) -> DaoPublisher[Dto]:
|
|
127
127
|
"""*To be implemented by the provider. Input validation is done outside of this
|
|
128
128
|
method.*
|
|
129
129
|
"""
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
|
+
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
"""A protocol for consuming events published through the DaoPublisherFactoryProtocol."""
|
|
18
|
+
|
|
19
|
+
from abc import ABC, abstractmethod
|
|
20
|
+
from typing import Generic, TypeVar
|
|
21
|
+
|
|
22
|
+
from pydantic import BaseModel
|
|
23
|
+
|
|
24
|
+
Dto = TypeVar("Dto", bound=BaseModel)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class DtoValidationError(ValueError):
|
|
28
|
+
"""Raised when the payload of a received event was not formatted as expected."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class DaoSubscriberProtocol(ABC, Generic[Dto]):
|
|
32
|
+
"""A protocol for consuming events published through the DaoPublisherFactoryProtocol.
|
|
33
|
+
|
|
34
|
+
In addition to the methods described below, implementations shall expose the
|
|
35
|
+
the following attributes:
|
|
36
|
+
event_topic:
|
|
37
|
+
The name of the topic from which updates for the given resource are
|
|
38
|
+
consumed.
|
|
39
|
+
dto_model:
|
|
40
|
+
A pydantic model representing the data transfer object (DTO) for the
|
|
41
|
+
payload of changed events.
|
|
42
|
+
|
|
43
|
+
If the upstream provider using this protocol fails to convert the payload of a
|
|
44
|
+
change event to the expected DTO model, it should raise a DtoValidationError.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
event_topic: str
|
|
48
|
+
dto_model: type[Dto]
|
|
49
|
+
|
|
50
|
+
@abstractmethod
|
|
51
|
+
async def changed(self, resource_id: str, update: Dto) -> None:
|
|
52
|
+
"""Consume a change event (created or updated) for the resource with the given
|
|
53
|
+
ID.
|
|
54
|
+
"""
|
|
55
|
+
...
|
|
56
|
+
|
|
57
|
+
@abstractmethod
|
|
58
|
+
async def deleted(self, resource_id: str) -> None:
|
|
59
|
+
"""Consume an event indicating the deletion of the resource with the given ID."""
|
|
60
|
+
...
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -45,27 +45,47 @@ class EventSubscriberProtocol(ABC):
|
|
|
45
45
|
topics_of_interest: list[Ascii]
|
|
46
46
|
types_of_interest: list[Ascii]
|
|
47
47
|
|
|
48
|
-
async def consume(
|
|
48
|
+
async def consume(
|
|
49
|
+
self,
|
|
50
|
+
*,
|
|
51
|
+
payload: JsonObject,
|
|
52
|
+
type_: Ascii,
|
|
53
|
+
topic: Ascii,
|
|
54
|
+
key: Ascii,
|
|
55
|
+
) -> None:
|
|
49
56
|
"""Receive an event of interest and process it according to its type.
|
|
50
57
|
|
|
51
58
|
Args:
|
|
52
|
-
payload
|
|
53
|
-
type_
|
|
54
|
-
topic
|
|
59
|
+
payload: The data/payload to send with the event.
|
|
60
|
+
type_: The type of the event.
|
|
61
|
+
topic: Name of the topic the event was published to.
|
|
62
|
+
key: A key used for routing the event.
|
|
55
63
|
"""
|
|
56
64
|
check_ascii(type_, topic)
|
|
57
|
-
|
|
65
|
+
|
|
66
|
+
if key:
|
|
67
|
+
check_ascii(key)
|
|
68
|
+
|
|
69
|
+
await self._consume_validated(
|
|
70
|
+
payload=payload, type_=type_, topic=topic, key=key
|
|
71
|
+
)
|
|
58
72
|
|
|
59
73
|
@abstractmethod
|
|
60
74
|
async def _consume_validated(
|
|
61
|
-
self,
|
|
75
|
+
self,
|
|
76
|
+
*,
|
|
77
|
+
payload: JsonObject,
|
|
78
|
+
type_: Ascii,
|
|
79
|
+
topic: Ascii,
|
|
80
|
+
key: Ascii,
|
|
62
81
|
) -> None:
|
|
63
82
|
"""
|
|
64
|
-
Receive and process an event with already validated topic and
|
|
83
|
+
Receive and process an event with already validated topic, type, and key.
|
|
65
84
|
|
|
66
85
|
Args:
|
|
67
|
-
payload
|
|
68
|
-
type_
|
|
69
|
-
topic
|
|
86
|
+
payload: The data/payload to send with the event.
|
|
87
|
+
type_: The type of the event.
|
|
88
|
+
topic: Name of the topic the event was published to.
|
|
89
|
+
key: A key used for routing the event.
|
|
70
90
|
"""
|
|
71
91
|
...
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -19,10 +19,16 @@ They correspond to the `EventPublisherProtocol` and `EventSubscriberProtocol`,
|
|
|
19
19
|
respectively.
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
|
+
from .config import KafkaConfig
|
|
22
23
|
from .provider import (
|
|
23
|
-
KafkaConfig,
|
|
24
24
|
KafkaEventPublisher,
|
|
25
25
|
KafkaEventSubscriber,
|
|
26
|
+
KafkaOutboxSubscriber,
|
|
26
27
|
)
|
|
27
28
|
|
|
28
|
-
__all__ = [
|
|
29
|
+
__all__ = [
|
|
30
|
+
"KafkaEventPublisher",
|
|
31
|
+
"KafkaEventSubscriber",
|
|
32
|
+
"KafkaConfig",
|
|
33
|
+
"KafkaOutboxSubscriber",
|
|
34
|
+
]
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
|
+
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
"""Apache Kafka specific configuration."""
|
|
18
|
+
|
|
19
|
+
from typing import Literal
|
|
20
|
+
|
|
21
|
+
from pydantic import Field, SecretStr
|
|
22
|
+
from pydantic_settings import BaseSettings
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class KafkaConfig(BaseSettings):
|
|
26
|
+
"""Config parameters needed for connecting to Apache Kafka."""
|
|
27
|
+
|
|
28
|
+
service_name: str = Field(
|
|
29
|
+
...,
|
|
30
|
+
examples=["my-cool-special-service"],
|
|
31
|
+
description="The name of the (micro-)service from which messages are published.",
|
|
32
|
+
)
|
|
33
|
+
service_instance_id: str = Field(
|
|
34
|
+
...,
|
|
35
|
+
examples=["germany-bw-instance-001"],
|
|
36
|
+
description=(
|
|
37
|
+
"A string that uniquely identifies this instance across all instances of"
|
|
38
|
+
+ " this service. A globally unique Kafka client ID will be created by"
|
|
39
|
+
+ " concatenating the service_name and the service_instance_id."
|
|
40
|
+
),
|
|
41
|
+
)
|
|
42
|
+
kafka_servers: list[str] = Field(
|
|
43
|
+
...,
|
|
44
|
+
examples=[["localhost:9092"]],
|
|
45
|
+
description="A list of connection strings to connect to Kafka bootstrap servers.",
|
|
46
|
+
)
|
|
47
|
+
kafka_security_protocol: Literal["PLAINTEXT", "SSL"] = Field(
|
|
48
|
+
default="PLAINTEXT",
|
|
49
|
+
description="Protocol used to communicate with brokers. "
|
|
50
|
+
+ "Valid values are: PLAINTEXT, SSL.",
|
|
51
|
+
)
|
|
52
|
+
kafka_ssl_cafile: str = Field(
|
|
53
|
+
default="",
|
|
54
|
+
description="Certificate Authority file path containing certificates"
|
|
55
|
+
+ " used to sign broker certificates. If a CA is not specified, the default"
|
|
56
|
+
+ " system CA will be used if found by OpenSSL.",
|
|
57
|
+
)
|
|
58
|
+
kafka_ssl_certfile: str = Field(
|
|
59
|
+
default="",
|
|
60
|
+
description="Optional filename of client certificate, as well as any"
|
|
61
|
+
+ " CA certificates needed to establish the certificate's authenticity.",
|
|
62
|
+
)
|
|
63
|
+
kafka_ssl_keyfile: str = Field(
|
|
64
|
+
default="", description="Optional filename containing the client private key."
|
|
65
|
+
)
|
|
66
|
+
kafka_ssl_password: SecretStr = Field(
|
|
67
|
+
default="",
|
|
68
|
+
description="Optional password to be used for the client private key.",
|
|
69
|
+
)
|
|
70
|
+
generate_correlation_id: bool = Field(
|
|
71
|
+
default=True,
|
|
72
|
+
examples=[True, False],
|
|
73
|
+
description=(
|
|
74
|
+
"A flag, which, if False, will result in an error when trying to publish an"
|
|
75
|
+
+ " event without a valid correlation ID set for the context. If True, the a"
|
|
76
|
+
+ " newly correlation ID will be generated and used in the event header."
|
|
77
|
+
),
|
|
78
|
+
)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
|
+
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
"""A subpackage containing all Apache Kafka-specific providers.
|
|
18
|
+
|
|
19
|
+
Require dependencies of the `akafka` extra.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from .daosub import KafkaOutboxSubscriber
|
|
23
|
+
from .eventpub import KafkaEventPublisher
|
|
24
|
+
from .eventsub import (
|
|
25
|
+
ConsumerEvent,
|
|
26
|
+
KafkaEventSubscriber,
|
|
27
|
+
get_header_value,
|
|
28
|
+
headers_as_dict,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"KafkaEventPublisher",
|
|
33
|
+
"KafkaEventSubscriber",
|
|
34
|
+
"ConsumerEvent",
|
|
35
|
+
"get_header_value",
|
|
36
|
+
"headers_as_dict",
|
|
37
|
+
"KafkaOutboxSubscriber",
|
|
38
|
+
]
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
|
+
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
"""Apache Kafka-specific provider using implementations of the
|
|
18
|
+
`DaoSubscriberProtocol`.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import logging
|
|
22
|
+
from collections.abc import Sequence
|
|
23
|
+
from contextlib import asynccontextmanager
|
|
24
|
+
|
|
25
|
+
from aiokafka import AIOKafkaConsumer
|
|
26
|
+
from pydantic import ValidationError
|
|
27
|
+
|
|
28
|
+
from hexkit.base import InboundProviderBase
|
|
29
|
+
from hexkit.custom_types import Ascii, JsonObject
|
|
30
|
+
from hexkit.protocols.daosub import (
|
|
31
|
+
DaoSubscriberProtocol,
|
|
32
|
+
DtoValidationError,
|
|
33
|
+
)
|
|
34
|
+
from hexkit.protocols.eventsub import EventSubscriberProtocol
|
|
35
|
+
from hexkit.providers.akafka.config import KafkaConfig
|
|
36
|
+
from hexkit.providers.akafka.provider.eventsub import (
|
|
37
|
+
KafkaConsumerCompatible,
|
|
38
|
+
KafkaEventSubscriber,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
CHANGE_EVENT_TYPE = "upserted"
|
|
42
|
+
DELETE_EVENT_TYPE = "deleted"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class TranslatorConverter(EventSubscriberProtocol):
|
|
46
|
+
"""Takes a list of translators implementing the `DaoSubscriberProtocol` to
|
|
47
|
+
create a single translator implementing the `EventSubscriberProtocol`.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
types_of_interest = [CHANGE_EVENT_TYPE, DELETE_EVENT_TYPE]
|
|
51
|
+
|
|
52
|
+
def __init__(self, *, translators: Sequence[DaoSubscriberProtocol]):
|
|
53
|
+
self.topics_of_interest = [translator.event_topic for translator in translators]
|
|
54
|
+
|
|
55
|
+
if len(set(self.topics_of_interest)) != len(self.topics_of_interest):
|
|
56
|
+
raise ValueError(
|
|
57
|
+
"Got multiple DaoSubscriberProtocol-compliant translators trying to"
|
|
58
|
+
+ " consume from the same event topic."
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
self._translator_by_topic = {
|
|
62
|
+
translator.event_topic: translator for translator in translators
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async def _consume_validated(
|
|
66
|
+
self, *, payload: JsonObject, type_: Ascii, topic: Ascii, key: Ascii
|
|
67
|
+
) -> None:
|
|
68
|
+
"""
|
|
69
|
+
Receive and process an event with already validated topic, type, and key.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
payload: The data/payload to send with the event.
|
|
73
|
+
type_: The type of the event.
|
|
74
|
+
topic: Name of the topic the event was published to.
|
|
75
|
+
key: A key used for routing the event.
|
|
76
|
+
"""
|
|
77
|
+
translator = self._translator_by_topic.get(topic)
|
|
78
|
+
|
|
79
|
+
if translator is None:
|
|
80
|
+
# This should never happen, as the topic should have been filtered out:
|
|
81
|
+
raise RuntimeError
|
|
82
|
+
|
|
83
|
+
if type_ == CHANGE_EVENT_TYPE:
|
|
84
|
+
try:
|
|
85
|
+
dto = translator.dto_model.model_validate(payload)
|
|
86
|
+
except ValidationError as error:
|
|
87
|
+
message = (
|
|
88
|
+
f"The event of type {type_} on topic {topic} was not valid wrt. the"
|
|
89
|
+
+ " DTO model."
|
|
90
|
+
)
|
|
91
|
+
logging.error(message)
|
|
92
|
+
raise DtoValidationError(message) from error
|
|
93
|
+
|
|
94
|
+
await translator.changed(resource_id=key, update=dto)
|
|
95
|
+
|
|
96
|
+
else:
|
|
97
|
+
# a deletion event:
|
|
98
|
+
await translator.deleted(resource_id=key)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class KafkaOutboxSubscriber(InboundProviderBase):
|
|
102
|
+
"""Apache Kafka-specific provider using translators that implement the
|
|
103
|
+
`DaoSubscriberProtocol`.
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
@classmethod
|
|
107
|
+
@asynccontextmanager
|
|
108
|
+
async def construct(
|
|
109
|
+
cls,
|
|
110
|
+
*,
|
|
111
|
+
config: KafkaConfig,
|
|
112
|
+
translators: Sequence[DaoSubscriberProtocol],
|
|
113
|
+
kafka_consumer_cls: type[KafkaConsumerCompatible] = AIOKafkaConsumer,
|
|
114
|
+
):
|
|
115
|
+
"""Setup and teardown an instance of the provider.
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
config: MongoDB-specific config parameters.
|
|
119
|
+
|
|
120
|
+
Returns:
|
|
121
|
+
An instance of the provider.
|
|
122
|
+
"""
|
|
123
|
+
translator_converter = TranslatorConverter(translators=translators)
|
|
124
|
+
|
|
125
|
+
async with KafkaEventSubscriber.construct(
|
|
126
|
+
config=config,
|
|
127
|
+
translator=translator_converter,
|
|
128
|
+
kafka_consumer_cls=kafka_consumer_cls,
|
|
129
|
+
) as event_subscriber:
|
|
130
|
+
yield cls(event_subscriber=event_subscriber)
|
|
131
|
+
|
|
132
|
+
def __init__(self, *, event_subscriber: KafkaEventSubscriber):
|
|
133
|
+
"""Please do not call directly! Should be called by the `construct` method."""
|
|
134
|
+
self._event_subscriber = event_subscriber
|
|
135
|
+
|
|
136
|
+
async def run(self, forever: bool = True) -> None:
|
|
137
|
+
"""Start consuming events from the Kafka outbox.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
forever: Whether to run the consumer indefinitely. Defaults to True.
|
|
141
|
+
"""
|
|
142
|
+
await self._event_subscriber.run(forever=forever)
|