hexkit 2.1.1__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hexkit-2.1.1 → hexkit-3.0.0}/LICENSE +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/MANIFEST.in +1 -0
- {hexkit-2.1.1/src/hexkit.egg-info → hexkit-3.0.0}/PKG-INFO +7 -6
- {hexkit-2.1.1 → hexkit-3.0.0}/pyproject.toml +31 -27
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/__init__.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/__main__.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/base.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/config.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/correlation.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/custom_types.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/log.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/protocols/__init__.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/protocols/dao.py +36 -22
- hexkit-3.0.0/src/hexkit/protocols/daopub.py +130 -0
- hexkit-3.0.0/src/hexkit/protocols/daosub.py +60 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/protocols/eventpub.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/protocols/eventsub.py +31 -11
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/protocols/objstorage.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/__init__.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/akafka/__init__.py +9 -3
- hexkit-3.0.0/src/hexkit/providers/akafka/config.py +78 -0
- hexkit-3.0.0/src/hexkit/providers/akafka/provider/__init__.py +38 -0
- hexkit-3.0.0/src/hexkit/providers/akafka/provider/daosub.py +142 -0
- hexkit-3.0.0/src/hexkit/providers/akafka/provider/eventpub.py +176 -0
- hexkit-2.1.1/src/hexkit/providers/akafka/provider.py → hexkit-3.0.0/src/hexkit/providers/akafka/provider/eventsub.py +13 -236
- hexkit-3.0.0/src/hexkit/providers/akafka/provider/utils.py +46 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/akafka/testcontainer.py +5 -25
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/akafka/testutils.py +44 -59
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/mongodb/__init__.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/mongodb/provider.py +124 -83
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/mongodb/testutils.py +2 -1
- hexkit-3.0.0/src/hexkit/providers/mongokafka/__init__.py +19 -0
- hexkit-3.0.0/src/hexkit/providers/mongokafka/provider.py +467 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/s3/__init__.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/s3/provider.py +13 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/__init__.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/s3/testutils.py +32 -25
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/testing/__init__.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/testing/eventpub.py +1 -1
- hexkit-3.0.0/src/hexkit/py.typed +1 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/utils.py +1 -1
- {hexkit-2.1.1 → hexkit-3.0.0/src/hexkit.egg-info}/PKG-INFO +7 -6
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit.egg-info/SOURCES.txt +11 -1
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit.egg-info/requires.txt +5 -5
- {hexkit-2.1.1 → hexkit-3.0.0}/README.md +0 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/setup.cfg +0 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file1.yaml +0 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file2.yaml +0 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file3.yaml +0 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit/providers/s3/test_files/test_file4.yaml +0 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit.egg-info/dependency_links.txt +0 -0
- {hexkit-2.1.1 → hexkit-3.0.0}/src/hexkit.egg-info/top_level.txt +0 -0
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright 2021 -
|
|
189
|
+
Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
190
190
|
for the German Human Genome-Phenome Archive (GHGA)
|
|
191
191
|
|
|
192
192
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: hexkit
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
|
|
5
5
|
Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
|
|
6
6
|
License: Apache 2.0
|
|
@@ -10,6 +10,7 @@ Classifier: Operating System :: POSIX :: Linux
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
14
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
15
|
Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
|
|
15
16
|
Classifier: Topic :: Software Development :: Libraries
|
|
@@ -21,8 +22,8 @@ Requires-Dist: pydantic<3,>=2
|
|
|
21
22
|
Requires-Dist: pydantic_settings<3,>=2
|
|
22
23
|
Requires-Dist: PyYAML<7,>=6.0
|
|
23
24
|
Provides-Extra: akafka
|
|
24
|
-
Requires-Dist: aiokafka~=0.
|
|
25
|
-
Requires-Dist: jsonschema<5,>=4.
|
|
25
|
+
Requires-Dist: aiokafka~=0.10.0; extra == "akafka"
|
|
26
|
+
Requires-Dist: jsonschema<5,>=4.21; extra == "akafka"
|
|
26
27
|
Provides-Extra: s3
|
|
27
28
|
Requires-Dist: boto3<2,>=1.26.50; extra == "s3"
|
|
28
29
|
Requires-Dist: botocore<2,>=1.29.50; extra == "s3"
|
|
@@ -30,13 +31,13 @@ Provides-Extra: mongodb
|
|
|
30
31
|
Requires-Dist: motor<4,>=3.1.1; extra == "mongodb"
|
|
31
32
|
Provides-Extra: test-akafka
|
|
32
33
|
Requires-Dist: hexkit[akafka]; extra == "test-akafka"
|
|
33
|
-
Requires-Dist: testcontainers[kafka]<4
|
|
34
|
+
Requires-Dist: testcontainers[kafka]<5,>=4.3.3; extra == "test-akafka"
|
|
34
35
|
Provides-Extra: test-s3
|
|
35
36
|
Requires-Dist: hexkit[s3]; extra == "test-s3"
|
|
36
|
-
Requires-Dist: testcontainers<4
|
|
37
|
+
Requires-Dist: testcontainers<5,>=4.3.3; extra == "test-s3"
|
|
37
38
|
Provides-Extra: test-mongodb
|
|
38
39
|
Requires-Dist: hexkit[mongodb]; extra == "test-mongodb"
|
|
39
|
-
Requires-Dist: testcontainers[mongo]<4
|
|
40
|
+
Requires-Dist: testcontainers[mongo]<5,>=4.3.3; extra == "test-mongodb"
|
|
40
41
|
Provides-Extra: test
|
|
41
42
|
Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
|
|
42
43
|
Provides-Extra: all
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[build-system]
|
|
2
2
|
requires = [
|
|
3
|
-
"setuptools>=
|
|
3
|
+
"setuptools>=69",
|
|
4
4
|
]
|
|
5
5
|
build-backend = "setuptools.build_meta"
|
|
6
6
|
|
|
@@ -16,13 +16,14 @@ classifiers = [
|
|
|
16
16
|
"Programming Language :: Python :: 3.9",
|
|
17
17
|
"Programming Language :: Python :: 3.10",
|
|
18
18
|
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
19
20
|
"License :: OSI Approved :: Apache Software License",
|
|
20
21
|
"Topic :: Internet :: WWW/HTTP :: HTTP Servers",
|
|
21
22
|
"Topic :: Software Development :: Libraries",
|
|
22
23
|
"Intended Audience :: Developers",
|
|
23
24
|
]
|
|
24
25
|
name = "hexkit"
|
|
25
|
-
version = "
|
|
26
|
+
version = "3.0.0"
|
|
26
27
|
description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
|
|
27
28
|
dependencies = [
|
|
28
29
|
"pydantic >=2, <3",
|
|
@@ -35,8 +36,8 @@ text = "Apache 2.0"
|
|
|
35
36
|
|
|
36
37
|
[project.optional-dependencies]
|
|
37
38
|
akafka = [
|
|
38
|
-
"aiokafka~=0.
|
|
39
|
-
"jsonschema >=4.
|
|
39
|
+
"aiokafka~=0.10.0",
|
|
40
|
+
"jsonschema >=4.21, <5",
|
|
40
41
|
]
|
|
41
42
|
s3 = [
|
|
42
43
|
"boto3 >=1.26.50, <2",
|
|
@@ -47,15 +48,15 @@ mongodb = [
|
|
|
47
48
|
]
|
|
48
49
|
test-akafka = [
|
|
49
50
|
"hexkit[akafka]",
|
|
50
|
-
"testcontainers[kafka] >=3.
|
|
51
|
+
"testcontainers[kafka] >=4.3.3, <5",
|
|
51
52
|
]
|
|
52
53
|
test-s3 = [
|
|
53
54
|
"hexkit[s3]",
|
|
54
|
-
"testcontainers >=3.
|
|
55
|
+
"testcontainers >=4.3.3, <5",
|
|
55
56
|
]
|
|
56
57
|
test-mongodb = [
|
|
57
58
|
"hexkit[mongodb]",
|
|
58
|
-
"testcontainers[mongo] >=3.
|
|
59
|
+
"testcontainers[mongo] >=4.3.3, <5",
|
|
59
60
|
]
|
|
60
61
|
test = [
|
|
61
62
|
"hexkit[test-akafka,test-s3,test-mongodb]",
|
|
@@ -80,9 +81,25 @@ exclude = [
|
|
|
80
81
|
"build",
|
|
81
82
|
"dist",
|
|
82
83
|
]
|
|
84
|
+
line-length = 88
|
|
85
|
+
src = [
|
|
86
|
+
"src",
|
|
87
|
+
"tests",
|
|
88
|
+
"examples",
|
|
89
|
+
"scripts",
|
|
90
|
+
]
|
|
91
|
+
target-version = "py39"
|
|
92
|
+
|
|
93
|
+
[tool.ruff.lint]
|
|
94
|
+
fixable = [
|
|
95
|
+
"UP",
|
|
96
|
+
"I",
|
|
97
|
+
"D",
|
|
98
|
+
]
|
|
83
99
|
ignore = [
|
|
84
|
-
"
|
|
85
|
-
"
|
|
100
|
+
"E111",
|
|
101
|
+
"E114",
|
|
102
|
+
"E116",
|
|
86
103
|
"PLW",
|
|
87
104
|
"RUF001",
|
|
88
105
|
"RUF010",
|
|
@@ -97,7 +114,6 @@ ignore = [
|
|
|
97
114
|
"D206",
|
|
98
115
|
"D300",
|
|
99
116
|
]
|
|
100
|
-
line-length = 88
|
|
101
117
|
select = [
|
|
102
118
|
"C90",
|
|
103
119
|
"F",
|
|
@@ -111,23 +127,11 @@ select = [
|
|
|
111
127
|
"SIM",
|
|
112
128
|
"D",
|
|
113
129
|
]
|
|
114
|
-
fixable = [
|
|
115
|
-
"UP",
|
|
116
|
-
"I",
|
|
117
|
-
"D",
|
|
118
|
-
]
|
|
119
|
-
src = [
|
|
120
|
-
"src",
|
|
121
|
-
"tests",
|
|
122
|
-
"examples",
|
|
123
|
-
"scripts",
|
|
124
|
-
]
|
|
125
|
-
target-version = "py39"
|
|
126
130
|
|
|
127
|
-
[tool.ruff.mccabe]
|
|
131
|
+
[tool.ruff.lint.mccabe]
|
|
128
132
|
max-complexity = 10
|
|
129
133
|
|
|
130
|
-
[tool.ruff.per-file-ignores]
|
|
134
|
+
[tool.ruff.lint.per-file-ignores]
|
|
131
135
|
"scripts/*" = [
|
|
132
136
|
"PL",
|
|
133
137
|
"S",
|
|
@@ -153,7 +157,7 @@ max-complexity = 10
|
|
|
153
157
|
"D",
|
|
154
158
|
]
|
|
155
159
|
|
|
156
|
-
[tool.ruff.pydocstyle]
|
|
160
|
+
[tool.ruff.lint.pydocstyle]
|
|
157
161
|
convention = "pep257"
|
|
158
162
|
|
|
159
163
|
[tool.mypy]
|
|
@@ -168,7 +172,7 @@ check_untyped_defs = true
|
|
|
168
172
|
no_site_packages = false
|
|
169
173
|
|
|
170
174
|
[tool.pytest.ini_options]
|
|
171
|
-
minversion = "
|
|
175
|
+
minversion = "8.0"
|
|
172
176
|
asyncio_mode = "strict"
|
|
173
177
|
|
|
174
178
|
[tool.coverage.paths]
|
|
@@ -179,4 +183,4 @@ source = [
|
|
|
179
183
|
]
|
|
180
184
|
|
|
181
185
|
[tool.tox]
|
|
182
|
-
legacy_tox_ini = " [tox]\n
|
|
186
|
+
legacy_tox_ini = " [tox]\n env_list = py3{9,12}\n\n [gh-actions]\n python =\n 3.9: py39\n 3.10: py310\n 3.11: py311\n 3.12: py312\n\n [testenv]\n pass_env =\n TC_HOST\n DOCKER_HOST\n deps =\n --no-deps -r ./lock/requirements-dev.txt\n commands = pytest {posargs}\n"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -268,10 +268,8 @@ async def uuid4_id_generator() -> AsyncGenerator[str, None]:
|
|
|
268
268
|
yield str(uuid4())
|
|
269
269
|
|
|
270
270
|
|
|
271
|
-
class
|
|
272
|
-
"""A
|
|
273
|
-
that are enclosed in transactional scopes.
|
|
274
|
-
"""
|
|
271
|
+
class DaoFactoryBase:
|
|
272
|
+
"""A base for Data Access Objects (DAO) Factory protocols."""
|
|
275
273
|
|
|
276
274
|
class IdFieldNotFoundError(ValueError):
|
|
277
275
|
"""Raised when the dto_model did not contain the expected id_field."""
|
|
@@ -335,6 +333,32 @@ class DaoFactoryProtocol(ABC):
|
|
|
335
333
|
f"Provided index fields are invalid: {error}"
|
|
336
334
|
) from error
|
|
337
335
|
|
|
336
|
+
@classmethod
|
|
337
|
+
def _validate(
|
|
338
|
+
cls,
|
|
339
|
+
*,
|
|
340
|
+
dto_model: type[Dto],
|
|
341
|
+
id_field: str,
|
|
342
|
+
dto_creation_model: Optional[type[DtoCreation]],
|
|
343
|
+
fields_to_index: Optional[Collection[str]],
|
|
344
|
+
) -> None:
|
|
345
|
+
"""Validates the input parameters of the get_dao method."""
|
|
346
|
+
cls._validate_dto_model_id(dto_model=dto_model, id_field=id_field)
|
|
347
|
+
cls._validate_dto_creation_model(
|
|
348
|
+
dto_model=dto_model,
|
|
349
|
+
dto_creation_model=dto_creation_model,
|
|
350
|
+
id_field=id_field,
|
|
351
|
+
)
|
|
352
|
+
cls._validate_fields_to_index(
|
|
353
|
+
dto_model=dto_model, fields_to_index=fields_to_index
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
class DaoFactoryProtocol(DaoFactoryBase, ABC):
|
|
358
|
+
"""A protocol describing a factory to produce Data Access Objects (DAO) objects
|
|
359
|
+
that are enclosed in transactional scopes.
|
|
360
|
+
"""
|
|
361
|
+
|
|
338
362
|
@overload
|
|
339
363
|
async def get_dao(
|
|
340
364
|
self,
|
|
@@ -344,8 +368,7 @@ class DaoFactoryProtocol(ABC):
|
|
|
344
368
|
id_field: str,
|
|
345
369
|
fields_to_index: Optional[Collection[str]] = None,
|
|
346
370
|
id_generator: Optional[AsyncGenerator[str, None]] = None,
|
|
347
|
-
) -> DaoNaturalId[Dto]:
|
|
348
|
-
...
|
|
371
|
+
) -> DaoNaturalId[Dto]: ...
|
|
349
372
|
|
|
350
373
|
@overload
|
|
351
374
|
async def get_dao(
|
|
@@ -357,8 +380,7 @@ class DaoFactoryProtocol(ABC):
|
|
|
357
380
|
dto_creation_model: type[DtoCreation],
|
|
358
381
|
fields_to_index: Optional[Collection[str]] = None,
|
|
359
382
|
id_generator: Optional[AsyncGenerator[str, None]] = None,
|
|
360
|
-
) -> DaoSurrogateId[Dto, DtoCreation]:
|
|
361
|
-
...
|
|
383
|
+
) -> DaoSurrogateId[Dto, DtoCreation]: ...
|
|
362
384
|
|
|
363
385
|
async def get_dao(
|
|
364
386
|
self,
|
|
@@ -408,16 +430,11 @@ class DaoFactoryProtocol(ABC):
|
|
|
408
430
|
self.IdFieldNotFoundError:
|
|
409
431
|
Raised when the dto_model did not contain the expected id_field.
|
|
410
432
|
"""
|
|
411
|
-
self.
|
|
412
|
-
|
|
413
|
-
self._validate_dto_creation_model(
|
|
433
|
+
self._validate(
|
|
414
434
|
dto_model=dto_model,
|
|
415
|
-
dto_creation_model=dto_creation_model,
|
|
416
435
|
id_field=id_field,
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
self._validate_fields_to_index(
|
|
420
|
-
dto_model=dto_model, fields_to_index=fields_to_index
|
|
436
|
+
dto_creation_model=dto_creation_model,
|
|
437
|
+
fields_to_index=fields_to_index,
|
|
421
438
|
)
|
|
422
439
|
|
|
423
440
|
if id_generator is None:
|
|
@@ -430,7 +447,6 @@ class DaoFactoryProtocol(ABC):
|
|
|
430
447
|
id_field=id_field,
|
|
431
448
|
fields_to_index=fields_to_index,
|
|
432
449
|
dto_creation_model=dto_creation_model,
|
|
433
|
-
# (above behavior by mypy seems incorrect)
|
|
434
450
|
id_generator=id_generator,
|
|
435
451
|
)
|
|
436
452
|
|
|
@@ -445,8 +461,7 @@ class DaoFactoryProtocol(ABC):
|
|
|
445
461
|
dto_creation_model: None,
|
|
446
462
|
fields_to_index: Optional[Collection[str]],
|
|
447
463
|
id_generator: AsyncGenerator[str, None],
|
|
448
|
-
) -> DaoNaturalId[Dto]:
|
|
449
|
-
...
|
|
464
|
+
) -> DaoNaturalId[Dto]: ...
|
|
450
465
|
|
|
451
466
|
@overload
|
|
452
467
|
@abstractmethod
|
|
@@ -459,8 +474,7 @@ class DaoFactoryProtocol(ABC):
|
|
|
459
474
|
dto_creation_model: type[DtoCreation],
|
|
460
475
|
fields_to_index: Optional[Collection[str]],
|
|
461
476
|
id_generator: AsyncGenerator[str, None],
|
|
462
|
-
) -> DaoSurrogateId[Dto, DtoCreation]:
|
|
463
|
-
...
|
|
477
|
+
) -> DaoSurrogateId[Dto, DtoCreation]: ...
|
|
464
478
|
|
|
465
479
|
@abstractmethod
|
|
466
480
|
async def _get_dao(
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
|
+
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
"""Protocol for creating Data Access Objects to perform CRUD (plus find) interactions
|
|
18
|
+
with the database plus automatically publish the changes as events using a modification
|
|
19
|
+
of the outbox pattern.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
# ruff: noqa: PLR0913
|
|
23
|
+
|
|
24
|
+
import typing
|
|
25
|
+
from abc import ABC, abstractmethod
|
|
26
|
+
from collections.abc import Collection
|
|
27
|
+
from typing import Callable, Optional
|
|
28
|
+
|
|
29
|
+
from hexkit.custom_types import JsonObject
|
|
30
|
+
from hexkit.protocols.dao import (
|
|
31
|
+
DaoFactoryBase,
|
|
32
|
+
DaoNaturalId,
|
|
33
|
+
Dto,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class DaoPublisher(DaoNaturalId[Dto], typing.Protocol[Dto]):
|
|
38
|
+
"""A Data Access Object (DAO) that automatically publishes changes according to the
|
|
39
|
+
outbox pattern.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
async def publish_pending(self) -> None:
|
|
43
|
+
"""Publishes all non-published changes."""
|
|
44
|
+
...
|
|
45
|
+
|
|
46
|
+
async def republish(self) -> None:
|
|
47
|
+
"""Republishes the state of all resources independent of whether they have
|
|
48
|
+
already been published or not.
|
|
49
|
+
"""
|
|
50
|
+
...
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
|
|
54
|
+
"""A protocol describing a factory to produce Data Access Objects (DAO) objects
|
|
55
|
+
which automatically publish changes according to the outbox pattern.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
async def get_dao(
|
|
59
|
+
self,
|
|
60
|
+
*,
|
|
61
|
+
name: str,
|
|
62
|
+
dto_model: type[Dto],
|
|
63
|
+
id_field: str,
|
|
64
|
+
fields_to_index: Optional[Collection[str]] = None,
|
|
65
|
+
dto_to_event: Callable[[Dto], JsonObject],
|
|
66
|
+
event_topic: str,
|
|
67
|
+
autopublish: bool = True,
|
|
68
|
+
) -> DaoPublisher[Dto]:
|
|
69
|
+
"""Constructs an Outbox DAO for interacting with resources in a database.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
name:
|
|
73
|
+
The name of the resource type (roughly equivalent to the name of a
|
|
74
|
+
database table or collection).
|
|
75
|
+
dto_model:
|
|
76
|
+
A DTO (Data Transfer Object) model describing the shape of resources.
|
|
77
|
+
id_field:
|
|
78
|
+
The name of the field of the `dto_model` that serves as resource ID.
|
|
79
|
+
(DAO implementation might use this field as primary key.)
|
|
80
|
+
fields_to_index:
|
|
81
|
+
Optionally, provide any fields that should be indexed in addition to the
|
|
82
|
+
`id_field`. Defaults to None.
|
|
83
|
+
dto_to_event:
|
|
84
|
+
A function that takes a DTO and returns the payload for an event.
|
|
85
|
+
event_topic:
|
|
86
|
+
The topic to which events should be published.
|
|
87
|
+
autopublish:
|
|
88
|
+
Whether to automatically publish changes. Defaults to True.
|
|
89
|
+
|
|
90
|
+
Returns:
|
|
91
|
+
A DAO of type DaoPublisher, which requires ID specification upon resource
|
|
92
|
+
creation.
|
|
93
|
+
|
|
94
|
+
Raises:
|
|
95
|
+
self.IdFieldNotFoundError:
|
|
96
|
+
Raised when the dto_model did not contain the expected id_field.
|
|
97
|
+
"""
|
|
98
|
+
self._validate(
|
|
99
|
+
dto_model=dto_model,
|
|
100
|
+
id_field=id_field,
|
|
101
|
+
dto_creation_model=None,
|
|
102
|
+
fields_to_index=fields_to_index,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
return await self._get_dao(
|
|
106
|
+
name=name,
|
|
107
|
+
dto_model=dto_model,
|
|
108
|
+
id_field=id_field,
|
|
109
|
+
fields_to_index=fields_to_index,
|
|
110
|
+
dto_to_event=dto_to_event,
|
|
111
|
+
event_topic=event_topic,
|
|
112
|
+
autopublish=autopublish,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
@abstractmethod
|
|
116
|
+
async def _get_dao(
|
|
117
|
+
self,
|
|
118
|
+
*,
|
|
119
|
+
name: str,
|
|
120
|
+
dto_model: type[Dto],
|
|
121
|
+
id_field: str,
|
|
122
|
+
fields_to_index: Optional[Collection[str]],
|
|
123
|
+
dto_to_event: Callable[[Dto], JsonObject],
|
|
124
|
+
event_topic: str,
|
|
125
|
+
autopublish: bool,
|
|
126
|
+
) -> DaoPublisher[Dto]:
|
|
127
|
+
"""*To be implemented by the provider. Input validation is done outside of this
|
|
128
|
+
method.*
|
|
129
|
+
"""
|
|
130
|
+
...
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
|
+
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
"""A protocol for consuming events published through the DaoPublisherFactoryProtocol."""
|
|
18
|
+
|
|
19
|
+
from abc import ABC, abstractmethod
|
|
20
|
+
from typing import Generic, TypeVar
|
|
21
|
+
|
|
22
|
+
from pydantic import BaseModel
|
|
23
|
+
|
|
24
|
+
Dto = TypeVar("Dto", bound=BaseModel)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class DtoValidationError(ValueError):
|
|
28
|
+
"""Raised when the payload of a received event was not formatted as expected."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class DaoSubscriberProtocol(ABC, Generic[Dto]):
|
|
32
|
+
"""A protocol for consuming events published through the DaoPublisherFactoryProtocol.
|
|
33
|
+
|
|
34
|
+
In addition to the methods described below, implementations shall expose the
|
|
35
|
+
the following attributes:
|
|
36
|
+
event_topic:
|
|
37
|
+
The name of the topic from which updates for the given resource are
|
|
38
|
+
consumed.
|
|
39
|
+
dto_model:
|
|
40
|
+
A pydantic model representing the data transfer object (DTO) for the
|
|
41
|
+
payload of changed events.
|
|
42
|
+
|
|
43
|
+
If the upstream provider using this protocol fails to convert the payload of a
|
|
44
|
+
change event to the expected DTO model, it should raise a DtoValidationError.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
event_topic: str
|
|
48
|
+
dto_model: type[Dto]
|
|
49
|
+
|
|
50
|
+
@abstractmethod
|
|
51
|
+
async def changed(self, resource_id: str, update: Dto) -> None:
|
|
52
|
+
"""Consume a change event (created or updated) for the resource with the given
|
|
53
|
+
ID.
|
|
54
|
+
"""
|
|
55
|
+
...
|
|
56
|
+
|
|
57
|
+
@abstractmethod
|
|
58
|
+
async def deleted(self, resource_id: str) -> None:
|
|
59
|
+
"""Consume an event indicating the deletion of the resource with the given ID."""
|
|
60
|
+
...
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -45,27 +45,47 @@ class EventSubscriberProtocol(ABC):
|
|
|
45
45
|
topics_of_interest: list[Ascii]
|
|
46
46
|
types_of_interest: list[Ascii]
|
|
47
47
|
|
|
48
|
-
async def consume(
|
|
48
|
+
async def consume(
|
|
49
|
+
self,
|
|
50
|
+
*,
|
|
51
|
+
payload: JsonObject,
|
|
52
|
+
type_: Ascii,
|
|
53
|
+
topic: Ascii,
|
|
54
|
+
key: Ascii,
|
|
55
|
+
) -> None:
|
|
49
56
|
"""Receive an event of interest and process it according to its type.
|
|
50
57
|
|
|
51
58
|
Args:
|
|
52
|
-
payload
|
|
53
|
-
type_
|
|
54
|
-
topic
|
|
59
|
+
payload: The data/payload to send with the event.
|
|
60
|
+
type_: The type of the event.
|
|
61
|
+
topic: Name of the topic the event was published to.
|
|
62
|
+
key: A key used for routing the event.
|
|
55
63
|
"""
|
|
56
64
|
check_ascii(type_, topic)
|
|
57
|
-
|
|
65
|
+
|
|
66
|
+
if key:
|
|
67
|
+
check_ascii(key)
|
|
68
|
+
|
|
69
|
+
await self._consume_validated(
|
|
70
|
+
payload=payload, type_=type_, topic=topic, key=key
|
|
71
|
+
)
|
|
58
72
|
|
|
59
73
|
@abstractmethod
|
|
60
74
|
async def _consume_validated(
|
|
61
|
-
self,
|
|
75
|
+
self,
|
|
76
|
+
*,
|
|
77
|
+
payload: JsonObject,
|
|
78
|
+
type_: Ascii,
|
|
79
|
+
topic: Ascii,
|
|
80
|
+
key: Ascii,
|
|
62
81
|
) -> None:
|
|
63
82
|
"""
|
|
64
|
-
Receive and process an event with already validated topic and
|
|
83
|
+
Receive and process an event with already validated topic, type, and key.
|
|
65
84
|
|
|
66
85
|
Args:
|
|
67
|
-
payload
|
|
68
|
-
type_
|
|
69
|
-
topic
|
|
86
|
+
payload: The data/payload to send with the event.
|
|
87
|
+
type_: The type of the event.
|
|
88
|
+
topic: Name of the topic the event was published to.
|
|
89
|
+
key: A key used for routing the event.
|
|
70
90
|
"""
|
|
71
91
|
...
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -19,10 +19,16 @@ They correspond to the `EventPublisherProtocol` and `EventSubscriberProtocol`,
|
|
|
19
19
|
respectively.
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
|
+
from .config import KafkaConfig
|
|
22
23
|
from .provider import (
|
|
23
|
-
KafkaConfig,
|
|
24
24
|
KafkaEventPublisher,
|
|
25
25
|
KafkaEventSubscriber,
|
|
26
|
+
KafkaOutboxSubscriber,
|
|
26
27
|
)
|
|
27
28
|
|
|
28
|
-
__all__ = [
|
|
29
|
+
__all__ = [
|
|
30
|
+
"KafkaEventPublisher",
|
|
31
|
+
"KafkaEventSubscriber",
|
|
32
|
+
"KafkaConfig",
|
|
33
|
+
"KafkaOutboxSubscriber",
|
|
34
|
+
]
|