hexkit 4.1.1__tar.gz → 4.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hexkit-4.1.1 → hexkit-4.3.0}/LICENSE +1 -1
- {hexkit-4.1.1/src/hexkit.egg-info → hexkit-4.3.0}/PKG-INFO +26 -22
- {hexkit-4.1.1 → hexkit-4.3.0}/README.md +13 -12
- {hexkit-4.1.1 → hexkit-4.3.0}/pyproject.toml +16 -12
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/__init__.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/__main__.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/base.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/config.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/correlation.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/custom_types.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/log.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/protocols/__init__.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/protocols/dao.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/protocols/daopub.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/protocols/daosub.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/protocols/eventpub.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/protocols/eventsub.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/protocols/objstorage.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/__init__.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/akafka/__init__.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/akafka/config.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/akafka/provider/__init__.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/akafka/provider/daosub.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/akafka/provider/eventpub.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/akafka/provider/eventsub.py +72 -33
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/akafka/provider/utils.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/akafka/testcontainer.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/akafka/testutils.py +2 -2
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/mongodb/__init__.py +1 -1
- {hexkit-4.1.1/src/hexkit/providers/mongokafka → hexkit-4.3.0/src/hexkit/providers/mongodb/migrations}/__init__.py +19 -5
- hexkit-4.3.0/src/hexkit/providers/mongodb/migrations/_manager.py +403 -0
- hexkit-4.3.0/src/hexkit/providers/mongodb/migrations/_utils.py +380 -0
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/mongodb/provider.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/mongodb/testutils.py +1 -1
- hexkit-4.3.0/src/hexkit/providers/mongokafka/__init__.py +37 -0
- hexkit-4.3.0/src/hexkit/providers/mongokafka/provider/__init__.py +36 -0
- hexkit-4.3.0/src/hexkit/providers/mongokafka/provider/config.py +36 -0
- hexkit-4.1.1/src/hexkit/providers/mongokafka/provider.py → hexkit-4.3.0/src/hexkit/providers/mongokafka/provider/daopub.py +3 -19
- hexkit-4.3.0/src/hexkit/providers/mongokafka/provider/persistent_pub.py +256 -0
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/mongokafka/testutils.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/__init__.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/provider.py +3 -3
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/test_files/__init__.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/testutils/__init__.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/testutils/_fixtures.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/testutils/_typical_workflow.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/testutils/_utils.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/testing/__init__.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/testing/eventpub.py +1 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/utils.py +2 -2
- {hexkit-4.1.1 → hexkit-4.3.0/src/hexkit.egg-info}/PKG-INFO +26 -22
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit.egg-info/SOURCES.txt +7 -1
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit.egg-info/requires.txt +9 -8
- {hexkit-4.1.1 → hexkit-4.3.0}/MANIFEST.in +0 -0
- {hexkit-4.1.1 → hexkit-4.3.0}/setup.cfg +0 -0
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/test_files/test_file1.yaml +0 -0
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/test_files/test_file2.yaml +0 -0
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/test_files/test_file3.yaml +0 -0
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/providers/s3/test_files/test_file4.yaml +0 -0
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit/py.typed +0 -0
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit.egg-info/dependency_links.txt +0 -0
- {hexkit-4.1.1 → hexkit-4.3.0}/src/hexkit.egg-info/top_level.txt +0 -0
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright 2021 -
|
|
189
|
+
Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
190
190
|
for the German Human Genome-Phenome Archive (GHGA)
|
|
191
191
|
|
|
192
192
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: hexkit
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.3.0
|
|
4
4
|
Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
|
|
5
5
|
Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
|
|
6
6
|
License: Apache 2.0
|
|
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
15
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
16
|
Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
|
|
16
17
|
Classifier: Topic :: Software Development :: Libraries
|
|
@@ -18,30 +19,32 @@ Classifier: Intended Audience :: Developers
|
|
|
18
19
|
Requires-Python: >=3.9
|
|
19
20
|
Description-Content-Type: text/markdown
|
|
20
21
|
License-File: LICENSE
|
|
21
|
-
Requires-Dist:
|
|
22
|
-
Requires-Dist:
|
|
22
|
+
Requires-Dist: opentelemetry-api<2,>=1.31.1
|
|
23
|
+
Requires-Dist: pydantic<3,>=2.10
|
|
24
|
+
Requires-Dist: pydantic_settings<3,>=2.8
|
|
23
25
|
Requires-Dist: PyYAML<7,>=6
|
|
24
26
|
Provides-Extra: akafka
|
|
25
27
|
Requires-Dist: aiokafka~=0.12.0; extra == "akafka"
|
|
26
28
|
Requires-Dist: jsonschema<5,>=4.23; extra == "akafka"
|
|
27
29
|
Provides-Extra: s3
|
|
28
|
-
Requires-Dist: boto3<2,>=1.
|
|
29
|
-
Requires-Dist: botocore<2,>=1.
|
|
30
|
+
Requires-Dist: boto3<2,>=1.37; extra == "s3"
|
|
31
|
+
Requires-Dist: botocore<2,>=1.37; extra == "s3"
|
|
30
32
|
Provides-Extra: mongodb
|
|
31
|
-
Requires-Dist: motor<4,>=3.
|
|
33
|
+
Requires-Dist: motor<4,>=3.7; extra == "mongodb"
|
|
32
34
|
Provides-Extra: test-akafka
|
|
33
35
|
Requires-Dist: hexkit[akafka]; extra == "test-akafka"
|
|
34
|
-
Requires-Dist: testcontainers[kafka]<5,>=4.9
|
|
36
|
+
Requires-Dist: testcontainers[kafka]<5,>=4.9; extra == "test-akafka"
|
|
35
37
|
Provides-Extra: test-s3
|
|
36
38
|
Requires-Dist: hexkit[s3]; extra == "test-s3"
|
|
37
|
-
Requires-Dist: testcontainers<5,>=4.9
|
|
39
|
+
Requires-Dist: testcontainers<5,>=4.9; extra == "test-s3"
|
|
38
40
|
Provides-Extra: test-mongodb
|
|
39
41
|
Requires-Dist: hexkit[mongodb]; extra == "test-mongodb"
|
|
40
|
-
Requires-Dist: testcontainers[mongo]<5,>=4.9
|
|
42
|
+
Requires-Dist: testcontainers[mongo]<5,>=4.9; extra == "test-mongodb"
|
|
41
43
|
Provides-Extra: test
|
|
42
44
|
Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
|
|
43
45
|
Provides-Extra: all
|
|
44
46
|
Requires-Dist: hexkit[test]; extra == "all"
|
|
47
|
+
Dynamic: license-file
|
|
45
48
|
|
|
46
49
|

|
|
47
50
|
[](https://pypi.python.org/pypi/hexkit/)
|
|
@@ -49,7 +52,7 @@ Requires-Dist: hexkit[test]; extra == "all"
|
|
|
49
52
|
[](https://coveralls.io/github/ghga-de/hexkit?branch=main)
|
|
50
53
|
|
|
51
54
|
# hexkit
|
|
52
|
-
A chassis library for building domain-focused, infrastructure-agnostic and event-driven
|
|
55
|
+
A chassis library for building domain-focused, infrastructure-agnostic, and event-driven
|
|
53
56
|
microservices in Python.
|
|
54
57
|
|
|
55
58
|
## In a Nutshell
|
|
@@ -76,8 +79,8 @@ for an individual service is to implement service-specific translators between t
|
|
|
76
79
|
service's ports and the general-purpose protocols (in addition to implementing the
|
|
77
80
|
domain functionality of the service, of course).
|
|
78
81
|
|
|
79
|
-
Hexkit is designed as a general-purpose library. However,
|
|
80
|
-
|
|
82
|
+
Hexkit is designed as a general-purpose library. However, it currently contains only
|
|
83
|
+
a limited collection of protocol-provider pairs that are of immediate interest to the
|
|
81
84
|
authors. We like to add support for more protocols and technologies over time.
|
|
82
85
|
|
|
83
86
|
The following protocols and providers are currently available:
|
|
@@ -94,9 +97,9 @@ for the rest. For example, you could use hexkit for simplifying the exchange of
|
|
|
94
97
|
between microservices but use a classical web framework such as FastAPI for designing
|
|
95
98
|
REST API and an ORM like SQLAlchemy for interacting with databases.
|
|
96
99
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
+
Feel free to develop hexagonal components yourself, whether or not you use the base
|
|
101
|
+
classes of hexkit. Not every protocol or provider has to be general-purpose, however,
|
|
102
|
+
if they are, please consider contributing them to hexkit.
|
|
100
103
|
|
|
101
104
|
## Getting started
|
|
102
105
|
|
|
@@ -132,25 +135,26 @@ The following extras are available:
|
|
|
132
135
|
For setting up the development environment, we rely on the
|
|
133
136
|
[devcontainer feature](https://code.visualstudio.com/docs/remote/containers) of vscode.
|
|
134
137
|
|
|
135
|
-
To use it, you
|
|
138
|
+
To use it, you need Docker and VS Code with the "Remote - Containers" extension
|
|
139
|
+
(`ms-vscode-remote.remote-containers`) installed.
|
|
136
140
|
Then, you just have to open this repo in vscode and run the command
|
|
137
141
|
`Remote-Containers: Reopen in Container` from the vscode "Command Palette".
|
|
138
142
|
|
|
139
143
|
This will give you a full-fledged, pre-configured development environment including:
|
|
140
144
|
- infrastructural dependencies of the service (databases, etc.)
|
|
141
145
|
- all relevant vscode extensions pre-installed
|
|
142
|
-
- pre-configured linting and auto-
|
|
146
|
+
- pre-configured linting and auto-formatting
|
|
143
147
|
- a pre-configured debugger
|
|
144
148
|
- automatic license-header insertion
|
|
145
149
|
|
|
146
|
-
|
|
147
|
-
(
|
|
150
|
+
Additionally, the following convenience command is available inside the devcontainer
|
|
151
|
+
(type it in the integrated terminal of VS Code):
|
|
148
152
|
- `dev_install` - install the lib with all development dependencies and pre-commit hooks
|
|
149
153
|
(please run that if you are starting the devcontainer for the first time
|
|
150
154
|
or if added any python dependencies to the [`./setup.cfg`](./setup.cfg))
|
|
151
155
|
|
|
152
|
-
If you prefer not to use vscode, you could get a similar setup (without the editor
|
|
153
|
-
by running the following commands:
|
|
156
|
+
If you prefer not to use vscode, you could get a similar setup (without the editor
|
|
157
|
+
specific features) by running the following commands:
|
|
154
158
|
``` bash
|
|
155
159
|
# Execute in the repo's root dir:
|
|
156
160
|
cd ./.devcontainer
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
[](https://coveralls.io/github/ghga-de/hexkit?branch=main)
|
|
5
5
|
|
|
6
6
|
# hexkit
|
|
7
|
-
A chassis library for building domain-focused, infrastructure-agnostic and event-driven
|
|
7
|
+
A chassis library for building domain-focused, infrastructure-agnostic, and event-driven
|
|
8
8
|
microservices in Python.
|
|
9
9
|
|
|
10
10
|
## In a Nutshell
|
|
@@ -31,8 +31,8 @@ for an individual service is to implement service-specific translators between t
|
|
|
31
31
|
service's ports and the general-purpose protocols (in addition to implementing the
|
|
32
32
|
domain functionality of the service, of course).
|
|
33
33
|
|
|
34
|
-
Hexkit is designed as a general-purpose library. However,
|
|
35
|
-
|
|
34
|
+
Hexkit is designed as a general-purpose library. However, it currently contains only
|
|
35
|
+
a limited collection of protocol-provider pairs that are of immediate interest to the
|
|
36
36
|
authors. We like to add support for more protocols and technologies over time.
|
|
37
37
|
|
|
38
38
|
The following protocols and providers are currently available:
|
|
@@ -49,9 +49,9 @@ for the rest. For example, you could use hexkit for simplifying the exchange of
|
|
|
49
49
|
between microservices but use a classical web framework such as FastAPI for designing
|
|
50
50
|
REST API and an ORM like SQLAlchemy for interacting with databases.
|
|
51
51
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
52
|
+
Feel free to develop hexagonal components yourself, whether or not you use the base
|
|
53
|
+
classes of hexkit. Not every protocol or provider has to be general-purpose, however,
|
|
54
|
+
if they are, please consider contributing them to hexkit.
|
|
55
55
|
|
|
56
56
|
## Getting started
|
|
57
57
|
|
|
@@ -87,25 +87,26 @@ The following extras are available:
|
|
|
87
87
|
For setting up the development environment, we rely on the
|
|
88
88
|
[devcontainer feature](https://code.visualstudio.com/docs/remote/containers) of vscode.
|
|
89
89
|
|
|
90
|
-
To use it, you
|
|
90
|
+
To use it, you need Docker and VS Code with the "Remote - Containers" extension
|
|
91
|
+
(`ms-vscode-remote.remote-containers`) installed.
|
|
91
92
|
Then, you just have to open this repo in vscode and run the command
|
|
92
93
|
`Remote-Containers: Reopen in Container` from the vscode "Command Palette".
|
|
93
94
|
|
|
94
95
|
This will give you a full-fledged, pre-configured development environment including:
|
|
95
96
|
- infrastructural dependencies of the service (databases, etc.)
|
|
96
97
|
- all relevant vscode extensions pre-installed
|
|
97
|
-
- pre-configured linting and auto-
|
|
98
|
+
- pre-configured linting and auto-formatting
|
|
98
99
|
- a pre-configured debugger
|
|
99
100
|
- automatic license-header insertion
|
|
100
101
|
|
|
101
|
-
|
|
102
|
-
(
|
|
102
|
+
Additionally, the following convenience command is available inside the devcontainer
|
|
103
|
+
(type it in the integrated terminal of VS Code):
|
|
103
104
|
- `dev_install` - install the lib with all development dependencies and pre-commit hooks
|
|
104
105
|
(please run that if you are starting the devcontainer for the first time
|
|
105
106
|
or if added any python dependencies to the [`./setup.cfg`](./setup.cfg))
|
|
106
107
|
|
|
107
|
-
If you prefer not to use vscode, you could get a similar setup (without the editor
|
|
108
|
-
by running the following commands:
|
|
108
|
+
If you prefer not to use vscode, you could get a similar setup (without the editor
|
|
109
|
+
specific features) by running the following commands:
|
|
109
110
|
``` bash
|
|
110
111
|
# Execute in the repo's root dir:
|
|
111
112
|
cd ./.devcontainer
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[build-system]
|
|
2
2
|
requires = [
|
|
3
|
-
"setuptools>=
|
|
3
|
+
"setuptools>=78.1",
|
|
4
4
|
]
|
|
5
5
|
build-backend = "setuptools.build_meta"
|
|
6
6
|
|
|
@@ -17,17 +17,19 @@ classifiers = [
|
|
|
17
17
|
"Programming Language :: Python :: 3.10",
|
|
18
18
|
"Programming Language :: Python :: 3.11",
|
|
19
19
|
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
20
21
|
"License :: OSI Approved :: Apache Software License",
|
|
21
22
|
"Topic :: Internet :: WWW/HTTP :: HTTP Servers",
|
|
22
23
|
"Topic :: Software Development :: Libraries",
|
|
23
24
|
"Intended Audience :: Developers",
|
|
24
25
|
]
|
|
25
26
|
name = "hexkit"
|
|
26
|
-
version = "4.
|
|
27
|
+
version = "4.3.0"
|
|
27
28
|
description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
|
|
28
29
|
dependencies = [
|
|
29
|
-
"
|
|
30
|
-
"
|
|
30
|
+
"opentelemetry-api>=1.31.1, <2",
|
|
31
|
+
"pydantic >=2.10, <3",
|
|
32
|
+
"pydantic_settings >=2.8, <3",
|
|
31
33
|
"PyYAML >=6, <7",
|
|
32
34
|
]
|
|
33
35
|
|
|
@@ -40,23 +42,23 @@ akafka = [
|
|
|
40
42
|
"jsonschema >=4.23, <5",
|
|
41
43
|
]
|
|
42
44
|
s3 = [
|
|
43
|
-
"boto3 >=1.
|
|
44
|
-
"botocore >=1.
|
|
45
|
+
"boto3 >=1.37, <2",
|
|
46
|
+
"botocore >=1.37, <2",
|
|
45
47
|
]
|
|
46
48
|
mongodb = [
|
|
47
|
-
"motor >=3.
|
|
49
|
+
"motor >=3.7, <4",
|
|
48
50
|
]
|
|
49
51
|
test-akafka = [
|
|
50
52
|
"hexkit[akafka]",
|
|
51
|
-
"testcontainers[kafka] >=4.9
|
|
53
|
+
"testcontainers[kafka] >=4.9, <5",
|
|
52
54
|
]
|
|
53
55
|
test-s3 = [
|
|
54
56
|
"hexkit[s3]",
|
|
55
|
-
"testcontainers >=4.9
|
|
57
|
+
"testcontainers >=4.9, <5",
|
|
56
58
|
]
|
|
57
59
|
test-mongodb = [
|
|
58
60
|
"hexkit[mongodb]",
|
|
59
|
-
"testcontainers[mongo] >=4.9
|
|
61
|
+
"testcontainers[mongo] >=4.9, <5",
|
|
60
62
|
]
|
|
61
63
|
test = [
|
|
62
64
|
"hexkit[test-akafka,test-s3,test-mongodb]",
|
|
@@ -175,8 +177,9 @@ check_untyped_defs = true
|
|
|
175
177
|
no_site_packages = false
|
|
176
178
|
|
|
177
179
|
[tool.pytest.ini_options]
|
|
178
|
-
minversion = "8.
|
|
180
|
+
minversion = "8.3"
|
|
179
181
|
asyncio_mode = "strict"
|
|
182
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
180
183
|
|
|
181
184
|
[tool.coverage.paths]
|
|
182
185
|
source = [
|
|
@@ -188,7 +191,7 @@ source = [
|
|
|
188
191
|
[tool.tox]
|
|
189
192
|
legacy_tox_ini = """
|
|
190
193
|
[tox]
|
|
191
|
-
env_list = py3{9,12}
|
|
194
|
+
env_list = py3{9,12,13}
|
|
192
195
|
|
|
193
196
|
[gh-actions]
|
|
194
197
|
python =
|
|
@@ -196,6 +199,7 @@ legacy_tox_ini = """
|
|
|
196
199
|
3.10: py310
|
|
197
200
|
3.11: py311
|
|
198
201
|
3.12: py312
|
|
202
|
+
3.13: py313
|
|
199
203
|
|
|
200
204
|
[testenv]
|
|
201
205
|
pass_env =
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -33,6 +33,9 @@ from datetime import datetime, timezone
|
|
|
33
33
|
from typing import Any, Callable, Literal, Optional, Protocol, TypeVar, Union, cast
|
|
34
34
|
|
|
35
35
|
from aiokafka import AIOKafkaConsumer
|
|
36
|
+
from opentelemetry import trace
|
|
37
|
+
from opentelemetry.propagate import extract
|
|
38
|
+
from opentelemetry.propagators import textmap
|
|
36
39
|
from pydantic import ValidationError
|
|
37
40
|
|
|
38
41
|
from hexkit.base import InboundProviderBase
|
|
@@ -50,6 +53,32 @@ from hexkit.providers.akafka.provider.utils import (
|
|
|
50
53
|
CHANGE_EVENT_TYPE = "upserted"
|
|
51
54
|
DELETE_EVENT_TYPE = "deleted"
|
|
52
55
|
EventOrDaoSubProtocol = Union[DaoSubscriberProtocol, EventSubscriberProtocol]
|
|
56
|
+
TextmapHeaders = list[tuple[str, bytes]]
|
|
57
|
+
|
|
58
|
+
tracer = trace.get_tracer_provider().get_tracer("hexkit.providers.akafka")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class AIOKafkaOtelContextExtractor(textmap.Getter[TextmapHeaders]):
|
|
62
|
+
"""OpenTelemetry context extractor.
|
|
63
|
+
|
|
64
|
+
This is slighlty adapted from the aiokafka instrumentation library.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
def get(self, carrier: TextmapHeaders, key: str) -> Optional[list[str]]:
|
|
68
|
+
"""Extract specific OpenTelemetry header from propagated context"""
|
|
69
|
+
if carrier is None:
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
for item_key, value in carrier:
|
|
73
|
+
if item_key == key and value is not None:
|
|
74
|
+
return [value.decode()]
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
def keys(self, carrier: TextmapHeaders) -> list[str]:
|
|
78
|
+
"""Get all headers from propagated context"""
|
|
79
|
+
if carrier is None:
|
|
80
|
+
return []
|
|
81
|
+
return [key for (key, value) in carrier]
|
|
53
82
|
|
|
54
83
|
|
|
55
84
|
class ComboTranslator(EventSubscriberProtocol):
|
|
@@ -610,39 +639,49 @@ class KafkaEventSubscriber(InboundProviderBase):
|
|
|
610
639
|
|
|
611
640
|
async def _consume_event(self, event: ConsumerEvent) -> None:
|
|
612
641
|
"""Consume an event by passing it down to the translator via the protocol."""
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
642
|
+
otel_extractor = AIOKafkaOtelContextExtractor()
|
|
643
|
+
extracted_context = extract(
|
|
644
|
+
event.headers,
|
|
645
|
+
getter=otel_extractor,
|
|
646
|
+
)
|
|
647
|
+
with tracer.start_as_current_span(
|
|
648
|
+
name="KafkaEventSubscriber._consume_event", context=extracted_context
|
|
649
|
+
):
|
|
650
|
+
event_id = get_event_id(event, service_name=self._service_name)
|
|
651
|
+
event_info = self._extract_info(event)
|
|
652
|
+
try:
|
|
653
|
+
self._validate_extracted_info(event_info)
|
|
654
|
+
except RuntimeError as err:
|
|
655
|
+
logging.info(
|
|
656
|
+
"Ignored event of type '%s': %s, errors: %s",
|
|
657
|
+
event_info.type_,
|
|
658
|
+
event_id,
|
|
659
|
+
str(err),
|
|
660
|
+
)
|
|
661
|
+
# Always acknowledge event receipt for ignored events
|
|
662
|
+
await self._consumer.commit()
|
|
663
|
+
return
|
|
627
664
|
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
665
|
+
try:
|
|
666
|
+
logging.info(
|
|
667
|
+
"Consuming event of type '%s': %s", event_info.type_, event_id
|
|
668
|
+
)
|
|
669
|
+
correlation_id = event_info.headers[HeaderNames.CORRELATION_ID]
|
|
670
|
+
async with set_correlation_id(correlation_id):
|
|
671
|
+
await self._handle_consumption(event=event_info, event_id=event_id)
|
|
672
|
+
except Exception:
|
|
673
|
+
# Errors only bubble up here if the DLQ isn't used
|
|
674
|
+
logging.critical(
|
|
675
|
+
"An error occurred while processing event of type '%s': %s. It was NOT"
|
|
676
|
+
" placed in the DLQ topic (%s)",
|
|
677
|
+
event_info.type_,
|
|
678
|
+
event_id,
|
|
679
|
+
self._dlq_topic if self._enable_dlq else "DLQ is disabled",
|
|
680
|
+
)
|
|
681
|
+
raise
|
|
682
|
+
else:
|
|
683
|
+
# Only save consumed event offsets if it was successful or sent to DLQ
|
|
684
|
+
await self._consumer.commit()
|
|
646
685
|
|
|
647
686
|
async def run(self, forever: bool = True) -> None:
|
|
648
687
|
"""Start consuming events and passing them down to the translator.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -152,7 +152,7 @@ def check_recorded_events(
|
|
|
152
152
|
|
|
153
153
|
def get_field_mismatch_error(field, index):
|
|
154
154
|
return get_detailed_error(
|
|
155
|
-
details=f"the {field} of the recorded event no. {index+1}"
|
|
155
|
+
details=f"the {field} of the recorded event no. {index + 1}"
|
|
156
156
|
" does not match the expectations"
|
|
157
157
|
)
|
|
158
158
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 -
|
|
1
|
+
# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
2
|
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -12,10 +12,24 @@
|
|
|
12
12
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
13
|
# See the License for the specific language governing permissions and
|
|
14
14
|
# limitations under the License.
|
|
15
|
-
#
|
|
16
15
|
|
|
17
|
-
"""
|
|
16
|
+
"""Database migration tools for MongoDB"""
|
|
18
17
|
|
|
19
|
-
from .
|
|
18
|
+
from ._manager import (
|
|
19
|
+
MigrationConfig,
|
|
20
|
+
MigrationManager,
|
|
21
|
+
MigrationMap,
|
|
22
|
+
MigrationStepError,
|
|
23
|
+
)
|
|
24
|
+
from ._utils import Document, MigrationDefinition, Reversible, validate_doc
|
|
20
25
|
|
|
21
|
-
__all__ = [
|
|
26
|
+
__all__ = [
|
|
27
|
+
"Document",
|
|
28
|
+
"MigrationConfig",
|
|
29
|
+
"MigrationDefinition",
|
|
30
|
+
"MigrationManager",
|
|
31
|
+
"MigrationMap",
|
|
32
|
+
"MigrationStepError",
|
|
33
|
+
"Reversible",
|
|
34
|
+
"validate_doc",
|
|
35
|
+
]
|