hexkit 8.4.1__tar.gz → 8.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hexkit-8.4.1/src/hexkit.egg-info → hexkit-8.5.0}/PKG-INFO +83 -97
- hexkit-8.5.0/README.md +118 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/pyproject.toml +17 -11
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/log.py +1 -1
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/protocols/dao.py +8 -8
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/protocols/objstorage.py +28 -5
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/akafka/provider/daosub.py +3 -1
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/akafka/provider/eventpub.py +5 -3
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/akafka/provider/eventsub.py +16 -14
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/migrations/__init__.py +11 -2
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/migrations/_manager.py +40 -71
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/migrations/_utils.py +92 -3
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/provider/dao.py +10 -8
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongokafka/config.py +3 -1
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongokafka/provider/daopub.py +10 -8
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/provider/objstorage.py +83 -78
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/testutils/_fixtures.py +2 -4
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/testutils/_utils.py +6 -2
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/testing/dao.py +4 -6
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/testing/objstorage.py +17 -3
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/utils.py +1 -1
- {hexkit-8.4.1 → hexkit-8.5.0/src/hexkit.egg-info}/PKG-INFO +83 -97
- hexkit-8.4.1/README.md +0 -132
- {hexkit-8.4.1 → hexkit-8.5.0}/LICENSE +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/MANIFEST.in +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/setup.cfg +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/__main__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/base.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/config.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/correlation.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/custom_types.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/opentelemetry.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/protocols/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/protocols/daopub.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/protocols/daosub.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/protocols/eventpub.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/protocols/eventsub.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/protocols/kvstore.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/akafka/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/akafka/config.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/akafka/provider/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/akafka/provider/utils.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/akafka/testcontainer.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/akafka/testutils.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/config.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/migrations/helpers.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/provider/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/provider/client.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/provider/kvstore.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/provider/utils.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongodb/testutils.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongokafka/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongokafka/provider/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongokafka/provider/persistent_pub.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/mongokafka/testutils.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/redis/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/redis/config.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/redis/provider/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/redis/provider/client.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/redis/provider/kvstore.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/redis/testutils.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/config.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/provider/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/provider/kvstore.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/test_files/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/test_files/test_file1.yaml +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/test_files/test_file2.yaml +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/test_files/test_file3.yaml +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/test_files/test_file4.yaml +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/testutils/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/s3/testutils/_typical_workflow.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/testing/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/testing/eventpub.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/testing/kvstore.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/vault/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/vault/config.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/vault/provider/__init__.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/vault/provider/client.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/vault/provider/kvstore.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/providers/vault/testutils.py +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit/py.typed +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit.egg-info/SOURCES.txt +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit.egg-info/dependency_links.txt +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit.egg-info/requires.txt +0 -0
- {hexkit-8.4.1 → hexkit-8.5.0}/src/hexkit.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hexkit
|
|
3
|
-
Version: 8.
|
|
3
|
+
Version: 8.5.0
|
|
4
4
|
Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
|
|
5
5
|
Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
|
|
6
6
|
License: Apache 2.0
|
|
@@ -89,75 +89,83 @@ Requires-Dist: hexkit[test]; extra == "all"
|
|
|
89
89
|
Requires-Dist: hexkit[opentelemetry]; extra == "all"
|
|
90
90
|
Dynamic: license-file
|
|
91
91
|
|
|
92
|
-

|
|
93
92
|
[](https://pypi.python.org/pypi/hexkit/)
|
|
94
93
|
[](https://pypi.python.org/pypi/hexkit/)
|
|
94
|
+

|
|
95
95
|
[](https://coveralls.io/github/ghga-de/hexkit?branch=main)
|
|
96
|
+
[](https://github.com/astral-sh/ruff)
|
|
97
|
+
[](https://mypy-lang.org/)
|
|
96
98
|
|
|
97
99
|
# hexkit
|
|
98
100
|
|
|
99
|
-
A chassis library for building domain-focused, infrastructure-agnostic,
|
|
100
|
-
microservices in Python
|
|
101
|
+
*A chassis library for building domain-focused, infrastructure-agnostic,
|
|
102
|
+
and event-driven microservices in Python*
|
|
103
|
+
|
|
104
|
+
Read the short summary below, or jump straight to our full
|
|
105
|
+
📖 **[User Guide](https://ghga-de.github.io/hexkit/user-guide/)** to learn the abstract
|
|
106
|
+
concepts and patterns as well as the concrete infrastructure integrations and testing
|
|
107
|
+
tools that hexkit provides for building better microservices.
|
|
108
|
+
|
|
109
|
+
<!--
|
|
110
|
+
NOTE: This README doubles as the landing page of the documentation site built with
|
|
111
|
+
great-docs. Therefore, all links must be absolute URLs — repo-relative links like
|
|
112
|
+
./examples or ./src would resolve correctly on GitHub but break on the site.
|
|
113
|
+
-->
|
|
101
114
|
|
|
102
115
|
## In a Nutshell
|
|
103
116
|
|
|
104
|
-
|
|
105
|
-
|
|
117
|
+
hexkit implements the *Triple Hexagonal Architecture* pattern, an optimization of the
|
|
118
|
+
ordinary
|
|
106
119
|
[Hexagonal Architecture](https://alistair.cockburn.us/hexagonal-architecture/) for
|
|
107
|
-
use with microservices
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
technology
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
that are service-independent - hence protocols and providers. The only task that remains
|
|
123
|
-
for an individual service is to implement service-specific translators between the
|
|
124
|
-
service's ports and the general-purpose protocols (in addition to implementing the
|
|
125
|
-
domain functionality of the service, of course).
|
|
126
|
-
|
|
127
|
-
Hexkit is designed as a general-purpose library. However, it currently contains only
|
|
128
|
-
a limited collection of protocol-provider pairs that are of immediate interest to the
|
|
129
|
-
authors. We like to add support for more protocols and technologies over time.
|
|
120
|
+
use with microservices: adapters (as per the hexagonal architecture) are divided into
|
|
121
|
+
two parts. One part called *translator* is specific to one individual microservice
|
|
122
|
+
and its domain-oriented ports. The other part called *provider* is service-agnostic
|
|
123
|
+
but specific to one technology of the surrounding infrastructure. Both parts interact
|
|
124
|
+
through a high-level interface called *protocol* (not to be confused with Python's
|
|
125
|
+
typing.Protocol) which is neither specific to the technology nor to the microservice.
|
|
126
|
+
|
|
127
|
+
As a chassis lib, hexkit reduces redundancy and boilerplate across microservices by
|
|
128
|
+
providing the service-independent building blocks — protocols and providers — ready
|
|
129
|
+
to use. The only task that remains for an individual service is to implement
|
|
130
|
+
service-specific translators between the service's ports and the general-purpose
|
|
131
|
+
protocols (in addition to implementing the domain functionality of the service, of
|
|
132
|
+
course). For an in-depth introduction to the pattern, please read the
|
|
133
|
+
[Architectural Concepts](https://ghga-de.github.io/hexkit/user-guide/arch_concepts/)
|
|
134
|
+
chapter of the User Guide.
|
|
130
135
|
|
|
131
136
|
The following protocols and providers are currently available:
|
|
132
137
|
|
|
133
|
-
| Protocol
|
|
134
|
-
|
|
135
|
-
| Event Publishing
|
|
136
|
-
| Event Subscription | Apache Kafka
|
|
137
|
-
| Object
|
|
138
|
-
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
## Getting
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
[
|
|
159
|
-
|
|
160
|
-
|
|
138
|
+
| Protocol | Providers |
|
|
139
|
+
| ------------------ | -------------------------------------- |
|
|
140
|
+
| Event Publishing | Apache Kafka, MongoDB + Kafka (outbox) |
|
|
141
|
+
| Event Subscription | Apache Kafka |
|
|
142
|
+
| Data Access Object | MongoDB |
|
|
143
|
+
| Object Storage | S3(-compatible) |
|
|
144
|
+
| Key-Value Store | MongoDB, Redis, S3, HashiCorp Vault |
|
|
145
|
+
|
|
146
|
+
In-memory implementations and per-backend test utilities are also included to
|
|
147
|
+
support testing without real infrastructure.
|
|
148
|
+
|
|
149
|
+
You are not forced to go all-in on the idea of Triple Hexagonal Architecture.
|
|
150
|
+
You can use it just for the technologies where you see benefits and use another
|
|
151
|
+
approach for the rest. For example, you could use hexkit for simplifying the exchange
|
|
152
|
+
of events between microservices but use a classical web framework such as FastAPI for
|
|
153
|
+
designing REST APIs and an ORM like SQLAlchemy for interacting with databases.
|
|
154
|
+
|
|
155
|
+
## Getting Started
|
|
156
|
+
|
|
157
|
+
The 📖 [User Guide](https://ghga-de.github.io/hexkit/user-guide/) covers all
|
|
158
|
+
protocols, providers, testing utilities, and observability tools with example code,
|
|
159
|
+
and the [API Reference](https://ghga-de.github.io/hexkit/reference/) documents all
|
|
160
|
+
public classes and functions.
|
|
161
|
+
|
|
162
|
+
For a complete example service built with hexkit, have a look at the
|
|
163
|
+
[stream_calc](https://github.com/ghga-de/hexkit/tree/main/examples/stream_calc)
|
|
164
|
+
example application. We also put a lot of effort into making the code
|
|
165
|
+
self-documenting: you can find the protocols being defined at
|
|
166
|
+
[src/hexkit/protocols](https://github.com/ghga-de/hexkit/tree/main/src/hexkit/protocols)
|
|
167
|
+
and the providers being implemented at
|
|
168
|
+
[src/hexkit/providers](https://github.com/ghga-de/hexkit/tree/main/src/hexkit/providers).
|
|
161
169
|
|
|
162
170
|
## Installation
|
|
163
171
|
|
|
@@ -173,51 +181,29 @@ pip install hexkit
|
|
|
173
181
|
The following extras are available:
|
|
174
182
|
|
|
175
183
|
- `akafka`: when using the Apache Kafka-based event publishing or subscription
|
|
176
|
-
- `
|
|
177
|
-
- `
|
|
178
|
-
- `
|
|
179
|
-
- `
|
|
180
|
-
- `
|
|
184
|
+
- `mongodb`: when using MongoDB as backend for the DAO protocol or key-value stores
|
|
185
|
+
- `s3`: when interacting with S3-compatible object storages or key-value stores
|
|
186
|
+
- `redis`: when using Redis-based key-value stores
|
|
187
|
+
- `vault`: when using HashiCorp Vault-based key-value stores
|
|
188
|
+
- `opentelemetry`: observability instrumentation (also available per backend, e.g.
|
|
189
|
+
`opentelemetry-akafka`, `opentelemetry-mongodb`, `opentelemetry-fastapi`)
|
|
190
|
+
- `test-akafka`, `test-mongodb`, `test-s3`, `test-redis`, `test-vault`: testing
|
|
191
|
+
utils for the respective backend (`test` is a union of all of them)
|
|
181
192
|
- `all`: a union of all the above
|
|
182
193
|
|
|
183
|
-
##
|
|
184
|
-
|
|
185
|
-
For setting up the development environment, we rely on the
|
|
186
|
-
[devcontainer feature](https://code.visualstudio.com/docs/remote/containers) of vscode.
|
|
187
|
-
|
|
188
|
-
To use it, you need Docker and VS Code with the "Remote - Containers" extension
|
|
189
|
-
(`ms-vscode-remote.remote-containers`) installed.
|
|
190
|
-
Then, you just have to open this repo in vscode and run the command
|
|
191
|
-
`Remote-Containers: Reopen in Container` from the vscode "Command Palette".
|
|
192
|
-
|
|
193
|
-
This will give you a full-fledged, pre-configured development environment including:
|
|
194
|
-
|
|
195
|
-
- infrastructural dependencies of the service (databases, etc.)
|
|
196
|
-
- all relevant vscode extensions pre-installed
|
|
197
|
-
- pre-configured linting and auto-formatting
|
|
198
|
-
- a pre-configured debugger
|
|
199
|
-
- automatic license-header insertion
|
|
194
|
+
## Contributing
|
|
200
195
|
|
|
201
|
-
|
|
202
|
-
(
|
|
196
|
+
Contributions are welcome! Please see the
|
|
197
|
+
[contribution guide](https://github.com/ghga-de/hexkit/blob/main/CONTRIBUTING.md)
|
|
198
|
+
for how to set up the development environment and build the documentation locally.
|
|
203
199
|
|
|
204
|
-
|
|
205
|
-
(please run that if you are starting the devcontainer for the first time
|
|
206
|
-
or if added any python dependencies to the [`./pyproject.toml`](./pyproject.toml))
|
|
200
|
+
## Credits
|
|
207
201
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
``` bash
|
|
212
|
-
# Execute in the repo's root dir:
|
|
213
|
-
cd ./.devcontainer
|
|
214
|
-
|
|
215
|
-
# build and run the environment with docker-compose
|
|
216
|
-
docker build -t hexkit .
|
|
217
|
-
docker run -it hexkit /bin/bash
|
|
218
|
-
|
|
219
|
-
```
|
|
202
|
+
The hexkit library is developed and maintained by the developer team of GHGA. We would especially
|
|
203
|
+
like to thank its original author, Kersten Breuer, who came up with the idea for the
|
|
204
|
+
library and designed and coded its first versions.
|
|
220
205
|
|
|
221
206
|
## License
|
|
222
207
|
|
|
223
|
-
This repository is free to use and modify according to the
|
|
208
|
+
This repository is free to use and modify according to the
|
|
209
|
+
[Apache 2.0 License](https://github.com/ghga-de/hexkit/blob/main/LICENSE).
|
hexkit-8.5.0/README.md
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
[](https://pypi.python.org/pypi/hexkit/)
|
|
2
|
+
[](https://pypi.python.org/pypi/hexkit/)
|
|
3
|
+

|
|
4
|
+
[](https://coveralls.io/github/ghga-de/hexkit?branch=main)
|
|
5
|
+
[](https://github.com/astral-sh/ruff)
|
|
6
|
+
[](https://mypy-lang.org/)
|
|
7
|
+
|
|
8
|
+
# hexkit
|
|
9
|
+
|
|
10
|
+
*A chassis library for building domain-focused, infrastructure-agnostic,
|
|
11
|
+
and event-driven microservices in Python*
|
|
12
|
+
|
|
13
|
+
Read the short summary below, or jump straight to our full
|
|
14
|
+
📖 **[User Guide](https://ghga-de.github.io/hexkit/user-guide/)** to learn the abstract
|
|
15
|
+
concepts and patterns as well as the concrete infrastructure integrations and testing
|
|
16
|
+
tools that hexkit provides for building better microservices.
|
|
17
|
+
|
|
18
|
+
<!--
|
|
19
|
+
NOTE: This README doubles as the landing page of the documentation site built with
|
|
20
|
+
great-docs. Therefore, all links must be absolute URLs — repo-relative links like
|
|
21
|
+
./examples or ./src would resolve correctly on GitHub but break on the site.
|
|
22
|
+
-->
|
|
23
|
+
|
|
24
|
+
## In a Nutshell
|
|
25
|
+
|
|
26
|
+
hexkit implements the *Triple Hexagonal Architecture* pattern, an optimization of the
|
|
27
|
+
ordinary
|
|
28
|
+
[Hexagonal Architecture](https://alistair.cockburn.us/hexagonal-architecture/) for
|
|
29
|
+
use with microservices: adapters (as per the hexagonal architecture) are divided into
|
|
30
|
+
two parts. One part called *translator* is specific to one individual microservice
|
|
31
|
+
and its domain-oriented ports. The other part called *provider* is service-agnostic
|
|
32
|
+
but specific to one technology of the surrounding infrastructure. Both parts interact
|
|
33
|
+
through a high-level interface called *protocol* (not to be confused with Python's
|
|
34
|
+
typing.Protocol) which is neither specific to the technology nor to the microservice.
|
|
35
|
+
|
|
36
|
+
As a chassis lib, hexkit reduces redundancy and boilerplate across microservices by
|
|
37
|
+
providing the service-independent building blocks — protocols and providers — ready
|
|
38
|
+
to use. The only task that remains for an individual service is to implement
|
|
39
|
+
service-specific translators between the service's ports and the general-purpose
|
|
40
|
+
protocols (in addition to implementing the domain functionality of the service, of
|
|
41
|
+
course). For an in-depth introduction to the pattern, please read the
|
|
42
|
+
[Architectural Concepts](https://ghga-de.github.io/hexkit/user-guide/arch_concepts/)
|
|
43
|
+
chapter of the User Guide.
|
|
44
|
+
|
|
45
|
+
The following protocols and providers are currently available:
|
|
46
|
+
|
|
47
|
+
| Protocol | Providers |
|
|
48
|
+
| ------------------ | -------------------------------------- |
|
|
49
|
+
| Event Publishing | Apache Kafka, MongoDB + Kafka (outbox) |
|
|
50
|
+
| Event Subscription | Apache Kafka |
|
|
51
|
+
| Data Access Object | MongoDB |
|
|
52
|
+
| Object Storage | S3(-compatible) |
|
|
53
|
+
| Key-Value Store | MongoDB, Redis, S3, HashiCorp Vault |
|
|
54
|
+
|
|
55
|
+
In-memory implementations and per-backend test utilities are also included to
|
|
56
|
+
support testing without real infrastructure.
|
|
57
|
+
|
|
58
|
+
You are not forced to go all-in on the idea of Triple Hexagonal Architecture.
|
|
59
|
+
You can use it just for the technologies where you see benefits and use another
|
|
60
|
+
approach for the rest. For example, you could use hexkit for simplifying the exchange
|
|
61
|
+
of events between microservices but use a classical web framework such as FastAPI for
|
|
62
|
+
designing REST APIs and an ORM like SQLAlchemy for interacting with databases.
|
|
63
|
+
|
|
64
|
+
## Getting Started
|
|
65
|
+
|
|
66
|
+
The 📖 [User Guide](https://ghga-de.github.io/hexkit/user-guide/) covers all
|
|
67
|
+
protocols, providers, testing utilities, and observability tools with example code,
|
|
68
|
+
and the [API Reference](https://ghga-de.github.io/hexkit/reference/) documents all
|
|
69
|
+
public classes and functions.
|
|
70
|
+
|
|
71
|
+
For a complete example service built with hexkit, have a look at the
|
|
72
|
+
[stream_calc](https://github.com/ghga-de/hexkit/tree/main/examples/stream_calc)
|
|
73
|
+
example application. We also put a lot of effort into making the code
|
|
74
|
+
self-documenting: you can find the protocols being defined at
|
|
75
|
+
[src/hexkit/protocols](https://github.com/ghga-de/hexkit/tree/main/src/hexkit/protocols)
|
|
76
|
+
and the providers being implemented at
|
|
77
|
+
[src/hexkit/providers](https://github.com/ghga-de/hexkit/tree/main/src/hexkit/providers).
|
|
78
|
+
|
|
79
|
+
## Installation
|
|
80
|
+
|
|
81
|
+
This package is available at PyPI:
|
|
82
|
+
<https://pypi.org/project/hexkit>
|
|
83
|
+
|
|
84
|
+
You can install it from there using:
|
|
85
|
+
|
|
86
|
+
```sh
|
|
87
|
+
pip install hexkit
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The following extras are available:
|
|
91
|
+
|
|
92
|
+
- `akafka`: when using the Apache Kafka-based event publishing or subscription
|
|
93
|
+
- `mongodb`: when using MongoDB as backend for the DAO protocol or key-value stores
|
|
94
|
+
- `s3`: when interacting with S3-compatible object storages or key-value stores
|
|
95
|
+
- `redis`: when using Redis-based key-value stores
|
|
96
|
+
- `vault`: when using HashiCorp Vault-based key-value stores
|
|
97
|
+
- `opentelemetry`: observability instrumentation (also available per backend, e.g.
|
|
98
|
+
`opentelemetry-akafka`, `opentelemetry-mongodb`, `opentelemetry-fastapi`)
|
|
99
|
+
- `test-akafka`, `test-mongodb`, `test-s3`, `test-redis`, `test-vault`: testing
|
|
100
|
+
utils for the respective backend (`test` is a union of all of them)
|
|
101
|
+
- `all`: a union of all the above
|
|
102
|
+
|
|
103
|
+
## Contributing
|
|
104
|
+
|
|
105
|
+
Contributions are welcome! Please see the
|
|
106
|
+
[contribution guide](https://github.com/ghga-de/hexkit/blob/main/CONTRIBUTING.md)
|
|
107
|
+
for how to set up the development environment and build the documentation locally.
|
|
108
|
+
|
|
109
|
+
## Credits
|
|
110
|
+
|
|
111
|
+
The hexkit library is developed and maintained by the developer team of GHGA. We would especially
|
|
112
|
+
like to thank its original author, Kersten Breuer, who came up with the idea for the
|
|
113
|
+
library and designed and coded its first versions.
|
|
114
|
+
|
|
115
|
+
## License
|
|
116
|
+
|
|
117
|
+
This repository is free to use and modify according to the
|
|
118
|
+
[Apache 2.0 License](https://github.com/ghga-de/hexkit/blob/main/LICENSE).
|
|
@@ -23,7 +23,7 @@ classifiers = [
|
|
|
23
23
|
"Intended Audience :: Developers",
|
|
24
24
|
]
|
|
25
25
|
name = "hexkit"
|
|
26
|
-
version = "8.
|
|
26
|
+
version = "8.5.0"
|
|
27
27
|
description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
|
|
28
28
|
dependencies = [
|
|
29
29
|
"opentelemetry-api >=1.39, <2",
|
|
@@ -157,6 +157,9 @@ ignore = [
|
|
|
157
157
|
"E111",
|
|
158
158
|
"E114",
|
|
159
159
|
"E116",
|
|
160
|
+
"E117",
|
|
161
|
+
"E501",
|
|
162
|
+
"W191",
|
|
160
163
|
"PLW",
|
|
161
164
|
"RUF001",
|
|
162
165
|
"RUF010",
|
|
@@ -185,6 +188,13 @@ select = [
|
|
|
185
188
|
"RUF",
|
|
186
189
|
"SIM",
|
|
187
190
|
"D",
|
|
191
|
+
"E",
|
|
192
|
+
"W",
|
|
193
|
+
"ASYNC",
|
|
194
|
+
"LOG",
|
|
195
|
+
"PERF",
|
|
196
|
+
"RET",
|
|
197
|
+
"C4",
|
|
188
198
|
]
|
|
189
199
|
|
|
190
200
|
[tool.ruff.lint.mccabe]
|
|
@@ -196,6 +206,7 @@ max-complexity = 10
|
|
|
196
206
|
"S",
|
|
197
207
|
"SIM",
|
|
198
208
|
"D",
|
|
209
|
+
"E402",
|
|
199
210
|
]
|
|
200
211
|
"tests/*" = [
|
|
201
212
|
"S",
|
|
@@ -256,19 +267,14 @@ legacy_tox_ini = """
|
|
|
256
267
|
|
|
257
268
|
[testenv:docs]
|
|
258
269
|
basepython = python3.13
|
|
259
|
-
allowlist_externals =
|
|
260
|
-
mv
|
|
261
|
-
rm
|
|
262
270
|
deps =
|
|
263
271
|
--no-deps -r ./lock/requirements-dev.txt
|
|
272
|
+
commands_pre =
|
|
273
|
+
# great-docs (Python >= 3.11 only) is kept out of the shared dev lock so it
|
|
274
|
+
# does not break the py310 test env; install it here where docs are built.
|
|
275
|
+
pip install great-docs==0.14.1
|
|
264
276
|
commands =
|
|
265
|
-
|
|
266
|
-
pdoc -d google -o docs/api hexkit
|
|
267
|
-
mv docs/api/index.html docs/api/index.md
|
|
268
|
-
mkdocs build
|
|
269
|
-
mv docs/api/index.md docs/api/index.html
|
|
270
|
-
rm -rf site/api
|
|
271
|
-
mv docs/api site/api
|
|
277
|
+
great-docs build
|
|
272
278
|
|
|
273
279
|
[testenv]
|
|
274
280
|
pass_env =
|
|
@@ -105,7 +105,7 @@ class JsonFormatter(Formatter):
|
|
|
105
105
|
"""
|
|
106
106
|
# Create a log record dictionary
|
|
107
107
|
log_record = record.__dict__
|
|
108
|
-
output: dict[str, Any] =
|
|
108
|
+
output: dict[str, Any] = {}
|
|
109
109
|
|
|
110
110
|
output["timestamp"] = log_record["timestamp"]
|
|
111
111
|
output["service"] = log_record["service"]
|
|
@@ -243,10 +243,10 @@ class Dao(typing.Protocol[Dto]):
|
|
|
243
243
|
It is expected that at most one resource matches the constraints.
|
|
244
244
|
An exception is raised if no or multiple hits are found.
|
|
245
245
|
|
|
246
|
-
The values in the mapping are used to filter the resources
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
on the specific provider.
|
|
246
|
+
The values in the mapping are used to filter the resources. Provide them
|
|
247
|
+
using the same Python types as the corresponding DTO model fields, e.g. a
|
|
248
|
+
UUID object for a UUID field and a datetime object for a datetime field.
|
|
249
|
+
The behavior for non-scalar values depends on the specific provider.
|
|
250
250
|
|
|
251
251
|
Args:
|
|
252
252
|
mapping:
|
|
@@ -275,10 +275,10 @@ class Dao(typing.Protocol[Dto]):
|
|
|
275
275
|
) -> "FindResult[Dto]":
|
|
276
276
|
"""Find all resources that match the specified mapping.
|
|
277
277
|
|
|
278
|
-
The values in the mapping are used to filter the resources
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
on the specific provider.
|
|
278
|
+
The values in the mapping are used to filter the resources. Provide them
|
|
279
|
+
using the same Python types as the corresponding DTO model fields, e.g. a
|
|
280
|
+
UUID object for a UUID field and a datetime object for a datetime field.
|
|
281
|
+
The behavior for non-scalar values depends on the specific provider.
|
|
282
282
|
|
|
283
283
|
Args:
|
|
284
284
|
mapping:
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
import re
|
|
22
22
|
from abc import ABC, abstractmethod
|
|
23
|
-
from typing import NamedTuple
|
|
23
|
+
from typing import Any, NamedTuple
|
|
24
24
|
|
|
25
25
|
__all__ = ["ObjectStorageProtocol", "PresignedPostURL"]
|
|
26
26
|
|
|
@@ -214,8 +214,8 @@ class ObjectStorageProtocol(ABC):
|
|
|
214
214
|
start at 1, not 0.
|
|
215
215
|
|
|
216
216
|
Raises:
|
|
217
|
-
ValueError if `max_parts` or `first_part_no` are invalid.
|
|
218
|
-
MultiPartUploadNotFoundError if no upload with `upload_id` exists.
|
|
217
|
+
ValueError: if `max_parts` or `first_part_no` are invalid.
|
|
218
|
+
MultiPartUploadNotFoundError: if no upload with `upload_id` exists.
|
|
219
219
|
"""
|
|
220
220
|
self._validate_bucket_id(bucket_id)
|
|
221
221
|
self._validate_object_id(object_id)
|
|
@@ -278,6 +278,14 @@ class ObjectStorageProtocol(ABC):
|
|
|
278
278
|
self._validate_object_id(object_id)
|
|
279
279
|
return await self._get_object_size(bucket_id=bucket_id, object_id=object_id)
|
|
280
280
|
|
|
281
|
+
async def get_object_metadata(
|
|
282
|
+
self, *, bucket_id: str, object_id: str
|
|
283
|
+
) -> dict[str, Any]:
|
|
284
|
+
"""Returns object metadata without downloading the actual object."""
|
|
285
|
+
self._validate_bucket_id(bucket_id)
|
|
286
|
+
self._validate_object_id(object_id)
|
|
287
|
+
return await self._get_object_metadata(bucket_id=bucket_id, object_id=object_id)
|
|
288
|
+
|
|
281
289
|
async def copy_object(
|
|
282
290
|
self,
|
|
283
291
|
*,
|
|
@@ -310,6 +318,9 @@ class ObjectStorageProtocol(ABC):
|
|
|
310
318
|
async def delete_object(self, *, bucket_id: str, object_id: str) -> None:
|
|
311
319
|
"""Delete an object with the specified id (`object_id`) in the bucket with the
|
|
312
320
|
specified id (`bucket_id`).
|
|
321
|
+
|
|
322
|
+
Deleting an object that does not exist succeeds silently. However, a
|
|
323
|
+
`BucketNotFoundError` is raised if the bucket does not exist.
|
|
313
324
|
"""
|
|
314
325
|
self._validate_bucket_id(bucket_id)
|
|
315
326
|
self._validate_object_id(object_id)
|
|
@@ -492,8 +503,8 @@ class ObjectStorageProtocol(ABC):
|
|
|
492
503
|
start at 1, not 0.
|
|
493
504
|
|
|
494
505
|
Raises:
|
|
495
|
-
ValueError if `max_parts` or `first_part_no` are invalid.
|
|
496
|
-
MultiPartUploadNotFoundError if no upload with `upload_id` exists.
|
|
506
|
+
ValueError: if `max_parts` or `first_part_no` are invalid.
|
|
507
|
+
MultiPartUploadNotFoundError: if no upload with `upload_id` exists.
|
|
497
508
|
"""
|
|
498
509
|
...
|
|
499
510
|
|
|
@@ -508,6 +519,7 @@ class ObjectStorageProtocol(ABC):
|
|
|
508
519
|
*To be implemented by the provider. Input validation is done outside of this
|
|
509
520
|
method.*
|
|
510
521
|
"""
|
|
522
|
+
...
|
|
511
523
|
|
|
512
524
|
@abstractmethod
|
|
513
525
|
async def _get_object_download_url(
|
|
@@ -530,6 +542,7 @@ class ObjectStorageProtocol(ABC):
|
|
|
530
542
|
*To be implemented by the provider. Input validation is done outside of this
|
|
531
543
|
method.*
|
|
532
544
|
"""
|
|
545
|
+
...
|
|
533
546
|
|
|
534
547
|
@abstractmethod
|
|
535
548
|
async def _get_object_size(self, *, bucket_id: str, object_id: str) -> int:
|
|
@@ -541,6 +554,13 @@ class ObjectStorageProtocol(ABC):
|
|
|
541
554
|
"""
|
|
542
555
|
...
|
|
543
556
|
|
|
557
|
+
@abstractmethod
|
|
558
|
+
async def _get_object_metadata(
|
|
559
|
+
self, *, bucket_id: str, object_id: str
|
|
560
|
+
) -> dict[str, Any]:
|
|
561
|
+
"""Returns object metadata without downloading the actual object."""
|
|
562
|
+
...
|
|
563
|
+
|
|
544
564
|
@abstractmethod
|
|
545
565
|
async def _does_object_exist(
|
|
546
566
|
self, *, bucket_id: str, object_id: str, object_md5sum: str | None = None
|
|
@@ -585,6 +605,9 @@ class ObjectStorageProtocol(ABC):
|
|
|
585
605
|
"""Delete an object with the specified id (`object_id`) in the bucket with the
|
|
586
606
|
specified id (`bucket_id`).
|
|
587
607
|
|
|
608
|
+
Deleting an object that does not exist succeeds silently. However, a
|
|
609
|
+
`BucketNotFoundError` is raised if the bucket does not exist.
|
|
610
|
+
|
|
588
611
|
*To be implemented by the provider. Input validation is done outside of this
|
|
589
612
|
method.*
|
|
590
613
|
"""
|
|
@@ -38,6 +38,8 @@ from hexkit.providers.akafka.provider.eventsub import (
|
|
|
38
38
|
ComboTranslator as TranslatorConverter, # for backwards compatibility
|
|
39
39
|
)
|
|
40
40
|
|
|
41
|
+
log = logging.getLogger(__name__)
|
|
42
|
+
|
|
41
43
|
|
|
42
44
|
class KafkaOutboxSubscriber(InboundProviderBase):
|
|
43
45
|
"""Apache Kafka-specific provider using translators that implement the
|
|
@@ -76,7 +78,7 @@ class KafkaOutboxSubscriber(InboundProviderBase):
|
|
|
76
78
|
|
|
77
79
|
if config.kafka_enable_dlq and dlq_publisher is None:
|
|
78
80
|
error = ValueError("A publisher is required when the DLQ is enabled.")
|
|
79
|
-
|
|
81
|
+
log.error(error)
|
|
80
82
|
raise error
|
|
81
83
|
|
|
82
84
|
async with KafkaEventSubscriber.construct(
|
|
@@ -45,6 +45,8 @@ from hexkit.providers.akafka.provider.utils import (
|
|
|
45
45
|
generate_ssl_context,
|
|
46
46
|
)
|
|
47
47
|
|
|
48
|
+
log = logging.getLogger(__name__)
|
|
49
|
+
|
|
48
50
|
RESERVED_HEADERS = ["type", "correlation_id", "event_id"]
|
|
49
51
|
|
|
50
52
|
|
|
@@ -198,7 +200,7 @@ class KafkaEventPublisher(EventPublisherProtocol):
|
|
|
198
200
|
raise
|
|
199
201
|
|
|
200
202
|
correlation_id = new_correlation_id()
|
|
201
|
-
|
|
203
|
+
log.info("Generated new correlation ID: %s.", correlation_id)
|
|
202
204
|
|
|
203
205
|
# Create a shallow copy of the headers
|
|
204
206
|
headers_copy = dict(headers) # used for validation and logging only
|
|
@@ -220,12 +222,12 @@ class KafkaEventPublisher(EventPublisherProtocol):
|
|
|
220
222
|
f"The '{header}' header shouldn't be supplied, but was."
|
|
221
223
|
f" Overwriting old value ({old_value}) with {headers_copy[header]}."
|
|
222
224
|
)
|
|
223
|
-
|
|
225
|
+
log.warning(log_msg, extra={header: old_value})
|
|
224
226
|
|
|
225
227
|
await self._producer.send_and_wait(
|
|
226
228
|
topic, key=key, value=payload, headers=encoded_headers_list
|
|
227
229
|
)
|
|
228
|
-
|
|
230
|
+
log.info(
|
|
229
231
|
"Published event: topic=%s, type=%s, key=%s, event_id=%s.",
|
|
230
232
|
topic,
|
|
231
233
|
type_,
|