hexkit 3.4.0__tar.gz → 3.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hexkit-3.4.0/src/hexkit.egg-info → hexkit-3.6.0}/PKG-INFO +1 -1
- {hexkit-3.4.0 → hexkit-3.6.0}/pyproject.toml +1 -1
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/config.py +7 -30
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/dao.py +15 -1
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/config.py +7 -1
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/provider/eventpub.py +4 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/provider/eventsub.py +4 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongodb/provider.py +18 -1
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongokafka/provider.py +28 -2
- hexkit-3.6.0/src/hexkit/providers/s3/testutils/__init__.py +94 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/testutils/_fixtures.py +148 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/testutils/_utils.py +2 -2
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/testing/eventpub.py +2 -0
- {hexkit-3.4.0 → hexkit-3.6.0/src/hexkit.egg-info}/PKG-INFO +1 -1
- hexkit-3.4.0/src/hexkit/providers/s3/testutils/__init__.py +0 -47
- {hexkit-3.4.0 → hexkit-3.6.0}/LICENSE +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/MANIFEST.in +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/README.md +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/setup.cfg +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/__init__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/__main__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/base.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/correlation.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/custom_types.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/log.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/__init__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/daopub.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/daosub.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/eventpub.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/eventsub.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/objstorage.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/__init__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/__init__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/provider/__init__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/provider/daosub.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/provider/utils.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/testcontainer.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/testutils.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongodb/__init__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongodb/testutils.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongokafka/__init__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongokafka/testutils.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/__init__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/provider.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/test_files/__init__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/test_files/test_file1.yaml +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/test_files/test_file2.yaml +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/test_files/test_file3.yaml +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/test_files/test_file4.yaml +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/testutils/_typical_workflow.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/testing/__init__.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/py.typed +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/utils.py +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit.egg-info/SOURCES.txt +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit.egg-info/dependency_links.txt +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit.egg-info/requires.txt +0 -0
- {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit.egg-info/top_level.txt +0 -0
|
@@ -17,13 +17,13 @@
|
|
|
17
17
|
|
|
18
18
|
import os
|
|
19
19
|
from pathlib import Path
|
|
20
|
-
from typing import
|
|
20
|
+
from typing import Callable, Final, Optional
|
|
21
21
|
|
|
22
|
-
import yaml
|
|
23
22
|
from pydantic_settings import (
|
|
24
23
|
BaseSettings,
|
|
25
24
|
PydanticBaseSettingsSource,
|
|
26
25
|
SettingsConfigDict,
|
|
26
|
+
YamlConfigSettingsSource,
|
|
27
27
|
)
|
|
28
28
|
|
|
29
29
|
# Default config prefix:
|
|
@@ -79,29 +79,6 @@ def get_default_config_yaml(prefix: str) -> Optional[Path]:
|
|
|
79
79
|
return None
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
def yaml_settings_factory(
|
|
83
|
-
config_yaml: Optional[Path] = None,
|
|
84
|
-
) -> Callable[[], dict[str, Any]]:
|
|
85
|
-
"""
|
|
86
|
-
A factory of source methods for pydantic's BaseSettings Config that load
|
|
87
|
-
settings from a yaml file.
|
|
88
|
-
|
|
89
|
-
Args:
|
|
90
|
-
config_yaml (str, Optional):
|
|
91
|
-
Path to the yaml file to read from.
|
|
92
|
-
"""
|
|
93
|
-
|
|
94
|
-
def yaml_settings() -> dict[str, Any]:
|
|
95
|
-
"""Source method for loading pydantic BaseSettings from a yaml file"""
|
|
96
|
-
if config_yaml is None:
|
|
97
|
-
return {}
|
|
98
|
-
|
|
99
|
-
with open(config_yaml, encoding="utf8") as yaml_file:
|
|
100
|
-
return yaml.safe_load(yaml_file)
|
|
101
|
-
|
|
102
|
-
return yaml_settings
|
|
103
|
-
|
|
104
|
-
|
|
105
82
|
def config_from_yaml(
|
|
106
83
|
prefix: str = DEFAULT_CONFIG_PREFIX,
|
|
107
84
|
) -> Callable:
|
|
@@ -119,7 +96,7 @@ def config_from_yaml(
|
|
|
119
96
|
|
|
120
97
|
Args:
|
|
121
98
|
prefix: (str, optional):
|
|
122
|
-
When defining parameters via
|
|
99
|
+
When defining parameters via environment variables, all variables
|
|
123
100
|
have to be prefixed with this string following this pattern
|
|
124
101
|
"{prefix}_{actual_variable_name}". Moreover, this prefix is used
|
|
125
102
|
to derive the default location for the config yaml file
|
|
@@ -161,12 +138,12 @@ def config_from_yaml(
|
|
|
161
138
|
raise ConfigYamlDoesNotExist(path=config_yaml)
|
|
162
139
|
|
|
163
140
|
class ModSettings(settings):
|
|
164
|
-
"""Modifies the
|
|
141
|
+
"""Modifies the original Settings class provided by the user"""
|
|
165
142
|
|
|
166
143
|
model_config = SettingsConfigDict(frozen=True, env_prefix=f"{prefix}_")
|
|
167
144
|
|
|
168
145
|
@classmethod
|
|
169
|
-
def settings_customise_sources(
|
|
146
|
+
def settings_customise_sources(
|
|
170
147
|
cls,
|
|
171
148
|
settings_cls: type[BaseSettings],
|
|
172
149
|
init_settings: PydanticBaseSettingsSource,
|
|
@@ -174,12 +151,12 @@ def config_from_yaml(
|
|
|
174
151
|
dotenv_settings: PydanticBaseSettingsSource,
|
|
175
152
|
file_secret_settings: PydanticBaseSettingsSource,
|
|
176
153
|
):
|
|
177
|
-
"""Add custom yaml source"""
|
|
154
|
+
"""Add custom yaml source, but maintain the priority of sources."""
|
|
178
155
|
return (
|
|
179
156
|
init_settings,
|
|
180
157
|
env_settings,
|
|
181
158
|
file_secret_settings,
|
|
182
|
-
|
|
159
|
+
YamlConfigSettingsSource(settings_cls, config_yaml),
|
|
183
160
|
)
|
|
184
161
|
|
|
185
162
|
# construct settings class:
|
|
@@ -21,6 +21,7 @@ with the database.
|
|
|
21
21
|
# ruff: noqa: PLR0913
|
|
22
22
|
|
|
23
23
|
import typing
|
|
24
|
+
import warnings
|
|
24
25
|
from abc import ABC, abstractmethod
|
|
25
26
|
from collections.abc import AsyncGenerator, AsyncIterator, Collection, Mapping
|
|
26
27
|
from contextlib import AbstractAsyncContextManager
|
|
@@ -29,6 +30,7 @@ from typing import Any, Optional, TypeVar, Union, overload
|
|
|
29
30
|
from uuid import uuid4
|
|
30
31
|
|
|
31
32
|
from pydantic import BaseModel
|
|
33
|
+
from typing_extensions import deprecated
|
|
32
34
|
|
|
33
35
|
from hexkit.utils import FieldNotInModelError, validate_fields_in_model
|
|
34
36
|
|
|
@@ -61,7 +63,7 @@ class ResourceAlreadyExistsError(RuntimeError):
|
|
|
61
63
|
"""Raised when a resource did unexpectedly exist."""
|
|
62
64
|
|
|
63
65
|
def __init__(self, *, id_: str):
|
|
64
|
-
message = f'The resource with the id "{id_}"
|
|
66
|
+
message = f'The resource with the id "{id_}" already exists.'
|
|
65
67
|
super().__init__(message)
|
|
66
68
|
|
|
67
69
|
|
|
@@ -259,6 +261,9 @@ class DaoNaturalId(DaoCommons[Dto], typing.Protocol[Dto]):
|
|
|
259
261
|
...
|
|
260
262
|
|
|
261
263
|
|
|
264
|
+
@deprecated(
|
|
265
|
+
"uuid4_id_generator is deprecated as of v3.6 and will be removed in hexkit v4."
|
|
266
|
+
)
|
|
262
267
|
async def uuid4_id_generator() -> AsyncGenerator[str, None]:
|
|
263
268
|
"""Generates a new ID using the UUID4 algorithm.
|
|
264
269
|
This is an AsyncGenerator to be compliant with the id_generator requirements of the
|
|
@@ -441,6 +446,15 @@ class DaoFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
441
446
|
# instantiate the default ID generator:
|
|
442
447
|
id_generator = uuid4_id_generator()
|
|
443
448
|
|
|
449
|
+
if dto_creation_model is not None:
|
|
450
|
+
dep_msg = (
|
|
451
|
+
"The DAO API (and existing providers) are deprecated as of v3.6."
|
|
452
|
+
+ " They will be simplified in hexkit v4."
|
|
453
|
+
+ " Please plan to provide IDs during resource creation. The same model"
|
|
454
|
+
+ " should be used for both creation and retrieval."
|
|
455
|
+
)
|
|
456
|
+
warnings.warn(dep_msg, category=DeprecationWarning, stacklevel=2)
|
|
457
|
+
|
|
444
458
|
return await self._get_dao(
|
|
445
459
|
name=name,
|
|
446
460
|
dto_model=dto_model,
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
|
|
19
19
|
from typing import Literal
|
|
20
20
|
|
|
21
|
-
from pydantic import Field, SecretStr
|
|
21
|
+
from pydantic import Field, PositiveInt, SecretStr
|
|
22
22
|
from pydantic_settings import BaseSettings
|
|
23
23
|
|
|
24
24
|
|
|
@@ -76,3 +76,9 @@ class KafkaConfig(BaseSettings):
|
|
|
76
76
|
+ " newly correlation ID will be generated and used in the event header."
|
|
77
77
|
),
|
|
78
78
|
)
|
|
79
|
+
kafka_max_message_size: PositiveInt = Field(
|
|
80
|
+
default=1024 * 1024, # 1 MiB
|
|
81
|
+
description="The largest message size that can be transmitted, in bytes. Only"
|
|
82
|
+
+ " services that have a need to send/receive larger messages should set this.",
|
|
83
|
+
examples=[1024 * 1024, 16 * 1024 * 1024],
|
|
84
|
+
)
|
|
@@ -54,6 +54,7 @@ class KafkaProducerCompatible(Protocol):
|
|
|
54
54
|
client_id: str,
|
|
55
55
|
key_serializer: Callable[[Any], bytes],
|
|
56
56
|
value_serializer: Callable[[Any], bytes],
|
|
57
|
+
max_request_size: int,
|
|
57
58
|
):
|
|
58
59
|
"""
|
|
59
60
|
Initialize the producer with some config params.
|
|
@@ -67,6 +68,8 @@ class KafkaProducerCompatible(Protocol):
|
|
|
67
68
|
Function to serialize the keys into bytes.
|
|
68
69
|
value_serializer:
|
|
69
70
|
Function to serialize the values into bytes.
|
|
71
|
+
max_request_size:
|
|
72
|
+
Maximum sendable message size.
|
|
70
73
|
"""
|
|
71
74
|
...
|
|
72
75
|
|
|
@@ -116,6 +119,7 @@ class KafkaEventPublisher(EventPublisherProtocol):
|
|
|
116
119
|
value_serializer=lambda event_value: json.dumps(event_value).encode(
|
|
117
120
|
"ascii"
|
|
118
121
|
),
|
|
122
|
+
max_request_size=config.kafka_max_message_size,
|
|
119
123
|
)
|
|
120
124
|
|
|
121
125
|
try:
|
|
@@ -90,6 +90,7 @@ class KafkaConsumerCompatible(Protocol):
|
|
|
90
90
|
enable_auto_commit: bool,
|
|
91
91
|
key_deserializer: Callable[[bytes], str],
|
|
92
92
|
value_deserializer: Callable[[bytes], str],
|
|
93
|
+
max_partition_fetch_bytes: int,
|
|
93
94
|
):
|
|
94
95
|
"""
|
|
95
96
|
Initialize the consumer with some config params.
|
|
@@ -108,6 +109,8 @@ class KafkaConsumerCompatible(Protocol):
|
|
|
108
109
|
Function to deserialize the keys into strings.
|
|
109
110
|
value_serializer:
|
|
110
111
|
Function to deserialize the values into strings.
|
|
112
|
+
max_partition_fetch_bytes:
|
|
113
|
+
Maximum receivable message size
|
|
111
114
|
"""
|
|
112
115
|
...
|
|
113
116
|
|
|
@@ -176,6 +179,7 @@ class KafkaEventSubscriber(InboundProviderBase):
|
|
|
176
179
|
value_deserializer=lambda event_value: json.loads(
|
|
177
180
|
event_value.decode("ascii")
|
|
178
181
|
),
|
|
182
|
+
max_partition_fetch_bytes=config.kafka_max_message_size,
|
|
179
183
|
)
|
|
180
184
|
try:
|
|
181
185
|
await consumer.start()
|
|
@@ -22,6 +22,7 @@ Utilities for testing are located in `./testutils.py`.
|
|
|
22
22
|
# ruff: noqa: PLR0913
|
|
23
23
|
|
|
24
24
|
import json
|
|
25
|
+
import warnings
|
|
25
26
|
from abc import ABC
|
|
26
27
|
from collections.abc import AsyncGenerator, AsyncIterator, Collection, Mapping
|
|
27
28
|
from contextlib import AbstractAsyncContextManager
|
|
@@ -31,6 +32,7 @@ from motor.core import AgnosticCollection
|
|
|
31
32
|
from motor.motor_asyncio import AsyncIOMotorClient
|
|
32
33
|
from pydantic import Field, SecretStr
|
|
33
34
|
from pydantic_settings import BaseSettings
|
|
35
|
+
from pymongo.errors import DuplicateKeyError
|
|
34
36
|
|
|
35
37
|
from hexkit.protocols.dao import (
|
|
36
38
|
DaoFactoryProtocol,
|
|
@@ -42,6 +44,7 @@ from hexkit.protocols.dao import (
|
|
|
42
44
|
InvalidFindMappingError,
|
|
43
45
|
MultipleHitsFoundError,
|
|
44
46
|
NoHitsFoundError,
|
|
47
|
+
ResourceAlreadyExistsError,
|
|
45
48
|
ResourceNotFoundError,
|
|
46
49
|
)
|
|
47
50
|
from hexkit.utils import FieldNotInModelError, validate_fields_in_model
|
|
@@ -161,6 +164,17 @@ class MongoDbDaoBase(ABC, Generic[Dto]):
|
|
|
161
164
|
self._document_to_dto = document_to_dto
|
|
162
165
|
self._dto_to_document = dto_to_document
|
|
163
166
|
|
|
167
|
+
warnings.warn(
|
|
168
|
+
"The MongoDbDaoBase class and subclasses are deprecated as of v3.6 and will"
|
|
169
|
+
+ " be replaced in hexkit v4 with the new MongoDbDao class. To replace the"
|
|
170
|
+
+ " MongoDbDaoNaturalId: just use the new class. To replace the"
|
|
171
|
+
+ " MongoDbDaoSurrogateId: use the new class, remove the creation model and"
|
|
172
|
+
+ " id_generator parameters, and make sure the supplied DTO model includes"
|
|
173
|
+
+ " the ID field with a default factory for generating new IDs.",
|
|
174
|
+
category=DeprecationWarning,
|
|
175
|
+
stacklevel=2,
|
|
176
|
+
)
|
|
177
|
+
|
|
164
178
|
async def get_by_id(self, id_: str) -> Dto:
|
|
165
179
|
"""Get a resource by providing its ID.
|
|
166
180
|
|
|
@@ -386,7 +400,10 @@ class MongoDbDaoNaturalId(MongoDbDaoBase[Dto]):
|
|
|
386
400
|
when a resource with the ID specified in the dto does already exist.
|
|
387
401
|
"""
|
|
388
402
|
document = self._dto_to_document(dto)
|
|
389
|
-
|
|
403
|
+
try:
|
|
404
|
+
await self._collection.insert_one(document)
|
|
405
|
+
except DuplicateKeyError as error:
|
|
406
|
+
raise ResourceAlreadyExistsError(id_=document["_id"]) from error
|
|
390
407
|
|
|
391
408
|
async def upsert(self, dto: Dto) -> None:
|
|
392
409
|
"""Update the provided resource if it already exists, create it otherwise.
|
|
@@ -20,6 +20,7 @@ Require dependencies of the `akafka` and `mongodb` extras.
|
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
22
|
import json
|
|
23
|
+
import logging
|
|
23
24
|
from collections.abc import AsyncIterator, Awaitable, Collection, Mapping
|
|
24
25
|
from contextlib import AbstractAsyncContextManager, asynccontextmanager, contextmanager
|
|
25
26
|
from typing import Any, Callable, Generic, Optional
|
|
@@ -27,6 +28,7 @@ from typing import Any, Callable, Generic, Optional
|
|
|
27
28
|
from aiokafka import AIOKafkaProducer
|
|
28
29
|
from motor.core import AgnosticCollection
|
|
29
30
|
from motor.motor_asyncio import AsyncIOMotorClient
|
|
31
|
+
from pydantic import field_validator
|
|
30
32
|
|
|
31
33
|
from hexkit.correlation import get_correlation_id, set_correlation_id
|
|
32
34
|
from hexkit.custom_types import JsonObject
|
|
@@ -253,7 +255,14 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
|
|
|
253
255
|
document = self._dao._dto_to_document(dto)
|
|
254
256
|
document.setdefault("__metadata__", {})["correlation_id"] = correlation_id
|
|
255
257
|
result = await self._collection.replace_one(
|
|
256
|
-
{
|
|
258
|
+
{
|
|
259
|
+
"_id": document["_id"],
|
|
260
|
+
"$or": [
|
|
261
|
+
{"__metadata__": {"$exists": False}},
|
|
262
|
+
{"__metadata__.deleted": False},
|
|
263
|
+
],
|
|
264
|
+
},
|
|
265
|
+
document,
|
|
257
266
|
)
|
|
258
267
|
if result.matched_count == 0:
|
|
259
268
|
raise ResourceNotFoundError(id_=document["_id"])
|
|
@@ -280,7 +289,14 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
|
|
|
280
289
|
},
|
|
281
290
|
}
|
|
282
291
|
result = await self._collection.replace_one(
|
|
283
|
-
{
|
|
292
|
+
{
|
|
293
|
+
"_id": document["_id"],
|
|
294
|
+
"$or": [
|
|
295
|
+
{"__metadata__": {"$exists": False}},
|
|
296
|
+
{"__metadata__.deleted": False},
|
|
297
|
+
],
|
|
298
|
+
},
|
|
299
|
+
document,
|
|
284
300
|
)
|
|
285
301
|
if result.matched_count == 0:
|
|
286
302
|
raise ResourceNotFoundError(id_=id_)
|
|
@@ -396,6 +412,16 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
|
|
|
396
412
|
class MongoKafkaConfig(MongoDbConfig, KafkaConfig):
|
|
397
413
|
"""Config parameters and their defaults."""
|
|
398
414
|
|
|
415
|
+
@field_validator("kafka_max_message_size", mode="after")
|
|
416
|
+
@classmethod
|
|
417
|
+
def validate_max_message_size(cls, value: int) -> int:
|
|
418
|
+
"""Validate the maximum message size."""
|
|
419
|
+
if value > 2**24: # 16 MiB
|
|
420
|
+
logging.warning(
|
|
421
|
+
f"Max message size ({value}) exceeds the 16 MiB document size limit for MongoDB!"
|
|
422
|
+
)
|
|
423
|
+
return value
|
|
424
|
+
|
|
399
425
|
|
|
400
426
|
class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
|
|
401
427
|
"""A provider implementing the DaoPublisherFactoryProtocol based on MongoDB and
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
|
+
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
"""S3-related test fixtures and support functions."""
|
|
17
|
+
|
|
18
|
+
from ._fixtures import (
|
|
19
|
+
LOCALSTACK_IMAGE,
|
|
20
|
+
MEBIBYTE,
|
|
21
|
+
TEST_FILE_DIR,
|
|
22
|
+
TEST_FILE_PATHS,
|
|
23
|
+
FederatedS3Fixture,
|
|
24
|
+
FileObject,
|
|
25
|
+
S3ContainerFixture,
|
|
26
|
+
S3Fixture,
|
|
27
|
+
S3MultiContainerFixture,
|
|
28
|
+
calc_md5,
|
|
29
|
+
clean_federated_s3_fixture,
|
|
30
|
+
clean_s3_fixture,
|
|
31
|
+
federated_s3_fixture,
|
|
32
|
+
get_clean_federated_s3_fixture,
|
|
33
|
+
get_clean_s3_fixture,
|
|
34
|
+
get_persistent_federated_s3_fixture,
|
|
35
|
+
get_persistent_s3_fixture,
|
|
36
|
+
get_s3_container_fixture,
|
|
37
|
+
get_s3_multi_container_fixture,
|
|
38
|
+
persistent_federated_s3_fixture,
|
|
39
|
+
persistent_s3_fixture,
|
|
40
|
+
populate_storage,
|
|
41
|
+
s3_container_fixture,
|
|
42
|
+
s3_fixture,
|
|
43
|
+
s3_multi_container_fixture,
|
|
44
|
+
temp_file_object,
|
|
45
|
+
tmp_file,
|
|
46
|
+
upload_file,
|
|
47
|
+
)
|
|
48
|
+
from ._typical_workflow import typical_workflow
|
|
49
|
+
from ._utils import (
|
|
50
|
+
check_part_size,
|
|
51
|
+
download_and_check_test_file,
|
|
52
|
+
multipart_upload_file,
|
|
53
|
+
upload_part,
|
|
54
|
+
upload_part_of_size,
|
|
55
|
+
upload_part_via_url,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
__all__ = [
|
|
59
|
+
"LOCALSTACK_IMAGE",
|
|
60
|
+
"MEBIBYTE",
|
|
61
|
+
"TEST_FILE_DIR",
|
|
62
|
+
"TEST_FILE_PATHS",
|
|
63
|
+
"FileObject",
|
|
64
|
+
"S3ContainerFixture",
|
|
65
|
+
"S3Fixture",
|
|
66
|
+
"calc_md5",
|
|
67
|
+
"clean_s3_fixture",
|
|
68
|
+
"get_clean_s3_fixture",
|
|
69
|
+
"get_persistent_s3_fixture",
|
|
70
|
+
"get_s3_container_fixture",
|
|
71
|
+
"persistent_s3_fixture",
|
|
72
|
+
"populate_storage",
|
|
73
|
+
"s3_container_fixture",
|
|
74
|
+
"s3_fixture",
|
|
75
|
+
"temp_file_object",
|
|
76
|
+
"tmp_file",
|
|
77
|
+
"upload_file",
|
|
78
|
+
"typical_workflow",
|
|
79
|
+
"check_part_size",
|
|
80
|
+
"download_and_check_test_file",
|
|
81
|
+
"multipart_upload_file",
|
|
82
|
+
"upload_part",
|
|
83
|
+
"upload_part_of_size",
|
|
84
|
+
"upload_part_via_url",
|
|
85
|
+
"FederatedS3Fixture",
|
|
86
|
+
"S3MultiContainerFixture",
|
|
87
|
+
"clean_federated_s3_fixture",
|
|
88
|
+
"federated_s3_fixture",
|
|
89
|
+
"get_clean_federated_s3_fixture",
|
|
90
|
+
"get_s3_multi_container_fixture",
|
|
91
|
+
"get_persistent_federated_s3_fixture",
|
|
92
|
+
"s3_multi_container_fixture",
|
|
93
|
+
"persistent_federated_s3_fixture",
|
|
94
|
+
]
|
|
@@ -344,3 +344,151 @@ def tmp_file() -> Generator[FileObject, None, None]:
|
|
|
344
344
|
"""A fixture that provides a temporary file."""
|
|
345
345
|
with temp_file_object() as temp_file:
|
|
346
346
|
yield temp_file
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
class FederatedS3Fixture:
|
|
350
|
+
"""Fixture containing multiple S3 fixtures to simulate federated storage."""
|
|
351
|
+
|
|
352
|
+
def __init__(self, storages: dict[str, S3Fixture]):
|
|
353
|
+
self.storages = storages
|
|
354
|
+
|
|
355
|
+
def get_configs_by_alias(self) -> dict[str, S3Config]:
|
|
356
|
+
"""Get the S3Config instance for each object storage in the fixture."""
|
|
357
|
+
return {alias: self.storages[alias].config for alias in self.storages}
|
|
358
|
+
|
|
359
|
+
async def populate_dummy_items(self, alias: str, contents: dict[str, list[str]]):
|
|
360
|
+
"""Convenience function to populate a specific S3Fixture.
|
|
361
|
+
|
|
362
|
+
Args:
|
|
363
|
+
- `alias`: The alias of the S3Fixture to populate.
|
|
364
|
+
- `contents`: A dictionary with bucket names as keys and lists of object names
|
|
365
|
+
as values. The buckets can be empty, and the objects are created with a size of
|
|
366
|
+
1 byte.
|
|
367
|
+
"""
|
|
368
|
+
if alias not in self.storages:
|
|
369
|
+
# This would indicate some kind of mismatch between config and fixture
|
|
370
|
+
raise RuntimeError(f"Alias '{alias}' not found in the federated S3 fixture")
|
|
371
|
+
storage = self.storages[alias]
|
|
372
|
+
|
|
373
|
+
# Populate the buckets so even empty buckets are established
|
|
374
|
+
await storage.populate_buckets([bucket for bucket in contents])
|
|
375
|
+
|
|
376
|
+
# Add the dummy items
|
|
377
|
+
for bucket, objects in contents.items():
|
|
378
|
+
for object in objects:
|
|
379
|
+
with temp_file_object(bucket, object, 1) as file:
|
|
380
|
+
await storage.populate_file_objects([file])
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
class S3MultiContainerFixture:
|
|
384
|
+
"""Fixture for managing multiple running S3 test containers in order to mimic
|
|
385
|
+
multiple object storages.
|
|
386
|
+
|
|
387
|
+
Without this fixture, separate S3Fixture instances would access the same
|
|
388
|
+
underlying storage resources.
|
|
389
|
+
"""
|
|
390
|
+
|
|
391
|
+
def __init__(self, s3_containers: dict[str, S3ContainerFixture]):
|
|
392
|
+
self.s3_containers = s3_containers
|
|
393
|
+
|
|
394
|
+
def __enter__(self):
|
|
395
|
+
"""Enter the context manager and start the S3 containers."""
|
|
396
|
+
for container in self.s3_containers.values():
|
|
397
|
+
container.__enter__()
|
|
398
|
+
return self
|
|
399
|
+
|
|
400
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
401
|
+
"""Exit the context manager and clean up the S3 containers."""
|
|
402
|
+
for container in self.s3_containers.values():
|
|
403
|
+
container.__exit__(exc_type, exc_val, exc_tb)
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _s3_multi_container_fixture(
|
|
407
|
+
request,
|
|
408
|
+
) -> Generator[S3MultiContainerFixture, None, None]:
|
|
409
|
+
"""Fixture function for getting multiple running S3 test containers."""
|
|
410
|
+
try:
|
|
411
|
+
storage_aliases = request.getfixturevalue("storage_aliases")
|
|
412
|
+
except pytest.FixtureLookupError as err:
|
|
413
|
+
raise NotImplementedError(
|
|
414
|
+
"You must provide a 'storage_aliases' fixture in your test setup."
|
|
415
|
+
+ " It must have the same scope as 's3_multi_container'"
|
|
416
|
+
+ " and it must return a list of the storage aliases to be used."
|
|
417
|
+
) from err
|
|
418
|
+
|
|
419
|
+
if not storage_aliases:
|
|
420
|
+
raise RuntimeError("The 'storage_aliases' list must not be empty.")
|
|
421
|
+
s3_containers = {
|
|
422
|
+
alias: S3ContainerFixture(name=f"{alias}_s3_container")
|
|
423
|
+
for alias in storage_aliases
|
|
424
|
+
}
|
|
425
|
+
with S3MultiContainerFixture(s3_containers) as s3_multi_container:
|
|
426
|
+
yield s3_multi_container
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def get_s3_multi_container_fixture(
|
|
430
|
+
scope: PytestScope = "session", name: str = "s3_multi_container"
|
|
431
|
+
):
|
|
432
|
+
"""Get a fixture containing multiple LocalStack test containers.
|
|
433
|
+
|
|
434
|
+
By default, the session scope is used for LocalStack test containers.
|
|
435
|
+
Requires that a 'storage_aliases' fixture is provided in the test setup.
|
|
436
|
+
"""
|
|
437
|
+
return pytest.fixture(_s3_multi_container_fixture, scope=scope, name=name)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
s3_multi_container_fixture = get_s3_multi_container_fixture()
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _persistent_federated_s3_fixture(
|
|
444
|
+
s3_multi_container: S3MultiContainerFixture,
|
|
445
|
+
) -> Generator[FederatedS3Fixture, None, None]:
|
|
446
|
+
"""Fixture function that creates a persistent FederatedS3Fixture.
|
|
447
|
+
|
|
448
|
+
The state of each S3 storage in the fixture is not cleaned up.
|
|
449
|
+
"""
|
|
450
|
+
s3_fixtures = {}
|
|
451
|
+
for alias, container in s3_multi_container.s3_containers.items():
|
|
452
|
+
config = container.s3_config
|
|
453
|
+
storage = S3ObjectStorage(config=config)
|
|
454
|
+
s3_fixtures[alias] = S3Fixture(config=config, storage=storage)
|
|
455
|
+
yield FederatedS3Fixture(s3_fixtures)
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def get_persistent_federated_s3_fixture(
|
|
459
|
+
scope: PytestScope = "function", name: str = "federated_s3"
|
|
460
|
+
):
|
|
461
|
+
"""Get a federated S3 storage fixture with desired scope.
|
|
462
|
+
|
|
463
|
+
The state of the S3 storage is not cleaned up by the fixture.
|
|
464
|
+
"""
|
|
465
|
+
return pytest.fixture(_persistent_federated_s3_fixture, scope=scope, name=name)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
persistent_federated_s3_fixture = get_persistent_federated_s3_fixture()
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
async def _clean_federated_s3_fixture(
|
|
472
|
+
s3_multi_container: S3MultiContainerFixture,
|
|
473
|
+
) -> AsyncGenerator[FederatedS3Fixture, None]:
|
|
474
|
+
"""Fixture function that creates a clean FederatedS3Fixture instance.
|
|
475
|
+
|
|
476
|
+
The state of each S3 storage is cleaned up before yielding the fixture.
|
|
477
|
+
"""
|
|
478
|
+
for federated_s3_fixture in _persistent_federated_s3_fixture(s3_multi_container):
|
|
479
|
+
for s3_fixture in federated_s3_fixture.storages.values():
|
|
480
|
+
await s3_fixture.delete_buckets()
|
|
481
|
+
yield federated_s3_fixture
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def get_clean_federated_s3_fixture(
|
|
485
|
+
scope: PytestScope = "function", name: str = "federated_s3"
|
|
486
|
+
):
|
|
487
|
+
"""Get a federated S3 storage fixture with desired scope.
|
|
488
|
+
|
|
489
|
+
The state of the S3 storage is not cleaned up by the fixture.
|
|
490
|
+
"""
|
|
491
|
+
return pytest_asyncio.fixture(_clean_federated_s3_fixture, scope=scope, name=name)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
federated_s3_fixture = clean_federated_s3_fixture = get_clean_federated_s3_fixture()
|
|
@@ -44,7 +44,7 @@ class FileObject(BaseModel):
|
|
|
44
44
|
bucket_id: str
|
|
45
45
|
object_id: str
|
|
46
46
|
|
|
47
|
-
@computed_field # type: ignore [
|
|
47
|
+
@computed_field # type: ignore [prop-decorator]
|
|
48
48
|
@property
|
|
49
49
|
def content(self) -> bytes:
|
|
50
50
|
"""Extract the content from the file at the provided path"""
|
|
@@ -53,7 +53,7 @@ class FileObject(BaseModel):
|
|
|
53
53
|
with open(self.file_path, "rb") as file:
|
|
54
54
|
return file.read()
|
|
55
55
|
|
|
56
|
-
@computed_field # type: ignore [
|
|
56
|
+
@computed_field # type: ignore [prop-decorator]
|
|
57
57
|
@property
|
|
58
58
|
def md5(self) -> str:
|
|
59
59
|
"""Calculate the md5 hash of the content"""
|
|
@@ -26,6 +26,8 @@ from typing import NamedTuple, Optional
|
|
|
26
26
|
from hexkit.custom_types import JsonObject
|
|
27
27
|
from hexkit.protocols.eventpub import EventPublisherProtocol
|
|
28
28
|
|
|
29
|
+
__all__ = ["InMemEventStore", "InMemEventPublisher", "TopicExhaustedError"]
|
|
30
|
+
|
|
29
31
|
|
|
30
32
|
class TopicExhaustedError(RuntimeError):
|
|
31
33
|
"""Thrown when now more event are queued in a topic."""
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
|
|
2
|
-
# for the German Human Genome-Phenome Archive (GHGA)
|
|
3
|
-
#
|
|
4
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
-
# you may not use this file except in compliance with the License.
|
|
6
|
-
# You may obtain a copy of the License at
|
|
7
|
-
#
|
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
#
|
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
-
# See the License for the specific language governing permissions and
|
|
14
|
-
# limitations under the License.
|
|
15
|
-
|
|
16
|
-
"""S3-related test fixtures and support functions."""
|
|
17
|
-
|
|
18
|
-
from ._fixtures import ( # noqa: F401
|
|
19
|
-
LOCALSTACK_IMAGE,
|
|
20
|
-
MEBIBYTE,
|
|
21
|
-
TEST_FILE_DIR,
|
|
22
|
-
TEST_FILE_PATHS,
|
|
23
|
-
FileObject,
|
|
24
|
-
S3ContainerFixture,
|
|
25
|
-
S3Fixture,
|
|
26
|
-
calc_md5,
|
|
27
|
-
clean_s3_fixture,
|
|
28
|
-
get_clean_s3_fixture,
|
|
29
|
-
get_persistent_s3_fixture,
|
|
30
|
-
get_s3_container_fixture,
|
|
31
|
-
persistent_s3_fixture,
|
|
32
|
-
populate_storage,
|
|
33
|
-
s3_container_fixture,
|
|
34
|
-
s3_fixture,
|
|
35
|
-
temp_file_object,
|
|
36
|
-
tmp_file,
|
|
37
|
-
upload_file,
|
|
38
|
-
)
|
|
39
|
-
from ._typical_workflow import typical_workflow # noqa: F401
|
|
40
|
-
from ._utils import ( # noqa: F401
|
|
41
|
-
check_part_size,
|
|
42
|
-
download_and_check_test_file,
|
|
43
|
-
multipart_upload_file,
|
|
44
|
-
upload_part,
|
|
45
|
-
upload_part_of_size,
|
|
46
|
-
upload_part_via_url,
|
|
47
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|