hexkit 3.4.0__tar.gz → 3.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {hexkit-3.4.0/src/hexkit.egg-info → hexkit-3.6.0}/PKG-INFO +1 -1
  2. {hexkit-3.4.0 → hexkit-3.6.0}/pyproject.toml +1 -1
  3. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/config.py +7 -30
  4. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/dao.py +15 -1
  5. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/config.py +7 -1
  6. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/provider/eventpub.py +4 -0
  7. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/provider/eventsub.py +4 -0
  8. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongodb/provider.py +18 -1
  9. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongokafka/provider.py +28 -2
  10. hexkit-3.6.0/src/hexkit/providers/s3/testutils/__init__.py +94 -0
  11. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/testutils/_fixtures.py +148 -0
  12. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/testutils/_utils.py +2 -2
  13. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/testing/eventpub.py +2 -0
  14. {hexkit-3.4.0 → hexkit-3.6.0/src/hexkit.egg-info}/PKG-INFO +1 -1
  15. hexkit-3.4.0/src/hexkit/providers/s3/testutils/__init__.py +0 -47
  16. {hexkit-3.4.0 → hexkit-3.6.0}/LICENSE +0 -0
  17. {hexkit-3.4.0 → hexkit-3.6.0}/MANIFEST.in +0 -0
  18. {hexkit-3.4.0 → hexkit-3.6.0}/README.md +0 -0
  19. {hexkit-3.4.0 → hexkit-3.6.0}/setup.cfg +0 -0
  20. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/__init__.py +0 -0
  21. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/__main__.py +0 -0
  22. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/base.py +0 -0
  23. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/correlation.py +0 -0
  24. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/custom_types.py +0 -0
  25. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/log.py +0 -0
  26. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/__init__.py +0 -0
  27. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/daopub.py +0 -0
  28. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/daosub.py +0 -0
  29. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/eventpub.py +0 -0
  30. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/eventsub.py +0 -0
  31. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/protocols/objstorage.py +0 -0
  32. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/__init__.py +0 -0
  33. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/__init__.py +0 -0
  34. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/provider/__init__.py +0 -0
  35. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/provider/daosub.py +0 -0
  36. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/provider/utils.py +0 -0
  37. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/testcontainer.py +0 -0
  38. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/akafka/testutils.py +0 -0
  39. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongodb/__init__.py +0 -0
  40. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongodb/testutils.py +0 -0
  41. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongokafka/__init__.py +0 -0
  42. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/mongokafka/testutils.py +0 -0
  43. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/__init__.py +0 -0
  44. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/provider.py +0 -0
  45. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/test_files/__init__.py +0 -0
  46. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/test_files/test_file1.yaml +0 -0
  47. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/test_files/test_file2.yaml +0 -0
  48. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/test_files/test_file3.yaml +0 -0
  49. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/test_files/test_file4.yaml +0 -0
  50. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/s3/testutils/_typical_workflow.py +0 -0
  51. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/providers/testing/__init__.py +0 -0
  52. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/py.typed +0 -0
  53. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit/utils.py +0 -0
  54. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit.egg-info/SOURCES.txt +0 -0
  55. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit.egg-info/dependency_links.txt +0 -0
  56. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit.egg-info/requires.txt +0 -0
  57. {hexkit-3.4.0 → hexkit-3.6.0}/src/hexkit.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: hexkit
3
- Version: 3.4.0
3
+ Version: 3.6.0
4
4
  Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
5
5
  Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
6
6
  License: Apache 2.0
@@ -23,7 +23,7 @@ classifiers = [
23
23
  "Intended Audience :: Developers",
24
24
  ]
25
25
  name = "hexkit"
26
- version = "3.4.0"
26
+ version = "3.6.0"
27
27
  description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
28
28
  dependencies = [
29
29
  "pydantic >=2, <3",
@@ -17,13 +17,13 @@
17
17
 
18
18
  import os
19
19
  from pathlib import Path
20
- from typing import Any, Callable, Final, Optional
20
+ from typing import Callable, Final, Optional
21
21
 
22
- import yaml
23
22
  from pydantic_settings import (
24
23
  BaseSettings,
25
24
  PydanticBaseSettingsSource,
26
25
  SettingsConfigDict,
26
+ YamlConfigSettingsSource,
27
27
  )
28
28
 
29
29
  # Default config prefix:
@@ -79,29 +79,6 @@ def get_default_config_yaml(prefix: str) -> Optional[Path]:
79
79
  return None
80
80
 
81
81
 
82
- def yaml_settings_factory(
83
- config_yaml: Optional[Path] = None,
84
- ) -> Callable[[], dict[str, Any]]:
85
- """
86
- A factory of source methods for pydantic's BaseSettings Config that load
87
- settings from a yaml file.
88
-
89
- Args:
90
- config_yaml (str, Optional):
91
- Path to the yaml file to read from.
92
- """
93
-
94
- def yaml_settings() -> dict[str, Any]:
95
- """Source method for loading pydantic BaseSettings from a yaml file"""
96
- if config_yaml is None:
97
- return {}
98
-
99
- with open(config_yaml, encoding="utf8") as yaml_file:
100
- return yaml.safe_load(yaml_file)
101
-
102
- return yaml_settings
103
-
104
-
105
82
  def config_from_yaml(
106
83
  prefix: str = DEFAULT_CONFIG_PREFIX,
107
84
  ) -> Callable:
@@ -119,7 +96,7 @@ def config_from_yaml(
119
96
 
120
97
  Args:
121
98
  prefix: (str, optional):
122
- When defining parameters via enviroment variables, all variables
99
+ When defining parameters via environment variables, all variables
123
100
  have to be prefixed with this string following this pattern
124
101
  "{prefix}_{actual_variable_name}". Moreover, this prefix is used
125
102
  to derive the default location for the config yaml file
@@ -161,12 +138,12 @@ def config_from_yaml(
161
138
  raise ConfigYamlDoesNotExist(path=config_yaml)
162
139
 
163
140
  class ModSettings(settings):
164
- """Modifies the orginal Settings class provided by the user"""
141
+ """Modifies the original Settings class provided by the user"""
165
142
 
166
143
  model_config = SettingsConfigDict(frozen=True, env_prefix=f"{prefix}_")
167
144
 
168
145
  @classmethod
169
- def settings_customise_sources( # noqa: PLR0913
146
+ def settings_customise_sources(
170
147
  cls,
171
148
  settings_cls: type[BaseSettings],
172
149
  init_settings: PydanticBaseSettingsSource,
@@ -174,12 +151,12 @@ def config_from_yaml(
174
151
  dotenv_settings: PydanticBaseSettingsSource,
175
152
  file_secret_settings: PydanticBaseSettingsSource,
176
153
  ):
177
- """Add custom yaml source"""
154
+ """Add custom yaml source, but maintain the priority of sources."""
178
155
  return (
179
156
  init_settings,
180
157
  env_settings,
181
158
  file_secret_settings,
182
- yaml_settings_factory(config_yaml),
159
+ YamlConfigSettingsSource(settings_cls, config_yaml),
183
160
  )
184
161
 
185
162
  # construct settings class:
@@ -21,6 +21,7 @@ with the database.
21
21
  # ruff: noqa: PLR0913
22
22
 
23
23
  import typing
24
+ import warnings
24
25
  from abc import ABC, abstractmethod
25
26
  from collections.abc import AsyncGenerator, AsyncIterator, Collection, Mapping
26
27
  from contextlib import AbstractAsyncContextManager
@@ -29,6 +30,7 @@ from typing import Any, Optional, TypeVar, Union, overload
29
30
  from uuid import uuid4
30
31
 
31
32
  from pydantic import BaseModel
33
+ from typing_extensions import deprecated
32
34
 
33
35
  from hexkit.utils import FieldNotInModelError, validate_fields_in_model
34
36
 
@@ -61,7 +63,7 @@ class ResourceAlreadyExistsError(RuntimeError):
61
63
  """Raised when a resource did unexpectedly exist."""
62
64
 
63
65
  def __init__(self, *, id_: str):
64
- message = f'The resource with the id "{id_}" does already exist.'
66
+ message = f'The resource with the id "{id_}" already exists.'
65
67
  super().__init__(message)
66
68
 
67
69
 
@@ -259,6 +261,9 @@ class DaoNaturalId(DaoCommons[Dto], typing.Protocol[Dto]):
259
261
  ...
260
262
 
261
263
 
264
+ @deprecated(
265
+ "uuid4_id_generator is deprecated as of v3.6 and will be removed in hexkit v4."
266
+ )
262
267
  async def uuid4_id_generator() -> AsyncGenerator[str, None]:
263
268
  """Generates a new ID using the UUID4 algorithm.
264
269
  This is an AsyncGenerator to be compliant with the id_generator requirements of the
@@ -441,6 +446,15 @@ class DaoFactoryProtocol(DaoFactoryBase, ABC):
441
446
  # instantiate the default ID generator:
442
447
  id_generator = uuid4_id_generator()
443
448
 
449
+ if dto_creation_model is not None:
450
+ dep_msg = (
451
+ "The DAO API (and existing providers) are deprecated as of v3.6."
452
+ + " They will be simplified in hexkit v4."
453
+ + " Please plan to provide IDs during resource creation. The same model"
454
+ + " should be used for both creation and retrieval."
455
+ )
456
+ warnings.warn(dep_msg, category=DeprecationWarning, stacklevel=2)
457
+
444
458
  return await self._get_dao(
445
459
  name=name,
446
460
  dto_model=dto_model,
@@ -18,7 +18,7 @@
18
18
 
19
19
  from typing import Literal
20
20
 
21
- from pydantic import Field, SecretStr
21
+ from pydantic import Field, PositiveInt, SecretStr
22
22
  from pydantic_settings import BaseSettings
23
23
 
24
24
 
@@ -76,3 +76,9 @@ class KafkaConfig(BaseSettings):
76
76
  + " newly correlation ID will be generated and used in the event header."
77
77
  ),
78
78
  )
79
+ kafka_max_message_size: PositiveInt = Field(
80
+ default=1024 * 1024, # 1 MiB
81
+ description="The largest message size that can be transmitted, in bytes. Only"
82
+ + " services that have a need to send/receive larger messages should set this.",
83
+ examples=[1024 * 1024, 16 * 1024 * 1024],
84
+ )
@@ -54,6 +54,7 @@ class KafkaProducerCompatible(Protocol):
54
54
  client_id: str,
55
55
  key_serializer: Callable[[Any], bytes],
56
56
  value_serializer: Callable[[Any], bytes],
57
+ max_request_size: int,
57
58
  ):
58
59
  """
59
60
  Initialize the producer with some config params.
@@ -67,6 +68,8 @@ class KafkaProducerCompatible(Protocol):
67
68
  Function to serialize the keys into bytes.
68
69
  value_serializer:
69
70
  Function to serialize the values into bytes.
71
+ max_request_size:
72
+ Maximum sendable message size.
70
73
  """
71
74
  ...
72
75
 
@@ -116,6 +119,7 @@ class KafkaEventPublisher(EventPublisherProtocol):
116
119
  value_serializer=lambda event_value: json.dumps(event_value).encode(
117
120
  "ascii"
118
121
  ),
122
+ max_request_size=config.kafka_max_message_size,
119
123
  )
120
124
 
121
125
  try:
@@ -90,6 +90,7 @@ class KafkaConsumerCompatible(Protocol):
90
90
  enable_auto_commit: bool,
91
91
  key_deserializer: Callable[[bytes], str],
92
92
  value_deserializer: Callable[[bytes], str],
93
+ max_partition_fetch_bytes: int,
93
94
  ):
94
95
  """
95
96
  Initialize the consumer with some config params.
@@ -108,6 +109,8 @@ class KafkaConsumerCompatible(Protocol):
108
109
  Function to deserialize the keys into strings.
109
110
  value_serializer:
110
111
  Function to deserialize the values into strings.
112
+ max_partition_fetch_bytes:
113
+ Maximum receivable message size
111
114
  """
112
115
  ...
113
116
 
@@ -176,6 +179,7 @@ class KafkaEventSubscriber(InboundProviderBase):
176
179
  value_deserializer=lambda event_value: json.loads(
177
180
  event_value.decode("ascii")
178
181
  ),
182
+ max_partition_fetch_bytes=config.kafka_max_message_size,
179
183
  )
180
184
  try:
181
185
  await consumer.start()
@@ -22,6 +22,7 @@ Utilities for testing are located in `./testutils.py`.
22
22
  # ruff: noqa: PLR0913
23
23
 
24
24
  import json
25
+ import warnings
25
26
  from abc import ABC
26
27
  from collections.abc import AsyncGenerator, AsyncIterator, Collection, Mapping
27
28
  from contextlib import AbstractAsyncContextManager
@@ -31,6 +32,7 @@ from motor.core import AgnosticCollection
31
32
  from motor.motor_asyncio import AsyncIOMotorClient
32
33
  from pydantic import Field, SecretStr
33
34
  from pydantic_settings import BaseSettings
35
+ from pymongo.errors import DuplicateKeyError
34
36
 
35
37
  from hexkit.protocols.dao import (
36
38
  DaoFactoryProtocol,
@@ -42,6 +44,7 @@ from hexkit.protocols.dao import (
42
44
  InvalidFindMappingError,
43
45
  MultipleHitsFoundError,
44
46
  NoHitsFoundError,
47
+ ResourceAlreadyExistsError,
45
48
  ResourceNotFoundError,
46
49
  )
47
50
  from hexkit.utils import FieldNotInModelError, validate_fields_in_model
@@ -161,6 +164,17 @@ class MongoDbDaoBase(ABC, Generic[Dto]):
161
164
  self._document_to_dto = document_to_dto
162
165
  self._dto_to_document = dto_to_document
163
166
 
167
+ warnings.warn(
168
+ "The MongoDbDaoBase class and subclasses are deprecated as of v3.6 and will"
169
+ + " be replaced in hexkit v4 with the new MongoDbDao class. To replace the"
170
+ + " MongoDbDaoNaturalId: just use the new class. To replace the"
171
+ + " MongoDbDaoSurrogateId: use the new class, remove the creation model and"
172
+ + " id_generator parameters, and make sure the supplied DTO model includes"
173
+ + " the ID field with a default factory for generating new IDs.",
174
+ category=DeprecationWarning,
175
+ stacklevel=2,
176
+ )
177
+
164
178
  async def get_by_id(self, id_: str) -> Dto:
165
179
  """Get a resource by providing its ID.
166
180
 
@@ -386,7 +400,10 @@ class MongoDbDaoNaturalId(MongoDbDaoBase[Dto]):
386
400
  when a resource with the ID specified in the dto does already exist.
387
401
  """
388
402
  document = self._dto_to_document(dto)
389
- await self._collection.insert_one(document)
403
+ try:
404
+ await self._collection.insert_one(document)
405
+ except DuplicateKeyError as error:
406
+ raise ResourceAlreadyExistsError(id_=document["_id"]) from error
390
407
 
391
408
  async def upsert(self, dto: Dto) -> None:
392
409
  """Update the provided resource if it already exists, create it otherwise.
@@ -20,6 +20,7 @@ Require dependencies of the `akafka` and `mongodb` extras.
20
20
  """
21
21
 
22
22
  import json
23
+ import logging
23
24
  from collections.abc import AsyncIterator, Awaitable, Collection, Mapping
24
25
  from contextlib import AbstractAsyncContextManager, asynccontextmanager, contextmanager
25
26
  from typing import Any, Callable, Generic, Optional
@@ -27,6 +28,7 @@ from typing import Any, Callable, Generic, Optional
27
28
  from aiokafka import AIOKafkaProducer
28
29
  from motor.core import AgnosticCollection
29
30
  from motor.motor_asyncio import AsyncIOMotorClient
31
+ from pydantic import field_validator
30
32
 
31
33
  from hexkit.correlation import get_correlation_id, set_correlation_id
32
34
  from hexkit.custom_types import JsonObject
@@ -253,7 +255,14 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
253
255
  document = self._dao._dto_to_document(dto)
254
256
  document.setdefault("__metadata__", {})["correlation_id"] = correlation_id
255
257
  result = await self._collection.replace_one(
256
- {"_id": document["_id"], "__metadata__.deleted": False}, document
258
+ {
259
+ "_id": document["_id"],
260
+ "$or": [
261
+ {"__metadata__": {"$exists": False}},
262
+ {"__metadata__.deleted": False},
263
+ ],
264
+ },
265
+ document,
257
266
  )
258
267
  if result.matched_count == 0:
259
268
  raise ResourceNotFoundError(id_=document["_id"])
@@ -280,7 +289,14 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
280
289
  },
281
290
  }
282
291
  result = await self._collection.replace_one(
283
- {"_id": document["_id"], "__metadata__.deleted": False}, document
292
+ {
293
+ "_id": document["_id"],
294
+ "$or": [
295
+ {"__metadata__": {"$exists": False}},
296
+ {"__metadata__.deleted": False},
297
+ ],
298
+ },
299
+ document,
284
300
  )
285
301
  if result.matched_count == 0:
286
302
  raise ResourceNotFoundError(id_=id_)
@@ -396,6 +412,16 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
396
412
  class MongoKafkaConfig(MongoDbConfig, KafkaConfig):
397
413
  """Config parameters and their defaults."""
398
414
 
415
+ @field_validator("kafka_max_message_size", mode="after")
416
+ @classmethod
417
+ def validate_max_message_size(cls, value: int) -> int:
418
+ """Validate the maximum message size."""
419
+ if value > 2**24: # 16 MiB
420
+ logging.warning(
421
+ f"Max message size ({value}) exceeds the 16 MiB document size limit for MongoDB!"
422
+ )
423
+ return value
424
+
399
425
 
400
426
  class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
401
427
  """A provider implementing the DaoPublisherFactoryProtocol based on MongoDB and
@@ -0,0 +1,94 @@
1
+ # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
+ # for the German Human Genome-Phenome Archive (GHGA)
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """S3-related test fixtures and support functions."""
17
+
18
+ from ._fixtures import (
19
+ LOCALSTACK_IMAGE,
20
+ MEBIBYTE,
21
+ TEST_FILE_DIR,
22
+ TEST_FILE_PATHS,
23
+ FederatedS3Fixture,
24
+ FileObject,
25
+ S3ContainerFixture,
26
+ S3Fixture,
27
+ S3MultiContainerFixture,
28
+ calc_md5,
29
+ clean_federated_s3_fixture,
30
+ clean_s3_fixture,
31
+ federated_s3_fixture,
32
+ get_clean_federated_s3_fixture,
33
+ get_clean_s3_fixture,
34
+ get_persistent_federated_s3_fixture,
35
+ get_persistent_s3_fixture,
36
+ get_s3_container_fixture,
37
+ get_s3_multi_container_fixture,
38
+ persistent_federated_s3_fixture,
39
+ persistent_s3_fixture,
40
+ populate_storage,
41
+ s3_container_fixture,
42
+ s3_fixture,
43
+ s3_multi_container_fixture,
44
+ temp_file_object,
45
+ tmp_file,
46
+ upload_file,
47
+ )
48
+ from ._typical_workflow import typical_workflow
49
+ from ._utils import (
50
+ check_part_size,
51
+ download_and_check_test_file,
52
+ multipart_upload_file,
53
+ upload_part,
54
+ upload_part_of_size,
55
+ upload_part_via_url,
56
+ )
57
+
58
+ __all__ = [
59
+ "LOCALSTACK_IMAGE",
60
+ "MEBIBYTE",
61
+ "TEST_FILE_DIR",
62
+ "TEST_FILE_PATHS",
63
+ "FileObject",
64
+ "S3ContainerFixture",
65
+ "S3Fixture",
66
+ "calc_md5",
67
+ "clean_s3_fixture",
68
+ "get_clean_s3_fixture",
69
+ "get_persistent_s3_fixture",
70
+ "get_s3_container_fixture",
71
+ "persistent_s3_fixture",
72
+ "populate_storage",
73
+ "s3_container_fixture",
74
+ "s3_fixture",
75
+ "temp_file_object",
76
+ "tmp_file",
77
+ "upload_file",
78
+ "typical_workflow",
79
+ "check_part_size",
80
+ "download_and_check_test_file",
81
+ "multipart_upload_file",
82
+ "upload_part",
83
+ "upload_part_of_size",
84
+ "upload_part_via_url",
85
+ "FederatedS3Fixture",
86
+ "S3MultiContainerFixture",
87
+ "clean_federated_s3_fixture",
88
+ "federated_s3_fixture",
89
+ "get_clean_federated_s3_fixture",
90
+ "get_s3_multi_container_fixture",
91
+ "get_persistent_federated_s3_fixture",
92
+ "s3_multi_container_fixture",
93
+ "persistent_federated_s3_fixture",
94
+ ]
@@ -344,3 +344,151 @@ def tmp_file() -> Generator[FileObject, None, None]:
344
344
  """A fixture that provides a temporary file."""
345
345
  with temp_file_object() as temp_file:
346
346
  yield temp_file
347
+
348
+
349
+ class FederatedS3Fixture:
350
+ """Fixture containing multiple S3 fixtures to simulate federated storage."""
351
+
352
+ def __init__(self, storages: dict[str, S3Fixture]):
353
+ self.storages = storages
354
+
355
+ def get_configs_by_alias(self) -> dict[str, S3Config]:
356
+ """Get the S3Config instance for each object storage in the fixture."""
357
+ return {alias: self.storages[alias].config for alias in self.storages}
358
+
359
+ async def populate_dummy_items(self, alias: str, contents: dict[str, list[str]]):
360
+ """Convenience function to populate a specific S3Fixture.
361
+
362
+ Args:
363
+ - `alias`: The alias of the S3Fixture to populate.
364
+ - `contents`: A dictionary with bucket names as keys and lists of object names
365
+ as values. The buckets can be empty, and the objects are created with a size of
366
+ 1 byte.
367
+ """
368
+ if alias not in self.storages:
369
+ # This would indicate some kind of mismatch between config and fixture
370
+ raise RuntimeError(f"Alias '{alias}' not found in the federated S3 fixture")
371
+ storage = self.storages[alias]
372
+
373
+ # Populate the buckets so even empty buckets are established
374
+ await storage.populate_buckets([bucket for bucket in contents])
375
+
376
+ # Add the dummy items
377
+ for bucket, objects in contents.items():
378
+ for object in objects:
379
+ with temp_file_object(bucket, object, 1) as file:
380
+ await storage.populate_file_objects([file])
381
+
382
+
383
+ class S3MultiContainerFixture:
384
+ """Fixture for managing multiple running S3 test containers in order to mimic
385
+ multiple object storages.
386
+
387
+ Without this fixture, separate S3Fixture instances would access the same
388
+ underlying storage resources.
389
+ """
390
+
391
+ def __init__(self, s3_containers: dict[str, S3ContainerFixture]):
392
+ self.s3_containers = s3_containers
393
+
394
+ def __enter__(self):
395
+ """Enter the context manager and start the S3 containers."""
396
+ for container in self.s3_containers.values():
397
+ container.__enter__()
398
+ return self
399
+
400
+ def __exit__(self, exc_type, exc_val, exc_tb):
401
+ """Exit the context manager and clean up the S3 containers."""
402
+ for container in self.s3_containers.values():
403
+ container.__exit__(exc_type, exc_val, exc_tb)
404
+
405
+
406
+ def _s3_multi_container_fixture(
407
+ request,
408
+ ) -> Generator[S3MultiContainerFixture, None, None]:
409
+ """Fixture function for getting multiple running S3 test containers."""
410
+ try:
411
+ storage_aliases = request.getfixturevalue("storage_aliases")
412
+ except pytest.FixtureLookupError as err:
413
+ raise NotImplementedError(
414
+ "You must provide a 'storage_aliases' fixture in your test setup."
415
+ + " It must have the same scope as 's3_multi_container'"
416
+ + " and it must return a list of the storage aliases to be used."
417
+ ) from err
418
+
419
+ if not storage_aliases:
420
+ raise RuntimeError("The 'storage_aliases' list must not be empty.")
421
+ s3_containers = {
422
+ alias: S3ContainerFixture(name=f"{alias}_s3_container")
423
+ for alias in storage_aliases
424
+ }
425
+ with S3MultiContainerFixture(s3_containers) as s3_multi_container:
426
+ yield s3_multi_container
427
+
428
+
429
+ def get_s3_multi_container_fixture(
430
+ scope: PytestScope = "session", name: str = "s3_multi_container"
431
+ ):
432
+ """Get a fixture containing multiple LocalStack test containers.
433
+
434
+ By default, the session scope is used for LocalStack test containers.
435
+ Requires that a 'storage_aliases' fixture is provided in the test setup.
436
+ """
437
+ return pytest.fixture(_s3_multi_container_fixture, scope=scope, name=name)
438
+
439
+
440
+ s3_multi_container_fixture = get_s3_multi_container_fixture()
441
+
442
+
443
+ def _persistent_federated_s3_fixture(
444
+ s3_multi_container: S3MultiContainerFixture,
445
+ ) -> Generator[FederatedS3Fixture, None, None]:
446
+ """Fixture function that creates a persistent FederatedS3Fixture.
447
+
448
+ The state of each S3 storage in the fixture is not cleaned up.
449
+ """
450
+ s3_fixtures = {}
451
+ for alias, container in s3_multi_container.s3_containers.items():
452
+ config = container.s3_config
453
+ storage = S3ObjectStorage(config=config)
454
+ s3_fixtures[alias] = S3Fixture(config=config, storage=storage)
455
+ yield FederatedS3Fixture(s3_fixtures)
456
+
457
+
458
+ def get_persistent_federated_s3_fixture(
459
+ scope: PytestScope = "function", name: str = "federated_s3"
460
+ ):
461
+ """Get a federated S3 storage fixture with desired scope.
462
+
463
+ The state of the S3 storage is not cleaned up by the fixture.
464
+ """
465
+ return pytest.fixture(_persistent_federated_s3_fixture, scope=scope, name=name)
466
+
467
+
468
+ persistent_federated_s3_fixture = get_persistent_federated_s3_fixture()
469
+
470
+
471
+ async def _clean_federated_s3_fixture(
472
+ s3_multi_container: S3MultiContainerFixture,
473
+ ) -> AsyncGenerator[FederatedS3Fixture, None]:
474
+ """Fixture function that creates a clean FederatedS3Fixture instance.
475
+
476
+ The state of each S3 storage is cleaned up before yielding the fixture.
477
+ """
478
+ for federated_s3_fixture in _persistent_federated_s3_fixture(s3_multi_container):
479
+ for s3_fixture in federated_s3_fixture.storages.values():
480
+ await s3_fixture.delete_buckets()
481
+ yield federated_s3_fixture
482
+
483
+
484
+ def get_clean_federated_s3_fixture(
485
+ scope: PytestScope = "function", name: str = "federated_s3"
486
+ ):
487
+ """Get a federated S3 storage fixture with desired scope.
488
+
489
+ The state of the S3 storage is not cleaned up by the fixture.
490
+ """
491
+ return pytest_asyncio.fixture(_clean_federated_s3_fixture, scope=scope, name=name)
492
+
493
+
494
+ federated_s3_fixture = clean_federated_s3_fixture = get_clean_federated_s3_fixture()
@@ -44,7 +44,7 @@ class FileObject(BaseModel):
44
44
  bucket_id: str
45
45
  object_id: str
46
46
 
47
- @computed_field # type: ignore [misc]
47
+ @computed_field # type: ignore [prop-decorator]
48
48
  @property
49
49
  def content(self) -> bytes:
50
50
  """Extract the content from the file at the provided path"""
@@ -53,7 +53,7 @@ class FileObject(BaseModel):
53
53
  with open(self.file_path, "rb") as file:
54
54
  return file.read()
55
55
 
56
- @computed_field # type: ignore [misc]
56
+ @computed_field # type: ignore [prop-decorator]
57
57
  @property
58
58
  def md5(self) -> str:
59
59
  """Calculate the md5 hash of the content"""
@@ -26,6 +26,8 @@ from typing import NamedTuple, Optional
26
26
  from hexkit.custom_types import JsonObject
27
27
  from hexkit.protocols.eventpub import EventPublisherProtocol
28
28
 
29
+ __all__ = ["InMemEventStore", "InMemEventPublisher", "TopicExhaustedError"]
30
+
29
31
 
30
32
  class TopicExhaustedError(RuntimeError):
31
33
  """Thrown when now more event are queued in a topic."""
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: hexkit
3
- Version: 3.4.0
3
+ Version: 3.6.0
4
4
  Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
5
5
  Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
6
6
  License: Apache 2.0
@@ -1,47 +0,0 @@
1
- # Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2
- # for the German Human Genome-Phenome Archive (GHGA)
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
-
16
- """S3-related test fixtures and support functions."""
17
-
18
- from ._fixtures import ( # noqa: F401
19
- LOCALSTACK_IMAGE,
20
- MEBIBYTE,
21
- TEST_FILE_DIR,
22
- TEST_FILE_PATHS,
23
- FileObject,
24
- S3ContainerFixture,
25
- S3Fixture,
26
- calc_md5,
27
- clean_s3_fixture,
28
- get_clean_s3_fixture,
29
- get_persistent_s3_fixture,
30
- get_s3_container_fixture,
31
- persistent_s3_fixture,
32
- populate_storage,
33
- s3_container_fixture,
34
- s3_fixture,
35
- temp_file_object,
36
- tmp_file,
37
- upload_file,
38
- )
39
- from ._typical_workflow import typical_workflow # noqa: F401
40
- from ._utils import ( # noqa: F401
41
- check_part_size,
42
- download_and_check_test_file,
43
- multipart_upload_file,
44
- upload_part,
45
- upload_part_of_size,
46
- upload_part_via_url,
47
- )
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes