hexkit 7.3.0__tar.gz → 8.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {hexkit-7.3.0/src/hexkit.egg-info → hexkit-8.0.0}/PKG-INFO +1 -1
  2. {hexkit-7.3.0 → hexkit-8.0.0}/pyproject.toml +3 -3
  3. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/dao.py +43 -20
  4. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/daopub.py +9 -17
  5. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/objstorage.py +48 -0
  6. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/__init__.py +2 -0
  7. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/provider/__init__.py +2 -0
  8. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/provider/dao.py +78 -8
  9. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/provider/daopub.py +9 -7
  10. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/provider.py +41 -10
  11. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/testing/s3.py +26 -0
  12. {hexkit-7.3.0 → hexkit-8.0.0/src/hexkit.egg-info}/PKG-INFO +1 -1
  13. {hexkit-7.3.0 → hexkit-8.0.0}/LICENSE +0 -0
  14. {hexkit-7.3.0 → hexkit-8.0.0}/MANIFEST.in +0 -0
  15. {hexkit-7.3.0 → hexkit-8.0.0}/README.md +0 -0
  16. {hexkit-7.3.0 → hexkit-8.0.0}/setup.cfg +0 -0
  17. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/__init__.py +0 -0
  18. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/__main__.py +0 -0
  19. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/base.py +0 -0
  20. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/config.py +0 -0
  21. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/correlation.py +0 -0
  22. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/custom_types.py +0 -0
  23. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/log.py +0 -0
  24. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/opentelemetry.py +0 -0
  25. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/__init__.py +0 -0
  26. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/daosub.py +0 -0
  27. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/eventpub.py +0 -0
  28. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/eventsub.py +0 -0
  29. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/kvstore.py +0 -0
  30. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/__init__.py +0 -0
  31. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/__init__.py +0 -0
  32. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/config.py +0 -0
  33. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/provider/__init__.py +0 -0
  34. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/provider/daosub.py +0 -0
  35. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/provider/eventpub.py +0 -0
  36. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/provider/eventsub.py +0 -0
  37. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/provider/utils.py +0 -0
  38. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/testcontainer.py +0 -0
  39. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/testutils.py +0 -0
  40. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/config.py +0 -0
  41. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/migrations/__init__.py +0 -0
  42. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/migrations/_manager.py +0 -0
  43. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/migrations/_utils.py +0 -0
  44. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/migrations/helpers.py +0 -0
  45. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/provider/client.py +0 -0
  46. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/provider/kvstore.py +0 -0
  47. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/provider/utils.py +0 -0
  48. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/testutils.py +0 -0
  49. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/__init__.py +0 -0
  50. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/config.py +0 -0
  51. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/provider/__init__.py +0 -0
  52. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/provider/persistent_pub.py +0 -0
  53. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/testutils.py +0 -0
  54. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/__init__.py +0 -0
  55. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/config.py +0 -0
  56. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/provider/__init__.py +0 -0
  57. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/provider/client.py +0 -0
  58. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/provider/kvstore.py +0 -0
  59. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/testutils.py +0 -0
  60. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/__init__.py +0 -0
  61. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/test_files/__init__.py +0 -0
  62. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/test_files/test_file1.yaml +0 -0
  63. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/test_files/test_file2.yaml +0 -0
  64. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/test_files/test_file3.yaml +0 -0
  65. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/test_files/test_file4.yaml +0 -0
  66. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/testutils/__init__.py +0 -0
  67. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/testutils/_fixtures.py +0 -0
  68. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/testutils/_typical_workflow.py +0 -0
  69. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/testutils/_utils.py +0 -0
  70. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/testing/__init__.py +0 -0
  71. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/testing/dao.py +0 -0
  72. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/testing/eventpub.py +0 -0
  73. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/py.typed +0 -0
  74. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/utils.py +0 -0
  75. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit.egg-info/SOURCES.txt +0 -0
  76. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit.egg-info/dependency_links.txt +0 -0
  77. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit.egg-info/requires.txt +0 -0
  78. {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hexkit
3
- Version: 7.3.0
3
+ Version: 8.0.0
4
4
  Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
5
5
  Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
6
6
  License: Apache 2.0
@@ -1,6 +1,6 @@
1
1
  [build-system]
2
2
  requires = [
3
- "setuptools>=80.9",
3
+ "setuptools>=80.10",
4
4
  ]
5
5
  build-backend = "setuptools.build_meta"
6
6
 
@@ -23,7 +23,7 @@ classifiers = [
23
23
  "Intended Audience :: Developers",
24
24
  ]
25
25
  name = "hexkit"
26
- version = "7.3.0"
26
+ version = "8.0.0"
27
27
  description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
28
28
  dependencies = [
29
29
  "opentelemetry-api >=1.39, <2",
@@ -221,7 +221,7 @@ check_untyped_defs = true
221
221
  no_site_packages = false
222
222
 
223
223
  [tool.pytest.ini_options]
224
- minversion = "8.4"
224
+ minversion = "9.0"
225
225
  asyncio_mode = "strict"
226
226
  asyncio_default_fixture_loop_scope = "function"
227
227
 
@@ -23,7 +23,7 @@ from abc import ABC, abstractmethod
23
23
  from collections.abc import AsyncIterator, Collection, Mapping
24
24
  from contextlib import AbstractAsyncContextManager
25
25
  from functools import partial
26
- from typing import Any, TypeVar
26
+ from typing import Any, Generic, TypeVar
27
27
  from uuid import uuid4
28
28
 
29
29
  from pydantic import BaseModel, Field
@@ -39,10 +39,22 @@ __all__ = [
39
39
  "ResourceAlreadyExistsError",
40
40
  "ResourceNotFoundError",
41
41
  "UUID4Field",
42
+ "UniqueConstraintViolationError",
42
43
  ]
43
44
 
45
+
46
+ class IndexBase(ABC):
47
+ """Base from which all DAO index classes must inherit"""
48
+
49
+ @abstractmethod
50
+ def list_field_names(self) -> list[str]:
51
+ """Returns a list of all fields specified by this index"""
52
+ ...
53
+
54
+
44
55
  # Type variable for handling Data Transfer Objects:
45
56
  Dto = TypeVar("Dto", bound=BaseModel)
57
+ Index = TypeVar("Index", bound=IndexBase)
46
58
 
47
59
 
48
60
  class DaoError(RuntimeError):
@@ -65,6 +77,20 @@ class ResourceAlreadyExistsError(DaoError):
65
77
  super().__init__(message)
66
78
 
67
79
 
80
+ class UniqueConstraintViolationError(DaoError):
81
+ """Raised when attempting to insert a resource violated a uniqueness constraint.
82
+
83
+ This does not apply to the default unique index automatically applied to
84
+ the ID field.
85
+ """
86
+
87
+ def __init__(self, *, unique_fields: dict[str, Any]):
88
+ message = (
89
+ f"A resource with the unique field value(s) {unique_fields} already exists."
90
+ )
91
+ super().__init__(message)
92
+
93
+
68
94
  class FindError(DaoError):
69
95
  """Base for all error related to DAO find operations."""
70
96
 
@@ -268,16 +294,19 @@ class DaoFactoryBase:
268
294
  cls,
269
295
  *,
270
296
  dto_model: type[Dto],
271
- fields_to_index: Collection[str] | None,
297
+ indexes: Collection[Index] | None,
272
298
  ) -> None:
273
299
  """Checks that all provided fields are present in the dto_model.
274
300
  Raises IndexFieldsInvalidError otherwise.
275
301
  """
276
- if fields_to_index is None:
302
+ if not indexes:
277
303
  return
278
304
 
279
305
  try:
280
- validate_fields_in_model(model=dto_model, fields=fields_to_index)
306
+ for index in indexes:
307
+ validate_fields_in_model(
308
+ model=dto_model, fields=index.list_field_names()
309
+ )
281
310
  except FieldNotInModelError as error:
282
311
  raise cls.IndexFieldsInvalidError(
283
312
  f"Provided index fields are invalid: {error}"
@@ -289,16 +318,14 @@ class DaoFactoryBase:
289
318
  *,
290
319
  dto_model: type[Dto],
291
320
  id_field: str,
292
- fields_to_index: Collection[str] | None,
321
+ indexes: Collection[Index] | None,
293
322
  ) -> None:
294
323
  """Validates the input parameters of the get_dao method."""
295
324
  cls._validate_dto_model_id(dto_model=dto_model, id_field=id_field)
296
- cls._validate_fields_to_index(
297
- dto_model=dto_model, fields_to_index=fields_to_index
298
- )
325
+ cls._validate_fields_to_index(dto_model=dto_model, indexes=indexes)
299
326
 
300
327
 
301
- class DaoFactoryProtocol(DaoFactoryBase, ABC):
328
+ class DaoFactoryProtocol(DaoFactoryBase, ABC, Generic[Index]):
302
329
  """A protocol describing a factory to produce Data Access Objects (DAO) objects
303
330
  that are enclosed in transactional scopes.
304
331
  """
@@ -309,7 +336,7 @@ class DaoFactoryProtocol(DaoFactoryBase, ABC):
309
336
  name: str,
310
337
  dto_model: type[Dto],
311
338
  id_field: str,
312
- fields_to_index: Collection[str] | None = None,
339
+ indexes: Collection[Index] | None = None,
313
340
  ) -> Dao[Dto]:
314
341
  """Constructs a DAO for interacting with resources in a database.
315
342
 
@@ -322,9 +349,9 @@ class DaoFactoryProtocol(DaoFactoryBase, ABC):
322
349
  id_field:
323
350
  The name of the field of the `dto_model` that serves as resource ID.
324
351
  (DAO implementation might use this field as primary key.)
325
- fields_to_index:
326
- Optionally, provide any fields that should be indexed in addition to the
327
- `id_field`. Defaults to None.
352
+ indexes:
353
+ Optionally, provide any indexes that should be created in addition to
354
+ the provider's default index on `id_field`. Defaults to None.
328
355
  Returns:
329
356
  A DAO specific to the provided DTO model.
330
357
 
@@ -334,17 +361,13 @@ class DaoFactoryProtocol(DaoFactoryBase, ABC):
334
361
  self.IdTypeNotSupportedError:
335
362
  Raised when the id_field of the dto_model has an unexpected type.
336
363
  """
337
- self._validate(
338
- dto_model=dto_model,
339
- id_field=id_field,
340
- fields_to_index=fields_to_index,
341
- )
364
+ self._validate(dto_model=dto_model, id_field=id_field, indexes=indexes)
342
365
 
343
366
  return await self._get_dao(
344
367
  name=name,
345
368
  dto_model=dto_model,
346
369
  id_field=id_field,
347
- fields_to_index=fields_to_index,
370
+ indexes=indexes,
348
371
  )
349
372
 
350
373
  @abstractmethod
@@ -354,7 +377,7 @@ class DaoFactoryProtocol(DaoFactoryBase, ABC):
354
377
  name: str,
355
378
  dto_model: type[Dto],
356
379
  id_field: str,
357
- fields_to_index: Collection[str] | None,
380
+ indexes: Collection[Index] | None,
358
381
  ) -> Dao[Dto]:
359
382
  """*To be implemented by the provider. Input validation is done outside of this
360
383
  method.*
@@ -26,11 +26,7 @@ from abc import ABC, abstractmethod
26
26
  from collections.abc import Callable, Collection
27
27
 
28
28
  from hexkit.custom_types import JsonObject
29
- from hexkit.protocols.dao import (
30
- Dao,
31
- DaoFactoryBase,
32
- Dto,
33
- )
29
+ from hexkit.protocols.dao import Dao, DaoFactoryBase, Dto, Index
34
30
 
35
31
 
36
32
  class DaoPublisher(Dao[Dto], typing.Protocol[Dto]):
@@ -49,7 +45,7 @@ class DaoPublisher(Dao[Dto], typing.Protocol[Dto]):
49
45
  ...
50
46
 
51
47
 
52
- class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
48
+ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC, typing.Generic[Index]):
53
49
  """A protocol describing a factory to produce Data Access Objects (DAO) objects
54
50
  which automatically publish changes according to the outbox pattern.
55
51
  """
@@ -60,7 +56,7 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
60
56
  name: str,
61
57
  dto_model: type[Dto],
62
58
  id_field: str,
63
- fields_to_index: Collection[str] | None = None,
59
+ indexes: Collection[Index] | None = None,
64
60
  dto_to_event: Callable[[Dto], JsonObject | None],
65
61
  event_topic: str,
66
62
  autopublish: bool = True,
@@ -76,9 +72,9 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
76
72
  id_field:
77
73
  The name of the field of the `dto_model` that serves as resource ID.
78
74
  (DAO implementation might use this field as primary key.)
79
- fields_to_index:
80
- Optionally, provide any fields that should be indexed in addition to the
81
- `id_field`. Defaults to None.
75
+ indexes:
76
+ Optionally, provide any indexes that should be created in addition to
77
+ the provider's default index on `id_field`. Defaults to None.
82
78
  dto_to_event:
83
79
  A function that takes a DTO and returns the payload for an event.
84
80
  If the returned payload is None, the event will not be published.
@@ -97,17 +93,13 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
97
93
  self.IdTypeNotSupportedError:
98
94
  Raised when the id_field of the dto_model has an unexpected type.
99
95
  """
100
- self._validate(
101
- dto_model=dto_model,
102
- id_field=id_field,
103
- fields_to_index=fields_to_index,
104
- )
96
+ self._validate(dto_model=dto_model, id_field=id_field, indexes=indexes)
105
97
 
106
98
  return await self._get_dao(
107
99
  name=name,
108
100
  dto_model=dto_model,
109
101
  id_field=id_field,
110
- fields_to_index=fields_to_index,
102
+ indexes=indexes,
111
103
  dto_to_event=dto_to_event,
112
104
  event_topic=event_topic,
113
105
  autopublish=autopublish,
@@ -120,7 +112,7 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
120
112
  name: str,
121
113
  dto_model: type[Dto],
122
114
  id_field: str,
123
- fields_to_index: Collection[str] | None,
115
+ indexes: Collection[Index] | None,
124
116
  dto_to_event: Callable[[Dto], JsonObject | None],
125
117
  event_topic: str,
126
118
  autopublish: bool,
@@ -182,6 +182,31 @@ class ObjectStorageProtocol(ABC):
182
182
  anticipated_part_size=anticipated_part_size,
183
183
  )
184
184
 
185
+ async def list_multipart_uploads_for_object(
186
+ self, *, bucket_id: str, object_id: str
187
+ ) -> list[str]:
188
+ """Lists all active multipart uploads for the given object ID.
189
+
190
+ Raises a `BucketNotFoundError` if the bucket does not exist.
191
+ """
192
+ self._validate_bucket_id(bucket_id)
193
+ self._validate_object_id(object_id)
194
+ return await self._list_multipart_uploads_for_object(
195
+ bucket_id=bucket_id, object_id=object_id
196
+ )
197
+
198
+ async def get_all_multipart_uploads(self, *, bucket_id: str) -> dict[str, str]:
199
+ """Gets all active multipart uploads for the given bucket ID.
200
+
201
+ Returns a dict where the keys are upload IDs and values are object IDs.
202
+ S3 allows multiple ongoing multi-part uploads, so it's possible for some upload
203
+ IDs to map to the same object ID.
204
+
205
+ Raises a `BucketNotFoundError` if the bucket does not exist.
206
+ """
207
+ self._validate_bucket_id(bucket_id)
208
+ return await self._get_all_multipart_uploads(bucket_id=bucket_id)
209
+
185
210
  async def get_object_download_url(
186
211
  self, *, bucket_id: str, object_id: str, expires_after: int = 86400
187
212
  ) -> str:
@@ -404,6 +429,29 @@ class ObjectStorageProtocol(ABC):
404
429
  """
405
430
  ...
406
431
 
432
+ @abstractmethod
433
+ async def _list_multipart_uploads_for_object(
434
+ self, *, bucket_id: str, object_id: str
435
+ ) -> list[str]:
436
+ """Lists all active multipart uploads for the given object ID.
437
+
438
+ *To be implemented by the provider. Input validation is done outside of this
439
+ method.*
440
+ """
441
+ ...
442
+
443
+ @abstractmethod
444
+ async def _get_all_multipart_uploads(self, *, bucket_id: str) -> dict[str, str]:
445
+ """Gets all active multipart uploads for the given bucket ID.
446
+
447
+ Returns a dict where the keys are upload IDs and values are object IDs.
448
+ S3 allows multiple ongoing multi-part uploads, so it's possible for some upload
449
+ IDs to map to the same object ID.
450
+
451
+ *To be implemented by the provider. Input validation is done outside of this
452
+ method.*
453
+ """
454
+
407
455
  @abstractmethod
408
456
  async def _get_object_download_url(
409
457
  self, *, bucket_id: str, object_id: str, expires_after: int = 86400
@@ -23,6 +23,7 @@ from .provider import (
23
23
  MongoDbDao,
24
24
  MongoDbDaoFactory,
25
25
  MongoDbDtoKeyValueStore,
26
+ MongoDbIndex,
26
27
  MongoDbJsonKeyValueStore,
27
28
  MongoDbStrKeyValueStore,
28
29
  translate_pymongo_errors,
@@ -35,6 +36,7 @@ __all__ = [
35
36
  "MongoDbDao",
36
37
  "MongoDbDaoFactory",
37
38
  "MongoDbDtoKeyValueStore",
39
+ "MongoDbIndex",
38
40
  "MongoDbJsonKeyValueStore",
39
41
  "MongoDbStrKeyValueStore",
40
42
  "translate_pymongo_errors",
@@ -23,6 +23,7 @@ from .client import ConfiguredMongoClient
23
23
  from .dao import (
24
24
  MongoDbDao,
25
25
  MongoDbDaoFactory,
26
+ MongoDbIndex,
26
27
  get_single_hit,
27
28
  replace_id_field_in_find_mapping,
28
29
  validate_find_mapping,
@@ -41,6 +42,7 @@ __all__ = [
41
42
  "MongoDbDao",
42
43
  "MongoDbDaoFactory",
43
44
  "MongoDbDtoKeyValueStore",
45
+ "MongoDbIndex",
44
46
  "MongoDbJsonKeyValueStore",
45
47
  "MongoDbStrKeyValueStore",
46
48
  "document_to_dto",
@@ -22,7 +22,7 @@ Utilities for testing are located in `./testutils.py`.
22
22
  from collections.abc import AsyncIterator, Callable, Collection, Mapping
23
23
  from contextlib import AbstractAsyncContextManager, asynccontextmanager
24
24
  from functools import partial
25
- from typing import Any, Generic
25
+ from typing import Any, Generic, Literal, TypeAlias
26
26
 
27
27
  from pymongo import AsyncMongoClient
28
28
  from pymongo.asynchronous.collection import AsyncCollection
@@ -33,11 +33,13 @@ from hexkit.protocols.dao import (
33
33
  Dao,
34
34
  DaoFactoryProtocol,
35
35
  Dto,
36
+ IndexBase,
36
37
  InvalidFindMappingError,
37
38
  MultipleHitsFoundError,
38
39
  NoHitsFoundError,
39
40
  ResourceAlreadyExistsError,
40
41
  ResourceNotFoundError,
42
+ UniqueConstraintViolationError,
41
43
  )
42
44
  from hexkit.providers.mongodb.config import MongoDbConfig
43
45
  from hexkit.providers.mongodb.provider.client import ConfiguredMongoClient
@@ -51,6 +53,7 @@ from hexkit.utils import FieldNotInModelError, validate_fields_in_model
51
53
  __all__ = [
52
54
  "MongoDbDao",
53
55
  "MongoDbDaoFactory",
56
+ "MongoDbIndex",
54
57
  "get_single_hit",
55
58
  "replace_id_field_in_find_mapping",
56
59
  "validate_find_mapping",
@@ -113,6 +116,67 @@ async def get_single_hit(
113
116
  raise MultipleHitsFoundError(mapping=mapping)
114
117
 
115
118
 
119
+ FieldName: TypeAlias = str
120
+ SortOrder: TypeAlias = Literal[1] | Literal[-1]
121
+
122
+
123
+ class MongoDbIndex(IndexBase):
124
+ """Information required to apply a single MongoDbIndex."""
125
+
126
+ fields: dict[FieldName, SortOrder]
127
+ properties: dict[str, Any] | None = None
128
+
129
+ def __init__(
130
+ self,
131
+ *,
132
+ fields: FieldName | dict[FieldName, SortOrder],
133
+ properties: dict[str, Any] | None = None,
134
+ ):
135
+ """Initialize the index.
136
+
137
+ **Args**
138
+ - `fields`: a single field name or a dict containing the field name and the sort
139
+ order. For sort order, 1 means ascending and -1 means descending. If the index
140
+ covers the model's `id_field`, then specify the actual field name instead of
141
+ "_id". When passing a field name instead of a dict, ascending sort order is used.
142
+ - `properties`: a dictionary where the keys are MongoDB index property names, and
143
+ the values are the value to pass for that property.
144
+
145
+ More information on index creation with Pymongo, including a list of the supported
146
+ index properties, can be found [here](https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.create_index)
147
+
148
+ Example:
149
+ ```python
150
+ # This creates a compound unique index over field_a and field_b.
151
+ MongoDbIndex(
152
+ fields={"field_a": 1, "field_b": -1},
153
+ properties={"unique": True}
154
+ )
155
+
156
+ # This creates an ascending-sorted index over field_a.
157
+ MongoDbIndex("field_a")
158
+ ```
159
+ """
160
+ if isinstance(fields, str):
161
+ self.fields = {fields: 1}
162
+ else:
163
+ self.fields = fields
164
+
165
+ self.properties = properties
166
+
167
+ def list_field_names(self) -> list[str]:
168
+ """Return a list of all the field names contained in this index"""
169
+ return list(self.fields)
170
+
171
+ def fields_with_id_replaced(
172
+ self, id_field: str
173
+ ) -> str | list[tuple[FieldName, SortOrder]]:
174
+ """Returns the `fields` property with all instances of `id_field`
175
+ replaced with `"_id"`.
176
+ """
177
+ return [("_id" if f == id_field else f, so) for f, so in self.fields.items()]
178
+
179
+
116
180
  class MongoDbDao(Generic[Dto]):
117
181
  """A MongoDb-specific DAO."""
118
182
 
@@ -295,6 +359,11 @@ class MongoDbDao(Generic[Dto]):
295
359
  try:
296
360
  await self._collection.insert_one(document)
297
361
  except DuplicateKeyError as error:
362
+ key_value = error.details.get("keyValue") # type: ignore
363
+ if key_value is not None and list(key_value) != ["_id"]:
364
+ raise UniqueConstraintViolationError(
365
+ unique_fields=key_value
366
+ ) from error
298
367
  raise ResourceAlreadyExistsError(id_=document["_id"]) from error
299
368
 
300
369
  async def upsert(self, dto: Dto) -> None:
@@ -312,7 +381,7 @@ class MongoDbDao(Generic[Dto]):
312
381
  )
313
382
 
314
383
 
315
- class MongoDbDaoFactory(DaoFactoryProtocol):
384
+ class MongoDbDaoFactory(DaoFactoryProtocol[MongoDbIndex]):
316
385
  """A MongoDB-based provider implementing the DaoFactoryProtocol."""
317
386
 
318
387
  @classmethod
@@ -354,7 +423,7 @@ class MongoDbDaoFactory(DaoFactoryProtocol):
354
423
  name: str,
355
424
  dto_model: type[Dto],
356
425
  id_field: str,
357
- fields_to_index: Collection[str] | None,
426
+ indexes: Collection[MongoDbIndex] | None = None,
358
427
  ) -> Dao[Dto]:
359
428
  """Constructs a DAO for interacting with resources in a MongoDB database.
360
429
 
@@ -366,13 +435,14 @@ class MongoDbDaoFactory(DaoFactoryProtocol):
366
435
  from the database server. Thus for compliance with the DaoFactoryProtocol, this
367
436
  method is async.
368
437
  """
369
- if fields_to_index is not None:
370
- raise NotImplementedError(
371
- "Indexing on non-ID fields has not been implemented, yet."
372
- )
373
-
374
438
  collection = self._db[name]
375
439
 
440
+ for index in indexes or []:
441
+ properties = index.properties or {}
442
+ await collection.create_index(
443
+ index.fields_with_id_replaced(id_field=id_field), **properties
444
+ )
445
+
376
446
  return MongoDbDao(
377
447
  collection=collection,
378
448
  dto_model=dto_model,
@@ -45,6 +45,7 @@ from hexkit.providers.mongodb.provider import (
45
45
  translate_pymongo_errors,
46
46
  validate_find_mapping,
47
47
  )
48
+ from hexkit.providers.mongodb.provider.dao import MongoDbIndex
48
49
  from hexkit.providers.mongokafka.config import MongoKafkaConfig
49
50
 
50
51
 
@@ -437,7 +438,7 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
437
438
  await self.publish_document(document)
438
439
 
439
440
 
440
- class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
441
+ class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol[MongoDbIndex]):
441
442
  """A provider implementing the DaoPublisherFactoryProtocol based on MongoDB and
442
443
  Apache Kafka.
443
444
  """
@@ -491,7 +492,7 @@ class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
491
492
  name: str,
492
493
  dto_model: type[Dto],
493
494
  id_field: str,
494
- fields_to_index: Collection[str] | None,
495
+ indexes: Collection[MongoDbIndex] | None,
495
496
  dto_to_event: Callable[[Dto], JsonObject | None],
496
497
  event_topic: str,
497
498
  autopublish: bool,
@@ -502,13 +503,14 @@ class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
502
503
  Please see the DaoPublisherFactoryProtocol superclass for documentation of
503
504
  parameters.
504
505
  """
505
- if fields_to_index is not None:
506
- raise NotImplementedError(
507
- "Indexing on non-ID fields has not been implemented, yet."
508
- )
509
-
510
506
  collection = self._db[name]
511
507
 
508
+ for index in indexes or []:
509
+ properties = index.properties or {}
510
+ await collection.create_index(
511
+ index.fields_with_id_replaced(id_field=id_field), **properties
512
+ )
513
+
512
514
  dao = MongoDbDao(
513
515
  collection=collection,
514
516
  dto_model=dto_model,
@@ -401,15 +401,17 @@ class S3ObjectStorage(ObjectStorageProtocol):
401
401
  url=presigned_url["url"], fields=presigned_url["fields"]
402
402
  )
403
403
 
404
- async def _list_multipart_upload_for_object(
404
+ async def _list_multipart_uploads_for_object(
405
405
  self, *, bucket_id: str, object_id: str
406
406
  ) -> list[str]:
407
- """Lists all active multipart uploads for the given object. Returns a list of
408
- their IDs.
407
+ """Lists all active multipart uploads for the given object ID.
409
408
 
410
- (S3 allows multiple ongoing multi-part uploads.)
409
+ Raises a `BucketNotFoundError` if the bucket does not exist.
410
+
411
+ Boto3 Documentation:
412
+ https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/list_multipart_uploads.html#S3.Client.list_multipart_uploads
411
413
  """
412
- uploads = []
414
+ uploads: list[str] = []
413
415
  try:
414
416
  response_iter = await asyncio.to_thread(
415
417
  self._client.get_paginator("list_multipart_uploads").paginate,
@@ -430,9 +432,38 @@ class S3ObjectStorage(ObjectStorageProtocol):
430
432
 
431
433
  return uploads
432
434
 
435
+ async def _get_all_multipart_uploads(self, *, bucket_id: str) -> dict[str, str]:
436
+ """Gets all active multipart uploads for the given bucket ID.
437
+
438
+ Returns a dict where the keys are upload IDs and values are object IDs.
439
+ S3 allows multiple ongoing multi-part uploads, so it's possible for some upload
440
+ IDs to map to the same object ID.
441
+
442
+ Raises a `BucketNotFoundError` if the bucket does not exist.
443
+ """
444
+ uploads: dict[str, str] = {}
445
+ try:
446
+ response_iter = await asyncio.to_thread(
447
+ self._client.get_paginator("list_multipart_uploads").paginate,
448
+ Bucket=bucket_id,
449
+ )
450
+ for response_page in response_iter:
451
+ uploads.update(
452
+ {
453
+ upload["UploadId"]: upload["Key"]
454
+ for upload in response_page.get("Uploads", [])
455
+ }
456
+ )
457
+ except botocore.exceptions.ClientError as error:
458
+ raise self._translate_s3_client_errors(
459
+ error, bucket_id=bucket_id
460
+ ) from error
461
+
462
+ return uploads
463
+
433
464
  async def _assert_no_multipart_upload(self, *, bucket_id: str, object_id: str):
434
465
  """Ensure that there are no active multi-part uploads for the given object."""
435
- upload_ids = await self._list_multipart_upload_for_object(
466
+ upload_ids = await self._list_multipart_uploads_for_object(
436
467
  bucket_id=bucket_id, object_id=object_id
437
468
  )
438
469
  if len(upload_ids) > 0:
@@ -454,14 +485,14 @@ class S3ObjectStorage(ObjectStorageProtocol):
454
485
  By default, also verifies that this upload is the only upload active for
455
486
  that file. Otherwise, raises MultipleActiveUploadsError.
456
487
  """
457
- upload_ids = await self._list_multipart_upload_for_object(
488
+ upload_ids = await self._list_multipart_uploads_for_object(
458
489
  bucket_id=bucket_id, object_id=object_id
459
490
  )
460
491
  n_uploads = len(upload_ids)
461
492
 
462
493
  if assert_exclusiveness and n_uploads > 1:
463
494
  raise self.MultipleActiveUploadsError(
464
- bucket_id=bucket_id, object_id=object_id, upload_ids=upload_ids
495
+ bucket_id=bucket_id, object_id=object_id, upload_ids=list(upload_ids)
465
496
  )
466
497
 
467
498
  if upload_id not in upload_ids:
@@ -877,12 +908,12 @@ class S3ObjectStorage(ObjectStorageProtocol):
877
908
  # There should only be one ongoing multipart upload/copy at at a time as long
878
909
  # as a new object ID is generated for each attempt
879
910
  try:
880
- upload_ids = await self._list_multipart_upload_for_object(
911
+ upload_ids = await self._list_multipart_uploads_for_object(
881
912
  bucket_id=dest_bucket_id, object_id=dest_object_id
882
913
  )
883
914
  if len(upload_ids) == 1:
884
915
  await self._abort_multipart_upload(
885
- upload_id=upload_ids[0],
916
+ upload_id=next(iter(upload_ids)),
886
917
  bucket_id=dest_bucket_id,
887
918
  object_id=dest_object_id,
888
919
  )
@@ -168,6 +168,32 @@ class InMemObjectStorage(ObjectStorageProtocol):
168
168
  uploads_for_object = self.uploads[bucket_id].get(object_id, set())
169
169
  return sorted(uploads_for_object)
170
170
 
171
+ async def _list_multipart_uploads_for_object(
172
+ self, *, bucket_id: str, object_id: str
173
+ ) -> list[str]:
174
+ """Lists all active multipart uploads for the given object ID.
175
+
176
+ Raises a `BucketNotFoundError` if the bucket does not exist.
177
+ """
178
+ await self._assert_bucket_exists(bucket_id)
179
+ return list(self.uploads[bucket_id].get(object_id, set()))
180
+
181
+ async def _get_all_multipart_uploads(self, *, bucket_id: str) -> dict[str, str]:
182
+ """Gets all active multipart uploads for the given bucket ID.
183
+
184
+ Returns a dict where the keys are upload IDs and values are object IDs.
185
+ S3 allows multiple ongoing multi-part uploads, so it's possible for some upload
186
+ IDs to map to the same object ID.
187
+
188
+ Raises a `BucketNotFoundError` if the bucket does not exist.
189
+ """
190
+ await self._assert_bucket_exists(bucket_id)
191
+ return {
192
+ upload_id: object_id
193
+ for object_id, uploads in self.uploads[bucket_id].items()
194
+ for upload_id in uploads
195
+ }
196
+
171
197
  async def _assert_no_multipart_upload(self, *, bucket_id: str, object_id: str):
172
198
  """Ensure that there are no active multi-part uploads for the given object.
173
199
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hexkit
3
- Version: 7.3.0
3
+ Version: 8.0.0
4
4
  Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
5
5
  Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
6
6
  License: Apache 2.0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes