hexkit 7.3.0__tar.gz → 8.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hexkit-7.3.0/src/hexkit.egg-info → hexkit-8.0.0}/PKG-INFO +1 -1
- {hexkit-7.3.0 → hexkit-8.0.0}/pyproject.toml +3 -3
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/dao.py +43 -20
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/daopub.py +9 -17
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/objstorage.py +48 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/__init__.py +2 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/provider/__init__.py +2 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/provider/dao.py +78 -8
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/provider/daopub.py +9 -7
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/provider.py +41 -10
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/testing/s3.py +26 -0
- {hexkit-7.3.0 → hexkit-8.0.0/src/hexkit.egg-info}/PKG-INFO +1 -1
- {hexkit-7.3.0 → hexkit-8.0.0}/LICENSE +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/MANIFEST.in +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/README.md +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/setup.cfg +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/__main__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/base.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/config.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/correlation.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/custom_types.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/log.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/opentelemetry.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/daosub.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/eventpub.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/eventsub.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/protocols/kvstore.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/config.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/provider/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/provider/daosub.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/provider/eventpub.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/provider/eventsub.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/provider/utils.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/testcontainer.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/akafka/testutils.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/config.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/migrations/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/migrations/_manager.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/migrations/_utils.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/migrations/helpers.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/provider/client.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/provider/kvstore.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/provider/utils.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongodb/testutils.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/config.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/provider/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/provider/persistent_pub.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/mongokafka/testutils.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/config.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/provider/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/provider/client.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/provider/kvstore.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/redis/testutils.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/test_files/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/test_files/test_file1.yaml +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/test_files/test_file2.yaml +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/test_files/test_file3.yaml +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/test_files/test_file4.yaml +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/testutils/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/testutils/_fixtures.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/testutils/_typical_workflow.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/s3/testutils/_utils.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/testing/__init__.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/testing/dao.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/providers/testing/eventpub.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/py.typed +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit/utils.py +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit.egg-info/SOURCES.txt +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit.egg-info/dependency_links.txt +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit.egg-info/requires.txt +0 -0
- {hexkit-7.3.0 → hexkit-8.0.0}/src/hexkit.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[build-system]
|
|
2
2
|
requires = [
|
|
3
|
-
"setuptools>=80.
|
|
3
|
+
"setuptools>=80.10",
|
|
4
4
|
]
|
|
5
5
|
build-backend = "setuptools.build_meta"
|
|
6
6
|
|
|
@@ -23,7 +23,7 @@ classifiers = [
|
|
|
23
23
|
"Intended Audience :: Developers",
|
|
24
24
|
]
|
|
25
25
|
name = "hexkit"
|
|
26
|
-
version = "
|
|
26
|
+
version = "8.0.0"
|
|
27
27
|
description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
|
|
28
28
|
dependencies = [
|
|
29
29
|
"opentelemetry-api >=1.39, <2",
|
|
@@ -221,7 +221,7 @@ check_untyped_defs = true
|
|
|
221
221
|
no_site_packages = false
|
|
222
222
|
|
|
223
223
|
[tool.pytest.ini_options]
|
|
224
|
-
minversion = "
|
|
224
|
+
minversion = "9.0"
|
|
225
225
|
asyncio_mode = "strict"
|
|
226
226
|
asyncio_default_fixture_loop_scope = "function"
|
|
227
227
|
|
|
@@ -23,7 +23,7 @@ from abc import ABC, abstractmethod
|
|
|
23
23
|
from collections.abc import AsyncIterator, Collection, Mapping
|
|
24
24
|
from contextlib import AbstractAsyncContextManager
|
|
25
25
|
from functools import partial
|
|
26
|
-
from typing import Any, TypeVar
|
|
26
|
+
from typing import Any, Generic, TypeVar
|
|
27
27
|
from uuid import uuid4
|
|
28
28
|
|
|
29
29
|
from pydantic import BaseModel, Field
|
|
@@ -39,10 +39,22 @@ __all__ = [
|
|
|
39
39
|
"ResourceAlreadyExistsError",
|
|
40
40
|
"ResourceNotFoundError",
|
|
41
41
|
"UUID4Field",
|
|
42
|
+
"UniqueConstraintViolationError",
|
|
42
43
|
]
|
|
43
44
|
|
|
45
|
+
|
|
46
|
+
class IndexBase(ABC):
|
|
47
|
+
"""Base from which all DAO index classes must inherit"""
|
|
48
|
+
|
|
49
|
+
@abstractmethod
|
|
50
|
+
def list_field_names(self) -> list[str]:
|
|
51
|
+
"""Returns a list of all fields specified by this index"""
|
|
52
|
+
...
|
|
53
|
+
|
|
54
|
+
|
|
44
55
|
# Type variable for handling Data Transfer Objects:
|
|
45
56
|
Dto = TypeVar("Dto", bound=BaseModel)
|
|
57
|
+
Index = TypeVar("Index", bound=IndexBase)
|
|
46
58
|
|
|
47
59
|
|
|
48
60
|
class DaoError(RuntimeError):
|
|
@@ -65,6 +77,20 @@ class ResourceAlreadyExistsError(DaoError):
|
|
|
65
77
|
super().__init__(message)
|
|
66
78
|
|
|
67
79
|
|
|
80
|
+
class UniqueConstraintViolationError(DaoError):
|
|
81
|
+
"""Raised when attempting to insert a resource violated a uniqueness constraint.
|
|
82
|
+
|
|
83
|
+
This does not apply to the default unique index automatically applied to
|
|
84
|
+
the ID field.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
def __init__(self, *, unique_fields: dict[str, Any]):
|
|
88
|
+
message = (
|
|
89
|
+
f"A resource with the unique field value(s) {unique_fields} already exists."
|
|
90
|
+
)
|
|
91
|
+
super().__init__(message)
|
|
92
|
+
|
|
93
|
+
|
|
68
94
|
class FindError(DaoError):
|
|
69
95
|
"""Base for all error related to DAO find operations."""
|
|
70
96
|
|
|
@@ -268,16 +294,19 @@ class DaoFactoryBase:
|
|
|
268
294
|
cls,
|
|
269
295
|
*,
|
|
270
296
|
dto_model: type[Dto],
|
|
271
|
-
|
|
297
|
+
indexes: Collection[Index] | None,
|
|
272
298
|
) -> None:
|
|
273
299
|
"""Checks that all provided fields are present in the dto_model.
|
|
274
300
|
Raises IndexFieldsInvalidError otherwise.
|
|
275
301
|
"""
|
|
276
|
-
if
|
|
302
|
+
if not indexes:
|
|
277
303
|
return
|
|
278
304
|
|
|
279
305
|
try:
|
|
280
|
-
|
|
306
|
+
for index in indexes:
|
|
307
|
+
validate_fields_in_model(
|
|
308
|
+
model=dto_model, fields=index.list_field_names()
|
|
309
|
+
)
|
|
281
310
|
except FieldNotInModelError as error:
|
|
282
311
|
raise cls.IndexFieldsInvalidError(
|
|
283
312
|
f"Provided index fields are invalid: {error}"
|
|
@@ -289,16 +318,14 @@ class DaoFactoryBase:
|
|
|
289
318
|
*,
|
|
290
319
|
dto_model: type[Dto],
|
|
291
320
|
id_field: str,
|
|
292
|
-
|
|
321
|
+
indexes: Collection[Index] | None,
|
|
293
322
|
) -> None:
|
|
294
323
|
"""Validates the input parameters of the get_dao method."""
|
|
295
324
|
cls._validate_dto_model_id(dto_model=dto_model, id_field=id_field)
|
|
296
|
-
cls._validate_fields_to_index(
|
|
297
|
-
dto_model=dto_model, fields_to_index=fields_to_index
|
|
298
|
-
)
|
|
325
|
+
cls._validate_fields_to_index(dto_model=dto_model, indexes=indexes)
|
|
299
326
|
|
|
300
327
|
|
|
301
|
-
class DaoFactoryProtocol(DaoFactoryBase, ABC):
|
|
328
|
+
class DaoFactoryProtocol(DaoFactoryBase, ABC, Generic[Index]):
|
|
302
329
|
"""A protocol describing a factory to produce Data Access Objects (DAO) objects
|
|
303
330
|
that are enclosed in transactional scopes.
|
|
304
331
|
"""
|
|
@@ -309,7 +336,7 @@ class DaoFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
309
336
|
name: str,
|
|
310
337
|
dto_model: type[Dto],
|
|
311
338
|
id_field: str,
|
|
312
|
-
|
|
339
|
+
indexes: Collection[Index] | None = None,
|
|
313
340
|
) -> Dao[Dto]:
|
|
314
341
|
"""Constructs a DAO for interacting with resources in a database.
|
|
315
342
|
|
|
@@ -322,9 +349,9 @@ class DaoFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
322
349
|
id_field:
|
|
323
350
|
The name of the field of the `dto_model` that serves as resource ID.
|
|
324
351
|
(DAO implementation might use this field as primary key.)
|
|
325
|
-
|
|
326
|
-
Optionally, provide any
|
|
327
|
-
`id_field`. Defaults to None.
|
|
352
|
+
indexes:
|
|
353
|
+
Optionally, provide any indexes that should be created in addition to
|
|
354
|
+
the provider's default index on `id_field`. Defaults to None.
|
|
328
355
|
Returns:
|
|
329
356
|
A DAO specific to the provided DTO model.
|
|
330
357
|
|
|
@@ -334,17 +361,13 @@ class DaoFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
334
361
|
self.IdTypeNotSupportedError:
|
|
335
362
|
Raised when the id_field of the dto_model has an unexpected type.
|
|
336
363
|
"""
|
|
337
|
-
self._validate(
|
|
338
|
-
dto_model=dto_model,
|
|
339
|
-
id_field=id_field,
|
|
340
|
-
fields_to_index=fields_to_index,
|
|
341
|
-
)
|
|
364
|
+
self._validate(dto_model=dto_model, id_field=id_field, indexes=indexes)
|
|
342
365
|
|
|
343
366
|
return await self._get_dao(
|
|
344
367
|
name=name,
|
|
345
368
|
dto_model=dto_model,
|
|
346
369
|
id_field=id_field,
|
|
347
|
-
|
|
370
|
+
indexes=indexes,
|
|
348
371
|
)
|
|
349
372
|
|
|
350
373
|
@abstractmethod
|
|
@@ -354,7 +377,7 @@ class DaoFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
354
377
|
name: str,
|
|
355
378
|
dto_model: type[Dto],
|
|
356
379
|
id_field: str,
|
|
357
|
-
|
|
380
|
+
indexes: Collection[Index] | None,
|
|
358
381
|
) -> Dao[Dto]:
|
|
359
382
|
"""*To be implemented by the provider. Input validation is done outside of this
|
|
360
383
|
method.*
|
|
@@ -26,11 +26,7 @@ from abc import ABC, abstractmethod
|
|
|
26
26
|
from collections.abc import Callable, Collection
|
|
27
27
|
|
|
28
28
|
from hexkit.custom_types import JsonObject
|
|
29
|
-
from hexkit.protocols.dao import
|
|
30
|
-
Dao,
|
|
31
|
-
DaoFactoryBase,
|
|
32
|
-
Dto,
|
|
33
|
-
)
|
|
29
|
+
from hexkit.protocols.dao import Dao, DaoFactoryBase, Dto, Index
|
|
34
30
|
|
|
35
31
|
|
|
36
32
|
class DaoPublisher(Dao[Dto], typing.Protocol[Dto]):
|
|
@@ -49,7 +45,7 @@ class DaoPublisher(Dao[Dto], typing.Protocol[Dto]):
|
|
|
49
45
|
...
|
|
50
46
|
|
|
51
47
|
|
|
52
|
-
class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
|
|
48
|
+
class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC, typing.Generic[Index]):
|
|
53
49
|
"""A protocol describing a factory to produce Data Access Objects (DAO) objects
|
|
54
50
|
which automatically publish changes according to the outbox pattern.
|
|
55
51
|
"""
|
|
@@ -60,7 +56,7 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
60
56
|
name: str,
|
|
61
57
|
dto_model: type[Dto],
|
|
62
58
|
id_field: str,
|
|
63
|
-
|
|
59
|
+
indexes: Collection[Index] | None = None,
|
|
64
60
|
dto_to_event: Callable[[Dto], JsonObject | None],
|
|
65
61
|
event_topic: str,
|
|
66
62
|
autopublish: bool = True,
|
|
@@ -76,9 +72,9 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
76
72
|
id_field:
|
|
77
73
|
The name of the field of the `dto_model` that serves as resource ID.
|
|
78
74
|
(DAO implementation might use this field as primary key.)
|
|
79
|
-
|
|
80
|
-
Optionally, provide any
|
|
81
|
-
`id_field`. Defaults to None.
|
|
75
|
+
indexes:
|
|
76
|
+
Optionally, provide any indexes that should be created in addition to
|
|
77
|
+
the provider's default index on `id_field`. Defaults to None.
|
|
82
78
|
dto_to_event:
|
|
83
79
|
A function that takes a DTO and returns the payload for an event.
|
|
84
80
|
If the returned payload is None, the event will not be published.
|
|
@@ -97,17 +93,13 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
97
93
|
self.IdTypeNotSupportedError:
|
|
98
94
|
Raised when the id_field of the dto_model has an unexpected type.
|
|
99
95
|
"""
|
|
100
|
-
self._validate(
|
|
101
|
-
dto_model=dto_model,
|
|
102
|
-
id_field=id_field,
|
|
103
|
-
fields_to_index=fields_to_index,
|
|
104
|
-
)
|
|
96
|
+
self._validate(dto_model=dto_model, id_field=id_field, indexes=indexes)
|
|
105
97
|
|
|
106
98
|
return await self._get_dao(
|
|
107
99
|
name=name,
|
|
108
100
|
dto_model=dto_model,
|
|
109
101
|
id_field=id_field,
|
|
110
|
-
|
|
102
|
+
indexes=indexes,
|
|
111
103
|
dto_to_event=dto_to_event,
|
|
112
104
|
event_topic=event_topic,
|
|
113
105
|
autopublish=autopublish,
|
|
@@ -120,7 +112,7 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
|
|
|
120
112
|
name: str,
|
|
121
113
|
dto_model: type[Dto],
|
|
122
114
|
id_field: str,
|
|
123
|
-
|
|
115
|
+
indexes: Collection[Index] | None,
|
|
124
116
|
dto_to_event: Callable[[Dto], JsonObject | None],
|
|
125
117
|
event_topic: str,
|
|
126
118
|
autopublish: bool,
|
|
@@ -182,6 +182,31 @@ class ObjectStorageProtocol(ABC):
|
|
|
182
182
|
anticipated_part_size=anticipated_part_size,
|
|
183
183
|
)
|
|
184
184
|
|
|
185
|
+
async def list_multipart_uploads_for_object(
|
|
186
|
+
self, *, bucket_id: str, object_id: str
|
|
187
|
+
) -> list[str]:
|
|
188
|
+
"""Lists all active multipart uploads for the given object ID.
|
|
189
|
+
|
|
190
|
+
Raises a `BucketNotFoundError` if the bucket does not exist.
|
|
191
|
+
"""
|
|
192
|
+
self._validate_bucket_id(bucket_id)
|
|
193
|
+
self._validate_object_id(object_id)
|
|
194
|
+
return await self._list_multipart_uploads_for_object(
|
|
195
|
+
bucket_id=bucket_id, object_id=object_id
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
async def get_all_multipart_uploads(self, *, bucket_id: str) -> dict[str, str]:
|
|
199
|
+
"""Gets all active multipart uploads for the given bucket ID.
|
|
200
|
+
|
|
201
|
+
Returns a dict where the keys are upload IDs and values are object IDs.
|
|
202
|
+
S3 allows multiple ongoing multi-part uploads, so it's possible for some upload
|
|
203
|
+
IDs to map to the same object ID.
|
|
204
|
+
|
|
205
|
+
Raises a `BucketNotFoundError` if the bucket does not exist.
|
|
206
|
+
"""
|
|
207
|
+
self._validate_bucket_id(bucket_id)
|
|
208
|
+
return await self._get_all_multipart_uploads(bucket_id=bucket_id)
|
|
209
|
+
|
|
185
210
|
async def get_object_download_url(
|
|
186
211
|
self, *, bucket_id: str, object_id: str, expires_after: int = 86400
|
|
187
212
|
) -> str:
|
|
@@ -404,6 +429,29 @@ class ObjectStorageProtocol(ABC):
|
|
|
404
429
|
"""
|
|
405
430
|
...
|
|
406
431
|
|
|
432
|
+
@abstractmethod
|
|
433
|
+
async def _list_multipart_uploads_for_object(
|
|
434
|
+
self, *, bucket_id: str, object_id: str
|
|
435
|
+
) -> list[str]:
|
|
436
|
+
"""Lists all active multipart uploads for the given object ID.
|
|
437
|
+
|
|
438
|
+
*To be implemented by the provider. Input validation is done outside of this
|
|
439
|
+
method.*
|
|
440
|
+
"""
|
|
441
|
+
...
|
|
442
|
+
|
|
443
|
+
@abstractmethod
|
|
444
|
+
async def _get_all_multipart_uploads(self, *, bucket_id: str) -> dict[str, str]:
|
|
445
|
+
"""Gets all active multipart uploads for the given bucket ID.
|
|
446
|
+
|
|
447
|
+
Returns a dict where the keys are upload IDs and values are object IDs.
|
|
448
|
+
S3 allows multiple ongoing multi-part uploads, so it's possible for some upload
|
|
449
|
+
IDs to map to the same object ID.
|
|
450
|
+
|
|
451
|
+
*To be implemented by the provider. Input validation is done outside of this
|
|
452
|
+
method.*
|
|
453
|
+
"""
|
|
454
|
+
|
|
407
455
|
@abstractmethod
|
|
408
456
|
async def _get_object_download_url(
|
|
409
457
|
self, *, bucket_id: str, object_id: str, expires_after: int = 86400
|
|
@@ -23,6 +23,7 @@ from .provider import (
|
|
|
23
23
|
MongoDbDao,
|
|
24
24
|
MongoDbDaoFactory,
|
|
25
25
|
MongoDbDtoKeyValueStore,
|
|
26
|
+
MongoDbIndex,
|
|
26
27
|
MongoDbJsonKeyValueStore,
|
|
27
28
|
MongoDbStrKeyValueStore,
|
|
28
29
|
translate_pymongo_errors,
|
|
@@ -35,6 +36,7 @@ __all__ = [
|
|
|
35
36
|
"MongoDbDao",
|
|
36
37
|
"MongoDbDaoFactory",
|
|
37
38
|
"MongoDbDtoKeyValueStore",
|
|
39
|
+
"MongoDbIndex",
|
|
38
40
|
"MongoDbJsonKeyValueStore",
|
|
39
41
|
"MongoDbStrKeyValueStore",
|
|
40
42
|
"translate_pymongo_errors",
|
|
@@ -23,6 +23,7 @@ from .client import ConfiguredMongoClient
|
|
|
23
23
|
from .dao import (
|
|
24
24
|
MongoDbDao,
|
|
25
25
|
MongoDbDaoFactory,
|
|
26
|
+
MongoDbIndex,
|
|
26
27
|
get_single_hit,
|
|
27
28
|
replace_id_field_in_find_mapping,
|
|
28
29
|
validate_find_mapping,
|
|
@@ -41,6 +42,7 @@ __all__ = [
|
|
|
41
42
|
"MongoDbDao",
|
|
42
43
|
"MongoDbDaoFactory",
|
|
43
44
|
"MongoDbDtoKeyValueStore",
|
|
45
|
+
"MongoDbIndex",
|
|
44
46
|
"MongoDbJsonKeyValueStore",
|
|
45
47
|
"MongoDbStrKeyValueStore",
|
|
46
48
|
"document_to_dto",
|
|
@@ -22,7 +22,7 @@ Utilities for testing are located in `./testutils.py`.
|
|
|
22
22
|
from collections.abc import AsyncIterator, Callable, Collection, Mapping
|
|
23
23
|
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
|
24
24
|
from functools import partial
|
|
25
|
-
from typing import Any, Generic
|
|
25
|
+
from typing import Any, Generic, Literal, TypeAlias
|
|
26
26
|
|
|
27
27
|
from pymongo import AsyncMongoClient
|
|
28
28
|
from pymongo.asynchronous.collection import AsyncCollection
|
|
@@ -33,11 +33,13 @@ from hexkit.protocols.dao import (
|
|
|
33
33
|
Dao,
|
|
34
34
|
DaoFactoryProtocol,
|
|
35
35
|
Dto,
|
|
36
|
+
IndexBase,
|
|
36
37
|
InvalidFindMappingError,
|
|
37
38
|
MultipleHitsFoundError,
|
|
38
39
|
NoHitsFoundError,
|
|
39
40
|
ResourceAlreadyExistsError,
|
|
40
41
|
ResourceNotFoundError,
|
|
42
|
+
UniqueConstraintViolationError,
|
|
41
43
|
)
|
|
42
44
|
from hexkit.providers.mongodb.config import MongoDbConfig
|
|
43
45
|
from hexkit.providers.mongodb.provider.client import ConfiguredMongoClient
|
|
@@ -51,6 +53,7 @@ from hexkit.utils import FieldNotInModelError, validate_fields_in_model
|
|
|
51
53
|
__all__ = [
|
|
52
54
|
"MongoDbDao",
|
|
53
55
|
"MongoDbDaoFactory",
|
|
56
|
+
"MongoDbIndex",
|
|
54
57
|
"get_single_hit",
|
|
55
58
|
"replace_id_field_in_find_mapping",
|
|
56
59
|
"validate_find_mapping",
|
|
@@ -113,6 +116,67 @@ async def get_single_hit(
|
|
|
113
116
|
raise MultipleHitsFoundError(mapping=mapping)
|
|
114
117
|
|
|
115
118
|
|
|
119
|
+
FieldName: TypeAlias = str
|
|
120
|
+
SortOrder: TypeAlias = Literal[1] | Literal[-1]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class MongoDbIndex(IndexBase):
|
|
124
|
+
"""Information required to apply a single MongoDbIndex."""
|
|
125
|
+
|
|
126
|
+
fields: dict[FieldName, SortOrder]
|
|
127
|
+
properties: dict[str, Any] | None = None
|
|
128
|
+
|
|
129
|
+
def __init__(
|
|
130
|
+
self,
|
|
131
|
+
*,
|
|
132
|
+
fields: FieldName | dict[FieldName, SortOrder],
|
|
133
|
+
properties: dict[str, Any] | None = None,
|
|
134
|
+
):
|
|
135
|
+
"""Initialize the index.
|
|
136
|
+
|
|
137
|
+
**Args**
|
|
138
|
+
- `fields`: a single field name or a dict containing the field name and the sort
|
|
139
|
+
order. For sort order, 1 means ascending and -1 means descending. If the index
|
|
140
|
+
covers the model's `id_field`, then specify the actual field name instead of
|
|
141
|
+
"_id". When passing a field name instead of a dict, ascending sort order is used.
|
|
142
|
+
- `properties`: a dictionary where the keys are MongoDB index property names, and
|
|
143
|
+
the values are the value to pass for that property.
|
|
144
|
+
|
|
145
|
+
More information on index creation with Pymongo, including a list of the supported
|
|
146
|
+
index properties, can be found [here](https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.create_index)
|
|
147
|
+
|
|
148
|
+
Example:
|
|
149
|
+
```python
|
|
150
|
+
# This creates a compound unique index over field_a and field_b.
|
|
151
|
+
MongoDbIndex(
|
|
152
|
+
fields={"field_a": 1, "field_b": -1},
|
|
153
|
+
properties={"unique": True}
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
# This creates an ascending-sorted index over field_a.
|
|
157
|
+
MongoDbIndex("field_a")
|
|
158
|
+
```
|
|
159
|
+
"""
|
|
160
|
+
if isinstance(fields, str):
|
|
161
|
+
self.fields = {fields: 1}
|
|
162
|
+
else:
|
|
163
|
+
self.fields = fields
|
|
164
|
+
|
|
165
|
+
self.properties = properties
|
|
166
|
+
|
|
167
|
+
def list_field_names(self) -> list[str]:
|
|
168
|
+
"""Return a list of all the field names contained in this index"""
|
|
169
|
+
return list(self.fields)
|
|
170
|
+
|
|
171
|
+
def fields_with_id_replaced(
|
|
172
|
+
self, id_field: str
|
|
173
|
+
) -> str | list[tuple[FieldName, SortOrder]]:
|
|
174
|
+
"""Returns the `fields` property with all instances of `id_field`
|
|
175
|
+
replaced with `"_id"`.
|
|
176
|
+
"""
|
|
177
|
+
return [("_id" if f == id_field else f, so) for f, so in self.fields.items()]
|
|
178
|
+
|
|
179
|
+
|
|
116
180
|
class MongoDbDao(Generic[Dto]):
|
|
117
181
|
"""A MongoDb-specific DAO."""
|
|
118
182
|
|
|
@@ -295,6 +359,11 @@ class MongoDbDao(Generic[Dto]):
|
|
|
295
359
|
try:
|
|
296
360
|
await self._collection.insert_one(document)
|
|
297
361
|
except DuplicateKeyError as error:
|
|
362
|
+
key_value = error.details.get("keyValue") # type: ignore
|
|
363
|
+
if key_value is not None and list(key_value) != ["_id"]:
|
|
364
|
+
raise UniqueConstraintViolationError(
|
|
365
|
+
unique_fields=key_value
|
|
366
|
+
) from error
|
|
298
367
|
raise ResourceAlreadyExistsError(id_=document["_id"]) from error
|
|
299
368
|
|
|
300
369
|
async def upsert(self, dto: Dto) -> None:
|
|
@@ -312,7 +381,7 @@ class MongoDbDao(Generic[Dto]):
|
|
|
312
381
|
)
|
|
313
382
|
|
|
314
383
|
|
|
315
|
-
class MongoDbDaoFactory(DaoFactoryProtocol):
|
|
384
|
+
class MongoDbDaoFactory(DaoFactoryProtocol[MongoDbIndex]):
|
|
316
385
|
"""A MongoDB-based provider implementing the DaoFactoryProtocol."""
|
|
317
386
|
|
|
318
387
|
@classmethod
|
|
@@ -354,7 +423,7 @@ class MongoDbDaoFactory(DaoFactoryProtocol):
|
|
|
354
423
|
name: str,
|
|
355
424
|
dto_model: type[Dto],
|
|
356
425
|
id_field: str,
|
|
357
|
-
|
|
426
|
+
indexes: Collection[MongoDbIndex] | None = None,
|
|
358
427
|
) -> Dao[Dto]:
|
|
359
428
|
"""Constructs a DAO for interacting with resources in a MongoDB database.
|
|
360
429
|
|
|
@@ -366,13 +435,14 @@ class MongoDbDaoFactory(DaoFactoryProtocol):
|
|
|
366
435
|
from the database server. Thus for compliance with the DaoFactoryProtocol, this
|
|
367
436
|
method is async.
|
|
368
437
|
"""
|
|
369
|
-
if fields_to_index is not None:
|
|
370
|
-
raise NotImplementedError(
|
|
371
|
-
"Indexing on non-ID fields has not been implemented, yet."
|
|
372
|
-
)
|
|
373
|
-
|
|
374
438
|
collection = self._db[name]
|
|
375
439
|
|
|
440
|
+
for index in indexes or []:
|
|
441
|
+
properties = index.properties or {}
|
|
442
|
+
await collection.create_index(
|
|
443
|
+
index.fields_with_id_replaced(id_field=id_field), **properties
|
|
444
|
+
)
|
|
445
|
+
|
|
376
446
|
return MongoDbDao(
|
|
377
447
|
collection=collection,
|
|
378
448
|
dto_model=dto_model,
|
|
@@ -45,6 +45,7 @@ from hexkit.providers.mongodb.provider import (
|
|
|
45
45
|
translate_pymongo_errors,
|
|
46
46
|
validate_find_mapping,
|
|
47
47
|
)
|
|
48
|
+
from hexkit.providers.mongodb.provider.dao import MongoDbIndex
|
|
48
49
|
from hexkit.providers.mongokafka.config import MongoKafkaConfig
|
|
49
50
|
|
|
50
51
|
|
|
@@ -437,7 +438,7 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
|
|
|
437
438
|
await self.publish_document(document)
|
|
438
439
|
|
|
439
440
|
|
|
440
|
-
class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
|
|
441
|
+
class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol[MongoDbIndex]):
|
|
441
442
|
"""A provider implementing the DaoPublisherFactoryProtocol based on MongoDB and
|
|
442
443
|
Apache Kafka.
|
|
443
444
|
"""
|
|
@@ -491,7 +492,7 @@ class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
|
|
|
491
492
|
name: str,
|
|
492
493
|
dto_model: type[Dto],
|
|
493
494
|
id_field: str,
|
|
494
|
-
|
|
495
|
+
indexes: Collection[MongoDbIndex] | None,
|
|
495
496
|
dto_to_event: Callable[[Dto], JsonObject | None],
|
|
496
497
|
event_topic: str,
|
|
497
498
|
autopublish: bool,
|
|
@@ -502,13 +503,14 @@ class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
|
|
|
502
503
|
Please see the DaoPublisherFactoryProtocol superclass for documentation of
|
|
503
504
|
parameters.
|
|
504
505
|
"""
|
|
505
|
-
if fields_to_index is not None:
|
|
506
|
-
raise NotImplementedError(
|
|
507
|
-
"Indexing on non-ID fields has not been implemented, yet."
|
|
508
|
-
)
|
|
509
|
-
|
|
510
506
|
collection = self._db[name]
|
|
511
507
|
|
|
508
|
+
for index in indexes or []:
|
|
509
|
+
properties = index.properties or {}
|
|
510
|
+
await collection.create_index(
|
|
511
|
+
index.fields_with_id_replaced(id_field=id_field), **properties
|
|
512
|
+
)
|
|
513
|
+
|
|
512
514
|
dao = MongoDbDao(
|
|
513
515
|
collection=collection,
|
|
514
516
|
dto_model=dto_model,
|
|
@@ -401,15 +401,17 @@ class S3ObjectStorage(ObjectStorageProtocol):
|
|
|
401
401
|
url=presigned_url["url"], fields=presigned_url["fields"]
|
|
402
402
|
)
|
|
403
403
|
|
|
404
|
-
async def
|
|
404
|
+
async def _list_multipart_uploads_for_object(
|
|
405
405
|
self, *, bucket_id: str, object_id: str
|
|
406
406
|
) -> list[str]:
|
|
407
|
-
"""Lists all active multipart uploads for the given object.
|
|
408
|
-
their IDs.
|
|
407
|
+
"""Lists all active multipart uploads for the given object ID.
|
|
409
408
|
|
|
410
|
-
|
|
409
|
+
Raises a `BucketNotFoundError` if the bucket does not exist.
|
|
410
|
+
|
|
411
|
+
Boto3 Documentation:
|
|
412
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/list_multipart_uploads.html#S3.Client.list_multipart_uploads
|
|
411
413
|
"""
|
|
412
|
-
uploads = []
|
|
414
|
+
uploads: list[str] = []
|
|
413
415
|
try:
|
|
414
416
|
response_iter = await asyncio.to_thread(
|
|
415
417
|
self._client.get_paginator("list_multipart_uploads").paginate,
|
|
@@ -430,9 +432,38 @@ class S3ObjectStorage(ObjectStorageProtocol):
|
|
|
430
432
|
|
|
431
433
|
return uploads
|
|
432
434
|
|
|
435
|
+
async def _get_all_multipart_uploads(self, *, bucket_id: str) -> dict[str, str]:
|
|
436
|
+
"""Gets all active multipart uploads for the given bucket ID.
|
|
437
|
+
|
|
438
|
+
Returns a dict where the keys are upload IDs and values are object IDs.
|
|
439
|
+
S3 allows multiple ongoing multi-part uploads, so it's possible for some upload
|
|
440
|
+
IDs to map to the same object ID.
|
|
441
|
+
|
|
442
|
+
Raises a `BucketNotFoundError` if the bucket does not exist.
|
|
443
|
+
"""
|
|
444
|
+
uploads: dict[str, str] = {}
|
|
445
|
+
try:
|
|
446
|
+
response_iter = await asyncio.to_thread(
|
|
447
|
+
self._client.get_paginator("list_multipart_uploads").paginate,
|
|
448
|
+
Bucket=bucket_id,
|
|
449
|
+
)
|
|
450
|
+
for response_page in response_iter:
|
|
451
|
+
uploads.update(
|
|
452
|
+
{
|
|
453
|
+
upload["UploadId"]: upload["Key"]
|
|
454
|
+
for upload in response_page.get("Uploads", [])
|
|
455
|
+
}
|
|
456
|
+
)
|
|
457
|
+
except botocore.exceptions.ClientError as error:
|
|
458
|
+
raise self._translate_s3_client_errors(
|
|
459
|
+
error, bucket_id=bucket_id
|
|
460
|
+
) from error
|
|
461
|
+
|
|
462
|
+
return uploads
|
|
463
|
+
|
|
433
464
|
async def _assert_no_multipart_upload(self, *, bucket_id: str, object_id: str):
|
|
434
465
|
"""Ensure that there are no active multi-part uploads for the given object."""
|
|
435
|
-
upload_ids = await self.
|
|
466
|
+
upload_ids = await self._list_multipart_uploads_for_object(
|
|
436
467
|
bucket_id=bucket_id, object_id=object_id
|
|
437
468
|
)
|
|
438
469
|
if len(upload_ids) > 0:
|
|
@@ -454,14 +485,14 @@ class S3ObjectStorage(ObjectStorageProtocol):
|
|
|
454
485
|
By default, also verifies that this upload is the only upload active for
|
|
455
486
|
that file. Otherwise, raises MultipleActiveUploadsError.
|
|
456
487
|
"""
|
|
457
|
-
upload_ids = await self.
|
|
488
|
+
upload_ids = await self._list_multipart_uploads_for_object(
|
|
458
489
|
bucket_id=bucket_id, object_id=object_id
|
|
459
490
|
)
|
|
460
491
|
n_uploads = len(upload_ids)
|
|
461
492
|
|
|
462
493
|
if assert_exclusiveness and n_uploads > 1:
|
|
463
494
|
raise self.MultipleActiveUploadsError(
|
|
464
|
-
bucket_id=bucket_id, object_id=object_id, upload_ids=upload_ids
|
|
495
|
+
bucket_id=bucket_id, object_id=object_id, upload_ids=list(upload_ids)
|
|
465
496
|
)
|
|
466
497
|
|
|
467
498
|
if upload_id not in upload_ids:
|
|
@@ -877,12 +908,12 @@ class S3ObjectStorage(ObjectStorageProtocol):
|
|
|
877
908
|
# There should only be one ongoing multipart upload/copy at at a time as long
|
|
878
909
|
# as a new object ID is generated for each attempt
|
|
879
910
|
try:
|
|
880
|
-
upload_ids = await self.
|
|
911
|
+
upload_ids = await self._list_multipart_uploads_for_object(
|
|
881
912
|
bucket_id=dest_bucket_id, object_id=dest_object_id
|
|
882
913
|
)
|
|
883
914
|
if len(upload_ids) == 1:
|
|
884
915
|
await self._abort_multipart_upload(
|
|
885
|
-
upload_id=upload_ids
|
|
916
|
+
upload_id=next(iter(upload_ids)),
|
|
886
917
|
bucket_id=dest_bucket_id,
|
|
887
918
|
object_id=dest_object_id,
|
|
888
919
|
)
|
|
@@ -168,6 +168,32 @@ class InMemObjectStorage(ObjectStorageProtocol):
|
|
|
168
168
|
uploads_for_object = self.uploads[bucket_id].get(object_id, set())
|
|
169
169
|
return sorted(uploads_for_object)
|
|
170
170
|
|
|
171
|
+
async def _list_multipart_uploads_for_object(
|
|
172
|
+
self, *, bucket_id: str, object_id: str
|
|
173
|
+
) -> list[str]:
|
|
174
|
+
"""Lists all active multipart uploads for the given object ID.
|
|
175
|
+
|
|
176
|
+
Raises a `BucketNotFoundError` if the bucket does not exist.
|
|
177
|
+
"""
|
|
178
|
+
await self._assert_bucket_exists(bucket_id)
|
|
179
|
+
return list(self.uploads[bucket_id].get(object_id, set()))
|
|
180
|
+
|
|
181
|
+
async def _get_all_multipart_uploads(self, *, bucket_id: str) -> dict[str, str]:
|
|
182
|
+
"""Gets all active multipart uploads for the given bucket ID.
|
|
183
|
+
|
|
184
|
+
Returns a dict where the keys are upload IDs and values are object IDs.
|
|
185
|
+
S3 allows multiple ongoing multi-part uploads, so it's possible for some upload
|
|
186
|
+
IDs to map to the same object ID.
|
|
187
|
+
|
|
188
|
+
Raises a `BucketNotFoundError` if the bucket does not exist.
|
|
189
|
+
"""
|
|
190
|
+
await self._assert_bucket_exists(bucket_id)
|
|
191
|
+
return {
|
|
192
|
+
upload_id: object_id
|
|
193
|
+
for object_id, uploads in self.uploads[bucket_id].items()
|
|
194
|
+
for upload_id in uploads
|
|
195
|
+
}
|
|
196
|
+
|
|
171
197
|
async def _assert_no_multipart_upload(self, *, bucket_id: str, object_id: str):
|
|
172
198
|
"""Ensure that there are no active multi-part uploads for the given object.
|
|
173
199
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|