boto3-assist 0.35.0__py3-none-any.whl → 0.36.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- boto3_assist/__init__.py +10 -0
- boto3_assist/connection.py +85 -6
- boto3_assist/connection_pool.py +179 -0
- boto3_assist/dynamodb/__init__.py +9 -0
- boto3_assist/dynamodb/dynamodb.py +215 -168
- boto3_assist/dynamodb/dynamodb_connection.py +2 -0
- boto3_assist/s3/__init__.py +9 -0
- boto3_assist/s3/s3.py +43 -0
- boto3_assist/s3/s3_connection.py +2 -0
- boto3_assist/sqs/sqs_connection.py +43 -0
- boto3_assist/version.py +1 -1
- {boto3_assist-0.35.0.dist-info → boto3_assist-0.36.0.dist-info}/METADATA +1 -1
- {boto3_assist-0.35.0.dist-info → boto3_assist-0.36.0.dist-info}/RECORD +16 -13
- {boto3_assist-0.35.0.dist-info → boto3_assist-0.36.0.dist-info}/WHEEL +0 -0
- {boto3_assist-0.35.0.dist-info → boto3_assist-0.36.0.dist-info}/licenses/LICENSE-EXPLAINED.txt +0 -0
- {boto3_assist-0.35.0.dist-info → boto3_assist-0.36.0.dist-info}/licenses/LICENSE.txt +0 -0
|
@@ -46,6 +46,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
46
46
|
assume_role_arn: Optional[str] = None,
|
|
47
47
|
assume_role_chain: Optional[List[str]] = None,
|
|
48
48
|
assume_role_duration_seconds: Optional[int] = 3600,
|
|
49
|
+
use_connection_pool: bool = False,
|
|
49
50
|
) -> None:
|
|
50
51
|
super().__init__(
|
|
51
52
|
aws_profile=aws_profile,
|
|
@@ -55,6 +56,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
55
56
|
assume_role_arn=assume_role_arn,
|
|
56
57
|
assume_role_chain=assume_role_chain,
|
|
57
58
|
assume_role_duration_seconds=assume_role_duration_seconds,
|
|
59
|
+
use_connection_pool=use_connection_pool,
|
|
58
60
|
)
|
|
59
61
|
self.helpers: DynamoDBHelpers = DynamoDBHelpers()
|
|
60
62
|
self.log_dynamodb_item_size: bool = bool(
|
|
@@ -65,19 +67,59 @@ class DynamoDB(DynamoDBConnection):
|
|
|
65
67
|
)
|
|
66
68
|
logger.setLevel(os.getenv("LOG_LEVEL", "INFO"))
|
|
67
69
|
|
|
70
|
+
@classmethod
|
|
71
|
+
def from_pool(
|
|
72
|
+
cls,
|
|
73
|
+
aws_profile: Optional[str] = None,
|
|
74
|
+
aws_region: Optional[str] = None,
|
|
75
|
+
aws_end_point_url: Optional[str] = None,
|
|
76
|
+
**kwargs,
|
|
77
|
+
) -> "DynamoDB":
|
|
78
|
+
"""
|
|
79
|
+
Create DynamoDB connection using connection pool (recommended for Lambda).
|
|
80
|
+
|
|
81
|
+
This is the recommended pattern for Lambda functions as it reuses
|
|
82
|
+
boto3 sessions across invocations in warm containers.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
aws_profile: AWS profile name (optional)
|
|
86
|
+
aws_region: AWS region (optional)
|
|
87
|
+
aws_end_point_url: Custom endpoint URL (optional, for moto testing)
|
|
88
|
+
**kwargs: Additional DynamoDB parameters
|
|
89
|
+
|
|
90
|
+
Returns:
|
|
91
|
+
DynamoDB instance configured to use connection pool
|
|
92
|
+
|
|
93
|
+
Example:
|
|
94
|
+
>>> # Recommended pattern for Lambda
|
|
95
|
+
>>> db = DynamoDB.from_pool()
|
|
96
|
+
>>> result = db.get(table_name="my-table", key={"id": "123"})
|
|
97
|
+
>>>
|
|
98
|
+
>>> # Subsequent calls reuse the same connection
|
|
99
|
+
>>> db2 = DynamoDB.from_pool()
|
|
100
|
+
>>> assert db.session is db2.session
|
|
101
|
+
"""
|
|
102
|
+
return cls(
|
|
103
|
+
aws_profile=aws_profile,
|
|
104
|
+
aws_region=aws_region,
|
|
105
|
+
aws_end_point_url=aws_end_point_url,
|
|
106
|
+
use_connection_pool=True,
|
|
107
|
+
**kwargs,
|
|
108
|
+
)
|
|
109
|
+
|
|
68
110
|
def _apply_decimal_conversion(self, response: Dict[str, Any]) -> Dict[str, Any]:
|
|
69
111
|
"""
|
|
70
112
|
Apply decimal conversion to DynamoDB response if enabled.
|
|
71
|
-
|
|
113
|
+
|
|
72
114
|
Args:
|
|
73
115
|
response: The DynamoDB response dictionary
|
|
74
|
-
|
|
116
|
+
|
|
75
117
|
Returns:
|
|
76
118
|
The response with decimal conversion applied if enabled
|
|
77
119
|
"""
|
|
78
120
|
if not self.convert_decimals:
|
|
79
121
|
return response
|
|
80
|
-
|
|
122
|
+
|
|
81
123
|
return DecimalConversionUtility.convert_decimals_to_native_types(response)
|
|
82
124
|
|
|
83
125
|
def save(
|
|
@@ -92,7 +134,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
92
134
|
) -> dict:
|
|
93
135
|
"""
|
|
94
136
|
Save an item to the database with optional conditional expressions.
|
|
95
|
-
|
|
137
|
+
|
|
96
138
|
Args:
|
|
97
139
|
item (dict): DynamoDB Dictionary Object or DynamoDBModelBase.
|
|
98
140
|
Supports the "client" or "resource" syntax
|
|
@@ -116,14 +158,14 @@ class DynamoDB(DynamoDBConnection):
|
|
|
116
158
|
Returns:
|
|
117
159
|
dict: The Response from DynamoDB's put_item actions.
|
|
118
160
|
It does not return the saved object, only the response.
|
|
119
|
-
|
|
161
|
+
|
|
120
162
|
Examples:
|
|
121
163
|
>>> # Simple save
|
|
122
164
|
>>> db.save(item=user, table_name="users")
|
|
123
|
-
|
|
165
|
+
|
|
124
166
|
>>> # Prevent duplicates
|
|
125
167
|
>>> db.save(item=user, table_name="users", fail_if_exists=True)
|
|
126
|
-
|
|
168
|
+
|
|
127
169
|
>>> # Optimistic locking with version check
|
|
128
170
|
>>> db.save(
|
|
129
171
|
... item=user,
|
|
@@ -155,7 +197,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
155
197
|
|
|
156
198
|
if isinstance(item, dict):
|
|
157
199
|
self.__log_item_size(item=item)
|
|
158
|
-
|
|
200
|
+
|
|
159
201
|
# Convert native numeric types to Decimal for DynamoDB
|
|
160
202
|
# (DynamoDB doesn't accept float, requires Decimal)
|
|
161
203
|
item = DecimalConversionUtility.convert_native_types_to_decimals(item)
|
|
@@ -167,7 +209,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
167
209
|
"TableName": table_name,
|
|
168
210
|
"Item": item,
|
|
169
211
|
}
|
|
170
|
-
|
|
212
|
+
|
|
171
213
|
# Handle conditional expressions
|
|
172
214
|
if condition_expression:
|
|
173
215
|
# Custom condition provided
|
|
@@ -175,42 +217,48 @@ class DynamoDB(DynamoDBConnection):
|
|
|
175
217
|
if expression_attribute_names:
|
|
176
218
|
params["ExpressionAttributeNames"] = expression_attribute_names
|
|
177
219
|
if expression_attribute_values:
|
|
178
|
-
params["ExpressionAttributeValues"] =
|
|
220
|
+
params["ExpressionAttributeValues"] = (
|
|
221
|
+
expression_attribute_values
|
|
222
|
+
)
|
|
179
223
|
elif fail_if_exists:
|
|
180
224
|
# only insert if the item does *not* already exist
|
|
181
225
|
params["ConditionExpression"] = (
|
|
182
226
|
"attribute_not_exists(#pk) AND attribute_not_exists(#sk)"
|
|
183
227
|
)
|
|
184
228
|
params["ExpressionAttributeNames"] = {"#pk": "pk", "#sk": "sk"}
|
|
185
|
-
|
|
229
|
+
|
|
186
230
|
response = dict(self.dynamodb_client.put_item(**params))
|
|
187
231
|
|
|
188
232
|
else:
|
|
189
233
|
# Use boto3.resource syntax
|
|
190
234
|
table = self.dynamodb_resource.Table(table_name)
|
|
191
|
-
|
|
235
|
+
|
|
192
236
|
# Build put_item parameters
|
|
193
237
|
put_params = {"Item": item}
|
|
194
|
-
|
|
238
|
+
|
|
195
239
|
# Handle conditional expressions
|
|
196
240
|
if condition_expression:
|
|
197
241
|
# Custom condition provided
|
|
198
242
|
# Convert string condition to boto3 condition object if needed
|
|
199
243
|
put_params["ConditionExpression"] = condition_expression
|
|
200
244
|
if expression_attribute_names:
|
|
201
|
-
put_params["ExpressionAttributeNames"] =
|
|
245
|
+
put_params["ExpressionAttributeNames"] = (
|
|
246
|
+
expression_attribute_names
|
|
247
|
+
)
|
|
202
248
|
if expression_attribute_values:
|
|
203
|
-
put_params["ExpressionAttributeValues"] =
|
|
249
|
+
put_params["ExpressionAttributeValues"] = (
|
|
250
|
+
expression_attribute_values
|
|
251
|
+
)
|
|
204
252
|
elif fail_if_exists:
|
|
205
253
|
put_params["ConditionExpression"] = (
|
|
206
254
|
Attr("pk").not_exists() & Attr("sk").not_exists()
|
|
207
255
|
)
|
|
208
|
-
|
|
256
|
+
|
|
209
257
|
response = dict(table.put_item(**put_params))
|
|
210
258
|
|
|
211
259
|
except ClientError as e:
|
|
212
260
|
error_code = e.response["Error"]["Code"]
|
|
213
|
-
|
|
261
|
+
|
|
214
262
|
if error_code == "ConditionalCheckFailedException":
|
|
215
263
|
# Enhanced error message for conditional check failures
|
|
216
264
|
if fail_if_exists:
|
|
@@ -379,10 +427,10 @@ class DynamoDB(DynamoDBConnection):
|
|
|
379
427
|
) -> dict:
|
|
380
428
|
"""
|
|
381
429
|
Update an item in DynamoDB with an update expression.
|
|
382
|
-
|
|
430
|
+
|
|
383
431
|
Update expressions allow you to modify specific attributes without replacing
|
|
384
432
|
the entire item. Supports SET, ADD, REMOVE, and DELETE operations.
|
|
385
|
-
|
|
433
|
+
|
|
386
434
|
Args:
|
|
387
435
|
table_name: The DynamoDB table name
|
|
388
436
|
key: Primary key dict, e.g., {"pk": "user#123", "sk": "user#123"}
|
|
@@ -396,14 +444,14 @@ class DynamoDB(DynamoDBConnection):
|
|
|
396
444
|
- "UPDATED_OLD": Only updated attributes before update
|
|
397
445
|
- "ALL_NEW": All attributes after update
|
|
398
446
|
- "UPDATED_NEW": Only updated attributes after update
|
|
399
|
-
|
|
447
|
+
|
|
400
448
|
Returns:
|
|
401
449
|
dict: DynamoDB response with optional Attributes based on return_values
|
|
402
|
-
|
|
450
|
+
|
|
403
451
|
Raises:
|
|
404
452
|
RuntimeError: If condition expression fails
|
|
405
453
|
ClientError: For other DynamoDB errors
|
|
406
|
-
|
|
454
|
+
|
|
407
455
|
Examples:
|
|
408
456
|
>>> # Simple SET operation
|
|
409
457
|
>>> db.update_item(
|
|
@@ -412,7 +460,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
412
460
|
... update_expression="SET email = :email",
|
|
413
461
|
... expression_attribute_values={":email": "new@example.com"}
|
|
414
462
|
... )
|
|
415
|
-
|
|
463
|
+
|
|
416
464
|
>>> # Atomic counter
|
|
417
465
|
>>> db.update_item(
|
|
418
466
|
... table_name="users",
|
|
@@ -420,7 +468,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
420
468
|
... update_expression="ADD view_count :inc",
|
|
421
469
|
... expression_attribute_values={":inc": 1}
|
|
422
470
|
... )
|
|
423
|
-
|
|
471
|
+
|
|
424
472
|
>>> # Multiple operations with reserved word
|
|
425
473
|
>>> db.update_item(
|
|
426
474
|
... table_name="users",
|
|
@@ -429,7 +477,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
429
477
|
... expression_attribute_names={"#status": "status"},
|
|
430
478
|
... expression_attribute_values={":status": "active", ":now": "2024-10-15"}
|
|
431
479
|
... )
|
|
432
|
-
|
|
480
|
+
|
|
433
481
|
>>> # Conditional update with return value
|
|
434
482
|
>>> response = db.update_item(
|
|
435
483
|
... table_name="users",
|
|
@@ -442,38 +490,38 @@ class DynamoDB(DynamoDBConnection):
|
|
|
442
490
|
>>> updated_user = response['Attributes']
|
|
443
491
|
"""
|
|
444
492
|
table = self.dynamodb_resource.Table(table_name)
|
|
445
|
-
|
|
493
|
+
|
|
446
494
|
# Build update parameters
|
|
447
495
|
params = {
|
|
448
496
|
"Key": key,
|
|
449
497
|
"UpdateExpression": update_expression,
|
|
450
|
-
"ReturnValues": return_values
|
|
498
|
+
"ReturnValues": return_values,
|
|
451
499
|
}
|
|
452
|
-
|
|
500
|
+
|
|
453
501
|
if expression_attribute_values:
|
|
454
502
|
params["ExpressionAttributeValues"] = expression_attribute_values
|
|
455
|
-
|
|
503
|
+
|
|
456
504
|
if expression_attribute_names:
|
|
457
505
|
params["ExpressionAttributeNames"] = expression_attribute_names
|
|
458
|
-
|
|
506
|
+
|
|
459
507
|
if condition_expression:
|
|
460
508
|
params["ConditionExpression"] = condition_expression
|
|
461
|
-
|
|
509
|
+
|
|
462
510
|
try:
|
|
463
511
|
response = dict(table.update_item(**params))
|
|
464
|
-
|
|
512
|
+
|
|
465
513
|
# Apply decimal conversion if response contains attributes
|
|
466
514
|
return self._apply_decimal_conversion(response)
|
|
467
|
-
|
|
515
|
+
|
|
468
516
|
except ClientError as e:
|
|
469
517
|
error_code = e.response["Error"]["Code"]
|
|
470
|
-
|
|
518
|
+
|
|
471
519
|
if error_code == "ConditionalCheckFailedException":
|
|
472
520
|
raise RuntimeError(
|
|
473
521
|
f"Conditional check failed for update in {table_name}. "
|
|
474
522
|
f"Condition: {condition_expression}"
|
|
475
523
|
) from e
|
|
476
|
-
|
|
524
|
+
|
|
477
525
|
logger.exception(f"Error in update_item: {str(e)}")
|
|
478
526
|
raise
|
|
479
527
|
|
|
@@ -702,11 +750,11 @@ class DynamoDB(DynamoDBConnection):
|
|
|
702
750
|
) -> dict:
|
|
703
751
|
"""
|
|
704
752
|
Retrieve multiple items from DynamoDB in a single request.
|
|
705
|
-
|
|
753
|
+
|
|
706
754
|
DynamoDB allows up to 100 items per batch_get_item call. This method
|
|
707
755
|
automatically chunks larger requests and handles unprocessed keys with
|
|
708
756
|
exponential backoff retry logic.
|
|
709
|
-
|
|
757
|
+
|
|
710
758
|
Args:
|
|
711
759
|
keys: List of key dictionaries. Each dict must contain the primary key
|
|
712
760
|
(and sort key if applicable) for the items to retrieve.
|
|
@@ -715,13 +763,13 @@ class DynamoDB(DynamoDBConnection):
|
|
|
715
763
|
projection_expression: Optional comma-separated list of attributes to retrieve
|
|
716
764
|
expression_attribute_names: Optional dict mapping attribute name placeholders to actual names
|
|
717
765
|
consistent_read: If True, uses strongly consistent reads (costs more RCUs)
|
|
718
|
-
|
|
766
|
+
|
|
719
767
|
Returns:
|
|
720
768
|
dict: Response containing:
|
|
721
769
|
- 'Items': List of retrieved items (with Decimal conversion applied)
|
|
722
770
|
- 'UnprocessedKeys': Any keys that couldn't be processed after retries
|
|
723
771
|
- 'ConsumedCapacity': Capacity units consumed (if available)
|
|
724
|
-
|
|
772
|
+
|
|
725
773
|
Example:
|
|
726
774
|
>>> keys = [
|
|
727
775
|
... {"pk": "user#user-001", "sk": "user#user-001"},
|
|
@@ -731,7 +779,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
731
779
|
>>> response = db.batch_get_item(keys=keys, table_name="users")
|
|
732
780
|
>>> items = response['Items']
|
|
733
781
|
>>> print(f"Retrieved {len(items)} items")
|
|
734
|
-
|
|
782
|
+
|
|
735
783
|
Note:
|
|
736
784
|
- Maximum 100 items per request (automatically chunked)
|
|
737
785
|
- Each item can be up to 400 KB
|
|
@@ -739,52 +787,53 @@ class DynamoDB(DynamoDBConnection):
|
|
|
739
787
|
- Unprocessed keys are automatically retried with exponential backoff
|
|
740
788
|
"""
|
|
741
789
|
import time
|
|
742
|
-
|
|
790
|
+
|
|
743
791
|
all_items = []
|
|
744
792
|
unprocessed_keys = []
|
|
745
|
-
|
|
793
|
+
|
|
746
794
|
# DynamoDB limit: 100 items per batch_get_item call
|
|
747
795
|
BATCH_SIZE = 100
|
|
748
|
-
|
|
796
|
+
|
|
749
797
|
# Chunk keys into batches of 100
|
|
750
798
|
for i in range(0, len(keys), BATCH_SIZE):
|
|
751
|
-
batch_keys = keys[i:i + BATCH_SIZE]
|
|
752
|
-
|
|
799
|
+
batch_keys = keys[i : i + BATCH_SIZE]
|
|
800
|
+
|
|
753
801
|
# Build request parameters
|
|
754
802
|
request_items = {
|
|
755
|
-
table_name: {
|
|
756
|
-
'Keys': batch_keys,
|
|
757
|
-
'ConsistentRead': consistent_read
|
|
758
|
-
}
|
|
803
|
+
table_name: {"Keys": batch_keys, "ConsistentRead": consistent_read}
|
|
759
804
|
}
|
|
760
|
-
|
|
805
|
+
|
|
761
806
|
# Add projection if provided
|
|
762
807
|
if projection_expression:
|
|
763
|
-
request_items[table_name][
|
|
808
|
+
request_items[table_name][
|
|
809
|
+
"ProjectionExpression"
|
|
810
|
+
] = projection_expression
|
|
764
811
|
if expression_attribute_names:
|
|
765
|
-
request_items[table_name][
|
|
766
|
-
|
|
812
|
+
request_items[table_name][
|
|
813
|
+
"ExpressionAttributeNames"
|
|
814
|
+
] = expression_attribute_names
|
|
815
|
+
|
|
767
816
|
# Retry logic for unprocessed keys
|
|
768
817
|
max_retries = 5
|
|
769
818
|
retry_count = 0
|
|
770
819
|
backoff_time = 0.1 # Start with 100ms
|
|
771
|
-
|
|
820
|
+
|
|
772
821
|
while retry_count <= max_retries:
|
|
773
822
|
try:
|
|
774
823
|
response = self.dynamodb_resource.meta.client.batch_get_item(
|
|
775
824
|
RequestItems=request_items
|
|
776
825
|
)
|
|
777
|
-
|
|
826
|
+
|
|
778
827
|
# Collect items from this batch
|
|
779
|
-
if
|
|
780
|
-
batch_items = response[
|
|
828
|
+
if "Responses" in response and table_name in response["Responses"]:
|
|
829
|
+
batch_items = response["Responses"][table_name]
|
|
781
830
|
all_items.extend(batch_items)
|
|
782
|
-
|
|
831
|
+
|
|
783
832
|
# Check for unprocessed keys
|
|
784
|
-
if
|
|
785
|
-
if table_name in response[
|
|
786
|
-
unprocessed = response[
|
|
787
|
-
|
|
833
|
+
if "UnprocessedKeys" in response and response["UnprocessedKeys"]:
|
|
834
|
+
if table_name in response["UnprocessedKeys"]:
|
|
835
|
+
unprocessed = response["UnprocessedKeys"][table_name]
|
|
836
|
+
|
|
788
837
|
if retry_count < max_retries:
|
|
789
838
|
# Retry with exponential backoff
|
|
790
839
|
logger.warning(
|
|
@@ -801,15 +850,18 @@ class DynamoDB(DynamoDBConnection):
|
|
|
801
850
|
logger.error(
|
|
802
851
|
f"Max retries reached. {len(unprocessed['Keys'])} keys remain unprocessed"
|
|
803
852
|
)
|
|
804
|
-
unprocessed_keys.extend(unprocessed[
|
|
853
|
+
unprocessed_keys.extend(unprocessed["Keys"])
|
|
805
854
|
break
|
|
806
855
|
else:
|
|
807
856
|
# No unprocessed keys, we're done with this batch
|
|
808
857
|
break
|
|
809
|
-
|
|
858
|
+
|
|
810
859
|
except ClientError as e:
|
|
811
|
-
error_code = e.response[
|
|
812
|
-
if
|
|
860
|
+
error_code = e.response["Error"]["Code"]
|
|
861
|
+
if (
|
|
862
|
+
error_code == "ProvisionedThroughputExceededException"
|
|
863
|
+
and retry_count < max_retries
|
|
864
|
+
):
|
|
813
865
|
logger.warning(
|
|
814
866
|
f"Throughput exceeded. Retrying in {backoff_time}s (attempt {retry_count + 1}/{max_retries})"
|
|
815
867
|
)
|
|
@@ -820,43 +872,39 @@ class DynamoDB(DynamoDBConnection):
|
|
|
820
872
|
else:
|
|
821
873
|
logger.exception(f"Error in batch_get_item: {str(e)}")
|
|
822
874
|
raise
|
|
823
|
-
|
|
875
|
+
|
|
824
876
|
# Apply decimal conversion to all items
|
|
825
877
|
result = {
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
878
|
+
"Items": all_items,
|
|
879
|
+
"Count": len(all_items),
|
|
880
|
+
"UnprocessedKeys": unprocessed_keys,
|
|
829
881
|
}
|
|
830
|
-
|
|
882
|
+
|
|
831
883
|
return self._apply_decimal_conversion(result)
|
|
832
|
-
|
|
884
|
+
|
|
833
885
|
def batch_write_item(
|
|
834
|
-
self,
|
|
835
|
-
items: list[dict],
|
|
836
|
-
table_name: str,
|
|
837
|
-
*,
|
|
838
|
-
operation: str = "put"
|
|
886
|
+
self, items: list[dict], table_name: str, *, operation: str = "put"
|
|
839
887
|
) -> dict:
|
|
840
888
|
"""
|
|
841
889
|
Write or delete multiple items in a single request.
|
|
842
|
-
|
|
890
|
+
|
|
843
891
|
DynamoDB allows up to 25 write operations per batch_write_item call.
|
|
844
892
|
This method automatically chunks larger requests and handles unprocessed
|
|
845
893
|
items with exponential backoff retry logic.
|
|
846
|
-
|
|
894
|
+
|
|
847
895
|
Args:
|
|
848
896
|
items: List of items to write or delete
|
|
849
897
|
- For 'put': Full item dictionaries
|
|
850
898
|
- For 'delete': Key-only dictionaries (pk, sk)
|
|
851
899
|
table_name: The DynamoDB table name
|
|
852
900
|
operation: Either 'put' (default) or 'delete'
|
|
853
|
-
|
|
901
|
+
|
|
854
902
|
Returns:
|
|
855
903
|
dict: Response containing:
|
|
856
904
|
- 'UnprocessedItems': Items that couldn't be processed after retries
|
|
857
905
|
- 'ProcessedCount': Number of successfully processed items
|
|
858
906
|
- 'UnprocessedCount': Number of unprocessed items
|
|
859
|
-
|
|
907
|
+
|
|
860
908
|
Example (Put):
|
|
861
909
|
>>> items = [
|
|
862
910
|
... {"pk": "user#1", "sk": "user#1", "name": "Alice"},
|
|
@@ -865,7 +913,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
865
913
|
... ]
|
|
866
914
|
>>> response = db.batch_write_item(items=items, table_name="users")
|
|
867
915
|
>>> print(f"Processed {response['ProcessedCount']} items")
|
|
868
|
-
|
|
916
|
+
|
|
869
917
|
Example (Delete):
|
|
870
918
|
>>> keys = [
|
|
871
919
|
... {"pk": "user#1", "sk": "user#1"},
|
|
@@ -876,7 +924,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
876
924
|
... table_name="users",
|
|
877
925
|
... operation="delete"
|
|
878
926
|
... )
|
|
879
|
-
|
|
927
|
+
|
|
880
928
|
Note:
|
|
881
929
|
- Maximum 25 operations per request (automatically chunked)
|
|
882
930
|
- Each item can be up to 400 KB
|
|
@@ -885,51 +933,53 @@ class DynamoDB(DynamoDBConnection):
|
|
|
885
933
|
- Unprocessed items are automatically retried with exponential backoff
|
|
886
934
|
"""
|
|
887
935
|
import time
|
|
888
|
-
|
|
889
|
-
if operation not in [
|
|
890
|
-
raise ValueError(
|
|
891
|
-
|
|
936
|
+
|
|
937
|
+
if operation not in ["put", "delete"]:
|
|
938
|
+
raise ValueError(
|
|
939
|
+
f"Invalid operation '{operation}'. Must be 'put' or 'delete'"
|
|
940
|
+
)
|
|
941
|
+
|
|
892
942
|
# DynamoDB limit: 25 operations per batch_write_item call
|
|
893
943
|
BATCH_SIZE = 25
|
|
894
|
-
|
|
944
|
+
|
|
895
945
|
total_processed = 0
|
|
896
946
|
all_unprocessed = []
|
|
897
|
-
|
|
947
|
+
|
|
898
948
|
# Chunk items into batches of 25
|
|
899
949
|
for i in range(0, len(items), BATCH_SIZE):
|
|
900
|
-
batch_items = items[i:i + BATCH_SIZE]
|
|
901
|
-
|
|
950
|
+
batch_items = items[i : i + BATCH_SIZE]
|
|
951
|
+
|
|
902
952
|
# Build request items
|
|
903
953
|
write_requests = []
|
|
904
954
|
for item in batch_items:
|
|
905
|
-
if operation ==
|
|
906
|
-
write_requests.append({
|
|
955
|
+
if operation == "put":
|
|
956
|
+
write_requests.append({"PutRequest": {"Item": item}})
|
|
907
957
|
else: # delete
|
|
908
|
-
write_requests.append({
|
|
909
|
-
|
|
958
|
+
write_requests.append({"DeleteRequest": {"Key": item}})
|
|
959
|
+
|
|
910
960
|
request_items = {table_name: write_requests}
|
|
911
|
-
|
|
961
|
+
|
|
912
962
|
# Retry logic for unprocessed items
|
|
913
963
|
max_retries = 5
|
|
914
964
|
retry_count = 0
|
|
915
965
|
backoff_time = 0.1 # Start with 100ms
|
|
916
|
-
|
|
966
|
+
|
|
917
967
|
while retry_count <= max_retries:
|
|
918
968
|
try:
|
|
919
969
|
response = self.dynamodb_resource.meta.client.batch_write_item(
|
|
920
970
|
RequestItems=request_items
|
|
921
971
|
)
|
|
922
|
-
|
|
972
|
+
|
|
923
973
|
# Count processed items from this batch
|
|
924
974
|
processed_in_batch = len(batch_items)
|
|
925
|
-
|
|
975
|
+
|
|
926
976
|
# Check for unprocessed items
|
|
927
|
-
if
|
|
928
|
-
if table_name in response[
|
|
929
|
-
unprocessed = response[
|
|
977
|
+
if "UnprocessedItems" in response and response["UnprocessedItems"]:
|
|
978
|
+
if table_name in response["UnprocessedItems"]:
|
|
979
|
+
unprocessed = response["UnprocessedItems"][table_name]
|
|
930
980
|
unprocessed_count = len(unprocessed)
|
|
931
981
|
processed_in_batch -= unprocessed_count
|
|
932
|
-
|
|
982
|
+
|
|
933
983
|
if retry_count < max_retries:
|
|
934
984
|
# Retry with exponential backoff
|
|
935
985
|
logger.warning(
|
|
@@ -948,14 +998,17 @@ class DynamoDB(DynamoDBConnection):
|
|
|
948
998
|
)
|
|
949
999
|
all_unprocessed.extend(unprocessed)
|
|
950
1000
|
break
|
|
951
|
-
|
|
1001
|
+
|
|
952
1002
|
# Successfully processed this batch
|
|
953
1003
|
total_processed += processed_in_batch
|
|
954
1004
|
break
|
|
955
|
-
|
|
1005
|
+
|
|
956
1006
|
except ClientError as e:
|
|
957
|
-
error_code = e.response[
|
|
958
|
-
if
|
|
1007
|
+
error_code = e.response["Error"]["Code"]
|
|
1008
|
+
if (
|
|
1009
|
+
error_code == "ProvisionedThroughputExceededException"
|
|
1010
|
+
and retry_count < max_retries
|
|
1011
|
+
):
|
|
959
1012
|
logger.warning(
|
|
960
1013
|
f"Throughput exceeded. Retrying in {backoff_time}s (attempt {retry_count + 1}/{max_retries})"
|
|
961
1014
|
)
|
|
@@ -966,28 +1019,28 @@ class DynamoDB(DynamoDBConnection):
|
|
|
966
1019
|
else:
|
|
967
1020
|
logger.exception(f"Error in batch_write_item: {str(e)}")
|
|
968
1021
|
raise
|
|
969
|
-
|
|
1022
|
+
|
|
970
1023
|
return {
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
1024
|
+
"ProcessedCount": total_processed,
|
|
1025
|
+
"UnprocessedCount": len(all_unprocessed),
|
|
1026
|
+
"UnprocessedItems": all_unprocessed,
|
|
974
1027
|
}
|
|
975
|
-
|
|
1028
|
+
|
|
976
1029
|
def transact_write_items(
|
|
977
1030
|
self,
|
|
978
1031
|
operations: list[dict],
|
|
979
1032
|
*,
|
|
980
1033
|
client_request_token: Optional[str] = None,
|
|
981
1034
|
return_consumed_capacity: str = "NONE",
|
|
982
|
-
return_item_collection_metrics: str = "NONE"
|
|
1035
|
+
return_item_collection_metrics: str = "NONE",
|
|
983
1036
|
) -> dict:
|
|
984
1037
|
"""
|
|
985
1038
|
Execute multiple write operations as an atomic transaction.
|
|
986
|
-
|
|
1039
|
+
|
|
987
1040
|
All operations succeed or all fail together. This is critical for
|
|
988
1041
|
maintaining data consistency across multiple items. Supports up to
|
|
989
1042
|
100 operations per transaction (increased from 25 in 2023).
|
|
990
|
-
|
|
1043
|
+
|
|
991
1044
|
Args:
|
|
992
1045
|
operations: List of transaction operation dictionaries. Each dict must
|
|
993
1046
|
have one of: 'Put', 'Update', 'Delete', or 'ConditionCheck'
|
|
@@ -1011,19 +1064,19 @@ class DynamoDB(DynamoDBConnection):
|
|
|
1011
1064
|
client_request_token: Optional idempotency token for retry safety
|
|
1012
1065
|
return_consumed_capacity: 'INDEXES', 'TOTAL', or 'NONE' (default)
|
|
1013
1066
|
return_item_collection_metrics: 'SIZE' or 'NONE' (default)
|
|
1014
|
-
|
|
1067
|
+
|
|
1015
1068
|
Returns:
|
|
1016
1069
|
dict: Transaction response containing:
|
|
1017
1070
|
- 'ConsumedCapacity': Capacity consumed (if requested)
|
|
1018
1071
|
- 'ItemCollectionMetrics': Metrics (if requested)
|
|
1019
|
-
|
|
1072
|
+
|
|
1020
1073
|
Raises:
|
|
1021
1074
|
TransactionCanceledException: If transaction fails due to:
|
|
1022
1075
|
- Conditional check failure
|
|
1023
1076
|
- Item size too large
|
|
1024
1077
|
- Throughput exceeded
|
|
1025
1078
|
- Duplicate request
|
|
1026
|
-
|
|
1079
|
+
|
|
1027
1080
|
Example:
|
|
1028
1081
|
>>> # Transfer money between accounts atomically
|
|
1029
1082
|
>>> operations = [
|
|
@@ -1046,7 +1099,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
1046
1099
|
... }
|
|
1047
1100
|
... ]
|
|
1048
1101
|
>>> response = db.transact_write_items(operations=operations)
|
|
1049
|
-
|
|
1102
|
+
|
|
1050
1103
|
Note:
|
|
1051
1104
|
- Maximum 100 operations per transaction (AWS limit as of 2023)
|
|
1052
1105
|
- Each item can be up to 400 KB
|
|
@@ -1057,62 +1110,59 @@ class DynamoDB(DynamoDBConnection):
|
|
|
1057
1110
|
"""
|
|
1058
1111
|
if not operations:
|
|
1059
1112
|
raise ValueError("At least one operation is required")
|
|
1060
|
-
|
|
1113
|
+
|
|
1061
1114
|
if len(operations) > 100:
|
|
1062
1115
|
raise ValueError(
|
|
1063
1116
|
f"Transaction supports maximum 100 operations, got {len(operations)}. "
|
|
1064
1117
|
"Consider splitting into multiple transactions."
|
|
1065
1118
|
)
|
|
1066
|
-
|
|
1119
|
+
|
|
1067
1120
|
params = {
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1121
|
+
"TransactItems": operations,
|
|
1122
|
+
"ReturnConsumedCapacity": return_consumed_capacity,
|
|
1123
|
+
"ReturnItemCollectionMetrics": return_item_collection_metrics,
|
|
1071
1124
|
}
|
|
1072
|
-
|
|
1125
|
+
|
|
1073
1126
|
if client_request_token:
|
|
1074
|
-
params[
|
|
1075
|
-
|
|
1127
|
+
params["ClientRequestToken"] = client_request_token
|
|
1128
|
+
|
|
1076
1129
|
try:
|
|
1077
1130
|
response = self.dynamodb_resource.meta.client.transact_write_items(**params)
|
|
1078
1131
|
return response
|
|
1079
|
-
|
|
1132
|
+
|
|
1080
1133
|
except ClientError as e:
|
|
1081
|
-
error_code = e.response[
|
|
1082
|
-
|
|
1083
|
-
if error_code ==
|
|
1134
|
+
error_code = e.response["Error"]["Code"]
|
|
1135
|
+
|
|
1136
|
+
if error_code == "TransactionCanceledException":
|
|
1084
1137
|
# Parse cancellation reasons
|
|
1085
|
-
reasons = e.response.get(
|
|
1138
|
+
reasons = e.response.get("CancellationReasons", [])
|
|
1086
1139
|
logger.error(f"Transaction cancelled. Reasons: {reasons}")
|
|
1087
|
-
|
|
1140
|
+
|
|
1088
1141
|
# Enhance error message with specific reason
|
|
1089
1142
|
if reasons:
|
|
1090
1143
|
reason_messages = []
|
|
1091
1144
|
for idx, reason in enumerate(reasons):
|
|
1092
|
-
if reason.get(
|
|
1145
|
+
if reason.get("Code"):
|
|
1093
1146
|
reason_messages.append(
|
|
1094
1147
|
f"Operation {idx}: {reason['Code']} - {reason.get('Message', '')}"
|
|
1095
1148
|
)
|
|
1096
|
-
|
|
1149
|
+
|
|
1097
1150
|
raise RuntimeError(
|
|
1098
1151
|
f"Transaction failed: {'; '.join(reason_messages)}"
|
|
1099
1152
|
) from e
|
|
1100
|
-
|
|
1153
|
+
|
|
1101
1154
|
logger.exception(f"Error in transact_write_items: {str(e)}")
|
|
1102
1155
|
raise
|
|
1103
|
-
|
|
1156
|
+
|
|
1104
1157
|
def transact_get_items(
|
|
1105
|
-
self,
|
|
1106
|
-
keys: list[dict],
|
|
1107
|
-
*,
|
|
1108
|
-
return_consumed_capacity: str = "NONE"
|
|
1158
|
+
self, keys: list[dict], *, return_consumed_capacity: str = "NONE"
|
|
1109
1159
|
) -> dict:
|
|
1110
1160
|
"""
|
|
1111
1161
|
Retrieve multiple items with strong consistency as a transaction.
|
|
1112
|
-
|
|
1162
|
+
|
|
1113
1163
|
Unlike batch_get_item, this provides a consistent snapshot across all items
|
|
1114
1164
|
using strongly consistent reads. Maximum 100 items per transaction.
|
|
1115
|
-
|
|
1165
|
+
|
|
1116
1166
|
Args:
|
|
1117
1167
|
keys: List of get operation dictionaries. Each dict must specify:
|
|
1118
1168
|
- 'Key': The item's primary key
|
|
@@ -1133,12 +1183,12 @@ class DynamoDB(DynamoDBConnection):
|
|
|
1133
1183
|
}
|
|
1134
1184
|
]
|
|
1135
1185
|
return_consumed_capacity: 'INDEXES', 'TOTAL', or 'NONE' (default)
|
|
1136
|
-
|
|
1186
|
+
|
|
1137
1187
|
Returns:
|
|
1138
1188
|
dict: Response containing:
|
|
1139
1189
|
- 'Items': List of retrieved items (with Decimal conversion)
|
|
1140
1190
|
- 'ConsumedCapacity': Capacity consumed (if requested)
|
|
1141
|
-
|
|
1191
|
+
|
|
1142
1192
|
Example:
|
|
1143
1193
|
>>> keys = [
|
|
1144
1194
|
... {
|
|
@@ -1152,7 +1202,7 @@ class DynamoDB(DynamoDBConnection):
|
|
|
1152
1202
|
... ]
|
|
1153
1203
|
>>> response = db.transact_get_items(keys=keys)
|
|
1154
1204
|
>>> items = response['Items']
|
|
1155
|
-
|
|
1205
|
+
|
|
1156
1206
|
Note:
|
|
1157
1207
|
- Maximum 100 items per transaction
|
|
1158
1208
|
- Always uses strongly consistent reads
|
|
@@ -1162,45 +1212,42 @@ class DynamoDB(DynamoDBConnection):
|
|
|
1162
1212
|
"""
|
|
1163
1213
|
if not keys:
|
|
1164
1214
|
raise ValueError("At least one key is required")
|
|
1165
|
-
|
|
1215
|
+
|
|
1166
1216
|
if len(keys) > 100:
|
|
1167
1217
|
raise ValueError(
|
|
1168
1218
|
f"Transaction supports maximum 100 items, got {len(keys)}. "
|
|
1169
1219
|
"Use batch_get_item for larger requests."
|
|
1170
1220
|
)
|
|
1171
|
-
|
|
1221
|
+
|
|
1172
1222
|
# Build transaction get items
|
|
1173
1223
|
transact_items = []
|
|
1174
1224
|
for key_spec in keys:
|
|
1175
|
-
get_item = {
|
|
1225
|
+
get_item = {"Get": key_spec}
|
|
1176
1226
|
transact_items.append(get_item)
|
|
1177
|
-
|
|
1227
|
+
|
|
1178
1228
|
params = {
|
|
1179
|
-
|
|
1180
|
-
|
|
1229
|
+
"TransactItems": transact_items,
|
|
1230
|
+
"ReturnConsumedCapacity": return_consumed_capacity,
|
|
1181
1231
|
}
|
|
1182
|
-
|
|
1232
|
+
|
|
1183
1233
|
try:
|
|
1184
1234
|
response = self.dynamodb_resource.meta.client.transact_get_items(**params)
|
|
1185
|
-
|
|
1235
|
+
|
|
1186
1236
|
# Extract items from response
|
|
1187
1237
|
items = []
|
|
1188
|
-
if
|
|
1189
|
-
for item_response in response[
|
|
1190
|
-
if
|
|
1191
|
-
items.append(item_response[
|
|
1192
|
-
|
|
1193
|
-
result = {
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
if 'ConsumedCapacity' in response:
|
|
1199
|
-
result['ConsumedCapacity'] = response['ConsumedCapacity']
|
|
1200
|
-
|
|
1238
|
+
if "Responses" in response:
|
|
1239
|
+
for item_response in response["Responses"]:
|
|
1240
|
+
if "Item" in item_response:
|
|
1241
|
+
items.append(item_response["Item"])
|
|
1242
|
+
|
|
1243
|
+
result = {"Items": items, "Count": len(items)}
|
|
1244
|
+
|
|
1245
|
+
if "ConsumedCapacity" in response:
|
|
1246
|
+
result["ConsumedCapacity"] = response["ConsumedCapacity"]
|
|
1247
|
+
|
|
1201
1248
|
# Apply decimal conversion
|
|
1202
1249
|
return self._apply_decimal_conversion(result)
|
|
1203
|
-
|
|
1250
|
+
|
|
1204
1251
|
except ClientError as e:
|
|
1205
1252
|
logger.exception(f"Error in transact_get_items: {str(e)}")
|
|
1206
1253
|
raise
|