clear-skies-aws 1.10.2__py3-none-any.whl → 2.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {clear_skies_aws-1.10.2.dist-info → clear_skies_aws-2.0.2.dist-info}/METADATA +36 -35
- clear_skies_aws-2.0.2.dist-info/RECORD +63 -0
- {clear_skies_aws-1.10.2.dist-info → clear_skies_aws-2.0.2.dist-info}/WHEEL +1 -1
- clear_skies_aws-2.0.2.dist-info/licenses/LICENSE +21 -0
- clearskies_aws/__init__.py +15 -2
- clearskies_aws/actions/__init__.py +13 -106
- clearskies_aws/actions/action_aws.py +74 -57
- clearskies_aws/actions/assume_role.py +43 -30
- clearskies_aws/actions/ses.py +82 -73
- clearskies_aws/actions/sns.py +27 -30
- clearskies_aws/actions/sqs.py +32 -33
- clearskies_aws/actions/step_function.py +38 -31
- clearskies_aws/backends/__init__.py +11 -4
- clearskies_aws/backends/backend.py +106 -0
- clearskies_aws/backends/dynamo_db_backend.py +150 -155
- clearskies_aws/backends/dynamo_db_condition_parser.py +40 -80
- clearskies_aws/backends/dynamo_db_parti_ql_backend.py +179 -337
- clearskies_aws/backends/sqs_backend.py +32 -51
- clearskies_aws/configs/__init__.py +0 -0
- clearskies_aws/contexts/__init__.py +23 -10
- clearskies_aws/contexts/cli_web_socket_mock.py +19 -0
- clearskies_aws/contexts/lambda_alb.py +76 -0
- clearskies_aws/contexts/lambda_api_gateway.py +75 -28
- clearskies_aws/contexts/lambda_api_gateway_web_socket.py +56 -29
- clearskies_aws/contexts/lambda_invocation.py +15 -44
- clearskies_aws/contexts/lambda_sns.py +8 -33
- clearskies_aws/contexts/lambda_sqs_standard_partial_batch.py +14 -36
- clearskies_aws/di/__init__.py +6 -1
- clearskies_aws/di/aws_additional_config_auto_import.py +37 -0
- clearskies_aws/di/inject/__init__.py +6 -0
- clearskies_aws/di/inject/boto3.py +15 -0
- clearskies_aws/di/inject/boto3_session.py +13 -0
- clearskies_aws/di/inject/parameter_store.py +15 -0
- clearskies_aws/{handlers → endpoints}/secrets_manager_rotation.py +76 -55
- clearskies_aws/endpoints/simple_body_routing.py +41 -0
- clearskies_aws/input_outputs/__init__.py +21 -8
- clearskies_aws/input_outputs/{cli_websocket_mock.py → cli_web_socket_mock.py} +9 -3
- clearskies_aws/input_outputs/lambda_alb.py +53 -0
- clearskies_aws/input_outputs/lambda_api_gateway.py +106 -88
- clearskies_aws/input_outputs/lambda_api_gateway_web_socket.py +69 -6
- clearskies_aws/input_outputs/lambda_input_output.py +87 -0
- clearskies_aws/input_outputs/lambda_invocation.py +77 -26
- clearskies_aws/input_outputs/lambda_sns.py +66 -39
- clearskies_aws/input_outputs/lambda_sqs_standard.py +70 -40
- clearskies_aws/mocks/actions/ses.py +25 -19
- clearskies_aws/mocks/actions/sns.py +18 -12
- clearskies_aws/mocks/actions/sqs.py +18 -12
- clearskies_aws/mocks/actions/step_function.py +19 -13
- clearskies_aws/models/__init__.py +0 -0
- clearskies_aws/models/web_socket_connection_model.py +182 -0
- clearskies_aws/secrets/__init__.py +13 -7
- clearskies_aws/secrets/additional_configs/__init__.py +10 -2
- clearskies_aws/secrets/additional_configs/iam_db_auth.py +26 -16
- clearskies_aws/secrets/additional_configs/iam_db_auth_with_ssm.py +43 -39
- clearskies_aws/secrets/additional_configs/mysql_connection_dynamic_producer_via_ssh_cert_bastion.py +30 -31
- clearskies_aws/secrets/additional_configs/mysql_connection_dynamic_producer_via_ssm_bastion.py +70 -49
- clearskies_aws/secrets/akeyless_with_ssm_cache.py +32 -18
- clearskies_aws/secrets/parameter_store.py +34 -32
- clearskies_aws/secrets/secrets.py +16 -0
- clearskies_aws/secrets/secrets_manager.py +78 -57
- clear_skies_aws-1.10.2.dist-info/LICENSE +0 -7
- clear_skies_aws-1.10.2.dist-info/RECORD +0 -71
- clearskies_aws/actions/assume_role_test.py +0 -72
- clearskies_aws/actions/ses_test.py +0 -89
- clearskies_aws/actions/sns_test.py +0 -77
- clearskies_aws/actions/sqs_test.py +0 -127
- clearskies_aws/actions/step_function_test.py +0 -103
- clearskies_aws/backends/dynamo_db_backend_test.py +0 -300
- clearskies_aws/backends/dynamo_db_condition_parser_test.py +0 -266
- clearskies_aws/backends/dynamo_db_parti_ql_backend_test.py +0 -544
- clearskies_aws/backends/sqs_backend_test.py +0 -31
- clearskies_aws/contexts/cli.py +0 -19
- clearskies_aws/contexts/cli_websocket_mock.py +0 -33
- clearskies_aws/contexts/lambda_elb.py +0 -30
- clearskies_aws/contexts/lambda_http_gateway.py +0 -30
- clearskies_aws/contexts/lambda_sqs_standard_partial_batch_test.py +0 -66
- clearskies_aws/contexts/wsgi.py +0 -19
- clearskies_aws/di/standard_dependencies.py +0 -60
- clearskies_aws/handlers/simple_body_routing.py +0 -39
- clearskies_aws/input_outputs/lambda_api_gateway_test.py +0 -87
- clearskies_aws/input_outputs/lambda_elb.py +0 -21
- clearskies_aws/input_outputs/lambda_http_gateway.py +0 -12
- clearskies_aws/secrets/parameter_store_test.py +0 -18
- clearskies_aws/secrets/secrets_manager_test.py +0 -18
- clearskies_aws/web_socket_connection_model.py +0 -43
- clearskies_aws/{handlers → endpoints}/__init__.py +1 -1
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import base64
|
|
2
4
|
import binascii
|
|
3
5
|
import json
|
|
4
6
|
import logging
|
|
5
7
|
import re
|
|
6
8
|
from decimal import Decimal, InvalidOperation
|
|
7
|
-
from typing import Any, Callable,
|
|
9
|
+
from typing import Any, Callable, Generator
|
|
8
10
|
|
|
9
11
|
from boto3.session import Session as Boto3Session
|
|
10
12
|
from botocore.exceptions import ClientError
|
|
@@ -35,7 +37,7 @@ class DynamoDBPartiQLCursor:
|
|
|
35
37
|
|
|
36
38
|
def __init__(self, boto3_session: Boto3Session) -> None:
|
|
37
39
|
"""
|
|
38
|
-
|
|
40
|
+
Create the DynamoDBPartiQLCursor.
|
|
39
41
|
|
|
40
42
|
Args:
|
|
41
43
|
boto3_session: An initialized Boto3 Session object.
|
|
@@ -46,10 +48,10 @@ class DynamoDBPartiQLCursor:
|
|
|
46
48
|
def execute(
|
|
47
49
|
self,
|
|
48
50
|
statement: str,
|
|
49
|
-
parameters:
|
|
50
|
-
Limit:
|
|
51
|
-
NextToken:
|
|
52
|
-
ConsistentRead:
|
|
51
|
+
parameters: list[AttributeValueTypeDef] | None = None,
|
|
52
|
+
Limit: int | None = None,
|
|
53
|
+
NextToken: str | None = None,
|
|
54
|
+
ConsistentRead: bool | None = None,
|
|
53
55
|
) -> ExecuteStatementOutputTypeDef:
|
|
54
56
|
"""
|
|
55
57
|
Execute a PartiQL statement against DynamoDB.
|
|
@@ -70,9 +72,7 @@ class DynamoDBPartiQLCursor:
|
|
|
70
72
|
try:
|
|
71
73
|
call_args: ExecuteStatementInputTypeDef = {"Statement": statement}
|
|
72
74
|
# Only include 'Parameters' if it's not None AND not empty
|
|
73
|
-
if
|
|
74
|
-
parameters
|
|
75
|
-
): # This implies parameters is not None and parameters is not an empty list
|
|
75
|
+
if parameters: # This implies parameters is not None and parameters is not an empty list
|
|
76
76
|
call_args["Parameters"] = parameters
|
|
77
77
|
if Limit is not None:
|
|
78
78
|
call_args["Limit"] = Limit
|
|
@@ -81,11 +81,9 @@ class DynamoDBPartiQLCursor:
|
|
|
81
81
|
if ConsistentRead is not None:
|
|
82
82
|
call_args["ConsistentRead"] = ConsistentRead
|
|
83
83
|
|
|
84
|
-
output: ExecuteStatementOutputTypeDef = self._client.execute_statement(
|
|
85
|
-
**call_args
|
|
86
|
-
)
|
|
84
|
+
output: ExecuteStatementOutputTypeDef = self._client.execute_statement(**call_args)
|
|
87
85
|
except ClientError as err:
|
|
88
|
-
error_response:
|
|
86
|
+
error_response: dict[str, Any] = err.response.get("Error", {}) # type: ignore
|
|
89
87
|
error_code: str = error_response.get("Code", "UnknownCode")
|
|
90
88
|
error_message: str = error_response.get("Message", "Unknown error")
|
|
91
89
|
|
|
@@ -113,13 +111,14 @@ class DynamoDBPartiQLCursor:
|
|
|
113
111
|
class DynamoDBPartiQLBackend(CursorBackend):
|
|
114
112
|
"""
|
|
115
113
|
DynamoDB backend implementation that uses PartiQL for database interactions.
|
|
114
|
+
|
|
116
115
|
Supports querying base tables and attempts to use Global Secondary Indexes (GSIs)
|
|
117
116
|
when appropriate based on query conditions and sorting.
|
|
118
117
|
The count() method uses native DynamoDB Query/Scan operations for accuracy.
|
|
119
118
|
"""
|
|
120
119
|
|
|
121
120
|
_cursor: DynamoDBPartiQLCursor
|
|
122
|
-
_allowed_configs:
|
|
121
|
+
_allowed_configs: list[str] = [
|
|
123
122
|
"table_name",
|
|
124
123
|
"wheres",
|
|
125
124
|
"sorts",
|
|
@@ -131,20 +130,16 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
131
130
|
"group_by_column",
|
|
132
131
|
"joins",
|
|
133
132
|
]
|
|
134
|
-
_required_configs:
|
|
133
|
+
_required_configs: list[str] = ["table_name"]
|
|
135
134
|
|
|
136
135
|
def __init__(self, dynamo_db_parti_ql_cursor: DynamoDBPartiQLCursor) -> None:
|
|
137
|
-
"""
|
|
138
|
-
Initializes the DynamoDBPartiQLBackend.
|
|
139
|
-
"""
|
|
136
|
+
"""Initialize the DynamoDBPartiQLBackend."""
|
|
140
137
|
super().__init__(dynamo_db_parti_ql_cursor)
|
|
141
138
|
self.condition_parser: DynamoDBConditionParser = DynamoDBConditionParser()
|
|
142
|
-
self._table_descriptions_cache:
|
|
139
|
+
self._table_descriptions_cache: dict[str, dict[str, Any]] = {}
|
|
143
140
|
|
|
144
|
-
def _get_table_description(self, table_name: str) ->
|
|
145
|
-
"""
|
|
146
|
-
Retrieves and caches the DynamoDB table description.
|
|
147
|
-
"""
|
|
141
|
+
def _get_table_description(self, table_name: str) -> dict[str, Any]:
|
|
142
|
+
"""Retrieve and cache the DynamoDB table description."""
|
|
148
143
|
if table_name not in self._table_descriptions_cache:
|
|
149
144
|
try:
|
|
150
145
|
self._table_descriptions_cache[table_name] = self._cursor._client.describe_table(TableName=table_name) # type: ignore
|
|
@@ -154,19 +149,15 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
154
149
|
return self._table_descriptions_cache[table_name].get("Table", {})
|
|
155
150
|
|
|
156
151
|
def _table_escape_character(self) -> str:
|
|
157
|
-
"""
|
|
152
|
+
"""Return the character used to escape table/index names."""
|
|
158
153
|
return '"'
|
|
159
154
|
|
|
160
155
|
def _column_escape_character(self) -> str:
|
|
161
|
-
"""
|
|
156
|
+
"""Return the character used to escape column names."""
|
|
162
157
|
return '"'
|
|
163
158
|
|
|
164
|
-
def _finalize_table_name(
|
|
165
|
-
|
|
166
|
-
) -> str:
|
|
167
|
-
"""
|
|
168
|
-
Escapes a table name and optionally an index name for use in a PartiQL FROM clause.
|
|
169
|
-
"""
|
|
159
|
+
def _finalize_table_name(self, table_name: str, index_name: str | None = None) -> str:
|
|
160
|
+
"""Escapes a table name and optionally an index name for use in a PartiQL FROM clause."""
|
|
170
161
|
if not table_name:
|
|
171
162
|
return ""
|
|
172
163
|
esc: str = self._table_escape_character()
|
|
@@ -176,33 +167,29 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
176
167
|
return final_name
|
|
177
168
|
|
|
178
169
|
def _conditions_as_wheres_and_parameters(
|
|
179
|
-
self, conditions:
|
|
180
|
-
) ->
|
|
181
|
-
"""
|
|
182
|
-
Converts where conditions into a PartiQL WHERE clause and parameters.
|
|
183
|
-
"""
|
|
170
|
+
self, conditions: list[dict[str, Any]], default_table_name: str
|
|
171
|
+
) -> tuple[str, list[AttributeValueTypeDef]]:
|
|
172
|
+
"""Convert where conditions into a PartiQL WHERE clause and parameters."""
|
|
184
173
|
if not conditions:
|
|
185
174
|
return "", []
|
|
186
175
|
|
|
187
|
-
where_parts:
|
|
188
|
-
parameters:
|
|
176
|
+
where_parts: list[str] = []
|
|
177
|
+
parameters: list[AttributeValueTypeDef] = []
|
|
189
178
|
|
|
190
179
|
for where in conditions:
|
|
191
180
|
if not isinstance(where, dict):
|
|
192
181
|
logger.warning(f"Skipping non-dictionary where condition: {where}")
|
|
193
182
|
continue
|
|
194
183
|
|
|
195
|
-
column:
|
|
196
|
-
operator:
|
|
197
|
-
values:
|
|
184
|
+
column: str | None = where.get("column")
|
|
185
|
+
operator: str | None = where.get("operator")
|
|
186
|
+
values: list[Any] | None = where.get("values")
|
|
198
187
|
|
|
199
188
|
if not column or not operator or values is None:
|
|
200
|
-
logger.warning(
|
|
201
|
-
f"Skipping malformed structured where condition: {where}"
|
|
202
|
-
)
|
|
189
|
+
logger.warning(f"Skipping malformed structured where condition: {where}")
|
|
203
190
|
continue
|
|
204
191
|
|
|
205
|
-
value_parts:
|
|
192
|
+
value_parts: list[str] = []
|
|
206
193
|
for v in values:
|
|
207
194
|
if isinstance(v, str):
|
|
208
195
|
value_parts.append(f"'{v}'")
|
|
@@ -220,14 +207,10 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
220
207
|
elif op_lower in self.condition_parser.operators_without_placeholders:
|
|
221
208
|
condition_string = f"{column} {operator}"
|
|
222
209
|
else:
|
|
223
|
-
condition_string =
|
|
224
|
-
f"{column} {operator} {value_parts[0] if value_parts else ''}"
|
|
225
|
-
)
|
|
210
|
+
condition_string = f"{column} {operator} {value_parts[0] if value_parts else ''}"
|
|
226
211
|
|
|
227
212
|
try:
|
|
228
|
-
parsed:
|
|
229
|
-
condition_string
|
|
230
|
-
)
|
|
213
|
+
parsed: dict[str, Any] = self.condition_parser.parse_condition(condition_string)
|
|
231
214
|
where_parts.append(parsed["parsed"])
|
|
232
215
|
parameters.extend(parsed["values"])
|
|
233
216
|
except ValueError as e:
|
|
@@ -238,100 +221,83 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
238
221
|
return "", []
|
|
239
222
|
return " WHERE " + " AND ".join(where_parts), parameters
|
|
240
223
|
|
|
241
|
-
def as_sql(
|
|
242
|
-
|
|
243
|
-
) -> Tuple[str, List[AttributeValueTypeDef], Optional[int], Optional[str]]:
|
|
244
|
-
"""
|
|
245
|
-
Constructs a PartiQL statement and parameters from a query configuration.
|
|
246
|
-
"""
|
|
224
|
+
def as_sql(self, configuration: dict[str, Any]) -> tuple[str, list[AttributeValueTypeDef], int | None, str | None]:
|
|
225
|
+
"""Construct a PartiQL statement and parameters from a query configuration."""
|
|
247
226
|
escape: str = self._column_escape_character()
|
|
248
227
|
table_name: str = configuration.get("table_name", "")
|
|
249
|
-
chosen_index_name:
|
|
228
|
+
chosen_index_name: str | None = configuration.get("_chosen_index_name")
|
|
250
229
|
|
|
251
|
-
wheres, parameters = self._conditions_as_wheres_and_parameters(
|
|
252
|
-
configuration.get("wheres", []), table_name
|
|
253
|
-
)
|
|
230
|
+
wheres, parameters = self._conditions_as_wheres_and_parameters(configuration.get("wheres", []), table_name)
|
|
254
231
|
|
|
255
|
-
from_clause_target: str = self._finalize_table_name(
|
|
256
|
-
table_name, chosen_index_name
|
|
257
|
-
)
|
|
232
|
+
from_clause_target: str = self._finalize_table_name(table_name, chosen_index_name)
|
|
258
233
|
|
|
259
|
-
selects:
|
|
234
|
+
selects: list[str] | None = configuration.get("selects")
|
|
260
235
|
select_clause: str
|
|
261
236
|
if selects:
|
|
262
|
-
select_clause = ", ".join(
|
|
263
|
-
[f"{escape}{s.strip(escape)}{escape}" for s in selects]
|
|
264
|
-
)
|
|
237
|
+
select_clause = ", ".join([f"{escape}{s.strip(escape)}{escape}" for s in selects])
|
|
265
238
|
if configuration.get("select_all"):
|
|
266
|
-
logger.warning(
|
|
267
|
-
"Both 'select_all=True' and specific 'selects' were provided. Using specific 'selects'."
|
|
268
|
-
)
|
|
239
|
+
logger.warning("Both 'select_all=True' and specific 'selects' were provided. Using specific 'selects'.")
|
|
269
240
|
else:
|
|
270
241
|
select_clause = "*"
|
|
271
242
|
|
|
272
243
|
order_by: str = ""
|
|
273
|
-
sorts:
|
|
244
|
+
sorts: list[dict[str, str]] | None = configuration.get("sorts")
|
|
274
245
|
if sorts:
|
|
275
|
-
sort_parts:
|
|
246
|
+
sort_parts: list[str] = []
|
|
276
247
|
for sort in sorts:
|
|
277
248
|
column_name: str = sort["column"]
|
|
278
249
|
direction: str = sort.get("direction", "ASC").upper()
|
|
279
|
-
sort_parts.append(
|
|
280
|
-
f"{escape}{column_name.strip(escape)}{escape} {direction}"
|
|
281
|
-
)
|
|
250
|
+
sort_parts.append(f"{escape}{column_name.strip(escape)}{escape} {direction}")
|
|
282
251
|
if sort_parts:
|
|
283
252
|
order_by = " ORDER BY " + ", ".join(sort_parts)
|
|
284
253
|
|
|
285
254
|
if configuration.get("group_by_column"):
|
|
255
|
+
group_by_column = configuration.get("group_by_column")
|
|
286
256
|
logger.warning(
|
|
287
|
-
"Configuration included 'group_by_column="
|
|
288
|
-
|
|
257
|
+
"Configuration included 'group_by_column="
|
|
258
|
+
+ (group_by_column if group_by_column is not None else "")
|
|
259
|
+
+ "', "
|
|
260
|
+
+ "but GROUP BY is not supported by this DynamoDB PartiQL backend and will be ignored for SQL generation."
|
|
289
261
|
)
|
|
290
262
|
|
|
291
263
|
if configuration.get("joins"):
|
|
292
264
|
logger.warning(
|
|
293
|
-
"Configuration included 'joins="
|
|
294
|
-
|
|
265
|
+
"Configuration included 'joins="
|
|
266
|
+
+ str(configuration.get("joins"))
|
|
267
|
+
+ "', "
|
|
268
|
+
+ "but JOINs are not supported by this DynamoDB PartiQL backend and will be ignored for SQL generation."
|
|
295
269
|
)
|
|
296
270
|
|
|
297
|
-
limit:
|
|
271
|
+
limit: int | None = configuration.get("limit")
|
|
298
272
|
if limit is not None:
|
|
299
273
|
limit = int(limit)
|
|
300
274
|
|
|
301
|
-
pagination:
|
|
302
|
-
next_token:
|
|
275
|
+
pagination: dict[str, Any] = configuration.get("pagination", {})
|
|
276
|
+
next_token: str | None = pagination.get("next_token")
|
|
303
277
|
if next_token is not None:
|
|
304
278
|
next_token = str(next_token)
|
|
305
279
|
|
|
306
280
|
if not from_clause_target:
|
|
307
281
|
raise ValueError("Table name is required for constructing SQL query.")
|
|
308
282
|
|
|
309
|
-
statement: str = (
|
|
310
|
-
f"SELECT {select_clause} FROM {from_clause_target}{wheres}{order_by}".strip()
|
|
311
|
-
)
|
|
283
|
+
statement: str = f"SELECT {select_clause} FROM {from_clause_target}{wheres}{order_by}".strip()
|
|
312
284
|
|
|
313
285
|
return statement, parameters, limit, next_token
|
|
314
286
|
|
|
315
287
|
def records(
|
|
316
288
|
self,
|
|
317
|
-
configuration:
|
|
289
|
+
configuration: dict[str, Any],
|
|
318
290
|
model: ClearSkiesModel,
|
|
319
291
|
next_page_data: dict[str, Any] = {},
|
|
320
|
-
) -> Generator[
|
|
321
|
-
"""
|
|
322
|
-
Fetches records from DynamoDB based on the provided configuration using PartiQL.
|
|
323
|
-
"""
|
|
292
|
+
) -> Generator[dict[str, Any], None, None]:
|
|
293
|
+
"""Fetch records from DynamoDB based on the provided configuration using PartiQL."""
|
|
324
294
|
configuration = self._check_query_configuration(configuration, model)
|
|
325
295
|
|
|
326
|
-
statement, params, limit, client_next_token_from_as_sql = self.as_sql(
|
|
327
|
-
configuration
|
|
328
|
-
)
|
|
296
|
+
statement, params, limit, client_next_token_from_as_sql = self.as_sql(configuration)
|
|
329
297
|
|
|
330
|
-
ddb_token_for_this_call:
|
|
331
|
-
client_next_token_from_as_sql
|
|
332
|
-
)
|
|
298
|
+
ddb_token_for_this_call: str | None = self.restore_next_token_from_config(client_next_token_from_as_sql)
|
|
333
299
|
|
|
334
|
-
cursor_limit:
|
|
300
|
+
cursor_limit: int | None = None
|
|
335
301
|
if limit is not None and limit > 0:
|
|
336
302
|
cursor_limit = limit
|
|
337
303
|
|
|
@@ -343,38 +309,37 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
343
309
|
NextToken=ddb_token_for_this_call,
|
|
344
310
|
)
|
|
345
311
|
except Exception as e:
|
|
346
|
-
logger.error(
|
|
347
|
-
f"Error executing PartiQL statement in records(): {statement}, error: {e}"
|
|
348
|
-
)
|
|
312
|
+
logger.error(f"Error executing PartiQL statement in records(): {statement}, error: {e}")
|
|
349
313
|
next_page_data = {}
|
|
350
314
|
raise
|
|
351
315
|
|
|
352
|
-
items_from_response:
|
|
316
|
+
items_from_response: list[dict[str, Any]] = response.get("Items", [])
|
|
353
317
|
|
|
354
318
|
for item_raw in items_from_response:
|
|
355
319
|
yield self._map_from_boto3(item_raw)
|
|
356
320
|
|
|
357
|
-
next_token_from_ddb:
|
|
321
|
+
next_token_from_ddb: str | None = response.get("NextToken")
|
|
358
322
|
if next_token_from_ddb:
|
|
359
|
-
next_page_data["next_token"] = self.serialize_next_token_for_response(
|
|
360
|
-
next_token_from_ddb
|
|
361
|
-
)
|
|
323
|
+
next_page_data["next_token"] = self.serialize_next_token_for_response(next_token_from_ddb)
|
|
362
324
|
|
|
363
325
|
def _wheres_to_native_dynamo_expressions(
|
|
364
326
|
self,
|
|
365
|
-
conditions:
|
|
366
|
-
partition_key_name:
|
|
367
|
-
sort_key_name:
|
|
368
|
-
) ->
|
|
327
|
+
conditions: list[dict[str, Any]],
|
|
328
|
+
partition_key_name: str | None,
|
|
329
|
+
sort_key_name: str | None = None,
|
|
330
|
+
) -> dict[str, Any]:
|
|
369
331
|
"""
|
|
370
|
-
|
|
371
|
-
|
|
332
|
+
Convert 'where' conditions to DynamoDB expressions.
|
|
333
|
+
|
|
334
|
+
Transforms a list of condition dictionaries into PartiQL expression strings and attribute maps
|
|
335
|
+
for Query/Scan operations.
|
|
336
|
+
|
|
372
337
|
This implementation is more comprehensive than the previous one.
|
|
373
338
|
"""
|
|
374
|
-
expression_attribute_names:
|
|
375
|
-
expression_attribute_values:
|
|
376
|
-
key_condition_parts:
|
|
377
|
-
filter_expression_parts:
|
|
339
|
+
expression_attribute_names: dict[str, str] = {}
|
|
340
|
+
expression_attribute_values: dict[str, AttributeValueTypeDef] = {}
|
|
341
|
+
key_condition_parts: list[str] = []
|
|
342
|
+
filter_expression_parts: list[str] = []
|
|
378
343
|
|
|
379
344
|
name_counter = 0
|
|
380
345
|
value_counter = 0
|
|
@@ -393,9 +358,7 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
393
358
|
def get_value_placeholder(value: Any) -> str:
|
|
394
359
|
nonlocal value_counter
|
|
395
360
|
placeholder = f":val{value_counter}"
|
|
396
|
-
expression_attribute_values[placeholder] = (
|
|
397
|
-
self.condition_parser.to_dynamodb_attribute_value(value)
|
|
398
|
-
)
|
|
361
|
+
expression_attribute_values[placeholder] = self.condition_parser.to_dynamodb_attribute_value(value)
|
|
399
362
|
value_counter += 1
|
|
400
363
|
return placeholder
|
|
401
364
|
|
|
@@ -406,11 +369,7 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
406
369
|
pk_condition_index = -1
|
|
407
370
|
if partition_key_name:
|
|
408
371
|
for i, cond in enumerate(conditions):
|
|
409
|
-
if (
|
|
410
|
-
cond.get("column") == partition_key_name
|
|
411
|
-
and cond.get("operator") == "="
|
|
412
|
-
and cond.get("values")
|
|
413
|
-
):
|
|
372
|
+
if cond.get("column") == partition_key_name and cond.get("operator") == "=" and cond.get("values"):
|
|
414
373
|
pk_condition_index = i
|
|
415
374
|
break
|
|
416
375
|
|
|
@@ -449,18 +408,12 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
449
408
|
if len(cond["values"]) == 2:
|
|
450
409
|
sk_value1_ph = get_value_placeholder(cond["values"][0])
|
|
451
410
|
sk_value2_ph = get_value_placeholder(cond["values"][1])
|
|
452
|
-
key_condition_parts.append(
|
|
453
|
-
f"{sk_name_ph} BETWEEN {sk_value1_ph} AND {sk_value2_ph}"
|
|
454
|
-
)
|
|
411
|
+
key_condition_parts.append(f"{sk_name_ph} BETWEEN {sk_value1_ph} AND {sk_value2_ph}")
|
|
455
412
|
else:
|
|
456
|
-
logger.warning(
|
|
457
|
-
f"Skipping malformed BETWEEN condition for sort key: {cond}"
|
|
458
|
-
)
|
|
413
|
+
logger.warning(f"Skipping malformed BETWEEN condition for sort key: {cond}")
|
|
459
414
|
elif op_lower == "begins_with":
|
|
460
415
|
sk_value_ph = get_value_placeholder(cond["values"][0])
|
|
461
|
-
key_condition_parts.append(
|
|
462
|
-
f"begins_with({sk_name_ph}, {sk_value_ph})"
|
|
463
|
-
)
|
|
416
|
+
key_condition_parts.append(f"begins_with({sk_name_ph}, {sk_value_ph})")
|
|
464
417
|
else:
|
|
465
418
|
# Other operators for sort key are not part of KeyConditionExpression
|
|
466
419
|
# They will be handled in FilterExpression below
|
|
@@ -505,9 +458,7 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
505
458
|
if len(vals) == 2:
|
|
506
459
|
value1_ph = get_value_placeholder(vals[0])
|
|
507
460
|
value2_ph = get_value_placeholder(vals[1])
|
|
508
|
-
filter_expression_parts.append(
|
|
509
|
-
f"{name_ph} BETWEEN {value1_ph} AND {value2_ph}"
|
|
510
|
-
)
|
|
461
|
+
filter_expression_parts.append(f"{name_ph} BETWEEN {value1_ph} AND {value2_ph}")
|
|
511
462
|
else:
|
|
512
463
|
logger.warning(f"Skipping malformed BETWEEN condition: {cond}")
|
|
513
464
|
elif op_lower == "in":
|
|
@@ -524,16 +475,12 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
524
475
|
filter_expression_parts.append(f"begins_with({name_ph}, {value_ph})")
|
|
525
476
|
elif op_lower == "not begins_with":
|
|
526
477
|
value_ph = get_value_placeholder(vals[0])
|
|
527
|
-
filter_expression_parts.append(
|
|
528
|
-
f"NOT begins_with({name_ph}, {value_ph})"
|
|
529
|
-
)
|
|
478
|
+
filter_expression_parts.append(f"NOT begins_with({name_ph}, {value_ph})")
|
|
530
479
|
elif op_lower == "is null":
|
|
531
480
|
filter_expression_parts.append(f"attribute_not_exists({name_ph})")
|
|
532
481
|
elif op_lower == "is not null":
|
|
533
482
|
filter_expression_parts.append(f"attribute_exists({name_ph})")
|
|
534
|
-
elif
|
|
535
|
-
op_lower == "like"
|
|
536
|
-
): # Clearskies 'like' usually translates to begins_with or contains
|
|
483
|
+
elif op_lower == "like": # Clearskies 'like' usually translates to begins_with or contains
|
|
537
484
|
# This is a simplification. A full implementation might need to inspect '%' position.
|
|
538
485
|
# For now, if it contains '%', assume 'contains'. If it ends with '%', assume 'begins_with'.
|
|
539
486
|
# If no '%', it's an equality.
|
|
@@ -541,25 +488,19 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
541
488
|
like_value = vals[0]
|
|
542
489
|
if like_value.startswith("%") and like_value.endswith("%"):
|
|
543
490
|
value_ph = get_value_placeholder(like_value.strip("%"))
|
|
544
|
-
filter_expression_parts.append(
|
|
545
|
-
f"contains({name_ph}, {value_ph})"
|
|
546
|
-
)
|
|
491
|
+
filter_expression_parts.append(f"contains({name_ph}, {value_ph})")
|
|
547
492
|
elif like_value.endswith("%"):
|
|
548
493
|
value_ph = get_value_placeholder(like_value.rstrip("%"))
|
|
549
|
-
filter_expression_parts.append(
|
|
550
|
-
f"begins_with({name_ph}, {value_ph})"
|
|
551
|
-
)
|
|
494
|
+
filter_expression_parts.append(f"begins_with({name_ph}, {value_ph})")
|
|
552
495
|
else: # Treat as equality if no wildcards or complex pattern
|
|
553
496
|
value_ph = get_value_placeholder(like_value)
|
|
554
497
|
filter_expression_parts.append(f"{name_ph} = {value_ph}")
|
|
555
498
|
else:
|
|
556
499
|
logger.warning(f"Skipping unsupported LIKE condition: {cond}")
|
|
557
500
|
else:
|
|
558
|
-
logger.warning(
|
|
559
|
-
f"Skipping unsupported operator '{op}' for native DynamoDB expressions: {cond}"
|
|
560
|
-
)
|
|
501
|
+
logger.warning(f"Skipping unsupported operator '{op}' for native DynamoDB expressions: {cond}")
|
|
561
502
|
|
|
562
|
-
result:
|
|
503
|
+
result: dict[str, Any] = {}
|
|
563
504
|
if key_condition_parts:
|
|
564
505
|
result["KeyConditionExpression"] = " AND ".join(key_condition_parts)
|
|
565
506
|
if filter_expression_parts:
|
|
@@ -571,23 +512,19 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
571
512
|
|
|
572
513
|
return result
|
|
573
514
|
|
|
574
|
-
def count(self, configuration:
|
|
575
|
-
"""
|
|
576
|
-
Counts records in DynamoDB using native Query or Scan operations.
|
|
577
|
-
"""
|
|
515
|
+
def count(self, configuration: dict[str, Any], model: ClearSkiesModel) -> int:
|
|
516
|
+
"""Count records in DynamoDB using native Query or Scan operations."""
|
|
578
517
|
configuration = self._check_query_configuration(configuration, model)
|
|
579
518
|
|
|
580
519
|
table_name: str = configuration["table_name"]
|
|
581
|
-
chosen_index_name:
|
|
582
|
-
partition_key_for_target:
|
|
583
|
-
"_partition_key_for_target"
|
|
584
|
-
)
|
|
520
|
+
chosen_index_name: str | None = configuration.get("_chosen_index_name")
|
|
521
|
+
partition_key_for_target: str | None = configuration.get("_partition_key_for_target")
|
|
585
522
|
# Get sort key for the chosen target (base table or GSI)
|
|
586
|
-
sort_key_for_target:
|
|
523
|
+
sort_key_for_target: str | None = None
|
|
587
524
|
table_description = self._get_table_description(table_name)
|
|
588
525
|
if chosen_index_name:
|
|
589
|
-
gsi_definitions:
|
|
590
|
-
|
|
526
|
+
gsi_definitions: list[GlobalSecondaryIndexDescriptionTypeDef] = table_description.get(
|
|
527
|
+
"GlobalSecondaryIndexes", []
|
|
591
528
|
)
|
|
592
529
|
for gsi in gsi_definitions:
|
|
593
530
|
if gsi.get("IndexName", "") == chosen_index_name:
|
|
@@ -597,9 +534,7 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
597
534
|
break
|
|
598
535
|
break
|
|
599
536
|
else:
|
|
600
|
-
base_table_key_schema:
|
|
601
|
-
table_description.get("KeySchema", [])
|
|
602
|
-
)
|
|
537
|
+
base_table_key_schema: list[KeySchemaElementTypeDef] = table_description.get("KeySchema", [])
|
|
603
538
|
for key_element in base_table_key_schema:
|
|
604
539
|
if key_element["KeyType"] == "RANGE":
|
|
605
540
|
sort_key_for_target = key_element["AttributeName"]
|
|
@@ -611,7 +546,7 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
611
546
|
wheres_config, partition_key_for_target, sort_key_for_target
|
|
612
547
|
)
|
|
613
548
|
|
|
614
|
-
params_for_native_call:
|
|
549
|
+
params_for_native_call: dict[str, Any] = {
|
|
615
550
|
"TableName": table_name,
|
|
616
551
|
"Select": "COUNT",
|
|
617
552
|
}
|
|
@@ -628,40 +563,28 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
628
563
|
in native_expressions.get("ExpressionAttributeNames", {})
|
|
629
564
|
and native_expressions.get("KeyConditionExpression")
|
|
630
565
|
and f"#{re.sub(r'[^a-zA-Z0-9_]', '', partition_key_for_target)}_0 = :val0"
|
|
631
|
-
in native_expressions[
|
|
632
|
-
"KeyConditionExpression"
|
|
633
|
-
] # Simplified check, assumes first value is PK
|
|
566
|
+
in native_expressions["KeyConditionExpression"] # Simplified check, assumes first value is PK
|
|
634
567
|
):
|
|
635
568
|
can_use_query_for_count = True
|
|
636
|
-
params_for_native_call["KeyConditionExpression"] = native_expressions[
|
|
637
|
-
"KeyConditionExpression"
|
|
638
|
-
]
|
|
569
|
+
params_for_native_call["KeyConditionExpression"] = native_expressions["KeyConditionExpression"]
|
|
639
570
|
if native_expressions.get("FilterExpression"):
|
|
640
|
-
params_for_native_call["FilterExpression"] = native_expressions[
|
|
641
|
-
"FilterExpression"
|
|
642
|
-
]
|
|
571
|
+
params_for_native_call["FilterExpression"] = native_expressions["FilterExpression"]
|
|
643
572
|
else:
|
|
644
573
|
# Fall back to Scan, and all conditions (including any potential key conditions that
|
|
645
574
|
# couldn't be used for a Query) go into FilterExpression.
|
|
646
575
|
if native_expressions.get("FilterExpression"):
|
|
647
|
-
params_for_native_call["FilterExpression"] = native_expressions[
|
|
648
|
-
"FilterExpression"
|
|
649
|
-
]
|
|
576
|
+
params_for_native_call["FilterExpression"] = native_expressions["FilterExpression"]
|
|
650
577
|
# If there's a KeyConditionExpression but no PK equality, it should also be part of the filter for scan.
|
|
651
578
|
# This logic is now handled more robustly within _wheres_to_native_dynamo_expressions
|
|
652
579
|
# by ensuring only true PK/SK conditions go to KeyConditionExpression initially.
|
|
653
580
|
|
|
654
581
|
if native_expressions.get("ExpressionAttributeNames"):
|
|
655
|
-
params_for_native_call["ExpressionAttributeNames"] = native_expressions[
|
|
656
|
-
"ExpressionAttributeNames"
|
|
657
|
-
]
|
|
582
|
+
params_for_native_call["ExpressionAttributeNames"] = native_expressions["ExpressionAttributeNames"]
|
|
658
583
|
if native_expressions.get("ExpressionAttributeValues"):
|
|
659
|
-
params_for_native_call["ExpressionAttributeValues"] = native_expressions[
|
|
660
|
-
"ExpressionAttributeValues"
|
|
661
|
-
]
|
|
584
|
+
params_for_native_call["ExpressionAttributeValues"] = native_expressions["ExpressionAttributeValues"]
|
|
662
585
|
|
|
663
586
|
total_count = 0
|
|
664
|
-
exclusive_start_key:
|
|
587
|
+
exclusive_start_key: dict[str, AttributeValueTypeDef] | None = None
|
|
665
588
|
|
|
666
589
|
while True:
|
|
667
590
|
if exclusive_start_key:
|
|
@@ -669,14 +592,10 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
669
592
|
|
|
670
593
|
try:
|
|
671
594
|
if can_use_query_for_count:
|
|
672
|
-
logger.debug(
|
|
673
|
-
f"Executing native DynamoDB Query (for count) with params: {params_for_native_call}"
|
|
674
|
-
)
|
|
595
|
+
logger.debug(f"Executing native DynamoDB Query (for count) with params: {params_for_native_call}")
|
|
675
596
|
response = self._cursor._client.query(**params_for_native_call) # type: ignore
|
|
676
597
|
else:
|
|
677
|
-
logger.debug(
|
|
678
|
-
f"Executing native DynamoDB Scan (for count) with params: {params_for_native_call}"
|
|
679
|
-
)
|
|
598
|
+
logger.debug(f"Executing native DynamoDB Scan (for count) with params: {params_for_native_call}")
|
|
680
599
|
response = self._cursor._client.scan(**params_for_native_call) # type: ignore
|
|
681
600
|
except ClientError as e:
|
|
682
601
|
logger.error(
|
|
@@ -691,10 +610,8 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
691
610
|
|
|
692
611
|
return total_count
|
|
693
612
|
|
|
694
|
-
def create(self, data:
|
|
695
|
-
"""
|
|
696
|
-
Creates a new record in DynamoDB using PartiQL INSERT.
|
|
697
|
-
"""
|
|
613
|
+
def create(self, data: dict[str, Any], model: ClearSkiesModel) -> dict[str, Any]:
|
|
614
|
+
"""Create a new record in DynamoDB using PartiQL INSERT."""
|
|
698
615
|
table_name: str = self._finalize_table_name(model.get_table_name())
|
|
699
616
|
|
|
700
617
|
if not data:
|
|
@@ -702,10 +619,10 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
702
619
|
return {}
|
|
703
620
|
|
|
704
621
|
# Prepare parameters
|
|
705
|
-
parameters:
|
|
622
|
+
parameters: list[AttributeValueTypeDef] = []
|
|
706
623
|
|
|
707
624
|
# Build the 'VALUE {key: ?, key: ?}' part and collect parameters
|
|
708
|
-
value_struct_parts:
|
|
625
|
+
value_struct_parts: list[str] = []
|
|
709
626
|
for key, value in data.items():
|
|
710
627
|
# Use single quotes around the key to match PartiQL documentation examples
|
|
711
628
|
value_struct_parts.append(f"'{key}': ?")
|
|
@@ -722,31 +639,21 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
722
639
|
)
|
|
723
640
|
return data
|
|
724
641
|
except Exception as e:
|
|
725
|
-
logger.error(
|
|
726
|
-
f"Error executing INSERT PartiQL statement: {statement}, data: {data}, error: {e}"
|
|
727
|
-
)
|
|
642
|
+
logger.error(f"Error executing INSERT PartiQL statement: {statement}, data: {data}, error: {e}")
|
|
728
643
|
raise
|
|
729
644
|
|
|
730
|
-
def update(
|
|
731
|
-
|
|
732
|
-
) -> Dict[str, Any]:
|
|
733
|
-
"""
|
|
734
|
-
Updates an existing record in DynamoDB using PartiQL UPDATE.
|
|
735
|
-
"""
|
|
645
|
+
def update(self, id_value: Any, data: dict[str, Any], model: ClearSkiesModel) -> dict[str, Any]:
|
|
646
|
+
"""Update an existing record in DynamoDB using PartiQL UPDATE."""
|
|
736
647
|
table_name: str = self._finalize_table_name(model.get_table_name())
|
|
737
648
|
id_column_name: str = model.id_column_name
|
|
738
|
-
escaped_id_column: str = (
|
|
739
|
-
f"{self._column_escape_character()}{id_column_name}{self._column_escape_character()}"
|
|
740
|
-
)
|
|
649
|
+
escaped_id_column: str = f"{self._column_escape_character()}{id_column_name}{self._column_escape_character()}"
|
|
741
650
|
|
|
742
651
|
if not data:
|
|
743
|
-
logger.warning(
|
|
744
|
-
f"Update called with empty data for ID {id_value}. Returning ID only."
|
|
745
|
-
)
|
|
652
|
+
logger.warning(f"Update called with empty data for ID {id_value}. Returning ID only.")
|
|
746
653
|
return {id_column_name: id_value}
|
|
747
654
|
|
|
748
|
-
set_clauses:
|
|
749
|
-
parameters:
|
|
655
|
+
set_clauses: list[str] = []
|
|
656
|
+
parameters: list[AttributeValueTypeDef] = []
|
|
750
657
|
col_esc: str = self._column_escape_character()
|
|
751
658
|
|
|
752
659
|
for key, value in data.items():
|
|
@@ -756,17 +663,13 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
756
663
|
parameters.append(self.condition_parser.to_dynamodb_attribute_value(value))
|
|
757
664
|
|
|
758
665
|
if not set_clauses:
|
|
759
|
-
logger.warning(
|
|
760
|
-
f"Update called for ID {id_value} but no updatable fields found in data. Returning ID only."
|
|
761
|
-
)
|
|
666
|
+
logger.warning(f"Update called for ID {id_value} but no updatable fields found in data. Returning ID only.")
|
|
762
667
|
return {id_column_name: id_value}
|
|
763
668
|
|
|
764
669
|
parameters.append(self.condition_parser.to_dynamodb_attribute_value(id_value))
|
|
765
670
|
|
|
766
671
|
set_statement: str = ", ".join(set_clauses)
|
|
767
|
-
statement: str =
|
|
768
|
-
f"UPDATE {table_name} SET {set_statement} WHERE {escaped_id_column} = ? RETURNING ALL NEW *"
|
|
769
|
-
)
|
|
672
|
+
statement: str = f"UPDATE {table_name} SET {set_statement} WHERE {escaped_id_column} = ? RETURNING ALL NEW *"
|
|
770
673
|
|
|
771
674
|
try:
|
|
772
675
|
response = self._cursor.execute(statement=statement, parameters=parameters)
|
|
@@ -785,33 +688,26 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
785
688
|
raise
|
|
786
689
|
|
|
787
690
|
def delete(self, id_value: Any, model: ClearSkiesModel) -> bool:
|
|
788
|
-
"""
|
|
789
|
-
Deletes a record from DynamoDB using PartiQL DELETE.
|
|
790
|
-
"""
|
|
691
|
+
"""Delete a record from DynamoDB using PartiQL DELETE."""
|
|
791
692
|
table_name: str = self._finalize_table_name(model.get_table_name())
|
|
792
693
|
id_column_name: str = model.id_column_name
|
|
793
|
-
escaped_id_column: str = (
|
|
794
|
-
f"{self._column_escape_character()}{id_column_name}{self._column_escape_character()}"
|
|
795
|
-
)
|
|
694
|
+
escaped_id_column: str = f"{self._column_escape_character()}{id_column_name}{self._column_escape_character()}"
|
|
796
695
|
|
|
797
|
-
parameters:
|
|
798
|
-
self.condition_parser.to_dynamodb_attribute_value(id_value)
|
|
799
|
-
]
|
|
696
|
+
parameters: list[AttributeValueTypeDef] = [self.condition_parser.to_dynamodb_attribute_value(id_value)]
|
|
800
697
|
statement: str = f"DELETE FROM {table_name} WHERE {escaped_id_column} = ?"
|
|
801
698
|
|
|
802
699
|
try:
|
|
803
700
|
self._cursor.execute(statement=statement, parameters=parameters)
|
|
804
701
|
return True
|
|
805
702
|
except Exception as e:
|
|
806
|
-
logger.error(
|
|
807
|
-
f"Error executing DELETE PartiQL statement: {statement}, id: {id_value}, error: {e}"
|
|
808
|
-
)
|
|
703
|
+
logger.error(f"Error executing DELETE PartiQL statement: {statement}, id: {id_value}, error: {e}")
|
|
809
704
|
raise
|
|
810
705
|
|
|
811
|
-
def _map_from_boto3(self, record:
|
|
706
|
+
def _map_from_boto3(self, record: dict[str, Any]) -> dict[str, Any]:
|
|
812
707
|
"""
|
|
813
|
-
|
|
814
|
-
|
|
708
|
+
Convert DynamoDB record to Python-native dictionary.
|
|
709
|
+
|
|
710
|
+
Maps AttributeValueTypeDef values from DynamoDB to standard Python types for easier processing.
|
|
815
711
|
|
|
816
712
|
Args:
|
|
817
713
|
record: A dictionary representing a record item from DynamoDB,
|
|
@@ -820,13 +716,11 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
820
716
|
Returns:
|
|
821
717
|
A dictionary with values unwrapped to Python native types.
|
|
822
718
|
"""
|
|
823
|
-
return {
|
|
824
|
-
key: self._map_from_boto3_value(value) for (key, value) in record.items()
|
|
825
|
-
}
|
|
719
|
+
return {key: self._map_from_boto3_value(value) for (key, value) in record.items()}
|
|
826
720
|
|
|
827
721
|
def _map_from_boto3_value(self, attribute_value: AttributeValueTypeDef) -> Any:
|
|
828
722
|
"""
|
|
829
|
-
|
|
723
|
+
Convert a single DynamoDB AttributeValueTypeDef to its Python native equivalent.
|
|
830
724
|
|
|
831
725
|
Args:
|
|
832
726
|
attribute_value: A DynamoDB AttributeValueTypeDef dictionary.
|
|
@@ -843,9 +737,7 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
843
737
|
try:
|
|
844
738
|
return Decimal(attribute_value["N"])
|
|
845
739
|
except InvalidOperation: # Changed from DecimalException
|
|
846
|
-
logger.warning(
|
|
847
|
-
f"Could not convert N value '{attribute_value['N']}' to Decimal."
|
|
848
|
-
)
|
|
740
|
+
logger.warning(f"Could not convert N value '{attribute_value['N']}' to Decimal.")
|
|
849
741
|
return attribute_value["N"]
|
|
850
742
|
if "BOOL" in attribute_value:
|
|
851
743
|
return attribute_value["BOOL"]
|
|
@@ -855,26 +747,19 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
855
747
|
try:
|
|
856
748
|
return base64.b64decode(attribute_value["B"])
|
|
857
749
|
except (binascii.Error, TypeError) as e:
|
|
858
|
-
logger.warning(
|
|
859
|
-
f"Failed to decode base64 binary value: {attribute_value['B']}, error: {e}"
|
|
860
|
-
)
|
|
750
|
+
logger.warning(f"Failed to decode base64 binary value: {attribute_value['B']}, error: {e}")
|
|
861
751
|
return attribute_value["B"] # Return raw if decoding fails
|
|
862
752
|
if "L" in attribute_value:
|
|
863
753
|
return [self._map_from_boto3_value(item) for item in attribute_value["L"]]
|
|
864
754
|
if "M" in attribute_value:
|
|
865
|
-
return {
|
|
866
|
-
key: self._map_from_boto3_value(val)
|
|
867
|
-
for key, val in attribute_value["M"].items()
|
|
868
|
-
}
|
|
755
|
+
return {key: self._map_from_boto3_value(val) for key, val in attribute_value["M"].items()}
|
|
869
756
|
if "SS" in attribute_value:
|
|
870
757
|
return set(attribute_value["SS"])
|
|
871
758
|
if "NS" in attribute_value:
|
|
872
759
|
try:
|
|
873
760
|
return set(Decimal(n_val) for n_val in attribute_value["NS"])
|
|
874
761
|
except InvalidOperation: # Changed from DecimalException
|
|
875
|
-
logger.warning(
|
|
876
|
-
f"Could not convert one or more NS values in '{attribute_value['NS']}' to Decimal."
|
|
877
|
-
)
|
|
762
|
+
logger.warning(f"Could not convert one or more NS values in '{attribute_value['NS']}' to Decimal.")
|
|
878
763
|
return set(attribute_value["NS"])
|
|
879
764
|
if "BS" in attribute_value:
|
|
880
765
|
try:
|
|
@@ -888,19 +773,18 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
888
773
|
logger.warning(f"Unrecognized DynamoDB attribute type: {attribute_value}")
|
|
889
774
|
return attribute_value
|
|
890
775
|
|
|
891
|
-
def _check_query_configuration(
|
|
892
|
-
self, configuration: Dict[str, Any], model: ClearSkiesModel
|
|
893
|
-
) -> Dict[str, Any]:
|
|
776
|
+
def _check_query_configuration(self, configuration: dict[str, Any], model: ClearSkiesModel) -> dict[str, Any]:
|
|
894
777
|
"""
|
|
895
|
-
|
|
778
|
+
Validate and update query configuration.
|
|
779
|
+
|
|
780
|
+
Checks the configuration, sets defaults, and ensures required fields for a valid query.
|
|
896
781
|
select an appropriate GSI if sorting is requested and conditions allow.
|
|
782
|
+
|
|
897
783
|
It also stores the determined partition key for the target in the configuration.
|
|
898
784
|
"""
|
|
899
785
|
for key in list(configuration.keys()):
|
|
900
786
|
if key not in self._allowed_configs:
|
|
901
|
-
raise KeyError(
|
|
902
|
-
f"DynamoDBBackend does not support config '{key}'. You may be using the wrong backend"
|
|
903
|
-
)
|
|
787
|
+
raise KeyError(f"DynamoDBBackend does not support config '{key}'. You may be using the wrong backend")
|
|
904
788
|
for key in self._required_configs:
|
|
905
789
|
if not configuration.get(key):
|
|
906
790
|
raise KeyError(f"Missing required configuration key {key}")
|
|
@@ -927,33 +811,32 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
927
811
|
configuration["_chosen_index_name"] = None
|
|
928
812
|
configuration["_partition_key_for_target"] = None
|
|
929
813
|
|
|
930
|
-
if configuration.get("sorts") or configuration.get(
|
|
931
|
-
"wheres"
|
|
932
|
-
): # Check for index even if not sorting, for count
|
|
814
|
+
if configuration.get("sorts") or configuration.get("wheres"): # Check for index even if not sorting, for count
|
|
933
815
|
table_name_from_config: str = configuration.get("table_name", "")
|
|
934
816
|
table_description = self._get_table_description(table_name_from_config)
|
|
935
817
|
|
|
936
818
|
wheres = configuration.get("wheres", [])
|
|
819
|
+
sorts = configuration.get("sorts")
|
|
937
820
|
sort_column = (
|
|
938
|
-
|
|
939
|
-
if
|
|
821
|
+
sorts[0]["column"]
|
|
822
|
+
if sorts and len(sorts) > 0 and sorts[0] is not None and "column" in sorts[0]
|
|
940
823
|
else None
|
|
941
824
|
)
|
|
942
825
|
|
|
943
|
-
key_to_check_for_equality:
|
|
826
|
+
key_to_check_for_equality: str | None = None
|
|
944
827
|
target_name_for_error_msg: str = table_name_from_config
|
|
945
|
-
chosen_index_for_query:
|
|
946
|
-
partition_key_for_chosen_target:
|
|
828
|
+
chosen_index_for_query: str | None = None
|
|
829
|
+
partition_key_for_chosen_target: str | None = None
|
|
947
830
|
|
|
948
|
-
gsi_definitions:
|
|
949
|
-
|
|
831
|
+
gsi_definitions: list[GlobalSecondaryIndexDescriptionTypeDef] = table_description.get(
|
|
832
|
+
"GlobalSecondaryIndexes", []
|
|
950
833
|
)
|
|
951
834
|
if gsi_definitions:
|
|
952
835
|
for gsi in gsi_definitions:
|
|
953
836
|
gsi_name: str = gsi["IndexName"]
|
|
954
|
-
gsi_key_schema:
|
|
955
|
-
gsi_partition_key:
|
|
956
|
-
gsi_sort_key:
|
|
837
|
+
gsi_key_schema: list[KeySchemaElementTypeDef] = gsi["KeySchema"]
|
|
838
|
+
gsi_partition_key: str | None = None
|
|
839
|
+
gsi_sort_key: str | None = None
|
|
957
840
|
|
|
958
841
|
for key_element in gsi_key_schema:
|
|
959
842
|
if key_element["KeyType"] == "HASH":
|
|
@@ -962,8 +845,7 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
962
845
|
gsi_sort_key = key_element["AttributeName"]
|
|
963
846
|
|
|
964
847
|
if gsi_partition_key and any(
|
|
965
|
-
w.get("column") == gsi_partition_key
|
|
966
|
-
and w.get("operator") == "="
|
|
848
|
+
w.get("column") == gsi_partition_key and w.get("operator") == "="
|
|
967
849
|
for w in wheres
|
|
968
850
|
if isinstance(w, dict)
|
|
969
851
|
):
|
|
@@ -971,39 +853,29 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
971
853
|
if sort_column == gsi_partition_key and not gsi_sort_key:
|
|
972
854
|
key_to_check_for_equality = gsi_partition_key
|
|
973
855
|
chosen_index_for_query = gsi_name
|
|
974
|
-
target_name_for_error_msg = (
|
|
975
|
-
f"{table_name_from_config} (index: {gsi_name})"
|
|
976
|
-
)
|
|
856
|
+
target_name_for_error_msg = f"{table_name_from_config} (index: {gsi_name})"
|
|
977
857
|
partition_key_for_chosen_target = gsi_partition_key
|
|
978
858
|
break
|
|
979
859
|
if sort_column == gsi_sort_key:
|
|
980
860
|
key_to_check_for_equality = gsi_partition_key
|
|
981
861
|
chosen_index_for_query = gsi_name
|
|
982
|
-
target_name_for_error_msg = (
|
|
983
|
-
f"{table_name_from_config} (index: {gsi_name})"
|
|
984
|
-
)
|
|
862
|
+
target_name_for_error_msg = f"{table_name_from_config} (index: {gsi_name})"
|
|
985
863
|
partition_key_for_chosen_target = gsi_partition_key
|
|
986
864
|
break
|
|
987
865
|
else:
|
|
988
866
|
key_to_check_for_equality = gsi_partition_key
|
|
989
867
|
chosen_index_for_query = gsi_name
|
|
990
|
-
target_name_for_error_msg = (
|
|
991
|
-
f"{table_name_from_config} (index: {gsi_name})"
|
|
992
|
-
)
|
|
868
|
+
target_name_for_error_msg = f"{table_name_from_config} (index: {gsi_name})"
|
|
993
869
|
partition_key_for_chosen_target = gsi_partition_key
|
|
994
870
|
break
|
|
995
871
|
|
|
996
872
|
if not chosen_index_for_query:
|
|
997
|
-
base_table_key_schema:
|
|
998
|
-
table_description.get("KeySchema", [])
|
|
999
|
-
)
|
|
873
|
+
base_table_key_schema: list[KeySchemaElementTypeDef] = table_description.get("KeySchema", [])
|
|
1000
874
|
if base_table_key_schema:
|
|
1001
875
|
for key_element in base_table_key_schema:
|
|
1002
876
|
if key_element["KeyType"] == "HASH":
|
|
1003
877
|
key_to_check_for_equality = key_element["AttributeName"]
|
|
1004
|
-
partition_key_for_chosen_target = key_element[
|
|
1005
|
-
"AttributeName"
|
|
1006
|
-
]
|
|
878
|
+
partition_key_for_chosen_target = key_element["AttributeName"]
|
|
1007
879
|
break
|
|
1008
880
|
|
|
1009
881
|
configuration["_chosen_index_name"] = chosen_index_for_query
|
|
@@ -1017,8 +889,7 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
1017
889
|
)
|
|
1018
890
|
else:
|
|
1019
891
|
has_required_key_equality = any(
|
|
1020
|
-
w.get("column") == key_to_check_for_equality
|
|
1021
|
-
and w.get("operator") == "="
|
|
892
|
+
w.get("column") == key_to_check_for_equality and w.get("operator") == "="
|
|
1022
893
|
for w in wheres
|
|
1023
894
|
if isinstance(w, dict)
|
|
1024
895
|
)
|
|
@@ -1029,19 +900,12 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
1029
900
|
)
|
|
1030
901
|
return configuration
|
|
1031
902
|
|
|
1032
|
-
def validate_pagination_kwargs(
|
|
1033
|
-
|
|
1034
|
-
) -> str:
|
|
1035
|
-
"""
|
|
1036
|
-
Validates pagination keyword arguments.
|
|
1037
|
-
"""
|
|
903
|
+
def validate_pagination_kwargs(self, kwargs: dict[str, Any], case_mapping: Callable[[str], str]) -> str:
|
|
904
|
+
"""Validate pagination keyword arguments."""
|
|
1038
905
|
extra_keys: set[str] = set(kwargs.keys()) - set(self.allowed_pagination_keys())
|
|
1039
906
|
key_name: str = case_mapping("next_token")
|
|
1040
907
|
if len(extra_keys):
|
|
1041
|
-
return (
|
|
1042
|
-
f"Invalid pagination key(s): '{','.join(sorted(list(extra_keys)))}'. "
|
|
1043
|
-
f"Only '{key_name}' is allowed"
|
|
1044
|
-
)
|
|
908
|
+
return f"Invalid pagination key(s): '{','.join(sorted(list(extra_keys)))}'. Only '{key_name}' is allowed"
|
|
1045
909
|
if "next_token" not in kwargs:
|
|
1046
910
|
return f"You must specify '{key_name}' when setting pagination"
|
|
1047
911
|
try:
|
|
@@ -1053,18 +917,12 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
1053
917
|
return f"The provided '{key_name}' appears to be invalid."
|
|
1054
918
|
return ""
|
|
1055
919
|
|
|
1056
|
-
def allowed_pagination_keys(self) ->
|
|
1057
|
-
"""
|
|
1058
|
-
Returns a list of allowed keys for pagination.
|
|
1059
|
-
"""
|
|
920
|
+
def allowed_pagination_keys(self) -> list[str]:
|
|
921
|
+
"""Return a list of allowed keys for pagination."""
|
|
1060
922
|
return ["next_token"]
|
|
1061
923
|
|
|
1062
|
-
def restore_next_token_from_config(
|
|
1063
|
-
|
|
1064
|
-
) -> Optional[Any]:
|
|
1065
|
-
"""
|
|
1066
|
-
Decodes a base64 encoded JSON string (next_token) into its original form.
|
|
1067
|
-
"""
|
|
924
|
+
def restore_next_token_from_config(self, next_token: str | None) -> Any | None:
|
|
925
|
+
"""Decode a base64 encoded JSON string (next_token) into its original form."""
|
|
1068
926
|
if not next_token or not isinstance(next_token, str):
|
|
1069
927
|
return None
|
|
1070
928
|
try:
|
|
@@ -1075,12 +933,8 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
1075
933
|
logger.warning(f"Failed to restore next_token: {next_token}")
|
|
1076
934
|
return None
|
|
1077
935
|
|
|
1078
|
-
def serialize_next_token_for_response(
|
|
1079
|
-
|
|
1080
|
-
) -> Optional[str]:
|
|
1081
|
-
"""
|
|
1082
|
-
Serializes a DynamoDB PartiQL NextToken string into a base64 encoded JSON string.
|
|
1083
|
-
"""
|
|
936
|
+
def serialize_next_token_for_response(self, ddb_next_token: str | None) -> str | None:
|
|
937
|
+
"""Serialize a DynamoDB PartiQL NextToken string into a base64 encoded JSON string."""
|
|
1084
938
|
if ddb_next_token is None:
|
|
1085
939
|
return None
|
|
1086
940
|
try:
|
|
@@ -1088,33 +942,21 @@ class DynamoDBPartiQLBackend(CursorBackend):
|
|
|
1088
942
|
encoded_bytes: bytes = base64.urlsafe_b64encode(json_string.encode("utf-8"))
|
|
1089
943
|
return encoded_bytes.decode("utf8")
|
|
1090
944
|
except (TypeError, ValueError) as e:
|
|
1091
|
-
logger.error(
|
|
1092
|
-
f"Error serializing DDB next_token: {ddb_next_token}, error: {e}"
|
|
1093
|
-
)
|
|
945
|
+
logger.error(f"Error serializing DDB next_token: {ddb_next_token}, error: {e}")
|
|
1094
946
|
return None
|
|
1095
947
|
|
|
1096
|
-
def documentation_pagination_next_page_response(
|
|
1097
|
-
|
|
1098
|
-
) -> List[AutoDocString]:
|
|
1099
|
-
"""
|
|
1100
|
-
Provides documentation for the 'next_page' (pagination token) in API responses.
|
|
1101
|
-
"""
|
|
948
|
+
def documentation_pagination_next_page_response(self, case_mapping: Callable[[str], str]) -> list[AutoDocString]:
|
|
949
|
+
"""Provide documentation for the 'next_page' (pagination token) in API responses."""
|
|
1102
950
|
return [AutoDocString(case_mapping("next_token"))]
|
|
1103
951
|
|
|
1104
|
-
def documentation_pagination_next_page_example(
|
|
1105
|
-
|
|
1106
|
-
) -> Dict[str, str]:
|
|
1107
|
-
"""
|
|
1108
|
-
Provides an example value for the 'next_page' (pagination token) in API responses.
|
|
1109
|
-
"""
|
|
952
|
+
def documentation_pagination_next_page_example(self, case_mapping: Callable[[str], str]) -> dict[str, str]:
|
|
953
|
+
"""Provide an example value for the 'next_page' (pagination token) in API responses."""
|
|
1110
954
|
return {case_mapping("next_token"): ""}
|
|
1111
955
|
|
|
1112
956
|
def documentation_pagination_parameters(
|
|
1113
957
|
self, case_mapping: Callable[[str], str]
|
|
1114
|
-
) ->
|
|
1115
|
-
"""
|
|
1116
|
-
Provides documentation for pagination parameters in API requests.
|
|
1117
|
-
"""
|
|
958
|
+
) -> list[tuple[AutoDocString, str]]:
|
|
959
|
+
"""Provide documentation for pagination parameters in API requests."""
|
|
1118
960
|
return [
|
|
1119
961
|
(
|
|
1120
962
|
AutoDocString(case_mapping("next_token"), example=""),
|