localstack-core 4.7.1.dev49__py3-none-any.whl → 4.10.1.dev12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. localstack/aws/api/cloudformation/__init__.py +18 -4
  2. localstack/aws/api/cloudwatch/__init__.py +41 -1
  3. localstack/aws/api/config/__init__.py +4 -0
  4. localstack/aws/api/core.py +6 -2
  5. localstack/aws/api/dynamodb/__init__.py +30 -0
  6. localstack/aws/api/ec2/__init__.py +1522 -65
  7. localstack/aws/api/iam/__init__.py +7 -0
  8. localstack/aws/api/kinesis/__init__.py +19 -0
  9. localstack/aws/api/kms/__init__.py +6 -0
  10. localstack/aws/api/lambda_/__init__.py +13 -0
  11. localstack/aws/api/logs/__init__.py +15 -0
  12. localstack/aws/api/redshift/__init__.py +9 -3
  13. localstack/aws/api/route53/__init__.py +5 -0
  14. localstack/aws/api/s3/__init__.py +12 -0
  15. localstack/aws/api/s3control/__init__.py +54 -0
  16. localstack/aws/api/ssm/__init__.py +2 -0
  17. localstack/aws/api/transcribe/__init__.py +17 -0
  18. localstack/aws/client.py +7 -2
  19. localstack/aws/forwarder.py +52 -5
  20. localstack/aws/handlers/analytics.py +1 -1
  21. localstack/aws/handlers/internal_requests.py +6 -1
  22. localstack/aws/handlers/logging.py +12 -2
  23. localstack/aws/handlers/metric_handler.py +41 -1
  24. localstack/aws/handlers/service.py +40 -20
  25. localstack/aws/mocking.py +2 -2
  26. localstack/aws/patches.py +2 -2
  27. localstack/aws/protocol/parser.py +459 -32
  28. localstack/aws/protocol/serializer.py +689 -69
  29. localstack/aws/protocol/service_router.py +120 -20
  30. localstack/aws/protocol/validate.py +1 -1
  31. localstack/aws/scaffold.py +1 -1
  32. localstack/aws/skeleton.py +4 -2
  33. localstack/aws/spec-patches.json +58 -0
  34. localstack/aws/spec.py +37 -16
  35. localstack/cli/exceptions.py +1 -1
  36. localstack/cli/localstack.py +6 -6
  37. localstack/cli/lpm.py +3 -4
  38. localstack/cli/plugins.py +1 -1
  39. localstack/cli/profiles.py +1 -2
  40. localstack/config.py +25 -18
  41. localstack/constants.py +4 -29
  42. localstack/dev/kubernetes/__main__.py +130 -7
  43. localstack/dev/run/configurators.py +1 -4
  44. localstack/dev/run/paths.py +1 -1
  45. localstack/dns/plugins.py +5 -1
  46. localstack/dns/server.py +13 -4
  47. localstack/logging/format.py +3 -3
  48. localstack/packages/api.py +9 -8
  49. localstack/packages/core.py +2 -2
  50. localstack/packages/plugins.py +0 -8
  51. localstack/runtime/analytics.py +3 -0
  52. localstack/runtime/hooks.py +1 -1
  53. localstack/runtime/init.py +2 -2
  54. localstack/runtime/main.py +5 -5
  55. localstack/runtime/patches.py +2 -2
  56. localstack/services/apigateway/helpers.py +1 -4
  57. localstack/services/apigateway/legacy/helpers.py +7 -8
  58. localstack/services/apigateway/legacy/integration.py +4 -3
  59. localstack/services/apigateway/legacy/invocations.py +6 -5
  60. localstack/services/apigateway/legacy/provider.py +148 -68
  61. localstack/services/apigateway/legacy/templates.py +1 -1
  62. localstack/services/apigateway/next_gen/execute_api/handlers/method_request.py +7 -2
  63. localstack/services/apigateway/next_gen/execute_api/handlers/resource_router.py +1 -2
  64. localstack/services/apigateway/next_gen/execute_api/integrations/aws.py +3 -0
  65. localstack/services/apigateway/next_gen/execute_api/integrations/http.py +3 -3
  66. localstack/services/apigateway/next_gen/execute_api/template_mapping.py +2 -2
  67. localstack/services/apigateway/next_gen/execute_api/test_invoke.py +114 -9
  68. localstack/services/apigateway/next_gen/provider.py +5 -0
  69. localstack/services/apigateway/resource_providers/aws_apigateway_resource.py +1 -1
  70. localstack/services/cloudformation/api_utils.py +4 -8
  71. localstack/services/cloudformation/cfn_utils.py +1 -1
  72. localstack/services/cloudformation/engine/entities.py +14 -4
  73. localstack/services/cloudformation/engine/template_deployer.py +6 -4
  74. localstack/services/cloudformation/engine/transformers.py +6 -4
  75. localstack/services/cloudformation/engine/v2/change_set_model.py +201 -13
  76. localstack/services/cloudformation/engine/v2/change_set_model_describer.py +52 -3
  77. localstack/services/cloudformation/engine/v2/change_set_model_executor.py +117 -76
  78. localstack/services/cloudformation/engine/v2/change_set_model_preproc.py +205 -52
  79. localstack/services/cloudformation/engine/v2/change_set_model_transform.py +350 -116
  80. localstack/services/cloudformation/engine/v2/change_set_model_validator.py +56 -14
  81. localstack/services/cloudformation/engine/v2/change_set_model_visitor.py +1 -0
  82. localstack/services/cloudformation/engine/v2/resolving.py +7 -5
  83. localstack/services/cloudformation/engine/yaml_parser.py +9 -2
  84. localstack/services/cloudformation/provider.py +7 -5
  85. localstack/services/cloudformation/resource_provider.py +7 -1
  86. localstack/services/cloudformation/resources.py +24149 -0
  87. localstack/services/cloudformation/service_models.py +2 -2
  88. localstack/services/cloudformation/v2/entities.py +19 -9
  89. localstack/services/cloudformation/v2/provider.py +336 -106
  90. localstack/services/cloudformation/v2/types.py +13 -7
  91. localstack/services/cloudformation/v2/utils.py +4 -1
  92. localstack/services/cloudwatch/alarm_scheduler.py +4 -1
  93. localstack/services/cloudwatch/provider.py +18 -13
  94. localstack/services/cloudwatch/provider_v2.py +25 -28
  95. localstack/services/dynamodb/packages.py +2 -1
  96. localstack/services/dynamodb/provider.py +42 -0
  97. localstack/services/dynamodb/server.py +2 -2
  98. localstack/services/dynamodb/v2/provider.py +42 -0
  99. localstack/services/ecr/resource_providers/aws_ecr_repository.py +5 -2
  100. localstack/services/edge.py +1 -1
  101. localstack/services/es/provider.py +2 -2
  102. localstack/services/events/event_rule_engine.py +31 -13
  103. localstack/services/events/models.py +4 -5
  104. localstack/services/events/provider.py +17 -14
  105. localstack/services/events/target.py +17 -9
  106. localstack/services/events/v1/provider.py +5 -5
  107. localstack/services/firehose/provider.py +14 -4
  108. localstack/services/iam/provider.py +11 -116
  109. localstack/services/iam/resources/policy_simulator.py +133 -0
  110. localstack/services/kinesis/models.py +15 -2
  111. localstack/services/kinesis/provider.py +86 -3
  112. localstack/services/kms/provider.py +14 -5
  113. localstack/services/lambda_/api_utils.py +6 -3
  114. localstack/services/lambda_/invocation/docker_runtime_executor.py +1 -1
  115. localstack/services/lambda_/invocation/event_manager.py +1 -1
  116. localstack/services/lambda_/invocation/internal_sqs_queue.py +5 -9
  117. localstack/services/lambda_/invocation/lambda_models.py +10 -7
  118. localstack/services/lambda_/invocation/lambda_service.py +5 -1
  119. localstack/services/lambda_/packages.py +1 -1
  120. localstack/services/lambda_/provider.py +4 -3
  121. localstack/services/lambda_/provider_utils.py +1 -1
  122. localstack/services/logs/provider.py +36 -19
  123. localstack/services/moto.py +2 -1
  124. localstack/services/opensearch/cluster.py +15 -7
  125. localstack/services/opensearch/packages.py +26 -7
  126. localstack/services/opensearch/provider.py +8 -2
  127. localstack/services/opensearch/versions.py +56 -7
  128. localstack/services/plugins.py +11 -7
  129. localstack/services/providers.py +10 -2
  130. localstack/services/redshift/provider.py +0 -21
  131. localstack/services/s3/constants.py +5 -2
  132. localstack/services/s3/cors.py +4 -4
  133. localstack/services/s3/models.py +1 -1
  134. localstack/services/s3/notifications.py +55 -39
  135. localstack/services/s3/presigned_url.py +35 -54
  136. localstack/services/s3/provider.py +73 -15
  137. localstack/services/s3/utils.py +42 -22
  138. localstack/services/s3/validation.py +46 -32
  139. localstack/services/s3/website_hosting.py +4 -2
  140. localstack/services/ses/provider.py +18 -8
  141. localstack/services/sns/constants.py +7 -1
  142. localstack/services/sns/executor.py +9 -2
  143. localstack/services/sns/provider.py +8 -5
  144. localstack/services/sns/publisher.py +31 -16
  145. localstack/services/sns/v2/models.py +167 -0
  146. localstack/services/sns/v2/provider.py +867 -0
  147. localstack/services/sns/v2/utils.py +130 -0
  148. localstack/services/sqs/constants.py +1 -1
  149. localstack/services/sqs/developer_api.py +205 -0
  150. localstack/services/sqs/models.py +48 -5
  151. localstack/services/sqs/provider.py +38 -311
  152. localstack/services/sqs/query_api.py +6 -2
  153. localstack/services/sqs/utils.py +121 -2
  154. localstack/services/ssm/provider.py +1 -1
  155. localstack/services/stepfunctions/asl/component/intrinsic/member.py +1 -1
  156. localstack/services/stepfunctions/asl/component/state/state_choice/comparison/comparison.py +5 -11
  157. localstack/services/stepfunctions/asl/component/state/state_choice/state_choice.py +2 -2
  158. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/state_map.py +2 -2
  159. localstack/services/stepfunctions/asl/component/state/state_execution/state_parallel/state_parallel.py +1 -1
  160. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/state_task.py +2 -2
  161. localstack/services/stepfunctions/asl/component/state/state_fail/state_fail.py +1 -1
  162. localstack/services/stepfunctions/asl/component/state/state_pass/state_pass.py +2 -2
  163. localstack/services/stepfunctions/asl/component/state/state_succeed/state_succeed.py +1 -1
  164. localstack/services/stepfunctions/asl/component/state/state_wait/state_wait.py +1 -1
  165. localstack/services/stepfunctions/asl/eval/environment.py +1 -1
  166. localstack/services/stepfunctions/asl/jsonata/jsonata.py +1 -1
  167. localstack/services/stepfunctions/backend/execution.py +2 -1
  168. localstack/services/stores.py +1 -1
  169. localstack/services/transcribe/provider.py +6 -1
  170. localstack/state/codecs.py +61 -0
  171. localstack/state/core.py +11 -5
  172. localstack/state/pickle.py +10 -49
  173. localstack/testing/aws/cloudformation_utils.py +1 -1
  174. localstack/testing/pytest/cloudformation/fixtures.py +3 -3
  175. localstack/testing/pytest/cloudformation/transformers.py +0 -0
  176. localstack/testing/pytest/container.py +4 -5
  177. localstack/testing/pytest/fixtures.py +33 -31
  178. localstack/testing/pytest/in_memory_localstack.py +0 -4
  179. localstack/testing/pytest/marking.py +38 -11
  180. localstack/testing/pytest/stepfunctions/utils.py +4 -3
  181. localstack/testing/pytest/util.py +1 -1
  182. localstack/testing/pytest/validation_tracking.py +1 -2
  183. localstack/testing/snapshots/transformer_utility.py +6 -1
  184. localstack/utils/analytics/events.py +2 -2
  185. localstack/utils/analytics/metadata.py +6 -4
  186. localstack/utils/analytics/metrics/counter.py +8 -15
  187. localstack/utils/analytics/publisher.py +1 -2
  188. localstack/utils/analytics/service_providers.py +19 -0
  189. localstack/utils/analytics/service_request_aggregator.py +2 -2
  190. localstack/utils/archives.py +11 -11
  191. localstack/utils/asyncio.py +2 -2
  192. localstack/utils/aws/arns.py +24 -29
  193. localstack/utils/aws/aws_responses.py +8 -8
  194. localstack/utils/aws/aws_stack.py +2 -3
  195. localstack/utils/aws/dead_letter_queue.py +1 -5
  196. localstack/utils/aws/message_forwarding.py +1 -2
  197. localstack/utils/aws/request_context.py +4 -5
  198. localstack/utils/aws/resources.py +1 -1
  199. localstack/utils/aws/templating.py +1 -1
  200. localstack/utils/batch_policy.py +3 -3
  201. localstack/utils/bootstrap.py +21 -13
  202. localstack/utils/catalog/catalog.py +139 -0
  203. localstack/utils/catalog/catalog_loader.py +119 -0
  204. localstack/utils/catalog/common.py +58 -0
  205. localstack/utils/catalog/plugins.py +28 -0
  206. localstack/utils/cloudwatch/cloudwatch_util.py +5 -5
  207. localstack/utils/collections.py +7 -8
  208. localstack/utils/config_listener.py +1 -1
  209. localstack/utils/container_networking.py +2 -3
  210. localstack/utils/container_utils/container_client.py +135 -136
  211. localstack/utils/container_utils/docker_cmd_client.py +85 -69
  212. localstack/utils/container_utils/docker_sdk_client.py +69 -66
  213. localstack/utils/crypto.py +10 -10
  214. localstack/utils/diagnose.py +3 -4
  215. localstack/utils/docker_utils.py +9 -5
  216. localstack/utils/files.py +33 -13
  217. localstack/utils/functions.py +4 -3
  218. localstack/utils/http.py +11 -11
  219. localstack/utils/json.py +20 -6
  220. localstack/utils/kinesis/kinesis_connector.py +2 -1
  221. localstack/utils/net.py +15 -9
  222. localstack/utils/no_exit_argument_parser.py +2 -2
  223. localstack/utils/numbers.py +9 -2
  224. localstack/utils/objects.py +7 -6
  225. localstack/utils/patch.py +10 -3
  226. localstack/utils/run.py +12 -11
  227. localstack/utils/scheduler.py +11 -11
  228. localstack/utils/server/tcp_proxy.py +2 -2
  229. localstack/utils/serving.py +3 -4
  230. localstack/utils/strings.py +15 -16
  231. localstack/utils/sync.py +126 -1
  232. localstack/utils/tagging.py +8 -6
  233. localstack/utils/testutil.py +8 -8
  234. localstack/utils/threads.py +2 -2
  235. localstack/utils/time.py +12 -4
  236. localstack/utils/urls.py +1 -3
  237. localstack/utils/xray/traceid.py +1 -1
  238. localstack/version.py +16 -3
  239. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/METADATA +18 -14
  240. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/RECORD +248 -239
  241. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/entry_points.txt +8 -4
  242. localstack_core-4.10.1.dev12.dist-info/plux.json +1 -0
  243. localstack/packages/terraform.py +0 -46
  244. localstack/services/cloudformation/deploy.html +0 -144
  245. localstack/services/cloudformation/deploy_ui.py +0 -47
  246. localstack/services/cloudformation/plugins.py +0 -12
  247. localstack_core-4.7.1.dev49.dist-info/plux.json +0 -1
  248. {localstack_core-4.7.1.dev49.data → localstack_core-4.10.1.dev12.data}/scripts/localstack +0 -0
  249. {localstack_core-4.7.1.dev49.data → localstack_core-4.10.1.dev12.data}/scripts/localstack-supervisor +0 -0
  250. {localstack_core-4.7.1.dev49.data → localstack_core-4.10.1.dev12.data}/scripts/localstack.bat +0 -0
  251. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/WHEEL +0 -0
  252. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/licenses/LICENSE.txt +0 -0
  253. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/top_level.txt +0 -0
@@ -1,5 +1,3 @@
1
- import copy
2
- import hashlib
3
1
  import json
4
2
  import logging
5
3
  import re
@@ -8,15 +6,12 @@ import time
8
6
  from collections.abc import Iterable
9
7
  from concurrent.futures.thread import ThreadPoolExecutor
10
8
  from itertools import islice
11
- from typing import Literal
12
9
 
13
10
  from botocore.utils import InvalidArnException
14
- from moto.sqs.models import BINARY_TYPE_FIELD_INDEX, STRING_TYPE_FIELD_INDEX
15
- from moto.sqs.models import Message as MotoMessage
16
11
  from werkzeug import Request as WerkzeugRequest
17
12
 
18
13
  from localstack import config
19
- from localstack.aws.api import CommonServiceException, RequestContext, ServiceException
14
+ from localstack.aws.api import RequestContext, ServiceException
20
15
  from localstack.aws.api.sqs import (
21
16
  ActionNameList,
22
17
  AttributeNameList,
@@ -70,11 +65,8 @@ from localstack.aws.api.sqs import (
70
65
  Token,
71
66
  TooManyEntriesInBatchRequest,
72
67
  )
73
- from localstack.aws.protocol.parser import create_parser
74
- from localstack.aws.protocol.serializer import aws_response_serializer
75
68
  from localstack.aws.spec import load_service
76
69
  from localstack.config import SQS_DISABLE_MAX_NUMBER_OF_MESSAGE_LIMIT
77
- from localstack.http import Request, route
78
70
  from localstack.services.edge import ROUTER
79
71
  from localstack.services.plugins import ServiceLifecycleHook
80
72
  from localstack.services.sqs import constants as sqs_constants
@@ -84,6 +76,7 @@ from localstack.services.sqs.constants import (
84
76
  HEADER_LOCALSTACK_SQS_OVERRIDE_WAIT_TIME_SECONDS,
85
77
  MAX_RESULT_LIMIT,
86
78
  )
79
+ from localstack.services.sqs.developer_api import SqsDeveloperApi
87
80
  from localstack.services.sqs.exceptions import (
88
81
  InvalidParameterValueException,
89
82
  MissingRequiredParameterException,
@@ -97,8 +90,10 @@ from localstack.services.sqs.models import (
97
90
  SqsStore,
98
91
  StandardQueue,
99
92
  sqs_stores,
93
+ to_sqs_api_message,
100
94
  )
101
95
  from localstack.services.sqs.utils import (
96
+ create_message_attribute_hash,
102
97
  decode_move_task_handle,
103
98
  generate_message_id,
104
99
  is_fifo_queue,
@@ -107,7 +102,6 @@ from localstack.services.sqs.utils import (
107
102
  )
108
103
  from localstack.services.stores import AccountRegionBundle
109
104
  from localstack.utils.aws.arns import parse_arn
110
- from localstack.utils.aws.request_context import extract_region_from_headers
111
105
  from localstack.utils.bootstrap import is_api_enabled
112
106
  from localstack.utils.cloudwatch.cloudwatch_util import (
113
107
  SqsMetricBatchData,
@@ -127,13 +121,6 @@ MAX_NUMBER_OF_MESSAGES = 10
127
121
  _STORE_LOCK = threading.RLock()
128
122
 
129
123
 
130
- class InvalidAddress(ServiceException):
131
- code = "InvalidAddress"
132
- message = "The address https://queue.amazonaws.com/ is not valid for this endpoint."
133
- sender_fault = True
134
- status_code = 404
135
-
136
-
137
124
  def assert_queue_name(queue_name: str, fifo: bool = False):
138
125
  if queue_name.endswith(".fifo"):
139
126
  if not fifo:
@@ -397,26 +384,34 @@ class QueueUpdateWorker:
397
384
 
398
385
  def iter_queues(self) -> Iterable[SqsQueue]:
399
386
  for account_id, region, store in sqs_stores.iter_stores():
400
- for queue in store.queues.values():
401
- yield queue
387
+ yield from store.queues.values()
402
388
 
403
389
  def do_update_all_queues(self):
404
390
  for queue in self.iter_queues():
405
391
  try:
406
392
  queue.requeue_inflight_messages()
407
393
  except Exception:
408
- LOG.exception("error re-queueing inflight messages")
394
+ LOG.error(
395
+ "error re-queueing inflight messages",
396
+ exc_info=LOG.isEnabledFor(logging.DEBUG),
397
+ )
409
398
 
410
399
  try:
411
400
  queue.enqueue_delayed_messages()
412
401
  except Exception:
413
- LOG.exception("error enqueueing delayed messages")
402
+ LOG.error(
403
+ "error enqueueing delayed messages",
404
+ exc_info=LOG.isEnabledFor(logging.DEBUG),
405
+ )
414
406
 
415
407
  if config.SQS_ENABLE_MESSAGE_RETENTION_PERIOD:
416
408
  try:
417
409
  queue.remove_expired_messages()
418
410
  except Exception:
419
- LOG.exception("error removing expired messages")
411
+ LOG.error(
412
+ "error removing expired messages",
413
+ exc_info=LOG.isEnabledFor(logging.DEBUG),
414
+ )
420
415
 
421
416
  def start(self):
422
417
  with self.mutex:
@@ -632,163 +627,6 @@ def check_fifo_id(fifo_id: str | None, parameter: str):
632
627
  )
633
628
 
634
629
 
635
- def get_sqs_protocol(request: Request) -> Literal["query", "json"]:
636
- content_type = request.headers.get("Content-Type")
637
- return "json" if content_type == "application/x-amz-json-1.0" else "query"
638
-
639
-
640
- def sqs_auto_protocol_aws_response_serializer(service_name: str, operation: str):
641
- def _decorate(fn):
642
- def _proxy(*args, **kwargs):
643
- # extract request from function invocation (decorator can be used for methods as well as for functions).
644
- if len(args) > 0 and isinstance(args[0], WerkzeugRequest):
645
- # function
646
- request = args[0]
647
- elif len(args) > 1 and isinstance(args[1], WerkzeugRequest):
648
- # method (arg[0] == self)
649
- request = args[1]
650
- elif "request" in kwargs:
651
- request = kwargs["request"]
652
- else:
653
- raise ValueError(f"could not find Request in signature of function {fn}")
654
-
655
- protocol = get_sqs_protocol(request)
656
- return aws_response_serializer(service_name, operation, protocol)(fn)(*args, **kwargs)
657
-
658
- return _proxy
659
-
660
- return _decorate
661
-
662
-
663
- class SqsDeveloperEndpoints:
664
- """
665
- A set of SQS developer tool endpoints:
666
-
667
- - ``/_aws/sqs/messages``: list SQS messages without side effects, compatible with ``ReceiveMessage``.
668
- """
669
-
670
- def __init__(self, stores=None):
671
- self.stores = stores or sqs_stores
672
-
673
- @route("/_aws/sqs/messages", methods=["GET", "POST"])
674
- @sqs_auto_protocol_aws_response_serializer("sqs", "ReceiveMessage")
675
- def list_messages(self, request: Request) -> ReceiveMessageResult:
676
- """
677
- This endpoint expects a ``QueueUrl`` request parameter (either as query arg or form parameter), similar to
678
- the ``ReceiveMessage`` operation. It will parse the Queue URL generated by one of the SQS endpoint strategies.
679
- """
680
-
681
- if "x-amz-" in request.mimetype or "x-www-form-urlencoded" in request.mimetype:
682
- # only parse the request using a parser if it comes from an AWS client
683
- protocol = get_sqs_protocol(request)
684
- operation, service_request = create_parser(
685
- load_service("sqs", protocol=protocol)
686
- ).parse(request)
687
- if operation.name != "ReceiveMessage":
688
- raise CommonServiceException(
689
- "InvalidRequest", "This endpoint only accepts ReceiveMessage calls"
690
- )
691
- else:
692
- service_request = dict(request.values)
693
-
694
- if not service_request.get("QueueUrl"):
695
- raise QueueDoesNotExist()
696
-
697
- try:
698
- account_id, region, queue_name = parse_queue_url(service_request.get("QueueUrl"))
699
- except ValueError:
700
- LOG.exception(
701
- "Error while parsing Queue URL from request values: %s", service_request.get
702
- )
703
- raise InvalidAddress()
704
-
705
- if not region:
706
- region = extract_region_from_headers(request.headers)
707
-
708
- return self._get_and_serialize_messages(request, region, account_id, queue_name)
709
-
710
- @route("/_aws/sqs/messages/<region>/<account_id>/<queue_name>")
711
- @sqs_auto_protocol_aws_response_serializer("sqs", "ReceiveMessage")
712
- def list_messages_for_queue_url(
713
- self, request: Request, region: str, account_id: str, queue_name: str
714
- ) -> ReceiveMessageResult:
715
- """
716
- This endpoint extracts the region, account_id, and queue_name directly from the URL rather than requiring the
717
- QueueUrl as parameter.
718
- """
719
- return self._get_and_serialize_messages(request, region, account_id, queue_name)
720
-
721
- def _get_and_serialize_messages(
722
- self,
723
- request: Request,
724
- region: str,
725
- account_id: str,
726
- queue_name: str,
727
- ) -> ReceiveMessageResult:
728
- show_invisible = request.values.get("ShowInvisible", "").lower() in ["true", "1"]
729
- show_delayed = request.values.get("ShowDelayed", "").lower() in ["true", "1"]
730
-
731
- try:
732
- store = SqsProvider.get_store(account_id, region)
733
- queue = store.queues[queue_name]
734
- except KeyError:
735
- LOG.info(
736
- "no queue named %s in region %s and account %s", queue_name, region, account_id
737
- )
738
- raise QueueDoesNotExist()
739
-
740
- messages = self._collect_messages(
741
- queue, show_invisible=show_invisible, show_delayed=show_delayed
742
- )
743
-
744
- return ReceiveMessageResult(Messages=messages)
745
-
746
- def _collect_messages(
747
- self, queue: SqsQueue, show_invisible: bool = False, show_delayed: bool = False
748
- ) -> list[Message]:
749
- """
750
- Retrieves from a given SqsQueue all visible messages without causing any side effects (not setting any
751
- receive timestamps, receive counts, or visibility state).
752
-
753
- :param queue: the queue
754
- :param show_invisible: show invisible messages as well
755
- :param show_delayed: show delayed messages as well
756
- :return: a list of messages
757
- """
758
- receipt_handle = "SQS/BACKDOOR/ACCESS" # dummy receipt handle
759
-
760
- sqs_messages: list[SqsMessage] = []
761
-
762
- if show_invisible:
763
- sqs_messages.extend(queue.inflight)
764
-
765
- if isinstance(queue, StandardQueue):
766
- sqs_messages.extend(queue.visible.queue)
767
- elif isinstance(queue, FifoQueue):
768
- for message_group in queue.message_groups.values():
769
- for sqs_message in message_group.messages:
770
- sqs_messages.append(sqs_message)
771
- else:
772
- raise ValueError(f"unknown queue type {type(queue)}")
773
-
774
- if show_delayed:
775
- sqs_messages.extend(queue.delayed)
776
-
777
- messages = []
778
-
779
- for sqs_message in sqs_messages:
780
- message: Message = to_sqs_api_message(sqs_message, [QueueAttributeName.All], ["All"])
781
- # these are all non-standard fields so we squelch the linter
782
- if show_invisible:
783
- message["Attributes"]["IsVisible"] = str(sqs_message.is_visible).lower() # noqa
784
- if show_delayed:
785
- message["Attributes"]["IsDelayed"] = str(sqs_message.is_delayed).lower() # noqa
786
- messages.append(message)
787
- message["ReceiptHandle"] = receipt_handle
788
-
789
- return messages
790
-
791
-
792
630
  class SqsProvider(SqsApi, ServiceLifecycleHook):
793
631
  """
794
632
  LocalStack SQS Provider.
@@ -825,9 +663,26 @@ class SqsProvider(SqsApi, ServiceLifecycleHook):
825
663
  def get_store(account_id: str, region: str) -> SqsStore:
826
664
  return sqs_stores[account_id][region]
827
665
 
666
+ def on_after_init(self):
667
+ # this configuration increases the processing power for Query protocol requests, which are form-encoded and by
668
+ # default are limited to 500kb payload size by Werkzeug. we make sure we only *increase* the limit if it's
669
+ # already set, and if it's already set to unlimited we leave it.
670
+ from rolo import Request as RoloRequest
671
+
672
+ # needed for the webserver integration (webservers create Werkzeug request objects)
673
+ if WerkzeugRequest.max_form_memory_size is not None:
674
+ WerkzeugRequest.max_form_memory_size = max(
675
+ WerkzeugRequest.max_form_memory_size, sqs_constants.DEFAULT_MAXIMUM_MESSAGE_SIZE * 2
676
+ )
677
+ # needed for internal/proxy requests (which create rolo request objects)
678
+ if RoloRequest.max_form_memory_size is not None:
679
+ RoloRequest.max_form_memory_size = max(
680
+ RoloRequest.max_form_memory_size, sqs_constants.DEFAULT_MAXIMUM_MESSAGE_SIZE * 2
681
+ )
682
+
828
683
  def on_before_start(self):
829
684
  query_api.register(ROUTER)
830
- self._router_rules = ROUTER.add(SqsDeveloperEndpoints())
685
+ self._router_rules = ROUTER.add(SqsDeveloperApi())
831
686
  self._queue_update_worker.start()
832
687
  self._start_cloudwatch_metrics_reporting()
833
688
 
@@ -1122,7 +977,7 @@ class SqsProvider(SqsApi, ServiceLifecycleHook):
1122
977
  MD5OfMessageBody=message["MD5OfBody"],
1123
978
  MD5OfMessageAttributes=message.get("MD5OfMessageAttributes"),
1124
979
  SequenceNumber=queue_item.sequence_number,
1125
- MD5OfMessageSystemAttributes=_create_message_attribute_hash(message_system_attributes),
980
+ MD5OfMessageSystemAttributes=create_message_attribute_hash(message_system_attributes),
1126
981
  )
1127
982
 
1128
983
  def send_message_batch(
@@ -1168,7 +1023,7 @@ class SqsProvider(SqsApi, ServiceLifecycleHook):
1168
1023
  MessageId=message.get("MessageId"),
1169
1024
  MD5OfMessageBody=message.get("MD5OfBody"),
1170
1025
  MD5OfMessageAttributes=message.get("MD5OfMessageAttributes"),
1171
- MD5OfMessageSystemAttributes=_create_message_attribute_hash(
1026
+ MD5OfMessageSystemAttributes=create_message_attribute_hash(
1172
1027
  message.get("message_system_attributes")
1173
1028
  ),
1174
1029
  SequenceNumber=queue_item.sequence_number,
@@ -1222,7 +1077,7 @@ class SqsProvider(SqsApi, ServiceLifecycleHook):
1222
1077
  MD5OfBody=md5(message_body),
1223
1078
  Body=message_body,
1224
1079
  Attributes=self._create_message_attributes(context, message_system_attributes),
1225
- MD5OfMessageAttributes=_create_message_attribute_hash(message_attributes),
1080
+ MD5OfMessageAttributes=create_message_attribute_hash(message_attributes),
1226
1081
  MessageAttributes=message_attributes,
1227
1082
  )
1228
1083
  if self._cloudwatch_dispatcher:
@@ -1782,34 +1637,6 @@ class SqsProvider(SqsApi, ServiceLifecycleHook):
1782
1637
  self._cloudwatch_dispatcher.shutdown()
1783
1638
 
1784
1639
 
1785
- # Method from moto's attribute_md5 of moto/sqs/models.py, separated from the Message Object
1786
- def _create_message_attribute_hash(message_attributes) -> str | None:
1787
- # To avoid the need to check for dict conformity everytime we invoke this function
1788
- if not isinstance(message_attributes, dict):
1789
- return
1790
- hash = hashlib.md5()
1791
-
1792
- for attrName in sorted(message_attributes.keys()):
1793
- attr_value = message_attributes[attrName]
1794
- # Encode name
1795
- MotoMessage.update_binary_length_and_value(hash, MotoMessage.utf8(attrName))
1796
- # Encode data type
1797
- MotoMessage.update_binary_length_and_value(hash, MotoMessage.utf8(attr_value["DataType"]))
1798
- # Encode transport type and value
1799
- if attr_value.get("StringValue"):
1800
- hash.update(bytearray([STRING_TYPE_FIELD_INDEX]))
1801
- MotoMessage.update_binary_length_and_value(
1802
- hash, MotoMessage.utf8(attr_value.get("StringValue"))
1803
- )
1804
- elif attr_value.get("BinaryValue"):
1805
- hash.update(bytearray([BINARY_TYPE_FIELD_INDEX]))
1806
- decoded_binary_value = attr_value.get("BinaryValue")
1807
- MotoMessage.update_binary_length_and_value(hash, decoded_binary_value)
1808
- # string_list_value, binary_list_value type is not implemented, reserved for the future use.
1809
- # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_MessageAttributeValue.html
1810
- return hash.hexdigest()
1811
-
1812
-
1813
1640
  def resolve_queue_location(
1814
1641
  context: RequestContext, queue_name: str | None = None, queue_url: str | None = None
1815
1642
  ) -> tuple[str, str | None, str]:
@@ -1834,106 +1661,6 @@ def resolve_queue_location(
1834
1661
  return context.account_id, context.region, queue_name
1835
1662
 
1836
1663
 
1837
- def to_sqs_api_message(
1838
- standard_message: SqsMessage,
1839
- attribute_names: AttributeNameList = None,
1840
- message_attribute_names: MessageAttributeNameList = None,
1841
- ) -> Message:
1842
- """
1843
- Utility function to convert an SQS message from LocalStack's internal representation to the AWS API
1844
- concept 'Message', which is the format returned by the ``ReceiveMessage`` operation.
1845
-
1846
- :param standard_message: A LocalStack SQS message
1847
- :param attribute_names: the attribute name list to filter
1848
- :param message_attribute_names: the message attribute names to filter
1849
- :return: a copy of the original Message with updated message attributes and MD5 attribute hash sums
1850
- """
1851
- # prepare message for receiver
1852
- message = copy.deepcopy(standard_message.message)
1853
-
1854
- # update system attributes of the message copy
1855
- message["Attributes"][MessageSystemAttributeName.ApproximateFirstReceiveTimestamp] = str(
1856
- int((standard_message.first_received or 0) * 1000)
1857
- )
1858
-
1859
- # filter attributes for receiver
1860
- message_filter_attributes(message, attribute_names)
1861
- message_filter_message_attributes(message, message_attribute_names)
1862
- if message.get("MessageAttributes"):
1863
- message["MD5OfMessageAttributes"] = _create_message_attribute_hash(
1864
- message["MessageAttributes"]
1865
- )
1866
- else:
1867
- # delete the value that was computed when creating the message
1868
- message.pop("MD5OfMessageAttributes", None)
1869
- return message
1870
-
1871
-
1872
- def message_filter_attributes(message: Message, names: AttributeNameList | None):
1873
- """
1874
- Utility function filter from the given message (in-place) the system attributes from the given list. It will
1875
- apply all rules according to:
1876
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Client.receive_message.
1877
-
1878
- :param message: The message to filter (it will be modified)
1879
- :param names: the attributes names/filters
1880
- """
1881
- if "Attributes" not in message:
1882
- return
1883
-
1884
- if not names:
1885
- del message["Attributes"]
1886
- return
1887
-
1888
- if QueueAttributeName.All in names:
1889
- return
1890
-
1891
- for k in list(message["Attributes"].keys()):
1892
- if k not in names:
1893
- del message["Attributes"][k]
1894
-
1895
-
1896
- def message_filter_message_attributes(message: Message, names: MessageAttributeNameList | None):
1897
- """
1898
- Utility function filter from the given message (in-place) the message attributes from the given list. It will
1899
- apply all rules according to:
1900
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Client.receive_message.
1901
-
1902
- :param message: The message to filter (it will be modified)
1903
- :param names: the attributes names/filters (can be 'All', '.*', '*' or prefix filters like 'Foo.*')
1904
- """
1905
- if not message.get("MessageAttributes"):
1906
- return
1907
-
1908
- if not names:
1909
- del message["MessageAttributes"]
1910
- return
1911
-
1912
- if "All" in names or ".*" in names or "*" in names:
1913
- return
1914
-
1915
- attributes = message["MessageAttributes"]
1916
- matched = []
1917
-
1918
- keys = [name for name in names if ".*" not in name]
1919
- prefixes = [name.split(".*")[0] for name in names if ".*" in name]
1920
-
1921
- # match prefix filters
1922
- for k in attributes:
1923
- if k in keys:
1924
- matched.append(k)
1925
- continue
1926
-
1927
- for prefix in prefixes:
1928
- if k.startswith(prefix):
1929
- matched.append(k)
1930
- break
1931
- if matched:
1932
- message["MessageAttributes"] = {k: attributes[k] for k in matched}
1933
- else:
1934
- message.pop("MessageAttributes")
1935
-
1936
-
1937
1664
  def extract_message_count_from_headers(context: RequestContext) -> int | None:
1938
1665
  if override := context.request.headers.get(
1939
1666
  HEADER_LOCALSTACK_SQS_OVERRIDE_MESSAGE_COUNT, default=None, type=int
@@ -35,7 +35,7 @@ LOG = logging.getLogger(__name__)
35
35
 
36
36
  service = load_service("sqs-query")
37
37
  parser = create_parser(service)
38
- serializer = create_serializer(service)
38
+ serializer = create_serializer(service, "query")
39
39
 
40
40
 
41
41
  @route(
@@ -166,7 +166,11 @@ def handle_request(request: Request, region: str) -> Response:
166
166
  op = service.operation_model(service.operation_names[0])
167
167
  return serializer.serialize_error_to_response(e, op, request.headers, request_id)
168
168
  except Exception as e:
169
- LOG.exception("exception")
169
+ LOG.error(
170
+ "Internal Server exception while executing SQS Query operation: '%s'",
171
+ e,
172
+ exc_info=LOG.isEnabledFor(logging.DEBUG),
173
+ )
170
174
  op = service.operation_model(service.operation_names[0])
171
175
  return serializer.serialize_error_to_response(
172
176
  CommonServiceException(
@@ -1,12 +1,20 @@
1
1
  import base64
2
+ import hashlib
2
3
  import itertools
3
4
  import json
4
5
  import re
6
+ import struct
5
7
  import time
6
- from typing import Literal, NamedTuple
8
+ from typing import Any, Literal, NamedTuple
7
9
  from urllib.parse import urlparse
8
10
 
9
- from localstack.aws.api.sqs import QueueAttributeName, ReceiptHandleIsInvalid
11
+ from localstack.aws.api.sqs import (
12
+ AttributeNameList,
13
+ Message,
14
+ MessageAttributeNameList,
15
+ QueueAttributeName,
16
+ ReceiptHandleIsInvalid,
17
+ )
10
18
  from localstack.services.sqs.constants import (
11
19
  DOMAIN_STRATEGY_URL_REGEX,
12
20
  LEGACY_STRATEGY_URL_REGEX,
@@ -22,6 +30,11 @@ DOMAIN_ENDPOINT = re.compile(DOMAIN_STRATEGY_URL_REGEX)
22
30
  PATH_ENDPOINT = re.compile(PATH_STRATEGY_URL_REGEX)
23
31
  LEGACY_ENDPOINT = re.compile(LEGACY_STRATEGY_URL_REGEX)
24
32
 
33
+ STRING_TYPE_FIELD_INDEX = 1
34
+ BINARY_TYPE_FIELD_INDEX = 2
35
+ STRING_LIST_TYPE_FIELD_INDEX = 3
36
+ BINARY_LIST_TYPE_FIELD_INDEX = 4
37
+
25
38
 
26
39
  def is_sqs_queue_url(url: str) -> bool:
27
40
  return any(
@@ -184,3 +197,109 @@ def global_message_sequence():
184
197
 
185
198
  def generate_message_id():
186
199
  return long_uid()
200
+
201
+
202
+ def message_filter_attributes(message: Message, names: AttributeNameList | None):
203
+ """
204
+ Utility function filter from the given message (in-place) the system attributes from the given list. It will
205
+ apply all rules according to:
206
+ https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Client.receive_message.
207
+
208
+ :param message: The message to filter (it will be modified)
209
+ :param names: the attributes names/filters
210
+ """
211
+ if "Attributes" not in message:
212
+ return
213
+
214
+ if not names:
215
+ del message["Attributes"]
216
+ return
217
+
218
+ if QueueAttributeName.All in names:
219
+ return
220
+
221
+ for k in list(message["Attributes"].keys()):
222
+ if k not in names:
223
+ del message["Attributes"][k]
224
+
225
+
226
+ def message_filter_message_attributes(message: Message, names: MessageAttributeNameList | None):
227
+ """
228
+ Utility function filter from the given message (in-place) the message attributes from the given list. It will
229
+ apply all rules according to:
230
+ https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sqs.html#SQS.Client.receive_message.
231
+
232
+ :param message: The message to filter (it will be modified)
233
+ :param names: the attributes names/filters (can be 'All', '.*', '*' or prefix filters like 'Foo.*')
234
+ """
235
+ if not message.get("MessageAttributes"):
236
+ return
237
+
238
+ if not names:
239
+ del message["MessageAttributes"]
240
+ return
241
+
242
+ if "All" in names or ".*" in names or "*" in names:
243
+ return
244
+
245
+ attributes = message["MessageAttributes"]
246
+ matched = []
247
+
248
+ keys = [name for name in names if ".*" not in name]
249
+ prefixes = [name.split(".*")[0] for name in names if ".*" in name]
250
+
251
+ # match prefix filters
252
+ for k in attributes:
253
+ if k in keys:
254
+ matched.append(k)
255
+ continue
256
+
257
+ for prefix in prefixes:
258
+ if k.startswith(prefix):
259
+ matched.append(k)
260
+ break
261
+ if matched:
262
+ message["MessageAttributes"] = {k: attributes[k] for k in matched}
263
+ else:
264
+ message.pop("MessageAttributes")
265
+
266
+
267
+ def _utf8(value: Any) -> bytes: # type: ignore[misc]
268
+ if isinstance(value, str):
269
+ return value.encode("utf-8")
270
+ return value
271
+
272
+
273
+ def _update_binary_length_and_value(md5: Any, value: bytes) -> None: # type: ignore[misc]
274
+ length_bytes = struct.pack("!I".encode("ascii"), len(value))
275
+ md5.update(length_bytes)
276
+ md5.update(value)
277
+
278
+
279
+ def create_message_attribute_hash(message_attributes) -> str | None:
280
+ """
281
+ Method from moto's attribute_md5 of moto/sqs/models.py, separated from the Message Object.
282
+ """
283
+ # To avoid the need to check for dict conformity everytime we invoke this function
284
+ if not isinstance(message_attributes, dict):
285
+ return
286
+
287
+ hash = hashlib.md5()
288
+
289
+ for attrName in sorted(message_attributes.keys()):
290
+ attr_value = message_attributes[attrName]
291
+ # Encode name
292
+ _update_binary_length_and_value(hash, _utf8(attrName))
293
+ # Encode data type
294
+ _update_binary_length_and_value(hash, _utf8(attr_value["DataType"]))
295
+ # Encode transport type and value
296
+ if attr_value.get("StringValue"):
297
+ hash.update(bytearray([STRING_TYPE_FIELD_INDEX]))
298
+ _update_binary_length_and_value(hash, _utf8(attr_value.get("StringValue")))
299
+ elif attr_value.get("BinaryValue"):
300
+ hash.update(bytearray([BINARY_TYPE_FIELD_INDEX]))
301
+ decoded_binary_value = attr_value.get("BinaryValue")
302
+ _update_binary_length_and_value(hash, decoded_binary_value)
303
+ # string_list_value, binary_list_value type is not implemented, reserved for the future use.
304
+ # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_MessageAttributeValue.html
305
+ return hash.hexdigest()
@@ -366,7 +366,7 @@ class SsmProvider(SsmApi, ABC):
366
366
  param_name = param_name.strip("/")
367
367
  param_name = param_name.replace("//", "/")
368
368
  if "/" in param_name:
369
- param_name = "/%s" % param_name
369
+ param_name = f"/{param_name}"
370
370
  return param_name
371
371
 
372
372
  @staticmethod
@@ -13,4 +13,4 @@ class IdentifiedMember(Member):
13
13
 
14
14
  class DollarMember(IdentifiedMember):
15
15
  def __init__(self):
16
- super(DollarMember, self).__init__(identifier="$")
16
+ super().__init__(identifier="$")