localstack-core 4.7.1.dev49__py3-none-any.whl → 4.10.1.dev12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. localstack/aws/api/cloudformation/__init__.py +18 -4
  2. localstack/aws/api/cloudwatch/__init__.py +41 -1
  3. localstack/aws/api/config/__init__.py +4 -0
  4. localstack/aws/api/core.py +6 -2
  5. localstack/aws/api/dynamodb/__init__.py +30 -0
  6. localstack/aws/api/ec2/__init__.py +1522 -65
  7. localstack/aws/api/iam/__init__.py +7 -0
  8. localstack/aws/api/kinesis/__init__.py +19 -0
  9. localstack/aws/api/kms/__init__.py +6 -0
  10. localstack/aws/api/lambda_/__init__.py +13 -0
  11. localstack/aws/api/logs/__init__.py +15 -0
  12. localstack/aws/api/redshift/__init__.py +9 -3
  13. localstack/aws/api/route53/__init__.py +5 -0
  14. localstack/aws/api/s3/__init__.py +12 -0
  15. localstack/aws/api/s3control/__init__.py +54 -0
  16. localstack/aws/api/ssm/__init__.py +2 -0
  17. localstack/aws/api/transcribe/__init__.py +17 -0
  18. localstack/aws/client.py +7 -2
  19. localstack/aws/forwarder.py +52 -5
  20. localstack/aws/handlers/analytics.py +1 -1
  21. localstack/aws/handlers/internal_requests.py +6 -1
  22. localstack/aws/handlers/logging.py +12 -2
  23. localstack/aws/handlers/metric_handler.py +41 -1
  24. localstack/aws/handlers/service.py +40 -20
  25. localstack/aws/mocking.py +2 -2
  26. localstack/aws/patches.py +2 -2
  27. localstack/aws/protocol/parser.py +459 -32
  28. localstack/aws/protocol/serializer.py +689 -69
  29. localstack/aws/protocol/service_router.py +120 -20
  30. localstack/aws/protocol/validate.py +1 -1
  31. localstack/aws/scaffold.py +1 -1
  32. localstack/aws/skeleton.py +4 -2
  33. localstack/aws/spec-patches.json +58 -0
  34. localstack/aws/spec.py +37 -16
  35. localstack/cli/exceptions.py +1 -1
  36. localstack/cli/localstack.py +6 -6
  37. localstack/cli/lpm.py +3 -4
  38. localstack/cli/plugins.py +1 -1
  39. localstack/cli/profiles.py +1 -2
  40. localstack/config.py +25 -18
  41. localstack/constants.py +4 -29
  42. localstack/dev/kubernetes/__main__.py +130 -7
  43. localstack/dev/run/configurators.py +1 -4
  44. localstack/dev/run/paths.py +1 -1
  45. localstack/dns/plugins.py +5 -1
  46. localstack/dns/server.py +13 -4
  47. localstack/logging/format.py +3 -3
  48. localstack/packages/api.py +9 -8
  49. localstack/packages/core.py +2 -2
  50. localstack/packages/plugins.py +0 -8
  51. localstack/runtime/analytics.py +3 -0
  52. localstack/runtime/hooks.py +1 -1
  53. localstack/runtime/init.py +2 -2
  54. localstack/runtime/main.py +5 -5
  55. localstack/runtime/patches.py +2 -2
  56. localstack/services/apigateway/helpers.py +1 -4
  57. localstack/services/apigateway/legacy/helpers.py +7 -8
  58. localstack/services/apigateway/legacy/integration.py +4 -3
  59. localstack/services/apigateway/legacy/invocations.py +6 -5
  60. localstack/services/apigateway/legacy/provider.py +148 -68
  61. localstack/services/apigateway/legacy/templates.py +1 -1
  62. localstack/services/apigateway/next_gen/execute_api/handlers/method_request.py +7 -2
  63. localstack/services/apigateway/next_gen/execute_api/handlers/resource_router.py +1 -2
  64. localstack/services/apigateway/next_gen/execute_api/integrations/aws.py +3 -0
  65. localstack/services/apigateway/next_gen/execute_api/integrations/http.py +3 -3
  66. localstack/services/apigateway/next_gen/execute_api/template_mapping.py +2 -2
  67. localstack/services/apigateway/next_gen/execute_api/test_invoke.py +114 -9
  68. localstack/services/apigateway/next_gen/provider.py +5 -0
  69. localstack/services/apigateway/resource_providers/aws_apigateway_resource.py +1 -1
  70. localstack/services/cloudformation/api_utils.py +4 -8
  71. localstack/services/cloudformation/cfn_utils.py +1 -1
  72. localstack/services/cloudformation/engine/entities.py +14 -4
  73. localstack/services/cloudformation/engine/template_deployer.py +6 -4
  74. localstack/services/cloudformation/engine/transformers.py +6 -4
  75. localstack/services/cloudformation/engine/v2/change_set_model.py +201 -13
  76. localstack/services/cloudformation/engine/v2/change_set_model_describer.py +52 -3
  77. localstack/services/cloudformation/engine/v2/change_set_model_executor.py +117 -76
  78. localstack/services/cloudformation/engine/v2/change_set_model_preproc.py +205 -52
  79. localstack/services/cloudformation/engine/v2/change_set_model_transform.py +350 -116
  80. localstack/services/cloudformation/engine/v2/change_set_model_validator.py +56 -14
  81. localstack/services/cloudformation/engine/v2/change_set_model_visitor.py +1 -0
  82. localstack/services/cloudformation/engine/v2/resolving.py +7 -5
  83. localstack/services/cloudformation/engine/yaml_parser.py +9 -2
  84. localstack/services/cloudformation/provider.py +7 -5
  85. localstack/services/cloudformation/resource_provider.py +7 -1
  86. localstack/services/cloudformation/resources.py +24149 -0
  87. localstack/services/cloudformation/service_models.py +2 -2
  88. localstack/services/cloudformation/v2/entities.py +19 -9
  89. localstack/services/cloudformation/v2/provider.py +336 -106
  90. localstack/services/cloudformation/v2/types.py +13 -7
  91. localstack/services/cloudformation/v2/utils.py +4 -1
  92. localstack/services/cloudwatch/alarm_scheduler.py +4 -1
  93. localstack/services/cloudwatch/provider.py +18 -13
  94. localstack/services/cloudwatch/provider_v2.py +25 -28
  95. localstack/services/dynamodb/packages.py +2 -1
  96. localstack/services/dynamodb/provider.py +42 -0
  97. localstack/services/dynamodb/server.py +2 -2
  98. localstack/services/dynamodb/v2/provider.py +42 -0
  99. localstack/services/ecr/resource_providers/aws_ecr_repository.py +5 -2
  100. localstack/services/edge.py +1 -1
  101. localstack/services/es/provider.py +2 -2
  102. localstack/services/events/event_rule_engine.py +31 -13
  103. localstack/services/events/models.py +4 -5
  104. localstack/services/events/provider.py +17 -14
  105. localstack/services/events/target.py +17 -9
  106. localstack/services/events/v1/provider.py +5 -5
  107. localstack/services/firehose/provider.py +14 -4
  108. localstack/services/iam/provider.py +11 -116
  109. localstack/services/iam/resources/policy_simulator.py +133 -0
  110. localstack/services/kinesis/models.py +15 -2
  111. localstack/services/kinesis/provider.py +86 -3
  112. localstack/services/kms/provider.py +14 -5
  113. localstack/services/lambda_/api_utils.py +6 -3
  114. localstack/services/lambda_/invocation/docker_runtime_executor.py +1 -1
  115. localstack/services/lambda_/invocation/event_manager.py +1 -1
  116. localstack/services/lambda_/invocation/internal_sqs_queue.py +5 -9
  117. localstack/services/lambda_/invocation/lambda_models.py +10 -7
  118. localstack/services/lambda_/invocation/lambda_service.py +5 -1
  119. localstack/services/lambda_/packages.py +1 -1
  120. localstack/services/lambda_/provider.py +4 -3
  121. localstack/services/lambda_/provider_utils.py +1 -1
  122. localstack/services/logs/provider.py +36 -19
  123. localstack/services/moto.py +2 -1
  124. localstack/services/opensearch/cluster.py +15 -7
  125. localstack/services/opensearch/packages.py +26 -7
  126. localstack/services/opensearch/provider.py +8 -2
  127. localstack/services/opensearch/versions.py +56 -7
  128. localstack/services/plugins.py +11 -7
  129. localstack/services/providers.py +10 -2
  130. localstack/services/redshift/provider.py +0 -21
  131. localstack/services/s3/constants.py +5 -2
  132. localstack/services/s3/cors.py +4 -4
  133. localstack/services/s3/models.py +1 -1
  134. localstack/services/s3/notifications.py +55 -39
  135. localstack/services/s3/presigned_url.py +35 -54
  136. localstack/services/s3/provider.py +73 -15
  137. localstack/services/s3/utils.py +42 -22
  138. localstack/services/s3/validation.py +46 -32
  139. localstack/services/s3/website_hosting.py +4 -2
  140. localstack/services/ses/provider.py +18 -8
  141. localstack/services/sns/constants.py +7 -1
  142. localstack/services/sns/executor.py +9 -2
  143. localstack/services/sns/provider.py +8 -5
  144. localstack/services/sns/publisher.py +31 -16
  145. localstack/services/sns/v2/models.py +167 -0
  146. localstack/services/sns/v2/provider.py +867 -0
  147. localstack/services/sns/v2/utils.py +130 -0
  148. localstack/services/sqs/constants.py +1 -1
  149. localstack/services/sqs/developer_api.py +205 -0
  150. localstack/services/sqs/models.py +48 -5
  151. localstack/services/sqs/provider.py +38 -311
  152. localstack/services/sqs/query_api.py +6 -2
  153. localstack/services/sqs/utils.py +121 -2
  154. localstack/services/ssm/provider.py +1 -1
  155. localstack/services/stepfunctions/asl/component/intrinsic/member.py +1 -1
  156. localstack/services/stepfunctions/asl/component/state/state_choice/comparison/comparison.py +5 -11
  157. localstack/services/stepfunctions/asl/component/state/state_choice/state_choice.py +2 -2
  158. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/state_map.py +2 -2
  159. localstack/services/stepfunctions/asl/component/state/state_execution/state_parallel/state_parallel.py +1 -1
  160. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/state_task.py +2 -2
  161. localstack/services/stepfunctions/asl/component/state/state_fail/state_fail.py +1 -1
  162. localstack/services/stepfunctions/asl/component/state/state_pass/state_pass.py +2 -2
  163. localstack/services/stepfunctions/asl/component/state/state_succeed/state_succeed.py +1 -1
  164. localstack/services/stepfunctions/asl/component/state/state_wait/state_wait.py +1 -1
  165. localstack/services/stepfunctions/asl/eval/environment.py +1 -1
  166. localstack/services/stepfunctions/asl/jsonata/jsonata.py +1 -1
  167. localstack/services/stepfunctions/backend/execution.py +2 -1
  168. localstack/services/stores.py +1 -1
  169. localstack/services/transcribe/provider.py +6 -1
  170. localstack/state/codecs.py +61 -0
  171. localstack/state/core.py +11 -5
  172. localstack/state/pickle.py +10 -49
  173. localstack/testing/aws/cloudformation_utils.py +1 -1
  174. localstack/testing/pytest/cloudformation/fixtures.py +3 -3
  175. localstack/testing/pytest/cloudformation/transformers.py +0 -0
  176. localstack/testing/pytest/container.py +4 -5
  177. localstack/testing/pytest/fixtures.py +33 -31
  178. localstack/testing/pytest/in_memory_localstack.py +0 -4
  179. localstack/testing/pytest/marking.py +38 -11
  180. localstack/testing/pytest/stepfunctions/utils.py +4 -3
  181. localstack/testing/pytest/util.py +1 -1
  182. localstack/testing/pytest/validation_tracking.py +1 -2
  183. localstack/testing/snapshots/transformer_utility.py +6 -1
  184. localstack/utils/analytics/events.py +2 -2
  185. localstack/utils/analytics/metadata.py +6 -4
  186. localstack/utils/analytics/metrics/counter.py +8 -15
  187. localstack/utils/analytics/publisher.py +1 -2
  188. localstack/utils/analytics/service_providers.py +19 -0
  189. localstack/utils/analytics/service_request_aggregator.py +2 -2
  190. localstack/utils/archives.py +11 -11
  191. localstack/utils/asyncio.py +2 -2
  192. localstack/utils/aws/arns.py +24 -29
  193. localstack/utils/aws/aws_responses.py +8 -8
  194. localstack/utils/aws/aws_stack.py +2 -3
  195. localstack/utils/aws/dead_letter_queue.py +1 -5
  196. localstack/utils/aws/message_forwarding.py +1 -2
  197. localstack/utils/aws/request_context.py +4 -5
  198. localstack/utils/aws/resources.py +1 -1
  199. localstack/utils/aws/templating.py +1 -1
  200. localstack/utils/batch_policy.py +3 -3
  201. localstack/utils/bootstrap.py +21 -13
  202. localstack/utils/catalog/catalog.py +139 -0
  203. localstack/utils/catalog/catalog_loader.py +119 -0
  204. localstack/utils/catalog/common.py +58 -0
  205. localstack/utils/catalog/plugins.py +28 -0
  206. localstack/utils/cloudwatch/cloudwatch_util.py +5 -5
  207. localstack/utils/collections.py +7 -8
  208. localstack/utils/config_listener.py +1 -1
  209. localstack/utils/container_networking.py +2 -3
  210. localstack/utils/container_utils/container_client.py +135 -136
  211. localstack/utils/container_utils/docker_cmd_client.py +85 -69
  212. localstack/utils/container_utils/docker_sdk_client.py +69 -66
  213. localstack/utils/crypto.py +10 -10
  214. localstack/utils/diagnose.py +3 -4
  215. localstack/utils/docker_utils.py +9 -5
  216. localstack/utils/files.py +33 -13
  217. localstack/utils/functions.py +4 -3
  218. localstack/utils/http.py +11 -11
  219. localstack/utils/json.py +20 -6
  220. localstack/utils/kinesis/kinesis_connector.py +2 -1
  221. localstack/utils/net.py +15 -9
  222. localstack/utils/no_exit_argument_parser.py +2 -2
  223. localstack/utils/numbers.py +9 -2
  224. localstack/utils/objects.py +7 -6
  225. localstack/utils/patch.py +10 -3
  226. localstack/utils/run.py +12 -11
  227. localstack/utils/scheduler.py +11 -11
  228. localstack/utils/server/tcp_proxy.py +2 -2
  229. localstack/utils/serving.py +3 -4
  230. localstack/utils/strings.py +15 -16
  231. localstack/utils/sync.py +126 -1
  232. localstack/utils/tagging.py +8 -6
  233. localstack/utils/testutil.py +8 -8
  234. localstack/utils/threads.py +2 -2
  235. localstack/utils/time.py +12 -4
  236. localstack/utils/urls.py +1 -3
  237. localstack/utils/xray/traceid.py +1 -1
  238. localstack/version.py +16 -3
  239. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/METADATA +18 -14
  240. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/RECORD +248 -239
  241. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/entry_points.txt +8 -4
  242. localstack_core-4.10.1.dev12.dist-info/plux.json +1 -0
  243. localstack/packages/terraform.py +0 -46
  244. localstack/services/cloudformation/deploy.html +0 -144
  245. localstack/services/cloudformation/deploy_ui.py +0 -47
  246. localstack/services/cloudformation/plugins.py +0 -12
  247. localstack_core-4.7.1.dev49.dist-info/plux.json +0 -1
  248. {localstack_core-4.7.1.dev49.data → localstack_core-4.10.1.dev12.data}/scripts/localstack +0 -0
  249. {localstack_core-4.7.1.dev49.data → localstack_core-4.10.1.dev12.data}/scripts/localstack-supervisor +0 -0
  250. {localstack_core-4.7.1.dev49.data → localstack_core-4.10.1.dev12.data}/scripts/localstack.bat +0 -0
  251. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/WHEEL +0 -0
  252. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/licenses/LICENSE.txt +0 -0
  253. {localstack_core-4.7.1.dev49.dist-info → localstack_core-4.10.1.dev12.dist-info}/top_level.txt +0 -0
@@ -40,15 +40,13 @@ from localstack.http.request import get_raw_path
40
40
  from localstack.services.s3.constants import (
41
41
  DEFAULT_PRE_SIGNED_ACCESS_KEY_ID,
42
42
  DEFAULT_PRE_SIGNED_SECRET_ACCESS_KEY,
43
+ S3_HOST_ID,
43
44
  SIGNATURE_V2_PARAMS,
44
45
  SIGNATURE_V4_PARAMS,
45
46
  )
46
47
  from localstack.services.s3.utils import (
47
- S3_VIRTUAL_HOST_FORWARDED_HEADER,
48
- _create_invalid_argument_exc,
49
48
  capitalize_header_name_from_snake_case,
50
49
  extract_bucket_name_and_key_from_headers_and_path,
51
- forwarded_from_virtual_host_addressed_request,
52
50
  is_bucket_name_valid,
53
51
  is_presigned_url_request,
54
52
  uses_host_addressing,
@@ -86,8 +84,6 @@ IGNORED_SIGV4_HEADERS = [
86
84
  "x-amz-content-sha256",
87
85
  ]
88
86
 
89
- FAKE_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg="
90
-
91
87
  HOST_COMBINATION_REGEX = r"^(.*)(:[\d]{0,6})"
92
88
  PORT_REPLACEMENT = [":80", ":443", f":{config.GATEWAY_LISTEN[0].port}", ""]
93
89
 
@@ -157,7 +153,7 @@ def create_signature_does_not_match_sig_v2(
157
153
  "The request signature we calculated does not match the signature you provided. Check your key and signing method."
158
154
  )
159
155
  ex.AWSAccessKeyId = access_key_id
160
- ex.HostId = FAKE_HOST_ID
156
+ ex.HostId = S3_HOST_ID
161
157
  ex.SignatureProvided = request_signature
162
158
  ex.StringToSign = string_to_sign
163
159
  ex.StringToSignBytes = to_bytes(string_to_sign).hex(sep=" ", bytes_per_sep=2).upper()
@@ -300,7 +296,7 @@ def is_valid_sig_v2(query_args: set) -> bool:
300
296
  LOG.info("Presign signature calculation failed")
301
297
  raise AccessDenied(
302
298
  "Query-string authentication requires the Signature, Expires and AWSAccessKeyId parameters",
303
- HostId=FAKE_HOST_ID,
299
+ HostId=S3_HOST_ID,
304
300
  )
305
301
 
306
302
  return True
@@ -318,7 +314,7 @@ def is_valid_sig_v4(query_args: set) -> bool:
318
314
  LOG.info("Presign signature calculation failed")
319
315
  raise AuthorizationQueryParametersError(
320
316
  "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
321
- HostId=FAKE_HOST_ID,
317
+ HostId=S3_HOST_ID,
322
318
  )
323
319
 
324
320
  return True
@@ -352,7 +348,7 @@ def validate_presigned_url_s3(context: RequestContext) -> None:
352
348
  )
353
349
  else:
354
350
  raise AccessDenied(
355
- "Request has expired", HostId=FAKE_HOST_ID, Expires=expires, ServerTime=time.time()
351
+ "Request has expired", HostId=S3_HOST_ID, Expires=expires, ServerTime=time.time()
356
352
  )
357
353
 
358
354
  auth_signer = HmacV1QueryAuthValidation(credentials=signing_credentials, expires=expires)
@@ -451,7 +447,7 @@ def validate_presigned_url_s3v4(context: RequestContext) -> None:
451
447
  else:
452
448
  raise AccessDenied(
453
449
  "There were headers present in the request which were not signed",
454
- HostId=FAKE_HOST_ID,
450
+ HostId=S3_HOST_ID,
455
451
  HeadersNotSigned=", ".join(sigv4_context.missing_signed_headers),
456
452
  )
457
453
 
@@ -483,7 +479,7 @@ def validate_presigned_url_s3v4(context: RequestContext) -> None:
483
479
  else:
484
480
  raise AccessDenied(
485
481
  "Request has expired",
486
- HostId=FAKE_HOST_ID,
482
+ HostId=S3_HOST_ID,
487
483
  Expires=expiration_time.timestamp(),
488
484
  ServerTime=time.time(),
489
485
  X_Amz_Expires=x_amz_expires,
@@ -569,34 +565,21 @@ class S3SigV4SignatureContext:
569
565
  self._query_parameters
570
566
  )
571
567
 
572
- if forwarded_from_virtual_host_addressed_request(self._headers):
573
- # FIXME: maybe move this so it happens earlier in the chain when using virtual host?
574
- if not is_bucket_name_valid(self._bucket):
575
- raise InvalidBucketName(BucketName=self._bucket)
576
- netloc = self._headers.get(S3_VIRTUAL_HOST_FORWARDED_HEADER)
577
- self.host = netloc
578
- self._original_host = netloc
579
- self.signed_headers["host"] = netloc
580
- # the request comes from the Virtual Host router, we need to remove the bucket from the path
568
+ netloc = urlparse.urlparse(self.request.url).netloc
569
+ self.host = netloc
570
+ self._original_host = netloc
571
+ if (host_addressed := uses_host_addressing(self._headers)) and not is_bucket_name_valid(
572
+ self._bucket
573
+ ):
574
+ raise InvalidBucketName(BucketName=self._bucket)
575
+
576
+ if not host_addressed and not self.request.path.startswith(f"/{self._bucket}"):
577
+ # if in path style, check that the path starts with the bucket
578
+ # our path has been sanitized, we should use the un-sanitized one
581
579
  splitted_path = self.request.path.split("/", maxsplit=2)
582
- self.path = f"/{splitted_path[-1]}"
583
-
580
+ self.path = f"/{self._bucket}/{splitted_path[-1]}"
584
581
  else:
585
- netloc = urlparse.urlparse(self.request.url).netloc
586
- self.host = netloc
587
- self._original_host = netloc
588
- if (host_addressed := uses_host_addressing(self._headers)) and not is_bucket_name_valid(
589
- self._bucket
590
- ):
591
- raise InvalidBucketName(BucketName=self._bucket)
592
-
593
- if not host_addressed and not self.request.path.startswith(f"/{self._bucket}"):
594
- # if in path style, check that the path starts with the bucket
595
- # our path has been sanitized, we should use the un-sanitized one
596
- splitted_path = self.request.path.split("/", maxsplit=2)
597
- self.path = f"/{self._bucket}/{splitted_path[-1]}"
598
- else:
599
- self.path = self.request.path
582
+ self.path = self.request.path
600
583
 
601
584
  # we need to URL encode the path, as the key needs to be urlencoded for the signature to match
602
585
  self.path = urlparse.quote(self.path)
@@ -715,7 +698,7 @@ class S3SigV4SignatureContext:
715
698
  if not (split_creds := credential.split("/")) or len(split_creds) != 5:
716
699
  raise AuthorizationQueryParametersError(
717
700
  'Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting "<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request".',
718
- HostId=FAKE_HOST_ID,
701
+ HostId=S3_HOST_ID,
719
702
  )
720
703
 
721
704
  return split_creds[2]
@@ -772,13 +755,12 @@ def validate_post_policy(
772
755
  :return: None
773
756
  """
774
757
  if not request_form.get("key"):
775
- ex: InvalidArgument = _create_invalid_argument_exc(
776
- message="Bucket POST must contain a field named 'key'. If it is specified, please check the order of the fields.",
777
- name="key",
778
- value="",
779
- host_id=FAKE_HOST_ID,
758
+ raise InvalidArgument(
759
+ "Bucket POST must contain a field named 'key'. If it is specified, please check the order of the fields.",
760
+ ArgumentName="key",
761
+ ArgumentValue="",
762
+ HostId=S3_HOST_ID,
780
763
  )
781
- raise ex
782
764
 
783
765
  form_dict = {k.lower(): v for k, v in request_form.items()}
784
766
 
@@ -793,7 +775,7 @@ def validate_post_policy(
793
775
 
794
776
  if not is_v2 and not is_v4:
795
777
  ex: AccessDenied = AccessDenied("Access Denied")
796
- ex.HostId = FAKE_HOST_ID
778
+ ex.HostId = S3_HOST_ID
797
779
  raise ex
798
780
 
799
781
  try:
@@ -812,7 +794,7 @@ def validate_post_policy(
812
794
  if expiration := policy_decoded.get("expiration"):
813
795
  if is_expired(_parse_policy_expiration_date(expiration)):
814
796
  ex: AccessDenied = AccessDenied("Invalid according to Policy: Policy expired.")
815
- ex.HostId = FAKE_HOST_ID
797
+ ex.HostId = S3_HOST_ID
816
798
  raise ex
817
799
 
818
800
  # TODO: validate the signature
@@ -834,7 +816,7 @@ def validate_post_policy(
834
816
  str_condition = str(condition).replace("'", '"')
835
817
  raise AccessDenied(
836
818
  f"Invalid according to Policy: Policy Condition failed: {str_condition}",
837
- HostId=FAKE_HOST_ID,
819
+ HostId=S3_HOST_ID,
838
820
  )
839
821
 
840
822
 
@@ -887,7 +869,7 @@ def _verify_condition(condition: list | dict, form: dict, additional_policy_meta
887
869
  "Your proposed upload exceeds the maximum allowed size",
888
870
  ProposedSize=size,
889
871
  MaxSizeAllowed=end,
890
- HostId=FAKE_HOST_ID,
872
+ HostId=S3_HOST_ID,
891
873
  )
892
874
  else:
893
875
  return True
@@ -932,13 +914,12 @@ def _is_match_with_signature_fields(
932
914
  if argument_name == "Awsaccesskeyid":
933
915
  argument_name = "AWSAccessKeyId"
934
916
 
935
- ex: InvalidArgument = _create_invalid_argument_exc(
936
- message=f"Bucket POST must contain a field named '{argument_name}'. If it is specified, please check the order of the fields.",
937
- name=argument_name,
938
- value="",
939
- host_id=FAKE_HOST_ID,
917
+ raise InvalidArgument(
918
+ f"Bucket POST must contain a field named '{argument_name}'. If it is specified, please check the order of the fields.",
919
+ ArgumentName=argument_name,
920
+ ArgumentValue="",
921
+ HostId=S3_HOST_ID,
940
922
  )
941
- raise ex
942
923
 
943
924
  return True
944
925
  return False
@@ -1,4 +1,5 @@
1
1
  import base64
2
+ import contextlib
2
3
  import copy
3
4
  import datetime
4
5
  import json
@@ -8,6 +9,7 @@ from collections import defaultdict
8
9
  from inspect import signature
9
10
  from io import BytesIO
10
11
  from operator import itemgetter
12
+ from threading import RLock
11
13
  from typing import IO
12
14
  from urllib import parse as urlparse
13
15
  from zoneinfo import ZoneInfo
@@ -23,6 +25,7 @@ from localstack.aws.api.s3 import (
23
25
  AccountId,
24
26
  AnalyticsConfiguration,
25
27
  AnalyticsId,
28
+ AuthorizationHeaderMalformed,
26
29
  BadDigest,
27
30
  Body,
28
31
  Bucket,
@@ -235,6 +238,7 @@ from localstack.services.s3.constants import (
235
238
  ARCHIVES_STORAGE_CLASSES,
236
239
  CHECKSUM_ALGORITHMS,
237
240
  DEFAULT_BUCKET_ENCRYPTION,
241
+ S3_HOST_ID,
238
242
  )
239
243
  from localstack.services.s3.cors import S3CorsHandler, s3_cors_request_handler
240
244
  from localstack.services.s3.exceptions import (
@@ -277,6 +281,7 @@ from localstack.services.s3.utils import (
277
281
  get_canned_acl,
278
282
  get_class_attrs_from_spec_class,
279
283
  get_failed_precondition_copy_source,
284
+ get_failed_upload_part_copy_source_preconditions,
280
285
  get_full_default_bucket_location,
281
286
  get_kms_key_arn,
282
287
  get_lifecycle_rule_from_object,
@@ -337,6 +342,8 @@ class S3Provider(S3Api, ServiceLifecycleHook):
337
342
  self._storage_backend = storage_backend or EphemeralS3ObjectStore(DEFAULT_S3_TMP_DIR)
338
343
  self._notification_dispatcher = NotificationDispatcher()
339
344
  self._cors_handler = S3CorsHandler(BucketCorsIndex())
345
+ # TODO: add lock for keys for PutObject, only way to support precondition writes for versioned buckets
346
+ self._preconditions_locks = defaultdict(lambda: defaultdict(RLock))
340
347
 
341
348
  # runtime cache of Lifecycle Expiration headers, as they need to be calculated everytime we fetch an object
342
349
  # in case the rules have changed
@@ -381,7 +388,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
381
388
  """
382
389
  if s3_bucket.notification_configuration:
383
390
  if not s3_notif_ctx:
384
- s3_notif_ctx = S3EventNotificationContext.from_request_context_native(
391
+ s3_notif_ctx = S3EventNotificationContext.from_request_context(
385
392
  context,
386
393
  s3_bucket=s3_bucket,
387
394
  s3_object=s3_object,
@@ -469,6 +476,16 @@ class S3Provider(S3Api, ServiceLifecycleHook):
469
476
  context: RequestContext,
470
477
  request: CreateBucketRequest,
471
478
  ) -> CreateBucketOutput:
479
+ if context.region == "aws-global":
480
+ # TODO: extend this logic to probably all the provider, and maybe all services. S3 is the most impacted
481
+ # right now so this will help users to properly set a region in their config
482
+ # See the `TestS3.test_create_bucket_aws_global` test
483
+ raise AuthorizationHeaderMalformed(
484
+ f"The authorization header is malformed; the region 'aws-global' is wrong; expecting '{AWS_REGION_US_EAST_1}'",
485
+ HostId=S3_HOST_ID,
486
+ Region=AWS_REGION_US_EAST_1,
487
+ )
488
+
472
489
  bucket_name = request["Bucket"]
473
490
 
474
491
  if not is_bucket_name_valid(bucket_name):
@@ -577,6 +594,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
577
594
  store.global_bucket_map.pop(bucket)
578
595
  self._cors_handler.invalidate_cache()
579
596
  self._expiration_cache.pop(bucket, None)
597
+ self._preconditions_locks.pop(bucket, None)
580
598
  # clean up the storage backend
581
599
  self._storage_backend.delete_bucket(bucket)
582
600
 
@@ -636,6 +654,16 @@ class S3Provider(S3Api, ServiceLifecycleHook):
636
654
  expected_bucket_owner: AccountId = None,
637
655
  **kwargs,
638
656
  ) -> HeadBucketOutput:
657
+ if context.region == "aws-global":
658
+ # TODO: extend this logic to probably all the provider, and maybe all services. S3 is the most impacted
659
+ # right now so this will help users to properly set a region in their config
660
+ # See the `TestS3.test_create_bucket_aws_global` test
661
+ raise AuthorizationHeaderMalformed(
662
+ f"The authorization header is malformed; the region 'aws-global' is wrong; expecting '{AWS_REGION_US_EAST_1}'",
663
+ HostId=S3_HOST_ID,
664
+ Region=AWS_REGION_US_EAST_1,
665
+ )
666
+
639
667
  store = self.get_store(context.account_id, context.region)
640
668
  if not (s3_bucket := store.buckets.get(bucket)):
641
669
  if not (account_id := store.global_bucket_map.get(bucket)):
@@ -727,6 +755,12 @@ class S3Provider(S3Api, ServiceLifecycleHook):
727
755
  system_metadata["ContentType"] = "binary/octet-stream"
728
756
 
729
757
  version_id = generate_version_id(s3_bucket.versioning_status)
758
+ if version_id != "null":
759
+ # if we are in a versioned bucket, we need to lock around the full key (all the versions)
760
+ # because object versions have locks per version
761
+ precondition_lock = self._preconditions_locks[bucket_name][key]
762
+ else:
763
+ precondition_lock = contextlib.nullcontext()
730
764
 
731
765
  etag_content_md5 = ""
732
766
  if content_md5 := request.get("ContentMD5"):
@@ -809,7 +843,10 @@ class S3Provider(S3Api, ServiceLifecycleHook):
809
843
  if encodings:
810
844
  s3_object.system_metadata["ContentEncoding"] = ",".join(encodings)
811
845
 
812
- with self._storage_backend.open(bucket_name, s3_object, mode="w") as s3_stored_object:
846
+ with (
847
+ precondition_lock,
848
+ self._storage_backend.open(bucket_name, s3_object, mode="w") as s3_stored_object,
849
+ ):
813
850
  # as we are inside the lock here, if multiple concurrent requests happen for the same object, it's the first
814
851
  # one to finish to succeed, and subsequent will raise exceptions. Once the first write finishes, we're
815
852
  # opening the lock and other requests can check this condition
@@ -1248,7 +1285,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1248
1285
  delete_marker_id = generate_version_id(s3_bucket.versioning_status)
1249
1286
  delete_marker = S3DeleteMarker(key=key, version_id=delete_marker_id)
1250
1287
  s3_bucket.objects.set(key, delete_marker)
1251
- s3_notif_ctx = S3EventNotificationContext.from_request_context_native(
1288
+ s3_notif_ctx = S3EventNotificationContext.from_request_context(
1252
1289
  context,
1253
1290
  s3_bucket=s3_bucket,
1254
1291
  s3_object=delete_marker,
@@ -1281,6 +1318,10 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1281
1318
  store.TAGS.tags.pop(get_unique_key_id(bucket, key, version_id), None)
1282
1319
  self._notify(context, s3_bucket=s3_bucket, s3_object=s3_object)
1283
1320
 
1321
+ if key not in s3_bucket.objects:
1322
+ # we clean up keys that do not have any object versions in them anymore
1323
+ self._preconditions_locks[bucket].pop(key, None)
1324
+
1284
1325
  return response
1285
1326
 
1286
1327
  def delete_objects(
@@ -1314,6 +1355,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1314
1355
  errors = []
1315
1356
 
1316
1357
  to_remove = []
1358
+ versioned_keys = set()
1317
1359
  for to_delete_object in objects:
1318
1360
  object_key = to_delete_object.get("Key")
1319
1361
  version_id = to_delete_object.get("VersionId")
@@ -1351,7 +1393,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1351
1393
  delete_marker_id = generate_version_id(s3_bucket.versioning_status)
1352
1394
  delete_marker = S3DeleteMarker(key=object_key, version_id=delete_marker_id)
1353
1395
  s3_bucket.objects.set(object_key, delete_marker)
1354
- s3_notif_ctx = S3EventNotificationContext.from_request_context_native(
1396
+ s3_notif_ctx = S3EventNotificationContext.from_request_context(
1355
1397
  context,
1356
1398
  s3_bucket=s3_bucket,
1357
1399
  s3_object=delete_marker,
@@ -1394,6 +1436,8 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1394
1436
  continue
1395
1437
 
1396
1438
  s3_bucket.objects.pop(object_key=object_key, version_id=version_id)
1439
+ versioned_keys.add(object_key)
1440
+
1397
1441
  if not quiet:
1398
1442
  deleted_object = DeletedObject(
1399
1443
  Key=object_key,
@@ -1411,6 +1455,11 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1411
1455
  self._notify(context, s3_bucket=s3_bucket, s3_object=found_object)
1412
1456
  store.TAGS.tags.pop(get_unique_key_id(bucket, object_key, version_id), None)
1413
1457
 
1458
+ for versioned_key in versioned_keys:
1459
+ # we clean up keys that do not have any object versions in them anymore
1460
+ if versioned_key not in s3_bucket.objects:
1461
+ self._preconditions_locks[bucket].pop(versioned_key, None)
1462
+
1414
1463
  # TODO: request charged
1415
1464
  self._storage_backend.remove(bucket, to_remove)
1416
1465
  response: DeleteObjectsOutput = {}
@@ -2179,7 +2228,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
2179
2228
  # TODO: add a way to transition from ongoing-request=true to false? for now it is instant
2180
2229
  s3_object.restore = f'ongoing-request="false", expiry-date="{restore_expiration_date}"'
2181
2230
 
2182
- s3_notif_ctx_initiated = S3EventNotificationContext.from_request_context_native(
2231
+ s3_notif_ctx_initiated = S3EventNotificationContext.from_request_context(
2183
2232
  context,
2184
2233
  s3_bucket=s3_bucket,
2185
2234
  s3_object=s3_object,
@@ -2483,10 +2532,6 @@ class S3Provider(S3Api, ServiceLifecycleHook):
2483
2532
  request: UploadPartCopyRequest,
2484
2533
  ) -> UploadPartCopyOutput:
2485
2534
  # TODO: handle following parameters:
2486
- # CopySourceIfMatch: Optional[CopySourceIfMatch]
2487
- # CopySourceIfModifiedSince: Optional[CopySourceIfModifiedSince]
2488
- # CopySourceIfNoneMatch: Optional[CopySourceIfNoneMatch]
2489
- # CopySourceIfUnmodifiedSince: Optional[CopySourceIfUnmodifiedSince]
2490
2535
  # SSECustomerAlgorithm: Optional[SSECustomerAlgorithm]
2491
2536
  # SSECustomerKey: Optional[SSECustomerKey]
2492
2537
  # SSECustomerKeyMD5: Optional[SSECustomerKeyMD5]
@@ -2549,6 +2594,14 @@ class S3Provider(S3Api, ServiceLifecycleHook):
2549
2594
  if source_range:
2550
2595
  range_data = parse_copy_source_range_header(source_range, src_s3_object.size)
2551
2596
 
2597
+ if precondition := get_failed_upload_part_copy_source_preconditions(
2598
+ request, src_s3_object.last_modified, src_s3_object.etag
2599
+ ):
2600
+ raise PreconditionFailed(
2601
+ "At least one of the pre-conditions you specified did not hold",
2602
+ Condition=precondition,
2603
+ )
2604
+
2552
2605
  s3_part = S3Part(part_number=part_number)
2553
2606
  if s3_multipart.checksum_algorithm:
2554
2607
  s3_part.checksum_algorithm = s3_multipart.checksum_algorithm
@@ -2622,6 +2675,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
2622
2675
  )
2623
2676
 
2624
2677
  elif if_none_match:
2678
+ # TODO: improve concurrency mechanism for `if_none_match` and `if_match`
2625
2679
  if if_none_match != "*":
2626
2680
  raise NotImplementedException(
2627
2681
  "A header you provided implies functionality that is not implemented",
@@ -3166,11 +3220,12 @@ class S3Provider(S3Api, ServiceLifecycleHook):
3166
3220
  if "TagSet" not in tagging:
3167
3221
  raise MalformedXML()
3168
3222
 
3169
- validate_tag_set(tagging["TagSet"], type_set="bucket")
3223
+ tag_set = tagging["TagSet"] or []
3224
+ validate_tag_set(tag_set, type_set="bucket")
3170
3225
 
3171
3226
  # remove the previous tags before setting the new ones, it overwrites the whole TagSet
3172
3227
  store.TAGS.tags.pop(s3_bucket.bucket_arn, None)
3173
- store.TAGS.tag_resource(s3_bucket.bucket_arn, tags=tagging["TagSet"])
3228
+ store.TAGS.tag_resource(s3_bucket.bucket_arn, tags=tag_set)
3174
3229
 
3175
3230
  def get_bucket_tagging(
3176
3231
  self,
@@ -3220,12 +3275,13 @@ class S3Provider(S3Api, ServiceLifecycleHook):
3220
3275
  if "TagSet" not in tagging:
3221
3276
  raise MalformedXML()
3222
3277
 
3223
- validate_tag_set(tagging["TagSet"], type_set="object")
3278
+ tag_set = tagging["TagSet"] or []
3279
+ validate_tag_set(tag_set, type_set="object")
3224
3280
 
3225
3281
  key_id = get_unique_key_id(bucket, key, s3_object.version_id)
3226
3282
  # remove the previous tags before setting the new ones, it overwrites the whole TagSet
3227
3283
  store.TAGS.tags.pop(key_id, None)
3228
- store.TAGS.tag_resource(key_id, tags=tagging["TagSet"])
3284
+ store.TAGS.tag_resource(key_id, tags=tag_set)
3229
3285
  response = PutObjectTaggingOutput()
3230
3286
  if s3_object.version_id:
3231
3287
  response["VersionId"] = s3_object.version_id
@@ -3291,7 +3347,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
3291
3347
 
3292
3348
  s3_object = s3_bucket.get_object(key=key, version_id=version_id, http_method="DELETE")
3293
3349
 
3294
- store.TAGS.tags.pop(get_unique_key_id(bucket, key, version_id), None)
3350
+ store.TAGS.tags.pop(get_unique_key_id(bucket, key, s3_object.version_id), None)
3295
3351
  response = DeleteObjectTaggingOutput()
3296
3352
  if s3_object.version_id:
3297
3353
  response["VersionId"] = s3_object.version_id
@@ -3852,7 +3908,9 @@ class S3Provider(S3Api, ServiceLifecycleHook):
3852
3908
  if retention and retention["RetainUntilDate"] < datetime.datetime.now(datetime.UTC):
3853
3909
  # weirdly, this date is format as following: Tue Dec 31 16:00:00 PST 2019
3854
3910
  # it contains the timezone as PST, even if you target a bucket in Europe or Asia
3855
- pst_datetime = retention["RetainUntilDate"].astimezone(tz=ZoneInfo("US/Pacific"))
3911
+ pst_datetime = retention["RetainUntilDate"].astimezone(
3912
+ tz=ZoneInfo("America/Los_Angeles")
3913
+ )
3856
3914
  raise InvalidArgument(
3857
3915
  "The retain until date must be in the future!",
3858
3916
  ArgumentName="RetainUntilDate",
@@ -7,6 +7,7 @@ import logging
7
7
  import re
8
8
  import time
9
9
  import zlib
10
+ from collections.abc import Mapping
10
11
  from enum import StrEnum
11
12
  from secrets import token_bytes
12
13
  from typing import Any, Literal, NamedTuple, Protocol
@@ -50,6 +51,7 @@ from localstack.aws.api.s3 import (
50
51
  SSEKMSKeyId,
51
52
  TaggingHeader,
52
53
  TagSet,
54
+ UploadPartCopyRequest,
53
55
  UploadPartRequest,
54
56
  )
55
57
  from localstack.aws.api.s3 import Type as GranteeType
@@ -62,7 +64,6 @@ from localstack.services.s3.constants import (
62
64
  AUTHENTICATED_USERS_ACL_GRANTEE,
63
65
  CHECKSUM_ALGORITHMS,
64
66
  LOG_DELIVERY_ACL_GRANTEE,
65
- S3_VIRTUAL_HOST_FORWARDED_HEADER,
66
67
  SIGNATURE_V2_PARAMS,
67
68
  SIGNATURE_V4_PARAMS,
68
69
  SYSTEM_METADATA_SETTABLE_HEADERS,
@@ -403,6 +404,45 @@ def parse_copy_source_range_header(copy_source_range: str, object_size: int) ->
403
404
  )
404
405
 
405
406
 
407
+ def get_failed_upload_part_copy_source_preconditions(
408
+ request: UploadPartCopyRequest, last_modified: datetime.datetime, etag: ETag
409
+ ) -> str | None:
410
+ """
411
+ Utility which parses the conditions from a S3 UploadPartCopy request.
412
+ Note: The order in which these conditions are checked if used in conjunction matters
413
+
414
+ :param UploadPartCopyRequest request: The S3 UploadPartCopy request.
415
+ :param datetime last_modified: The time the source object was last modified.
416
+ :param ETag etag: The ETag of the source object.
417
+
418
+ :returns: The name of the failed precondition.
419
+ """
420
+ if_match = request.get("CopySourceIfMatch")
421
+ if_none_match = request.get("CopySourceIfNoneMatch")
422
+ if_unmodified_since = request.get("CopySourceIfUnmodifiedSince")
423
+ if_modified_since = request.get("CopySourceIfModifiedSince")
424
+
425
+ if if_match:
426
+ if if_match.strip('"') != etag.strip('"'):
427
+ return "x-amz-copy-source-If-Match"
428
+ if if_modified_since and if_modified_since > last_modified:
429
+ return "x-amz-copy-source-If-Modified-Since"
430
+ # CopySourceIfMatch is unaffected by CopySourceIfUnmodifiedSince so return early
431
+ if if_unmodified_since:
432
+ return None
433
+
434
+ if if_unmodified_since and if_unmodified_since < last_modified:
435
+ return "x-amz-copy-source-If-Unmodified-Since"
436
+
437
+ if if_none_match and if_none_match.strip('"') == etag.strip('"'):
438
+ return "x-amz-copy-source-If-None-Match"
439
+
440
+ if if_modified_since and last_modified < if_modified_since < datetime.datetime.now(
441
+ tz=_gmt_zone_info
442
+ ):
443
+ return "x-amz-copy-source-If-Modified-Since"
444
+
445
+
406
446
  def get_full_default_bucket_location(bucket_name: BucketName) -> str:
407
447
  host_definition = localstack_host()
408
448
  if host_definition.host != constants.LOCALHOST_HOSTNAME:
@@ -482,7 +522,7 @@ def is_valid_canonical_id(canonical_id: str) -> bool:
482
522
  return False
483
523
 
484
524
 
485
- def uses_host_addressing(headers: dict[str, str]) -> str | None:
525
+ def uses_host_addressing(headers: Mapping[str, str]) -> str | None:
486
526
  """
487
527
  Determines if the request is targeting S3 with virtual host addressing
488
528
  :param headers: the request headers
@@ -511,15 +551,6 @@ def get_system_metadata_from_request(request: dict) -> Metadata:
511
551
  return metadata
512
552
 
513
553
 
514
- def forwarded_from_virtual_host_addressed_request(headers: dict[str, str]) -> bool:
515
- """
516
- Determines if the request was forwarded from a v-host addressing style into a path one
517
- """
518
- # we can assume that the host header we are receiving here is actually the header we originally received
519
- # from the client (because the edge service is forwarding the request in memory)
520
- return S3_VIRTUAL_HOST_FORWARDED_HEADER in headers
521
-
522
-
523
554
  def extract_bucket_name_and_key_from_headers_and_path(
524
555
  headers: dict[str, str], path: str
525
556
  ) -> tuple[str | None, str | None]:
@@ -574,17 +605,6 @@ def get_bucket_and_key_from_presign_url(presign_url: str) -> tuple[str, str]:
574
605
  return bucket, key
575
606
 
576
607
 
577
- def _create_invalid_argument_exc(
578
- message: str | None, name: str, value: str, host_id: str = None
579
- ) -> InvalidArgument:
580
- ex = InvalidArgument(message)
581
- ex.ArgumentName = name
582
- ex.ArgumentValue = value
583
- if host_id:
584
- ex.HostId = host_id
585
- return ex
586
-
587
-
588
608
  def capitalize_header_name_from_snake_case(header_name: str) -> str:
589
609
  return "-".join([part.capitalize() for part in header_name.split("-")])
590
610