localstack-core 4.6.1.dev60__py3-none-any.whl → 4.10.1.dev12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (672) hide show
  1. localstack/aws/api/apigateway/__init__.py +87 -110
  2. localstack/aws/api/cloudformation/__init__.py +18 -4
  3. localstack/aws/api/cloudwatch/__init__.py +41 -1
  4. localstack/aws/api/config/__init__.py +4 -0
  5. localstack/aws/api/core.py +8 -5
  6. localstack/aws/api/dynamodb/__init__.py +30 -0
  7. localstack/aws/api/ec2/__init__.py +1545 -66
  8. localstack/aws/api/events/__init__.py +12 -16
  9. localstack/aws/api/iam/__init__.py +7 -0
  10. localstack/aws/api/kinesis/__init__.py +19 -0
  11. localstack/aws/api/kms/__init__.py +6 -0
  12. localstack/aws/api/lambda_/__init__.py +36 -23
  13. localstack/aws/api/logs/__init__.py +20 -8
  14. localstack/aws/api/opensearch/__init__.py +16 -0
  15. localstack/aws/api/pipes/__init__.py +24 -32
  16. localstack/aws/api/redshift/__init__.py +9 -3
  17. localstack/aws/api/route53/__init__.py +5 -0
  18. localstack/aws/api/s3/__init__.py +12 -0
  19. localstack/aws/api/s3control/__init__.py +56 -0
  20. localstack/aws/api/scheduler/__init__.py +14 -16
  21. localstack/aws/api/ssm/__init__.py +2 -0
  22. localstack/aws/api/stepfunctions/__init__.py +88 -114
  23. localstack/aws/api/support/__init__.py +8 -9
  24. localstack/aws/api/transcribe/__init__.py +17 -0
  25. localstack/aws/chain.py +2 -2
  26. localstack/aws/client.py +14 -11
  27. localstack/aws/connect.py +42 -41
  28. localstack/aws/forwarder.py +57 -9
  29. localstack/aws/gateway.py +1 -3
  30. localstack/aws/handlers/analytics.py +2 -3
  31. localstack/aws/handlers/cors.py +4 -5
  32. localstack/aws/handlers/internal_requests.py +6 -1
  33. localstack/aws/handlers/logging.py +13 -4
  34. localstack/aws/handlers/metric_handler.py +44 -5
  35. localstack/aws/handlers/service.py +48 -28
  36. localstack/aws/mocking.py +18 -27
  37. localstack/aws/patches.py +2 -2
  38. localstack/aws/protocol/op_router.py +11 -10
  39. localstack/aws/protocol/parser.py +475 -49
  40. localstack/aws/protocol/serializer.py +723 -106
  41. localstack/aws/protocol/service_router.py +133 -33
  42. localstack/aws/protocol/validate.py +6 -6
  43. localstack/aws/scaffold.py +9 -10
  44. localstack/aws/serving/edge.py +5 -6
  45. localstack/aws/serving/hypercorn.py +2 -2
  46. localstack/aws/serving/twisted.py +1 -2
  47. localstack/aws/serving/werkzeug.py +2 -2
  48. localstack/aws/skeleton.py +12 -11
  49. localstack/aws/spec-patches.json +58 -0
  50. localstack/aws/spec.py +66 -46
  51. localstack/cli/exceptions.py +1 -1
  52. localstack/cli/localstack.py +11 -11
  53. localstack/cli/lpm.py +4 -5
  54. localstack/cli/plugins.py +1 -1
  55. localstack/cli/profiles.py +1 -2
  56. localstack/config.py +44 -30
  57. localstack/constants.py +4 -29
  58. localstack/deprecations.py +5 -5
  59. localstack/dev/kubernetes/__main__.py +130 -7
  60. localstack/dev/run/__main__.py +5 -5
  61. localstack/dev/run/configurators.py +1 -4
  62. localstack/dev/run/paths.py +6 -6
  63. localstack/dns/models.py +2 -1
  64. localstack/dns/plugins.py +5 -1
  65. localstack/dns/server.py +16 -6
  66. localstack/http/dispatcher.py +1 -2
  67. localstack/http/response.py +2 -2
  68. localstack/http/router.py +1 -1
  69. localstack/http/trace.py +2 -1
  70. localstack/logging/format.py +6 -6
  71. localstack/packages/api.py +13 -12
  72. localstack/packages/core.py +4 -4
  73. localstack/packages/debugpy.py +1 -3
  74. localstack/packages/ffmpeg.py +1 -2
  75. localstack/packages/java.py +38 -12
  76. localstack/packages/plugins.py +0 -8
  77. localstack/runtime/analytics.py +3 -0
  78. localstack/runtime/hooks.py +1 -1
  79. localstack/runtime/init.py +8 -9
  80. localstack/runtime/main.py +5 -5
  81. localstack/runtime/patches.py +2 -2
  82. localstack/runtime/shutdown.py +2 -1
  83. localstack/services/apigateway/exporter.py +1 -2
  84. localstack/services/apigateway/helpers.py +7 -10
  85. localstack/services/apigateway/legacy/context.py +21 -21
  86. localstack/services/apigateway/legacy/helpers.py +27 -28
  87. localstack/services/apigateway/legacy/integration.py +11 -10
  88. localstack/services/apigateway/legacy/invocations.py +6 -5
  89. localstack/services/apigateway/legacy/provider.py +148 -68
  90. localstack/services/apigateway/legacy/router_asf.py +2 -2
  91. localstack/services/apigateway/legacy/templates.py +6 -6
  92. localstack/services/apigateway/models.py +16 -16
  93. localstack/services/apigateway/next_gen/execute_api/api.py +2 -2
  94. localstack/services/apigateway/next_gen/execute_api/context.py +25 -25
  95. localstack/services/apigateway/next_gen/execute_api/handlers/api_key_validation.py +2 -5
  96. localstack/services/apigateway/next_gen/execute_api/handlers/method_request.py +7 -2
  97. localstack/services/apigateway/next_gen/execute_api/handlers/parse.py +1 -2
  98. localstack/services/apigateway/next_gen/execute_api/handlers/resource_router.py +2 -3
  99. localstack/services/apigateway/next_gen/execute_api/header_utils.py +1 -1
  100. localstack/services/apigateway/next_gen/execute_api/helpers.py +2 -2
  101. localstack/services/apigateway/next_gen/execute_api/integrations/aws.py +9 -6
  102. localstack/services/apigateway/next_gen/execute_api/integrations/http.py +12 -12
  103. localstack/services/apigateway/next_gen/execute_api/router.py +31 -0
  104. localstack/services/apigateway/next_gen/execute_api/template_mapping.py +2 -2
  105. localstack/services/apigateway/next_gen/execute_api/test_invoke.py +114 -9
  106. localstack/services/apigateway/next_gen/execute_api/variables.py +63 -63
  107. localstack/services/apigateway/next_gen/provider.py +5 -0
  108. localstack/services/apigateway/resource_providers/aws_apigateway_account.py +3 -3
  109. localstack/services/apigateway/resource_providers/aws_apigateway_account_plugin.py +1 -3
  110. localstack/services/apigateway/resource_providers/aws_apigateway_apikey.py +14 -14
  111. localstack/services/apigateway/resource_providers/aws_apigateway_apikey_plugin.py +1 -3
  112. localstack/services/apigateway/resource_providers/aws_apigateway_basepathmapping.py +5 -5
  113. localstack/services/apigateway/resource_providers/aws_apigateway_basepathmapping_plugin.py +1 -3
  114. localstack/services/apigateway/resource_providers/aws_apigateway_deployment.py +46 -46
  115. localstack/services/apigateway/resource_providers/aws_apigateway_deployment_plugin.py +1 -3
  116. localstack/services/apigateway/resource_providers/aws_apigateway_domainname.py +18 -18
  117. localstack/services/apigateway/resource_providers/aws_apigateway_domainname_plugin.py +1 -3
  118. localstack/services/apigateway/resource_providers/aws_apigateway_gatewayresponse.py +7 -7
  119. localstack/services/apigateway/resource_providers/aws_apigateway_gatewayresponse_plugin.py +1 -3
  120. localstack/services/apigateway/resource_providers/aws_apigateway_method.py +36 -36
  121. localstack/services/apigateway/resource_providers/aws_apigateway_method_plugin.py +1 -3
  122. localstack/services/apigateway/resource_providers/aws_apigateway_model.py +6 -6
  123. localstack/services/apigateway/resource_providers/aws_apigateway_model_plugin.py +1 -3
  124. localstack/services/apigateway/resource_providers/aws_apigateway_requestvalidator.py +6 -6
  125. localstack/services/apigateway/resource_providers/aws_apigateway_requestvalidator_plugin.py +1 -3
  126. localstack/services/apigateway/resource_providers/aws_apigateway_resource.py +6 -6
  127. localstack/services/apigateway/resource_providers/aws_apigateway_resource_plugin.py +1 -3
  128. localstack/services/apigateway/resource_providers/aws_apigateway_restapi.py +26 -26
  129. localstack/services/apigateway/resource_providers/aws_apigateway_restapi_plugin.py +1 -3
  130. localstack/services/apigateway/resource_providers/aws_apigateway_stage.py +33 -33
  131. localstack/services/apigateway/resource_providers/aws_apigateway_stage_plugin.py +1 -3
  132. localstack/services/apigateway/resource_providers/aws_apigateway_usageplan.py +18 -18
  133. localstack/services/apigateway/resource_providers/aws_apigateway_usageplan_plugin.py +1 -3
  134. localstack/services/apigateway/resource_providers/aws_apigateway_usageplankey.py +5 -5
  135. localstack/services/apigateway/resource_providers/aws_apigateway_usageplankey_plugin.py +1 -3
  136. localstack/services/cdk/resource_providers/cdk_metadata.py +4 -3
  137. localstack/services/cdk/resource_providers/cdk_metadata_plugin.py +1 -3
  138. localstack/services/certificatemanager/resource_providers/aws_certificatemanager_certificate.py +14 -14
  139. localstack/services/certificatemanager/resource_providers/aws_certificatemanager_certificate_plugin.py +1 -3
  140. localstack/services/cloudformation/api_utils.py +4 -8
  141. localstack/services/cloudformation/cfn_utils.py +2 -2
  142. localstack/services/cloudformation/deployment_utils.py +14 -12
  143. localstack/services/cloudformation/engine/changes.py +3 -3
  144. localstack/services/cloudformation/engine/entities.py +27 -17
  145. localstack/services/cloudformation/engine/parameters.py +4 -4
  146. localstack/services/cloudformation/engine/template_deployer.py +15 -14
  147. localstack/services/cloudformation/engine/template_utils.py +34 -12
  148. localstack/services/cloudformation/engine/transformers.py +11 -8
  149. localstack/services/cloudformation/engine/types.py +5 -4
  150. localstack/services/cloudformation/engine/v2/change_set_model.py +336 -39
  151. localstack/services/cloudformation/engine/v2/change_set_model_describer.py +96 -17
  152. localstack/services/cloudformation/engine/v2/change_set_model_executor.py +289 -136
  153. localstack/services/cloudformation/engine/v2/change_set_model_preproc.py +315 -146
  154. localstack/services/cloudformation/engine/v2/change_set_model_transform.py +380 -105
  155. localstack/services/cloudformation/engine/v2/change_set_model_validator.py +183 -0
  156. localstack/services/cloudformation/engine/v2/change_set_model_visitor.py +6 -0
  157. localstack/services/cloudformation/engine/v2/resolving.py +102 -0
  158. localstack/services/cloudformation/engine/yaml_parser.py +9 -2
  159. localstack/services/cloudformation/provider.py +11 -6
  160. localstack/services/cloudformation/provider_utils.py +34 -4
  161. localstack/services/cloudformation/resource_provider.py +31 -21
  162. localstack/services/cloudformation/resource_providers/aws_cloudformation_macro.py +7 -7
  163. localstack/services/cloudformation/resource_providers/aws_cloudformation_macro_plugin.py +1 -3
  164. localstack/services/cloudformation/resource_providers/aws_cloudformation_stack.py +9 -9
  165. localstack/services/cloudformation/resource_providers/aws_cloudformation_stack_plugin.py +1 -3
  166. localstack/services/cloudformation/resource_providers/aws_cloudformation_waitcondition.py +6 -6
  167. localstack/services/cloudformation/resource_providers/aws_cloudformation_waitcondition_plugin.py +1 -3
  168. localstack/services/cloudformation/resource_providers/aws_cloudformation_waitconditionhandle.py +2 -2
  169. localstack/services/cloudformation/resource_providers/aws_cloudformation_waitconditionhandle_plugin.py +1 -3
  170. localstack/services/cloudformation/resources.py +24149 -0
  171. localstack/services/cloudformation/scaffolding/__main__.py +13 -12
  172. localstack/services/cloudformation/scaffolding/propgen.py +2 -2
  173. localstack/services/cloudformation/service_models.py +2 -2
  174. localstack/services/cloudformation/stores.py +33 -22
  175. localstack/services/cloudformation/v2/entities.py +103 -88
  176. localstack/services/cloudformation/v2/provider.py +872 -131
  177. localstack/services/cloudformation/v2/types.py +38 -0
  178. localstack/services/cloudformation/v2/utils.py +4 -1
  179. localstack/services/cloudwatch/alarm_scheduler.py +11 -8
  180. localstack/services/cloudwatch/cloudwatch_database_helper.py +4 -5
  181. localstack/services/cloudwatch/models.py +5 -7
  182. localstack/services/cloudwatch/provider.py +30 -25
  183. localstack/services/cloudwatch/provider_v2.py +27 -32
  184. localstack/services/cloudwatch/resource_providers/aws_cloudwatch_alarm.py +40 -40
  185. localstack/services/cloudwatch/resource_providers/aws_cloudwatch_alarm_plugin.py +1 -3
  186. localstack/services/cloudwatch/resource_providers/aws_cloudwatch_compositealarm.py +12 -12
  187. localstack/services/cloudwatch/resource_providers/aws_cloudwatch_compositealarm_plugin.py +1 -3
  188. localstack/services/dynamodb/packages.py +3 -3
  189. localstack/services/dynamodb/provider.py +159 -18
  190. localstack/services/dynamodb/resource_providers/aws_dynamodb_globaltable.py +63 -63
  191. localstack/services/dynamodb/resource_providers/aws_dynamodb_globaltable_plugin.py +1 -3
  192. localstack/services/dynamodb/resource_providers/aws_dynamodb_table.py +57 -57
  193. localstack/services/dynamodb/resource_providers/aws_dynamodb_table_plugin.py +1 -3
  194. localstack/services/dynamodb/server.py +2 -2
  195. localstack/services/dynamodb/utils.py +13 -14
  196. localstack/services/dynamodb/v2/provider.py +157 -18
  197. localstack/services/dynamodbstreams/dynamodbstreams_api.py +2 -2
  198. localstack/services/dynamodbstreams/models.py +1 -3
  199. localstack/services/ec2/patches.py +26 -5
  200. localstack/services/ec2/provider.py +3 -3
  201. localstack/services/ec2/resource_providers/aws_ec2_dhcpoptions.py +10 -10
  202. localstack/services/ec2/resource_providers/aws_ec2_dhcpoptions_plugin.py +1 -3
  203. localstack/services/ec2/resource_providers/aws_ec2_instance.py +96 -96
  204. localstack/services/ec2/resource_providers/aws_ec2_instance_plugin.py +1 -3
  205. localstack/services/ec2/resource_providers/aws_ec2_internetgateway.py +5 -5
  206. localstack/services/ec2/resource_providers/aws_ec2_internetgateway_plugin.py +1 -3
  207. localstack/services/ec2/resource_providers/aws_ec2_keypair.py +10 -10
  208. localstack/services/ec2/resource_providers/aws_ec2_keypair_plugin.py +1 -3
  209. localstack/services/ec2/resource_providers/aws_ec2_natgateway.py +13 -13
  210. localstack/services/ec2/resource_providers/aws_ec2_natgateway_plugin.py +1 -3
  211. localstack/services/ec2/resource_providers/aws_ec2_networkacl.py +6 -6
  212. localstack/services/ec2/resource_providers/aws_ec2_networkacl_plugin.py +1 -3
  213. localstack/services/ec2/resource_providers/aws_ec2_prefixlist.py +14 -14
  214. localstack/services/ec2/resource_providers/aws_ec2_prefixlist_plugin.py +1 -3
  215. localstack/services/ec2/resource_providers/aws_ec2_route.py +15 -15
  216. localstack/services/ec2/resource_providers/aws_ec2_route_plugin.py +1 -3
  217. localstack/services/ec2/resource_providers/aws_ec2_routetable.py +6 -6
  218. localstack/services/ec2/resource_providers/aws_ec2_routetable_plugin.py +1 -3
  219. localstack/services/ec2/resource_providers/aws_ec2_securitygroup.py +29 -29
  220. localstack/services/ec2/resource_providers/aws_ec2_securitygroup_plugin.py +1 -3
  221. localstack/services/ec2/resource_providers/aws_ec2_subnet.py +19 -19
  222. localstack/services/ec2/resource_providers/aws_ec2_subnet_plugin.py +1 -3
  223. localstack/services/ec2/resource_providers/aws_ec2_subnetroutetableassociation.py +4 -4
  224. localstack/services/ec2/resource_providers/aws_ec2_subnetroutetableassociation_plugin.py +1 -3
  225. localstack/services/ec2/resource_providers/aws_ec2_transitgateway.py +16 -16
  226. localstack/services/ec2/resource_providers/aws_ec2_transitgateway_plugin.py +1 -3
  227. localstack/services/ec2/resource_providers/aws_ec2_transitgatewayattachment.py +9 -9
  228. localstack/services/ec2/resource_providers/aws_ec2_transitgatewayattachment_plugin.py +1 -3
  229. localstack/services/ec2/resource_providers/aws_ec2_vpc.py +15 -15
  230. localstack/services/ec2/resource_providers/aws_ec2_vpc_plugin.py +1 -3
  231. localstack/services/ec2/resource_providers/aws_ec2_vpcendpoint.py +29 -22
  232. localstack/services/ec2/resource_providers/aws_ec2_vpcendpoint_plugin.py +1 -3
  233. localstack/services/ec2/resource_providers/aws_ec2_vpcgatewayattachment.py +5 -5
  234. localstack/services/ec2/resource_providers/aws_ec2_vpcgatewayattachment_plugin.py +1 -3
  235. localstack/services/ecr/resource_providers/aws_ecr_repository.py +22 -19
  236. localstack/services/ecr/resource_providers/aws_ecr_repository_plugin.py +1 -3
  237. localstack/services/edge.py +6 -6
  238. localstack/services/es/provider.py +21 -21
  239. localstack/services/events/archive.py +2 -2
  240. localstack/services/events/connection.py +5 -5
  241. localstack/services/events/event_bus.py +9 -9
  242. localstack/services/events/event_rule_engine.py +31 -13
  243. localstack/services/events/models.py +29 -30
  244. localstack/services/events/provider.py +29 -26
  245. localstack/services/events/replay.py +3 -3
  246. localstack/services/events/resource_providers/aws_events_apidestination.py +8 -8
  247. localstack/services/events/resource_providers/aws_events_apidestination_plugin.py +1 -3
  248. localstack/services/events/resource_providers/aws_events_connection.py +27 -27
  249. localstack/services/events/resource_providers/aws_events_connection_plugin.py +1 -3
  250. localstack/services/events/resource_providers/aws_events_eventbus.py +9 -9
  251. localstack/services/events/resource_providers/aws_events_eventbus_plugin.py +1 -3
  252. localstack/services/events/resource_providers/aws_events_eventbuspolicy.py +11 -11
  253. localstack/services/events/resource_providers/aws_events_eventbuspolicy_plugin.py +1 -3
  254. localstack/services/events/resource_providers/aws_events_rule.py +82 -82
  255. localstack/services/events/resource_providers/aws_events_rule_plugin.py +1 -3
  256. localstack/services/events/rule.py +15 -15
  257. localstack/services/events/target.py +21 -13
  258. localstack/services/events/utils.py +7 -7
  259. localstack/services/events/v1/models.py +1 -3
  260. localstack/services/events/v1/provider.py +15 -15
  261. localstack/services/firehose/models.py +1 -3
  262. localstack/services/firehose/provider.py +25 -16
  263. localstack/services/iam/iam_patches.py +5 -6
  264. localstack/services/iam/provider.py +14 -119
  265. localstack/services/iam/resource_providers/aws_iam_accesskey.py +6 -6
  266. localstack/services/iam/resource_providers/aws_iam_accesskey_plugin.py +1 -3
  267. localstack/services/iam/resource_providers/aws_iam_group.py +9 -9
  268. localstack/services/iam/resource_providers/aws_iam_group_plugin.py +1 -3
  269. localstack/services/iam/resource_providers/aws_iam_instanceprofile.py +5 -5
  270. localstack/services/iam/resource_providers/aws_iam_instanceprofile_plugin.py +1 -3
  271. localstack/services/iam/resource_providers/aws_iam_managedpolicy.py +9 -9
  272. localstack/services/iam/resource_providers/aws_iam_managedpolicy_plugin.py +1 -3
  273. localstack/services/iam/resource_providers/aws_iam_policy.py +7 -7
  274. localstack/services/iam/resource_providers/aws_iam_policy_plugin.py +1 -3
  275. localstack/services/iam/resource_providers/aws_iam_role.py +16 -16
  276. localstack/services/iam/resource_providers/aws_iam_role_plugin.py +1 -3
  277. localstack/services/iam/resource_providers/aws_iam_servercertificate.py +10 -10
  278. localstack/services/iam/resource_providers/aws_iam_servercertificate_plugin.py +1 -3
  279. localstack/services/iam/resource_providers/aws_iam_servicelinkedrole.py +5 -5
  280. localstack/services/iam/resource_providers/aws_iam_servicelinkedrole_plugin.py +1 -3
  281. localstack/services/iam/resource_providers/aws_iam_user.py +17 -17
  282. localstack/services/iam/resource_providers/aws_iam_user_plugin.py +1 -3
  283. localstack/services/iam/resources/policy_simulator.py +133 -0
  284. localstack/services/kinesis/kinesis_mock_server.py +7 -8
  285. localstack/services/kinesis/models.py +17 -5
  286. localstack/services/kinesis/packages.py +3 -3
  287. localstack/services/kinesis/provider.py +86 -3
  288. localstack/services/kinesis/resource_providers/aws_kinesis_stream.py +13 -13
  289. localstack/services/kinesis/resource_providers/aws_kinesis_stream_plugin.py +1 -3
  290. localstack/services/kinesis/resource_providers/aws_kinesis_streamconsumer.py +7 -7
  291. localstack/services/kinesis/resource_providers/aws_kinesis_streamconsumer_plugin.py +1 -3
  292. localstack/services/kinesisfirehose/resource_providers/aws_kinesisfirehose_deliverystream.py +191 -191
  293. localstack/services/kinesisfirehose/resource_providers/aws_kinesisfirehose_deliverystream_plugin.py +1 -3
  294. localstack/services/kms/models.py +21 -18
  295. localstack/services/kms/provider.py +62 -26
  296. localstack/services/kms/resource_providers/aws_kms_alias.py +3 -3
  297. localstack/services/kms/resource_providers/aws_kms_alias_plugin.py +1 -3
  298. localstack/services/kms/resource_providers/aws_kms_key.py +14 -14
  299. localstack/services/kms/resource_providers/aws_kms_key_plugin.py +1 -3
  300. localstack/services/kms/utils.py +4 -3
  301. localstack/services/lambda_/api_utils.py +19 -16
  302. localstack/services/lambda_/custom_endpoints.py +2 -2
  303. localstack/services/lambda_/event_source_mapping/esm_worker_factory.py +4 -7
  304. localstack/services/lambda_/event_source_mapping/pipe_utils.py +2 -2
  305. localstack/services/lambda_/event_source_mapping/pollers/stream_poller.py +1 -1
  306. localstack/services/lambda_/event_source_mapping/senders/sender_utils.py +2 -1
  307. localstack/services/lambda_/hooks.py +6 -1
  308. localstack/services/lambda_/invocation/assignment.py +1 -2
  309. localstack/services/lambda_/invocation/docker_runtime_executor.py +7 -11
  310. localstack/services/lambda_/invocation/event_manager.py +1 -1
  311. localstack/services/lambda_/invocation/execution_environment.py +4 -4
  312. localstack/services/lambda_/invocation/executor_endpoint.py +8 -11
  313. localstack/services/lambda_/invocation/internal_sqs_queue.py +6 -10
  314. localstack/services/lambda_/invocation/lambda_models.py +33 -30
  315. localstack/services/lambda_/invocation/lambda_service.py +12 -5
  316. localstack/services/lambda_/invocation/logs.py +2 -3
  317. localstack/services/lambda_/invocation/runtime_executor.py +3 -3
  318. localstack/services/lambda_/invocation/version_manager.py +31 -8
  319. localstack/services/lambda_/ldm.py +14 -0
  320. localstack/services/lambda_/packages.py +3 -4
  321. localstack/services/lambda_/provider.py +9 -26
  322. localstack/services/lambda_/provider_utils.py +1 -1
  323. localstack/services/lambda_/resource_providers/aws_lambda_codesigningconfig.py +8 -8
  324. localstack/services/lambda_/resource_providers/aws_lambda_codesigningconfig_plugin.py +1 -3
  325. localstack/services/lambda_/resource_providers/aws_lambda_eventinvokeconfig.py +11 -11
  326. localstack/services/lambda_/resource_providers/aws_lambda_eventinvokeconfig_plugin.py +1 -3
  327. localstack/services/lambda_/resource_providers/aws_lambda_eventsourcemapping.py +39 -39
  328. localstack/services/lambda_/resource_providers/aws_lambda_eventsourcemapping_plugin.py +1 -3
  329. localstack/services/lambda_/resource_providers/aws_lambda_function.py +54 -54
  330. localstack/services/lambda_/resource_providers/aws_lambda_function_plugin.py +1 -3
  331. localstack/services/lambda_/resource_providers/aws_lambda_layerversion.py +11 -11
  332. localstack/services/lambda_/resource_providers/aws_lambda_layerversion_plugin.py +1 -3
  333. localstack/services/lambda_/resource_providers/aws_lambda_layerversionpermission.py +6 -6
  334. localstack/services/lambda_/resource_providers/aws_lambda_layerversionpermission_plugin.py +1 -3
  335. localstack/services/lambda_/resource_providers/aws_lambda_permission.py +10 -10
  336. localstack/services/lambda_/resource_providers/aws_lambda_permission_plugin.py +1 -3
  337. localstack/services/lambda_/resource_providers/aws_lambda_url.py +15 -15
  338. localstack/services/lambda_/resource_providers/aws_lambda_url_plugin.py +1 -3
  339. localstack/services/lambda_/resource_providers/aws_lambda_version.py +8 -8
  340. localstack/services/lambda_/resource_providers/aws_lambda_version_plugin.py +1 -3
  341. localstack/services/lambda_/resource_providers/lambda_alias.py +12 -12
  342. localstack/services/lambda_/resource_providers/lambda_alias_plugin.py +1 -3
  343. localstack/services/lambda_/runtimes.py +1 -3
  344. localstack/services/lambda_/urlrouter.py +14 -1
  345. localstack/services/logs/models.py +1 -3
  346. localstack/services/logs/provider.py +39 -22
  347. localstack/services/logs/resource_providers/aws_logs_loggroup.py +9 -9
  348. localstack/services/logs/resource_providers/aws_logs_loggroup_plugin.py +1 -3
  349. localstack/services/logs/resource_providers/aws_logs_logstream.py +4 -4
  350. localstack/services/logs/resource_providers/aws_logs_logstream_plugin.py +1 -3
  351. localstack/services/logs/resource_providers/aws_logs_subscriptionfilter.py +7 -7
  352. localstack/services/logs/resource_providers/aws_logs_subscriptionfilter_plugin.py +1 -3
  353. localstack/services/moto.py +5 -4
  354. localstack/services/opensearch/cluster.py +28 -20
  355. localstack/services/opensearch/cluster_manager.py +13 -14
  356. localstack/services/opensearch/models.py +1 -3
  357. localstack/services/opensearch/packages.py +27 -9
  358. localstack/services/opensearch/provider.py +15 -10
  359. localstack/services/opensearch/resource_providers/aws_elasticsearch_domain.py +61 -61
  360. localstack/services/opensearch/resource_providers/aws_elasticsearch_domain_plugin.py +1 -3
  361. localstack/services/opensearch/resource_providers/aws_opensearchservice_domain.py +89 -89
  362. localstack/services/opensearch/resource_providers/aws_opensearchservice_domain_plugin.py +1 -3
  363. localstack/services/opensearch/versions.py +57 -10
  364. localstack/services/plugins.py +37 -32
  365. localstack/services/providers.py +10 -2
  366. localstack/services/redshift/provider.py +0 -21
  367. localstack/services/redshift/resource_providers/aws_redshift_cluster.py +57 -57
  368. localstack/services/redshift/resource_providers/aws_redshift_cluster_plugin.py +1 -3
  369. localstack/services/resource_groups/resource_providers/aws_resourcegroups_group.py +21 -21
  370. localstack/services/resource_groups/resource_providers/aws_resourcegroups_group_plugin.py +1 -3
  371. localstack/services/route53/models.py +1 -3
  372. localstack/services/route53/provider.py +1 -2
  373. localstack/services/route53/resource_providers/aws_route53_healthcheck.py +6 -6
  374. localstack/services/route53/resource_providers/aws_route53_healthcheck_plugin.py +1 -3
  375. localstack/services/route53/resource_providers/aws_route53_recordset.py +27 -27
  376. localstack/services/route53/resource_providers/aws_route53_recordset_plugin.py +1 -3
  377. localstack/services/route53resolver/models.py +8 -10
  378. localstack/services/route53resolver/provider.py +12 -12
  379. localstack/services/s3/codec.py +2 -2
  380. localstack/services/s3/constants.py +5 -2
  381. localstack/services/s3/cors.py +8 -8
  382. localstack/services/s3/models.py +73 -73
  383. localstack/services/s3/notifications.py +74 -58
  384. localstack/services/s3/presigned_url.py +41 -59
  385. localstack/services/s3/provider.py +86 -29
  386. localstack/services/s3/resource_providers/aws_s3_bucket.py +180 -180
  387. localstack/services/s3/resource_providers/aws_s3_bucket_plugin.py +1 -3
  388. localstack/services/s3/resource_providers/aws_s3_bucketpolicy.py +4 -4
  389. localstack/services/s3/resource_providers/aws_s3_bucketpolicy_plugin.py +1 -3
  390. localstack/services/s3/storage/core.py +4 -3
  391. localstack/services/s3/storage/ephemeral.py +7 -6
  392. localstack/services/s3/utils.py +51 -31
  393. localstack/services/s3/validation.py +47 -33
  394. localstack/services/s3/website_hosting.py +9 -7
  395. localstack/services/scheduler/resource_providers/aws_scheduler_schedule.py +60 -60
  396. localstack/services/scheduler/resource_providers/aws_scheduler_schedule_plugin.py +1 -3
  397. localstack/services/scheduler/resource_providers/aws_scheduler_schedulegroup.py +9 -9
  398. localstack/services/scheduler/resource_providers/aws_scheduler_schedulegroup_plugin.py +1 -3
  399. localstack/services/secretsmanager/provider.py +10 -12
  400. localstack/services/secretsmanager/resource_providers/aws_secretsmanager_resourcepolicy.py +5 -5
  401. localstack/services/secretsmanager/resource_providers/aws_secretsmanager_resourcepolicy_plugin.py +1 -3
  402. localstack/services/secretsmanager/resource_providers/aws_secretsmanager_rotationschedule.py +21 -21
  403. localstack/services/secretsmanager/resource_providers/aws_secretsmanager_rotationschedule_plugin.py +1 -3
  404. localstack/services/secretsmanager/resource_providers/aws_secretsmanager_secret.py +23 -23
  405. localstack/services/secretsmanager/resource_providers/aws_secretsmanager_secret_plugin.py +1 -3
  406. localstack/services/secretsmanager/resource_providers/aws_secretsmanager_secrettargetattachment.py +5 -5
  407. localstack/services/secretsmanager/resource_providers/aws_secretsmanager_secrettargetattachment_plugin.py +1 -3
  408. localstack/services/ses/models.py +8 -1
  409. localstack/services/ses/provider.py +128 -55
  410. localstack/services/ses/resource_providers/aws_ses_emailidentity.py +21 -21
  411. localstack/services/ses/resource_providers/aws_ses_emailidentity_plugin.py +1 -3
  412. localstack/services/sns/constants.py +7 -1
  413. localstack/services/sns/executor.py +9 -2
  414. localstack/services/sns/models.py +25 -25
  415. localstack/services/sns/provider.py +9 -7
  416. localstack/services/sns/publisher.py +37 -23
  417. localstack/services/sns/resource_providers/aws_sns_subscription.py +13 -13
  418. localstack/services/sns/resource_providers/aws_sns_subscription_plugin.py +1 -3
  419. localstack/services/sns/resource_providers/aws_sns_topic.py +16 -16
  420. localstack/services/sns/resource_providers/aws_sns_topic_plugin.py +1 -3
  421. localstack/services/sns/resource_providers/aws_sns_topicpolicy.py +4 -4
  422. localstack/services/sns/resource_providers/aws_sns_topicpolicy_plugin.py +1 -3
  423. localstack/services/sns/v2/models.py +167 -0
  424. localstack/services/sns/v2/provider.py +867 -0
  425. localstack/services/sns/v2/utils.py +130 -0
  426. localstack/services/sqs/constants.py +2 -2
  427. localstack/services/sqs/developer_api.py +205 -0
  428. localstack/services/sqs/models.py +71 -36
  429. localstack/services/sqs/provider.py +56 -332
  430. localstack/services/sqs/query_api.py +8 -5
  431. localstack/services/sqs/resource_providers/aws_sqs_queue.py +21 -21
  432. localstack/services/sqs/resource_providers/aws_sqs_queue_plugin.py +1 -3
  433. localstack/services/sqs/resource_providers/aws_sqs_queuepolicy.py +7 -5
  434. localstack/services/sqs/resource_providers/aws_sqs_queuepolicy_plugin.py +1 -3
  435. localstack/services/sqs/utils.py +123 -4
  436. localstack/services/ssm/provider.py +3 -4
  437. localstack/services/ssm/resource_providers/aws_ssm_maintenancewindow.py +15 -15
  438. localstack/services/ssm/resource_providers/aws_ssm_maintenancewindow_plugin.py +1 -3
  439. localstack/services/ssm/resource_providers/aws_ssm_maintenancewindowtarget.py +10 -10
  440. localstack/services/ssm/resource_providers/aws_ssm_maintenancewindowtarget_plugin.py +1 -3
  441. localstack/services/ssm/resource_providers/aws_ssm_maintenancewindowtask.py +48 -48
  442. localstack/services/ssm/resource_providers/aws_ssm_maintenancewindowtask_plugin.py +1 -3
  443. localstack/services/ssm/resource_providers/aws_ssm_parameter.py +10 -10
  444. localstack/services/ssm/resource_providers/aws_ssm_parameter_plugin.py +1 -3
  445. localstack/services/ssm/resource_providers/aws_ssm_patchbaseline.py +29 -29
  446. localstack/services/ssm/resource_providers/aws_ssm_patchbaseline_plugin.py +1 -3
  447. localstack/services/stepfunctions/asl/antlt4utils/antlr4utils.py +3 -4
  448. localstack/services/stepfunctions/asl/component/common/assign/assign_decl.py +1 -1
  449. localstack/services/stepfunctions/asl/component/common/assign/assign_decl_binding.py +1 -1
  450. localstack/services/stepfunctions/asl/component/common/assign/assign_template_value_array.py +1 -1
  451. localstack/services/stepfunctions/asl/component/common/assign/assign_template_value_object.py +1 -1
  452. localstack/services/stepfunctions/asl/component/common/catch/catcher_decl.py +9 -9
  453. localstack/services/stepfunctions/asl/component/common/error_name/custom_error_name.py +2 -2
  454. localstack/services/stepfunctions/asl/component/common/error_name/error_name.py +4 -4
  455. localstack/services/stepfunctions/asl/component/common/error_name/failure_event.py +8 -8
  456. localstack/services/stepfunctions/asl/component/common/error_name/states_error_name_type.py +1 -1
  457. localstack/services/stepfunctions/asl/component/common/jsonata/jsonata_template_binding.py +2 -2
  458. localstack/services/stepfunctions/asl/component/common/jsonata/jsonata_template_value_array.py +1 -1
  459. localstack/services/stepfunctions/asl/component/common/jsonata/jsonata_template_value_object.py +1 -1
  460. localstack/services/stepfunctions/asl/component/common/path/input_path.py +4 -4
  461. localstack/services/stepfunctions/asl/component/common/path/output_path.py +4 -4
  462. localstack/services/stepfunctions/asl/component/common/path/result_path.py +3 -3
  463. localstack/services/stepfunctions/asl/component/common/payload/payloadvalue/payloadarr/payload_arr.py +1 -1
  464. localstack/services/stepfunctions/asl/component/common/payload/payloadvalue/payloadbinding/payload_binding.py +3 -3
  465. localstack/services/stepfunctions/asl/component/common/payload/payloadvalue/payloadtmpl/payload_tmpl.py +1 -1
  466. localstack/services/stepfunctions/asl/component/common/retry/retrier_decl.py +8 -8
  467. localstack/services/stepfunctions/asl/component/common/string/string_expression.py +3 -3
  468. localstack/services/stepfunctions/asl/component/common/timeouts/timeout.py +3 -3
  469. localstack/services/stepfunctions/asl/component/eval_component.py +2 -3
  470. localstack/services/stepfunctions/asl/component/intrinsic/argument/argument.py +4 -4
  471. localstack/services/stepfunctions/asl/component/intrinsic/function/statesfunction/array/array_partition.py +1 -1
  472. localstack/services/stepfunctions/asl/component/intrinsic/function/statesfunction/generic/string_format.py +1 -1
  473. localstack/services/stepfunctions/asl/component/intrinsic/function/statesfunction/states_function_array.py +1 -1
  474. localstack/services/stepfunctions/asl/component/intrinsic/function/statesfunction/states_function_format.py +1 -1
  475. localstack/services/stepfunctions/asl/component/intrinsic/jsonata.py +3 -3
  476. localstack/services/stepfunctions/asl/component/intrinsic/member.py +1 -1
  477. localstack/services/stepfunctions/asl/component/program/program.py +9 -9
  478. localstack/services/stepfunctions/asl/component/program/states.py +1 -1
  479. localstack/services/stepfunctions/asl/component/state/state.py +10 -11
  480. localstack/services/stepfunctions/asl/component/state/state_choice/choice_rule.py +11 -11
  481. localstack/services/stepfunctions/asl/component/state/state_choice/comparison/comparison.py +5 -11
  482. localstack/services/stepfunctions/asl/component/state/state_choice/comparison/operator/implementations/is_operator.py +2 -2
  483. localstack/services/stepfunctions/asl/component/state/state_choice/state_choice.py +3 -5
  484. localstack/services/stepfunctions/asl/component/state/state_execution/execute_state.py +9 -9
  485. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/item_reader/item_reader_decl.py +7 -7
  486. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/item_reader/reader_config/reader_config_decl.py +5 -5
  487. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/item_reader/resource_eval/resource_eval_s3.py +2 -1
  488. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/item_reader/resource_eval/resource_output_transformer/resource_output_transformer_csv.py +2 -2
  489. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/distributed_iteration_component.py +8 -8
  490. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/inline_iteration_component.py +7 -7
  491. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/itemprocessor/distributed_item_processor_worker.py +4 -4
  492. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/itemprocessor/inline_item_processor_worker.py +5 -5
  493. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/itemprocessor/map_run_record.py +10 -10
  494. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/iteration_component.py +3 -3
  495. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/iteration_declaration.py +3 -3
  496. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/iteration_worker.py +2 -2
  497. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/iterator/distributed_iterator_worker.py +3 -5
  498. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/iterator/inline_iterator.py +1 -2
  499. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/iterator/inline_iterator_worker.py +5 -5
  500. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/iteration/job.py +9 -9
  501. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/result_writer/resource_eval/resource_eval_s3.py +2 -1
  502. localstack/services/stepfunctions/asl/component/state/state_execution/state_map/state_map.py +12 -13
  503. localstack/services/stepfunctions/asl/component/state/state_execution/state_parallel/branch_worker.py +3 -3
  504. localstack/services/stepfunctions/asl/component/state/state_execution/state_parallel/branches_decl.py +7 -7
  505. localstack/services/stepfunctions/asl/component/state/state_execution/state_parallel/state_parallel.py +2 -3
  506. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/lambda_eval_utils.py +7 -7
  507. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/resource.py +3 -3
  508. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service.py +10 -10
  509. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_api_gateway.py +18 -18
  510. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_aws_sdk.py +1 -1
  511. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_batch.py +8 -7
  512. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_callback.py +18 -17
  513. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_dynamodb.py +2 -2
  514. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_ecs.py +5 -4
  515. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_events.py +4 -4
  516. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_glue.py +7 -6
  517. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_lambda.py +4 -4
  518. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_sfn.py +9 -8
  519. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_sns.py +2 -2
  520. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_sqs.py +4 -4
  521. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/state_task.py +5 -5
  522. localstack/services/stepfunctions/asl/component/state/state_execution/state_task/state_task_lambda.py +2 -2
  523. localstack/services/stepfunctions/asl/component/state/state_fail/state_fail.py +3 -5
  524. localstack/services/stepfunctions/asl/component/state/state_pass/state_pass.py +5 -7
  525. localstack/services/stepfunctions/asl/component/state/state_succeed/state_succeed.py +1 -1
  526. localstack/services/stepfunctions/asl/component/state/state_wait/state_wait.py +1 -1
  527. localstack/services/stepfunctions/asl/component/state/state_wait/wait_function/timestamp.py +2 -2
  528. localstack/services/stepfunctions/asl/eval/callback/callback.py +18 -18
  529. localstack/services/stepfunctions/asl/eval/environment.py +22 -24
  530. localstack/services/stepfunctions/asl/eval/evaluation_details.py +3 -5
  531. localstack/services/stepfunctions/asl/eval/event/event_manager.py +10 -10
  532. localstack/services/stepfunctions/asl/eval/event/logging.py +3 -3
  533. localstack/services/stepfunctions/asl/eval/program_state.py +8 -8
  534. localstack/services/stepfunctions/asl/eval/states.py +12 -12
  535. localstack/services/stepfunctions/asl/eval/test_state/environment.py +3 -5
  536. localstack/services/stepfunctions/asl/eval/variable_store.py +6 -6
  537. localstack/services/stepfunctions/asl/jsonata/jsonata.py +7 -6
  538. localstack/services/stepfunctions/asl/parse/asl_parser.py +1 -1
  539. localstack/services/stepfunctions/asl/parse/intrinsic/preprocessor.py +2 -3
  540. localstack/services/stepfunctions/asl/parse/preprocessor.py +44 -44
  541. localstack/services/stepfunctions/asl/parse/typed_props.py +2 -2
  542. localstack/services/stepfunctions/asl/static_analyser/intrinsic/variable_names_intrinsic_static_analyser.py +1 -1
  543. localstack/services/stepfunctions/asl/static_analyser/variable_references_static_analyser.py +2 -2
  544. localstack/services/stepfunctions/asl/utils/encoding.py +2 -2
  545. localstack/services/stepfunctions/asl/utils/json_path.py +2 -2
  546. localstack/services/stepfunctions/backend/activity.py +4 -4
  547. localstack/services/stepfunctions/backend/alias.py +8 -8
  548. localstack/services/stepfunctions/backend/execution.py +29 -30
  549. localstack/services/stepfunctions/backend/execution_worker.py +7 -7
  550. localstack/services/stepfunctions/backend/state_machine.py +28 -28
  551. localstack/services/stepfunctions/backend/test_state/execution.py +3 -4
  552. localstack/services/stepfunctions/backend/test_state/execution_worker.py +1 -3
  553. localstack/services/stepfunctions/mocking/mock_config.py +9 -9
  554. localstack/services/stepfunctions/mocking/mock_config_file.py +10 -10
  555. localstack/services/stepfunctions/packages.py +14 -5
  556. localstack/services/stepfunctions/provider.py +34 -44
  557. localstack/services/stepfunctions/quotas.py +2 -3
  558. localstack/services/stepfunctions/resource_providers/aws_stepfunctions_activity.py +6 -6
  559. localstack/services/stepfunctions/resource_providers/aws_stepfunctions_activity_plugin.py +1 -3
  560. localstack/services/stepfunctions/resource_providers/aws_stepfunctions_statemachine.py +25 -25
  561. localstack/services/stepfunctions/resource_providers/aws_stepfunctions_statemachine_plugin.py +1 -3
  562. localstack/services/stepfunctions/stepfunctions_utils.py +1 -2
  563. localstack/services/stores.py +8 -8
  564. localstack/services/transcribe/packages.py +1 -3
  565. localstack/services/transcribe/provider.py +8 -3
  566. localstack/state/codecs.py +61 -0
  567. localstack/state/core.py +11 -5
  568. localstack/state/inspect.py +4 -4
  569. localstack/state/pickle.py +36 -23
  570. localstack/testing/aws/asf_utils.py +3 -2
  571. localstack/testing/aws/cloudformation_utils.py +1 -1
  572. localstack/testing/aws/lambda_utils.py +15 -14
  573. localstack/testing/aws/util.py +3 -2
  574. localstack/testing/pytest/cloudformation/fixtures.py +68 -18
  575. localstack/testing/pytest/container.py +5 -5
  576. localstack/testing/pytest/filters.py +1 -3
  577. localstack/testing/pytest/fixtures.py +188 -49
  578. localstack/testing/pytest/in_memory_localstack.py +1 -3
  579. localstack/testing/pytest/marking.py +42 -15
  580. localstack/testing/pytest/path_filter.py +1 -1
  581. localstack/testing/pytest/stepfunctions/fixtures.py +4 -4
  582. localstack/testing/pytest/stepfunctions/utils.py +11 -10
  583. localstack/testing/pytest/util.py +1 -1
  584. localstack/testing/pytest/validation_tracking.py +3 -4
  585. localstack/testing/scenario/provisioning.py +11 -10
  586. localstack/testing/snapshots/transformer_utility.py +8 -3
  587. localstack/testing/testselection/matching.py +2 -2
  588. localstack/testing/testselection/opt_out.py +1 -1
  589. localstack/testing/testselection/scripts/filter_by_test_selection.py +1 -1
  590. localstack/testing/testselection/scripts/generate_test_selection.py +1 -1
  591. localstack/testing/testselection/testselection.py +2 -2
  592. localstack/utils/analytics/cli.py +2 -3
  593. localstack/utils/analytics/client.py +5 -5
  594. localstack/utils/analytics/events.py +2 -2
  595. localstack/utils/analytics/metadata.py +6 -4
  596. localstack/utils/analytics/metrics/counter.py +11 -18
  597. localstack/utils/analytics/metrics/registry.py +2 -2
  598. localstack/utils/analytics/publisher.py +4 -5
  599. localstack/utils/analytics/service_providers.py +19 -0
  600. localstack/utils/analytics/service_request_aggregator.py +4 -4
  601. localstack/utils/archives.py +12 -12
  602. localstack/utils/asyncio.py +2 -2
  603. localstack/utils/aws/arns.py +26 -31
  604. localstack/utils/aws/aws_responses.py +21 -28
  605. localstack/utils/aws/aws_stack.py +7 -12
  606. localstack/utils/aws/dead_letter_queue.py +4 -9
  607. localstack/utils/aws/message_forwarding.py +8 -11
  608. localstack/utils/aws/request_context.py +5 -6
  609. localstack/utils/aws/resources.py +1 -1
  610. localstack/utils/aws/templating.py +4 -4
  611. localstack/utils/batch_policy.py +4 -4
  612. localstack/utils/bootstrap.py +37 -30
  613. localstack/utils/catalog/catalog.py +139 -0
  614. localstack/utils/catalog/catalog_loader.py +119 -0
  615. localstack/utils/catalog/common.py +58 -0
  616. localstack/utils/catalog/plugins.py +28 -0
  617. localstack/utils/cloudwatch/cloudwatch_util.py +5 -5
  618. localstack/utils/collections.py +33 -27
  619. localstack/utils/config_listener.py +2 -2
  620. localstack/utils/container_networking.py +5 -6
  621. localstack/utils/container_utils/container_client.py +156 -160
  622. localstack/utils/container_utils/docker_cmd_client.py +97 -81
  623. localstack/utils/container_utils/docker_sdk_client.py +75 -72
  624. localstack/utils/crypto.py +12 -13
  625. localstack/utils/diagnose.py +11 -12
  626. localstack/utils/docker_utils.py +11 -7
  627. localstack/utils/files.py +34 -15
  628. localstack/utils/functions.py +5 -4
  629. localstack/utils/http.py +14 -14
  630. localstack/utils/iputils.py +2 -1
  631. localstack/utils/json.py +21 -7
  632. localstack/utils/kinesis/kinesis_connector.py +2 -1
  633. localstack/utils/net.py +25 -17
  634. localstack/utils/no_exit_argument_parser.py +2 -2
  635. localstack/utils/numbers.py +9 -2
  636. localstack/utils/objects.py +15 -14
  637. localstack/utils/patch.py +14 -7
  638. localstack/utils/platform.py +2 -2
  639. localstack/utils/run.py +15 -14
  640. localstack/utils/scheduler.py +13 -12
  641. localstack/utils/server/tcp_proxy.py +2 -2
  642. localstack/utils/serving.py +3 -4
  643. localstack/utils/strings.py +15 -16
  644. localstack/utils/sync.py +126 -1
  645. localstack/utils/tagging.py +10 -8
  646. localstack/utils/testutil.py +17 -17
  647. localstack/utils/threads.py +3 -3
  648. localstack/utils/time.py +12 -4
  649. localstack/utils/urls.py +1 -3
  650. localstack/utils/xml.py +1 -1
  651. localstack/utils/xray/traceid.py +1 -1
  652. localstack/version.py +16 -3
  653. {localstack_core-4.6.1.dev60.dist-info → localstack_core-4.10.1.dev12.dist-info}/METADATA +18 -13
  654. {localstack_core-4.6.1.dev60.dist-info → localstack_core-4.10.1.dev12.dist-info}/RECORD +663 -655
  655. {localstack_core-4.6.1.dev60.dist-info → localstack_core-4.10.1.dev12.dist-info}/entry_points.txt +8 -4
  656. localstack_core-4.10.1.dev12.dist-info/plux.json +1 -0
  657. localstack/packages/terraform.py +0 -47
  658. localstack/services/cloudformation/deploy.html +0 -144
  659. localstack/services/cloudformation/deploy_ui.py +0 -47
  660. localstack/services/cloudformation/plugins.py +0 -12
  661. localstack/services/lambda_/lambda_debug_mode/ldm.py +0 -375
  662. localstack/services/lambda_/lambda_debug_mode/ldm_config_file.py +0 -178
  663. localstack/services/lambda_/lambda_debug_mode/ldm_types.py +0 -11
  664. localstack/services/lambda_/lambda_debug_mode/ldm_utils.py +0 -43
  665. localstack_core-4.6.1.dev60.dist-info/plux.json +0 -1
  666. /localstack/{services/lambda_/lambda_debug_mode/__init__.py → testing/pytest/cloudformation/transformers.py} +0 -0
  667. {localstack_core-4.6.1.dev60.data → localstack_core-4.10.1.dev12.data}/scripts/localstack +0 -0
  668. {localstack_core-4.6.1.dev60.data → localstack_core-4.10.1.dev12.data}/scripts/localstack-supervisor +0 -0
  669. {localstack_core-4.6.1.dev60.data → localstack_core-4.10.1.dev12.data}/scripts/localstack.bat +0 -0
  670. {localstack_core-4.6.1.dev60.dist-info → localstack_core-4.10.1.dev12.dist-info}/WHEEL +0 -0
  671. {localstack_core-4.6.1.dev60.dist-info → localstack_core-4.10.1.dev12.dist-info}/licenses/LICENSE.txt +0 -0
  672. {localstack_core-4.6.1.dev60.dist-info → localstack_core-4.10.1.dev12.dist-info}/top_level.txt +0 -0
@@ -6,8 +6,9 @@ import logging
6
6
  import re
7
7
  import time
8
8
  from collections import namedtuple
9
+ from collections.abc import Mapping
9
10
  from functools import cache, cached_property
10
- from typing import Mapping, Optional, TypedDict
11
+ from typing import TypedDict
11
12
  from urllib import parse as urlparse
12
13
 
13
14
  from botocore.auth import HmacV1QueryAuth, S3SigV4QueryAuth
@@ -39,15 +40,13 @@ from localstack.http.request import get_raw_path
39
40
  from localstack.services.s3.constants import (
40
41
  DEFAULT_PRE_SIGNED_ACCESS_KEY_ID,
41
42
  DEFAULT_PRE_SIGNED_SECRET_ACCESS_KEY,
43
+ S3_HOST_ID,
42
44
  SIGNATURE_V2_PARAMS,
43
45
  SIGNATURE_V4_PARAMS,
44
46
  )
45
47
  from localstack.services.s3.utils import (
46
- S3_VIRTUAL_HOST_FORWARDED_HEADER,
47
- _create_invalid_argument_exc,
48
48
  capitalize_header_name_from_snake_case,
49
49
  extract_bucket_name_and_key_from_headers_and_path,
50
- forwarded_from_virtual_host_addressed_request,
51
50
  is_bucket_name_valid,
52
51
  is_presigned_url_request,
53
52
  uses_host_addressing,
@@ -85,8 +84,6 @@ IGNORED_SIGV4_HEADERS = [
85
84
  "x-amz-content-sha256",
86
85
  ]
87
86
 
88
- FAKE_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg="
89
-
90
87
  HOST_COMBINATION_REGEX = r"^(.*)(:[\d]{0,6})"
91
88
  PORT_REPLACEMENT = [":80", ":443", f":{config.GATEWAY_LISTEN[0].port}", ""]
92
89
 
@@ -105,7 +102,7 @@ class NotValidSigV4SignatureContext(TypedDict):
105
102
  canonical_request: str
106
103
 
107
104
 
108
- FindSigV4Result = tuple["S3SigV4SignatureContext", Optional[NotValidSigV4SignatureContext]]
105
+ FindSigV4Result = tuple["S3SigV4SignatureContext", NotValidSigV4SignatureContext | None]
109
106
 
110
107
 
111
108
  class HmacV1QueryAuthValidation(HmacV1QueryAuth):
@@ -156,7 +153,7 @@ def create_signature_does_not_match_sig_v2(
156
153
  "The request signature we calculated does not match the signature you provided. Check your key and signing method."
157
154
  )
158
155
  ex.AWSAccessKeyId = access_key_id
159
- ex.HostId = FAKE_HOST_ID
156
+ ex.HostId = S3_HOST_ID
160
157
  ex.SignatureProvided = request_signature
161
158
  ex.StringToSign = string_to_sign
162
159
  ex.StringToSignBytes = to_bytes(string_to_sign).hex(sep=" ", bytes_per_sep=2).upper()
@@ -258,7 +255,7 @@ def get_credentials_from_parameters(parameters: dict, region: str) -> PreSignedC
258
255
 
259
256
 
260
257
  @cache
261
- def get_secret_access_key_from_access_key_id(access_key_id: str, region: str) -> Optional[str]:
258
+ def get_secret_access_key_from_access_key_id(access_key_id: str, region: str) -> str | None:
262
259
  """
263
260
  We need to retrieve the internal secret access key in order to also sign the request on our side to validate it
264
261
  For now, we need to access Moto internals, as they are no public APIs to retrieve it for obvious reasons.
@@ -299,7 +296,7 @@ def is_valid_sig_v2(query_args: set) -> bool:
299
296
  LOG.info("Presign signature calculation failed")
300
297
  raise AccessDenied(
301
298
  "Query-string authentication requires the Signature, Expires and AWSAccessKeyId parameters",
302
- HostId=FAKE_HOST_ID,
299
+ HostId=S3_HOST_ID,
303
300
  )
304
301
 
305
302
  return True
@@ -317,7 +314,7 @@ def is_valid_sig_v4(query_args: set) -> bool:
317
314
  LOG.info("Presign signature calculation failed")
318
315
  raise AuthorizationQueryParametersError(
319
316
  "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
320
- HostId=FAKE_HOST_ID,
317
+ HostId=S3_HOST_ID,
321
318
  )
322
319
 
323
320
  return True
@@ -351,7 +348,7 @@ def validate_presigned_url_s3(context: RequestContext) -> None:
351
348
  )
352
349
  else:
353
350
  raise AccessDenied(
354
- "Request has expired", HostId=FAKE_HOST_ID, Expires=expires, ServerTime=time.time()
351
+ "Request has expired", HostId=S3_HOST_ID, Expires=expires, ServerTime=time.time()
355
352
  )
356
353
 
357
354
  auth_signer = HmacV1QueryAuthValidation(credentials=signing_credentials, expires=expires)
@@ -450,7 +447,7 @@ def validate_presigned_url_s3v4(context: RequestContext) -> None:
450
447
  else:
451
448
  raise AccessDenied(
452
449
  "There were headers present in the request which were not signed",
453
- HostId=FAKE_HOST_ID,
450
+ HostId=S3_HOST_ID,
454
451
  HeadersNotSigned=", ".join(sigv4_context.missing_signed_headers),
455
452
  )
456
453
 
@@ -472,7 +469,7 @@ def validate_presigned_url_s3v4(context: RequestContext) -> None:
472
469
  x_amz_expires = int(query_parameters["X-Amz-Expires"])
473
470
  x_amz_expires_dt = datetime.timedelta(seconds=x_amz_expires)
474
471
  expiration_time = x_amz_date + x_amz_expires_dt
475
- expiration_time = expiration_time.replace(tzinfo=datetime.timezone.utc)
472
+ expiration_time = expiration_time.replace(tzinfo=datetime.UTC)
476
473
 
477
474
  if is_expired(expiration_time):
478
475
  if config.S3_SKIP_SIGNATURE_VALIDATION:
@@ -482,7 +479,7 @@ def validate_presigned_url_s3v4(context: RequestContext) -> None:
482
479
  else:
483
480
  raise AccessDenied(
484
481
  "Request has expired",
485
- HostId=FAKE_HOST_ID,
482
+ HostId=S3_HOST_ID,
486
483
  Expires=expiration_time.timestamp(),
487
484
  ServerTime=time.time(),
488
485
  X_Amz_Expires=x_amz_expires,
@@ -568,34 +565,21 @@ class S3SigV4SignatureContext:
568
565
  self._query_parameters
569
566
  )
570
567
 
571
- if forwarded_from_virtual_host_addressed_request(self._headers):
572
- # FIXME: maybe move this so it happens earlier in the chain when using virtual host?
573
- if not is_bucket_name_valid(self._bucket):
574
- raise InvalidBucketName(BucketName=self._bucket)
575
- netloc = self._headers.get(S3_VIRTUAL_HOST_FORWARDED_HEADER)
576
- self.host = netloc
577
- self._original_host = netloc
578
- self.signed_headers["host"] = netloc
579
- # the request comes from the Virtual Host router, we need to remove the bucket from the path
568
+ netloc = urlparse.urlparse(self.request.url).netloc
569
+ self.host = netloc
570
+ self._original_host = netloc
571
+ if (host_addressed := uses_host_addressing(self._headers)) and not is_bucket_name_valid(
572
+ self._bucket
573
+ ):
574
+ raise InvalidBucketName(BucketName=self._bucket)
575
+
576
+ if not host_addressed and not self.request.path.startswith(f"/{self._bucket}"):
577
+ # if in path style, check that the path starts with the bucket
578
+ # our path has been sanitized, we should use the un-sanitized one
580
579
  splitted_path = self.request.path.split("/", maxsplit=2)
581
- self.path = f"/{splitted_path[-1]}"
582
-
580
+ self.path = f"/{self._bucket}/{splitted_path[-1]}"
583
581
  else:
584
- netloc = urlparse.urlparse(self.request.url).netloc
585
- self.host = netloc
586
- self._original_host = netloc
587
- if (host_addressed := uses_host_addressing(self._headers)) and not is_bucket_name_valid(
588
- self._bucket
589
- ):
590
- raise InvalidBucketName(BucketName=self._bucket)
591
-
592
- if not host_addressed and not self.request.path.startswith(f"/{self._bucket}"):
593
- # if in path style, check that the path starts with the bucket
594
- # our path has been sanitized, we should use the un-sanitized one
595
- splitted_path = self.request.path.split("/", maxsplit=2)
596
- self.path = f"/{self._bucket}/{splitted_path[-1]}"
597
- else:
598
- self.path = self.request.path
582
+ self.path = self.request.path
599
583
 
600
584
  # we need to URL encode the path, as the key needs to be urlencoded for the signature to match
601
585
  self.path = urlparse.quote(self.path)
@@ -714,7 +698,7 @@ class S3SigV4SignatureContext:
714
698
  if not (split_creds := credential.split("/")) or len(split_creds) != 5:
715
699
  raise AuthorizationQueryParametersError(
716
700
  'Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting "<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request".',
717
- HostId=FAKE_HOST_ID,
701
+ HostId=S3_HOST_ID,
718
702
  )
719
703
 
720
704
  return split_creds[2]
@@ -771,13 +755,12 @@ def validate_post_policy(
771
755
  :return: None
772
756
  """
773
757
  if not request_form.get("key"):
774
- ex: InvalidArgument = _create_invalid_argument_exc(
775
- message="Bucket POST must contain a field named 'key'. If it is specified, please check the order of the fields.",
776
- name="key",
777
- value="",
778
- host_id=FAKE_HOST_ID,
758
+ raise InvalidArgument(
759
+ "Bucket POST must contain a field named 'key'. If it is specified, please check the order of the fields.",
760
+ ArgumentName="key",
761
+ ArgumentValue="",
762
+ HostId=S3_HOST_ID,
779
763
  )
780
- raise ex
781
764
 
782
765
  form_dict = {k.lower(): v for k, v in request_form.items()}
783
766
 
@@ -792,7 +775,7 @@ def validate_post_policy(
792
775
 
793
776
  if not is_v2 and not is_v4:
794
777
  ex: AccessDenied = AccessDenied("Access Denied")
795
- ex.HostId = FAKE_HOST_ID
778
+ ex.HostId = S3_HOST_ID
796
779
  raise ex
797
780
 
798
781
  try:
@@ -811,7 +794,7 @@ def validate_post_policy(
811
794
  if expiration := policy_decoded.get("expiration"):
812
795
  if is_expired(_parse_policy_expiration_date(expiration)):
813
796
  ex: AccessDenied = AccessDenied("Invalid according to Policy: Policy expired.")
814
- ex.HostId = FAKE_HOST_ID
797
+ ex.HostId = S3_HOST_ID
815
798
  raise ex
816
799
 
817
800
  # TODO: validate the signature
@@ -833,7 +816,7 @@ def validate_post_policy(
833
816
  str_condition = str(condition).replace("'", '"')
834
817
  raise AccessDenied(
835
818
  f"Invalid according to Policy: Policy Condition failed: {str_condition}",
836
- HostId=FAKE_HOST_ID,
819
+ HostId=S3_HOST_ID,
837
820
  )
838
821
 
839
822
 
@@ -886,7 +869,7 @@ def _verify_condition(condition: list | dict, form: dict, additional_policy_meta
886
869
  "Your proposed upload exceeds the maximum allowed size",
887
870
  ProposedSize=size,
888
871
  MaxSizeAllowed=end,
889
- HostId=FAKE_HOST_ID,
872
+ HostId=S3_HOST_ID,
890
873
  )
891
874
  else:
892
875
  return True
@@ -905,7 +888,7 @@ def _parse_policy_expiration_date(expiration_string: str) -> datetime.datetime:
905
888
  dt = datetime.datetime.strptime(expiration_string, POLICY_EXPIRATION_FORMAT2)
906
889
 
907
890
  # both date formats assume a UTC timezone ('Z' suffix), but it's not parsed as tzinfo into the datetime object
908
- dt = dt.replace(tzinfo=datetime.timezone.utc)
891
+ dt = dt.replace(tzinfo=datetime.UTC)
909
892
  return dt
910
893
 
911
894
 
@@ -931,13 +914,12 @@ def _is_match_with_signature_fields(
931
914
  if argument_name == "Awsaccesskeyid":
932
915
  argument_name = "AWSAccessKeyId"
933
916
 
934
- ex: InvalidArgument = _create_invalid_argument_exc(
935
- message=f"Bucket POST must contain a field named '{argument_name}'. If it is specified, please check the order of the fields.",
936
- name=argument_name,
937
- value="",
938
- host_id=FAKE_HOST_ID,
917
+ raise InvalidArgument(
918
+ f"Bucket POST must contain a field named '{argument_name}'. If it is specified, please check the order of the fields.",
919
+ ArgumentName=argument_name,
920
+ ArgumentValue="",
921
+ HostId=S3_HOST_ID,
939
922
  )
940
- raise ex
941
923
 
942
924
  return True
943
925
  return False
@@ -1,4 +1,5 @@
1
1
  import base64
2
+ import contextlib
2
3
  import copy
3
4
  import datetime
4
5
  import json
@@ -8,7 +9,8 @@ from collections import defaultdict
8
9
  from inspect import signature
9
10
  from io import BytesIO
10
11
  from operator import itemgetter
11
- from typing import IO, Optional, Union
12
+ from threading import RLock
13
+ from typing import IO
12
14
  from urllib import parse as urlparse
13
15
  from zoneinfo import ZoneInfo
14
16
 
@@ -23,6 +25,7 @@ from localstack.aws.api.s3 import (
23
25
  AccountId,
24
26
  AnalyticsConfiguration,
25
27
  AnalyticsId,
28
+ AuthorizationHeaderMalformed,
26
29
  BadDigest,
27
30
  Body,
28
31
  Bucket,
@@ -235,6 +238,7 @@ from localstack.services.s3.constants import (
235
238
  ARCHIVES_STORAGE_CLASSES,
236
239
  CHECKSUM_ALGORITHMS,
237
240
  DEFAULT_BUCKET_ENCRYPTION,
241
+ S3_HOST_ID,
238
242
  )
239
243
  from localstack.services.s3.cors import S3CorsHandler, s3_cors_request_handler
240
244
  from localstack.services.s3.exceptions import (
@@ -277,6 +281,7 @@ from localstack.services.s3.utils import (
277
281
  get_canned_acl,
278
282
  get_class_attrs_from_spec_class,
279
283
  get_failed_precondition_copy_source,
284
+ get_failed_upload_part_copy_source_preconditions,
280
285
  get_full_default_bucket_location,
281
286
  get_kms_key_arn,
282
287
  get_lifecycle_rule_from_object,
@@ -337,6 +342,8 @@ class S3Provider(S3Api, ServiceLifecycleHook):
337
342
  self._storage_backend = storage_backend or EphemeralS3ObjectStore(DEFAULT_S3_TMP_DIR)
338
343
  self._notification_dispatcher = NotificationDispatcher()
339
344
  self._cors_handler = S3CorsHandler(BucketCorsIndex())
345
+ # TODO: add lock for keys for PutObject, only way to support precondition writes for versioned buckets
346
+ self._preconditions_locks = defaultdict(lambda: defaultdict(RLock))
340
347
 
341
348
  # runtime cache of Lifecycle Expiration headers, as they need to be calculated everytime we fetch an object
342
349
  # in case the rules have changed
@@ -381,7 +388,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
381
388
  """
382
389
  if s3_bucket.notification_configuration:
383
390
  if not s3_notif_ctx:
384
- s3_notif_ctx = S3EventNotificationContext.from_request_context_native(
391
+ s3_notif_ctx = S3EventNotificationContext.from_request_context(
385
392
  context,
386
393
  s3_bucket=s3_bucket,
387
394
  s3_object=s3_object,
@@ -469,6 +476,16 @@ class S3Provider(S3Api, ServiceLifecycleHook):
469
476
  context: RequestContext,
470
477
  request: CreateBucketRequest,
471
478
  ) -> CreateBucketOutput:
479
+ if context.region == "aws-global":
480
+ # TODO: extend this logic to probably all the provider, and maybe all services. S3 is the most impacted
481
+ # right now so this will help users to properly set a region in their config
482
+ # See the `TestS3.test_create_bucket_aws_global` test
483
+ raise AuthorizationHeaderMalformed(
484
+ f"The authorization header is malformed; the region 'aws-global' is wrong; expecting '{AWS_REGION_US_EAST_1}'",
485
+ HostId=S3_HOST_ID,
486
+ Region=AWS_REGION_US_EAST_1,
487
+ )
488
+
472
489
  bucket_name = request["Bucket"]
473
490
 
474
491
  if not is_bucket_name_valid(bucket_name):
@@ -577,6 +594,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
577
594
  store.global_bucket_map.pop(bucket)
578
595
  self._cors_handler.invalidate_cache()
579
596
  self._expiration_cache.pop(bucket, None)
597
+ self._preconditions_locks.pop(bucket, None)
580
598
  # clean up the storage backend
581
599
  self._storage_backend.delete_bucket(bucket)
582
600
 
@@ -636,6 +654,16 @@ class S3Provider(S3Api, ServiceLifecycleHook):
636
654
  expected_bucket_owner: AccountId = None,
637
655
  **kwargs,
638
656
  ) -> HeadBucketOutput:
657
+ if context.region == "aws-global":
658
+ # TODO: extend this logic to probably all the provider, and maybe all services. S3 is the most impacted
659
+ # right now so this will help users to properly set a region in their config
660
+ # See the `TestS3.test_create_bucket_aws_global` test
661
+ raise AuthorizationHeaderMalformed(
662
+ f"The authorization header is malformed; the region 'aws-global' is wrong; expecting '{AWS_REGION_US_EAST_1}'",
663
+ HostId=S3_HOST_ID,
664
+ Region=AWS_REGION_US_EAST_1,
665
+ )
666
+
639
667
  store = self.get_store(context.account_id, context.region)
640
668
  if not (s3_bucket := store.buckets.get(bucket)):
641
669
  if not (account_id := store.global_bucket_map.get(bucket)):
@@ -727,6 +755,12 @@ class S3Provider(S3Api, ServiceLifecycleHook):
727
755
  system_metadata["ContentType"] = "binary/octet-stream"
728
756
 
729
757
  version_id = generate_version_id(s3_bucket.versioning_status)
758
+ if version_id != "null":
759
+ # if we are in a versioned bucket, we need to lock around the full key (all the versions)
760
+ # because object versions have locks per version
761
+ precondition_lock = self._preconditions_locks[bucket_name][key]
762
+ else:
763
+ precondition_lock = contextlib.nullcontext()
730
764
 
731
765
  etag_content_md5 = ""
732
766
  if content_md5 := request.get("ContentMD5"):
@@ -809,7 +843,10 @@ class S3Provider(S3Api, ServiceLifecycleHook):
809
843
  if encodings:
810
844
  s3_object.system_metadata["ContentEncoding"] = ",".join(encodings)
811
845
 
812
- with self._storage_backend.open(bucket_name, s3_object, mode="w") as s3_stored_object:
846
+ with (
847
+ precondition_lock,
848
+ self._storage_backend.open(bucket_name, s3_object, mode="w") as s3_stored_object,
849
+ ):
813
850
  # as we are inside the lock here, if multiple concurrent requests happen for the same object, it's the first
814
851
  # one to finish to succeed, and subsequent will raise exceptions. Once the first write finishes, we're
815
852
  # opening the lock and other requests can check this condition
@@ -1248,7 +1285,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1248
1285
  delete_marker_id = generate_version_id(s3_bucket.versioning_status)
1249
1286
  delete_marker = S3DeleteMarker(key=key, version_id=delete_marker_id)
1250
1287
  s3_bucket.objects.set(key, delete_marker)
1251
- s3_notif_ctx = S3EventNotificationContext.from_request_context_native(
1288
+ s3_notif_ctx = S3EventNotificationContext.from_request_context(
1252
1289
  context,
1253
1290
  s3_bucket=s3_bucket,
1254
1291
  s3_object=delete_marker,
@@ -1281,6 +1318,10 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1281
1318
  store.TAGS.tags.pop(get_unique_key_id(bucket, key, version_id), None)
1282
1319
  self._notify(context, s3_bucket=s3_bucket, s3_object=s3_object)
1283
1320
 
1321
+ if key not in s3_bucket.objects:
1322
+ # we clean up keys that do not have any object versions in them anymore
1323
+ self._preconditions_locks[bucket].pop(key, None)
1324
+
1284
1325
  return response
1285
1326
 
1286
1327
  def delete_objects(
@@ -1314,6 +1355,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1314
1355
  errors = []
1315
1356
 
1316
1357
  to_remove = []
1358
+ versioned_keys = set()
1317
1359
  for to_delete_object in objects:
1318
1360
  object_key = to_delete_object.get("Key")
1319
1361
  version_id = to_delete_object.get("VersionId")
@@ -1351,7 +1393,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1351
1393
  delete_marker_id = generate_version_id(s3_bucket.versioning_status)
1352
1394
  delete_marker = S3DeleteMarker(key=object_key, version_id=delete_marker_id)
1353
1395
  s3_bucket.objects.set(object_key, delete_marker)
1354
- s3_notif_ctx = S3EventNotificationContext.from_request_context_native(
1396
+ s3_notif_ctx = S3EventNotificationContext.from_request_context(
1355
1397
  context,
1356
1398
  s3_bucket=s3_bucket,
1357
1399
  s3_object=delete_marker,
@@ -1394,6 +1436,8 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1394
1436
  continue
1395
1437
 
1396
1438
  s3_bucket.objects.pop(object_key=object_key, version_id=version_id)
1439
+ versioned_keys.add(object_key)
1440
+
1397
1441
  if not quiet:
1398
1442
  deleted_object = DeletedObject(
1399
1443
  Key=object_key,
@@ -1411,6 +1455,11 @@ class S3Provider(S3Api, ServiceLifecycleHook):
1411
1455
  self._notify(context, s3_bucket=s3_bucket, s3_object=found_object)
1412
1456
  store.TAGS.tags.pop(get_unique_key_id(bucket, object_key, version_id), None)
1413
1457
 
1458
+ for versioned_key in versioned_keys:
1459
+ # we clean up keys that do not have any object versions in them anymore
1460
+ if versioned_key not in s3_bucket.objects:
1461
+ self._preconditions_locks[bucket].pop(versioned_key, None)
1462
+
1414
1463
  # TODO: request charged
1415
1464
  self._storage_backend.remove(bucket, to_remove)
1416
1465
  response: DeleteObjectsOutput = {}
@@ -2179,7 +2228,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
2179
2228
  # TODO: add a way to transition from ongoing-request=true to false? for now it is instant
2180
2229
  s3_object.restore = f'ongoing-request="false", expiry-date="{restore_expiration_date}"'
2181
2230
 
2182
- s3_notif_ctx_initiated = S3EventNotificationContext.from_request_context_native(
2231
+ s3_notif_ctx_initiated = S3EventNotificationContext.from_request_context(
2183
2232
  context,
2184
2233
  s3_bucket=s3_bucket,
2185
2234
  s3_object=s3_object,
@@ -2483,10 +2532,6 @@ class S3Provider(S3Api, ServiceLifecycleHook):
2483
2532
  request: UploadPartCopyRequest,
2484
2533
  ) -> UploadPartCopyOutput:
2485
2534
  # TODO: handle following parameters:
2486
- # CopySourceIfMatch: Optional[CopySourceIfMatch]
2487
- # CopySourceIfModifiedSince: Optional[CopySourceIfModifiedSince]
2488
- # CopySourceIfNoneMatch: Optional[CopySourceIfNoneMatch]
2489
- # CopySourceIfUnmodifiedSince: Optional[CopySourceIfUnmodifiedSince]
2490
2535
  # SSECustomerAlgorithm: Optional[SSECustomerAlgorithm]
2491
2536
  # SSECustomerKey: Optional[SSECustomerKey]
2492
2537
  # SSECustomerKeyMD5: Optional[SSECustomerKeyMD5]
@@ -2545,10 +2590,18 @@ class S3Provider(S3Api, ServiceLifecycleHook):
2545
2590
  source_range = request.get("CopySourceRange")
2546
2591
  # TODO implement copy source IF
2547
2592
 
2548
- range_data: Optional[ObjectRange] = None
2593
+ range_data: ObjectRange | None = None
2549
2594
  if source_range:
2550
2595
  range_data = parse_copy_source_range_header(source_range, src_s3_object.size)
2551
2596
 
2597
+ if precondition := get_failed_upload_part_copy_source_preconditions(
2598
+ request, src_s3_object.last_modified, src_s3_object.etag
2599
+ ):
2600
+ raise PreconditionFailed(
2601
+ "At least one of the pre-conditions you specified did not hold",
2602
+ Condition=precondition,
2603
+ )
2604
+
2552
2605
  s3_part = S3Part(part_number=part_number)
2553
2606
  if s3_multipart.checksum_algorithm:
2554
2607
  s3_part.checksum_algorithm = s3_multipart.checksum_algorithm
@@ -2622,6 +2675,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
2622
2675
  )
2623
2676
 
2624
2677
  elif if_none_match:
2678
+ # TODO: improve concurrency mechanism for `if_none_match` and `if_match`
2625
2679
  if if_none_match != "*":
2626
2680
  raise NotImplementedException(
2627
2681
  "A header you provided implies functionality that is not implemented",
@@ -3166,11 +3220,12 @@ class S3Provider(S3Api, ServiceLifecycleHook):
3166
3220
  if "TagSet" not in tagging:
3167
3221
  raise MalformedXML()
3168
3222
 
3169
- validate_tag_set(tagging["TagSet"], type_set="bucket")
3223
+ tag_set = tagging["TagSet"] or []
3224
+ validate_tag_set(tag_set, type_set="bucket")
3170
3225
 
3171
3226
  # remove the previous tags before setting the new ones, it overwrites the whole TagSet
3172
3227
  store.TAGS.tags.pop(s3_bucket.bucket_arn, None)
3173
- store.TAGS.tag_resource(s3_bucket.bucket_arn, tags=tagging["TagSet"])
3228
+ store.TAGS.tag_resource(s3_bucket.bucket_arn, tags=tag_set)
3174
3229
 
3175
3230
  def get_bucket_tagging(
3176
3231
  self,
@@ -3220,12 +3275,13 @@ class S3Provider(S3Api, ServiceLifecycleHook):
3220
3275
  if "TagSet" not in tagging:
3221
3276
  raise MalformedXML()
3222
3277
 
3223
- validate_tag_set(tagging["TagSet"], type_set="object")
3278
+ tag_set = tagging["TagSet"] or []
3279
+ validate_tag_set(tag_set, type_set="object")
3224
3280
 
3225
3281
  key_id = get_unique_key_id(bucket, key, s3_object.version_id)
3226
3282
  # remove the previous tags before setting the new ones, it overwrites the whole TagSet
3227
3283
  store.TAGS.tags.pop(key_id, None)
3228
- store.TAGS.tag_resource(key_id, tags=tagging["TagSet"])
3284
+ store.TAGS.tag_resource(key_id, tags=tag_set)
3229
3285
  response = PutObjectTaggingOutput()
3230
3286
  if s3_object.version_id:
3231
3287
  response["VersionId"] = s3_object.version_id
@@ -3291,7 +3347,7 @@ class S3Provider(S3Api, ServiceLifecycleHook):
3291
3347
 
3292
3348
  s3_object = s3_bucket.get_object(key=key, version_id=version_id, http_method="DELETE")
3293
3349
 
3294
- store.TAGS.tags.pop(get_unique_key_id(bucket, key, version_id), None)
3350
+ store.TAGS.tags.pop(get_unique_key_id(bucket, key, s3_object.version_id), None)
3295
3351
  response = DeleteObjectTaggingOutput()
3296
3352
  if s3_object.version_id:
3297
3353
  response["VersionId"] = s3_object.version_id
@@ -3852,7 +3908,9 @@ class S3Provider(S3Api, ServiceLifecycleHook):
3852
3908
  if retention and retention["RetainUntilDate"] < datetime.datetime.now(datetime.UTC):
3853
3909
  # weirdly, this date is format as following: Tue Dec 31 16:00:00 PST 2019
3854
3910
  # it contains the timezone as PST, even if you target a bucket in Europe or Asia
3855
- pst_datetime = retention["RetainUntilDate"].astimezone(tz=ZoneInfo("US/Pacific"))
3911
+ pst_datetime = retention["RetainUntilDate"].astimezone(
3912
+ tz=ZoneInfo("America/Los_Angeles")
3913
+ )
3856
3914
  raise InvalidArgument(
3857
3915
  "The retain until date must be in the future!",
3858
3916
  ArgumentName="RetainUntilDate",
@@ -4820,14 +4878,12 @@ def get_part_range(s3_object: S3Object, part_number: PartNumber) -> ObjectRange:
4820
4878
 
4821
4879
 
4822
4880
  def get_acl_headers_from_request(
4823
- request: Union[
4824
- PutObjectRequest,
4825
- CreateMultipartUploadRequest,
4826
- CopyObjectRequest,
4827
- CreateBucketRequest,
4828
- PutBucketAclRequest,
4829
- PutObjectAclRequest,
4830
- ],
4881
+ request: PutObjectRequest
4882
+ | CreateMultipartUploadRequest
4883
+ | CopyObjectRequest
4884
+ | CreateBucketRequest
4885
+ | PutBucketAclRequest
4886
+ | PutObjectAclRequest,
4831
4887
  ) -> list[tuple[str, str]]:
4832
4888
  permission_keys = [
4833
4889
  "GrantFullControl",
@@ -4845,7 +4901,7 @@ def get_acl_headers_from_request(
4845
4901
 
4846
4902
 
4847
4903
  def get_access_control_policy_from_acl_request(
4848
- request: Union[PutBucketAclRequest, PutObjectAclRequest],
4904
+ request: PutBucketAclRequest | PutObjectAclRequest,
4849
4905
  owner: Owner,
4850
4906
  request_body: bytes,
4851
4907
  ) -> AccessControlPolicy:
@@ -4894,9 +4950,10 @@ def get_access_control_policy_from_acl_request(
4894
4950
 
4895
4951
 
4896
4952
  def get_access_control_policy_for_new_resource_request(
4897
- request: Union[
4898
- PutObjectRequest, CreateMultipartUploadRequest, CopyObjectRequest, CreateBucketRequest
4899
- ],
4953
+ request: PutObjectRequest
4954
+ | CreateMultipartUploadRequest
4955
+ | CopyObjectRequest
4956
+ | CreateBucketRequest,
4900
4957
  owner: Owner,
4901
4958
  ) -> AccessControlPolicy:
4902
4959
  # TODO: this is basic ACL, not taking into account Bucket settings. Revisit once we really implement ACLs.