localstack-core 4.7.1.dev139__py3-none-any.whl → 4.10.1.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- localstack/aws/api/cloudformation/__init__.py +1 -0
- localstack/aws/api/cloudwatch/__init__.py +41 -1
- localstack/aws/api/config/__init__.py +4 -0
- localstack/aws/api/core.py +4 -0
- localstack/aws/api/ec2/__init__.py +1113 -56
- localstack/aws/api/iam/__init__.py +7 -0
- localstack/aws/api/kinesis/__init__.py +19 -0
- localstack/aws/api/kms/__init__.py +6 -0
- localstack/aws/api/lambda_/__init__.py +13 -0
- localstack/aws/api/logs/__init__.py +15 -0
- localstack/aws/api/redshift/__init__.py +9 -3
- localstack/aws/api/route53/__init__.py +2 -0
- localstack/aws/api/s3/__init__.py +12 -0
- localstack/aws/api/s3control/__init__.py +32 -0
- localstack/aws/api/ssm/__init__.py +2 -0
- localstack/aws/client.py +7 -2
- localstack/aws/forwarder.py +52 -5
- localstack/aws/handlers/analytics.py +1 -1
- localstack/aws/handlers/logging.py +12 -2
- localstack/aws/handlers/metric_handler.py +41 -1
- localstack/aws/handlers/service.py +32 -9
- localstack/aws/protocol/parser.py +440 -21
- localstack/aws/protocol/serializer.py +684 -64
- localstack/aws/protocol/service_router.py +120 -20
- localstack/aws/skeleton.py +4 -2
- localstack/aws/spec-patches.json +58 -0
- localstack/aws/spec.py +33 -13
- localstack/cli/exceptions.py +1 -1
- localstack/cli/localstack.py +4 -4
- localstack/cli/lpm.py +3 -4
- localstack/cli/profiles.py +1 -2
- localstack/config.py +18 -12
- localstack/constants.py +4 -29
- localstack/dev/kubernetes/__main__.py +1 -1
- localstack/dev/run/paths.py +1 -1
- localstack/dns/plugins.py +5 -1
- localstack/dns/server.py +12 -3
- localstack/packages/api.py +9 -8
- localstack/packages/core.py +2 -2
- localstack/packages/plugins.py +0 -8
- localstack/runtime/init.py +1 -1
- localstack/services/apigateway/legacy/provider.py +53 -3
- localstack/services/apigateway/next_gen/execute_api/integrations/aws.py +3 -0
- localstack/services/apigateway/next_gen/execute_api/integrations/http.py +3 -3
- localstack/services/apigateway/next_gen/execute_api/test_invoke.py +50 -6
- localstack/services/apigateway/next_gen/provider.py +5 -0
- localstack/services/cloudformation/engine/entities.py +12 -1
- localstack/services/cloudformation/engine/v2/change_set_model.py +0 -3
- localstack/services/cloudformation/engine/v2/change_set_model_describer.py +14 -0
- localstack/services/cloudformation/engine/v2/change_set_model_executor.py +13 -15
- localstack/services/cloudformation/engine/v2/change_set_model_preproc.py +118 -24
- localstack/services/cloudformation/engine/v2/change_set_model_transform.py +4 -1
- localstack/services/cloudformation/engine/v2/change_set_model_validator.py +5 -14
- localstack/services/cloudformation/engine/v2/change_set_model_visitor.py +1 -0
- localstack/services/cloudformation/engine/v2/resolving.py +6 -4
- localstack/services/cloudformation/engine/yaml_parser.py +9 -2
- localstack/services/cloudformation/resource_provider.py +5 -1
- localstack/services/cloudformation/resources.py +24149 -0
- localstack/services/cloudformation/v2/entities.py +6 -3
- localstack/services/cloudformation/v2/provider.py +172 -27
- localstack/services/cloudformation/v2/types.py +8 -4
- localstack/services/cloudwatch/provider_v2.py +25 -28
- localstack/services/dynamodb/packages.py +2 -1
- localstack/services/dynamodb/provider.py +42 -0
- localstack/services/dynamodb/v2/provider.py +42 -0
- localstack/services/ecr/resource_providers/aws_ecr_repository.py +5 -2
- localstack/services/es/provider.py +2 -2
- localstack/services/events/event_rule_engine.py +31 -13
- localstack/services/events/models.py +4 -5
- localstack/services/events/target.py +17 -9
- localstack/services/iam/provider.py +11 -116
- localstack/services/iam/resources/policy_simulator.py +133 -0
- localstack/services/kinesis/models.py +15 -2
- localstack/services/kinesis/provider.py +77 -0
- localstack/services/kms/provider.py +14 -5
- localstack/services/lambda_/invocation/internal_sqs_queue.py +5 -9
- localstack/services/lambda_/packages.py +1 -1
- localstack/services/logs/provider.py +1 -1
- localstack/services/moto.py +2 -1
- localstack/services/opensearch/cluster.py +15 -7
- localstack/services/opensearch/packages.py +26 -7
- localstack/services/opensearch/provider.py +6 -1
- localstack/services/opensearch/versions.py +56 -7
- localstack/services/s3/constants.py +5 -2
- localstack/services/s3/cors.py +4 -4
- localstack/services/s3/notifications.py +1 -1
- localstack/services/s3/presigned_url.py +27 -43
- localstack/services/s3/provider.py +67 -11
- localstack/services/s3/utils.py +42 -11
- localstack/services/ses/provider.py +16 -7
- localstack/services/sns/constants.py +7 -1
- localstack/services/sns/v2/models.py +167 -0
- localstack/services/sns/v2/provider.py +860 -2
- localstack/services/sns/v2/utils.py +130 -0
- localstack/services/sqs/developer_api.py +205 -0
- localstack/services/sqs/models.py +42 -3
- localstack/services/sqs/provider.py +8 -309
- localstack/services/sqs/query_api.py +1 -1
- localstack/services/sqs/utils.py +121 -2
- localstack/services/stepfunctions/asl/jsonata/jsonata.py +1 -1
- localstack/testing/aws/cloudformation_utils.py +1 -1
- localstack/testing/pytest/cloudformation/fixtures.py +3 -3
- localstack/testing/pytest/container.py +4 -5
- localstack/testing/pytest/fixtures.py +20 -19
- localstack/testing/pytest/in_memory_localstack.py +0 -4
- localstack/testing/pytest/marking.py +13 -4
- localstack/testing/pytest/stepfunctions/utils.py +4 -3
- localstack/testing/pytest/util.py +1 -1
- localstack/testing/pytest/validation_tracking.py +1 -2
- localstack/testing/snapshots/transformer_utility.py +5 -0
- localstack/utils/analytics/events.py +2 -2
- localstack/utils/analytics/metadata.py +1 -2
- localstack/utils/analytics/metrics/counter.py +6 -8
- localstack/utils/analytics/publisher.py +1 -2
- localstack/utils/analytics/service_request_aggregator.py +2 -2
- localstack/utils/archives.py +11 -11
- localstack/utils/aws/arns.py +17 -9
- localstack/utils/aws/aws_responses.py +7 -7
- localstack/utils/aws/aws_stack.py +2 -3
- localstack/utils/aws/message_forwarding.py +1 -2
- localstack/utils/aws/request_context.py +4 -5
- localstack/utils/batch_policy.py +3 -3
- localstack/utils/bootstrap.py +7 -7
- localstack/utils/catalog/catalog.py +139 -0
- localstack/utils/catalog/catalog_loader.py +11 -0
- localstack/utils/catalog/common.py +58 -0
- localstack/utils/catalog/plugins.py +28 -0
- localstack/utils/cloudwatch/cloudwatch_util.py +5 -5
- localstack/utils/collections.py +7 -8
- localstack/utils/config_listener.py +1 -1
- localstack/utils/container_networking.py +2 -3
- localstack/utils/container_utils/container_client.py +115 -131
- localstack/utils/container_utils/docker_cmd_client.py +42 -42
- localstack/utils/container_utils/docker_sdk_client.py +63 -62
- localstack/utils/diagnose.py +2 -3
- localstack/utils/docker_utils.py +3 -4
- localstack/utils/files.py +31 -7
- localstack/utils/functions.py +3 -2
- localstack/utils/http.py +4 -5
- localstack/utils/json.py +19 -5
- localstack/utils/kinesis/kinesis_connector.py +2 -1
- localstack/utils/net.py +6 -6
- localstack/utils/no_exit_argument_parser.py +2 -2
- localstack/utils/numbers.py +9 -2
- localstack/utils/objects.py +6 -5
- localstack/utils/patch.py +2 -1
- localstack/utils/run.py +10 -9
- localstack/utils/scheduler.py +11 -11
- localstack/utils/server/tcp_proxy.py +2 -2
- localstack/utils/serving.py +2 -3
- localstack/utils/strings.py +10 -11
- localstack/utils/sync.py +126 -1
- localstack/utils/tagging.py +1 -4
- localstack/utils/testutil.py +5 -4
- localstack/utils/threads.py +2 -2
- localstack/utils/time.py +11 -3
- localstack/utils/urls.py +1 -3
- localstack/version.py +2 -2
- {localstack_core-4.7.1.dev139.dist-info → localstack_core-4.10.1.dev7.dist-info}/METADATA +17 -12
- {localstack_core-4.7.1.dev139.dist-info → localstack_core-4.10.1.dev7.dist-info}/RECORD +168 -164
- {localstack_core-4.7.1.dev139.dist-info → localstack_core-4.10.1.dev7.dist-info}/entry_points.txt +4 -2
- localstack_core-4.10.1.dev7.dist-info/plux.json +1 -0
- localstack/packages/terraform.py +0 -46
- localstack/services/cloudformation/deploy.html +0 -144
- localstack/services/cloudformation/deploy_ui.py +0 -47
- localstack/services/cloudformation/plugins.py +0 -12
- localstack_core-4.7.1.dev139.dist-info/plux.json +0 -1
- {localstack_core-4.7.1.dev139.data → localstack_core-4.10.1.dev7.data}/scripts/localstack +0 -0
- {localstack_core-4.7.1.dev139.data → localstack_core-4.10.1.dev7.data}/scripts/localstack-supervisor +0 -0
- {localstack_core-4.7.1.dev139.data → localstack_core-4.10.1.dev7.data}/scripts/localstack.bat +0 -0
- {localstack_core-4.7.1.dev139.dist-info → localstack_core-4.10.1.dev7.dist-info}/WHEEL +0 -0
- {localstack_core-4.7.1.dev139.dist-info → localstack_core-4.10.1.dev7.dist-info}/licenses/LICENSE.txt +0 -0
- {localstack_core-4.7.1.dev139.dist-info → localstack_core-4.10.1.dev7.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import logging
|
|
2
3
|
import os
|
|
4
|
+
import re
|
|
3
5
|
import time
|
|
4
6
|
from random import random
|
|
5
7
|
|
|
@@ -8,14 +10,18 @@ from localstack.aws.api import RequestContext
|
|
|
8
10
|
from localstack.aws.api.kinesis import (
|
|
9
11
|
ConsumerARN,
|
|
10
12
|
Data,
|
|
13
|
+
GetResourcePolicyOutput,
|
|
11
14
|
HashKey,
|
|
12
15
|
KinesisApi,
|
|
13
16
|
PartitionKey,
|
|
17
|
+
Policy,
|
|
14
18
|
ProvisionedThroughputExceededException,
|
|
15
19
|
PutRecordOutput,
|
|
16
20
|
PutRecordsOutput,
|
|
17
21
|
PutRecordsRequestEntryList,
|
|
18
22
|
PutRecordsResultEntry,
|
|
23
|
+
ResourceARN,
|
|
24
|
+
ResourceNotFoundException,
|
|
19
25
|
SequenceNumber,
|
|
20
26
|
ShardId,
|
|
21
27
|
StartingPosition,
|
|
@@ -24,6 +30,7 @@ from localstack.aws.api.kinesis import (
|
|
|
24
30
|
SubscribeToShardEvent,
|
|
25
31
|
SubscribeToShardEventStream,
|
|
26
32
|
SubscribeToShardOutput,
|
|
33
|
+
ValidationException,
|
|
27
34
|
)
|
|
28
35
|
from localstack.aws.connect import connect_to
|
|
29
36
|
from localstack.constants import LOCALHOST
|
|
@@ -39,6 +46,13 @@ LOG = logging.getLogger(__name__)
|
|
|
39
46
|
MAX_SUBSCRIPTION_SECONDS = 300
|
|
40
47
|
SERVER_STARTUP_TIMEOUT = 120
|
|
41
48
|
|
|
49
|
+
DATA_STREAM_ARN_REGEX = re.compile(
|
|
50
|
+
r"^arn:aws(?:-[a-z]+)*:kinesis:[a-z0-9-]+:\d{12}:stream\/[a-zA-Z0-9_.\-]+$"
|
|
51
|
+
)
|
|
52
|
+
CONSUMER_ARN_REGEX = re.compile(
|
|
53
|
+
r"^arn:aws(?:-[a-z]+)*:kinesis:[a-z0-9-]+:\d{12}:stream\/[a-zA-Z0-9_.\-]+\/consumer\/[a-zA-Z0-9_.\-]+:\d+$"
|
|
54
|
+
)
|
|
55
|
+
|
|
42
56
|
|
|
43
57
|
def find_stream_for_consumer(consumer_arn):
|
|
44
58
|
account_id = extract_account_id_from_arn(consumer_arn)
|
|
@@ -52,6 +66,11 @@ def find_stream_for_consumer(consumer_arn):
|
|
|
52
66
|
raise Exception(f"Unable to find stream for stream consumer {consumer_arn}")
|
|
53
67
|
|
|
54
68
|
|
|
69
|
+
def is_valid_kinesis_arn(resource_arn: ResourceARN) -> bool:
|
|
70
|
+
"""Check if the provided ARN is a valid Kinesis ARN."""
|
|
71
|
+
return bool(CONSUMER_ARN_REGEX.match(resource_arn) or DATA_STREAM_ARN_REGEX.match(resource_arn))
|
|
72
|
+
|
|
73
|
+
|
|
55
74
|
class KinesisProvider(KinesisApi, ServiceLifecycleHook):
|
|
56
75
|
server_manager: KinesisServerManager
|
|
57
76
|
|
|
@@ -81,6 +100,64 @@ class KinesisProvider(KinesisApi, ServiceLifecycleHook):
|
|
|
81
100
|
def get_store(account_id: str, region_name: str) -> KinesisStore:
|
|
82
101
|
return kinesis_stores[account_id][region_name]
|
|
83
102
|
|
|
103
|
+
def put_resource_policy(
|
|
104
|
+
self,
|
|
105
|
+
context: RequestContext,
|
|
106
|
+
resource_arn: ResourceARN,
|
|
107
|
+
policy: Policy,
|
|
108
|
+
**kwargs,
|
|
109
|
+
) -> None:
|
|
110
|
+
if not is_valid_kinesis_arn(resource_arn):
|
|
111
|
+
raise ValidationException(f"invalid kinesis arn {resource_arn}")
|
|
112
|
+
|
|
113
|
+
kinesis = connect_to(
|
|
114
|
+
aws_access_key_id=context.account_id, region_name=context.region
|
|
115
|
+
).kinesis
|
|
116
|
+
try:
|
|
117
|
+
kinesis.describe_stream_summary(StreamARN=resource_arn)
|
|
118
|
+
except kinesis.exceptions.ResourceNotFoundException:
|
|
119
|
+
raise ResourceNotFoundException(f"Stream with ARN {resource_arn} not found")
|
|
120
|
+
|
|
121
|
+
store = self.get_store(context.account_id, context.region)
|
|
122
|
+
store.resource_policies[resource_arn] = policy
|
|
123
|
+
|
|
124
|
+
def get_resource_policy(
|
|
125
|
+
self,
|
|
126
|
+
context: RequestContext,
|
|
127
|
+
resource_arn: ResourceARN,
|
|
128
|
+
**kwargs,
|
|
129
|
+
) -> GetResourcePolicyOutput:
|
|
130
|
+
if not is_valid_kinesis_arn(resource_arn):
|
|
131
|
+
raise ValidationException(f"invalid kinesis arn {resource_arn}")
|
|
132
|
+
|
|
133
|
+
kinesis = connect_to(
|
|
134
|
+
aws_access_key_id=context.account_id, region_name=context.region
|
|
135
|
+
).kinesis
|
|
136
|
+
try:
|
|
137
|
+
kinesis.describe_stream_summary(StreamARN=resource_arn)
|
|
138
|
+
except kinesis.exceptions.ResourceNotFoundException:
|
|
139
|
+
raise ResourceNotFoundException(f"Stream with ARN {resource_arn} not found")
|
|
140
|
+
|
|
141
|
+
store = self.get_store(context.account_id, context.region)
|
|
142
|
+
policy = store.resource_policies.get(resource_arn, json.dumps({}))
|
|
143
|
+
return GetResourcePolicyOutput(Policy=policy)
|
|
144
|
+
|
|
145
|
+
def delete_resource_policy(
|
|
146
|
+
self,
|
|
147
|
+
context: RequestContext,
|
|
148
|
+
resource_arn: ResourceARN,
|
|
149
|
+
**kwargs,
|
|
150
|
+
) -> None:
|
|
151
|
+
if not is_valid_kinesis_arn(resource_arn):
|
|
152
|
+
raise ValidationException(f"invalid kinesis arn {resource_arn}")
|
|
153
|
+
|
|
154
|
+
store = self.get_store(context.account_id, context.region)
|
|
155
|
+
if resource_arn not in store.resource_policies:
|
|
156
|
+
raise ResourceNotFoundException(
|
|
157
|
+
f"No resource policy found for resource ARN {resource_arn}"
|
|
158
|
+
)
|
|
159
|
+
del store.resource_policies[resource_arn]
|
|
160
|
+
|
|
84
161
|
def subscribe_to_shard(
|
|
85
162
|
self,
|
|
86
163
|
context: RequestContext,
|
|
@@ -85,6 +85,7 @@ from localstack.aws.api.kms import (
|
|
|
85
85
|
MacAlgorithmSpec,
|
|
86
86
|
MarkerType,
|
|
87
87
|
MultiRegionKey,
|
|
88
|
+
MultiRegionKeyType,
|
|
88
89
|
NotFoundException,
|
|
89
90
|
NullableBooleanType,
|
|
90
91
|
OriginType,
|
|
@@ -490,24 +491,32 @@ class KmsProvider(KmsApi, ServiceLifecycleHook):
|
|
|
490
491
|
self, context: RequestContext, request: ReplicateKeyRequest
|
|
491
492
|
) -> ReplicateKeyResponse:
|
|
492
493
|
account_id = context.account_id
|
|
493
|
-
|
|
494
|
-
key_id =
|
|
495
|
-
|
|
494
|
+
primary_key = self._get_kms_key(account_id, context.region, request.get("KeyId"))
|
|
495
|
+
key_id = primary_key.metadata.get("KeyId")
|
|
496
|
+
key_arn = primary_key.metadata.get("Arn")
|
|
497
|
+
if not primary_key.metadata.get("MultiRegion"):
|
|
496
498
|
raise UnsupportedOperationException(
|
|
497
499
|
f"Unable to replicate a non-MultiRegion key {key_id}"
|
|
498
500
|
)
|
|
499
501
|
replica_region = request.get("ReplicaRegion")
|
|
500
502
|
replicate_to_store = kms_stores[account_id][replica_region]
|
|
503
|
+
|
|
504
|
+
if (
|
|
505
|
+
primary_key.metadata.get("MultiRegionConfiguration", {}).get("MultiRegionKeyType")
|
|
506
|
+
!= MultiRegionKeyType.PRIMARY
|
|
507
|
+
):
|
|
508
|
+
raise UnsupportedOperationException(f"{key_arn} is not a multi-region primary key.")
|
|
509
|
+
|
|
501
510
|
if key_id in replicate_to_store.keys:
|
|
502
511
|
raise AlreadyExistsException(
|
|
503
512
|
f"Unable to replicate key {key_id} to region {replica_region}, as the key "
|
|
504
513
|
f"already exist there"
|
|
505
514
|
)
|
|
506
|
-
replica_key = copy.deepcopy(
|
|
515
|
+
replica_key = copy.deepcopy(primary_key)
|
|
507
516
|
replica_key.replicate_metadata(request, account_id, replica_region)
|
|
508
517
|
replicate_to_store.keys[key_id] = replica_key
|
|
509
518
|
|
|
510
|
-
self.update_primary_key_with_replica_keys(
|
|
519
|
+
self.update_primary_key_with_replica_keys(primary_key, replica_key, replica_region)
|
|
511
520
|
|
|
512
521
|
return ReplicateKeyResponse(ReplicaKeyMetadata=replica_key.metadata)
|
|
513
522
|
|
|
@@ -19,13 +19,9 @@ from localstack.aws.api.sqs import (
|
|
|
19
19
|
String,
|
|
20
20
|
TagMap,
|
|
21
21
|
)
|
|
22
|
-
from localstack.services.sqs.models import SqsQueue, StandardQueue
|
|
23
|
-
from localstack.services.sqs.provider import
|
|
24
|
-
|
|
25
|
-
_create_message_attribute_hash,
|
|
26
|
-
to_sqs_api_message,
|
|
27
|
-
)
|
|
28
|
-
from localstack.services.sqs.utils import generate_message_id
|
|
22
|
+
from localstack.services.sqs.models import SqsQueue, StandardQueue, to_sqs_api_message
|
|
23
|
+
from localstack.services.sqs.provider import QueueUpdateWorker
|
|
24
|
+
from localstack.services.sqs.utils import create_message_attribute_hash, generate_message_id
|
|
29
25
|
from localstack.utils.objects import singleton_factory
|
|
30
26
|
from localstack.utils.strings import md5
|
|
31
27
|
from localstack.utils.time import now
|
|
@@ -189,7 +185,7 @@ class FakeSqsClient:
|
|
|
189
185
|
MD5OfBody=md5(MessageBody),
|
|
190
186
|
Body=MessageBody,
|
|
191
187
|
Attributes=self._create_message_attributes(MessageSystemAttributes),
|
|
192
|
-
MD5OfMessageAttributes=
|
|
188
|
+
MD5OfMessageAttributes=create_message_attribute_hash(MessageAttributes),
|
|
193
189
|
MessageAttributes=MessageAttributes,
|
|
194
190
|
)
|
|
195
191
|
queue_item = queue.put(
|
|
@@ -204,7 +200,7 @@ class FakeSqsClient:
|
|
|
204
200
|
"MD5OfMessageBody": message["MD5OfBody"],
|
|
205
201
|
"MD5OfMessageAttributes": message.get("MD5OfMessageAttributes"),
|
|
206
202
|
"SequenceNumber": queue_item.sequence_number,
|
|
207
|
-
"MD5OfMessageSystemAttributes":
|
|
203
|
+
"MD5OfMessageSystemAttributes": create_message_attribute_hash(MessageSystemAttributes),
|
|
208
204
|
}
|
|
209
205
|
|
|
210
206
|
|
|
@@ -12,7 +12,7 @@ from localstack.utils.platform import get_arch
|
|
|
12
12
|
"""Customized LocalStack version of the AWS Lambda Runtime Interface Emulator (RIE).
|
|
13
13
|
https://github.com/localstack/lambda-runtime-init/blob/localstack/README-LOCALSTACK.md
|
|
14
14
|
"""
|
|
15
|
-
LAMBDA_RUNTIME_DEFAULT_VERSION = "v0.1.
|
|
15
|
+
LAMBDA_RUNTIME_DEFAULT_VERSION = "v0.1.36-pre"
|
|
16
16
|
LAMBDA_RUNTIME_VERSION = config.LAMBDA_INIT_RELEASE_VERSION or LAMBDA_RUNTIME_DEFAULT_VERSION
|
|
17
17
|
LAMBDA_RUNTIME_INIT_URL = "https://github.com/localstack/lambda-runtime-init/releases/download/{version}/aws-lambda-rie-{arch}"
|
|
18
18
|
|
|
@@ -105,7 +105,7 @@ class LogsProvider(LogsApi, ServiceLifecycleHook):
|
|
|
105
105
|
"LogGroup name prefix and LogGroup name pattern are mutually exclusive parameters."
|
|
106
106
|
)
|
|
107
107
|
|
|
108
|
-
copy_groups = copy.deepcopy(region_backend.groups)
|
|
108
|
+
copy_groups = copy.deepcopy(dict(region_backend.groups))
|
|
109
109
|
|
|
110
110
|
groups = [
|
|
111
111
|
group.to_describe_dict()
|
localstack/services/moto.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"""
|
|
2
|
-
This module provides tools to call
|
|
2
|
+
This module provides tools to call Moto service implementations.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
import copy
|
|
@@ -65,6 +65,7 @@ def call_moto_with_request(
|
|
|
65
65
|
action=context.operation.name,
|
|
66
66
|
parameters=service_request,
|
|
67
67
|
region=context.region,
|
|
68
|
+
protocol=context.protocol,
|
|
68
69
|
)
|
|
69
70
|
# we keep the headers from the original request, but override them with the ones created from the `service_request`
|
|
70
71
|
headers = copy.deepcopy(context.request.headers)
|
|
@@ -18,7 +18,12 @@ from localstack.http.client import SimpleRequestsClient
|
|
|
18
18
|
from localstack.http.proxy import ProxyHandler
|
|
19
19
|
from localstack.services.edge import ROUTER
|
|
20
20
|
from localstack.services.opensearch import versions
|
|
21
|
-
from localstack.services.opensearch.packages import
|
|
21
|
+
from localstack.services.opensearch.packages import (
|
|
22
|
+
ELASTICSEARCH_DEFAULT_VERSION,
|
|
23
|
+
OPENSEARCH_DEFAULT_VERSION,
|
|
24
|
+
elasticsearch_package,
|
|
25
|
+
opensearch_package,
|
|
26
|
+
)
|
|
22
27
|
from localstack.utils.aws.arns import parse_arn
|
|
23
28
|
from localstack.utils.common import (
|
|
24
29
|
ShellCommandThread,
|
|
@@ -37,6 +42,9 @@ LOG = logging.getLogger(__name__)
|
|
|
37
42
|
INTERNAL_USER_AUTH = ("localstack-internal", "localstack-internal")
|
|
38
43
|
DEFAULT_BACKEND_HOST = "127.0.0.1"
|
|
39
44
|
|
|
45
|
+
# user that starts the opensearch process if the current user is root
|
|
46
|
+
OS_USER_OPENSEARCH = "localstack"
|
|
47
|
+
|
|
40
48
|
CommandSettings = dict[str, str]
|
|
41
49
|
|
|
42
50
|
|
|
@@ -314,7 +322,7 @@ class OpensearchCluster(Server):
|
|
|
314
322
|
|
|
315
323
|
@property
|
|
316
324
|
def default_version(self) -> str:
|
|
317
|
-
return
|
|
325
|
+
return OPENSEARCH_DEFAULT_VERSION
|
|
318
326
|
|
|
319
327
|
@property
|
|
320
328
|
def version(self) -> str:
|
|
@@ -336,7 +344,7 @@ class OpensearchCluster(Server):
|
|
|
336
344
|
|
|
337
345
|
@property
|
|
338
346
|
def os_user(self):
|
|
339
|
-
return
|
|
347
|
+
return OS_USER_OPENSEARCH
|
|
340
348
|
|
|
341
349
|
def health(self) -> str | None:
|
|
342
350
|
return get_cluster_health_status(self.url, auth=self.auth)
|
|
@@ -580,7 +588,7 @@ class EdgeProxiedOpensearchCluster(Server):
|
|
|
580
588
|
|
|
581
589
|
@property
|
|
582
590
|
def default_version(self):
|
|
583
|
-
return
|
|
591
|
+
return OPENSEARCH_DEFAULT_VERSION
|
|
584
592
|
|
|
585
593
|
@property
|
|
586
594
|
def url(self) -> str:
|
|
@@ -658,7 +666,7 @@ class ElasticsearchCluster(OpensearchCluster):
|
|
|
658
666
|
|
|
659
667
|
@property
|
|
660
668
|
def default_version(self) -> str:
|
|
661
|
-
return
|
|
669
|
+
return ELASTICSEARCH_DEFAULT_VERSION
|
|
662
670
|
|
|
663
671
|
@property
|
|
664
672
|
def bin_name(self) -> str:
|
|
@@ -666,7 +674,7 @@ class ElasticsearchCluster(OpensearchCluster):
|
|
|
666
674
|
|
|
667
675
|
@property
|
|
668
676
|
def os_user(self):
|
|
669
|
-
return
|
|
677
|
+
return OS_USER_OPENSEARCH
|
|
670
678
|
|
|
671
679
|
def _ensure_installed(self):
|
|
672
680
|
elasticsearch_package.install(self.version)
|
|
@@ -710,7 +718,7 @@ class ElasticsearchCluster(OpensearchCluster):
|
|
|
710
718
|
class EdgeProxiedElasticsearchCluster(EdgeProxiedOpensearchCluster):
|
|
711
719
|
@property
|
|
712
720
|
def default_version(self):
|
|
713
|
-
return
|
|
721
|
+
return ELASTICSEARCH_DEFAULT_VERSION
|
|
714
722
|
|
|
715
723
|
def _backend_cluster(self) -> OpensearchCluster:
|
|
716
724
|
return ElasticsearchCluster(
|
|
@@ -9,13 +9,6 @@ import threading
|
|
|
9
9
|
import semver
|
|
10
10
|
|
|
11
11
|
from localstack import config
|
|
12
|
-
from localstack.constants import (
|
|
13
|
-
ELASTICSEARCH_DEFAULT_VERSION,
|
|
14
|
-
ELASTICSEARCH_DELETE_MODULES,
|
|
15
|
-
ELASTICSEARCH_PLUGIN_LIST,
|
|
16
|
-
OPENSEARCH_DEFAULT_VERSION,
|
|
17
|
-
OPENSEARCH_PLUGIN_LIST,
|
|
18
|
-
)
|
|
19
12
|
from localstack.packages import InstallTarget, Package, PackageInstaller
|
|
20
13
|
from localstack.packages.java import java_package
|
|
21
14
|
from localstack.services.opensearch import versions
|
|
@@ -32,6 +25,32 @@ from localstack.utils.sync import SynchronizedDefaultDict, retry
|
|
|
32
25
|
|
|
33
26
|
LOG = logging.getLogger(__name__)
|
|
34
27
|
|
|
28
|
+
# the version of opensearch which is used by default
|
|
29
|
+
OPENSEARCH_DEFAULT_VERSION = "OpenSearch_3.1"
|
|
30
|
+
|
|
31
|
+
# See https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-plugins.html
|
|
32
|
+
OPENSEARCH_PLUGIN_LIST = [
|
|
33
|
+
"ingest-attachment",
|
|
34
|
+
"analysis-kuromoji",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
# the version of elasticsearch that is pre-seeded into the base image (sync with Dockerfile.base)
|
|
38
|
+
ELASTICSEARCH_DEFAULT_VERSION = "Elasticsearch_7.10"
|
|
39
|
+
|
|
40
|
+
# See https://docs.aws.amazon.com/ja_jp/elasticsearch-service/latest/developerguide/aes-supported-plugins.html
|
|
41
|
+
ELASTICSEARCH_PLUGIN_LIST = [
|
|
42
|
+
"analysis-icu",
|
|
43
|
+
"ingest-attachment",
|
|
44
|
+
"analysis-kuromoji",
|
|
45
|
+
"mapper-murmur3",
|
|
46
|
+
"mapper-size",
|
|
47
|
+
"analysis-phonetic",
|
|
48
|
+
"analysis-smartcn",
|
|
49
|
+
"analysis-stempel",
|
|
50
|
+
"analysis-ukrainian",
|
|
51
|
+
]
|
|
52
|
+
# Default ES modules to exclude (save apprx 66MB in the final image)
|
|
53
|
+
ELASTICSEARCH_DELETE_MODULES = ["ingest-geoip"]
|
|
35
54
|
|
|
36
55
|
_OPENSEARCH_INSTALL_LOCKS = SynchronizedDefaultDict(threading.RLock)
|
|
37
56
|
|
|
@@ -26,6 +26,7 @@ from localstack.aws.api.opensearch import (
|
|
|
26
26
|
CognitoOptions,
|
|
27
27
|
CognitoOptionsStatus,
|
|
28
28
|
ColdStorageOptions,
|
|
29
|
+
CompatibleVersionsMap,
|
|
29
30
|
CreateDomainRequest,
|
|
30
31
|
CreateDomainResponse,
|
|
31
32
|
DeleteDomainResponse,
|
|
@@ -75,7 +76,6 @@ from localstack.aws.api.opensearch import (
|
|
|
75
76
|
VolumeType,
|
|
76
77
|
VPCDerivedInfoStatus,
|
|
77
78
|
)
|
|
78
|
-
from localstack.constants import OPENSEARCH_DEFAULT_VERSION
|
|
79
79
|
from localstack.services.opensearch import versions
|
|
80
80
|
from localstack.services.opensearch.cluster import SecurityOptions
|
|
81
81
|
from localstack.services.opensearch.cluster_manager import (
|
|
@@ -84,6 +84,7 @@ from localstack.services.opensearch.cluster_manager import (
|
|
|
84
84
|
create_cluster_manager,
|
|
85
85
|
)
|
|
86
86
|
from localstack.services.opensearch.models import OpenSearchStore, opensearch_stores
|
|
87
|
+
from localstack.services.opensearch.packages import OPENSEARCH_DEFAULT_VERSION
|
|
87
88
|
from localstack.services.plugins import ServiceLifecycleHook
|
|
88
89
|
from localstack.state import AssetDirectory, StateVisitor
|
|
89
90
|
from localstack.utils.aws.arns import parse_arn
|
|
@@ -650,6 +651,10 @@ class OpensearchProvider(OpensearchApi, ServiceLifecycleHook):
|
|
|
650
651
|
for comp in versions.compatible_versions
|
|
651
652
|
if comp["SourceVersion"] == version_filter
|
|
652
653
|
]
|
|
654
|
+
if not compatible_versions:
|
|
655
|
+
compatible_versions = [
|
|
656
|
+
CompatibleVersionsMap(SourceVersion=version_filter, TargetVersions=[])
|
|
657
|
+
]
|
|
653
658
|
return GetCompatibleVersionsResponse(CompatibleVersions=compatible_versions)
|
|
654
659
|
|
|
655
660
|
def describe_domain_config(
|
|
@@ -13,20 +13,24 @@ from localstack.utils.common import get_arch
|
|
|
13
13
|
|
|
14
14
|
# Internal representation of the OpenSearch versions (without the "OpenSearch_" prefix)
|
|
15
15
|
_opensearch_install_versions = {
|
|
16
|
+
"3.1": "3.1.0",
|
|
17
|
+
"2.19": "2.19.3",
|
|
18
|
+
"2.17": "2.17.1",
|
|
19
|
+
"2.15": "2.15.0",
|
|
16
20
|
"2.13": "2.13.0",
|
|
17
21
|
"2.11": "2.11.1",
|
|
18
22
|
"2.9": "2.9.0",
|
|
19
23
|
"2.7": "2.7.0",
|
|
20
24
|
"2.5": "2.5.0",
|
|
21
25
|
"2.3": "2.3.0",
|
|
22
|
-
"1.3": "1.3.
|
|
26
|
+
"1.3": "1.3.20",
|
|
23
27
|
"1.2": "1.2.4",
|
|
24
28
|
"1.1": "1.1.0",
|
|
25
29
|
"1.0": "1.0.0",
|
|
26
30
|
}
|
|
27
31
|
# Internal representation of the Elasticsearch versions (without the "Elasticsearch_" prefix)
|
|
28
32
|
_elasticsearch_install_versions = {
|
|
29
|
-
"7.10": "7.10.
|
|
33
|
+
"7.10": "7.10.2",
|
|
30
34
|
"7.9": "7.9.3",
|
|
31
35
|
"7.8": "7.8.1",
|
|
32
36
|
"7.7": "7.7.1",
|
|
@@ -221,6 +225,9 @@ compatible_versions = [
|
|
|
221
225
|
"OpenSearch_2.9",
|
|
222
226
|
"OpenSearch_2.11",
|
|
223
227
|
"OpenSearch_2.13",
|
|
228
|
+
"OpenSearch_2.15",
|
|
229
|
+
"OpenSearch_2.17",
|
|
230
|
+
"OpenSearch_2.19",
|
|
224
231
|
],
|
|
225
232
|
),
|
|
226
233
|
CompatibleVersionsMap(
|
|
@@ -231,28 +238,68 @@ compatible_versions = [
|
|
|
231
238
|
"OpenSearch_2.9",
|
|
232
239
|
"OpenSearch_2.11",
|
|
233
240
|
"OpenSearch_2.13",
|
|
241
|
+
"OpenSearch_2.15",
|
|
242
|
+
"OpenSearch_2.17",
|
|
243
|
+
"OpenSearch_2.19",
|
|
234
244
|
],
|
|
235
245
|
),
|
|
236
246
|
CompatibleVersionsMap(
|
|
237
247
|
SourceVersion="OpenSearch_2.5",
|
|
238
|
-
TargetVersions=[
|
|
248
|
+
TargetVersions=[
|
|
249
|
+
"OpenSearch_2.7",
|
|
250
|
+
"OpenSearch_2.9",
|
|
251
|
+
"OpenSearch_2.11",
|
|
252
|
+
"OpenSearch_2.13",
|
|
253
|
+
"OpenSearch_2.15",
|
|
254
|
+
"OpenSearch_2.17",
|
|
255
|
+
"OpenSearch_2.19",
|
|
256
|
+
],
|
|
239
257
|
),
|
|
240
258
|
CompatibleVersionsMap(
|
|
241
259
|
SourceVersion="OpenSearch_2.7",
|
|
242
|
-
TargetVersions=[
|
|
260
|
+
TargetVersions=[
|
|
261
|
+
"OpenSearch_2.9",
|
|
262
|
+
"OpenSearch_2.11",
|
|
263
|
+
"OpenSearch_2.13",
|
|
264
|
+
"OpenSearch_2.15",
|
|
265
|
+
"OpenSearch_2.17",
|
|
266
|
+
"OpenSearch_2.19",
|
|
267
|
+
],
|
|
243
268
|
),
|
|
244
269
|
CompatibleVersionsMap(
|
|
245
270
|
SourceVersion="OpenSearch_2.9",
|
|
246
|
-
TargetVersions=[
|
|
271
|
+
TargetVersions=[
|
|
272
|
+
"OpenSearch_2.11",
|
|
273
|
+
"OpenSearch_2.13",
|
|
274
|
+
"OpenSearch_2.15",
|
|
275
|
+
"OpenSearch_2.17",
|
|
276
|
+
"OpenSearch_2.19",
|
|
277
|
+
],
|
|
247
278
|
),
|
|
248
279
|
CompatibleVersionsMap(
|
|
249
280
|
SourceVersion="OpenSearch_2.11",
|
|
250
|
-
TargetVersions=["OpenSearch_2.13"],
|
|
281
|
+
TargetVersions=["OpenSearch_2.13", "OpenSearch_2.15", "OpenSearch_2.17", "OpenSearch_2.19"],
|
|
282
|
+
),
|
|
283
|
+
CompatibleVersionsMap(
|
|
284
|
+
SourceVersion="OpenSearch_2.13",
|
|
285
|
+
TargetVersions=["OpenSearch_2.15", "OpenSearch_2.17", "OpenSearch_2.19"],
|
|
286
|
+
),
|
|
287
|
+
CompatibleVersionsMap(
|
|
288
|
+
SourceVersion="OpenSearch_2.15",
|
|
289
|
+
TargetVersions=["OpenSearch_2.17", "OpenSearch_2.19"],
|
|
290
|
+
),
|
|
291
|
+
CompatibleVersionsMap(
|
|
292
|
+
SourceVersion="OpenSearch_2.17",
|
|
293
|
+
TargetVersions=["OpenSearch_2.19"],
|
|
294
|
+
),
|
|
295
|
+
CompatibleVersionsMap(
|
|
296
|
+
SourceVersion="OpenSearch_2.19",
|
|
297
|
+
TargetVersions=["OpenSearch_3.1"],
|
|
251
298
|
),
|
|
252
299
|
]
|
|
253
300
|
|
|
254
301
|
|
|
255
|
-
def get_install_type_and_version(version: str) ->
|
|
302
|
+
def get_install_type_and_version(version: str) -> tuple[EngineType, str]:
|
|
256
303
|
engine_type = EngineType(version.split("_")[0])
|
|
257
304
|
|
|
258
305
|
if version not in install_versions:
|
|
@@ -297,6 +344,8 @@ def get_download_url(install_version: str, engine_type: EngineType) -> str:
|
|
|
297
344
|
return _opensearch_url(install_version)
|
|
298
345
|
elif engine_type == EngineType.Elasticsearch:
|
|
299
346
|
return _es_url(install_version)
|
|
347
|
+
else:
|
|
348
|
+
raise ValueError(f"Unknown OpenSearch engine type: {engine_type}")
|
|
300
349
|
|
|
301
350
|
|
|
302
351
|
def fetch_latest_versions() -> dict[str, str]: # pragma: no cover
|
|
@@ -10,8 +10,6 @@ from localstack.aws.api.s3 import (
|
|
|
10
10
|
)
|
|
11
11
|
from localstack.aws.api.s3 import Type as GranteeType
|
|
12
12
|
|
|
13
|
-
S3_VIRTUAL_HOST_FORWARDED_HEADER = "x-s3-vhost-forwarded-for"
|
|
14
|
-
|
|
15
13
|
S3_UPLOAD_PART_MIN_SIZE = 5242880
|
|
16
14
|
"""
|
|
17
15
|
This is minimum size allowed by S3 when uploading more than one part for a Multipart Upload, except for the last part
|
|
@@ -21,6 +19,11 @@ This is minimum size allowed by S3 when uploading more than one part for a Multi
|
|
|
21
19
|
DEFAULT_PRE_SIGNED_ACCESS_KEY_ID = "test"
|
|
22
20
|
DEFAULT_PRE_SIGNED_SECRET_ACCESS_KEY = "test"
|
|
23
21
|
|
|
22
|
+
S3_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg="
|
|
23
|
+
"""
|
|
24
|
+
S3 is returning a Host Id as part of its exceptions
|
|
25
|
+
"""
|
|
26
|
+
|
|
24
27
|
AUTHENTICATED_USERS_ACL_GROUP = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
|
|
25
28
|
ALL_USERS_ACL_GROUP = "http://acs.amazonaws.com/groups/global/AllUsers"
|
|
26
29
|
LOG_DELIVERY_ACL_GROUP = "http://acs.amazonaws.com/groups/s3/LogDelivery"
|
localstack/services/s3/cors.py
CHANGED
|
@@ -21,13 +21,13 @@ from localstack.aws.protocol.op_router import RestServiceOperationRouter
|
|
|
21
21
|
from localstack.aws.spec import get_service_catalog
|
|
22
22
|
from localstack.config import S3_VIRTUAL_HOSTNAME
|
|
23
23
|
from localstack.http import Request, Response
|
|
24
|
+
from localstack.services.s3.constants import S3_HOST_ID
|
|
24
25
|
from localstack.services.s3.utils import S3_VIRTUAL_HOSTNAME_REGEX
|
|
25
26
|
|
|
26
27
|
# TODO: add more logging statements
|
|
27
28
|
LOG = logging.getLogger(__name__)
|
|
28
29
|
|
|
29
30
|
_s3_virtual_host_regex = re.compile(S3_VIRTUAL_HOSTNAME_REGEX)
|
|
30
|
-
FAKE_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg="
|
|
31
31
|
|
|
32
32
|
# TODO: refactor those to expose the needed methods maybe in another way that both can import
|
|
33
33
|
add_default_headers = CorsResponseEnricher.add_cors_headers
|
|
@@ -135,7 +135,7 @@ class S3CorsHandler(Handler):
|
|
|
135
135
|
if is_options_request:
|
|
136
136
|
context.operation = self._get_op_from_request(request)
|
|
137
137
|
raise BadRequest(
|
|
138
|
-
"Insufficient information. Origin request header needed.", HostId=
|
|
138
|
+
"Insufficient information. Origin request header needed.", HostId=S3_HOST_ID
|
|
139
139
|
)
|
|
140
140
|
else:
|
|
141
141
|
# If the header is missing, Amazon S3 doesn't treat the request as a cross-origin request,
|
|
@@ -167,7 +167,7 @@ class S3CorsHandler(Handler):
|
|
|
167
167
|
context.operation = self._get_op_from_request(request)
|
|
168
168
|
raise AccessForbidden(
|
|
169
169
|
message,
|
|
170
|
-
HostId=
|
|
170
|
+
HostId=S3_HOST_ID,
|
|
171
171
|
Method=request.headers.get("Access-Control-Request-Method", "OPTIONS"),
|
|
172
172
|
ResourceType="BUCKET",
|
|
173
173
|
)
|
|
@@ -182,7 +182,7 @@ class S3CorsHandler(Handler):
|
|
|
182
182
|
context.operation = self._get_op_from_request(request)
|
|
183
183
|
raise AccessForbidden(
|
|
184
184
|
"CORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted by the resource's CORS spec.",
|
|
185
|
-
HostId=
|
|
185
|
+
HostId=S3_HOST_ID,
|
|
186
186
|
Method=request.headers.get("Access-Control-Request-Method"),
|
|
187
187
|
ResourceType="OBJECT",
|
|
188
188
|
)
|