localstack-core 4.6.1.dev60__py3-none-any.whl → 4.10.1.dev12__py3-none-any.whl

localstack/services/opensearch/cluster.py

@@ -2,7 +2,7 @@ import dataclasses
 import logging
 import os
 import threading
-from typing import Dict, List, NamedTuple, Optional, Tuple
+from typing import NamedTuple
 from urllib.parse import urlparse
 
 import requests
@@ -18,7 +18,12 @@ from localstack.http.client import SimpleRequestsClient
 from localstack.http.proxy import ProxyHandler
 from localstack.services.edge import ROUTER
 from localstack.services.opensearch import versions
-from localstack.services.opensearch.packages import elasticsearch_package, opensearch_package
+from localstack.services.opensearch.packages import (
+    ELASTICSEARCH_DEFAULT_VERSION,
+    OPENSEARCH_DEFAULT_VERSION,
+    elasticsearch_package,
+    opensearch_package,
+)
 from localstack.utils.aws.arns import parse_arn
 from localstack.utils.common import (
     ShellCommandThread,
@@ -37,7 +42,10 @@ LOG = logging.getLogger(__name__)
 INTERNAL_USER_AUTH = ("localstack-internal", "localstack-internal")
 DEFAULT_BACKEND_HOST = "127.0.0.1"
 
-CommandSettings = Dict[str, str]
+# user that starts the opensearch process if the current user is root
+OS_USER_OPENSEARCH = "localstack"
+
+CommandSettings = dict[str, str]
 
 
 class Directories(NamedTuple):
@@ -49,8 +57,8 @@ class Directories(NamedTuple):
 
 
 def get_cluster_health_status(
-    url: str, auth: Tuple[str, str] | None, host: str | None = None
-) -> Optional[str]:
+    url: str, auth: tuple[str, str] | None, host: str | None = None
+) -> str | None:
     """
     Queries the health endpoint of OpenSearch/Elasticsearch and returns either the status ('green', 'yellow',
     ...) or None if the response returned a non-200 response.
@@ -125,7 +133,7 @@ def resolve_directories(version: str, cluster_path: str, data_root: str = None)
     return Directories(install_dir, tmp_dir, modules_dir, data_dir, backup_dir)
 
 
-def build_cluster_run_command(cluster_bin: str, settings: CommandSettings) -> List[str]:
+def build_cluster_run_command(cluster_bin: str, settings: CommandSettings) -> list[str]:
     """
     Takes the command settings dict and builds the actual command (which can then be executed as a shell command).
 
@@ -169,13 +177,13 @@ class SecurityOptions:
     master_password: str | None
 
     @property
-    def auth(self) -> Tuple[str, str] | None:
+    def auth(self) -> tuple[str, str] | None:
         """Returns an auth tuple which can be used for HTTP requests or None, if disabled."""
         return None if not self.enabled else (self.master_username, self.master_password)
 
     @staticmethod
     def from_input(
-        advanced_security_options: Optional[AdvancedSecurityOptionsInput],
+        advanced_security_options: AdvancedSecurityOptionsInput | None,
     ) -> "SecurityOptions":
         """
         Parses the given AdvancedSecurityOptionsInput, performs some validation, and returns the parsed SecurityOptions.
@@ -215,7 +223,7 @@ class SecurityOptions:
 
 def register_cluster(
     host: str, path: str, forward_url: str, custom_endpoint: CustomEndpoint
-) -> List[Rule]:
+) -> list[Rule]:
     """
     Registers routes for a cluster at the edge router.
     Depending on which endpoint strategy is employed, and if a custom endpoint is enabled, different routes are
@@ -314,7 +322,7 @@ class OpensearchCluster(Server):
 
     @property
     def default_version(self) -> str:
-        return constants.OPENSEARCH_DEFAULT_VERSION
+        return OPENSEARCH_DEFAULT_VERSION
 
     @property
     def version(self) -> str:
@@ -336,9 +344,9 @@ class OpensearchCluster(Server):
 
     @property
     def os_user(self):
-        return constants.OS_USER_OPENSEARCH
+        return OS_USER_OPENSEARCH
 
-    def health(self) -> Optional[str]:
+    def health(self) -> str | None:
         return get_cluster_health_status(self.url, auth=self.auth)
 
     def do_start_thread(self) -> FuncThread:
@@ -451,8 +459,8 @@ class OpensearchCluster(Server):
         return settings
 
     def _create_run_command(
-        self, directories: Directories, additional_settings: Optional[CommandSettings] = None
-    ) -> List[str]:
+        self, directories: Directories, additional_settings: CommandSettings | None = None
+    ) -> list[str]:
         # delete opensearch data that may be cached locally from a previous test run
         bin_path = os.path.join(directories.install, "bin", self.bin_name)
 
@@ -464,7 +472,7 @@ class OpensearchCluster(Server):
         cmd = build_cluster_run_command(bin_path, settings)
         return cmd
 
-    def _create_env_vars(self, directories: Directories) -> Dict:
+    def _create_env_vars(self, directories: Directories) -> dict:
         env_vars = {
             "JAVA_HOME": os.path.join(directories.install, "jdk"),
             "OPENSEARCH_JAVA_OPTS": os.environ.get("OPENSEARCH_JAVA_OPTS", "-Xms200m -Xmx600m"),
@@ -580,7 +588,7 @@ class EdgeProxiedOpensearchCluster(Server):
 
     @property
     def default_version(self):
-        return constants.OPENSEARCH_DEFAULT_VERSION
+        return OPENSEARCH_DEFAULT_VERSION
 
     @property
     def url(self) -> str:
@@ -658,7 +666,7 @@ class ElasticsearchCluster(OpensearchCluster):
 
     @property
     def default_version(self) -> str:
-        return constants.ELASTICSEARCH_DEFAULT_VERSION
+        return ELASTICSEARCH_DEFAULT_VERSION
 
     @property
     def bin_name(self) -> str:
@@ -666,7 +674,7 @@ class ElasticsearchCluster(OpensearchCluster):
 
     @property
     def os_user(self):
-        return constants.OS_USER_OPENSEARCH
+        return OS_USER_OPENSEARCH
 
     def _ensure_installed(self):
         elasticsearch_package.install(self.version)
@@ -699,7 +707,7 @@ class ElasticsearchCluster(OpensearchCluster):
 
         return settings
 
-    def _create_env_vars(self, directories: Directories) -> Dict:
+    def _create_env_vars(self, directories: Directories) -> dict:
         return {
             **elasticsearch_package.get_installer(self.version).get_java_env_vars(),
             "ES_JAVA_OPTS": os.environ.get("ES_JAVA_OPTS", "-Xms200m -Xmx600m"),
@@ -710,7 +718,7 @@ class ElasticsearchCluster(OpensearchCluster):
 class EdgeProxiedElasticsearchCluster(EdgeProxiedOpensearchCluster):
     @property
     def default_version(self):
-        return constants.ELASTICSEARCH_DEFAULT_VERSION
+        return ELASTICSEARCH_DEFAULT_VERSION
 
     def _backend_cluster(self) -> OpensearchCluster:
         return ElasticsearchCluster(

localstack/services/opensearch/cluster_manager.py

@@ -1,7 +1,6 @@
 import dataclasses
 import logging
 import threading
-from typing import Dict, Optional
 
 from botocore.utils import ArnParser
 
@@ -80,9 +79,9 @@ class DomainKey:
 
 def build_cluster_endpoint(
     domain_key: DomainKey,
-    custom_endpoint: Optional[CustomEndpoint] = None,
+    custom_endpoint: CustomEndpoint | None = None,
     engine_type: EngineType = EngineType.OpenSearch,
-    preferred_port: Optional[int] = None,
+    preferred_port: int | None = None,
 ) -> str:
     """
     Builds the cluster endpoint from and optional custom_endpoint and the localstack opensearch config. Example
@@ -131,7 +130,7 @@ def build_cluster_endpoint(
 
 def determine_custom_endpoint(
     domain_endpoint_options: DomainEndpointOptions,
-) -> Optional[CustomEndpoint]:
+) -> CustomEndpoint | None:
     if not domain_endpoint_options:
         return
 
@@ -146,7 +145,7 @@ def determine_custom_endpoint(
 
 
 class ClusterManager:
-    clusters: Dict[str, Server]
+    clusters: dict[str, Server]
 
     def __init__(self) -> None:
         self.clusters = {}
@@ -155,9 +154,9 @@ class ClusterManager:
         self,
         arn: str,
         version: str,
-        endpoint_options: Optional[DomainEndpointOptions] = None,
-        security_options: Optional[SecurityOptions] = None,
-        preferred_port: Optional[int] = None,
+        endpoint_options: DomainEndpointOptions | None = None,
+        security_options: SecurityOptions | None = None,
+        preferred_port: int | None = None,
     ) -> Server:
         """
         Creates a new cluster.
@@ -190,7 +189,7 @@ class ClusterManager:
         self.clusters[arn] = cluster
         return cluster
 
-    def get(self, arn: str) -> Optional[Server]:
+    def get(self, arn: str) -> Server | None:
         return self.clusters.get(arn)
 
     def remove(self, arn: str):
@@ -257,8 +256,8 @@ class MultiplexingClusterManager(ClusterManager):
     - OPENSEARCH_ENDPOINT_STRATEGY = domain / path
     """
 
-    cluster: Optional[Server]
-    endpoints: Dict[str, ClusterEndpoint]
+    cluster: Server | None
+    endpoints: dict[str, ClusterEndpoint]
 
     def __init__(self) -> None:
         super().__init__()
@@ -366,7 +365,7 @@ class SingletonClusterManager(ClusterManager):
     - ES_MULTI_CLUSTER == False
     """
 
-    cluster: Optional[Server]
+    cluster: Server | None
 
     def __init__(self) -> None:
         super().__init__()
@@ -378,8 +377,8 @@ class SingletonClusterManager(ClusterManager):
         self,
         arn: str,
         version: str,
-        endpoint_options: Optional[DomainEndpointOptions] = None,
-        security_options: Optional[SecurityOptions] = None,
+        endpoint_options: DomainEndpointOptions | None = None,
+        security_options: SecurityOptions | None = None,
         preferred_port: int = None,
     ) -> Server:
         with self.mutex:

localstack/services/opensearch/models.py

@@ -1,5 +1,3 @@
-from typing import Dict
-
 from localstack.aws.api.opensearch import DomainStatus
 from localstack.services.stores import (
     AccountRegionBundle,
@@ -12,7 +10,7 @@ from localstack.utils.tagging import TaggingService
 
 class OpenSearchStore(BaseStore):
     # storage for domain resources (access should be protected with the _domain_mutex)
-    opensearch_domains: Dict[str, DomainStatus] = LocalAttribute(default=dict)
+    opensearch_domains: dict[str, DomainStatus] = LocalAttribute(default=dict)
 
     # static tagging service instance
     TAGS = CrossRegionAttribute(default=TaggingService)

localstack/services/opensearch/packages.py

@@ -5,18 +5,10 @@ import re
 import shutil
 import textwrap
 import threading
-from typing import List
 
 import semver
 
 from localstack import config
-from localstack.constants import (
-    ELASTICSEARCH_DEFAULT_VERSION,
-    ELASTICSEARCH_DELETE_MODULES,
-    ELASTICSEARCH_PLUGIN_LIST,
-    OPENSEARCH_DEFAULT_VERSION,
-    OPENSEARCH_PLUGIN_LIST,
-)
 from localstack.packages import InstallTarget, Package, PackageInstaller
 from localstack.packages.java import java_package
 from localstack.services.opensearch import versions
@@ -33,6 +25,32 @@ from localstack.utils.sync import SynchronizedDefaultDict, retry
 
 LOG = logging.getLogger(__name__)
 
+# the version of opensearch which is used by default
+OPENSEARCH_DEFAULT_VERSION = "OpenSearch_3.1"
+
+# See https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-plugins.html
+OPENSEARCH_PLUGIN_LIST = [
+    "ingest-attachment",
+    "analysis-kuromoji",
+]
+
+# the version of elasticsearch that is pre-seeded into the base image (sync with Dockerfile.base)
+ELASTICSEARCH_DEFAULT_VERSION = "Elasticsearch_7.10"
+
+# See https://docs.aws.amazon.com/ja_jp/elasticsearch-service/latest/developerguide/aes-supported-plugins.html
+ELASTICSEARCH_PLUGIN_LIST = [
+    "analysis-icu",
+    "ingest-attachment",
+    "analysis-kuromoji",
+    "mapper-murmur3",
+    "mapper-size",
+    "analysis-phonetic",
+    "analysis-smartcn",
+    "analysis-stempel",
+    "analysis-ukrainian",
+]
+# Default ES modules to exclude (save apprx 66MB in the final image)
+ELASTICSEARCH_DELETE_MODULES = ["ingest-geoip"]
 
 _OPENSEARCH_INSTALL_LOCKS = SynchronizedDefaultDict(threading.RLock)
 
@@ -49,7 +67,7 @@ class OpensearchPackage(Package):
         else:
             return OpensearchPackageInstaller(version)
 
-    def get_versions(self) -> List[str]:
+    def get_versions(self) -> list[str]:
         return list(versions.install_versions.keys())
 
 

localstack/services/opensearch/provider.py

@@ -3,9 +3,8 @@ import os
 import re
 import threading
 from copy import deepcopy
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from random import randint
-from typing import Dict, Optional
 from urllib.parse import urlparse
 
 from localstack import config
@@ -27,6 +26,7 @@ from localstack.aws.api.opensearch import (
     CognitoOptions,
     CognitoOptionsStatus,
     ColdStorageOptions,
+    CompatibleVersionsMap,
     CreateDomainRequest,
     CreateDomainResponse,
     DeleteDomainResponse,
@@ -76,7 +76,6 @@ from localstack.aws.api.opensearch import (
     VolumeType,
     VPCDerivedInfoStatus,
 )
-from localstack.constants import OPENSEARCH_DEFAULT_VERSION
 from localstack.services.opensearch import versions
 from localstack.services.opensearch.cluster import SecurityOptions
 from localstack.services.opensearch.cluster_manager import (
@@ -85,6 +84,7 @@ from localstack.services.opensearch.cluster_manager import (
     create_cluster_manager,
 )
 from localstack.services.opensearch.models import OpenSearchStore, opensearch_stores
+from localstack.services.opensearch.packages import OPENSEARCH_DEFAULT_VERSION
 from localstack.services.plugins import ServiceLifecycleHook
 from localstack.state import AssetDirectory, StateVisitor
 from localstack.utils.aws.arns import parse_arn
@@ -143,9 +143,9 @@ def _run_cluster_startup_monitor(cluster: Server, domain_name: str, region: str)
 def create_cluster(
     domain_key: DomainKey,
     engine_version: str,
-    domain_endpoint_options: Optional[DomainEndpointOptions],
-    security_options: Optional[SecurityOptions],
-    preferred_port: Optional[int] = None,
+    domain_endpoint_options: DomainEndpointOptions | None,
+    security_options: SecurityOptions | None,
+    preferred_port: int | None = None,
 ):
     """
     Uses the ClusterManager to create a new cluster for the given domain key. NOT thread safe, needs to be called
@@ -369,7 +369,7 @@ def get_domain_status(
             Cancellable=False,
             UpdateStatus=DeploymentStatus.COMPLETED,
             Description="There is no software update available for this domain.",
-            AutomatedUpdateDate=datetime.fromtimestamp(0, tz=timezone.utc),
+            AutomatedUpdateDate=datetime.fromtimestamp(0, tz=UTC),
             OptionalDeployment=True,
         ),
         DomainEndpointOptions=stored_status.get("DomainEndpointOptions")
@@ -399,7 +399,7 @@ def _ensure_domain_exists(arn: ARN) -> None:
 
 
 def _update_domain_config_request_to_status(request: UpdateDomainConfigRequest) -> DomainStatus:
-    request: Dict
+    request: dict
     request.pop("DryRun", None)
     request.pop("DomainName", None)
     return request
@@ -467,10 +467,11 @@ class OpensearchProvider(OpensearchApi, ServiceLifecycleHook):
                         preferred_port=preferred_port,
                     )
                 except Exception:
-                    LOG.exception(
+                    LOG.error(
                         "Could not restore domain %s in region %s.",
                         domain_name,
                         region,
+                        exc_info=LOG.isEnabledFor(logging.DEBUG),
                     )
 
     def on_before_state_reset(self):
@@ -581,7 +582,7 @@ class OpensearchProvider(OpensearchApi, ServiceLifecycleHook):
             if domain_status is None:
                 raise ResourceNotFoundException(f"Domain not found: {domain_key.domain_name}")
 
-            status_update: Dict = _update_domain_config_request_to_status(payload)
+            status_update: dict = _update_domain_config_request_to_status(payload)
             domain_status.update(status_update)
 
         return UpdateDomainConfigResponse(DomainConfig=_status_to_config(domain_status))
@@ -650,6 +651,10 @@ class OpensearchProvider(OpensearchApi, ServiceLifecycleHook):
                 for comp in versions.compatible_versions
                 if comp["SourceVersion"] == version_filter
             ]
+            if not compatible_versions:
+                compatible_versions = [
+                    CompatibleVersionsMap(SourceVersion=version_filter, TargetVersions=[])
+                ]
         return GetCompatibleVersionsResponse(CompatibleVersions=compatible_versions)
 
     def describe_domain_config(

localstack/services/opensearch/resource_providers/aws_elasticsearch_domain.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 
 import copy
 from pathlib import Path
-from typing import Optional, TypedDict
+from typing import TypedDict
 
 import localstack.services.cloudformation.provider_utils as util
 from localstack.aws.api.es import CreateElasticsearchDomainRequest
@@ -17,105 +17,105 @@ from localstack.utils.collections import convert_to_typed_dict
 
 
 class ElasticsearchDomainProperties(TypedDict):
-    AccessPolicies: Optional[dict]
-    AdvancedOptions: Optional[dict]
-    AdvancedSecurityOptions: Optional[AdvancedSecurityOptionsInput]
-    Arn: Optional[str]
-    CognitoOptions: Optional[CognitoOptions]
-    DomainArn: Optional[str]
-    DomainEndpoint: Optional[str]
-    DomainEndpointOptions: Optional[DomainEndpointOptions]
-    DomainName: Optional[str]
-    EBSOptions: Optional[EBSOptions]
-    ElasticsearchClusterConfig: Optional[ElasticsearchClusterConfig]
-    ElasticsearchVersion: Optional[str]
-    EncryptionAtRestOptions: Optional[EncryptionAtRestOptions]
-    Id: Optional[str]
-    LogPublishingOptions: Optional[dict]
-    NodeToNodeEncryptionOptions: Optional[NodeToNodeEncryptionOptions]
-    SnapshotOptions: Optional[SnapshotOptions]
-    Tags: Optional[list[Tag]]
-    VPCOptions: Optional[VPCOptions]
+    AccessPolicies: dict | None
+    AdvancedOptions: dict | None
+    AdvancedSecurityOptions: AdvancedSecurityOptionsInput | None
+    Arn: str | None
+    CognitoOptions: CognitoOptions | None
+    DomainArn: str | None
+    DomainEndpoint: str | None
+    DomainEndpointOptions: DomainEndpointOptions | None
+    DomainName: str | None
+    EBSOptions: EBSOptions | None
+    ElasticsearchClusterConfig: ElasticsearchClusterConfig | None
+    ElasticsearchVersion: str | None
+    EncryptionAtRestOptions: EncryptionAtRestOptions | None
+    Id: str | None
+    LogPublishingOptions: dict | None
+    NodeToNodeEncryptionOptions: NodeToNodeEncryptionOptions | None
+    SnapshotOptions: SnapshotOptions | None
+    Tags: list[Tag] | None
+    VPCOptions: VPCOptions | None
 
 
 class ZoneAwarenessConfig(TypedDict):
-    AvailabilityZoneCount: Optional[int]
+    AvailabilityZoneCount: int | None
 
 
 class ColdStorageOptions(TypedDict):
-    Enabled: Optional[bool]
+    Enabled: bool | None
 
 
 class ElasticsearchClusterConfig(TypedDict):
-    ColdStorageOptions: Optional[ColdStorageOptions]
-    DedicatedMasterCount: Optional[int]
-    DedicatedMasterEnabled: Optional[bool]
-    DedicatedMasterType: Optional[str]
-    InstanceCount: Optional[int]
-    InstanceType: Optional[str]
-    WarmCount: Optional[int]
-    WarmEnabled: Optional[bool]
-    WarmType: Optional[str]
-    ZoneAwarenessConfig: Optional[ZoneAwarenessConfig]
-    ZoneAwarenessEnabled: Optional[bool]
+    ColdStorageOptions: ColdStorageOptions | None
+    DedicatedMasterCount: int | None
+    DedicatedMasterEnabled: bool | None
+    DedicatedMasterType: str | None
+    InstanceCount: int | None
+    InstanceType: str | None
+    WarmCount: int | None
+    WarmEnabled: bool | None
+    WarmType: str | None
+    ZoneAwarenessConfig: ZoneAwarenessConfig | None
+    ZoneAwarenessEnabled: bool | None
 
 
 class SnapshotOptions(TypedDict):
-    AutomatedSnapshotStartHour: Optional[int]
+    AutomatedSnapshotStartHour: int | None
 
 
 class VPCOptions(TypedDict):
-    SecurityGroupIds: Optional[list[str]]
-    SubnetIds: Optional[list[str]]
+    SecurityGroupIds: list[str] | None
+    SubnetIds: list[str] | None
 
 
 class NodeToNodeEncryptionOptions(TypedDict):
-    Enabled: Optional[bool]
+    Enabled: bool | None
 
 
 class DomainEndpointOptions(TypedDict):
-    CustomEndpoint: Optional[str]
-    CustomEndpointCertificateArn: Optional[str]
-    CustomEndpointEnabled: Optional[bool]
-    EnforceHTTPS: Optional[bool]
-    TLSSecurityPolicy: Optional[str]
+    CustomEndpoint: str | None
+    CustomEndpointCertificateArn: str | None
+    CustomEndpointEnabled: bool | None
+    EnforceHTTPS: bool | None
+    TLSSecurityPolicy: str | None
 
 
 class CognitoOptions(TypedDict):
-    Enabled: Optional[bool]
-    IdentityPoolId: Optional[str]
-    RoleArn: Optional[str]
-    UserPoolId: Optional[str]
+    Enabled: bool | None
+    IdentityPoolId: str | None
+    RoleArn: str | None
+    UserPoolId: str | None
 
 
 class MasterUserOptions(TypedDict):
-    MasterUserARN: Optional[str]
-    MasterUserName: Optional[str]
-    MasterUserPassword: Optional[str]
+    MasterUserARN: str | None
+    MasterUserName: str | None
+    MasterUserPassword: str | None
 
 
 class AdvancedSecurityOptionsInput(TypedDict):
-    AnonymousAuthEnabled: Optional[bool]
-    Enabled: Optional[bool]
-    InternalUserDatabaseEnabled: Optional[bool]
-    MasterUserOptions: Optional[MasterUserOptions]
+    AnonymousAuthEnabled: bool | None
+    Enabled: bool | None
+    InternalUserDatabaseEnabled: bool | None
+    MasterUserOptions: MasterUserOptions | None
 
 
 class EBSOptions(TypedDict):
-    EBSEnabled: Optional[bool]
-    Iops: Optional[int]
-    VolumeSize: Optional[int]
-    VolumeType: Optional[str]
+    EBSEnabled: bool | None
+    Iops: int | None
+    VolumeSize: int | None
+    VolumeType: str | None
 
 
 class EncryptionAtRestOptions(TypedDict):
-    Enabled: Optional[bool]
-    KmsKeyId: Optional[str]
+    Enabled: bool | None
+    KmsKeyId: str | None
 
 
 class Tag(TypedDict):
-    Key: Optional[str]
-    Value: Optional[str]
+    Key: str | None
+    Value: str | None
 
 
 REPEATED_INVOCATION = "repeated_invocation"

localstack/services/opensearch/resource_providers/aws_elasticsearch_domain_plugin.py

@@ -1,5 +1,3 @@
-from typing import Optional, Type
-
 from localstack.services.cloudformation.resource_provider import (
     CloudFormationResourceProviderPlugin,
     ResourceProvider,
@@ -10,7 +8,7 @@ class ElasticsearchDomainProviderPlugin(CloudFormationResourceProviderPlugin):
     name = "AWS::Elasticsearch::Domain"
 
     def __init__(self):
-        self.factory: Optional[Type[ResourceProvider]] = None
+        self.factory: type[ResourceProvider] | None = None
 
     def load(self):
         from localstack.services.opensearch.resource_providers.aws_elasticsearch_domain import (