sagemaker-core 1.0.47__py3-none-any.whl → 2.1.1__py3-none-any.whl

sagemaker/core/modules/utils.py

@@ -0,0 +1,194 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Utils module."""
+from __future__ import absolute_import
+
+import os
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+
+from datetime import datetime
+from typing import Literal, Any
+
+from sagemaker.core.shapes import Unassigned
+from sagemaker.core.modules import logger
+
+
+def _is_valid_s3_uri(path: str, path_type: Literal["File", "Directory", "Any"] = "Any") -> bool:
+    """Check if the path is a valid S3 URI.
+
+    This method checks if the path is a valid S3 URI. If the path_type is specified,
+    it will also check if the path is a file or a directory.
+    This method does not check if the S3 bucket or object exists.
+
+    Args:
+        path (str): S3 URI to validate
+        path_type (Optional(Literal["File", "Directory", "Any"])): The type of the path to validate.
+            Defaults to "Any".
+
+    Returns:
+        bool: True if the path is a valid S3 URI, False otherwise
+    """
+    # Check if the path is a valid S3 URI
+    if not path.startswith("s3://"):
+        return False
+
+    if path_type == "File":
+        # If it's a file, it should not end with a slash
+        return not path.endswith("/")
+    if path_type == "Directory":
+        # If it's a directory, it should end with a slash
+        return path.endswith("/")
+
+    return path_type == "Any"
+
+
+def _is_valid_path(path: str, path_type: Literal["File", "Directory", "Any"] = "Any") -> bool:
+    """Check if the path is a valid local path.
+
+    Args:
+        path (str): Local path to validate
+        path_type (Optional(Literal["File", "Directory", "Any"])): The type of the path to validate.
+            Defaults to "Any".
+
+    Returns:
+        bool: True if the path is a valid local path, False otherwise
+    """
+    if not os.path.exists(path):
+        return False
+
+    if path_type == "File":
+        return os.path.isfile(path)
+    if path_type == "Directory":
+        return os.path.isdir(path)
+
+    return path_type == "Any"
+
+
+def _get_unique_name(base, max_length=63):
+    """Generate a unique name based on the base name.
+
+    This method generates a unique name based on the base name.
+    The unique name is generated by appending the current timestamp
+    to the base name.
+
+    Args:
+        base (str): The base name to use
+        max_length (int): The maximum length of the unique name. Defaults to 63.
+
+    Returns:
+        str: The unique name
+    """
+    current_time = datetime.now().strftime("%Y%m%d%H%M%S")
+    base = base.replace("_", "-")
+    unique_name = f"{base}-{current_time}"
+    unique_name = unique_name[:max_length]  # Truncate to max_length
+    return unique_name
+
+
+def _get_repo_name_from_image(image: str) -> str:
+    """Get the repository name from the image URI.
+
+    Example:
+    ``` python
+    _get_repo_name_from_image("123456789012.dkr.ecr.us-west-2.amazonaws.com/my-repo:latest")
+    # Returns "my-repo"
+    ```
+
+    Args:
+        image (str): The image URI
+
+    Returns:
+        str: The repository name
+    """
+    return image.split("/")[-1].split(":")[0]
+
+
+def convert_unassigned_to_none(instance) -> Any:
+    """Convert Unassigned values to None for any instance."""
+    for name, value in instance.__dict__.items():
+        if isinstance(value, Unassigned):
+            setattr(instance, name, None)
+    return instance
+
+
+def safe_serialize(data):
+    """Serialize the data without wrapping strings in quotes.
+
+    This function handles the following cases:
+    1. If `data` is a string, it returns the string as-is without wrapping in quotes.
+    2. If `data` is serializable (e.g., a dictionary, list, int, float), it returns
+       the JSON-encoded string using `json.dumps()`.
+    3. If `data` cannot be serialized (e.g., a custom object), it returns the string
+       representation of the data using `str(data)`.
+
+    Args:
+        data (Any): The data to serialize.
+
+    Returns:
+        str: The serialized JSON-compatible string or the string representation of the input.
+    """
+    if isinstance(data, str):
+        return data
+    try:
+        return json.dumps(data)
+    except TypeError:
+        return str(data)
+
+
+def _run_clone_command_silent(repo_url, dest_dir):
+    """Run the 'git clone' command with the repo url and the directory to clone the repo into.
+
+    Args:
+        repo_url (str): Git repo url to be cloned.
+        dest_dir: (str): Local path where the repo should be cloned into.
+
+    Raises:
+        CalledProcessError: If failed to clone git repo.
+    """
+    my_env = os.environ.copy()
+    if repo_url.startswith("https://"):
+        try:
+            my_env["GIT_TERMINAL_PROMPT"] = "0"
+            subprocess.check_call(
+                ["git", "clone", repo_url, dest_dir],
+                env=my_env,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Failed to clone repository: {repo_url}")
+            logger.error(f"Error output:\n{e}")
+            raise
+    elif repo_url.startswith("git@") or repo_url.startswith("ssh://"):
+        try:
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                custom_ssh_executable = Path(tmp_dir) / "ssh_batch"
+                with open(custom_ssh_executable, "w") as pipe:
+                    print("#!/bin/sh", file=pipe)
+                    print("ssh -oBatchMode=yes $@", file=pipe)
+                os.chmod(custom_ssh_executable, 0o511)
+                my_env["GIT_SSH"] = str(custom_ssh_executable)
+                subprocess.check_call(
+                    ["git", "clone", repo_url, dest_dir],
+                    env=my_env,
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL,
+                )
+        except subprocess.CalledProcessError as e:
+            del my_env["GIT_SSH"]
+            logger.error(f"Failed to clone repository: {repo_url}")
+            logger.error(f"Error output:\n{e}")
+            raise

sagemaker/core/network.py

@@ -0,0 +1,185 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This file contains code related to network configuration.
+
+It also includes encryption, network isolation, and VPC configurations.
+"""
+from __future__ import absolute_import
+
+from typing import Union, Optional, List
+
+from sagemaker.core.helper.pipeline_variable import PipelineVariable
+
+
+class NetworkConfig(object):
+    """Accepts network configuration parameters for conversion to request dict.
+
+    The `_to_request_dict` provides a method to turn the parameters into a dict.
+    """
+
+    def __init__(
+        self,
+        enable_network_isolation: Union[bool, PipelineVariable] = None,
+        security_group_ids: Optional[List[Union[str, PipelineVariable]]] = None,
+        subnets: Optional[List[Union[str, PipelineVariable]]] = None,
+        encrypt_inter_container_traffic: Optional[Union[bool, PipelineVariable]] = None,
+    ):
+        """Initialize a ``NetworkConfig`` instance.
+
+        NetworkConfig accepts network configuration parameters and provides a method to turn
+        these parameters into a dictionary.
+
+        Args:
+            enable_network_isolation (bool or PipelineVariable): Boolean that determines
+                whether to enable network isolation.
+            security_group_ids (list[str] or list[PipelineVariable]): A list of strings representing
+                security group IDs.
+            subnets (list[str] or list[PipelineVariable]): A list of strings representing subnets.
+            encrypt_inter_container_traffic (bool or PipelineVariable): Boolean that determines
+                whether to encrypt inter-container traffic. Default value is None.
+        """
+        self.enable_network_isolation = enable_network_isolation
+        self.security_group_ids = security_group_ids
+        self.subnets = subnets
+        self.encrypt_inter_container_traffic = encrypt_inter_container_traffic
+
+    def _to_request_dict(self):
+        """Generates a request dictionary using the parameters provided to the class."""
+        # Enable Network Isolation should default to False if it is not provided.
+        enable_network_isolation = (
+            False if self.enable_network_isolation is None else self.enable_network_isolation
+        )
+        network_config_request = {"EnableNetworkIsolation": enable_network_isolation}
+
+        if self.encrypt_inter_container_traffic is not None:
+            network_config_request["EnableInterContainerTrafficEncryption"] = (
+                self.encrypt_inter_container_traffic
+            )
+
+        if self.security_group_ids is not None or self.subnets is not None:
+            network_config_request["VpcConfig"] = {}
+
+        if self.security_group_ids is not None:
+            network_config_request["VpcConfig"]["SecurityGroupIds"] = self.security_group_ids
+
+        if self.subnets is not None:
+            network_config_request["VpcConfig"]["Subnets"] = self.subnets
+
+        return network_config_request
+
+
+# VPC Utilities (merged from vpc_utils.py)
+
+SUBNETS_KEY = "Subnets"
+SECURITY_GROUP_IDS_KEY = "SecurityGroupIds"
+VPC_CONFIG_KEY = "VpcConfig"
+
+# A global constant value for methods which can optionally override VpcConfig
+VPC_CONFIG_DEFAULT = "VPC_CONFIG_DEFAULT"
+
+
+def to_dict(subnets, security_group_ids):
+    """Prepares a VpcConfig dict containing keys 'Subnets' and 'SecurityGroupIds'.
+
+    This is the dict format expected by SageMaker CreateTrainingJob and CreateModel APIs.
+    See https://docs.aws.amazon.com/sagemaker/latest/dg/API_VpcConfig.html
+
+    Args:
+        subnets (list): list of subnet IDs to use in VpcConfig
+        security_group_ids (list): list of security group IDs to use in
+            VpcConfig
+
+    Returns:
+        A VpcConfig dict containing keys 'Subnets' and 'SecurityGroupIds' If
+        either or both parameters are None, returns None
+    """
+    if subnets is None or security_group_ids is None:
+        return None
+    return {SUBNETS_KEY: subnets, SECURITY_GROUP_IDS_KEY: security_group_ids}
+
+
+def from_dict(vpc_config, do_sanitize=False):
+    """Extracts subnets and security group ids as lists from a VpcConfig dict
+
+    Args:
+        vpc_config (dict): a VpcConfig dict containing 'Subnets' and
+            'SecurityGroupIds'
+        do_sanitize (bool): whether to sanitize the VpcConfig dict before
+            extracting values
+
+    Returns:
+        Tuple of lists as (subnets, security_group_ids) If vpc_config parameter
+        is None, returns (None, None)
+
+    Raises:
+        * ValueError if sanitize enabled and vpc_config is invalid
+
+        * KeyError if sanitize disabled and vpc_config is missing key(s)
+    """
+    if do_sanitize:
+        vpc_config = sanitize(vpc_config)
+    if vpc_config is None:
+        return None, None
+    return vpc_config[SUBNETS_KEY], vpc_config[SECURITY_GROUP_IDS_KEY]
+
+
+def sanitize(vpc_config):
+    """Checks and removes unexpected keys from VpcConfig or raises error for violations.
+
+    Checks that an instance of VpcConfig has the expected keys and values,
+    removes unexpected keys, and raises ValueErrors if any expectations are
+    violated.
+
+    Args:
+        vpc_config (dict): a VpcConfig dict containing 'Subnets' and
+            'SecurityGroupIds'
+
+    Returns:
+        A valid VpcConfig dict containing only 'Subnets' and 'SecurityGroupIds'
+        from the vpc_config parameter If vpc_config parameter is None, returns
+        None
+
+    Raises:
+        ValueError if any expectations are violated:
+            * vpc_config must be a non-empty dict
+            * vpc_config must have key `Subnets` and the value must be a non-empty list
+            * vpc_config must have key `SecurityGroupIds` and the value must be a non-empty list
+    """
+    if vpc_config is None:
+        return vpc_config
+    if not isinstance(vpc_config, dict):
+        raise ValueError("vpc_config is not a dict: {}".format(vpc_config))
+    if not vpc_config:
+        raise ValueError("vpc_config is empty")
+
+    subnets = vpc_config.get(SUBNETS_KEY)
+    if subnets is None:
+        raise ValueError("vpc_config is missing key: {}".format(SUBNETS_KEY))
+    if not isinstance(subnets, list):
+        raise ValueError("vpc_config value for {} is not a list: {}".format(SUBNETS_KEY, subnets))
+    if not subnets:
+        raise ValueError("vpc_config value for {} is empty".format(SUBNETS_KEY))
+
+    security_group_ids = vpc_config.get(SECURITY_GROUP_IDS_KEY)
+    if security_group_ids is None:
+        raise ValueError("vpc_config is missing key: {}".format(SECURITY_GROUP_IDS_KEY))
+    if not isinstance(security_group_ids, list):
+        raise ValueError(
+            "vpc_config value for {} is not a list: {}".format(
+                SECURITY_GROUP_IDS_KEY, security_group_ids
+            )
+        )
+    if not security_group_ids:
+        raise ValueError("vpc_config value for {} is empty".format(SECURITY_GROUP_IDS_KEY))
+
+    return to_dict(subnets, security_group_ids)

sagemaker/core/parameter.py

@@ -0,0 +1,173 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Placeholder docstring"""
+from __future__ import absolute_import
+
+import json
+from typing import Union
+
+from sagemaker.core.helper.pipeline_variable import PipelineVariable
+from sagemaker.core.common_utils import to_string
+
+
+class ParameterRange(object):
+    """Base class for representing parameter ranges.
+
+    This is used to define what hyperparameters to tune for an Amazon SageMaker
+    hyperparameter tuning job and to verify hyperparameters for Marketplace Algorithms.
+    """
+
+    __all_types__ = ("Continuous", "Categorical", "Integer")
+
+    def __init__(
+        self,
+        min_value: Union[int, float, PipelineVariable],
+        max_value: Union[int, float, PipelineVariable],
+        scaling_type: Union[str, PipelineVariable] = "Auto",
+    ):
+        """Initialize a parameter range.
+
+        Args:
+            min_value (float or int or PipelineVariable): The minimum value for the range.
+            max_value (float or int or PipelineVariable): The maximum value for the range.
+            scaling_type (str or PipelineVariable): The scale used for searching the range during
+                tuning (default: 'Auto'). Valid values: 'Auto', 'Linear',
+                'Logarithmic' and 'ReverseLogarithmic'.
+        """
+        self.min_value = min_value
+        self.max_value = max_value
+        self.scaling_type = scaling_type
+
+    def is_valid(self, value):
+        """Determine if a value is valid within this ParameterRange.
+
+        Args:
+            value (float or int): The value to be verified.
+
+        Returns:
+            bool: True if valid, False otherwise.
+        """
+        return self.min_value <= value <= self.max_value
+
+    @classmethod
+    def cast_to_type(cls, value):
+        """Placeholder docstring"""
+        return float(value)
+
+    def as_tuning_range(self, name):
+        """Represent the parameter range as a dictionary.
+
+        It is suitable for a request to create an Amazon SageMaker hyperparameter tuning job.
+
+        Args:
+            name (str): The name of the hyperparameter.
+
+        Returns:
+            dict[str, str]: A dictionary that contains the name and values of
+            the hyperparameter.
+        """
+        return {
+            "Name": name,
+            "MinValue": to_string(self.min_value),
+            "MaxValue": to_string(self.max_value),
+            "ScalingType": self.scaling_type,
+        }
+
+
+class ContinuousParameter(ParameterRange):
+    """A class for representing hyperparameters that have a continuous range of possible values.
+
+    Args:
+            min_value (float): The minimum value for the range.
+            max_value (float): The maximum value for the range.
+    """
+
+    __name__ = "Continuous"
+
+    @classmethod
+    def cast_to_type(cls, value):
+        """Placeholder docstring"""
+        return float(value)
+
+
+class CategoricalParameter(ParameterRange):
+    """A class for representing hyperparameters that have a discrete list of possible values."""
+
+    __name__ = "Categorical"
+
+    def __init__(self, values):  # pylint: disable=super-init-not-called
+        """Initialize a ``CategoricalParameter``.
+
+        Args:
+            values (list or object): The possible values for the hyperparameter.
+                This input will be converted into a list of strings.
+        """
+        values = values if isinstance(values, list) else [values]
+        self.values = [to_string(v) for v in values]
+
+    def as_tuning_range(self, name):
+        """Represent the parameter range as a dictionary.
+
+        It is suitable for a request to create an Amazon SageMaker hyperparameter tuning job.
+
+        Args:
+            name (str): The name of the hyperparameter.
+
+        Returns:
+            dict[str, list[str]]: A dictionary that contains the name and values
+            of the hyperparameter.
+        """
+        return {"Name": name, "Values": self.values}
+
+    def as_json_range(self, name):
+        """Represent the parameter range as a dictionary.
+
+        Dictionary is suitable for a request to create an Amazon SageMaker hyperparameter tuning job
+        using one of the deep learning frameworks.
+
+        The deep learning framework images require that hyperparameters be
+        serialized as JSON.
+
+        Args:
+            name (str): The name of the hyperparameter.
+
+        Returns:
+            dict[str, list[str]]: A dictionary that contains the name and values of the
+            hyperparameter, where the values are serialized as JSON.
+        """
+        return {"Name": name, "Values": [json.dumps(v) for v in self.values]}
+
+    def is_valid(self, value):
+        """Placeholder docstring"""
+        return value in self.values
+
+    @classmethod
+    def cast_to_type(cls, value):
+        """Placeholder docstring"""
+        return str(value)
+
+
+class IntegerParameter(ParameterRange):
+    """A class for representing hyperparameters that have an integer range of possible values.
+
+    Args:
+        min_value (int): The minimum value for the range.
+        max_value (int): The maximum value for the range.
+    """
+
+    __name__ = "Integer"
+
+    @classmethod
+    def cast_to_type(cls, value):
+        """Placeholder docstring"""
+        return int(value)