sagemaker-core 1.0.62__py3-none-any.whl → 2.1.1__py3-none-any.whl

sagemaker/core/jumpstart/validators.py

@@ -0,0 +1,257 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This module contains validators related to SageMaker JumpStart."""
+from __future__ import absolute_import
+from typing import Any, Dict, List, Optional
+from sagemaker.core.helper.session_helper import Session
+
+from sagemaker.core.jumpstart.enums import (
+    HyperparameterValidationMode,
+    JumpStartScriptScope,
+    VariableScope,
+    VariableTypes,
+)
+from sagemaker.core.jumpstart.exceptions import JumpStartHyperparametersError
+from sagemaker.core.jumpstart.types import JumpStartHyperparameter
+from sagemaker.core.jumpstart.utils import get_region_fallback, verify_model_region_and_return_specs
+
+
+def _validate_hyperparameter(
+    hyperparameter_name: str,
+    hyperparameter_value: Any,
+    hyperparameter_specs: List[JumpStartHyperparameter],
+) -> None:
+    """Perform low-level hyperparameter validation on single parameter.
+
+    Args:
+      hyperparameter_name (str): The name of the hyperparameter to validate.
+      hyperparameter_value (Any): The value of the hyperparemter to validate.
+      hyperparameter_specs (List[JumpStartHyperparameter]): List of ``JumpStartHyperparameter`` to
+        use when validating the hyperparameter.
+
+    Raises:
+        JumpStartHyperparametersError: If the hyperparameter is not formatted correctly,
+            according to its specs in the model metadata.
+    """
+    hyperparameter_spec = [
+        spec for spec in hyperparameter_specs if spec.name == hyperparameter_name
+    ]
+    if len(hyperparameter_spec) == 0:
+        raise JumpStartHyperparametersError(
+            f"Unable to perform validation -- cannot find hyperparameter '{hyperparameter_name}' "
+            "in model specs."
+        )
+
+    if len(hyperparameter_spec) > 1:
+        raise JumpStartHyperparametersError(
+            "Unable to perform validation -- found multiple hyperparameter "
+            f"'{hyperparameter_name}' in model specs."
+        )
+
+    hyperparameter_spec = hyperparameter_spec[0]
+
+    if hyperparameter_spec.type == VariableTypes.BOOL.value:
+        if isinstance(hyperparameter_value, bool):
+            return
+        if not isinstance(hyperparameter_value, str):
+            raise JumpStartHyperparametersError(
+                f"Expecting boolean valued hyperparameter, but got '{str(hyperparameter_value)}'."
+            )
+        if str(hyperparameter_value).lower() not in ["true", "false"]:
+            raise JumpStartHyperparametersError(
+                f"Expecting boolean valued hyperparameter, but got '{str(hyperparameter_value)}'."
+            )
+    elif hyperparameter_spec.type == VariableTypes.TEXT.value:
+        if not isinstance(hyperparameter_value, str):
+            raise JumpStartHyperparametersError(
+                "Expecting text valued hyperparameter to have string type."
+            )
+
+        if hasattr(hyperparameter_spec, "options"):
+            if hyperparameter_value not in hyperparameter_spec.options:
+                raise JumpStartHyperparametersError(
+                    f"Hyperparameter '{hyperparameter_name}' must have one of the following "
+                    f"values: {', '.join(hyperparameter_spec.options)}."
+                )
+
+        if hasattr(hyperparameter_spec, "min"):
+            if len(hyperparameter_value) < hyperparameter_spec.min:
+                raise JumpStartHyperparametersError(
+                    f"Hyperparameter '{hyperparameter_name}' must have length no less than "
+                    f"{hyperparameter_spec.min}."
+                )
+
+        if hasattr(hyperparameter_spec, "exclusive_min"):
+            if len(hyperparameter_value) <= hyperparameter_spec.exclusive_min:
+                raise JumpStartHyperparametersError(
+                    f"Hyperparameter '{hyperparameter_name}' must have length greater than "
+                    f"{hyperparameter_spec.exclusive_min}."
+                )
+
+        if hasattr(hyperparameter_spec, "max"):
+            if len(hyperparameter_value) > hyperparameter_spec.max:
+                raise JumpStartHyperparametersError(
+                    f"Hyperparameter '{hyperparameter_name}' must have length no greater than "
+                    f"{hyperparameter_spec.max}."
+                )
+
+        if hasattr(hyperparameter_spec, "exclusive_max"):
+            if len(hyperparameter_value) >= hyperparameter_spec.exclusive_max:
+                raise JumpStartHyperparametersError(
+                    f"Hyperparameter '{hyperparameter_name}' must have length less than "
+                    f"{hyperparameter_spec.exclusive_max}."
+                )
+
+    # validate numeric types
+    elif hyperparameter_spec.type in [VariableTypes.INT.value, VariableTypes.FLOAT.value]:
+        try:
+            numeric_hyperparam_value = float(hyperparameter_value)
+        except ValueError:
+            raise JumpStartHyperparametersError(
+                f"Hyperparameter '{hyperparameter_name}' must be numeric type "
+                f"('{hyperparameter_value}')."
+            )
+
+        if hyperparameter_spec.type == VariableTypes.INT.value:
+            hyperparameter_value_str = str(hyperparameter_value)
+            start_index = 0
+            if hyperparameter_value_str[0] in ["+", "-"]:
+                start_index = 1
+            if not hyperparameter_value_str[start_index:].isdigit():
+                raise JumpStartHyperparametersError(
+                    f"Hyperparameter '{hyperparameter_name}' must be integer type "
+                    f"('{hyperparameter_value}')."
+                )
+
+        if hasattr(hyperparameter_spec, "min"):
+            if numeric_hyperparam_value < hyperparameter_spec.min:
+                raise JumpStartHyperparametersError(
+                    f"Hyperparameter '{hyperparameter_name}' can be no less than "
+                    f"{hyperparameter_spec.min}."
+                )
+
+        if hasattr(hyperparameter_spec, "max"):
+            if numeric_hyperparam_value > hyperparameter_spec.max:
+                raise JumpStartHyperparametersError(
+                    f"Hyperparameter '{hyperparameter_name}' can be no greater than "
+                    f"{hyperparameter_spec.max}."
+                )
+
+        if hasattr(hyperparameter_spec, "exclusive_min"):
+            if numeric_hyperparam_value <= hyperparameter_spec.exclusive_min:
+                raise JumpStartHyperparametersError(
+                    f"Hyperparameter '{hyperparameter_name}' must be greater than "
+                    f"{hyperparameter_spec.exclusive_min}."
+                )
+
+        if hasattr(hyperparameter_spec, "exclusive_max"):
+            if numeric_hyperparam_value >= hyperparameter_spec.exclusive_max:
+                raise JumpStartHyperparametersError(
+                    f"Hyperparameter '{hyperparameter_name}' must be less than "
+                    f"{hyperparameter_spec.exclusive_max}."
+                )
+
+
+def validate_hyperparameters(
+    model_id: str,
+    model_version: str,
+    hyperparameters: Dict[str, Any],
+    validation_mode: HyperparameterValidationMode = HyperparameterValidationMode.VALIDATE_PROVIDED,
+    hub_arn: Optional[str] = None,
+    region: Optional[str] = None,
+    sagemaker_session: Optional[Session] = None,
+    tolerate_vulnerable_model: bool = False,
+    tolerate_deprecated_model: bool = False,
+    config_name: Optional[str] = None,
+) -> None:
+    """Validate hyperparameters for JumpStart models.
+
+    Args:
+        model_id (str): Model ID of the model for which to validate hyperparameters.
+        model_version (str): Version of the model for which to validate hyperparameters.
+        hyperparameters (dict): Hyperparameters to validate.
+        validation_mode (HyperparameterValidationMode): Method of validation to use with
+          hyperparameters. If set to ``VALIDATE_PROVIDED``, only hyperparameters provided
+          to this function will be validated, the missing hyperparameters will be ignored.
+          If set to``VALIDATE_ALGORITHM``, all algorithm hyperparameters will be validated.
+          If set to ``VALIDATE_ALL``, all hyperparameters for the model will be validated.
+        region (str): Region for which to validate hyperparameters. (Default: None).
+        sagemaker_session (Optional[Session]): Custom SageMaker Session to use.
+          (Default: sagemaker.jumpstart.constants.DEFAULT_JUMPSTART_SAGEMAKER_SESSION).
+        tolerate_vulnerable_model (bool): True if vulnerable versions of model
+           specifications should be tolerated (exception not raised). If False, raises an
+           exception if the script used by this version of the model has dependencies with known
+           security vulnerabilities. (Default: False).
+        tolerate_deprecated_model (bool): True if deprecated models should be tolerated
+           (exception not raised). False if these models should raise an exception.
+           (Default: False).
+        config_name (Optional[str]): Name of the JumpStart Model config to apply. (Default: None).
+
+    Raises:
+        JumpStartHyperparametersError: If the hyperparameters are not formatted correctly,
+            according to their metadata specs.
+
+    """
+
+    region = region or get_region_fallback(
+        sagemaker_session=sagemaker_session,
+    )
+    if validation_mode is None:
+        validation_mode = HyperparameterValidationMode.VALIDATE_PROVIDED
+
+    region = region or get_region_fallback(
+        sagemaker_session=sagemaker_session,
+    )
+
+    model_specs = verify_model_region_and_return_specs(
+        model_id=model_id,
+        version=model_version,
+        hub_arn=hub_arn,
+        region=region,
+        scope=JumpStartScriptScope.TRAINING,
+        sagemaker_session=sagemaker_session,
+        tolerate_deprecated_model=tolerate_deprecated_model,
+        tolerate_vulnerable_model=tolerate_vulnerable_model,
+        config_name=config_name,
+    )
+    hyperparameters_specs = model_specs.hyperparameters
+
+    if validation_mode == HyperparameterValidationMode.VALIDATE_PROVIDED:
+        for hyperparam_name, hyperparam_value in hyperparameters.items():
+            _validate_hyperparameter(hyperparam_name, hyperparam_value, hyperparameters_specs)
+
+    elif validation_mode == HyperparameterValidationMode.VALIDATE_ALGORITHM:
+        for hyperparam in hyperparameters_specs:
+            if hyperparam.scope == VariableScope.ALGORITHM:
+                if hyperparam.name not in hyperparameters:
+                    raise JumpStartHyperparametersError(
+                        f"Cannot find algorithm hyperparameter for '{hyperparam.name}'."
+                    )
+                _validate_hyperparameter(
+                    hyperparam.name, hyperparameters[hyperparam.name], hyperparameters_specs
+                )
+
+    elif validation_mode == HyperparameterValidationMode.VALIDATE_ALL:
+        for hyperparam in hyperparameters_specs:
+            if hyperparam.name not in hyperparameters:
+                raise JumpStartHyperparametersError(
+                    f"Cannot find hyperparameter for '{hyperparam.name}'."
+                )
+            _validate_hyperparameter(
+                hyperparam.name, hyperparameters[hyperparam.name], hyperparameters_specs
+            )
+
+    else:
+        raise NotImplementedError(
+            f"Unable to handle validation for the mode '{validation_mode.value}'."
+        )

sagemaker/core/lambda_helper.py

@@ -0,0 +1,312 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This module contains helper methods related to Lambda."""
+from __future__ import print_function, absolute_import
+
+from io import BytesIO
+import zipfile
+import time
+from botocore.exceptions import ClientError
+
+from sagemaker.core import s3
+from sagemaker.core.helper.session_helper import Session
+
+
+class Lambda:
+    """Contains lambda boto3 wrappers to Create, Update, Delete and Invoke Lambda functions."""
+
+    def __init__(
+        self,
+        function_arn: str = None,
+        function_name: str = None,
+        execution_role_arn: str = None,
+        zipped_code_dir: str = None,
+        s3_bucket: str = None,
+        script: str = None,
+        handler: str = None,
+        session: Session = None,
+        timeout: int = 120,
+        memory_size: int = 128,
+        runtime: str = "python3.8",
+        vpc_config: dict = None,
+        environment: dict = None,
+        layers: list = None,
+    ):
+        """Constructs a Lambda instance.
+
+        This instance represents a Lambda function and provides methods for updating,
+        deleting and invoking the function.
+
+        This class can be used either for creating a new Lambda function or using an existing one.
+        When using an existing Lambda function, only the function_arn argument is required.
+        When creating a new one the function_name, execution_role_arn and handler arguments
+        are required, as well as either script or zipped_code_dir.
+
+        Args:
+            function_arn (str): The arn of the Lambda function.
+            function_name (str): The name of the Lambda function.
+                Function name must be provided to create a Lambda function.
+            execution_role_arn (str): The role to be attached to Lambda function.
+            zipped_code_dir (str): The path of the zipped code package of the Lambda function.
+            s3_bucket (str): The bucket where zipped code is uploaded.
+                If not provided, default session bucket is used to upload zipped_code_dir.
+            script (str): The path of Lambda function script for direct zipped upload
+            handler (str): The Lambda handler. The format for handler should be
+                file_name.function_name. For ex: if the name of the Lambda script is
+                hello_world.py and Lambda function definition in that script is
+                lambda_handler(event, context), the handler should be hello_world.lambda_handler
+            session (sagemaker.session.Session): Session object which manages interactions
+                with Amazon SageMaker APIs and any other AWS services needed.
+                If not specified, new session is created.
+            timeout (int): Timeout of the Lambda function in seconds. Default is 120 seconds.
+            memory_size (int): Memory of the Lambda function in megabytes. Default is 128 MB.
+            runtime (str): Runtime of the Lambda function. Default is set to python3.8.
+            vpc_config (dict): VPC to deploy the Lambda function to. Default is None.
+            environment (dict): Environment Variables for the Lambda function. Default is None.
+            layers (list): List of Lambda layers for the Lambda function. Default is None.
+        """
+        self.function_arn = function_arn
+        self.function_name = function_name
+        self.zipped_code_dir = zipped_code_dir
+        self.s3_bucket = s3_bucket
+        self.script = script
+        self.handler = handler
+        self.execution_role_arn = execution_role_arn
+        self.session = session if session is not None else Session()
+        self.timeout = timeout
+        self.memory_size = memory_size
+        self.runtime = runtime
+        self.vpc_config = vpc_config or {}
+        self.environment = environment or {}
+        self.layers = layers or []
+
+        if function_arn is None and function_name is None:
+            raise ValueError("Either function_arn or function_name must be provided.")
+
+        if function_name is not None:
+            if execution_role_arn is None:
+                raise ValueError("execution_role_arn must be provided.")
+            if zipped_code_dir is None and script is None:
+                raise ValueError("Either zipped_code_dir or script must be provided.")
+            if zipped_code_dir and script:
+                raise ValueError("Provide either script or zipped_code_dir, not both.")
+            if handler is None:
+                raise ValueError("Lambda handler must be provided.")
+
+        if function_arn is not None:
+            if zipped_code_dir and script:
+                raise ValueError("Provide either script or zipped_code_dir, not both.")
+
+    def create(self):
+        """Method to create a lambda function.
+
+        Returns: boto3 response from Lambda's create_function method.
+        """
+        lambda_client = _get_lambda_client(self.session)
+
+        if self.function_name is None:
+            raise ValueError("FunctionName must be provided to create a Lambda function.")
+
+        if self.script is not None:
+            code = {"ZipFile": _zip_lambda_code(self.script)}
+        else:
+            bucket, key_prefix = s3.determine_bucket_and_prefix(
+                bucket=self.s3_bucket, key_prefix=None, sagemaker_session=self.session
+            )
+            key = _upload_to_s3(
+                s3_client=_get_s3_client(self.session),
+                function_name=self.function_name,
+                zipped_code_dir=self.zipped_code_dir,
+                s3_bucket=bucket,
+                s3_key_prefix=key_prefix,
+            )
+            code = {"S3Bucket": bucket, "S3Key": key}
+
+        try:
+            response = lambda_client.create_function(
+                FunctionName=self.function_name,
+                Runtime=self.runtime,
+                Handler=self.handler,
+                Role=self.execution_role_arn,
+                Code=code,
+                Timeout=self.timeout,
+                MemorySize=self.memory_size,
+                VpcConfig=self.vpc_config,
+                Environment=self.environment,
+                Layers=self.layers,
+            )
+            return response
+        except ClientError as e:
+            error = e.response["Error"]
+            raise ValueError(error)
+
+    def update(self):
+        """Method to update a lambda function.
+
+        Returns: boto3 response from Lambda's update_function method.
+        """
+        lambda_client = _get_lambda_client(self.session)
+        retry_attempts = 7
+        for i in range(retry_attempts):
+            try:
+                if self.script is not None:
+                    response = lambda_client.update_function_code(
+                        FunctionName=self.function_name or self.function_arn,
+                        ZipFile=_zip_lambda_code(self.script),
+                    )
+                else:
+                    bucket, key_prefix = s3.determine_bucket_and_prefix(
+                        bucket=self.s3_bucket, key_prefix=None, sagemaker_session=self.session
+                    )
+
+                    # get function name to be used in S3 upload path
+                    if self.function_arn:
+                        versioned_function_name = self.function_arn.split("funtion:")[-1]
+                        if ":" in versioned_function_name:
+                            function_name_for_s3 = versioned_function_name.split(":")[0]
+                        else:
+                            function_name_for_s3 = versioned_function_name
+                    else:
+                        function_name_for_s3 = self.function_name
+
+                    response = lambda_client.update_function_code(
+                        FunctionName=(self.function_name or self.function_arn),
+                        S3Bucket=bucket,
+                        S3Key=_upload_to_s3(
+                            s3_client=_get_s3_client(self.session),
+                            function_name=function_name_for_s3,
+                            zipped_code_dir=self.zipped_code_dir,
+                            s3_bucket=bucket,
+                            s3_key_prefix=key_prefix,
+                        ),
+                    )
+                return response
+            except ClientError as e:
+                error = e.response["Error"]
+                code = error["Code"]
+                if code == "ResourceConflictException":
+                    if i == retry_attempts - 1:
+                        raise ValueError(error)
+                    # max wait time = 2**0 + 2**1 + .. + 2**6 = 127 seconds
+                    time.sleep(2**i)
+                else:
+                    raise ValueError(error)
+
+    def upsert(self):
+        """Method to create a lambda function or update it if it already exists
+
+        Returns: boto3 response from Lambda's methods.
+        """
+        try:
+            return self.create()
+        except ValueError as error:
+            if "ResourceConflictException" in str(error):
+                return self.update()
+            raise
+
+    def invoke(self):
+        """Method to invoke a lambda function.
+
+        Returns: boto3 response from Lambda's invoke method.
+        """
+        lambda_client = _get_lambda_client(self.session)
+        try:
+            response = lambda_client.invoke(
+                FunctionName=self.function_name or self.function_arn,
+                InvocationType="RequestResponse",
+            )
+            return response
+        except ClientError as e:
+            error = e.response["Error"]
+            raise ValueError(error)
+
+    def delete(self):
+        """Method to delete a lambda function.
+
+        Returns: boto3 response from Lambda's delete_function method.
+        """
+        lambda_client = _get_lambda_client(self.session)
+        try:
+            response = lambda_client.delete_function(
+                FunctionName=self.function_name or self.function_arn
+            )
+            return response
+        except ClientError as e:
+            error = e.response["Error"]
+            raise ValueError(error)
+
+
+def _get_s3_client(session):
+    """Method to get a boto3 s3 client.
+
+    Returns: a s3 client.
+    """
+    sagemaker_session = session or Session()
+    if sagemaker_session.s3_client is None:
+        s3_client = sagemaker_session.boto_session.client(
+            "s3", region_name=sagemaker_session.boto_region_name
+        )
+    else:
+        s3_client = sagemaker_session.s3_client
+    return s3_client
+
+
+def _get_lambda_client(session):
+    """Method to get a boto3 lambda client.
+
+    Returns: a lambda client.
+    """
+    sagemaker_session = session or Session()
+    if sagemaker_session.lambda_client is None:
+        lambda_client = sagemaker_session.boto_session.client(
+            "lambda", region_name=sagemaker_session.boto_region_name
+        )
+    else:
+        lambda_client = sagemaker_session.lambda_client
+    return lambda_client
+
+
+def _upload_to_s3(s3_client, function_name, zipped_code_dir, s3_bucket, s3_key_prefix=None):
+    """Upload the zipped code to S3 bucket provided in the Lambda instance.
+
+    Lambda instance must have a path to the zipped code folder and a S3 bucket to upload
+    the code. The key will lambda/function_name/code and the S3 URI where the code is
+    uploaded is in this format: s3://bucket_name/lambda/function_name/code.
+
+    Returns: the S3 key where the code is uploaded.
+    """
+
+    key = s3.s3_path_join(
+        s3_key_prefix,
+        "lambda",
+        function_name,
+        "code",
+    )
+    s3_client.upload_file(zipped_code_dir, s3_bucket, key)
+    return key
+
+
+def _zip_lambda_code(script):
+    """This method zips the lambda function script.
+
+    Lambda function script is provided in the lambda instance and reads that zipped file.
+
+    Returns: A buffer of zipped lambda function script.
+    """
+    buffer = BytesIO()
+    code_dir = script.split("/")[-1]
+
+    with zipfile.ZipFile(buffer, "w") as z:
+        z.write(script, code_dir)
+    buffer.seek(0)
+    return buffer.read()

sagemaker/core/lineage/__init__.py

@@ -0,0 +1,42 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""SageMaker Lineage tracking and artifact management."""
+from __future__ import absolute_import
+
+from sagemaker.core.lineage.action import Action  # noqa: F401
+from sagemaker.core.lineage.artifact import Artifact  # noqa: F401
+from sagemaker.core.lineage.association import Association  # noqa: F401
+from sagemaker.core.lineage.context import Context  # noqa: F401
+from sagemaker.core.lineage.lineage_trial_component import LineageTrialComponent  # noqa: F401
+from sagemaker.core.lineage.query import (  # noqa: F401
+    LineageEntityEnum,
+    LineageFilter,
+    LineageQuery,
+    LineageQueryDirectionEnum,
+    LineageSourceEnum,
+)
+from sagemaker.core.lineage.visualizer import LineageTableVisualizer  # noqa: F401
+
+__all__ = [
+    "Action",
+    "Artifact",
+    "Association",
+    "Context",
+    "LineageEntityEnum",
+    "LineageFilter",
+    "LineageQuery",
+    "LineageQueryDirectionEnum",
+    "LineageSourceEnum",
+    "LineageTableVisualizer",
+    "LineageTrialComponent",
+]