sagemaker-core 1.0.62__py3-none-any.whl → 2.3.1__py3-none-any.whl

sagemaker/core/workflow/entities.py

@@ -0,0 +1,58 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Defines the base entities used in workflow."""
+from __future__ import absolute_import
+
+import abc
+
+from enum import EnumMeta
+from typing import Any, Dict, List, Union, TYPE_CHECKING
+
+try:
+    from pydantic_core import core_schema
+except ImportError:
+    core_schema = None
+
+if TYPE_CHECKING:
+    pass
+
+# Import PipelineVariable from the migrated location
+from sagemaker.core.helper.pipeline_variable import PipelineVariable  # noqa: F401
+
+PrimitiveType = Union[str, int, bool, float, None]
+RequestType = Union[Dict[str, Any], List[Dict[str, Any]]]
+
+
+class Entity(abc.ABC):
+    """Base object for workflow entities.
+
+    Entities must implement the to_request method.
+    """
+
+    @abc.abstractmethod
+    def to_request(self) -> RequestType:
+        """Get the request structure for workflow service calls."""
+
+
+class DefaultEnumMeta(EnumMeta):
+    """An EnumMeta which defaults to the first value in the Enum list."""
+
+    default = object()
+
+    def __call__(cls, *args, value=default, **kwargs):
+        """Defaults to the first value in the Enum list."""
+        if value is DefaultEnumMeta.default:
+            return next(iter(cls))
+        return super().__call__(value, *args, **kwargs)
+
+    factory = __call__

sagemaker/core/workflow/execution_variables.py

@@ -0,0 +1,89 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Pipeline parameters and conditions for workflow."""
+from __future__ import absolute_import
+
+from typing import List, TYPE_CHECKING
+from sagemaker.core.helper.pipeline_variable import (
+    RequestType,
+    PipelineVariable,
+)
+
+if TYPE_CHECKING:
+    from sagemaker.mlops.workflow.steps import Step
+
+
+class ExecutionVariable(PipelineVariable):
+    """Pipeline execution variables for workflow."""
+
+    def __init__(self, name: str):
+        """Create a pipeline execution variable.
+
+        Args:
+            name (str): The name of the execution variable.
+        """
+        self.name = name
+
+    def __eq__(self, other):
+        """Override default equals method"""
+        if not isinstance(other, ExecutionVariable):
+            return NotImplemented
+        return self.name == other.name
+
+    def to_string(self) -> PipelineVariable:
+        """Prompt the pipeline to convert the pipeline variable to String in runtime
+
+        As ExecutionVariable is treated as String in runtime, no extra actions are needed.
+        """
+        return self
+
+    @property
+    def expr(self) -> RequestType:
+        """The 'Get' expression dict for an `ExecutionVariable`."""
+        return {"Get": f"Execution.{self.name}"}
+
+    @property
+    def _pickleable(self):
+        """The pickleable object that can be passed to a remote function invocation."""
+
+        from sagemaker.core.remote_function.core.pipeline_variables import _ExecutionVariable
+
+        return _ExecutionVariable(name=self.name)
+
+    @property
+    def _referenced_steps(self) -> List["Step"]:
+        """List of step that this function depends on."""
+        return []
+
+
+class ExecutionVariables:
+    """Provide access to all available execution variables:
+
+    - ExecutionVariables.START_DATETIME
+    - ExecutionVariables.CURRENT_DATETIME
+    - ExecutionVariables.PIPELINE_NAME
+    - ExecutionVariables.PIPELINE_ARN
+    - ExecutionVariables.PIPELINE_EXECUTION_ID
+    - ExecutionVariables.PIPELINE_EXECUTION_ARN
+    - ExecutionVariables.TRAINING_JOB_NAME
+    - ExecutionVariables.PROCESSING_JOB_NAME
+    """
+
+    START_DATETIME = ExecutionVariable("StartDateTime")
+    CURRENT_DATETIME = ExecutionVariable("CurrentDateTime")
+    PIPELINE_NAME = ExecutionVariable("PipelineName")
+    PIPELINE_ARN = ExecutionVariable("PipelineArn")
+    PIPELINE_EXECUTION_ID = ExecutionVariable("PipelineExecutionId")
+    PIPELINE_EXECUTION_ARN = ExecutionVariable("PipelineExecutionArn")
+    TRAINING_JOB_NAME = ExecutionVariable("TrainingJobName")
+    PROCESSING_JOB_NAME = ExecutionVariable("ProcessingJobName")

sagemaker/core/workflow/functions.py

@@ -0,0 +1,193 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""The step definitions for workflow."""
+from __future__ import absolute_import
+
+from typing import List, Union, Optional, TYPE_CHECKING
+
+import attr
+
+from sagemaker.core.helper.pipeline_variable import PipelineVariable
+from sagemaker.core.workflow.execution_variables import ExecutionVariable
+from sagemaker.core.workflow.parameters import Parameter
+from sagemaker.core.workflow.properties import PropertyFile, Properties
+
+
+def is_pipeline_variable(var: object) -> bool:
+    """Check if the variable is a pipeline variable
+
+    Args:
+        var (object): The variable to be verified.
+    Returns:
+         bool: True if it is, False otherwise.
+    """
+    return isinstance(var, PipelineVariable)
+
+
+if TYPE_CHECKING:
+    from sagemaker.mlops.workflow.steps import Step
+
+
+@attr.s
+class Join(PipelineVariable):
+    """Join together properties.
+
+    Examples:
+    Build a Amazon S3 Uri with bucket name parameter and pipeline execution Id and use it
+    as training input::
+
+        bucket = ParameterString('bucket', default_value='my-bucket')
+
+        TrainingInput(
+            s3_data=Join(
+                on='/',
+                values=['s3:/', bucket, ExecutionVariables.PIPELINE_EXECUTION_ID]
+            ),
+            content_type="text/csv")
+
+    Attributes:
+        values (List[Union[PrimitiveType, Parameter, PipelineVariable]]):
+            The primitive type values, parameters, step properties, expressions to join.
+        on (str): The string to join the values on (Defaults to "").
+    """
+
+    on: str = attr.ib(factory=str)
+    values: List = attr.ib(factory=list)
+
+    def to_string(self) -> PipelineVariable:
+        """Prompt the pipeline to convert the pipeline variable to String in runtime
+
+        As Join is treated as String in runtime, no extra actions are needed.
+        """
+        return self
+
+    @property
+    def expr(self):
+        """The expression dict for a `Join` function."""
+
+        return {
+            "Std:Join": {
+                "On": self.on,
+                "Values": [
+                    value.expr if hasattr(value, "expr") else value for value in self.values
+                ],
+            },
+        }
+
+    @property
+    def _referenced_steps(self) -> List[Union["Step", str]]:
+        """List of step names that this function depends on."""
+        steps = []
+        for value in self.values:
+            if isinstance(value, PipelineVariable):
+                steps.extend(value._referenced_steps)
+        return steps
+
+
+@attr.s
+class JsonGet(PipelineVariable):
+    """Get JSON properties from PropertyFiles or S3 location.
+
+    Attributes:
+        step_name (str): The step name from which to get the property file.
+        property_file (Optional[Union[PropertyFile, str]]): Either a PropertyFile instance
+            or the name of a property file.
+        json_path (str): The JSON path expression to the requested value.
+        s3_uri (Optional[sagemaker.workflow.functions.Join]): The S3 location from which to fetch
+            a Json file. The Json file is the output of a step defined with ``@step`` decorator.
+        step (Step): The upstream step object which the s3_uri is associated to.
+    """
+
+    # pylint: disable=W0613
+    def _check_property_file_s3_uri(self, attribute, value):
+        """Validate mutually exclusive property file / s3uri"""
+        if self.property_file and self.s3_uri:
+            raise ValueError(
+                "Please specify either a property file or s3 uri as an input, but not both."
+            )
+        if not self.property_file and not self.s3_uri:
+            raise ValueError(
+                "Missing s3uri or property file as a required input to JsonGet."
+                "Please specify either a property file or s3 uri as an input, but not both."
+            )
+        if self.s3_uri:
+            self._validate_json_get_s3_uri()
+
+    step_name: str = attr.ib(default=None)
+    property_file: Optional[Union[PropertyFile, str]] = attr.ib(
+        default=None, validator=_check_property_file_s3_uri
+    )
+    json_path: str = attr.ib(default=None)
+    s3_uri: Optional[Join] = attr.ib(default=None, validator=_check_property_file_s3_uri)
+    step: "Step" = attr.ib(default=None)
+
+    # pylint: disable=R1710
+    @property
+    def expr(self):
+        """The expression dict for a `JsonGet` function."""
+
+        if self.property_file:
+            if not isinstance(self.step_name, str) or not self.step_name:
+                raise ValueError("Please give a valid step name as a string.")
+            if isinstance(self.property_file, PropertyFile):
+                name = self.property_file.name
+            else:
+                name = self.property_file
+            return {
+                "Std:JsonGet": {
+                    "PropertyFile": {"Get": f"Steps.{self.step_name}.PropertyFiles.{name}"},
+                    "Path": self.json_path,
+                }
+            }
+
+        # ConditionStep uses a JoinFunction to provide this non-static, built s3Uri in
+        # the case of Lightsaber steps.
+        if self.s3_uri:
+            return {
+                "Std:JsonGet": {
+                    "S3Uri": (
+                        self.s3_uri.expr
+                        if isinstance(self.s3_uri, PipelineVariable)
+                        else self.s3_uri
+                    ),
+                    "Path": self.json_path,
+                }
+            }
+
+    @property
+    def _referenced_steps(self) -> List[Union["Step", str]]:
+        """List of step that this function depends on."""
+        if self.step:
+            return [self.step]
+        if self.step_name:
+            return [self.step_name]
+
+        return []
+
+    def _validate_json_get_s3_uri(self):
+        """Validate the s3 uri in JsonGet"""
+        s3_uri = self.s3_uri
+        if not isinstance(s3_uri, Join):
+            raise ValueError(
+                f"Invalid JsonGet function {self.expr}. JsonGet "
+                "function's s3_uri can only be a sagemaker.workflow.functions.Join object."
+            )
+        for join_arg in s3_uri.values:
+            if not is_pipeline_variable(join_arg):
+                continue
+            if not isinstance(join_arg, (Parameter, ExecutionVariable, Properties)):
+                raise ValueError(
+                    f"Invalid JsonGet function {self.expr}. "
+                    f"The Join values in JsonGet's s3_uri can only be a primitive object, "
+                    f"Parameter, ExecutionVariable or Properties."
+                )

sagemaker/core/workflow/parameters.py

@@ -0,0 +1,222 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Pipeline parameters and conditions for workflow."""
+from __future__ import absolute_import
+
+from enum import Enum
+from functools import partial
+from typing import Dict, List, Type
+
+import attr
+
+from sagemaker.core.workflow.entities import (
+    DefaultEnumMeta,
+    Entity,
+)
+from sagemaker.core.helper.pipeline_variable import (
+    PrimitiveType,
+    RequestType,
+    PipelineVariable,
+)
+
+
+class ParameterTypeEnum(Enum, metaclass=DefaultEnumMeta):
+    """Parameter type enum."""
+
+    STRING = "String"
+    INTEGER = "Integer"
+    BOOLEAN = "Boolean"
+    FLOAT = "Float"
+
+    @property
+    def python_type(self) -> Type:
+        """Provide the Python type of the enum value."""
+        mapping = {
+            ParameterTypeEnum.STRING: str,
+            ParameterTypeEnum.INTEGER: int,
+            ParameterTypeEnum.BOOLEAN: bool,
+            ParameterTypeEnum.FLOAT: float,
+        }
+        return mapping[self]
+
+
+@attr.s
+class Parameter(PipelineVariable, Entity):
+    """Pipeline parameter for workflow.
+
+    Attributes:
+        name (str): The name of the parameter.
+        parameter_type (ParameterTypeEnum): The type of the parameter.
+        default_value (PrimitiveType): The default value of the parameter.
+    """
+
+    name: str = attr.ib(factory=str)
+    parameter_type: ParameterTypeEnum = attr.ib(factory=ParameterTypeEnum.factory)
+    default_value: PrimitiveType = attr.ib(default=None)
+
+    @default_value.validator
+    def _check_default_value(self, _, value):
+        """Check whether the default value is compatible with the parameter type.
+
+        Args:
+            _: unused argument required by attrs validator decorator.
+            value: The value to check the type for.
+
+        Raises:
+            `TypeError` if the value is not compatible with the instance's Python type.
+        """
+        self._check_default_value_type(value, self.parameter_type.python_type)
+
+    def to_request(self) -> RequestType:
+        """Get the request structure for workflow service calls."""
+        value = {
+            "Name": self.name,
+            "Type": self.parameter_type.value,
+        }
+        if self.default_value is not None:
+            value["DefaultValue"] = self.default_value
+        return value
+
+    @property
+    def expr(self) -> Dict[str, str]:
+        """The 'Get' expression dict for a `Parameter`."""
+        return Parameter._expr(self.name)
+
+    @property
+    def _pickleable(self):
+        """The pickleable object that can be passed to a remote function invocation."""
+
+        from sagemaker.core.remote_function.core.pipeline_variables import (
+            _ParameterString,
+            _ParameterInteger,
+            _ParameterBoolean,
+            _ParameterFloat,
+        )
+
+        if self.parameter_type == ParameterTypeEnum.STRING:
+            return _ParameterString(self.name)
+        if self.parameter_type == ParameterTypeEnum.INTEGER:
+            return _ParameterInteger(self.name)
+        if self.parameter_type == ParameterTypeEnum.BOOLEAN:
+            return _ParameterBoolean(self.name)
+        if self.parameter_type == ParameterTypeEnum.FLOAT:
+            return _ParameterFloat(self.name)
+        raise ValueError(f"Unsupported parameter type: {self.parameter_type}")
+
+    @property
+    def _referenced_steps(self) -> List[str]:
+        """List of step names that this function depends on."""
+        return []
+
+    @classmethod
+    def _expr(cls, name):
+        """An internal classmethod for the 'Get' expression dict for a `Parameter`.
+
+        Args:
+            name (str): The name of the parameter.
+        """
+        return {"Get": f"Parameters.{name}"}
+
+    @classmethod
+    def _check_default_value_type(cls, value, python_type):
+        """Check whether the default value is compatible with the parameter type.
+
+        Args:
+            value: The value to check the type for.
+            python_type: The type to check the value against.
+
+        Raises:
+            `TypeError` if the value is not compatible with the instance's Python type.
+        """
+        if value and not isinstance(value, python_type):
+            raise TypeError("The default value specified does not match the Parameter Python type.")
+
+
+# NOTE: partials do not handle metadata well, but make for at least "partial" syntactic sugar :-P
+# proper implementation postponed, for "reasons": https://bugs.python.org/issue33419
+# NOTE: cannot subclass bool: http://mail.python.org/pipermail/python-dev/2002-March/020822.html
+ParameterBoolean = partial(Parameter, parameter_type=ParameterTypeEnum.BOOLEAN)
+
+
+class ParameterString(Parameter):
+    """String parameter for pipelines."""
+
+    def __init__(self, name: str, default_value: str = None, enum_values: List[str] = None):
+        """Create a pipeline string parameter.
+
+        Args:
+            name (str): The name of the parameter.
+            default_value (str): The default value of the parameter.
+                The default value could be overridden at start of an execution.
+                If not set or it is set to None, a value must be provided
+                at the start of the execution.
+            enum_values (List[str]): Enum values for this parameter.
+        """
+        super(ParameterString, self).__init__(
+            name=name, parameter_type=ParameterTypeEnum.STRING, default_value=default_value
+        )
+        self.enum_values = enum_values
+
+    def __hash__(self):
+        """Hash function for parameter types"""
+        return hash(tuple(self.to_request()))
+
+    def to_string(self) -> PipelineVariable:
+        """Prompt the pipeline to convert the pipeline variable to String in runtime
+
+        As ParameterString is treated as String in runtime, no extra actions are needed.
+        """
+        return self
+
+    def to_request(self) -> RequestType:
+        """Get the request structure for workflow service calls."""
+        request_dict = super(ParameterString, self).to_request()
+        if self.enum_values:
+            request_dict["EnumValues"] = self.enum_values
+        return request_dict
+
+
+class ParameterInteger(Parameter):
+    """Integer parameter for pipelines."""
+
+    def __init__(self, name: str, default_value: int = None):
+        """Create a pipeline integer parameter.
+
+        Args:
+            name (str): The name of the parameter.
+            default_value (int): The default value of the parameter.
+                The default value could be overridden at start of an execution.
+                If not set or it is set to None, a value must be provided
+                at the start of the execution.
+        """
+        super(ParameterInteger, self).__init__(
+            name=name, parameter_type=ParameterTypeEnum.INTEGER, default_value=default_value
+        )
+
+
+class ParameterFloat(Parameter):
+    """Float parameter for pipelines."""
+
+    def __init__(self, name: str, default_value: float = None):
+        """Create a pipeline float parameter.
+
+        Args:
+            name (str): The name of the parameter.
+            default_value (float): The default value of the parameter.
+                The default value could be overridden at start of an execution.
+                If not set or it is set to None, a value must be provided
+                at the start of the execution.
+        """
+        super(ParameterFloat, self).__init__(
+            name=name, parameter_type=ParameterTypeEnum.FLOAT, default_value=default_value
+        )