sagemaker-core 1.0.62__py3-none-any.whl → 2.3.1__py3-none-any.whl

sagemaker/core/local/utils.py

@@ -0,0 +1,246 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Placeholder docstring"""
+from __future__ import absolute_import
+
+import os
+import logging
+import shutil
+import subprocess
+import json
+import re
+import errno
+
+from sagemaker.core import s3
+from six.moves.urllib.parse import urlparse
+
+
+logger = logging.getLogger(__name__)
+
+STUDIO_APP_TYPES = ["KernelGateway", "CodeEditor", "JupyterLab"]
+
+
+def copy_directory_structure(destination_directory, relative_path):
+    """Creates intermediate directory structure for relative_path.
+
+    Create all the intermediate directories required for relative_path to
+    exist within destination_directory. This assumes that relative_path is a
+    directory located within root_dir.
+
+    Examples:
+        destination_directory: /tmp/destination relative_path: test/unit/
+        will create: /tmp/destination/test/unit
+
+    Args:
+        destination_directory (str): root of the destination directory where the
+            directory structure will be created.
+        relative_path (str): relative path that will be created within
+            destination_directory
+    """
+    full_path = os.path.join(destination_directory, relative_path)
+    os.makedirs(full_path, exist_ok=True)
+
+
+def move_to_destination(source, destination, job_name, sagemaker_session, prefix=""):
+    """Move source to destination.
+
+    Can handle uploading to S3.
+
+    Args:
+        source (str): root directory to move
+        destination (str): file:// or s3:// URI that source will be moved to.
+        job_name (str): SageMaker job name.
+        sagemaker_session (sagemaker.Session): a sagemaker_session to interact
+            with S3 if needed
+        prefix (str, optional): the directory on S3 used to save files, default
+            to the root of  ``destination``
+
+    Returns:
+        (str): destination URI
+    """
+    parsed_uri = urlparse(destination)
+    if parsed_uri.scheme == "file":
+        dir_path = os.path.abspath(parsed_uri.netloc + parsed_uri.path)
+        recursive_copy(source, dir_path)
+        final_uri = destination
+    elif parsed_uri.scheme == "s3":
+        bucket = parsed_uri.netloc
+        path = s3.s3_path_join(parsed_uri.path, job_name, prefix)
+        final_uri = s3.s3_path_join("s3://", bucket, path)
+        sagemaker_session.upload_data(source, bucket, path)
+    else:
+        raise ValueError("Invalid destination URI, must be s3:// or file://, got: %s" % destination)
+
+    try:
+        shutil.rmtree(source)
+    except OSError as exc:
+        # on Linux, when docker writes to any mounted volume, it uses the container's user. In most
+        # cases this is root. When the container exits and we try to delete them we can't because
+        # root owns those files. We expect this to happen, so we handle EACCESS. Any other error
+        # we will raise the exception up.
+        if exc.errno == errno.EACCES:
+            logger.warning("Failed to delete: %s Please remove it manually.", source)
+        else:
+            logger.error("Failed to delete: %s", source)
+            raise
+
+    return final_uri
+
+
+def recursive_copy(source, destination):
+    """A wrapper around shutil.copy_tree.
+
+    This won't throw any exception when the source directory does not exist.
+
+    Args:
+        source (str): source path
+        destination (str): destination path
+    """
+    if os.path.isdir(source):
+        shutil.copytree(source, destination, dirs_exist_ok=True)
+
+
+def kill_child_processes(pid):
+    """Kill child processes
+
+    Kills all nested child process ids for a specific pid
+
+    Args:
+        pid (int): process id
+    """
+    child_pids = get_child_process_ids(pid)
+    for child_pid in child_pids:
+        os.kill(child_pid, 15)
+
+
+def get_child_process_ids(pid):
+    """Retrieve all child pids for a certain pid
+
+    Recursively scan each childs process tree and add it to the output
+
+    Args:
+        pid (int): process id
+
+    Returns:
+        (List[int]): Child process ids
+    """
+    if not str(pid).isdigit():
+        raise ValueError("Invalid PID")
+    
+    cmd = ["pgrep", "-P", str(pid)]
+    
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    output, err = process.communicate()
+    if err:
+        return []
+    pids = [int(pid) for pid in output.decode("utf-8").split()]
+    if pids:
+        for child_pid in pids:
+            return pids + get_child_process_ids(child_pid)
+    else:
+        return []
+
+
+def get_docker_host():
+    """Discover remote docker host address (if applicable) or use "localhost"
+
+    Use "docker context inspect" to read current docker host endpoint url,
+    url must start with "tcp://"
+
+    Args:
+
+    Returns:
+        docker_host (str): Docker host DNS or IP address
+    """
+    cmd = "docker context inspect".split()
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    output, err = process.communicate()
+    if err:
+        return "localhost"
+    docker_context_string = output.decode("utf-8")
+    docker_context_host_url = json.loads(docker_context_string)[0]["Endpoints"]["docker"]["Host"]
+    parsed_url = urlparse(docker_context_host_url)
+    if parsed_url.hostname and parsed_url.scheme == "tcp":
+        return parsed_url.hostname
+    return "localhost"
+
+
+def get_using_dot_notation(dictionary, keys):
+    """Extract `keys` from dictionary where keys is a string in dot notation.
+
+    Args:
+        dictionary (Dict)
+        keys (str)
+
+    Returns:
+        Nested object within dictionary as defined by "keys"
+
+    Raises:
+     ValueError if the provided key does not exist in input dictionary
+    """
+    try:
+        if keys is None:
+            return dictionary
+        split_keys = keys.split(".", 1)
+        key = split_keys[0]
+        rest = None
+        if len(split_keys) > 1:
+            rest = split_keys[1]
+        bracket_accessors = re.findall(r"\[(.+?)]", key)
+        if bracket_accessors:
+            pre_bracket_key = key.split("[", 1)[0]
+            inner_dict = dictionary[pre_bracket_key]
+        else:
+            inner_dict = dictionary[key]
+        for bracket_accessor in bracket_accessors:
+            if (
+                bracket_accessor.startswith("'")
+                and bracket_accessor.endswith("'")
+                or bracket_accessor.startswith('"')
+                and bracket_accessor.endswith('"')
+            ):
+                # key accessor
+                inner_key = bracket_accessor[1:-1]
+            else:
+                # list accessor
+                inner_key = int(bracket_accessor)
+            inner_dict = inner_dict[inner_key]
+        return get_using_dot_notation(inner_dict, rest)
+    except (KeyError, IndexError, TypeError):
+        raise ValueError(f"{keys} does not exist in input dictionary.")
+
+
+def check_for_studio():
+    """Helper function to determine if the run environment is studio.
+
+    Returns (bool): Returns True if valid Studio request.
+
+    Raises:
+        NotImplementedError:
+            if run environment = Studio and AppType not in STUDIO_APP_TYPES
+    """
+    is_studio = False
+    if os.path.exists("/opt/ml/metadata/resource-metadata.json"):
+        with open("/opt/ml/metadata/resource-metadata.json", "r") as handle:
+            metadata = json.load(handle)
+        app_type = metadata.get("AppType")
+        if app_type:
+            # check if the execution is triggered from Studio KernelGateway App
+            if app_type in STUDIO_APP_TYPES:
+                is_studio = True
+            else:
+                raise NotImplementedError(
+                    f"AppType {app_type} in Studio does not support Local Mode."
+                )
+        # if no apptype, case of classic notebooks
+    return is_studio

sagemaker/core/logs.py

@@ -0,0 +1,181 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Placeholder docstring"""
+from __future__ import absolute_import
+
+import collections
+import functools
+import os
+import sys
+
+##############################################################################
+#
+# Support for reading logs
+#
+##############################################################################
+
+
+class ColorWrap(object):
+    """A callable that will print text in a different color depending on the instance.
+
+    Up to 6 if standard output is a terminal or a Jupyter notebook cell.
+    """
+
+    # For what color each number represents, see
+    # https://misc.flogisoft.com/bash/tip_colors_and_formatting#colors
+    _stream_colors = [34, 35, 32, 36, 33]
+
+    def __init__(self, force=False):
+        """Initialize the class.
+
+        Args:
+            force (bool): If True, render colorizes output no matter where the
+                output is (default: False).
+        """
+        self.colorize = force or sys.stdout.isatty() or os.environ.get("JPY_PARENT_PID", None)
+
+    def __call__(self, index, s):
+        """Print the output, colorized or not, depending on the environment.
+
+        Args:
+            index (int): The instance number.
+            s (str): The string to print.
+        """
+        if self.colorize:
+            self._color_wrap(index, s)
+        else:
+            print(s)
+
+    def _color_wrap(self, index, s):
+        """Placeholder docstring"""
+        print("\x1b[{}m{}\x1b[0m".format(self._stream_colors[index % len(self._stream_colors)], s))
+
+
+def argmin(arr, f):
+    """Return the index, i, in arr that minimizes f(arr[i])
+
+    Args:
+        arr:
+        f:
+    """
+    m = None
+    i = None
+    for idx, item in enumerate(arr):
+        if item is not None:
+            if m is None or f(item) < m:
+                m = f(item)
+                i = idx
+    return i
+
+
+def some(arr):
+    """Return True iff there is an element, a, of arr such that a is not None.
+
+    Args:
+        arr:
+    """
+    return functools.reduce(lambda x, y: x or (y is not None), arr, False)
+
+
+# Position is a tuple that includes the last read timestamp and the number of items that were read
+# at that time. This is used to figure out which event to start with on the next read.
+Position = collections.namedtuple("Position", ["timestamp", "skip"])
+
+
+def multi_stream_iter(client, log_group, streams, positions=None):
+    """Iterate over the available events coming from a set of log streams.
+
+    Log streams are in a single log group interleaving the events from each stream
+    so they're yielded in timestamp order.
+
+    Args:
+        client (boto3 client): The boto client for logs.
+        log_group (str): The name of the log group.
+        streams (list of str): A list of the log stream names. The position of the stream in
+        this list is the stream number.
+        positions: (list of Positions): A list of pairs of (timestamp, skip) which represents
+        the last record read from each stream.
+
+    Yields:
+        A tuple of (stream number, cloudwatch log event).
+    """
+    positions = positions or {s: Position(timestamp=0, skip=0) for s in streams}
+    event_iters = [
+        log_stream(client, log_group, s, positions[s].timestamp, positions[s].skip) for s in streams
+    ]
+    events = []
+    for s in event_iters:
+        if not s:
+            events.append(None)
+            continue
+        try:
+            events.append(next(s))
+        except StopIteration:
+            events.append(None)
+
+    while some(events):
+        i = argmin(events, lambda x: x["timestamp"] if x else 9999999999)
+        yield (i, events[i])
+        try:
+            events[i] = next(event_iters[i])
+        except StopIteration:
+            events[i] = None
+
+
+def log_stream(client, log_group, stream_name, start_time=0, skip=0):
+    """A generator for log items in a single stream.
+
+    This will yield all the items that are available at the current moment.
+
+    Args:
+        client (boto3.CloudWatchLogs.Client): The Boto client for CloudWatch logs.
+        log_group (str): The name of the log group.
+        stream_name (str): The name of the specific stream.
+        start_time (int): The time stamp value to start reading the logs from (default: 0).
+        skip (int): The number of log entries to skip at the start (default: 0). This is for
+        when there are multiple entries at the same timestamp.
+
+    Yields:
+       dict: A CloudWatch log event with the following key-value pairs:
+           'timestamp' (int): The time of the event.
+           'message' (str): The log event data.
+           'ingestionTime' (int): The time the event was ingested.
+    """
+
+    next_token = None
+
+    event_count = 1
+    while event_count > 0:
+        if next_token is not None:
+            token_arg = {"nextToken": next_token}
+        else:
+            token_arg = {}
+
+        response = client.get_log_events(
+            logGroupName=log_group,
+            logStreamName=stream_name,
+            startTime=start_time,
+            startFromHead=True,
+            **token_arg,
+        )
+        next_token = response["nextForwardToken"]
+        events = response["events"]
+        event_count = len(events)
+        if event_count > skip:
+            events = events[skip:]
+            skip = 0
+        else:
+            skip = skip - event_count
+            events = []
+        for ev in events:
+            yield ev

sagemaker/core/metadata_properties.py

@@ -0,0 +1,56 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This file contains code related to metadata properties."""
+from __future__ import absolute_import
+
+from typing import Optional, Union
+
+from sagemaker.core.helper.pipeline_variable import PipelineVariable
+
+
+class MetadataProperties(object):
+    """Accepts metadata properties parameters for conversion to request dict."""
+
+    def __init__(
+        self,
+        commit_id: Optional[Union[str, PipelineVariable]] = None,
+        repository: Optional[Union[str, PipelineVariable]] = None,
+        generated_by: Optional[Union[str, PipelineVariable]] = None,
+        project_id: Optional[Union[str, PipelineVariable]] = None,
+    ):
+        """Initialize a ``MetadataProperties`` instance and turn parameters into dict.
+
+        # TODO: flesh out docstrings
+        Args:
+            commit_id (str or PipelineVariable):
+            repository (str or PipelineVariable):
+            generated_by (str or PipelineVariable):
+            project_id (str or PipelineVariable):
+        """
+        self.commit_id = commit_id
+        self.repository = repository
+        self.generated_by = generated_by
+        self.project_id = project_id
+
+    def _to_request_dict(self):
+        """Generates a request dictionary using the parameters provided to the class."""
+        metadata_properties_request = dict()
+        if self.commit_id:
+            metadata_properties_request["CommitId"] = self.commit_id
+        if self.repository:
+            metadata_properties_request["Repository"] = self.repository
+        if self.generated_by:
+            metadata_properties_request["GeneratedBy"] = self.generated_by
+        if self.project_id:
+            metadata_properties_request["ProjectId"] = self.project_id
+        return metadata_properties_request

sagemaker/core/metric_definitions.py

@@ -0,0 +1,91 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Accessors to retrieve metric definition for training jobs."""
+
+from __future__ import absolute_import
+
+import logging
+from typing import Dict, Optional, List
+
+from sagemaker.core.jumpstart import utils as jumpstart_utils
+from sagemaker.core.jumpstart import artifacts
+from sagemaker.core.jumpstart.constants import DEFAULT_JUMPSTART_SAGEMAKER_SESSION
+from sagemaker.core.jumpstart.enums import JumpStartModelType
+from sagemaker.core.helper.session_helper import Session
+
+logger = logging.getLogger(__name__)
+
+
+def retrieve_default(
+    region: Optional[str] = None,
+    model_id: Optional[str] = None,
+    model_version: Optional[str] = None,
+    hub_arn: Optional[str] = None,
+    instance_type: Optional[str] = None,
+    tolerate_vulnerable_model: bool = False,
+    tolerate_deprecated_model: bool = False,
+    sagemaker_session: Session = DEFAULT_JUMPSTART_SAGEMAKER_SESSION,
+    config_name: Optional[str] = None,
+    model_type: JumpStartModelType = JumpStartModelType.OPEN_WEIGHTS,
+) -> Optional[List[Dict[str, str]]]:
+    """Retrieves the default training metric definitions for the model matching the given arguments.
+
+    Args:
+        region (str): The AWS Region for which to retrieve the default default training metric
+            definitions. Defaults to ``None``.
+        model_id (str): The model ID of the model for which to
+            retrieve the default training metric definitions. (Default: None).
+        model_version (str): The version of the model for which to retrieve the
+            default training metric definitions. (Default: None).
+        hub_arn (str): The arn of the SageMaker Hub for which to retrieve
+            model details from. (default: None).
+        instance_type (str): An instance type to optionally supply in order to get
+            metric definitions specific for the instance type.
+        tolerate_vulnerable_model (bool): True if vulnerable versions of model
+            specifications should be tolerated (exception not raised). If False, raises an
+            exception if the script used by this version of the model has dependencies with known
+            security vulnerabilities. (Default: False).
+        tolerate_deprecated_model (bool): True if deprecated models should be tolerated
+            (exception not raised). False if these models should raise an exception.
+            (Default: False).
+        sagemaker_session (sagemaker.session.Session): A SageMaker Session
+            object, used for SageMaker interactions. If not
+            specified, one is created using the default AWS configuration
+            chain. (Default: sagemaker.jumpstart.constants.DEFAULT_JUMPSTART_SAGEMAKER_SESSION).
+        config_name (Optional[str]): Name of the JumpStart Model config to apply. (Default: None).
+        model_type (JumpStartModelType): The type of the model, can be open weights model
+            or proprietary model. (Default: JumpStartModelType.OPEN_WEIGHTS).
+    Returns:
+        list: The default metric definitions to use for the model or None.
+
+    Raises:
+        ValueError: If the combination of arguments specified is not supported.
+    """
+    if not jumpstart_utils.is_jumpstart_model_input(model_id, model_version):
+        raise ValueError(
+            "Must specify JumpStart `model_id` and `model_version` "
+            "when retrieving default training metric definitions."
+        )
+
+    return artifacts._retrieve_default_training_metric_definitions(
+        model_id=model_id,
+        model_version=model_version,
+        hub_arn=hub_arn,
+        instance_type=instance_type,
+        region=region,
+        tolerate_vulnerable_model=tolerate_vulnerable_model,
+        tolerate_deprecated_model=tolerate_deprecated_model,
+        sagemaker_session=sagemaker_session,
+        config_name=config_name,
+        model_type=model_type,
+    )

sagemaker/core/mlflow/__init__.py

@@ -0,0 +1,38 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""MLflow integration module - stub.
+
+This module provides stubs for MLflow integration functionality.
+MLflow integration is an optional feature that requires the mlflow package.
+
+NOTE: This is a stub module. Full MLflow integration will be implemented
+in a future release.
+"""
+from __future__ import absolute_import
+
+__all__ = ["forward_sagemaker_metrics"]
+
+
+def forward_sagemaker_metrics(*args, **kwargs):
+    """Stub for MLflow metrics forwarding.
+
+    This function is not yet implemented. MLflow integration is an optional
+    feature that will be added in a future release.
+
+    Raises:
+        NotImplementedError: Always raised as this is a stub.
+    """
+    raise NotImplementedError(
+        "MLflow integration is not yet implemented. "
+        "This feature will be added in a future release."
+    )

sagemaker/core/mlflow/forward_sagemaker_metrics.py

@@ -0,0 +1,44 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""MLflow metrics forwarding module - stub.
+
+This module provides stubs for forwarding SageMaker metrics to MLflow.
+MLflow integration is an optional feature that requires the mlflow package.
+
+NOTE: This is a stub module. Full MLflow integration will be implemented
+in a future release.
+"""
+from __future__ import absolute_import
+
+__all__ = ["log_sagemaker_job_to_mlflow"]
+
+
+def log_sagemaker_job_to_mlflow(job_name, *args, **kwargs):
+    """Stub for logging SageMaker job metrics to MLflow.
+
+    This function is not yet implemented. MLflow integration is an optional
+    feature that will be added in a future release.
+
+    Args:
+        job_name (str): Name of the SageMaker training job
+        *args: Additional positional arguments (ignored)
+        **kwargs: Additional keyword arguments (ignored)
+
+    Raises:
+        NotImplementedError: Always raised as this is a stub.
+    """
+    raise NotImplementedError(
+        "MLflow integration is not yet implemented. "
+        "This feature will be added in a future release. "
+        f"Cannot log job '{job_name}' to MLflow."
+    )

sagemaker/core/model_card/__init__.py

@@ -0,0 +1,26 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Model Card utilities for SageMaker Python SDK."""
+from __future__ import absolute_import
+
+# Re-export ModelCard from resources
+from sagemaker.core.resources import ModelCard, ModelCardExportJob  # noqa: F401
+
+# Re-export from shapes
+from sagemaker.core.shapes.model_card_shapes import (  # noqa: F401
+    ModelCardContent,
+    TrainingDetails,
+)
+
+# Alias for backward compatibility
+ModelPackageModelCard = ModelCardContent