sagemaker-core 1.0.62__py3-none-any.whl → 2.3.1__py3-none-any.whl

sagemaker/core/experiments/_utils.py

@@ -0,0 +1,216 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Contains the SageMaker Experiment utility methods."""
+from __future__ import absolute_import
+
+import logging
+import os
+
+import mimetypes
+import urllib
+from functools import wraps
+from typing import Optional
+
+from sagemaker.core.helper.session_helper import Session
+from sagemaker.core.apiutils import _utils
+from sagemaker.core.experiments._environment import _RunEnvironment, _EnvironmentType
+from sagemaker.core.experiments.trial_component import _TrialComponent
+from sagemaker.core.utilities.search_expression import Filter, Operator, SearchExpression
+from sagemaker.core.common_utils import retry_with_backoff
+
+
+def resolve_artifact_name(file_path):
+    """Resolve artifact name from given file path.
+
+    If not specified, will auto create one.
+
+    Args:
+        file_path (str): Path to the file.
+
+    Returns:
+        str: The resolved artifact name.
+    """
+    _, filename = os.path.split(file_path)
+    if filename:
+        return filename
+
+    return _utils.name("artifact")
+
+
+def guess_media_type(file_path):
+    """Infer the media type of a file based on its file name.
+
+    Args:
+        file_path (str): Path to the file.
+
+    Returns:
+        str: The guessed media type.
+    """
+    file_url = urllib.parse.urljoin("file:", urllib.request.pathname2url(file_path))
+    guessed_media_type, _ = mimetypes.guess_type(file_url, strict=False)
+    return guessed_media_type
+
+
+def verify_length_of_true_and_predicted(true_labels, predicted_attrs, predicted_attrs_name):
+    """Verify if lengths match between lists of true labels and predicted attributes.
+
+    Args:
+        true_labels (list or array): The list of the true labels.
+        predicted_attrs (list or array): The list of the predicted labels/probabilities/scores.
+        predicted_attrs_name (str): The name of the predicted attributes.
+
+    Raises:
+        ValueError: If lengths mismatch between true labels and predicted attributes.
+    """
+    if len(true_labels) != len(predicted_attrs):
+        raise ValueError(
+            "Lengths mismatch between true labels and {}: "
+            "({} vs {}).".format(predicted_attrs_name, len(true_labels), len(predicted_attrs))
+        )
+
+
+def validate_invoked_inside_run_context(func):
+    """A Decorator to force the decorated method called under Run context."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        self_instance = args[0]
+        if not self_instance._inside_load_context and not self_instance._inside_init_context:
+            raise RuntimeError("This method should be called inside context of 'with' statement.")
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def is_already_exist_error(error):
+    """Check if the error indicates resource already exists
+
+    Args:
+        error (dict): The "Error" field in the response of the
+            `botocore.exceptions.ClientError`
+    """
+    return error["Code"] == "ValidationException" and "already exists" in error["Message"]
+
+
+def get_tc_and_exp_config_from_job_env(
+    environment: _RunEnvironment,
+    sagemaker_session: Session,
+) -> dict:
+    """Retrieve an experiment config from the job environment.
+
+    Args:
+        environment (_RunEnvironment): The run environment object with job specific data.
+        sagemaker_session (sagemaker.core.helper.session_helper.Session): Session object which
+            manages interactions with Amazon SageMaker APIs and any other
+            AWS services needed. If not specified, one is created using the
+            default AWS configuration chain.
+    """
+    job_name = environment.source_arn.split("/")[-1]
+    if environment.environment_type == _EnvironmentType.SageMakerTrainingJob:
+        job_response = retry_with_backoff(
+            callable_func=lambda: sagemaker_session.describe_training_job(job_name),
+            num_attempts=4,
+        )
+    elif environment.environment_type == _EnvironmentType.SageMakerProcessingJob:
+        job_response = retry_with_backoff(
+            callable_func=lambda: sagemaker_session.describe_processing_job(job_name),
+            num_attempts=4,
+        )
+    else:  # environment.environment_type == _EnvironmentType.SageMakerTransformJob
+        job_response = retry_with_backoff(
+            callable_func=lambda: sagemaker_session.describe_transform_job(job_name),
+            num_attempts=4,
+        )
+
+    job_exp_config = job_response.get("ExperimentConfig", dict())
+    from sagemaker.core.experiments.run import RUN_NAME
+
+    if job_exp_config.get(RUN_NAME, None):
+        return job_exp_config
+    raise RuntimeError(
+        "Not able to fetch RunName in ExperimentConfig of the sagemaker job. "
+        "Please make sure the ExperimentConfig is correctly set."
+    )
+
+
+def verify_load_input_names(
+    run_name: Optional[str] = None,
+    experiment_name: Optional[str] = None,
+):
+    """Verify the run_name and the experiment_name inputs in load_run.
+
+    Args:
+        run_name (str): The run_name supplied by the user (default: None).
+        experiment_name (str): The experiment_name supplied by the user
+            (default: None).
+
+    Raises:
+        ValueError: If run_name is supplied while experiment_name is not.
+    """
+    if not run_name and experiment_name:
+        logging.warning(
+            "No run_name is supplied. Ignoring the provided experiment_name "
+            "since it only takes effect along with run_name. "
+            "Will load the Run object from the job environment or current Run context."
+        )
+    if run_name and not experiment_name:
+        raise ValueError(
+            "Invalid input: experiment_name is missing when run_name is supplied. "
+            "Please supply a valid experiment_name when the run_name is not None."
+        )
+
+
+def is_run_trial_component(trial_component_name: str, sagemaker_session: Session) -> bool:
+    """Check if a trial component is generated by `sagemaker.experiments.Run`
+
+    Args:
+        trial_component_name (str): The name of the trial component.
+        sagemaker_session (sagemaker.core.helper.session_helper.Session): Session object which
+            manages interactions with Amazon SageMaker APIs and any other
+            AWS services needed. If not specified, one is created using the
+            default AWS configuration chain.
+
+    Returns:
+        bool: Indicate whether the trial component is created by
+            `sagemaker.experiments.Run` or not.
+    """
+    search_filter = Filter(
+        name="TrialComponentName",
+        operator=Operator.EQUALS,
+        value=trial_component_name,
+    )
+    search_expression = SearchExpression(filters=[search_filter])
+
+    def search():
+        return list(
+            _TrialComponent.search(
+                search_expression=search_expression,
+                max_results=1,  # TrialComponentName is unique in an account
+                sagemaker_session=sagemaker_session,
+            )
+        )[0]
+
+    try:
+        tc_search_res = retry_with_backoff(search, 4)
+        from sagemaker.core.experiments.run import RUN_TC_TAG
+
+        if not tc_search_res.tags or RUN_TC_TAG not in tc_search_res.tags:
+            return False
+        return True
+    except Exception as ex:  # pylint: disable=broad-except
+        logging.warning(
+            "Failed to inspect the type of the trial component (%s), due to (%s)",
+            trial_component_name,
+            str(ex),
+        )
+        return False

sagemaker/core/experiments/experiment.py

@@ -0,0 +1,247 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Contains the SageMaker Experiment class."""
+from __future__ import absolute_import
+
+import time
+
+from botocore.exceptions import ClientError
+
+from sagemaker.core.apiutils import _base_types
+from sagemaker.core.experiments.trial import _Trial
+from sagemaker.core.experiments.trial_component import _TrialComponent
+from sagemaker.core.common_utils import format_tags
+from sagemaker.core.telemetry.telemetry_logging import _telemetry_emitter
+from sagemaker.core.telemetry.constants import Feature
+
+
+class Experiment(_base_types.Record):
+    """An Amazon SageMaker experiment, which is a collection of related trials.
+
+    New experiments are created by calling `experiments.experiment.Experiment.create`.
+    Existing experiments can be reloaded by calling `experiments.experiment.Experiment.load`.
+
+    Attributes:
+        experiment_name (str): The name of the experiment. The name must be unique
+            within an account.
+        display_name (str): Name of the experiment that will appear in UI,
+            such as SageMaker Studio.
+        description (str): A description of the experiment.
+        tags (List[Dict[str, str]]): A list of tags to associate with the experiment.
+    """
+
+    experiment_name = None
+    display_name = None
+    description = None
+    tags = None
+
+    _boto_create_method = "create_experiment"
+    _boto_load_method = "describe_experiment"
+    _boto_update_method = "update_experiment"
+    _boto_delete_method = "delete_experiment"
+
+    _boto_update_members = ["experiment_name", "description", "display_name"]
+    _boto_delete_members = ["experiment_name"]
+
+    _MAX_DELETE_ALL_ATTEMPTS = 3
+
+    def save(self):
+        """Save the state of this Experiment to SageMaker.
+
+        Returns:
+            dict: Update experiment API response.
+        """
+        return self._invoke_api(self._boto_update_method, self._boto_update_members)
+
+    def delete(self):
+        """Delete this Experiment from SageMaker.
+
+        Deleting an Experiment does not delete associated Trials and their Trial Components.
+        It requires that each Trial in the Experiment is first deleted.
+
+        Returns:
+            dict: Delete experiment API response.
+        """
+        return self._invoke_api(self._boto_delete_method, self._boto_delete_members)
+
+    @classmethod
+    def load(cls, experiment_name, sagemaker_session=None):
+        """Load an existing experiment and return an `Experiment` object representing it.
+
+        Args:
+            experiment_name: (str): Name of the experiment
+            sagemaker_session (sagemaker.core.helper.session_helper.Session): Session object which
+                manages interactions with Amazon SageMaker APIs and any other
+                AWS services needed. If not specified, one is created using the
+                default AWS configuration chain.
+
+        Returns:
+            experiments.experiment.Experiment: A SageMaker `Experiment` object
+        """
+        return cls._construct(
+            cls._boto_load_method,
+            experiment_name=experiment_name,
+            sagemaker_session=sagemaker_session,
+        )
+
+    @classmethod
+    @_telemetry_emitter(feature=Feature.MLOPS, func_name="experiment.create")
+    def create(
+        cls,
+        experiment_name,
+        display_name=None,
+        description=None,
+        tags=None,
+        sagemaker_session=None,
+    ):
+        """Create a new experiment in SageMaker and return an `Experiment` object.
+
+        Args:
+            experiment_name: (str): Name of the experiment. Must be unique. Required.
+            display_name: (str): Name of the experiment that will appear in UI,
+                such as SageMaker Studio (default: None).
+            description: (str): Description of the experiment (default: None).
+            sagemaker_session (sagemaker.core.helper.session_helper.Session): Session object which
+                manages interactions with Amazon SageMaker APIs and any other
+                AWS services needed. If not specified, one is created using the
+                default AWS configuration chain.
+            tags (Optional[Tags]): A list of tags to associate with the experiment
+                (default: None).
+
+        Returns:
+            experiments.experiment.Experiment: A SageMaker `Experiment` object
+        """
+        return cls._construct(
+            cls._boto_create_method,
+            experiment_name=experiment_name,
+            display_name=display_name,
+            description=description,
+            tags=format_tags(tags),
+            sagemaker_session=sagemaker_session,
+        )
+
+    @classmethod
+    def _load_or_create(
+        cls,
+        experiment_name,
+        display_name=None,
+        description=None,
+        tags=None,
+        sagemaker_session=None,
+    ):
+        """Load an experiment by name and create a new one if it does not exist.
+
+        Args:
+            experiment_name: (str): Name of the experiment. Must be unique. Required.
+            display_name: (str): Name of the experiment that will appear in UI,
+                such as SageMaker Studio (default: None). This is used only when the
+                given `experiment_name` does not exist and a new experiment has to be created.
+            description: (str): Description of the experiment (default: None).
+                This is used only when the given `experiment_name` does not exist and
+                a new experiment has to be created.
+            sagemaker_session (sagemaker.core.helper.session_helper.Session): Session object which
+                manages interactions with Amazon SageMaker APIs and any other
+                AWS services needed. If not specified, one is created using the
+                default AWS configuration chain.
+            tags (Optional[Tags]): A list of tags to associate with the experiment
+                (default: None). This is used only when the given `experiment_name` does not
+                exist and a new experiment has to be created.
+
+        Returns:
+            experiments.experiment.Experiment: A SageMaker `Experiment` object
+        """
+        try:
+            experiment = Experiment.create(
+                experiment_name=experiment_name,
+                display_name=display_name,
+                description=description,
+                tags=format_tags(tags),
+                sagemaker_session=sagemaker_session,
+            )
+        except ClientError as ce:
+            error_code = ce.response["Error"]["Code"]
+            error_message = ce.response["Error"]["Message"]
+            if not (error_code == "ValidationException" and "already exists" in error_message):
+                raise ce
+            # already exists
+            experiment = Experiment.load(experiment_name, sagemaker_session)
+        return experiment
+
+    def list_trials(self, created_before=None, created_after=None, sort_by=None, sort_order=None):
+        """List trials in this experiment matching the specified criteria.
+
+        Args:
+            created_before (datetime.datetime): Return trials created before this instant
+                (default: None).
+            created_after (datetime.datetime): Return trials created after this instant
+                (default: None).
+            sort_by (str): Which property to sort results by. One of 'Name', 'CreationTime'
+                (default: None).
+            sort_order (str): One of 'Ascending', or 'Descending' (default: None).
+
+        Returns:
+            collections.Iterator[experiments._api_types.TrialSummary] :
+                An iterator over trials matching the criteria.
+        """
+        return _Trial.list(
+            experiment_name=self.experiment_name,
+            created_before=created_before,
+            created_after=created_after,
+            sort_by=sort_by,
+            sort_order=sort_order,
+            sagemaker_session=self.sagemaker_session,
+        )
+
+    def _delete_all(self, action):
+        """Force to delete the experiment and associated trials, trial components.
+
+        Args:
+            action (str): The string '--force' is required to pass in to confirm recursively
+                delete the experiments, and all its trials and trial components.
+        """
+        if action != "--force":
+            raise ValueError(
+                "Must confirm with string '--force' in order to delete the experiment and "
+                "associated trials, trial components."
+            )
+
+        delete_attempt_count = 0
+        last_exception = None
+        while True:
+            if delete_attempt_count == self._MAX_DELETE_ALL_ATTEMPTS:
+                raise Exception("Failed to delete, please try again.") from last_exception
+            try:
+                for trial_summary in self.list_trials():
+                    trial = _Trial.load(
+                        sagemaker_session=self.sagemaker_session,
+                        trial_name=trial_summary.trial_name,
+                    )
+                    for (
+                        trial_component_summary
+                    ) in trial.list_trial_components():  # pylint: disable=no-member
+                        tc = _TrialComponent.load(
+                            sagemaker_session=self.sagemaker_session,
+                            trial_component_name=trial_component_summary.trial_component_name,
+                        )
+                        tc.delete(force_disassociate=True)
+                        # to prevent throttling
+                        time.sleep(1.2)
+                    trial.delete()  # pylint: disable=no-member
+                    # to prevent throttling
+                    time.sleep(1.2)
+                self.delete()
+                break
+            except Exception as ex:  # pylint: disable=broad-except
+                last_exception = ex
+            finally:
+                delete_attempt_count = delete_attempt_count + 1