sagemaker-core 1.0.47__py3-none-any.whl → 2.1.1__py3-none-any.whl

sagemaker/core/config/config_utils.py

@@ -0,0 +1,297 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This file contains util functions for the sagemaker Defaults Config.
+
+These utils may be used inside or outside the config module.
+"""
+from __future__ import absolute_import
+from collections import deque
+
+import logging
+import sys
+from typing import Callable, List, TYPE_CHECKING
+import re
+from copy import deepcopy
+
+if TYPE_CHECKING:
+    from sagemaker.core.common_utils import TagsDict
+
+from sagemaker.core.config.config_schema import KEY
+
+
+def get_sagemaker_config_logger():
+    """Return a logger with the name 'sagemaker.config'
+
+    If the logger to be returned has no level or handlers set, this will get level and handler
+    attributes. (So if the SDK user has setup loggers in a certain way, that setup will not be
+    changed by this function.) It is safe to make repeat calls to this function.
+    """
+    sagemaker_config_logger = logging.getLogger("sagemaker.config")
+    sagemaker_logger = logging.getLogger("sagemaker")
+
+    if sagemaker_config_logger.level == logging.NOTSET:
+        sagemaker_config_logger.setLevel(logging.INFO)
+
+    # check sagemaker_logger here as well, so that if handlers were set for the parent logger
+    # already, we dont change behavior for the child logger
+    if len(sagemaker_config_logger.handlers) == 0 and len(sagemaker_logger.handlers) == 0:
+        # use sys.stdout so logs dont show up with a red background in a notebook
+        handler = logging.StreamHandler(sys.stdout)
+
+        formatter = logging.Formatter("%(name)s %(levelname)-4s - %(message)s")
+        handler.setFormatter(formatter)
+        sagemaker_config_logger.addHandler(handler)
+
+        # if a handler is being added, we dont want the root handler to also process the same events
+        sagemaker_config_logger.propagate = False
+
+    return sagemaker_config_logger
+
+
+def _log_sagemaker_config_single_substitution(source_value, config_value, config_key_path: str):
+    """Informs the SDK user whether a config value was present and automatically substituted
+
+    Args:
+        source_value: The value that will be used if it exists. Usually, this is user-provided
+            input to a Class or to a session.py method, or None if no input was provided.
+        config_value: The value fetched from sagemaker_config. If it exists, this is the value that
+            will be used if direct_input is None.
+        config_key_path: A string denoting the path of keys that point to the config value in the
+            sagemaker_config.
+
+    Returns:
+        None. Logs information to the "sagemaker.config" logger.
+    """
+    logger = get_sagemaker_config_logger()
+
+    source_value_log_copy = deepcopy(source_value)
+    config_value_log_copy = deepcopy(config_value)
+
+    if isinstance(source_value_log_copy, dict):
+        for key in source_value_log_copy.keys():
+            if re.search(r"(secret|password|key|token)", key, re.IGNORECASE):
+                source_value_log_copy[key] = "***"
+
+    if isinstance(config_value_log_copy, dict):
+        for key in config_value_log_copy.keys():
+            if re.search(r"(secret|password|key|token)", key, re.IGNORECASE):
+                config_value_log_copy[key] = "***"
+
+    if config_value is not None:
+
+        if source_value is None:
+            # Sagemaker Config value is going to be used. By default the user should know about
+            # this scenario because the behavior they expect could change because of a new config
+            # value being injected in.
+            # However, it may not be safe to log ARNs to stdout by default. We can include more
+            # diagnostic info if the user enabled DEBUG logs though.
+            if logger.isEnabledFor(logging.DEBUG):
+                logger.debug(
+                    "Applied value\n  config key = %s\n  config value that will be used = %s",
+                    config_key_path,
+                    config_value_log_copy,
+                )
+            else:
+                logger.info(
+                    "Applied value from config key = %s",
+                    config_key_path,
+                )
+
+        # The cases below here are logged as just debug statements because this info can be useful
+        # when debugging the config, but should not affect current behavior with/without the config.
+
+        elif source_value is not None and config_value == source_value:
+            # Sagemaker Config had a value defined that is NOT going to be used here.
+            # Either (1) the config value was already fetched and applied earlier, or
+            # (2) the user happened to pass in the same value.
+            logger.debug(
+                (
+                    "Skipped value\n"
+                    "  config key = %s\n"
+                    "  config value = %s\n"
+                    "  source value that will be used = %s"
+                ),
+                config_key_path,
+                config_value_log_copy,
+                source_value_log_copy,
+            )
+        elif source_value is not None and config_value != source_value:
+            # Sagemaker Config had a value defined that is NOT going to be used
+            # and the config value has not already been applied earlier (we know because the values
+            # are different).
+            logger.debug(
+                (
+                    "Skipped value\n"
+                    "  config key = %s\n"
+                    "  config value = %s\n"
+                    "  source value that will be used = %s",
+                ),
+                config_key_path,
+                config_value_log_copy,
+                source_value_log_copy,
+            )
+    else:
+        # nothing was specified in the config and nothing is being automatically applied
+        logger.debug("Skipped value because no value defined\n  config key = %s", config_key_path)
+
+
+def _log_sagemaker_config_merge(
+    source_value=None,
+    config_value=None,
+    merged_source_and_config_value=None,
+    config_key_path: str = None,
+):
+    """Informs the SDK user whether a config value was present and automatically substituted
+
+    Args:
+        source_value: The dict or object that would be used if no default values existed. Usually,
+            this is user-provided input to a Class or to a session.py method, or None if no input
+            was provided.
+        config_value: The dict or object fetched from sagemaker_config. If it exists, this is the
+            value that will be used if source_value is None.
+        merged_source_and_config_value: The value that results from the merging of source_value and
+            original_config_value. This will be the value used.
+        config_key_path: A string denoting the path of keys that point to the config value in the
+            sagemaker_config.
+
+    Returns:
+        None. Logs information to the "sagemaker.config" logger.
+    """
+    logger = get_sagemaker_config_logger()
+
+    if config_value:
+
+        if source_value != merged_source_and_config_value:
+            # Sagemaker Config value(s) were used and affected the final object/dictionary. By
+            # default the user should know about this scenario because the behavior they expect
+            # could change because of new config values being injected in.
+            # However, it may not be safe to log ARNs to stdout by default. We can include more
+            # diagnostic info if the user enabled DEBUG logs though.
+            if logger.isEnabledFor(logging.DEBUG):
+                logger.debug(
+                    (
+                        "Applied value(s)\n"
+                        "  config key = %s\n"
+                        "  config value = %s\n"
+                        "  source value = %s\n"
+                        "  combined value that will be used = %s"
+                    ),
+                    config_key_path,
+                    config_value,
+                    source_value,
+                    merged_source_and_config_value,
+                )
+            else:
+                logger.info(
+                    "Applied value(s) from config key = %s",
+                    config_key_path,
+                )
+
+        # The cases below here are logged as just debug statements because this info can be useful
+        # when debugging the config, but should not affect current behavior with/without the config.
+
+        else:
+            # Sagemaker Config had a value defined that is NOT going to be used here.
+            # Either (1) the config value was already fetched and applied earlier, or
+            # (2) the user happened to pass in the same values.
+            logger.debug(
+                (
+                    "Skipped value(s)\n"
+                    "  config key = %s\n"
+                    "  config value = %s\n"
+                    "  source value that will be used = %s"
+                ),
+                config_key_path,
+                config_value,
+                merged_source_and_config_value,
+            )
+
+    else:
+        # nothing was specified in the config and nothing is being automatically applied
+        logger.debug("Skipped value because no value defined\n  config key = %s", config_key_path)
+
+
+def non_repeating_log_factory(logger: logging.Logger, method: str, cache_size=100) -> Callable:
+    """Create log function that filters the repeated messages.
+
+    By default. It only keeps track of last 100 messages, if a repeated
+    message arrives after the ``cache_size`` messages, it will be displayed.
+
+    Args:
+        logger (logging.Logger): the logger to be used to dispatch the message.
+        method (str): the log method, can be info, warning or debug.
+        cache_size (int): the number of last log messages to keep in cache.
+            Default to 100
+
+    Returns:
+        (Callable): the new log method
+    """
+    if method not in ["info", "warning", "debug"]:
+        raise ValueError("Not supported logging method.")
+
+    _caches = deque(maxlen=cache_size)
+    log_method = getattr(logger, method)
+
+    def new_log_method(msg, *args, **kwargs):
+        key = f"{msg}:{args}"
+        if key not in _caches:
+            log_method(msg, *args, **kwargs)
+            _caches.append(key)
+
+    return new_log_method
+
+
+def _append_sagemaker_config_tags(
+    sagemaker_session, tags: List["TagsDict"], config_path_to_tags: str
+):
+    """Appends tags specified in the sagemaker_config to the given list of tags.
+
+    To minimize the chance of duplicate tags being applied, this is intended to be used
+    immediately before calls to sagemaker_client, rather than during initialization of
+    classes like EstimatorBase.
+
+    Args:
+        tags: The list of tags to append to.
+        config_path_to_tags: The path to look up tags in the config.
+
+    Returns:
+        A list of tags.
+    """
+    from sagemaker.core.config.config_manager import SageMakerConfig
+
+    config_tags = SageMakerConfig().get_sagemaker_config_value(
+        sagemaker_session, config_path_to_tags
+    )
+
+    if config_tags is None or len(config_tags) == 0:
+        return tags
+
+    all_tags = tags or []
+    for config_tag in config_tags:
+        config_tag_key = config_tag[KEY]
+        if not any(tag.get("Key", None) == config_tag_key for tag in all_tags):
+            # This check prevents new tags with duplicate keys from being added
+            # (to prevent API failure and/or overwriting of tags). If there is a conflict,
+            # the user-provided tag should take precedence over the config-provided tag.
+            # Note: this does not check user-provided tags for conflicts with other
+            # user-provided tags.
+            all_tags.append(config_tag)
+
+    _log_sagemaker_config_merge(
+        source_value=tags,
+        config_value=config_tags,
+        merged_source_and_config_value=all_tags,
+        config_key_path=config_path_to_tags,
+    )
+
+    return all_tags