cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/core/tests/test_versions.py

@@ -0,0 +1,76 @@
+import pytest
+
+from cumulusci.core.versions import PackageType, PackageVersionNumber, VersionTypeEnum
+
+
+class TestPackageVersionNumber:
+    def test_parse_format(self):
+        assert PackageVersionNumber.parse("1.2.3.4").format() == "1.2.3.4"
+        assert (
+            str(
+                PackageVersionNumber.parse(
+                    "1.2.3.4", package_type=PackageType.FIRST_GEN
+                )
+            )
+            == "1.2.3 (Beta 4)"
+        )
+        assert PackageVersionNumber.parse("1.2.3 (Beta 4)").format() == "1.2.3 (Beta 4)"
+        assert PackageVersionNumber.parse("1.2.3 (beta 4)").format() == "1.2.3 (Beta 4)"
+        assert PackageVersionNumber.parse("1.2.3").format() == "1.2.3"
+
+        assert (
+            PackageVersionNumber.parse_tag(
+                "release/1.2.3", "release/", "beta/"
+            ).format()
+            == "1.2.3"
+        )
+        assert (
+            PackageVersionNumber.parse_tag(
+                "beta/1.2.3-Beta_4", "release/", "beta/"
+            ).format()
+            == "1.2.3 (Beta 4)"
+        )
+        assert (
+            PackageVersionNumber.parse_tag("beta/1.2.3.4", "release/", "beta/").format()
+            == "1.2.3.4"
+        )
+        assert (
+            PackageVersionNumber.parse_tag(
+                "beta/1.2.3.4", "release/", "beta/"
+            ).format_tag("beta/")
+            == "beta/1.2.3.4"
+        )
+        assert (
+            PackageVersionNumber.parse_tag(
+                "beta/1.2.3-Beta_4", "release/", "beta/"
+            ).format_tag("beta/")
+            == "beta/1.2.3-Beta_4"
+        )
+
+    def test_parse__invalid(self):
+        with pytest.raises(ValueError):
+            PackageVersionNumber.parse("asdf")
+
+    def test_increment(self):
+        assert (
+            PackageVersionNumber.parse("1.0.0.1")
+            .increment(VersionTypeEnum.major)
+            .format()
+            == "2.0.0.NEXT"
+        )
+        assert (
+            PackageVersionNumber.parse("1.0.0.1")
+            .increment(VersionTypeEnum.minor)
+            .format()
+            == "1.1.0.NEXT"
+        )
+        assert (
+            PackageVersionNumber.parse("1.0.0.1")
+            .increment(VersionTypeEnum.patch)
+            .format()
+            == "1.0.1.NEXT"
+        )
+
+    def test_increment__1gp(self):
+        with pytest.raises(ValueError):
+            PackageVersionNumber.parse("1.0 (Beta 4)").increment(VersionTypeEnum.major)

cumulusci/core/tests/untrusted_repo_child/cumulusci.yml

@@ -0,0 +1,7 @@
+project:
+    name: Untrusted-Repo-Child
+
+tasks:
+    example_task:
+        description: A task that should be inaccessible to parent tasks
+        class_path: tasks.untrusted_child.ExampleTask

cumulusci/core/tests/untrusted_repo_child/tasks/untrusted_child.py

@@ -0,0 +1,6 @@
+from cumulusci.core.tasks import BaseTask
+
+
+class UntrustedChildExampleTask(BaseTask):
+    def _run_task(self):
+        raise AssertionError("This should not be callable (child)")

cumulusci/core/tests/untrusted_repo_parent/cumulusci.yml

@@ -0,0 +1,26 @@
+project:
+    name: Untrusted-Repo-Parent
+
+sources:
+    # trusted by the parent but not the root cumulusci
+    untrusted_child:
+        path: cumulusci/core/tests/untrusted_repo_child
+        allow_remote_code: True
+
+tasks:
+    example_task:
+        description: An example project-level task that does nothing
+        class_path: tasks.untrusted_parent.ExampleTask
+
+    untrusted_child_task:
+        description: An example project-level task that does nothing
+        class_path: tasks.untrusted_child.ExampleTask
+
+flows:
+    untrusted:
+        steps:
+            1:
+                task: example_task
+
+            2:
+                task: untrusted_child:example_task

cumulusci/core/tests/untrusted_repo_parent/tasks/untrusted_parent.py

@@ -0,0 +1,6 @@
+from cumulusci.core.tasks import BaseTask
+
+
+class UntrustedParentExampleTask(BaseTask):
+    def _run_task(self):
+        raise AssertionError("This should not be callable! (parent)")

cumulusci/core/tests/utils.py

@@ -0,0 +1,116 @@
+""" Utilities for testing CumulusCI
+
+MockLoggingHandler: a logging handler that we can assert"""
+
+import collections
+import logging
+import os
+
+import cumulusci
+
+
+class MockLoggingHandler(logging.Handler):
+    """Mock logging handler to check for expected logs.
+
+    Messages are available from an instance's ``messages`` dict, in order,
+    indexed by a lowercase log level string (e.g., 'debug', 'info', etc.).
+
+    TODO: Should be replaced by caplog at some point.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.messages = {
+            "debug": [],
+            "info": [],
+            "warning": [],
+            "error": [],
+            "critical": [],
+        }
+        super().__init__(*args, **kwargs)
+
+    def emit(self, record):
+        "Store a message from ``record`` in the instance's ``messages`` dict."
+        self.acquire()
+        try:
+            self.messages[record.levelname.lower()].append(record.getMessage())
+        finally:
+            self.release()
+
+    def reset(self):
+        """Reset the handler in TestCase.setup_method() to clear the msg list"""
+        self.acquire()
+        try:
+            for message_list in list(self.messages.values()):
+                del message_list[:]
+        finally:
+            self.release()
+
+
+class EnvironmentVarGuard(collections.abc.MutableMapping):
+
+    """Class to help protect the environment variable properly.  Can be used as
+    a context manager."""
+
+    def __init__(self):
+        self._environ = os.environ
+        self._changed = {}
+
+    def __getitem__(self, envvar):
+        return self._environ[envvar]
+
+    def __setitem__(self, envvar, value):
+        # Remember the initial value on the first access
+        if envvar not in self._changed:
+            self._changed[envvar] = self._environ.get(envvar)
+        self._environ[envvar] = value
+
+    def __delitem__(self, envvar):
+        # Remember the initial value on the first access
+        if envvar not in self._changed:
+            self._changed[envvar] = self._environ.get(envvar)
+        if envvar in self._environ:
+            del self._environ[envvar]
+
+    def keys(self):
+        return self._environ.keys()
+
+    def __iter__(self):
+        return iter(self._environ)
+
+    def __len__(self):
+        return len(self._environ)
+
+    def set(self, envvar, value):
+        self[envvar] = value
+
+    def unset(self, envvar):
+        del self[envvar]
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *ignore_exc):
+        for (k, v) in self._changed.items():
+            if v is None:
+                if k in self._environ:
+                    del self._environ[k]
+            else:
+                self._environ[k] = v
+        os.environ = self._environ
+
+
+class MockLoggerMixin(object):
+    @classmethod
+    def setup_class(cls):
+        logger = logging.getLogger(cumulusci.core.tasks.__name__)
+        logger.setLevel(logging.DEBUG)
+        cls._task_log_handler = MockLoggingHandler(logging.DEBUG)
+        logger.addHandler(cls._task_log_handler)
+
+
+class MockLookup:
+    def __init__(self, **kwargs):
+        self.dict = kwargs
+
+    def __call__(self, name, default=None):
+        return self.dict[name]

cumulusci/core/utils.py

@@ -0,0 +1,402 @@
+""" Utilities for CumulusCI Core"""
+
+import copy
+import glob
+import json
+import time
+import typing as T
+import warnings
+from datetime import datetime, timedelta
+from logging import getLogger
+from pathlib import Path
+
+import pytz
+
+from cumulusci.core.debug import get_debug_mode
+from cumulusci.core.exceptions import (
+    ConfigMergeError,
+    CumulusCIException,
+    TaskOptionsError,
+)
+from cumulusci.utils.options import parse_list_of_pairs_dict_arg
+
+
+def import_global(path: str):
+    """Import a class from a string module class path"""
+    components = path.split(".")
+    module = components[:-1]
+    module = ".".join(module)
+
+    if get_debug_mode():
+        import sys
+
+        logger = getLogger(__file__)
+        logger.info(f"Importing {path}")
+        logger.info(f"Looking in sys.path {sys.path}")
+        if components[0] == "tasks":
+            logger.info(f"With tasks.__path__ {sys.modules['tasks'].__path__}")
+
+    mod = __import__(module, fromlist=[str(components[-1])])
+    return getattr(mod, str(components[-1]))
+
+
+# For backwards-compatibility
+import_class = import_global
+
+
+def parse_datetime(dt_str, format):
+    """Create a timezone-aware datetime object from a datetime string."""
+    t = time.strptime(dt_str, format)
+    return datetime(t[0], t[1], t[2], t[3], t[4], t[5], t[6], pytz.UTC)
+
+
+def process_bool_arg(arg: T.Union[int, str, None]):
+    """Determine True/False from argument.
+
+    Similar to parts of the Salesforce API, there are a few true-ish and false-ish strings,
+        but "True" and "False" are the canonical ones.
+
+    None is accepted as "False" for backwards compatiblity reasons, but this usage is deprecated.
+    """
+    if isinstance(arg, (int, bool)):
+        return bool(arg)
+    elif arg is None:
+        # backwards compatible behaviour that some tasks
+        # rely upon.
+        import traceback
+
+        warnings.warn("".join(traceback.format_stack(limit=4)), DeprecationWarning)
+        warnings.warn(
+            "Future versions of CCI will not accept 'None' as an argument to process_bool_arg",
+            DeprecationWarning,
+        )
+
+        return False
+    elif isinstance(arg, str):
+        # these are values that Salesforce's bulk loader accepts
+        # there doesn't seem to be any harm in acccepting the
+        # full list to be coordinated with a "Salesforce standard"
+        if arg.lower() in ["yes", "y", "true", "on", "1"]:
+            return True
+        elif arg.lower() in ["no", "n", "false", "off", "0"]:
+            return False
+    raise TypeError(f"Cannot interpret as boolean: `{arg}`")
+
+
+def process_glob_list_arg(arg):
+    """Convert a list of glob patterns or filenames into a list of files
+    The initial list can take the form of a comma-separated string or
+    a proper list. Order is preserved, but duplicates will be removed.
+
+    Note: this function processes glob patterns, but doesn't validate
+    that the files actually exist. For example, if the pattern is
+    'foo.bar' and there is no file named 'foo.bar', the literal string
+    'foo.bar' will be included in the returned files.
+
+    Similarly, if the pattern is '*.baz' and it doesn't match any files,
+    the literal string '*.baz' will be returned.
+    """
+    initial_list = process_list_arg(arg)
+
+    if not arg:
+        return []
+
+    files = []
+    for path in initial_list:
+        more_files = glob.glob(path, recursive=True)
+        if len(more_files):
+            files += sorted(more_files)
+        else:
+            files.append(path)
+    # In python 3.6+ dict is ordered, so we'll use it to weed
+    # out duplicates. We can't use a set because sets aren't ordered.
+    return list(dict.fromkeys(files))
+
+
+def process_list_arg(arg):
+    """Parse a string into a list separated by commas with whitespace stripped"""
+    if isinstance(arg, Path):
+        arg = str(arg)
+
+    if isinstance(arg, list):
+        return arg
+    elif isinstance(arg, str):
+        args = []
+        for part in arg.split(","):
+            args.append(part.strip())
+        return args
+    elif arg is None:
+        # backwards compatible behaviour.
+        return None
+    else:
+        getLogger(__file__).warn(
+            f"Unknown option type `{type(arg)}` for value `{arg}`."
+            "This will be an error in a future version of CCI."
+        )
+
+
+def process_list_of_pairs_dict_arg(arg):
+    """Process an arg in the format "aa:bb,cc:dd" """
+    try:
+        return parse_list_of_pairs_dict_arg(arg)
+    except TypeError as e:
+        raise TaskOptionsError(e) from e
+
+
+def decode_to_unicode(content):
+    """decode ISO-8859-1 to unicode, when using sf api"""
+    if content and not isinstance(content, str):
+        try:
+            # Try to decode ISO-8859-1 to unicode
+            return content.decode("ISO-8859-1")
+        except UnicodeEncodeError:
+            # Assume content is unicode already
+            return content
+    return content
+
+
+def merge_config(configs):
+    """
+    First remove any flow steps that are being overridden so that there are no conflicts
+    or different step types after merging. Then recursively deep-merge the configs into
+    one another (highest priority comes first)
+    """
+    config_copies = {name: copy.deepcopy(config) for name, config in configs.items()}
+    cleaned_configs = cleanup_flow_step_override_conflicts(config_copies)
+
+    new_config = {}
+    for name, config in cleaned_configs.items():
+        new_config = dictmerge(new_config, config, name)
+
+    return new_config
+
+
+def cleanup_flow_step_override_conflicts(configs: T.List[dict]) -> T.List[dict]:
+    """
+    If a flow step is been overridden with a step of a different type (i.e. tas, flow),
+    then we need to set the step that is _lower_ in precedence order to an empty dict ({}).
+    If we don't, then we will end up with both "flow" and "task" listed in the step_config which
+    leads to an error when CumulusCI attempts to run that step in the flow.
+
+    We need to also account for scenarios where a single flow step
+    is being overriden more than once.
+
+    Example:
+    A flow step in the universal config is being overridden somewhere
+    in the project_config, which is being overridden by additional_yaml.
+
+    The while loop below is how we account for these scenarios.
+    """
+    config_precedence_order = [
+        "additional_yaml",
+        "project_local_config",
+        "project_config",
+        "plugins_config",
+        "global_config",
+        "universal_config",
+    ]
+    while len(config_precedence_order) > 1:
+        overriding_config = config_precedence_order[0]
+        config_precedence_order = config_precedence_order[1:]
+        for config_to_override in config_precedence_order:
+            if configs_present_and_not_empty(
+                [config_to_override, overriding_config], configs
+            ):
+                remove_overridden_flow_steps_in_config(
+                    configs[config_to_override], configs[overriding_config]
+                )
+
+    return configs
+
+
+def configs_present_and_not_empty(
+    configs_to_check: T.List[dict], configs: dict
+) -> bool:
+    return all(c in configs and c != {} for c in configs_to_check)
+
+
+def remove_overridden_flow_steps_in_config(
+    config_to_override: dict, overriding_config: dict
+):
+    """If any steps of flows from the config_to_override are being overridden in overriding_config,
+    then we need to set those steps in the config_to_override to an empty dict so that we don't have
+    both a "task" and a "flow" listed in a flow step after merging the configs with `dictmerge()`.
+    """
+
+    if "flows" not in config_to_override or "flows" not in overriding_config:
+        return
+
+    for flow, flow_config in overriding_config["flows"].items():
+        for (
+            step_num,
+            overriding_step_config,
+        ) in flow_config.get("steps", {}).items():
+            cleanup_old_flow_step_replace_syntax(overriding_step_config)
+            both_configs_have_flow_and_step = config_has_flow_and_step_num(
+                config_to_override, flow, step_num
+            )
+            if both_configs_have_flow_and_step:
+                step_config_to_override = config_to_override["flows"][flow]["steps"][
+                    step_num
+                ]
+                steps_same_type = steps_are_same_type(
+                    overriding_step_config, step_config_to_override
+                )
+
+                link_missing_task_or_flow(
+                    step_config_to_override, overriding_step_config
+                )
+
+                if not steps_same_type:
+                    config_to_override["flows"][flow]["steps"][step_num] = {}
+
+
+def link_missing_task_or_flow(
+    step_config_to_override: dict, overriding_step_config: dict
+):
+    """If the incoming override does not have task/flow defined then inherit from
+    the flow step that we're overridding."""
+    if "flow" not in overriding_step_config and "task" not in overriding_step_config:
+        if "task" in step_config_to_override:
+            overriding_step_config["task"] = step_config_to_override["task"]
+        elif "flow" in step_config_to_override:
+            overriding_step_config["flow"] = step_config_to_override["flow"]
+
+
+def cleanup_old_flow_step_replace_syntax(step_config: dict):
+    """When replacing flow steps with a step of a different type, the old syntax
+    had users declare the original step type as 'None' along with the new step type.
+    If both are present, we want to remove the one that has a value of 'None'."""
+    if all(s_type in step_config for s_type in ("task", "flow")):
+        if step_config["flow"] == "None" and step_config["task"] == "None":
+            raise CumulusCIException(
+                "Cannot have both step types declared with a value of 'None'."
+                "For information on replacing a flow step see: https://cumulusci.readthedocs.io/en/latest/config.html#replace-a-flow-step"
+            )
+        elif step_config["flow"] == "None":
+            del step_config["flow"]
+        else:
+            del step_config["task"]
+
+
+def config_has_flow_and_step_num(config: dict, flow_name: str, step_num: int) -> bool:
+    return (
+        flow_name in config["flows"] and step_num in config["flows"][flow_name]["steps"]
+    )
+
+
+def steps_are_same_type(step_one_config: dict, step_two_config: dict) -> bool:
+    """If both steps are of the same type returns True, else False."""
+    config_one_type = "task" if "task" in step_one_config else "flow"
+    config_two_type = "task" if "task" in step_two_config else "flow"
+
+    return config_one_type == config_two_type
+
+
+def dictmerge(a, b, name=None):
+    """Deeply merge two ``dict``s that consist of lists, dicts, and scalars.
+    This function (recursively) merges ``b`` INTO ``a``, does not copy any values, and returns ``a``.
+
+    based on https://stackoverflow.com/a/15836901/5042831
+    NOTE: tuples and arbitrary objects are NOT handled and will raise TypeError"""
+
+    key = None
+
+    if b is None:
+        return a
+
+    try:
+        if a is None or isinstance(a, (bytes, int, str, float)):
+            # first run, or if ``a``` is a scalar
+            a = b
+        elif isinstance(a, list):
+            # lists can be only appended
+            if isinstance(b, list):
+                # merge lists
+                a.extend(b)
+            else:
+                # append to list
+                a.append(b)
+        elif isinstance(a, dict):
+            # dicts must be merged
+            if isinstance(b, dict):
+                for key in b:
+                    if key in a:
+                        a[key] = dictmerge(a[key], b[key], name)
+                    else:
+                        a[key] = copy.deepcopy(b[key])
+            else:
+                raise TypeError(
+                    f'Cannot merge non-dict of type "{type(b)}" into dict "{a}"'
+                )
+        else:
+            raise TypeError(
+                f'dictmerge does not supporting merging "{type(b)}" into "{type(a)}"'
+            )
+    except TypeError as e:
+        raise ConfigMergeError(
+            f'TypeError "{e}" in key "{key}" when merging "{type(b)}" into "{type(a)}"',
+            config_name=name,
+        )
+    return a
+
+
+def format_duration(duration: timedelta):
+    hours, remainder = divmod(duration.total_seconds(), 3600)
+    minutes, seconds = divmod(remainder, 60)
+    hours = f"{int(hours)}h:" if hours > 0 else ""
+    minutes = f"{int(minutes)}m:" if (hours or minutes) else ""
+    seconds = f"{str(int(seconds))}s"
+    return hours + minutes + seconds
+
+
+def make_jsonable(x):
+    """Attempts to json serialize an object.
+    If it is not serializable,
+    returns a list if it's a set
+    or a string representation for anything else.
+    """
+    if isinstance(x, set):
+        return list(x)
+    try:
+        json.dumps(x)
+        return x
+    except (TypeError, OverflowError):
+        return str(x)
+
+
+def determine_managed_mode(options, project_config, org_config):
+    """Determine the managed mode based on options, project config, and org config.
+
+    Args:
+        options: Dict of task options that may contain 'managed' or 'unmanaged' flags
+        project_config: Project configuration object with package info
+        org_config: Org configuration object with installed packages and namespace info
+
+    Returns:
+        bool: True if in managed mode, False if in unmanaged mode
+
+    Note: The changes allows multiple package development under same namespace.
+    """
+    if "managed" in options:
+        return process_bool_arg(options["managed"])
+
+    # Get package and namespace information
+    package_name = getattr(project_config, "project__package__name", None)
+    namespace = getattr(
+        project_config, "project__package__namespace", None
+    ) or options.get("namespace", None)
+    installed_packages = getattr(org_config, "installed_packages", {})
+
+    if "unmanaged" in options:
+        # Explicit unmanaged flag always takes precedence
+        return not process_bool_arg(options.get("unmanaged", True))
+    elif package_name and any(package_name in key for key in installed_packages.keys()):
+        # If this specific package is installed (or there is any installed package with a Name that contains package_name), we're in managed context
+        return True
+    elif bool(namespace) and namespace == getattr(org_config, "namespace", None):
+        # We're in a namespaced org (packaging org) developing unmanaged code
+        return False
+    else:
+        # Fall back to checking namespace in installed packages
+        return bool(namespace) and namespace in installed_packages