cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/core/keychain/environment_project_keychain.py

@@ -0,0 +1,7 @@
+from cumulusci.core.keychain.encrypted_file_project_keychain import (
+    EncryptedFileProjectKeychain,
+)
+
+# All logic for loading orgs and services from the environment
+# now lives in EncryptedFileProjectKeychain.
+EnvironmentProjectKeychain = EncryptedFileProjectKeychain  # pragma: no cover

cumulusci/core/keychain/serialization.py

@@ -0,0 +1,152 @@
+import json
+import os
+import pickle
+from datetime import date, datetime
+from logging import Logger
+from typing import NamedTuple, Optional
+
+# Delay saving as JSON for a few CumulusCI releases because
+# people might downgrade a release and then their
+# CCI can't parse their JSON orgfiles
+#
+# Thus we roll out the ability to parse JSON configs a bit
+# ahead of the write behaviour.
+SHOULD_SAVE_AS_JSON = os.environ.get("SHOULD_SAVE_AS_JSON", "False") != "False"
+
+
+def load_config_from_json_or_pickle(b: bytes) -> dict:
+    """Input should be plaintext JSON or Pickle"""
+    assert isinstance(b, bytes)
+
+    try:
+        data = try_load_config_from_json_or_pickle(b)
+    except pickle.PickleError as e:
+        # we use ValueError because Pickle and Crypto both do too
+        raise ValueError(str(e)) from e
+
+    return data
+
+
+class JSONSerializer(NamedTuple):
+    type: str
+    to_json: callable
+    from_json: callable
+
+    @property
+    def name(self):
+        return self.type.__name__
+
+
+# make sure that datetime comes before date
+string_serializers = [
+    JSONSerializer(
+        datetime,
+        to_json=lambda x: x.isoformat(),
+        from_json=lambda x: datetime.fromisoformat(x),
+    ),
+    JSONSerializer(
+        date, to_json=lambda x: x.isoformat(), from_json=lambda x: date.fromisoformat(x)
+    ),
+    JSONSerializer(
+        bytes,
+        to_json=lambda x: x.decode("unicode_escape"),
+        from_json=lambda x: x.encode("unicode_escape"),
+    ),
+]
+
+
+def encode_value(x):
+    """Encode a value that JSON does not support natively"""
+    for serializer in string_serializers:
+        if isinstance(x, serializer.type):
+            return {"$type": serializer.name, "$value": serializer.to_json(x)}
+
+    raise TypeError(type(x))  # pragma: no cover
+
+
+def decode_dict(x: dict):
+    """Decode a dict from JSON"""
+    assert isinstance(x, dict)
+    if "$type" in x:
+        return decode_typed_value(x)
+    else:
+        return x
+
+
+def decode_typed_value(x: dict):
+    """Decode a value that JSON does not support natively"""
+    for serializer in string_serializers:
+        if x["$type"] == serializer.name:
+            return serializer.from_json(x["$value"])
+
+    raise TypeError(f"Unknown $type: {x['$type']}")  # pragma: no cover
+
+
+def try_load_config_from_json_or_pickle(data: bytes) -> dict:
+    """Load JSON or Pickle into a dict"""
+    try:
+        config = json.loads(data.decode("utf-8"), object_hook=decode_dict)
+        # remove this debugging tool after transition
+        config["serialization_format"] = "json"
+        return config
+    except ValueError as e1:
+        try:
+            # first byte in a Pickle must be part of
+            # OPCODE Proto == \x80 == 128
+            # https://github.com/python/cpython/blob/1b293b60067f6f4a95984d064ce0f6b6d34c1216/Lib/pickletools.py#L2124
+            if data[0] != 128:
+                raise ValueError("Decryption error")
+            config = pickle.loads(data, encoding="bytes")
+            # remove this debugging tool after transition
+            config["serialization_format"] = "pickle"
+            return config
+        except pickle.UnpicklingError as e2:
+            raise ValueError(f"{e1}\n{e2}")
+
+
+def report_error(msg: str, e: Exception, logger: Logger):
+    logger.error(
+        "\n".join(
+            (
+                msg,
+                str(e),
+                "Please report it to the CumulusCI team.",
+                "For now this is just a warning. By January 2023 it may become a real error.",
+                "When you report it to the CumulusCI team, they will investigate",
+                "whether the error is in your config or in CumulusCI",
+            )
+        )
+    )
+
+
+def check_round_trip(data: dict, logger: Logger) -> Optional[bytes]:
+    """Return JSON bytes if possible, else None"""
+    try:
+        as_json_text = json.dumps(data, default=encode_value).encode("utf-8")
+    except Exception as e:
+        report_error("CumulusCI found an unusual datatype in your config:", e, logger)
+        return None
+    try:
+        test_load = load_config_from_json_or_pickle(as_json_text)
+        assert _simplify_config(test_load) == _simplify_config(
+            data
+        ), f"JSON did not round-trip-cleanly {test_load}, {data}"
+    except Exception as e:  # pragma: no cover
+        report_error("CumulusCI found a problem saving your config:", e, logger)
+        return None
+    assert isinstance(as_json_text, bytes)
+    return as_json_text
+
+
+def _simplify_config(config: dict):
+    return {k: v for k, v in config.items() if k != "serialization_format"}
+
+
+def serialize_config_to_json_or_pickle(config: dict, logger: Logger) -> bytes:
+    """Serialize a dict to JSON if possible or Pickle otherwise"""
+    as_json_text = check_round_trip(config, logger)
+    if as_json_text and SHOULD_SAVE_AS_JSON:
+        assert isinstance(as_json_text, bytes)
+        return as_json_text
+    else:
+        return pickle.dumps(config, protocol=2)

cumulusci/core/keychain/subprocess_keychain.py

@@ -0,0 +1,24 @@
+import typing as T
+
+from cumulusci.core.exceptions import ServiceNotConfigured
+
+
+class SubprocessKeychain(T.NamedTuple):
+    """A pretend, in-memory keychain that knows about connected apps and nothing else.
+
+    This keychain is used by Snowfakery to make the connected app available to
+    tasks that are running in sub-processes. If Snowfakery ever needs a broader
+    range of services thaan just the connected app, this keychain will need to
+    become more complex.
+    """
+
+    connected_app: T.Any = None
+
+    def get_service(self, name, alias=None):
+        if name == "connected_app" and self.connected_app:
+            return self.connected_app
+
+        raise ServiceNotConfigured(name)
+
+    def set_org(self, *args):
+        pass

cumulusci/core/keychain/tests/conftest.py

@@ -0,0 +1,50 @@
+import pytest
+
+from cumulusci.core.config import (
+    BaseProjectConfig,
+    OrgConfig,
+    ServiceConfig,
+    UniversalConfig,
+)
+from cumulusci.core.config.scratch_org_config import ScratchOrgConfig
+
+
+@pytest.fixture()
+def project_config():
+    universal_config = UniversalConfig()
+    project_config = BaseProjectConfig(universal_config, config={"no_yaml": True})
+    project_config.config["services"] = {
+        "connected_app": {"attributes": {"test": {"required": True}}},
+        "github": {"attributes": {"name": {"required": True}, "password": {}}},
+        "github_enterprise": {
+            "attributes": {"name": {"required": True}, "password": {}}
+        },
+        "not_configured": {"attributes": {"foo": {"required": True}}},
+        "devhub": {"attributes": {"foo": {"required": True}}},
+        "marketing_cloud": {
+            "class_path": "cumulusci.core.config.marketing_cloud_service_config.MarketingCloudServiceConfig",
+            "attributes": {"foo": {"required": True}},
+        },
+    }
+    project_config.project__name = "TestProject"
+    return project_config
+
+
+@pytest.fixture
+def org_config():
+    return OrgConfig({"foo": "bar"}, "test")
+
+
+@pytest.fixture
+def scratch_org_config():
+    return ScratchOrgConfig({"foo": "bar"}, "test")
+
+
+@pytest.fixture
+def key():
+    return "0123456789123456"
+
+
+@pytest.fixture
+def service_config():
+    return ServiceConfig({"name": "bar@baz.biz", "password": "test123"})

cumulusci/core/keychain/tests/test_base_project_keychain.py

@@ -0,0 +1,299 @@
+import re
+from unittest import mock
+
+import pytest
+
+from cumulusci.core.config import (
+    BaseProjectConfig,
+    OrgConfig,
+    ScratchOrgConfig,
+    ServiceConfig,
+    UniversalConfig,
+)
+from cumulusci.core.exceptions import (
+    CumulusCIException,
+    OrgNotFound,
+    ServiceNotConfigured,
+    ServiceNotValid,
+)
+from cumulusci.core.keychain import DEFAULT_CONNECTED_APP, BaseProjectKeychain
+
+
+@pytest.fixture
+def service_configs():
+    return {
+        "connected_app": ServiceConfig({"test": "value"}),
+        "github": ServiceConfig({"name": "hub"}),
+    }
+
+
+@pytest.fixture
+def scratch_org_config():
+    return ScratchOrgConfig({"foo": "bar", "scratch": True}, "test_scratch")
+
+
+@pytest.fixture()
+def keychain(project_config, key):
+    keychain = BaseProjectKeychain(project_config, key)
+    assert keychain.project_config == project_config
+    assert keychain.key == key
+    return keychain
+
+
+class TestBaseProjectKeychain:
+    @pytest.mark.parametrize("key", ["0123456789123456", None])
+    def test_set_non_existant_service(self, project_config, key):
+        keychain = BaseProjectKeychain(project_config, key)
+        keychain.key = key
+        with pytest.raises(ServiceNotValid):
+            keychain.set_service("doesnotexist", "alias", ServiceConfig({"name": ""}))
+
+    def test_set_invalid_service(self, keychain):
+        with pytest.raises(ServiceNotValid):
+            keychain.set_service("github", "alias", ServiceConfig({"name": ""}))
+
+    def test_get_service__type_not_in_configuration(self, keychain):
+        """This service is not listed as a service type in cumulusci.yml"""
+        with pytest.raises(ServiceNotConfigured):
+            keychain.get_service("not_in_configuration")
+
+    def test_get_service__alias_not_found(self, keychain):
+        """This service is not listed as a service type in cumulusci.yml"""
+        with pytest.raises(ServiceNotConfigured):
+            keychain.get_service("connected_app", "not-in-configuration")
+
+    def test_get_service_not_configured(self, keychain):
+        """This service is supported by CumulusCI but is not currently configured"""
+        with pytest.raises(ServiceNotConfigured):
+            keychain.get_service("not_configured")
+
+    def test_set_service__github(self, keychain, service_config):
+        keychain.set_service("github", "alias", service_config)
+        assert keychain.get_service("github", "alias").config == service_config.config
+
+    def test_get_service__default_service(self, keychain):
+        keychain._default_services = {"devhub": "baz"}
+        keychain.config["services"] = {
+            "devhub": {"foo": "config1", "bar": "config2", "baz": "config3"}
+        }
+        # If we don't specify an alias to get_service() we get
+        # the default service for the given type
+        default_github_service = keychain.get_service("devhub")
+
+        assert default_github_service == "config3"
+
+    def test_get_service__default_service_not_set(self, keychain):
+        keychain._default_services = {"github": "baz"}
+        keychain.config["services"] = {
+            "devhub": {"foo": "config1", "bar": "config2", "baz": "config3"}
+        }
+
+        with pytest.raises(
+            CumulusCIException,
+            match="No default service currently set for service type: devhub",
+        ):
+            keychain.get_service("devhub")
+
+    def test_get_service__service_not_loaded(self, keychain, service_config):
+        keychain.project_config.config["services"] = {}
+        with pytest.raises(
+            ServiceNotValid,
+            match=re.escape("Expecting services to be loaded, but none were found."),
+        ):
+            keychain.get_service(
+                "test-service", "alias"
+            ).config == service_config.config
+
+    def test_get_service__service_type_not_in_config(self, keychain, service_config):
+        with pytest.raises(
+            ServiceNotConfigured,
+            match="Service type is not configured: test-service",
+        ):
+            keychain.get_service(
+                "test-service", "alias"
+            ).config == service_config.config
+
+    def test_get_service__DEFAULT_CONNECTED_APP(self, keychain):
+        keychain._load_default_connected_app()
+        service = keychain.get_service("connected_app")
+        assert service is DEFAULT_CONNECTED_APP
+
+    def test_set_service__private_method(self, keychain, service_config):
+        alias = "ziggy"
+        keychain.services = {"github": {}}
+        keychain._set_service("github", alias, service_config)
+        assert alias in keychain.services["github"].keys()
+        assert keychain.services["github"][alias].config == service_config.config
+
+    @pytest.mark.parametrize(
+        "service_type,expected", [("github", "foo"), ("connected_app", None)]
+    )
+    def test_get_default_service_name(self, service_type, expected, keychain):
+        keychain._default_services = {"github": "foo"}
+        assert keychain.get_default_service_name(service_type) == expected
+
+    def test_set_and_get_org(self, keychain, org_config):
+        org_config.global_org = False
+        keychain.set_org(org_config, global_org=False)
+        assert list(keychain.orgs.keys()) == ["test"]
+        assert keychain.get_org("test").config == org_config.config
+
+    def test_set_and_get_scratch_org(self, keychain, scratch_org_config):
+        keychain.set_org(scratch_org_config, global_org=False)
+        assert list(keychain.orgs.keys()) == ["test_scratch"]
+        org = keychain.get_org("test_scratch")
+        assert org.config == scratch_org_config.config
+        assert org.__class__ == ScratchOrgConfig
+
+    @pytest.mark.parametrize("key", ["0123456789123456", None])
+    def test_create_scratch_org(self, key):
+        project_config = BaseProjectConfig(
+            UniversalConfig, {"orgs": {"scratch": {"dev": {}}}}
+        )
+        keychain = BaseProjectKeychain(project_config, key)
+        keychain.key = key
+        keychain.create_scratch_org("test", "dev", days=3, release="previous")
+        org_config = keychain.get_org("test").config
+        assert org_config["days"] == 3
+        assert org_config["release"] == "previous"
+
+    def test_load_scratch_orgs(self, keychain):
+        assert list(keychain.orgs) == []
+
+    @pytest.mark.parametrize("key", ["0123456789123456", None])
+    def test_get_org__existing_scratch_org(self, project_config, key):
+        project_config.config["orgs"] = {}
+        project_config.config["orgs"]["scratch"] = {}
+        project_config.config["orgs"]["scratch"]["test_scratch_auto"] = {}
+        keychain = BaseProjectKeychain(project_config, key)
+        keychain.key = key
+        keychain._load_scratch_orgs()
+        assert list(keychain.orgs) == ["test_scratch_auto"]
+
+    @pytest.mark.parametrize("key", ["0123456789123456", None])
+    def test_get_org__existing_org(self, project_config, key):
+        project_config.config["orgs"] = {}
+        project_config.config["orgs"]["scratch"] = {}
+        project_config.config["orgs"]["scratch"]["test"] = {}
+
+        keychain = BaseProjectKeychain(project_config, key)
+        keychain.key = key
+        keychain.set_org(OrgConfig({}, "test"))
+
+        assert list(keychain.orgs) == ["test"]
+        org = keychain.get_org("test")
+        assert org.scratch is None
+
+    def test_get_org__not_found(self, keychain):
+        with pytest.raises(OrgNotFound):
+            keychain.get_org("test")
+
+    @mock.patch("sarge.Command")
+    def test_set_and_get_default_org(self, Command, keychain, org_config):
+        org_config = OrgConfig({"created": True}, "test", keychain=keychain)
+        org_config.save()
+        keychain.set_default_org("test")
+        org_config.config["default"] = True
+
+        assert Command.call_count == 2
+        assert keychain.get_default_org()[1].config == org_config.config
+
+    @mock.patch("sarge.Command")
+    def test_unset_default_org(self, Command, keychain, org_config):
+        org_config = org_config.config.copy()
+        org_config = OrgConfig(org_config, "test")
+        org_config.config["default"] = True
+        keychain.set_org(org_config)
+        keychain.unset_default_org()
+
+        Command.assert_called_once()
+        assert keychain.get_default_org()[1] is None
+
+    def test_list_orgs(self, keychain, org_config):
+        keychain.set_org(org_config)
+        assert keychain.list_orgs() == ["test"]
+
+    def test_list_orgs__empty(self, keychain):
+        assert keychain.list_orgs() == []
+
+    def test_get_default_org__no_default(self, keychain):
+        """Name and config for default org should be empty"""
+        assert keychain.get_default_org() == (None, None)
+
+    def test_validate_service_attributes(self, keychain):
+        # config is missing the "name" attribute
+        service_config = ServiceConfig({"password": "test123"})
+        error_message = re.escape(
+            "Missing required attribute(s) for github service: ['name']"
+        )
+        with pytest.raises(
+            ServiceNotValid,
+            match=error_message,
+        ):
+            keychain._validate_service_attributes("github", service_config)
+
+    def test_validate_service_alias__same_as_service_type(self, keychain):
+        with pytest.raises(
+            ServiceNotValid,
+            match=re.escape("Service name cannot be the same as the service type."),
+        ):
+            keychain._validate_service_alias("service_type", "service_type")
+
+    def test_list_services(self, keychain):
+        service_config = ServiceConfig({"foo": "bar"})
+        keychain.services = {
+            "devhub": {
+                "foo_alias": service_config,
+                "bar_alias": service_config,
+            },
+            "github": {
+                "zed_alias": service_config,
+                "zoo_alias": service_config,
+            },
+        }
+        services = keychain.list_services()
+        assert len(list(services.keys())) == 2
+        assert services["devhub"] == ["bar_alias", "foo_alias"]
+        assert services["github"] == ["zed_alias", "zoo_alias"]
+
+    def test_get_services_for_type(self, keychain):
+        service_config = ServiceConfig({"foo": "bar"})
+        keychain.services = {
+            "devhub": {
+                "foo_alias": service_config,
+                "bar_alias": service_config,
+            },
+            "github": {
+                "zed_alias": service_config,
+                "zoo_alias": service_config,
+                "zap_alias": service_config,
+            },
+            "github_enterprise": {
+                "meep_alias": service_config,
+                "morp_alias": service_config,
+            },
+        }
+        devhub_services = keychain.get_services_for_type("devhub")
+        github_services = keychain.get_services_for_type("github")
+        ghent_services = keychain.get_services_for_type("github_enterprise")
+
+        assert len(devhub_services) == 2
+        assert len(github_services) == 3
+        assert len(ghent_services) == 2
+
+    def test_remove_org(
+        self,
+        keychain,
+        org_config,
+    ):
+        keychain.cleanup_org_cache_dirs = mock.Mock()
+        keychain.set_org(org_config)
+        keychain.remove_org("test")
+        assert "test" not in keychain.orgs
+        keychain.cleanup_org_cache_dirs.assert_called_once()
+
+    def test_org_definition__missing(self, project_config, key):
+        """What if a scratch org was created with a YAML definition which was deleted more recently?"""
+        keychain = BaseProjectKeychain(project_config, key)
+        with pytest.raises(OrgNotFound, match="No such org"):
+            keychain.create_scratch_org("no_such_org", "no_such_org")