cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/utils/yaml/tests/test_cumulusci_yml.py

@@ -0,0 +1,286 @@
+import os
+from io import StringIO
+from pathlib import Path
+from unittest.mock import MagicMock, Mock, patch
+
+import pytest
+from pydantic import ValidationError
+
+from cumulusci.utils import temporary_dir
+from cumulusci.utils.yaml.cumulusci_yml import (
+    GitHubSourceModel,
+    _validate_files,
+    _validate_url,
+    cci_safe_load,
+    parse_from_yaml,
+)
+
+
+class TestCumulusciYml:
+    def test_cumulusci_yaml(self):
+        cciyml = parse_from_yaml("cumulusci.yml")
+        assert cciyml.project.package.name == "CumulusCI"
+        assert cciyml["project"]["package"]["name"] == "CumulusCI"
+        assert (
+            cciyml.tasks["robot"].options["suites"]
+            == cciyml["tasks"]["robot"]["options"]["suites"]
+            == "cumulusci/robotframework/tests"
+        )
+
+    def test_cumulusci_cumulusci_yaml(self, cumulusci_test_repo_root):
+        cciyml = parse_from_yaml(cumulusci_test_repo_root / "cumulusci/cumulusci.yml")
+        assert cciyml.tasks["connected_app"].options["overwrite"] is False
+
+    def test_parse_cumulusci_yaml(self):
+        yaml = """xyz:
+                    y: abc"""
+        lf = Mock()
+        cciyml = cci_safe_load(StringIO(yaml), "foo", on_error=lf)
+        assert isinstance(cciyml, dict)  # should parse despite model errors
+        lf.assert_called()
+        assert "foo" in str(lf.mock_calls[0][1][0])
+        assert "xyz" in str(lf.mock_calls[0][1][0])
+
+    @patch("cumulusci.utils.yaml.cumulusci_yml.validate_data")
+    def test_unexpected_exceptions(self, validate_data):
+        validate_data.side_effect = AssertionError("Boom!")
+        yaml = """xyz:
+            y: abc"""
+        logfunc = Mock()
+        cciyml = cci_safe_load(StringIO(yaml), "foo", on_error=logfunc)
+
+        assert isinstance(cciyml, dict)  # should parse despite model errors
+        logfunc.assert_called()
+        validate_data.assert_called()
+
+    @pytest.mark.large_vcr()
+    def test_from_web(self):
+        good_urls = """
+            https://raw.githubusercontent.com/SalesforceFoundation/NPSP/master/cumulusci.yml
+            https://raw.githubusercontent.com/SalesforceFoundation/EDA/master/cumulusci.yml
+        """
+
+        bad_urls = """
+            https://raw.githubusercontent.com/SFDO-Tooling/CumulusCI-Test/master/cumulusci.yml
+            https://raw.githubusercontent.com/SalesforceFoundation/Relationships/master/cumulusci.yml
+            https://raw.githubusercontent.com/SalesforceFoundation/Volunteers-for-Salesforce/master/cumulusci.yml
+            https://raw.githubusercontent.com/SalesforceFoundation/Recurring_Donations/master/cumulusci.yml
+        """
+
+        def raise_exception(error):
+            raise Exception(error)
+
+        def test_urls(urls_string, on_error_callback):
+            urls = (url.strip() for url in good_urls.split("\n"))
+            results = [
+                (url, cci_safe_load(url, on_error=on_error_callback))
+                for url in urls
+                if url
+            ]
+            for url, data in results:
+                assert "flows" in data, url
+
+        # these ones should not trigger the error handler
+        test_urls(good_urls, raise_exception)
+        # these ones might, but they should still parse smoothly
+        test_urls(bad_urls, lambda x: x)
+
+    @pytest.mark.skipif(
+        not os.environ.get("CCI_LOCAL_DIRECTORY_TESTS"),
+        reason="CCI_LOCAL_DIRECTORY_TESTS environment variable not set",
+    )  # you can turn this on if you happen to have this local file structure
+    def test_from_local(self):
+        assert parse_from_yaml("../Abacus/cumulusci.yml")
+        assert parse_from_yaml("../NPSP/cumulusci.yml")
+        assert parse_from_yaml("../CaseMan/cumulusci.yml")
+
+    def test_steps_flow_and_task_heterogenous(self, caplog):
+        yaml = """flows:
+                    my_flow:
+                        steps:
+                            1:
+                                flow: a
+                            2:
+                                task: b
+                            """
+        cci_data = cci_safe_load(StringIO(yaml))
+        assert not caplog.text
+        assert cci_data["flows"]["my_flow"]["steps"][1]["flow"] == "a"
+        assert cci_data["flows"]["my_flow"]["steps"][2]["task"] == "b"
+
+    def test_steps_as_list(self, caplog):
+        yaml = """flows:
+                    my_flow:
+                        steps:
+                            - A
+                            - B
+                            - C """
+        assert not caplog.text
+        cci_safe_load(StringIO(yaml))
+        assert "my_flow" in caplog.text
+        assert "steps" in caplog.text
+        assert "dict" in caplog.text
+
+    def test_individual_steps_as_list(self, caplog):
+        yaml = """flows:
+                    my_flow:
+                        steps:
+                            1:
+                                - task : b
+"""
+        assert not caplog.text
+        cci_safe_load(StringIO(yaml))
+        print(caplog.text)
+        assert "steps" in caplog.text
+        assert "my_flow" in caplog.text
+        assert "dict" in caplog.text
+
+    def test_flow_and_task_confusion(self, caplog):
+        yaml = """flows:
+                    my_flow:
+                        steps:
+                            1:
+                                task: b
+                                flow: c
+"""
+        assert not caplog.text
+        cci_safe_load(StringIO(yaml))
+        assert "steps" in caplog.text
+        assert "my_flow" in caplog.text
+
+    def test_validate_files__no_errors(self, caplog):
+        import cumulusci
+
+        codedir = Path(cumulusci.__file__).parent.parent
+        errs = _validate_files([str(codedir / "cumulusci.yml")])
+        assert not errs
+
+    def test_validate_files__with_errors(self, caplog):
+        codedir = Path(__file__).parent
+        errs = _validate_files([str(Path(codedir / "bad_cci.yml"))])
+        assert errs
+
+    @pytest.mark.vcr()
+    def test_validate_url__with_errors(self, caplog):
+        url = "https://raw.githubusercontent.com/SFDO-Tooling/CumulusCI/8b8d1eb9a1593503bf625030fa702b6d4651cb55/cumulusci/tasks/bulkdata/tests/snowfakery/simple_snowfakery.load.yml"
+        errs = _validate_url(url)
+        assert "sf_object" in str(errs)
+        assert "extra fields not permitted" in str(errs)
+        assert "snowfakery.load.yml" in str(errs)
+
+    def test_validate_empty(self, caplog):
+        out = cci_safe_load(StringIO(""))
+        assert not caplog.text, caplog.text
+        assert out == {}
+
+    def test_custom(self, caplog):
+        yaml = """project:
+                    custom:
+                        foo: X
+                        bar:
+                            - a
+                            - b
+                            - c
+                        baz:
+                            a: aa
+                            b: bb
+                            c: cc
+"""
+        cciyml = cci_safe_load(StringIO(yaml))
+        assert not caplog.text
+        assert cciyml["project"]["custom"]["foo"] == "X"
+        assert cciyml["project"]["custom"]["bar"] == ["a", "b", "c"]
+        assert cciyml["project"]["custom"]["baz"] == {
+            "a": "aa",
+            "b": "bb",
+            "c": "cc",
+        }
+
+
+@pytest.fixture
+def cci_yml_file():
+    "Yields a cumulusci file obj for writing to. Cleans up when finished."
+    with temporary_dir() as temp_dir:
+        cumulusci_yml_filepath = Path(temp_dir) / "cumulusci.yml"
+
+        with open(cumulusci_yml_filepath, "w+") as cumulusci_yml:
+            yield cumulusci_yml
+
+
+def test_mutually_exclusive_options():
+    logger = MagicMock()
+    with pytest.raises(AssertionError):
+        cci_safe_load(StringIO(""), on_error=lambda *args: args, logger=logger)
+
+
+def test_github_source():
+    with pytest.raises(ValidationError):
+        GitHubSourceModel(
+            github="https://github.com/Test/TestRepo", commit="abcdef", release="foo"
+        )
+
+    GitHubSourceModel(github="https://github.com/Test/TestRepo", release="latest")
+
+    assert (
+        GitHubSourceModel(github="https://github.com/Test/TestRepo").resolution_strategy
+        == "production"
+    )
+
+
+def test_single_primary_plan(caplog):
+    yaml = """
+plans:
+  first:
+    tier: primary
+  second:
+    tier: primary
+    """
+    cci_safe_load(StringIO(yaml))
+    assert "Only one plan can be defined as 'primary' or 'secondary'" in caplog.text
+
+
+def test_single_primary_plan_implicit(caplog):
+    yaml = """
+plans:
+  explicit:
+    tier: primary
+  implicit:
+    slug: implicit_primary
+    """
+    cci_safe_load(StringIO(yaml))
+    assert "Only one plan can be defined as 'primary' or 'secondary'" in caplog.text
+
+
+def test_single_secondary_plan(caplog):
+    yaml = """
+plans:
+  first:
+    tier: secondary
+  second:
+    tier: secondary
+    """
+    cci_safe_load(StringIO(yaml))
+    assert "Only one plan can be defined as 'primary' or 'secondary'" in caplog.text
+
+
+def test_multiple_additional_plan(caplog):
+    yaml = """
+plans:
+  first:
+    slug: implicit_primary
+  second:
+    tier: secondary
+  third:
+    tier: additional
+  fourth:
+    tier: additional
+    """
+    parsed_yaml: dict = cci_safe_load(StringIO(yaml))
+    expected: dict = {
+        "first": {"slug": "implicit_primary"},
+        "second": {"tier": "secondary"},
+        "third": {"tier": "additional"},
+        "fourth": {"tier": "additional"},
+    }
+    assert "" == caplog.text
+    assert expected == parsed_yaml["plans"]

cumulusci/utils/yaml/tests/test_model_parser.py

@@ -0,0 +1,175 @@
+from io import StringIO
+from unittest.mock import Mock
+
+import pytest
+from pydantic import Field
+
+from cumulusci.utils.yaml.model_parser import CCIDictModel, CCIModel, ValidationError
+
+
+class Foo(CCIModel):
+    bar: str = None
+    fields_ = Field([], alias="fields")
+
+
+class Document(CCIModel):
+    __root__: Foo
+
+
+class TestCCIModel:
+    def test_fields_property(self):
+        # JSON is YAML. Strange but true.
+        foo = Document.parse_from_yaml(StringIO("{bar: 'blah'}"))
+        assert type(foo) == Foo
+        assert foo.fields_ == []
+        assert foo.fields == []
+
+        foo = Document.parse_from_yaml(StringIO("{bar: 'blah', fields: [1,2]}"))
+        assert foo.fields == [1, 2]
+
+        foo.fields = ["a", "b"]
+        assert foo.fields == ["a", "b"]
+
+    def test_parse_from_dict(self):
+        assert Document.parse_obj({"bar": "blah"})
+
+    def test_validate_data__success(self):
+        assert Document.validate_data({"bar": "blah"})
+
+    def test_validate_data__without_error_handler(self):
+        assert not Document.validate_data({"foo": "fail"}, context="pytest")
+
+    def test_validate_data__with_error_handler(self):
+        lf = Mock()
+        assert not Document.validate_data(
+            {"foo": "fail"}, context="pytest", on_error=lf
+        )
+        lf.assert_called()
+        assert "pytest" in str(lf.mock_calls[0][1][0])
+        assert "foo" in str(lf.mock_calls[0][1][0])
+
+    def test_validate_on_error_param(self):
+        with pytest.raises(Exception) as e:
+            assert not Document.validate_data({"qqq": "zzz"}, on_error="barn")
+        assert e.value.__class__ in [ValueError, TypeError]
+
+    def test_getattr_missing(self):
+        with pytest.raises(AttributeError):
+            x = Document.parse_obj({})
+            assert x
+            x.foo
+
+    def test_error_messages(self):
+        class FooWithError(CCIModel):
+            bar: int = None
+
+        class DocumentWithError(CCIModel):
+            __root__: FooWithError
+
+        s = StringIO("{bar: 'blah'}")
+        s.name = "some_filename"
+        with pytest.raises(ValidationError) as e:
+            DocumentWithError.parse_from_yaml(s)
+        assert "some_filename" in str(e.value)
+
+    def test_error_messages__nested(self):
+        class Foo(CCIModel):
+            bar: int  # required
+
+        class Bar(CCIModel):
+            foo: Foo = None
+
+        class Baz(CCIModel):
+            bar: Bar = None
+
+        class Document(CCIModel):
+            __root__: Baz
+
+        s = StringIO("{bar: {foo: {}}}")
+        s.name = "some_filename"
+        with pytest.raises(ValidationError) as e:
+            Document.parse_from_yaml(s)
+        assert "some_filename" in str(e.value)
+
+    def test_fields_no_alias(self):
+        class Foo(CCIDictModel):
+            bar: str = None
+
+        x = Foo.parse_obj({})
+        assert x
+        with pytest.raises(AttributeError):
+            x.fields
+
+    def test_copy(self):
+        class Foo(CCIDictModel):
+            bar: str = None
+
+        x = Foo(bar="abc")
+        y = x.copy()
+        y.bar = "def"
+        assert x.bar == "abc"
+        assert y.bar == "def"
+
+
+class TestCCIDictModel:
+    def test_fields_items(self):
+        class Foo(CCIDictModel):
+            bar: str = None
+            fields_ = Field([], alias="fields")
+
+        class Document(CCIDictModel):
+            __root__: Foo
+
+        # JSON is YAML. Strange but true.
+        foo = Document.parse_from_yaml(StringIO("{bar: 'blah'}"))
+        assert type(foo) == Foo
+        assert foo["fields"] == []
+
+        foo = Document.parse_from_yaml(StringIO("{bar: 'blah', fields: [1,2]}"))
+        assert foo["fields"] == [1, 2]
+
+        foo["fields"] = ["a", "b"]
+        assert foo["fields"] == ["a", "b"]
+
+    def test_getitem_missing(self):
+        class Foo(CCIDictModel):
+            bar: str = None
+            fields_ = Field([], alias="fields")
+
+        x = Foo.parse_obj({})
+        assert x
+        with pytest.raises(IndexError):
+            x["foo"]
+
+        assert "bar" in x
+        assert "fields" in x
+        assert x["fields"] == []
+
+    def test_get(self):
+        class Foo(CCIDictModel):
+            bar: str = None
+            fields_ = Field([], alias="fields")
+
+        x = Foo.parse_obj({"bar": "q"})
+        assert x.get("bar") == x.bar == x["bar"] == "q"
+        assert x.get("xyzzy", 0) == 0
+        assert x.get("xyzzy") is None
+        assert x.get("fields") == []
+
+    def test_del(self):
+        class Foo(CCIDictModel):
+            bar: str = None
+            fields_ = Field([], alias="fields")
+
+        x = Foo.parse_obj({"bar": "q"})
+        assert x["bar"] == x.bar == "q"
+        assert "bar" in x
+        del x["bar"]
+        assert "bar" not in x
+        assert x.get("bar") is None
+
+        assert x["fields"] == x.fields == []
+        assert "fields" in x
+        del x["fields"]
+        assert "fields" not in x
+        assert x.get("fields") is None

cumulusci/utils/yaml/tests/test_safer_loader.py

@@ -0,0 +1,88 @@
+from io import StringIO
+from unittest.mock import patch
+
+import pytest
+
+from cumulusci.core.exceptions import YAMLParseException
+from cumulusci.utils.yaml.safer_loader import _replace_nbsp, load_yaml_data
+
+
+@pytest.fixture
+def incorrect_yaml_file(tmp_path):
+    with (tmp_path / "foo.yml").open("w+") as cci_yml_file:
+        invalid_yml = """xyz: abc   \n>>>\nefg: lmn\n"""
+        cci_yml_file.write(invalid_yml)
+        cci_yml_file.seek(0)
+        yield cci_yml_file
+
+
+def test_simple_load(caplog):
+    yaml = """xyz:
+        y: abc"""
+    cciyml = load_yaml_data(StringIO(yaml))
+    assert not caplog.text
+
+    assert isinstance(cciyml, dict)  # should parse despite funny character
+    assert cciyml["xyz"]["y"] == "abc", cciyml
+
+
+def test_convert_nbsp(caplog):
+    yaml = """xyz:
+        \u00A0 y: abc"""
+    cciyml = load_yaml_data(StringIO(yaml))
+    assert "space character" in caplog.text
+
+    assert isinstance(cciyml, dict)  # should parse despite funny character
+    assert cciyml["xyz"]["y"] == "abc", cciyml
+
+
+def test_converter():
+    inp = """xyz:
+        \u00A0 y: abc"""
+    outp = """xyz:
+          y: abc"""
+
+    rc = _replace_nbsp(inp, "filename")
+    assert rc == outp
+
+
+def test_converter_is_selective():
+    inp = """xyz:
+            y: abc\u00A0"""
+
+    rc = _replace_nbsp(inp, "filename")
+    assert rc == inp
+
+
+def test_invalid_cumulusci_yml_file(incorrect_yaml_file):
+    with pytest.raises(
+        YAMLParseException,
+        match="An error occurred parsing yaml file at line 2, column 1.*",
+    ):
+        load_yaml_data(incorrect_yaml_file)
+
+
+def test_invalid_string_io():
+    invalid_yml_string = """xyz: abc   \n>>>\nefg: lmn\n"""
+    with pytest.raises(YAMLParseException) as error:
+        load_yaml_data(StringIO(invalid_yml_string))
+
+    assert "Error message: " in error.value.args[0]
+
+
+@patch("cumulusci.utils.yaml.safer_loader.yaml.safe_load")
+def test_generic_exception__with_name_attr(safe_load, incorrect_yaml_file):
+    safe_load.side_effect = Exception("generic")
+    with pytest.raises(YAMLParseException):
+        load_yaml_data(incorrect_yaml_file)
+
+
+@patch("cumulusci.utils.yaml.safer_loader.yaml.safe_load")
+def test_generic_exception__without_name_attr(safe_load):
+    invalid_yaml = """xyz: abc   \n>>>\nefg: lmn\n"""
+    safe_load.side_effect = Exception("generic")
+    with pytest.raises(
+        YAMLParseException,
+        match="An error occurred parsing a yaml file.\nError message: generic",
+    ):
+        load_yaml_data(StringIO(invalid_yaml))

cumulusci/utils/ziputils.py

@@ -0,0 +1,61 @@
+import hashlib
+import io
+import zipfile
+
+
+def zip_subfolder(zip_src: zipfile.ZipFile, path):
+    if path and not path.endswith("/"):
+        path = path + "/"
+
+    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
+    for name in zip_src.namelist():
+        if not name.startswith(path):
+            continue
+
+        content = zip_src.read(name)
+        rel_name = name.replace(path, "", 1)
+
+        if rel_name:
+            zip_dest.writestr(rel_name, content)
+
+    return zip_dest
+
+
+def process_text_in_zipfile(zf, process_file):
+    """Process each file in a zip file using the `process_file` function.
+
+    Returns a new zip file.
+
+    `process_file` should be a function which accepts a filename and content as text
+    and returns a (possibly modified) filename and content.  The file will be
+    replaced with the new content, and renamed if necessary.
+
+    Files with content that cannot be decoded as UTF-8 will be skipped.
+    """
+
+    new_zf = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
+    for name in zf.namelist():
+        content = zf.read(name)
+        try:
+            content = content.decode("utf-8")
+        except UnicodeDecodeError:
+            # Probably a binary file; don't change it
+            pass
+        else:
+            name, content = process_file(name, content)
+        # writestr handles either bytes or text, and will implicitly encode text as utf-8
+        new_zf.writestr(name, content)
+    zf.close()
+    return new_zf
+
+
+def hash_zipfile_contents(zf):
+    """Returns a hash of a zipfile's file contents.
+
+    Ignores things like file mode, modtime
+    """
+    h = hashlib.blake2b()
+    for name in zf.namelist():
+        h.update(name.encode("utf-8"))
+        h.update(zf.read(name))
+    return h.hexdigest()