cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/tasks/bulkdata/tests/test_factory_utils.py

@@ -0,0 +1,55 @@
+import os
+
+import factory
+import pytest
+
+from cumulusci.tasks.bulkdata import factory_utils
+from cumulusci.tasks.bulkdata.tests.dummy_data_factory import (
+    ContactFactory,
+    GenerateDummyData,
+)
+from cumulusci.tasks.bulkdata.tests.utils import _make_task
+from cumulusci.utils import temporary_dir
+
+
+class TestFactoryUtils:
+    def test_factory(self):
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        with temporary_dir() as d:
+            tmp_db_path = os.path.join(d, "temp.db")
+            dburl = "sqlite:///" + tmp_db_path
+            task = _make_task(
+                GenerateDummyData,
+                {
+                    "options": {
+                        "num_records": 12,
+                        "mapping": mapping_file,
+                        "database_url": dburl,
+                    }
+                },
+            )
+            task()
+
+
+class TestAdder:
+    def test_adder(self):
+        a = factory_utils.Adder(10)
+        b = a(20)
+        assert b == 30
+        c = a(0)
+        assert c == 30
+        d = a(-5)
+        assert d == 25
+        a.reset(3)
+        assert a(0) == 3
+
+
+class TestFactories:
+    def test_factories(self):
+        class Broken(factory.alchemy.SQLAlchemyModelFactory):
+            class Meta:
+                model = "xyzzy"
+
+        with pytest.raises(KeyError):
+            factory_utils.Factories(None, {}, {"A": ContactFactory, "B": Broken})

cumulusci/tasks/bulkdata/tests/test_generate_and_load.py

@@ -0,0 +1,252 @@
+import os.path
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest import mock
+
+import pytest
+
+from cumulusci.core.exceptions import TaskOptionsError
+from cumulusci.tasks.bulkdata import GenerateAndLoadData
+
+from .utils import _make_task
+
+
+class TestGenerateAndLoadData:
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    def test_generate_and_load_data(self, _dataload):
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        task = _make_task(
+            GenerateAndLoadData,
+            {
+                "options": {
+                    "num_records": 12,
+                    "mapping": mapping_file,
+                    "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                }
+            },
+        )
+
+        task()
+        _dataload.assert_called_once()
+
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    def test_generate_and_load_data_batched(self, _dataload):
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        task = _make_task(
+            GenerateAndLoadData,
+            {
+                "options": {
+                    "num_records": 20,
+                    "mapping": mapping_file,
+                    "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                    "batch_size": 8,
+                }
+            },
+        )
+
+        task()
+        calls = _dataload.mock_calls
+        assert len(calls) == 3
+        task_options = [call[1][0] for call in calls]  # get at the args
+        for i in range(0, 3):
+            assert task_options[i]["current_batch_number"] == i
+            assert task_options[i]["batch_size"] == 8
+            assert task_options[i]["mapping"] == mapping_file
+        assert task_options[0]["num_records"] == task_options[1]["num_records"] == 8
+        assert task_options[2]["num_records"] == 4
+
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    def test_batchsize_zero(self, _dataload):
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        options = {
+            "options": {
+                "num_records": 20,
+                "mapping": mapping_file,
+                "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                "batch_size": 0,
+            }
+        }
+
+        with pytest.raises(TaskOptionsError):
+            _make_task(GenerateAndLoadData, options)
+
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    def test_no_datageneration_task_specified(self, _dataload):
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        options = {"options": {"num_records": 20, "mapping": mapping_file}}
+
+        with pytest.raises(TaskOptionsError) as e:
+            _make_task(GenerateAndLoadData, options)
+
+        assert "No data generation task" in str(e.value)
+
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    def test_batchsize_matches_numrecords(self, _dataload):
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        task = _make_task(
+            GenerateAndLoadData,
+            {
+                "options": {
+                    "num_records": 12,
+                    "mapping": mapping_file,
+                    "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                    "batch_size": 12,
+                }
+            },
+        )
+
+        task()
+        _dataload.assert_called_once()
+
+    def test_bad_mapping_file_path(self):
+        with pytest.raises(TaskOptionsError):
+            _make_task(
+                GenerateAndLoadData,
+                {
+                    "options": {
+                        "num_records": 12,
+                        "mapping": "does_not_exist",
+                        "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                        "batch_size": 12,
+                    }
+                },
+            )
+
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    def test_missing_mapping_file_when_needed(self, _dataload):
+        with pytest.raises(TaskOptionsError):
+            task = _make_task(
+                GenerateAndLoadData,
+                {
+                    "options": {
+                        "num_records": 12,
+                        "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                        "batch_size": 12,
+                    }
+                },
+            )
+            task()
+
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    def test_missing_mapping_file_path(self, _dataload):
+        task = _make_task(
+            GenerateAndLoadData,
+            {
+                "options": {
+                    "num_records": 12,
+                    "data_generation_task": "cumulusci.tasks.bulkdata.tests.mock_data_factory_without_mapping.GenerateDummyData",
+                    "batch_size": 12,
+                }
+            },
+        )
+        task()
+        calls = _dataload.mock_calls
+        assert calls[0][1][0]["generate_mapping_file"]
+
+    def test_bad_options(self):
+        with pytest.raises(TaskOptionsError):
+            _make_task(
+                GenerateAndLoadData,
+                {
+                    "options": {
+                        "num_records": 12,
+                        "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                        "database_url": "not_a_real_url:///non-url",
+                        "batch_size": -1,
+                    }
+                },
+            )
+
+    def test_loader_subtask(self):
+        class MockLoadData:
+            def __init__(self, *args, **kwargs):
+                options = kwargs["task_config"].options
+                assert options["num_records"] == 12
+                assert options["database_url"].startswith("sqlite")
+                assert options["bulk_mode"] == "Serial"
+                assert "mapping_vanilla_sf" in options["mapping"]
+
+            def __call__(self):
+                self.return_values = {}
+
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        with mock.patch(
+            "cumulusci.tasks.bulkdata.generate_and_load_data.LoadData", MockLoadData
+        ):
+            task = _make_task(
+                GenerateAndLoadData,
+                {
+                    "options": {
+                        "num_records": 12,
+                        "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                        "mapping": mapping_file,
+                        "bulk_mode": "Serial",
+                    }
+                },
+            )
+            task()
+
+    def test_error_on_tables_exist(self):
+        class FakeEngine:
+            pass
+
+        class FakeMetadata:
+            tables = {"foo": mock.MagicMock}
+
+        def _setup_engine(*args, **kwargs):
+            return FakeEngine(), FakeMetadata()
+
+        with mock.patch(
+            "cumulusci.tasks.bulkdata.generate_and_load_data.GenerateAndLoadData._setup_engine",
+            _setup_engine,
+        ):
+            with pytest.raises(TaskOptionsError):
+                task = _make_task(
+                    GenerateAndLoadData,
+                    {
+                        "options": {
+                            "num_records": 12,
+                            "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                            "batch_size": 12,
+                            "database_url": "not_a_real:///",
+                        }
+                    },
+                )
+
+                task()
+
+    def test_working_directory(self):
+        class MockLoadData:
+            def __init__(self, *args, **kwargs):
+                options = kwargs["task_config"].options
+                assert Path(options["working_directory"]).exists()
+
+            def __call__(self):
+                self.return_values = {}
+
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        with TemporaryDirectory() as t:
+            with mock.patch(
+                "cumulusci.tasks.bulkdata.generate_and_load_data.LoadData", MockLoadData
+            ):
+                assert not list(Path(t).glob("*"))
+                task = _make_task(
+                    GenerateAndLoadData,
+                    {
+                        "options": {
+                            "num_records": 12,
+                            "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                            "working_directory": t,
+                            "mapping": mapping_file,
+                        }
+                    },
+                )
+                task()
+                assert list(Path(t).glob("*"))

cumulusci/tasks/bulkdata/tests/test_generate_from_snowfakery_task.py

@@ -0,0 +1,343 @@
+from contextlib import contextmanager
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest import mock
+
+import pytest
+import yaml
+from sqlalchemy import create_engine
+
+from cumulusci.core.exceptions import TaskOptionsError
+from cumulusci.tasks.bulkdata.generate_and_load_data_from_yaml import (
+    GenerateAndLoadDataFromYaml,
+)
+from cumulusci.tasks.bulkdata.tests.utils import _make_task
+
+sample_yaml = Path(__file__).parent / "snowfakery/gen_npsp_standard_objects.recipe.yml"
+simple_yaml = Path(__file__).parent / "snowfakery/include_parent.yml"
+simple_snowfakery_yaml = (
+    Path(__file__).parent / "snowfakery/simple_snowfakery.recipe.yml"
+)
+
+from cumulusci.tasks.bulkdata.generate_from_yaml import GenerateDataFromYaml
+
+vanilla_mapping_file = Path(__file__).parent / "../tests/mapping_vanilla_sf.yml"
+
+
+@contextmanager
+def temporary_file_path(filename):
+    with TemporaryDirectory() as tmpdirname:
+        path = Path(tmpdirname) / filename
+        yield path
+
+
+@contextmanager
+def temp_sqlite_database_url():
+    with temporary_file_path("test.db") as path:
+        yield f"sqlite:///{str(path)}"
+
+
+@contextmanager
+def run_task(task=GenerateDataFromYaml, **options):
+    with temp_sqlite_database_url() as database_url:
+        options["database_url"] = database_url
+        task = _make_task(task, {"options": options})
+        task()
+        yield database_url
+
+
+class TestGenerateFromDataTask:
+    def assertRowsCreated(self, database_url):
+        engine = create_engine(database_url)
+        connection = engine.connect()
+        accounts = connection.execute("select * from Account")
+        accounts = list(accounts)
+        assert accounts and accounts[0] and accounts[0][1]
+        return accounts
+
+    def test_no_options(self):
+        with pytest.raises(Exception):
+            _make_task(GenerateDataFromYaml, {})
+
+    def test_simple(self):
+        with temp_sqlite_database_url() as database_url:
+            task = _make_task(
+                GenerateDataFromYaml,
+                {
+                    "options": {
+                        "generator_yaml": sample_yaml,
+                        "num_records": 1,
+                        "database_url": database_url,
+                        "num_records_tablename": "Account",
+                    }
+                },
+            )
+            task()
+            self.assertRowsCreated(database_url)
+
+    def test_inaccessible_generator_yaml(self):
+        with pytest.raises(TaskOptionsError):
+            task = _make_task(
+                GenerateDataFromYaml,
+                {
+                    "options": {
+                        "generator_yaml": sample_yaml / "junk",
+                        "num_records": 10,
+                        "num_records_tablename": "Account",
+                    }
+                },
+            )
+            task()
+
+    def test_vars(self):
+        with temp_sqlite_database_url() as database_url:
+            with pytest.warns(UserWarning):
+                task = _make_task(
+                    GenerateDataFromYaml,
+                    {
+                        "options": {
+                            "generator_yaml": sample_yaml,
+                            "vars": "xyzzy:foo",
+                            "database_url": database_url,
+                        }
+                    },
+                )
+                task()
+                self.assertRowsCreated(database_url)
+
+    def test_generate_mapping_file(self):
+        with temporary_file_path("mapping.yml") as temp_mapping:
+            with temp_sqlite_database_url() as database_url:
+                task = _make_task(
+                    GenerateDataFromYaml,
+                    {
+                        "options": {
+                            "generator_yaml": sample_yaml,
+                            "database_url": database_url,
+                            "generate_mapping_file": temp_mapping,
+                        }
+                    },
+                )
+                task()
+            mapping = yaml.safe_load(open(temp_mapping))
+            assert mapping["Insert Account"]["fields"]
+
+    def test_use_mapping_file(self):
+        assert vanilla_mapping_file.exists()
+        with temp_sqlite_database_url() as database_url:
+            task = _make_task(
+                GenerateDataFromYaml,
+                {
+                    "options": {
+                        "generator_yaml": sample_yaml,
+                        "database_url": database_url,
+                        "mapping": vanilla_mapping_file,
+                    }
+                },
+            )
+            task()
+            self.assertRowsCreated(database_url)
+
+    def test_num_records(self):
+        with temp_sqlite_database_url() as database_url:
+            task = _make_task(
+                GenerateDataFromYaml,
+                {
+                    "options": {
+                        "generator_yaml": simple_yaml,
+                        "database_url": database_url,
+                    }
+                },
+            )
+            task()
+            assert len(self.assertRowsCreated(database_url)) == 1, len(
+                self.assertRowsCreated(database_url)
+            )
+
+    @mock.patch(
+        "cumulusci.tasks.bulkdata.generate_and_load_data_from_yaml.GenerateAndLoadDataFromYaml._dataload"
+    )
+    def test_simple_generate_and_load_with_numrecords(self, _dataload):
+        task = _make_task(
+            GenerateAndLoadDataFromYaml,
+            {
+                "options": {
+                    "generator_yaml": simple_yaml,
+                    "num_records": 11,
+                    "num_records_tablename": "Account",
+                }
+            },
+        )
+        task()
+        assert len(_dataload.mock_calls) == 1
+
+    @mock.patch(
+        "cumulusci.tasks.bulkdata.generate_and_load_data_from_yaml.GenerateAndLoadDataFromYaml._dataload"
+    )
+    def test_simple_generate_and_load(self, _dataload):
+        task = _make_task(
+            GenerateAndLoadDataFromYaml,
+            {
+                "options": {
+                    "generator_yaml": simple_yaml,
+                    "num_records": 11,
+                    "num_records_tablename": "Account",
+                }
+            },
+        )
+        task()
+        assert len(_dataload.mock_calls) == 1
+
+    @mock.patch("cumulusci.tasks.bulkdata.generate_from_yaml.generate_data")
+    def test_exception_handled_cleanly(self, generate_data):
+        generate_data.side_effect = AssertionError("Foo")
+        with pytest.raises(AssertionError) as e:
+            task = _make_task(
+                GenerateAndLoadDataFromYaml,
+                {
+                    "options": {
+                        "generator_yaml": simple_yaml,
+                        "num_records": 11,
+                        "num_records_tablename": "Account",
+                    }
+                },
+            )
+            task()
+            assert "Foo" in str(e.value)
+        assert len(generate_data.mock_calls) == 1
+
+    @mock.patch(
+        "cumulusci.tasks.bulkdata.generate_and_load_data_from_yaml.GenerateAndLoadDataFromYaml._dataload"
+    )
+    def test_batching(self, _dataload):
+        with temp_sqlite_database_url() as database_url:
+            task = _make_task(
+                GenerateAndLoadDataFromYaml,
+                {
+                    "options": {
+                        "generator_yaml": simple_yaml,
+                        "num_records": 14,
+                        "batch_size": 6,
+                        "database_url": database_url,
+                        "num_records_tablename": "Account",
+                        "data_generation_task": "cumulusci.tasks.bulkdata.generate_from_yaml.GenerateDataFromYaml",
+                        "reset_oids": False,
+                    }
+                },
+            )
+            task()
+            assert len(_dataload.mock_calls) == 3
+            task = None  # clean up db?
+
+            engine = create_engine(database_url)
+            connection = engine.connect()
+            records = list(connection.execute("select * from Account"))
+            connection.close()
+            assert len(records) == 14 % 6  # leftovers
+
+    def test_mismatched_options(self):
+        with pytest.raises(TaskOptionsError) as e:
+            task = _make_task(
+                GenerateDataFromYaml,
+                {"options": {"generator_yaml": sample_yaml, "num_records": 10}},
+            )
+            task()
+        assert "without num_records_tablename" in str(e.value)
+
+    def test_with_nonexistent_continuation_file(self):
+        with pytest.raises(TaskOptionsError) as e:
+            with temp_sqlite_database_url() as database_url:
+                task = _make_task(
+                    GenerateDataFromYaml,
+                    {
+                        "options": {
+                            "generator_yaml": sample_yaml,
+                            "database_url": database_url,
+                            "mapping": vanilla_mapping_file,
+                            "continuation_file": "/tmp/foobar/baz/jazz/continuation.yml",
+                        }
+                    },
+                )
+                task()
+                rows = self.assertRowsCreated(database_url)
+                assert dict(rows[0])["id"] == 6
+
+        assert "jazz" in str(e.value)
+        assert "does not exist" in str(e.value)
+
+    def test_generate_continuation_file(self):
+        with temporary_file_path("cont.yml") as temp_continuation_file:
+            with temp_sqlite_database_url() as database_url:
+                task = _make_task(
+                    GenerateDataFromYaml,
+                    {
+                        "options": {
+                            "generator_yaml": sample_yaml,
+                            "database_url": database_url,
+                            "generate_continuation_file": temp_continuation_file,
+                        }
+                    },
+                )
+                task()
+            continuation_file = yaml.safe_load(open(temp_continuation_file))
+            assert continuation_file  # internals of this file are not important to CumulusCI
+
+    def _get_mapping_file(self, **options):
+        with temporary_file_path("mapping.yml") as temp_mapping:
+            with temp_sqlite_database_url() as database_url:
+                task = _make_task(
+                    GenerateDataFromYaml,
+                    {
+                        "options": {
+                            "database_url": database_url,
+                            "generate_mapping_file": temp_mapping,
+                            **options,
+                        }
+                    },
+                )
+                task()
+            with open(temp_mapping) as f:
+                mapping = yaml.safe_load(f)
+        return mapping
+
+    def test_generate_mapping_file__loadfile__inferred(self):
+        mapping = self._get_mapping_file(generator_yaml=simple_snowfakery_yaml)
+
+        assert mapping["Insert Account"]["api"] == "bulk"
+        assert mapping["Insert Contact"].get("bulk_mode") is None
+        assert list(mapping.keys()) == ["Insert Account", "Insert Contact"]
+
+    def test_generate_mapping_file__loadfile__overridden(self):
+        loading_rules = str(simple_snowfakery_yaml).replace(
+            ".recipe.yml", "_2.load.yml"
+        )
+        mapping = self._get_mapping_file(
+            generator_yaml=simple_snowfakery_yaml, loading_rules=str(loading_rules)
+        )
+
+        assert mapping["Insert Account"].get("api") is None
+        assert mapping["Insert Contact"]["bulk_mode"].lower() == "parallel"
+        assert list(mapping.keys()) == ["Insert Contact", "Insert Account"]
+
+    def test_generate_mapping_file__loadfile_multiple_files(self):
+        loading_rules = (
+            str(simple_snowfakery_yaml).replace(".recipe.yml", "_2.load.yml")
+            + ","
+            + str(simple_snowfakery_yaml).replace(".recipe.yml", ".load.yml")
+        )
+        mapping = self._get_mapping_file(
+            generator_yaml=simple_snowfakery_yaml, loading_rules=str(loading_rules)
+        )
+
+        assert mapping["Insert Account"]["api"] == "bulk"
+        assert mapping["Insert Contact"]["bulk_mode"].lower() == "parallel"
+        assert list(mapping.keys()) == ["Insert Contact", "Insert Account"]
+
+    def test_generate_mapping_file__loadfile_missing(self):
+        loading_rules = str(simple_snowfakery_yaml).replace(
+            ".recipe.yml", "_3.load.yml"
+        )
+        with pytest.raises(FileNotFoundError):
+            self._get_mapping_file(
+                generator_yaml=simple_snowfakery_yaml, loading_rules=str(loading_rules)
+            )