cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py

@@ -0,0 +1,330 @@
+from snowfakery.cci_mapping_files.declaration_parser import SObjectRuleDeclaration
+
+from cumulusci.tasks.bulkdata.extract_dataset_utils.extract_yml import (
+    ExtractDeclaration,
+)
+from cumulusci.tasks.bulkdata.extract_dataset_utils.tests.test_synthesize_extract_declarations import (
+    _fake_get_org_schema,
+    describe_for,
+)
+from cumulusci.tasks.bulkdata.generate_mapping_utils.generate_mapping_from_declarations import (
+    create_load_mapping_file_from_extract_declarations,
+    generate_load_mapping_file,
+)
+from cumulusci.tasks.bulkdata.mapping_parser import MappingStep
+from cumulusci.tasks.bulkdata.step import DataApi, DataOperationType
+
+
+class TestGenerateLoadMappingFromDeclarations:
+    def test_simple_generate_mapping_from_declarations(self, org_config):
+        declarations = [
+            ExtractDeclaration(sf_object="Account", fields=["Name", "Description"])
+        ]
+        object_counts = {"Account": 10, "Contact": 0, "Case": 5}
+        obj_describes = (
+            describe_for("Account"),
+            describe_for("Contact"),
+            describe_for("Case"),
+        )
+        with _fake_get_org_schema(
+            org_config,
+            obj_describes,
+            object_counts,
+            filters=[],
+            include_counts=True,
+        ) as schema:
+            mf = create_load_mapping_file_from_extract_declarations(
+                declarations, schema
+            )
+            assert mf == {
+                "Insert Account": {
+                    "sf_object": "Account",
+                    "table": "Account",
+                    "fields": ["Name", "Description"],
+                    "select_options": {},
+                }
+            }
+
+    def test_generate_mapping_from_both_kinds_of_declarations(self, org_config):
+        declarations = [
+            ExtractDeclaration(sf_object="Account", fields=["Name", "Description"]),
+            ExtractDeclaration(sf_object="Contact", fields=["FirstName", "LastName"]),
+        ]
+        object_counts = {"Account": 10, "Contact": 1, "Case": 5}
+        obj_describes = (
+            describe_for("Account"),
+            describe_for("Contact"),
+            describe_for("Case"),
+        )
+        with _fake_get_org_schema(
+            org_config,
+            obj_describes,
+            object_counts,
+            filters=[],
+            include_counts=True,
+        ) as schema:
+            loading_rules = [
+                SObjectRuleDeclaration(sf_object="Account", load_after="Contact")
+            ]
+            mf = create_load_mapping_file_from_extract_declarations(
+                declarations, schema, (), loading_rules
+            )
+            assert tuple(mf.items()) == tuple(
+                {
+                    "Insert Contact": {
+                        "sf_object": "Contact",
+                        "table": "Contact",
+                        "fields": ["FirstName", "LastName"],
+                        "select_options": {},
+                    },
+                    "Insert Account": {
+                        "sf_object": "Account",
+                        "table": "Account",
+                        "fields": ["Name", "Description"],
+                        "select_options": {},
+                    },
+                }.items()
+            )
+
+    def test_generate_load_mapping_from_declarations__lookups(self, org_config):
+        declarations = [
+            ExtractDeclaration(sf_object="Account", fields=["Name", "Description"]),
+            ExtractDeclaration(
+                sf_object="Contact", fields=["FirstName", "LastName", "AccountId"]
+            ),
+        ]
+        object_counts = {"Account": 10, "Contact": 5, "Case": 5}
+        obj_describes = (
+            describe_for("Account"),
+            describe_for("Contact"),
+            describe_for("Case"),
+        )
+        with _fake_get_org_schema(
+            org_config,
+            obj_describes,
+            object_counts,
+            filters=[],
+            include_counts=True,
+        ) as schema:
+            mf = create_load_mapping_file_from_extract_declarations(
+                declarations, schema
+            )
+            assert mf == {
+                "Insert Account": {
+                    "sf_object": "Account",
+                    "table": "Account",
+                    "fields": ["Name", "Description"],
+                    "select_options": {},
+                },
+                "Insert Contact": {
+                    "sf_object": "Contact",
+                    "table": "Contact",
+                    "fields": ["FirstName", "LastName"],
+                    "lookups": {
+                        "AccountId": {"table": ["Account"], "key_field": "AccountId"}
+                    },
+                    "select_options": {},
+                },
+            }
+
+    def test_generate_load_mapping_from_declarations__polymorphic_lookups(
+        self, org_config
+    ):
+        """Generate correct mapping file for sobjects with polymorphic lookup fields"""
+        declarations = [
+            ExtractDeclaration(sf_object="Account", fields=["Name", "Description"]),
+            ExtractDeclaration(
+                sf_object="Contact", fields=["FirstName", "LastName", "AccountId"]
+            ),
+            ExtractDeclaration(sf_object="Lead", fields=["LastName", "Company"]),
+            ExtractDeclaration(sf_object="Event", fields=["Subject", "WhoId"]),
+        ]
+        object_counts = {"Account": 10, "Contact": 5, "Case": 5, "Event": 2, "Lead": 1}
+        obj_describes = (
+            describe_for("Account"),
+            describe_for("Contact"),
+            describe_for("Case"),
+            describe_for("Event"),
+            describe_for("Lead"),
+        )
+        with _fake_get_org_schema(
+            org_config,
+            obj_describes,
+            object_counts,
+            filters=[],
+            include_counts=True,
+        ) as schema:
+            mf = create_load_mapping_file_from_extract_declarations(
+                declarations, schema
+            )
+            assert mf == {
+                "Insert Account": {
+                    "sf_object": "Account",
+                    "table": "Account",
+                    "fields": ["Name", "Description"],
+                    "select_options": {},
+                },
+                "Insert Contact": {
+                    "sf_object": "Contact",
+                    "table": "Contact",
+                    "fields": ["FirstName", "LastName"],
+                    "lookups": {
+                        "AccountId": {"table": ["Account"], "key_field": "AccountId"}
+                    },
+                    "select_options": {},
+                },
+                "Insert Lead": {
+                    "sf_object": "Lead",
+                    "table": "Lead",
+                    "fields": ["LastName", "Company"],
+                    "select_options": {},
+                },
+                "Insert Event": {
+                    "sf_object": "Event",
+                    "table": "Event",
+                    "fields": ["Subject"],
+                    "lookups": {
+                        "WhoId": {"table": ["Contact", "Lead"], "key_field": "WhoId"}
+                    },
+                    "select_options": {},
+                },
+            }
+
+    def test_generate_load_mapping_from_declarations__circular_lookups(
+        self, org_config
+    ):
+        declarations = [
+            ExtractDeclaration(
+                sf_object="Account",
+                fields=["Name", "Description", "Primary_Contact__c"],
+            ),
+            ExtractDeclaration(
+                sf_object="Contact", fields=["FirstName", "LastName", "AccountId"]
+            ),
+        ]
+        object_counts = {"Account": 10, "Contact": 5, "Case": 5}
+        obj_describes = (
+            describe_for("Account"),
+            describe_for("Contact"),
+            describe_for("Case"),
+        )
+        with _fake_get_org_schema(
+            org_config,
+            obj_describes,
+            object_counts,
+            filters=[],
+            include_counts=True,
+        ) as schema:
+            mf = create_load_mapping_file_from_extract_declarations(
+                declarations, schema
+            )
+            assert mf == {
+                "Insert Account": {
+                    "fields": ["Name", "Description"],
+                    "lookups": {
+                        "Primary_Contact__c": {
+                            "after": "Insert Contact",
+                            "key_field": "Primary_Contact__c",
+                            "table": ["Contact"],
+                        }
+                    },
+                    "sf_object": "Account",
+                    "table": "Account",
+                    "select_options": {},
+                },
+                "Insert Contact": {
+                    "sf_object": "Contact",
+                    "table": "Contact",
+                    "fields": ["FirstName", "LastName"],
+                    "lookups": {
+                        "AccountId": {"table": ["Account"], "key_field": "AccountId"}
+                    },
+                    "select_options": {},
+                },
+            }, mf
+
+    def test_generate_load_mapping__with_load_declarations(self, org_config):
+
+        mapping_steps = [
+            MappingStep(sf_object="Account"),
+            MappingStep(sf_object="Contact"),
+        ]
+        load_declarations = [
+            SObjectRuleDeclaration(sf_object="Account", api="rest"),
+            SObjectRuleDeclaration(sf_object="Contact", api="bulk"),
+        ]
+        mf = generate_load_mapping_file(
+            mapping_steps,
+            intertable_dependencies=set(),
+            load_declarations=load_declarations,
+        )
+        assert mf == {
+            "Insert Account": {
+                "sf_object": "Account",
+                "api": DataApi.REST,
+                "table": "Account",
+                "select_options": {},
+            },
+            "Insert Contact": {
+                "sf_object": "Contact",
+                "api": DataApi.BULK,
+                "table": "Contact",
+                "select_options": {},
+            },
+        }, mf
+
+    def test_generate_load_mapping__with_upserts(self, org_config):
+
+        mapping_steps = [
+            MappingStep(sf_object="Account"),
+            MappingStep(
+                sf_object="Account",
+                action="UPSERT",
+                update_key=["Name"],
+                fields=["Name"],
+            ),
+            MappingStep(
+                sf_object="Account",
+                action="ETL_UPSERT",
+                update_key=["AccountNumber", "Name"],
+                fields=["AccountNumber", "Name"],
+            ),
+            MappingStep(sf_object="Contact"),
+        ]
+
+        mf = generate_load_mapping_file(
+            mapping_steps,
+            intertable_dependencies=set(),
+            load_declarations=(),
+        )
+        assert mf == {
+            "Insert Account": {
+                "sf_object": "Account",
+                "table": "Account",
+                "select_options": {},
+            },
+            "Upsert Account Name": {
+                "sf_object": "Account",
+                "table": "Account",
+                "action": DataOperationType.UPSERT,
+                "update_key": ("Name",),
+                "fields": ["Name"],
+                "select_options": {},
+            },
+            "Etl_Upsert Account AccountNumber_Name": {
+                "sf_object": "Account",
+                "table": "Account",
+                "action": DataOperationType.ETL_UPSERT,
+                "update_key": ("AccountNumber", "Name"),
+                "fields": ["AccountNumber", "Name"],
+                "select_options": {},
+            },
+            "Insert Contact": {
+                "sf_object": "Contact",
+                "table": "Contact",
+                "select_options": {},
+            },
+        }, mf
+
+
+# TODO: Figure out where clauses

cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_mapping_generator_post_processes.py

@@ -0,0 +1,60 @@
+from cumulusci.tasks.bulkdata.generate_mapping_utils.mapping_generator_post_processes import (
+    add_after_statements,
+)
+from cumulusci.tasks.bulkdata.mapping_parser import MappingLookup, MappingStep
+
+
+class TestMappingGeneratorPostProcesses:
+    def test_add_after_statements(self):
+        "Test that the add_after_statements function will add an `after` statement to the correct mapping step"
+        mappings = {
+            "Insert Blah": MappingStep(sf_object="Blah"),
+            "Insert Accounts": MappingStep(
+                sf_object="Account",
+                lookups={"PrimaryContact__c": MappingLookup(table="Contact")},
+            ),
+            "Update Account": MappingStep(
+                sf_object="Account",
+                action="Update",
+            ),
+            "Insert Contact": MappingStep(
+                sf_object="Contact",
+                lookups={"AccountId": MappingLookup(table="Account")},
+            ),
+        }
+        add_after_statements(mappings)
+        assert (
+            mappings["Insert Accounts"].lookups["PrimaryContact__c"].after
+            == "Insert Contact"
+        )
+
+    def test_add_after_statements__polymorphic_lookups(self):
+        """Test that the add_after_statements function will add an `after` statement to the correct mapping step
+        for polymorphic lookups"""
+        mappings = {
+            "Insert Accounts": MappingStep(
+                sf_object="Account",
+                lookups={"PrimaryContact__c": MappingLookup(table="Contact")},
+            ),
+            "Update Account": MappingStep(
+                sf_object="Account",
+                action="Update",
+            ),
+            "Insert Contact": MappingStep(
+                sf_object="Contact",
+                lookups={"AccountId": MappingLookup(table="Account")},
+            ),
+            "Insert Events": MappingStep(
+                sf_object="Event",
+                lookups={"WhoId": MappingLookup(table=["Lead", "Contact"])},
+            ),
+            "Insert Lead": MappingStep(
+                sf_object="Lead",
+            ),
+        }
+        add_after_statements(mappings)
+        assert (
+            mappings["Insert Accounts"].lookups["PrimaryContact__c"].after
+            == "Insert Contact"
+        )
+        assert mappings["Insert Events"].lookups["WhoId"].after == "Insert Lead"

cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_mapping_transforms.py

@@ -0,0 +1,188 @@
+import random
+
+from cumulusci.tasks.bulkdata.extract_dataset_utils.calculate_dependencies import (
+    SObjDependency,
+)
+from cumulusci.tasks.bulkdata.generate_mapping_utils.dependency_map import DependencyMap
+from cumulusci.tasks.bulkdata.generate_mapping_utils.mapping_transforms import (
+    merge_matching_steps,
+    recategorize_lookups,
+    rename_record_type_fields,
+    sort_steps,
+)
+from cumulusci.tasks.bulkdata.mapping_parser import MappingStep
+
+
+class MappingTransformTesterBase:
+    @classmethod
+    def setup_class(_):
+        mttb = MappingTransformTesterBase
+        mttb.std_intertable_dependencies = [
+            SObjDependency("Contact", "Account", "AccountId"),
+            SObjDependency("Account", "Account", "ParentId"),
+            SObjDependency("Opportunity", "Contact", "ContactId"),
+            SObjDependency("Opportunity", "Account", "AccountId"),
+            SObjDependency("Account", "Custom__c", "Custom__c"),
+            SObjDependency("Custom__c", "Opportunity", "Opportunity__c"),
+            SObjDependency("Custom_2__c", "Custom__c", "Custom__c"),
+            SObjDependency("Custom_2__c", "Custom_3__c", "Custom_3__c"),
+            SObjDependency("Custom_3__c", "Custom_2__c", "Custom_2__c"),
+        ]
+        mttb.std_table_names = [
+            dep.table_name_from for dep in mttb.std_intertable_dependencies
+        ] + ["Unrelated__c"]
+
+        mttb.std_dep_map = DependencyMap(
+            mttb.std_table_names, mttb.std_intertable_dependencies
+        )
+
+
+class TestSortSteps(MappingTransformTesterBase):
+    def test_sort_steps(self):
+        steps = [
+            MappingStep(sf_object="Opportunity"),
+            MappingStep(sf_object="Custom__c"),
+            MappingStep(sf_object="Custom_2__c"),
+            MappingStep(sf_object="Custom_3__c"),
+            MappingStep(sf_object="Contact"),
+            MappingStep(sf_object="Account"),
+        ]
+
+        for i in range(0, 10):
+            random.Random(i).shuffle(steps)
+            out_steps = sort_steps(steps, self.std_dep_map)
+            enumerated = enumerate(step.sf_object for step in out_steps)
+            positions = {name: pos for (pos, name) in enumerated}
+            assert positions["Account"] < positions["Contact"], positions
+            assert positions["Account"] < positions["Opportunity"], positions
+            assert positions["Contact"] < positions["Opportunity"], positions
+            assert positions["Opportunity"] < positions["Custom__c"], positions
+            assert positions["Opportunity"] < positions["Custom__c"], positions
+            assert positions["Custom__c"] < positions["Custom_2__c"], positions
+            assert positions["Custom__c"] < positions["Custom_3__c"], positions
+
+    def test_sort_steps__with_priorities(self):
+        steps = [
+            MappingStep(sf_object="Opportunity"),
+            MappingStep(sf_object="Custom__c"),
+            MappingStep(sf_object="Custom_2__c"),
+            MappingStep(sf_object="Custom_3__c"),
+            MappingStep(sf_object="Contact"),
+            MappingStep(sf_object="Account"),
+        ]
+
+        for i in range(0, 10):
+            random.Random(i).shuffle(steps)
+            # add a high priority requirement that forces Custom_3__c to go before Account
+            deps = DependencyMap(
+                self.std_table_names,
+                self.std_intertable_dependencies
+                + [SObjDependency("Account", "Custom_3__c", "Custom_3__c", True)],
+            )
+            out_steps = sort_steps(steps, deps)
+            enumerated = enumerate(step.sf_object for step in out_steps)
+            positions = {name: pos for (pos, name) in enumerated}
+            assert positions["Custom_3__c"] < positions["Account"], positions
+            assert positions["Opportunity"] < positions["Custom__c"], positions
+            assert positions["Opportunity"] < positions["Custom__c"], positions
+            assert positions["Custom__c"] < positions["Custom_2__c"], positions
+            assert positions["Custom__c"] < positions["Custom_3__c"], positions
+
+    def test_sort_steps__with_priorities__2(self):
+        steps = [
+            MappingStep(sf_object="Opportunity"),
+            MappingStep(sf_object="Custom__c"),
+            MappingStep(sf_object="Custom_2__c"),
+            MappingStep(sf_object="Custom_3__c"),
+            MappingStep(sf_object="Contact"),
+            MappingStep(sf_object="Account"),
+        ]
+
+        for i in range(0, 10):
+            random.Random(i).shuffle(steps)
+            # add a high priority requirement that forces Custom_2__c to go before Custom_3__c
+            deps = DependencyMap(
+                self.std_table_names,
+                self.std_intertable_dependencies
+                + [SObjDependency("Custom_3__c", "Custom_2__c", "Custom_2__c", True)],
+            )
+            out_steps = sort_steps(steps, deps)
+            enumerated = enumerate(step.sf_object for step in out_steps)
+            positions = {name: pos for (pos, name) in enumerated}
+            assert positions["Custom_2__c"] < positions["Custom_3__c"], positions
+
+
+class TestMergeMatchingSteps(MappingTransformTesterBase):
+    def test_merge_matching_steps(self):
+        steps = [
+            MappingStep(sf_object="Account", fields={"recordtypeid": "recordtypeid"}),
+            MappingStep(sf_object="Account", fields={"Name": "Name"}),
+            MappingStep(sf_object="Contact", fields={"recordtype": "recordtype"}),
+            MappingStep(sf_object="Contact", fields={"description": "description"}),
+        ]
+        out_steps = merge_matching_steps(steps, self.std_dep_map)
+        by_name = {step.sf_object: step for step in out_steps}
+        assert tuple(step.sf_object for step in out_steps) == ("Account", "Contact")
+        assert tuple(by_name["Account"].fields) == ("recordtypeid", "Name")
+        assert tuple(by_name["Contact"].fields) == ("recordtype", "description")
+
+
+class TestRenameRecordTypeFields(MappingTransformTesterBase):
+    def test_rename_record_type_fields(self):
+        steps = [
+            MappingStep(
+                sf_object="Account", fields={"record_type_id": "record_type_id"}
+            ),
+            MappingStep(sf_object="Custom__c", fields={"recordtypeid": "recordtypeid"}),
+            MappingStep(sf_object="Custom_2__c", fields={"recordtype": "recordtype"}),
+            MappingStep(sf_object="Custom_3__c"),
+        ]
+        out_steps = rename_record_type_fields(steps, self.std_dep_map)
+
+        by_name = {step.sf_object: step for step in out_steps}
+        assert tuple(by_name.keys()) == (
+            "Account",
+            "Custom__c",
+            "Custom_2__c",
+            "Custom_3__c",
+        )
+
+        assert "RecordTypeId" in by_name["Account"].fields
+        assert "RecordTypeId" in by_name["Custom__c"].fields
+        assert "RecordTypeId" in by_name["Custom_2__c"].fields
+        assert "RecordTypeId" not in by_name["Custom_3__c"].fields
+
+
+class TestRecategorizeLookups(MappingTransformTesterBase):
+    def test_recategorize_lookups(self):
+        steps = [
+            MappingStep(
+                sf_object="Account",
+                fields={
+                    "ParentId": "ParentId",
+                    "Name": "Name",
+                    "Custom__c": "Custom__c",
+                },
+            ),
+            MappingStep(
+                sf_object="Contact",
+                fields={"AccountId": "AccountId", "Firstname": "Firstname"},
+            ),
+            MappingStep(
+                sf_object="Custom_2__c",
+                fields={"Custom__c": "Custom__c", "Name": "Name"},
+            ),
+            MappingStep(
+                sf_object="Custom_3__c",
+                fields={"Name": "Name"},
+            ),
+        ]
+        out_steps = recategorize_lookups(steps, self.std_dep_map)
+
+        by_name = {step.sf_object: step for step in out_steps}
+        assert tuple(by_name["Account"].fields) == ("Name",)
+        assert tuple(by_name["Account"].lookups) == ("ParentId", "Custom__c")
+        assert tuple(by_name["Contact"].fields) == ("Firstname",)
+        assert tuple(by_name["Contact"].lookups) == ("AccountId",)
+        assert tuple(by_name["Custom_2__c"].fields) == ("Name",)
+        assert tuple(by_name["Custom_2__c"].lookups) == ("Custom__c",)