cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/tasks/bulkdata/generate_mapping_utils/generate_mapping_from_declarations.py

@@ -0,0 +1,121 @@
+import typing as T
+from itertools import chain
+
+from snowfakery.cci_mapping_files.declaration_parser import SObjectRuleDeclaration
+
+from cumulusci.salesforce_api.org_schema import Schema
+from cumulusci.tasks.bulkdata.mapping_parser import MappingStep
+from cumulusci.utils.collections import OrderedSet
+from cumulusci.utils.iterators import partition
+
+from ..extract_dataset_utils.calculate_dependencies import SObjDependency
+from ..extract_dataset_utils.synthesize_extract_declarations import (
+    ExtractDeclaration,
+    SimplifiedExtractDeclaration,
+    flatten_declarations,
+)
+from .load_mapping_file_generator import generate_load_mapping_file
+
+
+class SimplifiedExtractDeclarationWithLookups(SimplifiedExtractDeclaration):
+    lookups: T.Dict[str, T.Union[str, T.Tuple[str, ...]]]
+
+
+def create_load_mapping_file_from_extract_declarations(
+    decls: T.Sequence[ExtractDeclaration],
+    schema: Schema,
+    opt_in_only: T.Sequence[str] = (),
+    loading_rules: T.Sequence[SObjectRuleDeclaration] = (),
+) -> T.Dict[str, dict]:
+    """Create a mapping file from Extract declarations"""
+    simplified_decls = flatten_declarations(decls, schema, opt_in_only)  # FIXME
+    simplified_decls_w_lookups = classify_and_filter_lookups(simplified_decls, schema)
+    intertable_dependencies = _discover_dependendencies(simplified_decls_w_lookups)
+
+    def _mapping_step(decl):
+        fields = tuple(chain(decl.fields, decl.lookups.keys()))
+        return MappingStep(
+            sf_object=decl.sf_object,
+            fields=dict(zip(fields, fields)),
+            # lookups=lookups,      # lookups can be re-created later, for simplicity
+        )
+
+    mapping_steps = [_mapping_step(decl) for decl in simplified_decls_w_lookups]
+
+    mappings = generate_load_mapping_file(
+        mapping_steps, intertable_dependencies, loading_rules
+    )
+    return mappings
+
+
+def _discover_dependendencies(simplified_decls: T.Sequence):
+    """Look at all of the lookups in a set of declarations to determine
+    what depends on what"""
+    intertable_dependencies = OrderedSet()
+
+    for decl in simplified_decls:
+        for fieldname, tablenames in decl.lookups.items():
+            intertable_dependencies.add(
+                SObjDependency(decl.sf_object, tablenames, fieldname)
+            )
+    return intertable_dependencies
+
+
+def classify_and_filter_lookups(
+    decls: T.Sequence[SimplifiedExtractDeclaration], schema: Schema
+) -> T.Sequence[SimplifiedExtractDeclarationWithLookups]:
+    """Move lookups into their own field, if they reference a table we're including"""
+    referenceable_tables = [decl.sf_object for decl in decls if decl.sf_object]
+    # the if statement above is just to shut up the type checker
+    return [_add_lookups_to_decl(decl, schema, referenceable_tables) for decl in decls]
+
+
+def _add_lookups_to_decl(
+    decl: SimplifiedExtractDeclaration,
+    schema: Schema,
+    referenceable_tables: T.Sequence[str],
+) -> SimplifiedExtractDeclarationWithLookups:
+    """Look at every declaration and check whether any of the fields it refers to
+    are actually lookups. If so, synthesize the lookups declarations."""
+    sobject_schema_info = schema[decl.sf_object]
+    fields, lookups_and_targets = _fields_and_lookups_for_decl(
+        decl, sobject_schema_info, referenceable_tables
+    )
+    new_decl_data = {
+        **dict(decl),
+        "fields": list(fields),
+        "lookups": dict(lookups_and_targets),
+    }
+    del new_decl_data["fields_"]
+    new_decl = SimplifiedExtractDeclarationWithLookups(**new_decl_data)
+    return new_decl
+
+
+def _fields_and_lookups_for_decl(decl, sobject_schema_info, referenceable_tables):
+    """Split fields versus lookups for a declaration"""
+
+    def is_lookup(field_name):
+        # Record types are not treated as lookup.
+        if field_name == "RecordTypeId":
+            return False
+        schema_info_for_field = sobject_schema_info.fields[field_name]
+        target = schema_info_for_field.referenceTo
+        return target
+
+    simple_fields, lookups = partition(is_lookup, decl.fields)
+
+    def target_table(field_info):
+        target = field_info.referenceTo
+        return target
+
+    lookups = list(lookups)
+
+    lookups_and_targets = (
+        (lookup, target_table(sobject_schema_info.fields[lookup])) for lookup in lookups
+    )
+    lookups_and_targets = (
+        (lookup, [table for table in tables if table in referenceable_tables])
+        for lookup, tables in lookups_and_targets
+        if any(table in referenceable_tables for table in tables)
+    )
+    return simple_fields, lookups_and_targets

cumulusci/tasks/bulkdata/generate_mapping_utils/load_mapping_file_generator.py

@@ -0,0 +1,127 @@
+import typing as T
+
+from snowfakery.cci_mapping_files.declaration_parser import SObjectRuleDeclaration
+
+from cumulusci.tasks.bulkdata.extract_dataset_utils.calculate_dependencies import (
+    SObjDependency,
+)
+from cumulusci.tasks.bulkdata.generate_mapping_utils.dependency_map import DependencyMap
+from cumulusci.tasks.bulkdata.generate_mapping_utils.mapping_transforms import (
+    merge_matching_steps,
+    recategorize_lookups,
+    rename_record_type_fields,
+    sort_steps,
+)
+from cumulusci.tasks.bulkdata.mapping_parser import MappingStep
+from cumulusci.utils.collections import OrderedSet, OrderedSetType
+
+# from ..extract_dataset_utils.extract_yml import ExtractDeclaration
+from .mapping_generator_post_processes import add_after_statements
+
+# Note that the code in this file is also used by Snowfakery, so
+# changes need to be coordinated.
+#
+# It is also intended to someday be used by CCI to generate
+# Load Mapping Files from Load Declarations.
+
+
+# Code adapted from Snowfakery
+def generate_load_mapping_file(
+    mapping_steps: T.Sequence[MappingStep],
+    intertable_dependencies: OrderedSetType[SObjDependency],
+    load_declarations: T.Sequence[SObjectRuleDeclaration] = (),
+) -> T.Dict[str, dict]:
+    """Generate a mapping file in optimal order with forward references etc.
+
+    Input is a set of a tables, dependencies between them and load declarations"""
+
+    load_declarations = list(load_declarations) or []
+    declared_dependencies = collect_user_specified_dependencies(load_declarations)
+    table_names = OrderedSet(mapping.sf_object for mapping in mapping_steps)
+    depmap = DependencyMap(
+        table_names, intertable_dependencies.union(declared_dependencies)
+    )
+
+    # Merge similar steps (output step count will be equal or less)
+    mapping_steps = merge_matching_steps(mapping_steps, depmap)
+
+    # Sort steps according to dependencies
+    mapping_steps = sort_steps(mapping_steps, depmap)
+
+    # Normalize various forms of record type fields
+    mapping_steps = rename_record_type_fields(mapping_steps, depmap)
+
+    # Detect which fields are lookups and properly set them up
+    mapping_steps = recategorize_lookups(mapping_steps, depmap)
+
+    # Apply user-specified declarations
+    mapping_steps = apply_declarations(mapping_steps, load_declarations)
+
+    mappings_dict = mappings_as_dicts(mapping_steps, depmap)
+    add_after_statements(mappings_dict)
+    return mappings_dict
+
+    # note that it is entirely possible for e.g.
+    # M=10 input mapping steps to relate to N=3 tables which generate P=7 load steps.
+    #
+    # This invariiant will always hold
+    #
+    # N <= P <= M
+    #
+    # e.g. 10 Snowfakery Templates generating Accounts, Contacts, Opportunities
+    # But Accounts have 5 different update keys and the other two have none.
+
+
+def collect_user_specified_dependencies(
+    declarations: T.Iterable[SObjectRuleDeclaration],
+) -> OrderedSetType[SObjDependency]:
+    """The user can specify load order dependencies."""
+    declared_dependencies = [decl for decl in declarations if decl.load_after]
+    return OrderedSet(
+        [
+            SObjDependency(decl.sf_object, decl.load_after, "DECLARED", True)
+            for decl in declared_dependencies
+        ]
+    )
+
+
+def mappings_as_dicts(
+    load_steps: T.List[MappingStep],
+    depmap: DependencyMap,
+) -> T.Mapping[str, T.Dict]:
+    """Generate mapping.yml data structures."""
+    mappings = {}
+    for mapping_step in load_steps:
+        step_name = f"{mapping_step.action.value.title()} {mapping_step.sf_object}"
+        # Will be used by Snowfakery to generate Upsert mapping files
+        if mapping_step.update_key:
+            step_name += f" {'_'.join(mapping_step.update_key)}"
+
+        assert step_name not in mappings
+
+        mappings[step_name] = mapping_step.dict(
+            by_alias=True,
+            exclude_defaults=True,
+        )
+    return mappings
+
+
+def apply_declarations(
+    mapping_steps: T.List[MappingStep],
+    load_declarations: T.List[SObjectRuleDeclaration] = None,
+) -> T.List[MappingStep]:
+    """Apply user-specified declarations.
+    Will be used when this code is used to generate load mappings
+    from schemas and SQL instead of from ExtractDeclarations."""
+
+    load_declarations = {decl.sf_object: decl for decl in load_declarations}
+
+    def doit(mapping_step: MappingStep) -> MappingStep:
+        if sobject_declarations := load_declarations.get(mapping_step.sf_object):
+            return MappingStep(
+                **{**mapping_step.dict(), **sobject_declarations.as_mapping()}
+            )
+        else:
+            return mapping_step
+
+    return [doit(step) for step in mapping_steps]

cumulusci/tasks/bulkdata/generate_mapping_utils/mapping_generator_post_processes.py

@@ -0,0 +1,53 @@
+from typing import NamedTuple
+
+
+def add_after_statements(mappings: dict):
+    """Automatically add CCI after: statements to the lookups
+    in a mapping file"""
+    indexed_by_sobject = _index_by_sobject(mappings)
+
+    for idx, (_, mapping) in enumerate(mappings.items()):
+        for lookup in mapping.get("lookups", {}).values():
+            target_table = lookup["table"]
+            # SnowfakeryPersonAccounts: Add this back in when Snowfakery is integrated with this code.
+            # PersonContacts are not real, so skip them
+            if target_table == "PersonContact":  # pragma: no cover
+                raise NotImplementedError(
+                    "This code is not yet tested for use with Snowfakery and Person Accounts"
+                )
+                # continue
+            if isinstance(target_table, list):
+                target_mapping_index = max(
+                    [indexed_by_sobject[table] for table in target_table],
+                    key=lambda index: index.first_instance,
+                )
+            else:
+                target_mapping_index = indexed_by_sobject[target_table]
+
+            if target_mapping_index.first_instance >= idx:
+                if not lookup.get("after"):
+                    lookup["after"] = target_mapping_index.last_step_name
+
+
+class MappingIndex(NamedTuple):  # info needed by the algorithm above
+    first_instance: int  # where was the first time this sobj was referenced?
+    last_step_name: str  # where was the last (so far)?
+
+
+def _index_by_sobject(mappings):
+    """Figure out the order in which sobjects are loaded and assign an index to each sobject
+
+    We can use this to detect sobjects out-of-order which require 'after' declarations."""
+    indexed_by_sobject = {}
+    for idx, (mapping_name, mapping) in enumerate(mappings.items()):
+        # make an index of the order of objects
+        sobject = mapping["sf_object"]
+        existing_index = indexed_by_sobject.get(sobject)
+
+        if existing_index:
+            new_mi = MappingIndex(existing_index.first_instance, mapping_name)
+        else:
+            new_mi = MappingIndex(idx, mapping_name)
+        indexed_by_sobject[sobject] = new_mi
+
+    return indexed_by_sobject

cumulusci/tasks/bulkdata/generate_mapping_utils/mapping_transforms.py

@@ -0,0 +1,139 @@
+import itertools
+import typing as T
+
+from snowfakery.salesforce import find_record_type_column
+
+from cumulusci.tasks.bulkdata.mapping_parser import MappingStep
+from cumulusci.tasks.bulkdata.step import DataOperationType
+from cumulusci.utils.yaml.model_parser import CCIModel
+
+"""A mapping transform is a function with the signature:
+
+def func(mapping_steps: T.List[MappingStep], depmap: "DependencyMap") -> T.List[MappingStep]:
+
+It should be idempotent and a pure function (don't mutate arguments).
+
+Ideally the output should be as similar to the input as possible (i.e.
+change what you need to change and leave the rest as it was). In particular,
+the order of the input steps should be preserved unless reordering them
+is the specific task of the transform.
+"""
+
+from cumulusci.tasks.bulkdata.generate_mapping_utils.dependency_map import DependencyMap
+
+
+def sort_steps(
+    mapping_steps: T.List[MappingStep], depmap: DependencyMap
+) -> T.List[MappingStep]:
+    """Sort mapping declaration steps in dependency order"""
+    table_order = depmap.get_dependency_order()
+
+    return sorted(mapping_steps, key=lambda step: table_order.index(step.sf_object))
+
+
+class _MappingStepSignature(CCIModel):
+    index: int
+    sf_object: T.Optional[str]
+    filters: T.Optional[T.Tuple[str]] = ()
+    soql_filter: T.Optional[str] = None  # soql_filter property
+    action: DataOperationType
+    update_key: T.Union[str, T.Tuple[str, ...]] = ()  # only for upserts
+
+
+def merge_matching_steps(
+    steps: T.Iterable[MappingStep], depmap: DependencyMap
+) -> T.List[MappingStep]:
+    """Merge mapping declaration steps with similar properties"""
+    table_order = {obj.sf_object: idx for idx, obj in enumerate(steps)}
+    grouped_steps = itertools.groupby(
+        steps,
+        lambda step: _MappingStepSignature(
+            index=table_order[step.sf_object],
+            sf_object=step.sf_object,
+            filters=tuple(step.filters) or None,
+            soql_filter=step.soql_filter,
+            action=step.action,
+            update_key=step.update_key,
+        ),
+    )
+
+    new_steps = [_merge_steps(steps) for _group, steps in grouped_steps]
+    return new_steps
+
+
+def _merge_steps(steps: T.Iterable[MappingStep]) -> MappingStep:
+    """Merge a groupp of matching steps together"""
+    steps = tuple(steps)
+    new_props = steps[0].dict()
+    new_props["fields"] = {}
+    new_props["lookups"] = {}
+    for step in steps:
+        new_props["fields"].update(step.fields)
+        new_props["lookups"].update(step.lookups)
+    return MappingStep(**new_props)
+
+
+def rename_record_type_fields(
+    mapping_steps: T.List[MappingStep], depmap: DependencyMap
+) -> T.List[MappingStep]:
+    """Rename fields like recordtype, recordtypeid, recordtype_id to RecordTypeId"""
+
+    def doit(mapping_step: MappingStep):
+        table_name = mapping_step.sf_object
+        record_type_col = find_record_type_column(table_name, mapping_step.fields)
+        if record_type_col:
+            fields = mapping_step.fields.copy()
+            fields["RecordTypeId"] = fields.pop(record_type_col)
+            return MappingStep(**{**mapping_step.dict(), "fields": fields})
+        else:
+            return mapping_step
+
+    return list(map(doit, mapping_steps))
+
+
+def recategorize_lookups(
+    mapping_steps: T.List[MappingStep], depmap: DependencyMap
+) -> T.List[MappingStep]:
+    """Categorize fields which are actually lookups
+
+    Put them in mapping_step.lookup and remove them from mapping_step.fields"""
+
+    def doit(mapping_step: MappingStep):
+        table_name = mapping_step.sf_object
+        lookups = mapping_step.lookups.copy() or {}
+        lookups.update(
+            {
+                fieldname: {
+                    "table": depmap.target_table_for(table_name, fieldname),
+                    "key_field": fieldname,
+                }
+                for fieldname in mapping_step.fields
+                if depmap.target_table_for(table_name, fieldname)
+            }
+        )
+
+        if lookups:
+            fields = [
+                field for field in mapping_step.fields if field not in lookups.keys()
+            ]
+            return MappingStep(
+                **{**mapping_step.dict(), "lookups": lookups, "fields": fields}
+            )
+        else:
+            return mapping_step
+
+    return list(map(doit, mapping_steps))
+
+
+# SnowfakeryPersonAccounts: This will be turned on when Snowfakery is
+# integrated with this code.
+#
+# def change_person_contact_sf_object(
+#     mapping_steps: T.List[MappingStep], depmap: DependencyMap
+# ) -> T.List[MappingStep]:
+#     def doit(mapping_step: MappingStep):
+
+#         if mapping_step.table_name == "PersonContact":
+#             mapping_step.sf_object = "Contact"
+
+#     return list(map(doit, mapping_steps))

cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_extract_mapping_from_declarations.py

@@ -0,0 +1,135 @@
+from cumulusci.tasks.bulkdata.extract_dataset_utils.extract_yml import (
+    ExtractDeclaration,
+)
+from cumulusci.tasks.bulkdata.extract_dataset_utils.tests.test_synthesize_extract_declarations import (
+    _fake_get_org_schema,
+    describe_for,
+)
+from cumulusci.tasks.bulkdata.generate_mapping_utils.extract_mapping_file_generator import (
+    create_extract_mapping_file_from_declarations,
+)
+
+
+class TestGenerateLoadMappingFromDeclarations:
+    def test_simple_generate_mapping_from_declarations(self, org_config):
+        declarations = [
+            ExtractDeclaration(sf_object="Account", fields=["Name", "Description"])
+        ]
+        object_counts = {"Account": 10, "Contact": 0, "Case": 5}
+        obj_describes = (
+            describe_for("Account"),
+            describe_for("Contact"),
+            describe_for("Case"),
+        )
+        with _fake_get_org_schema(
+            org_config,
+            obj_describes,
+            object_counts,
+            filters=[],
+            include_counts=True,
+        ) as schema:
+            mf = create_extract_mapping_file_from_declarations(declarations, schema, ())
+            assert mf == {
+                "Extract Account": {
+                    "api": "smart",
+                    "sf_object": "Account",
+                    "fields": ["Name", "Description"],
+                    "soql_filter": "Name != 'Sample Account for " "Entitlements'",
+                }
+            }
+
+    def test_generate_mapping_from_declarations__lookups(self, org_config):
+        declarations = [
+            ExtractDeclaration(sf_object="Account", fields=["Name", "Description"]),
+            ExtractDeclaration(sf_object="Contact", fields=["LastName", "AccountId"]),
+        ]
+        object_counts = {"Account": 10, "Contact": 5, "Case": 5}
+        obj_describes = (
+            describe_for("Account"),
+            describe_for("Contact"),
+            describe_for("Case"),
+        )
+        with _fake_get_org_schema(
+            org_config,
+            obj_describes,
+            object_counts,
+            filters=[],
+            include_counts=True,
+        ) as schema:
+            mf = create_extract_mapping_file_from_declarations(declarations, schema, ())
+            print(mf)
+            assert mf == {
+                "Extract Account": {
+                    "api": "smart",
+                    "sf_object": "Account",
+                    "soql_filter": "Name != 'Sample Account for Entitlements'",
+                    "fields": ["Name", "Description"],
+                },
+                "Extract Contact": {
+                    "api": "smart",
+                    "sf_object": "Contact",
+                    "fields": ["LastName"],
+                    "lookups": {"AccountId": {"table": ("Account",)}},
+                },
+            }
+
+    def test_generate_mapping_from_declarations__polymorphic_lookups(self, org_config):
+        """Generate correct mapping file for sobjects with polymorphic lookup fields"""
+        declarations = [
+            ExtractDeclaration(sf_object="Account", fields=["Name", "Description"]),
+            ExtractDeclaration(sf_object="Contact", fields=["LastName", "AccountId"]),
+            ExtractDeclaration(sf_object="Event", fields=["Subject", "WhoId"]),
+            ExtractDeclaration(sf_object="Lead", fields=["LastName", "Company"]),
+        ]
+        object_counts = {"Account": 10, "Contact": 5, "Case": 5, "Event": 1, "Lead": 2}
+        obj_describes = (
+            describe_for("Account"),
+            describe_for("Contact"),
+            describe_for("Case"),
+            describe_for("Lead"),
+            describe_for("Event"),
+        )
+        with _fake_get_org_schema(
+            org_config,
+            obj_describes,
+            object_counts,
+            filters=[],
+            include_counts=True,
+        ) as schema:
+            mf = create_extract_mapping_file_from_declarations(declarations, schema, ())
+            print(mf)
+            assert mf == {
+                "Extract Account": {
+                    "api": "smart",
+                    "sf_object": "Account",
+                    "soql_filter": "Name != 'Sample Account for Entitlements'",
+                    "fields": ["Name", "Description"],
+                },
+                "Extract Contact": {
+                    "api": "smart",
+                    "sf_object": "Contact",
+                    "fields": ["LastName"],
+                    "lookups": {"AccountId": {"table": ("Account",)}},
+                },
+                "Extract Lead": {
+                    "api": "smart",
+                    "sf_object": "Lead",
+                    "fields": ["LastName", "Company"],
+                },
+                "Extract Event": {
+                    "api": "smart",
+                    "sf_object": "Event",
+                    "fields": ["Subject"],
+                    "lookups": {
+                        "WhoId": {
+                            "table": (
+                                "Contact",
+                                "Lead",
+                            )
+                        }
+                    },
+                },
+            }
+
+
+# TODO: Figure out where clauses