cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/tasks/bulkdata/tests/test_utils.py

@@ -0,0 +1,158 @@
+import os
+from unittest import mock
+
+import responses
+from sqlalchemy import Column, Integer, MetaData, Table, Unicode, create_engine
+from sqlalchemy.orm import create_session, mapper
+
+from cumulusci.tasks import bulkdata
+from cumulusci.tasks.bulkdata.mapping_parser import parse_from_yaml
+from cumulusci.tasks.bulkdata.utils import (
+    create_table,
+    generate_batches,
+    sql_bulk_insert_from_records,
+)
+from cumulusci.utils import temporary_dir
+
+
+def create_db_file(filename):
+    """Create a SQLite file from a filename"""
+    db_url = "sqlite:///%s" % filename
+    engine = create_engine(db_url)
+    metadata = MetaData()
+    metadata.bind = engine
+    return engine, metadata
+
+
+def create_db_memory():
+    """Create a SQLite database in memory"""
+    db_url = "sqlite:///"
+    engine = create_engine(db_url)
+    metadata = MetaData()
+    metadata.bind = engine
+    return engine, metadata
+
+
+class TestSqlAlchemyMixin:
+    @mock.patch("cumulusci.tasks.bulkdata.utils.Table")
+    @mock.patch("cumulusci.tasks.bulkdata.utils.mapper")
+    def test_create_record_type_table(self, mapper, table):
+        util = bulkdata.utils.SqlAlchemyMixin()
+        util.models = {}
+        util.metadata = mock.Mock()
+
+        util._create_record_type_table("Account_rt_mapping")
+
+        assert "Account_rt_mapping" in util.models
+
+    @responses.activate
+    def test_extract_record_types(self):
+        util = bulkdata.utils.SqlAlchemyMixin()
+        util.sf = mock.Mock()
+        util.sf.query.return_value = {
+            "totalSize": 1,
+            "records": [
+                {
+                    "Id": "012000000000000",
+                    "DeveloperName": "Organization",
+                    "IsPersonType": "0",
+                }
+            ],
+        }
+        util.logger = mock.Mock()
+        util.metadata = mock.MagicMock()
+
+        conn = mock.MagicMock()
+        with mock.patch(
+            "cumulusci.tasks.bulkdata.utils.sql_bulk_insert_from_records"
+        ) as sql_bulk_insert_from_records:
+            util._extract_record_types("Account", "test_table", conn, True)
+
+        util.sf.query.assert_called_once_with(
+            "SELECT Id, DeveloperName, IsPersonType FROM RecordType WHERE SObjectType='Account'"
+        )
+        sql_bulk_insert_from_records.assert_called()
+        call = sql_bulk_insert_from_records.call_args_list[0][1]
+        assert call["connection"] == conn
+        assert call["table"] == util.metadata.tables["test_table"]
+        assert call["columns"] == ["record_type_id", "developer_name", "is_person_type"]
+        assert list(call["record_iterable"]) == [
+            ["012000000000000", "Organization", "0"]
+        ]
+
+    def test_sql_bulk_insert_from_records__sqlite(self):
+        engine, metadata = create_db_memory()
+        fields = [
+            Column("id", Integer(), primary_key=True, autoincrement=True),
+            Column("sf_id", Unicode(24)),
+        ]
+        id_t = Table("TestTable", metadata, *fields)
+        id_t.create()
+        model = type("TestModel", (object,), {})
+        mapper(model, id_t)
+
+        session = create_session(bind=engine, autocommit=False)
+        connection = session.connection()
+
+        sql_bulk_insert_from_records(
+            connection=connection,
+            table=id_t,
+            columns=("id", "sf_id"),
+            record_iterable=([f"{x}", f"00100000000000{x}"] for x in range(10)),
+        )
+
+        assert session.query(model).count() == 10
+
+
+class TestCreateTable:
+    def test_create_table_legacy_oid_mapping(self):
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_v1.yml")
+
+        content = parse_from_yaml(mapping_file)
+        account_mapping = content["Insert Contacts"]
+
+        with temporary_dir() as d:
+            tmp_db_path = os.path.join(d, "temp.db")
+
+            engine, metadata = create_db_file(tmp_db_path)
+            t = create_table(account_mapping, metadata)
+            assert t.name == "contacts"
+            assert isinstance(t.columns["sf_id"].type, Unicode)
+            assert isinstance(t.columns["first_name"].type, Unicode)
+            assert isinstance(t.columns["last_name"].type, Unicode)
+            assert isinstance(t.columns["email"].type, Unicode)
+
+    def test_create_table_modern_id_mapping(self):
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_v2.yml")
+        content = parse_from_yaml(mapping_file)
+        account_mapping = content["Insert Contacts"]
+
+        with temporary_dir() as d:
+            tmp_db_path = os.path.join(d, "temp.db")
+
+            engine, metadata = create_db_file(tmp_db_path)
+            t = create_table(account_mapping, metadata)
+            assert t.name == "contacts"
+            assert isinstance(t.columns["id"].type, Unicode)
+            assert isinstance(t.columns["first_name"].type, Unicode)
+            assert isinstance(t.columns["last_name"].type, Unicode)
+            assert isinstance(t.columns["email"].type, Unicode)
+
+
+class TestBatching:
+    def test_batching_no_remainder(self):
+        batches = list(generate_batches(num_records=20, batch_size=10))
+        assert batches == [(10, 0, 2), (10, 1, 2)], batches
+
+        batches = list(generate_batches(num_records=20, batch_size=5))
+        assert batches == [(5, 0, 4), (5, 1, 4), (5, 2, 4), (5, 3, 4)], batches
+
+        batches = list(generate_batches(num_records=3, batch_size=1))
+        assert batches == [(1, 0, 3), (1, 1, 3), (1, 2, 3)], batches
+
+        batches = list(generate_batches(num_records=3, batch_size=3))
+        assert batches == [(3, 0, 1)], batches
+
+    def test_batching_with_remainder(self):
+        batches = list(generate_batches(num_records=20, batch_size=7))
+        assert batches == [(7, 0, 3), (7, 1, 3), (6, 2, 3)], batches

cumulusci/tasks/bulkdata/tests/update_describe.py

@@ -0,0 +1,50 @@
+# generate Globals with a certain set of objects.
+
+import json
+
+import yaml
+
+from cumulusci.cli.runtime import CliRuntime
+
+filename = "cumulusci/tests/shared_cassettes/GET_sobjects_Global_describe.yaml"
+
+SELECTED_OBJS = [
+    "Account",
+    "Contact",
+    "Opportunity",
+    "Case",
+    "Campaign",
+    "CampaignMember",
+    "Organization",
+    "OpportunityContactRole",
+]
+
+
+def init_cci(orgname):
+    from cumulusci.salesforce_api.utils import get_simple_salesforce_connection
+
+    runtime = CliRuntime(load_keychain=True)
+    name, org_config = runtime.get_org(orgname)
+    sf = get_simple_salesforce_connection(runtime.project_config, org_config)
+    return sf, runtime, org_config
+
+
+sf, runtime, org_config = init_cci("qa")
+print("Org loaded")
+
+describe_data = sf.describe()
+print("Described")
+object_list = describe_data["sobjects"]
+relevant_objects = [
+    obj for obj in object_list if obj["name"] in SELECTED_OBJS
+] + object_list[-100:]
+describe_data["sobjects"] = relevant_objects
+
+with open(filename) as f:
+    yaml_cassette = yaml.safe_load(f)
+
+yaml_cassette["response"]["body"]["string"] = json.dumps(describe_data)
+
+with open(filename, "w") as f:
+    yaml.dump(yaml_cassette, f, sort_keys=False, indent=4)
+print("Done")

cumulusci/tasks/bulkdata/tests/update_person_accounts.yml

@@ -0,0 +1,23 @@
+Update Account:
+    sf_object: Account
+    table: Account
+    fields:
+        FirstName: FirstName
+        LastName: LastName
+        PersonMailingStreet: PersonMailingStreet
+        PersonMailingCity: PersonMailingCity
+        PersonMailingState: PersonMailingState
+        PersonMailingCountry: PersonMailingCountry
+        PersonMailingPostalCode: PersonMailingPostalCode
+        PersonEmail: PersonEmail
+        Phone: Phone
+        PersonMobilePhone: PersonMobilePhone
+Insert PersonContact:
+    sf_object: Contact
+    table: PersonContact
+    fields:
+        IsPersonAccount: IsPersonAccount
+    lookups:
+        AccountId:
+            table: Account
+            key_field: AccountId

cumulusci/tasks/bulkdata/tests/utils.py

@@ -0,0 +1,114 @@
+from cumulusci.core.config import BaseProjectConfig, TaskConfig, UniversalConfig
+from cumulusci.core.keychain import BaseProjectKeychain
+from cumulusci.tasks.bulkdata.step import (
+    BaseDmlOperation,
+    DataOperationJobResult,
+    DataOperationStatus,
+)
+from cumulusci.tests import util as cci_test_utils
+
+
+def _make_task(task_class, task_config, org_config=None):
+    task_config = TaskConfig(task_config)
+    universal_config = UniversalConfig()
+    project_config = BaseProjectConfig(
+        universal_config,
+        config={
+            "noyaml": True,
+            "project": {
+                "package": {"api_version": cci_test_utils.CURRENT_SF_API_VERSION}
+            },
+        },
+    )
+    keychain = BaseProjectKeychain(project_config, "")
+    project_config.set_keychain(keychain)
+    org_config = org_config or cci_test_utils.DummyOrgConfig(
+        {"instance_url": "https://example.com", "access_token": "abc123"}, "test"
+    )
+    return task_class(project_config, task_config, org_config)
+
+
+class FakeBulkAPI:
+    """Extremely simplistic mock of the bulk API
+
+    Can be improved as needed over time.
+    """
+
+    next_job_id = 0
+    next_batch_id = 0
+    endpoint = f"https://example.my.salesforce.com/services/async/{cci_test_utils.CURRENT_SF_API_VERSION}"
+
+    @classmethod
+    def create_job(cls, *args, **kwargs):
+        cls.next_job_id += 1
+        return cls.next_job_id
+
+    @classmethod
+    def create_query_job(cls, *args, **kwargs):
+        cls.next_job_id += 1
+        return cls.next_job_id
+
+    @classmethod
+    def post_batch(cls, *args, **kwargs):
+        cls.next_batch_id += 1
+        return cls.next_batch_id
+
+    def close_job(self, *args, **kwargs):
+        pass
+
+    def job_status(self, job_id):
+        return {
+            "numberBatchesCompleted": self.next_batch_id,
+            "numberBatchesTotal": self.next_batch_id,
+        }
+
+    def query(self, job_id, soql):
+        return self.post_batch()
+
+    def get_query_batch_result_ids(self, batch_id, job_id):
+        return range(0, 10)
+
+
+class FakeBulkAPIDmlOperation(BaseDmlOperation):
+    def __init__(
+        self, *, context, sobject=None, operation=None, api_options=None, fields=None
+    ):
+        super().__init__(
+            sobject=sobject,
+            operation=operation,
+            api_options=api_options,
+            context=context,
+            fields=fields,
+        )
+        self._results = []
+        self.records = []
+
+    def start(self):
+        self.job_id = "JOB"
+
+    def end(self):
+        records_processed = len(self.results)
+        self.job_result = DataOperationJobResult(
+            DataOperationStatus.SUCCESS, [], records_processed, 0
+        )
+
+    def get_prev_record_values(self, records):
+        pass
+
+    def load_records(self, records):
+        self.records.extend(records)
+
+    def select_records(self, records):
+        pass
+
+    def get_results(self):
+        return iter(self.results)
+
+    @property
+    def results(self):
+        return self._results
+
+    @results.setter
+    def results(self, results):
+        self._results = results
+        self.end()

cumulusci/tasks/bulkdata/update_data.py

@@ -0,0 +1,260 @@
+import csv
+import re
+from contextlib import contextmanager
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from snowfakery import SnowfakeryApplication, generate_data
+
+from cumulusci.core.exceptions import BulkDataException, TaskOptionsError
+from cumulusci.core.utils import (
+    process_bool_arg,
+    process_list_arg,
+    process_list_of_pairs_dict_arg,
+)
+from cumulusci.tasks.bulkdata.step import (
+    DataApi,
+    DataOperationStatus,
+    DataOperationType,
+    get_dml_operation,
+    get_query_operation,
+)
+from cumulusci.tasks.bulkdata.utils import RowErrorChecker
+from cumulusci.tasks.salesforce import BaseSalesforceApiTask
+
+
+class UpdateData(BaseSalesforceApiTask):
+    """Update records of an sObject matching a where-clause."""
+
+    task_options = {
+        "object": {
+            "description": "An SObject",
+            "required": True,
+        },
+        "where": {
+            "description": "A SOQL where-clause (without the keyword WHERE).",
+            "required": False,
+        },
+        "api": {
+            "description": "The desired Salesforce API to use, which may be 'rest', 'bulk', or "
+            "'smart' to auto-select based on record volume. The default is 'smart'.",
+            "required": False,
+        },
+        "fields": {
+            "description": "Fields to download as input to the Snowfakery recipe",
+            "required": False,
+        },
+        "recipe": {
+            "description": "Snowfakery recipe to be executed on each row",
+            "required": True,
+        },
+        "recipe_options": {
+            "required": False,
+            "description": """Pass values to override options in the format VAR1:foo,VAR2:bar
+
+            Example: --recipe_options weight:10,color:purple""",
+        },
+        "ignore_row_errors": {
+            "description": "If True, allow the operation to continue even if individual rows fail to delete."
+        },
+    }
+    row_warning_limit = 10
+
+    def _init_options(self, kwargs):
+        super()._init_options(kwargs)
+
+        self.sobject = self.options["object"]
+
+        def identifier(f):
+            return isinstance(f, str) and f.isidentifier()
+
+        if not identifier(self.sobject):
+            raise TaskOptionsError(
+                f"`object` option should be an sObject name, not {self.sobject}"
+            )
+
+        self.where = self.options.get("where")
+        self.fields = process_list_arg(self.options.get("fields", []))
+        bad_fields = [f for f in self.fields if not identifier(f)]
+        if bad_fields:
+            plural = "s" if len(bad_fields) > 1 else ""
+            verb = "are" if len(bad_fields) > 1 else "is"
+            article = "an " if len(bad_fields) == 1 else ""
+            raise TaskOptionsError(
+                f"Field{plural} `{'`,`'.join(bad_fields)}` {verb} not {article}identifier{plural}"
+            )
+
+        self.inject_namespaces = process_bool_arg(
+            self.task_options.get("inject_namespaces", True)
+        )
+        self.recipe = self.options.get("recipe")
+        self.recipe_options = process_list_of_pairs_dict_arg(
+            self.options.get("recipe_options", {})
+        )
+        self.ignore_row_errors = process_bool_arg(
+            self.options.get("ignore_row_errors", False)
+        )
+
+        try:
+            self.api = {
+                "bulk": DataApi.BULK,
+                "rest": DataApi.REST,
+                "smart": DataApi.SMART,
+            }[self.options.get("api", "smart").lower()]
+        except KeyError:
+            raise TaskOptionsError(
+                f"{self.options['api']} is not a valid value for API (valid: bulk, rest, smart)"
+            )
+
+    def _run_task(self):
+        obj = self._validate_and_inject_namespace_prefixes(
+            should_inject_namespaces=self.inject_namespaces,
+            sobjects_to_validate=[self.sobject],
+            operation_to_validate="updateable",
+        )[0]
+        fields = ["Id"] + self.fields
+        qs = self.query_objects(obj, fields)
+        if not qs:
+            self.logger.info(f"No records found, skipping update operation for {obj}")
+            return
+        with self.save_records(qs, fields) as csvfile, TemporaryDirectory() as outdir:
+            csv_out = self.generate_data(csvfile, Path(outdir))
+
+            with csv_out.open() as csv_out_open:
+                enriched_data = csv.DictReader(csv_out_open)
+                oid_index = enriched_data.fieldnames.index("Oid")
+                enriched_data.fieldnames[oid_index] = "Id"
+                ds = self.load_data(qs.job_result.records_processed, enriched_data)
+            records = ds.job_result.records_processed
+            errors = ds.job_result.total_row_errors
+
+        obj_description = self._object_description(obj).capitalize()
+        if errors:
+            self.logger.info(
+                f"{obj_description} processed ({records}). {errors} errors"
+            )
+        else:
+            self.logger.info(f"{obj_description} successfully updated ({records}).")
+
+        if errors and not self.ignore_row_errors:
+            plural = "s" if errors > 1 else ""
+            raise BulkDataException(f"{errors} update error{plural}")
+        else:
+            self.return_values = {**ds.job_result.simplify()}
+        return self.return_values
+
+    def generate_data(self, csvfile: Path, outdir: Path) -> Path:
+        generate_data(
+            yaml_file=self.recipe,
+            user_options=self.recipe_options,
+            update_input_file=csvfile,
+            update_passthrough_fields=["Oid"],
+            plugin_options={
+                "org_config": self.org_config,
+                "project_config": self.project_config,
+            },
+            output_folder=outdir,
+            output_format="csv",
+            parent_application=CumulusCIUpdatesApplication(self.logger),
+        )
+        created_csv = tuple(outdir.glob("*.csv"))
+        assert len(created_csv) == 1, "CSV was not created by Snowfakery"
+        return created_csv[0]
+
+    @contextmanager
+    def save_records(self, qs, fields):  # -> Path
+        with TemporaryDirectory() as t:
+            csvfile = Path(t) / "input.csv"
+            with csvfile.open("w") as f:
+                writer = csv.writer(f, quoting=csv.QUOTE_ALL)
+                snowfakery_fieldnames = ["Oid"] + fields[1:]
+                writer.writerow(snowfakery_fieldnames)
+                writer.writerows(qs.get_results())
+
+            yield csvfile
+
+    def query_objects(self, obj, fields):
+        query = f"SELECT {','.join(fields)} FROM {obj}"
+        if self.where:
+            query += f" WHERE {self.options['where']}"
+
+        qs = get_query_operation(
+            sobject=obj,
+            fields=fields,
+            api_options={},
+            context=self,
+            query=query,
+            api=DataApi.SMART,  # maybe this should always be bulk
+        )
+
+        self.logger.info(f"Querying for {self._object_description(obj)}")
+        qs.query()
+        if qs.job_result.status is not DataOperationStatus.SUCCESS:
+            raise BulkDataException(
+                f"Unable to query records for {obj}: {','.join(qs.job_result.job_errors)}"
+            )
+        if not qs.job_result.records_processed:
+            return None
+        return qs
+
+    def load_data(self, row_count: int, records: csv.DictReader):
+        obj = self.sobject
+        self.logger.info(f"Updating {row_count} {obj} records")
+        fieldnames = [
+            f for f in records.fieldnames if f != "id" and not f.startswith("_")
+        ]
+
+        ds = get_dml_operation(
+            sobject=obj,
+            operation=DataOperationType.UPDATE,
+            fields=fieldnames,
+            api_options={},
+            context=self,
+            api=self.api,
+            volume=row_count,
+        )
+
+        def cleanup(record):
+            return tuple(record[fieldname] for fieldname in fieldnames)
+
+        ds.start()
+        ds.load_records(cleanup(record) for record in records)
+        ds.end()
+
+        if ds.job_result.status not in [
+            DataOperationStatus.SUCCESS,
+            DataOperationStatus.ROW_FAILURE,
+        ]:
+            raise BulkDataException(
+                f"Unable to update records for {obj}: {','.join(ds.job_result.job_errors)}"
+            )
+
+        # "ignore" row errors because at this point it's too late. We're done the
+        # job entirely. Raising an exception isn't very helpful. We can raise
+        # it later.
+        error_checker = RowErrorChecker(self.logger, True, self.row_warning_limit)
+        for result in ds.get_results():
+            error_checker.check_for_row_error(result, result.id)
+        return ds
+
+    def _object_description(self, obj):
+        """Return a readable description of the object set to update."""
+        if self.where:
+            return f'{obj} objects matching "{self.options["where"]}"'
+        else:
+            return f"all {obj} objects"
+
+
+class CumulusCIUpdatesApplication(SnowfakeryApplication):
+    """Takes over Snowfakery logging so CumulusCI can control it"""
+
+    MATCHER = re.compile(r"^Created [^ ]+.(csv|json)$")
+
+    def __init__(self, logger) -> None:
+        self.logger = logger
+        super().__init__()
+
+    def echo(self, message, *args, **kwargs):
+        # skip CSV creation messages
+        if not self.MATCHER.match(message):
+            self.logger.info(message)