cumulusci-plus 5.0.21__py3-none-any.whl → 5.0.43__py3-none-any.whl

cumulusci/tasks/bulkdata/extract.py

@@ -213,7 +213,6 @@ class ExtractData(SqlAlchemyMixin, BaseSalesforceApiTask):
             IsPersonAccount_index = columns.index(mapping.fields["IsPersonAccount"])
 
             def strip_name_field(record):
-                nonlocal Name_index, IsPersonAccount_index
                 if record[IsPersonAccount_index].lower() == "true":
                     record[Name_index] = ""
                 return record

cumulusci/tasks/bulkdata/extract_dataset_utils/extract_yml.py

@@ -2,7 +2,7 @@ import re
 import typing as T
 from pathlib import Path
 
-from pydantic import Field, validator
+from pydantic.v1 import Field, validator
 
 from cumulusci.core.enums import StrEnum
 from cumulusci.tasks.bulkdata.utils import DataApi

cumulusci/tasks/bulkdata/extract_dataset_utils/synthesize_extract_declarations.py

@@ -2,7 +2,7 @@ import collections
 import re
 import typing as T
 
-from pydantic import validator
+from pydantic.v1 import validator
 
 from cumulusci.salesforce_api.org_schema import NOT_EXTRACTABLE, Field, Schema
 from cumulusci.utils.iterators import partition

cumulusci/tasks/bulkdata/extract_dataset_utils/tests/test_extract_yml.py

@@ -1,7 +1,7 @@
 from io import StringIO
 
 import pytest
-from pydantic import ValidationError
+from pydantic.v1 import ValidationError
 
 from cumulusci.tasks.bulkdata.extract_dataset_utils.extract_yml import (
     ExtractDeclaration,

cumulusci/tasks/bulkdata/generate_and_load_data.py

@@ -7,8 +7,13 @@ from sqlalchemy import MetaData, create_engine
 
 from cumulusci.core.config import TaskConfig
 from cumulusci.core.exceptions import TaskOptionsError
-from cumulusci.core.utils import import_global
+from cumulusci.core.utils import import_global, process_bool_arg
 from cumulusci.tasks.bulkdata import LoadData
+from cumulusci.tasks.bulkdata.mapping_parser import (
+    parse_from_yaml,
+    validate_and_inject_mapping,
+)
+from cumulusci.tasks.bulkdata.step import DataOperationType
 from cumulusci.tasks.bulkdata.utils import generate_batches
 from cumulusci.tasks.salesforce import BaseSalesforceApiTask
 
@@ -79,6 +84,10 @@ class GenerateAndLoadData(BaseSalesforceApiTask):
         "working_directory": {
             "description": "Store temporary files in working_directory for easier debugging."
         },
+        "validate_only": {
+            "description": "Boolean: if True, only validate the generated mapping against the org schema without loading data. "
+            "Defaults to False."
+        },
         **LoadData.task_options,
     }
     task_options["mapping"]["required"] = False
@@ -114,6 +123,7 @@ class GenerateAndLoadData(BaseSalesforceApiTask):
 
         self.working_directory = self.options.get("working_directory", None)
         self.database_url = self.options.get("database_url")
+        self.validate_only = process_bool_arg(self.options.get("validate_only", False))
 
         if self.database_url:
             engine, metadata = self._setup_engine(self.database_url)
@@ -132,6 +142,16 @@ class GenerateAndLoadData(BaseSalesforceApiTask):
             if working_directory:
                 tempdir = Path(working_directory)
                 tempdir.mkdir(exist_ok=True)
+
+            # Route to validation flow if validate_only is True
+            if self.validate_only:
+                return self._run_validation(
+                    database_url=self.database_url,
+                    tempdir=self.working_directory or tempdir,
+                    mapping_file=self.mapping_file,
+                )
+
+            # Normal data generation and loading flow
             if self.batch_size:
                 batches = generate_batches(self.num_records, self.batch_size)
             else:
@@ -186,6 +206,47 @@ class GenerateAndLoadData(BaseSalesforceApiTask):
         total_batches: int,
     ) -> dict:
         """Generate a batch in database_url or a tempfile if it isn't specified."""
+        # Setup and generate data
+        subtask_options = self._setup_and_generate_data(
+            database_url=database_url,
+            tempdir=tempdir,
+            mapping_file=mapping_file,
+            num_records=batch_size,
+            batch_index=index,
+        )
+
+        # Load the data
+        return self._dataload(subtask_options)
+
+    def _setup_engine(self, database_url):
+        """Set up the database engine"""
+        engine = create_engine(database_url)
+
+        metadata = MetaData(engine)
+        metadata.reflect()
+        return engine, metadata
+
+    def _setup_and_generate_data(
+        self,
+        *,
+        database_url: Optional[str],
+        tempdir: Union[Path, str, None],
+        mapping_file: Union[Path, str, None],
+        num_records: Optional[int],
+        batch_index: int,
+    ) -> dict:
+        """Setup database and generate data, returning subtask options with mapping.
+
+        Args:
+            database_url: Database URL or None to create temp SQLite
+            tempdir: Temporary directory for generated files
+            mapping_file: Path to mapping file or None to generate
+            num_records: Number of records to generate
+            batch_index: Current batch number
+
+        Returns:
+            dict: subtask_options with mapping file path set
+        """
         if not database_url:
             sqlite_path = Path(tempdir) / "generated_data.db"
             database_url = f"sqlite:///{sqlite_path}"
@@ -197,28 +258,91 @@ class GenerateAndLoadData(BaseSalesforceApiTask):
             "mapping": mapping_file,
             "reset_oids": False,
             "database_url": database_url,
-            "num_records": batch_size,
-            "current_batch_number": index,
+            "num_records": num_records,
+            "current_batch_number": batch_index,
             "working_directory": tempdir,
         }
 
-        # some generator tasks can generate the mapping file instead of reading it
+        # Generate mapping file if needed
         if not subtask_options.get("mapping"):
             temp_mapping = Path(tempdir) / "temp_mapping.yml"
             mapping_file = self.options.get("generate_mapping_file", temp_mapping)
             subtask_options["generate_mapping_file"] = mapping_file
+
+        # Run data generation
         self._datagen(subtask_options)
+
         if not subtask_options.get("mapping"):
-            subtask_options["mapping"] = mapping_file
-        return self._dataload(subtask_options)
+            subtask_options["mapping"] = subtask_options["generate_mapping_file"]
 
-    def _setup_engine(self, database_url):
-        """Set up the database engine"""
-        engine = create_engine(database_url)
+        return subtask_options
 
-        metadata = MetaData(engine)
-        metadata.reflect()
-        return engine, metadata
+    def _run_validation(
+        self,
+        *,
+        database_url: Optional[str],
+        tempdir: Union[Path, str, None],
+        mapping_file: Union[Path, str, None],
+    ):
+        """Run validation flow: generate data once and validate mapping.
+
+        Args:
+            database_url: Database URL or None to create temp SQLite
+            tempdir: Temporary directory for generated files
+            mapping_file: Path to mapping file or None to generate
+
+        Returns:
+            dict: return_values with validation_result
+        """
+        # Setup and generate minimal data to create mapping
+        subtask_options = self._setup_and_generate_data(
+            database_url=database_url,
+            tempdir=tempdir,
+            mapping_file=mapping_file,
+            num_records=1,  # Generate minimal data just to create mapping
+            batch_index=0,
+        )
+
+        # Validate the mapping
+        validation_result = self._validate_mapping(subtask_options)
+
+        self.return_values = {"validation_result": validation_result}
+        return self.return_values
+
+    def _validate_mapping(self, subtask_options):
+        """Validate the mapping against the org schema without loading data."""
+        mapping_file = subtask_options.get("mapping")
+        if not mapping_file:
+            raise TaskOptionsError("Mapping file path required for validation")
+
+        self.logger.info(f"Validating mapping file: {mapping_file}")
+        mapping = parse_from_yaml(mapping_file)
+
+        validation_result = validate_and_inject_mapping(
+            mapping=mapping,
+            sf=self.sf,
+            namespace=self.project_config.project__package__namespace,
+            data_operation=DataOperationType.INSERT,
+            inject_namespaces=self.options.get("inject_namespaces", False),
+            drop_missing=self.options.get("drop_missing_schema", False),
+            validate_only=True,
+        )
+
+        # Log summary message
+        self.logger.info("")
+        if validation_result and validation_result.has_errors():
+            self.logger.error("== Validation Failed ==")
+            self.logger.error(f"  Errors: {len(validation_result.errors)}")
+            if validation_result.warnings:
+                self.logger.warning(f"  Warnings: {len(validation_result.warnings)}")
+        elif validation_result and validation_result.warnings:
+            self.logger.warning("== Validation Successful (With Warnings) ==")
+            self.logger.warning(f"  Warnings: {len(validation_result.warnings)}")
+        else:
+            self.logger.info("== Validation Successful ==")
+        self.logger.info("")
+
+        return validation_result
 
     def _cleanup_object_tables(self, engine, metadata):
         """Delete all tables that do not relate to id->OID mapping"""

cumulusci/tasks/bulkdata/mapping_parser.py

@@ -7,7 +7,7 @@ from logging import getLogger
 from pathlib import Path
 from typing import IO, Any, Callable, Dict, List, Mapping, Optional, Tuple, Union
 
-from pydantic import Field, ValidationError, root_validator, validator
+from pydantic.v1 import Field, ValidationError, root_validator, validator
 from simple_salesforce import Salesforce
 from typing_extensions import Literal
 
@@ -23,6 +23,28 @@ from cumulusci.utils.yaml.model_parser import CCIDictModel
 logger = getLogger(__name__)
 
 
+class ValidationResult:
+    """Collects validation errors and warnings during mapping validation."""
+
+    def __init__(self):
+        self.errors = []
+        self.warnings = []
+
+    def add_error(self, message: str):
+        """Add an error message."""
+        self.errors.append(message)
+        logger.error(message)
+
+    def add_warning(self, message: str):
+        """Add a warning message."""
+        self.warnings.append(message)
+        logger.warning(message)
+
+    def has_errors(self) -> bool:
+        """Check if there are any errors."""
+        return len(self.errors) > 0
+
+
 class MappingLookup(CCIDictModel):
     "Lookup relationship between two tables."
     table: Union[str, List[str]]  # Support for polymorphic lookups
@@ -382,6 +404,7 @@ class MappingStep(CCIDictModel):
         strip: Optional[Callable[[str], str]],
         drop_missing: bool,
         data_operation_type: DataOperationType,
+        validation_result: Optional["ValidationResult"] = None,
     ) -> bool:
         ret = True
 
@@ -405,9 +428,11 @@ class MappingStep(CCIDictModel):
 
             if inject and self._is_injectable(f) and inject(f) not in orig_fields:
                 if f in describe and inject(f) in describe:
-                    logger.warning(
-                        f"Both {self.sf_object}.{f} and {self.sf_object}.{inject(f)} are present in the target org. Using {f}."
-                    )
+                    message = f"Both {self.sf_object}.{f} and {self.sf_object}.{inject(f)} are present in the target org. Using {f}."
+                    if validation_result:
+                        validation_result.add_warning(message)
+                    else:
+                        logger.warning(message)
 
                 f = replace_if_necessary(field_dict, f, inject(f))
             if strip:
@@ -417,9 +442,11 @@ class MappingStep(CCIDictModel):
             try:
                 new_name = describe.canonical_key(f)
             except KeyError:
-                logger.warning(
-                    f"Field {self.sf_object}.{f} does not exist or is not visible to the current user."
-                )
+                message = f"Field {self.sf_object}.{f} does not exist or is not visible to the current user."
+                if validation_result:
+                    validation_result.add_warning(message)
+                else:
+                    logger.warning(message)
             else:
                 del field_dict[f]
                 field_dict[new_name] = entry
@@ -434,9 +461,11 @@ class MappingStep(CCIDictModel):
             error_in_f = False
 
             if f not in describe:
-                logger.warning(
-                    f"Field {self.sf_object}.{f} does not exist or is not visible to the current user."
-                )
+                message = f"Field {self.sf_object}.{f} does not exist or is not visible to the current user."
+                if validation_result:
+                    validation_result.add_warning(message)
+                else:
+                    logger.warning(message)
                 error_in_f = True
             elif not self._check_field_permission(
                 describe,
@@ -446,10 +475,14 @@ class MappingStep(CCIDictModel):
                 relevant_permissions = self._get_required_permission_types(
                     relevant_operation
                 )
-                logger.warning(
+                message = (
                     f"Field {self.sf_object}.{f} does not have the correct permissions "
                     + f"{relevant_permissions} for this operation."
                 )
+                if validation_result:
+                    validation_result.add_warning(message)
+                else:
+                    logger.warning(message)
                 error_in_f = True
 
             if error_in_f:
@@ -466,6 +499,7 @@ class MappingStep(CCIDictModel):
         inject: Optional[Callable[[str], str]],
         strip: Optional[Callable[[str], str]],
         data_operation_type: DataOperationType,
+        validation_result: Optional["ValidationResult"] = None,
     ) -> bool:
         # Determine whether we need to inject or strip our sObject.
 
@@ -478,23 +512,29 @@ class MappingStep(CCIDictModel):
         try:
             self.sf_object = global_describe.canonical_key(self.sf_object)
         except KeyError:
-            logger.warning(
-                f"sObject {self.sf_object} does not exist or is not visible to the current user."
-            )
+            message = f"sObject {self.sf_object} does not exist or is not visible to the current user."
+            if validation_result:
+                validation_result.add_warning(message)
+            else:
+                logger.warning(message)
             return False
 
         # Validate our access to this sObject.
         if not self._check_object_permission(
             global_describe, self.sf_object, data_operation_type
         ):
-            logger.warning(
-                f"sObject {self.sf_object} does not have the correct permissions for {data_operation_type}."
-            )
+            message = f"sObject {self.sf_object} does not have the correct permissions for {data_operation_type}."
+            if validation_result:
+                validation_result.add_warning(message)
+            else:
+                logger.warning(message)
             return False
 
         return True
 
-    def check_required(self, fields_describe):
+    def check_required(
+        self, fields_describe, validation_result: Optional["ValidationResult"] = None
+    ):
         required_fields = set()
         for field in fields_describe:
             defaulted = (
@@ -508,9 +548,11 @@ class MappingStep(CCIDictModel):
             set(self.fields.keys()) | set(self.lookups)
         )
         if len(missing_fields) > 0:
-            logger.error(
-                f"One or more required fields are missing for loading on {self.sf_object} :{missing_fields}"
-            )
+            message = f"One or more required fields are missing for loading on {self.sf_object} :{missing_fields}"
+            if validation_result:
+                validation_result.add_error(message)
+            else:
+                logger.error(message)
             return False
         else:
             return True
@@ -523,6 +565,7 @@ class MappingStep(CCIDictModel):
         inject_namespaces: bool = False,
         drop_missing: bool = False,
         is_load: bool = False,
+        validation_result: Optional["ValidationResult"] = None,
     ):
         """Process the schema elements in this step.
 
@@ -554,7 +597,9 @@ class MappingStep(CCIDictModel):
         global_describe = CaseInsensitiveDict(
             {entry["name"]: entry for entry in sf.describe()["sobjects"]}
         )
-        if not self._validate_sobject(global_describe, inject, strip, operation):
+        if not self._validate_sobject(
+            global_describe, inject, strip, operation, validation_result
+        ):
             # Don't attempt to validate field permissions if the object doesn't exist.
             return False
 
@@ -562,16 +607,31 @@ class MappingStep(CCIDictModel):
         # By this point, we know the attribute is valid.
         describe = self.describe_data(sf)
         fields_correct = self._validate_field_dict(
-            describe, self.fields, inject, strip, drop_missing, operation
+            describe,
+            self.fields,
+            inject,
+            strip,
+            drop_missing,
+            operation,
+            validation_result,
         )
 
         lookups_correct = self._validate_field_dict(
-            describe, self.lookups, inject, strip, drop_missing, operation
+            describe,
+            self.lookups,
+            inject,
+            strip,
+            drop_missing,
+            operation,
+            validation_result,
         )
 
         if is_load:
-            # Show warning logs for unspecified required fields
-            self.check_required(describe)
+            # Check for unspecified required fields
+            required_fields_present = self.check_required(describe, validation_result)
+            # Only block if drop_missing is False, otherwise just warn
+            if not required_fields_present and not drop_missing:
+                return False
 
         if not (fields_correct and lookups_correct):
             return False
@@ -587,6 +647,7 @@ class MappingStep(CCIDictModel):
                 strip,
                 drop_missing=False,
                 data_operation_type=operation,
+                validation_result=validation_result,
             ):
                 return False
             self.update_key = tuple(update_keys.keys())
@@ -638,7 +699,11 @@ def parse_from_yaml(source: Union[str, Path, IO]) -> Dict:
     return MappingSteps.parse_from_yaml(source)
 
 
-def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce):
+def _infer_and_validate_lookups(
+    mapping: Dict,
+    sf: Salesforce,
+    validation_result: Optional["ValidationResult"] = None,
+):
     """Validate that all the lookup tables mentioned are valid references
     to the lookup. Also verify that the mapping for the tables are mentioned
     before they are mentioned in the lookups"""
@@ -673,14 +738,18 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce):
                     if sf_object in reference_to_objects:
                         target_objects.append(sf_object)
                     else:
-                        logger.error(
-                            f"The lookup {sf_object} is not a valid lookup for {lookup_name} in sf_object: {m.sf_object}"
-                        )
+                        message = f"The lookup {sf_object} is not a valid lookup for {lookup_name} in sf_object: {m.sf_object}"
+                        if validation_result:
+                            validation_result.add_error(message)
+                        else:
+                            logger.error(message)
                         fail = True
                 except KeyError:
-                    logger.error(
-                        f"The table {table} does not exist in the mapping file"
-                    )
+                    message = f"The table {table} does not exist in the mapping file"
+                    if validation_result:
+                        validation_result.add_error(message)
+                    else:
+                        logger.error(message)
                     fail = True
 
             if fail:
@@ -701,14 +770,18 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce):
                     list(sf_objects.values()).index(t) for t in target_objects
                 ]
                 if not all([target_index < idx for target_index in target_indices]):
-                    logger.error(
+                    message = (
                         f"All included target objects ({','.join(target_objects)}) for the field {m.sf_object}.{lookup_name} "
                         f"must precede {m.sf_object} in the mapping."
                     )
+                    if validation_result:
+                        validation_result.add_error(message)
+                    else:
+                        logger.error(message)
                     fail = True
                     continue
 
-    if fail:
+    if fail and validation_result is None:
         raise BulkDataException(
             "One or more relationship errors blocked the operation."
         )
@@ -723,23 +796,36 @@ def validate_and_inject_mapping(
     inject_namespaces: bool,
     drop_missing: bool,
     org_has_person_accounts_enabled: bool = False,
-):
+    validate_only: bool = False,
+) -> Optional[ValidationResult]:
     # Check if operation is load or extract
     is_load = True if data_operation == DataOperationType.INSERT else False
 
+    # Create ValidationResult if validate_only is True
+    validation_result = ValidationResult() if validate_only else None
+
     should_continue = [
         m.validate_and_inject_namespace(
-            sf, namespace, data_operation, inject_namespaces, drop_missing, is_load
+            sf,
+            namespace,
+            data_operation,
+            inject_namespaces,
+            drop_missing,
+            is_load,
+            validation_result,
         )
         for m in mapping.values()
     ]
 
     if not drop_missing and not all(should_continue):
-        raise BulkDataException(
-            "One or more schema or permissions errors blocked the operation.\n"
-            "If you would like to attempt the load regardless, you can specify "
-            "'--drop_missing_schema True' on the command option and ensure all required fields are included in the mapping file."
-        )
+        if validate_only and validation_result:
+            return validation_result
+        else:
+            raise BulkDataException(
+                "One or more schema or permissions errors blocked the operation.\n"
+                "If you would like to attempt the load regardless, you can specify "
+                "'--drop_missing_schema True' on the command option and ensure all required fields are included in the mapping file."
+            )
 
     if drop_missing:
         # Drop any steps with sObjects that are not present.
@@ -767,15 +853,22 @@ def validate_and_inject_mapping(
                     # Make sure this didn't cause the operation to be invalid
                     # by dropping a required field.
                     if not describe[field]["nillable"]:
-                        raise BulkDataException(
+                        message = (
                             f"{m.sf_object}.{field} is a required field, but the target object "
                             f"{describe[field]['referenceTo']} was removed from the operation "
                             "due to missing permissions."
                         )
+                        if validate_only and validation_result:
+                            validation_result.add_error(message)
+                            return validation_result
+                        else:
+                            raise BulkDataException(message)
 
     # Infer/validate lookups
     if is_load:
-        _infer_and_validate_lookups(mapping, sf)
+        _infer_and_validate_lookups(mapping, sf, validation_result)
+        if validate_only and validation_result:
+            return validation_result
 
     # If the org has person accounts enable, add a field mapping to track "IsPersonAccount".
     # IsPersonAccount field values are used to properly load person account records.
@@ -784,6 +877,8 @@ def validate_and_inject_mapping(
             if step["sf_object"] in ("Account", "Contact"):
                 step["fields"]["IsPersonAccount"] = "IsPersonAccount"
 
+    return validation_result
+
 
 def _inject_or_strip_name(name, transform, global_describe):
     if not transform:

cumulusci/tasks/bulkdata/select_utils.py

@@ -4,7 +4,7 @@ import re
 import typing as T
 from enum import Enum
 
-from pydantic import Field, root_validator, validator
+from pydantic.v1 import Field, root_validator, validator
 
 from cumulusci.core.enums import StrEnum
 from cumulusci.tasks.bulkdata.utils import CaseInsensitiveDict