cumulusci-plus 5.0.19__py3-none-any.whl → 5.0.35__py3-none-any.whl

cumulusci/tasks/bulkdata/snowfakery.py

@@ -140,6 +140,10 @@ class Snowfakery(BaseSalesforceApiTask):
             "description": "Boolean: should we continue loading even after running into row errors? "
             "Defaults to False."
         },
+        "validate_only": {
+            "description": "Boolean: if True, only validate the generated mapping against the org schema without loading data. "
+            "Defaults to False."
+        },
     }
 
     def _validate_options(self):
@@ -160,6 +164,7 @@ class Snowfakery(BaseSalesforceApiTask):
         self.drop_missing_schema = process_bool_arg(
             self.options.get("drop_missing_schema", False)
         )
+        self.validate_only = process_bool_arg(self.options.get("validate_only", False))
 
         loading_rules = process_list_arg(self.options.get("loading_rules")) or []
         self.loading_rules = [Path(path) for path in loading_rules if path]
@@ -230,14 +235,19 @@ class Snowfakery(BaseSalesforceApiTask):
     def _run_task(self):
         self.setup()
 
-        portions = PortionGenerator(
-            self.run_until.gap,
-            MIN_PORTION_SIZE,
-            MAX_PORTION_SIZE,
-        )
-
         working_directory = self.options.get("working_directory")
         with self.workingdir_or_tempdir(working_directory) as working_directory:
+            # Route to validation flow if validate_only is True
+            if self.validate_only:
+                return self._run_validation(working_directory)
+
+            # Normal data generation and loading flow
+            portions = PortionGenerator(
+                self.run_until.gap,
+                MIN_PORTION_SIZE,
+                MAX_PORTION_SIZE,
+            )
+
             self._setup_channels_and_queues(working_directory)
             self.logger.info(f"Working directory is {working_directory}")
 
@@ -544,32 +554,20 @@ class Snowfakery(BaseSalesforceApiTask):
 
         template_dir = Path(working_directory) / "template_1"
         template_dir.mkdir()
-        # changes here should often be reflected in
-        # data_generator_opts and data_loader_opts
 
         channel_decl = self.channel_configs[0]
 
-        plugin_options = {
-            "pid": "0",
-            "big_ids": "True",
-        }
         # if it's efficient to do the whole load in one go, let's just do that.
         if self.run_until.gap < MIN_PORTION_SIZE:
             num_records = self.run_until.gap
         else:
             num_records = 1  # smallest possible batch to get to parallelizing fast
+
+        batch_options = self._prepare_initial_batch_options(num_records)
         results = self._generate_and_load_batch(
             template_dir,
             channel_decl.org_config,
-            {
-                "generator_yaml": self.options.get("recipe"),
-                "num_records": num_records,
-                "num_records_tablename": self.run_until.sobject_name or COUNT_REPS,
-                "loading_rules": self.loading_rules,
-                "vars": channel_decl.merge_recipe_options(self.recipe_options),
-                "plugin_options": plugin_options,
-                "bulk_mode": self.bulk_mode,
-            },
+            batch_options,
         )
         self.update_running_totals_from_load_step_results(results)
 
@@ -595,9 +593,19 @@ class Snowfakery(BaseSalesforceApiTask):
 
         return template_dir, wd.relevant_sobjects()
 
-    def _generate_and_load_batch(self, tempdir, org_config, options) -> dict:
-        """Before the "full" dataload starts we do a single batch to
-        load singletons.
+    def _run_generate_and_load_subtask(
+        self, tempdir, org_config, options, validate_only=False
+    ):
+        """Run GenerateAndLoadDataFromYaml subtask with given options.
+
+        Args:
+            tempdir: Working directory for generated files
+            org_config: Org configuration
+            options: Options dict for the subtask
+            validate_only: If True, only validate mapping without loading
+
+        Returns:
+            dict: Subtask return values
         """
         options = {
             **options,
@@ -605,6 +613,7 @@ class Snowfakery(BaseSalesforceApiTask):
             "set_recently_viewed": False,
             "ignore_row_errors": self.ignore_row_errors,
             "drop_missing_schema": self.drop_missing_schema,
+            "validate_only": validate_only,
         }
         subtask_config = TaskConfig({"options": options})
         subtask = GenerateAndLoadDataFromYaml(
@@ -616,7 +625,73 @@ class Snowfakery(BaseSalesforceApiTask):
             stepnum=self.stepnum,
         )
         subtask()
-        return subtask.return_values["load_results"][0]
+        return subtask.return_values
+
+    def _prepare_initial_batch_options(self, num_records: int) -> dict:
+        """Prepare options for initial data generation batch.
+
+        Args:
+            num_records: Number of records to generate
+
+        Returns:
+            dict: Options for GenerateAndLoadDataFromYaml subtask
+        """
+        channel_decl = self.channel_configs[0]
+
+        plugin_options = {
+            "pid": "0",
+            "big_ids": "True",
+        }
+
+        return {
+            "generator_yaml": self.options.get("recipe"),
+            "num_records": num_records,
+            "num_records_tablename": self.run_until.sobject_name or COUNT_REPS,
+            "loading_rules": self.loading_rules,
+            "vars": channel_decl.merge_recipe_options(self.recipe_options),
+            "plugin_options": plugin_options,
+            "bulk_mode": self.bulk_mode,
+        }
+
+    def _run_validation(self, working_directory: Path):
+        """Run validation flow: generate minimal data and validate mapping.
+
+        Args:
+            working_directory: Working directory for generated files
+
+        Returns:
+            dict: return_values with validation_result
+        """
+        template_dir = Path(working_directory) / "template_validation"
+        template_dir.mkdir()
+
+        channel_decl = self.channel_configs[0]
+
+        # Prepare options for validation
+        batch_options = self._prepare_initial_batch_options(num_records=1)
+
+        # Run generation and validation
+        subtask_return_values = self._run_generate_and_load_subtask(
+            template_dir,
+            channel_decl.org_config,
+            batch_options,
+            validate_only=True,
+        )
+
+        # Set return values with validation result
+        self.return_values = {
+            "validation_result": subtask_return_values["validation_result"]
+        }
+        return self.return_values
+
+    def _generate_and_load_batch(self, tempdir, org_config, options) -> dict:
+        """Before the "full" dataload starts we do a single batch to
+        load singletons.
+        """
+        subtask_return_values = self._run_generate_and_load_subtask(
+            tempdir, org_config, options, validate_only=False
+        )
+        return subtask_return_values["load_results"][0]
 
     def _cleanup_object_tables(self, engine, metadata):
         """Delete all tables that do not relate to id->OID mapping"""

cumulusci/tasks/bulkdata/tests/test_generate_and_load.py

@@ -250,3 +250,162 @@ class TestGenerateAndLoadData:
                 )
                 task()
                 assert list(Path(t).glob("*"))
+
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    @mock.patch(
+        "cumulusci.tasks.bulkdata.generate_and_load_data.validate_and_inject_mapping"
+    )
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._datagen")
+    def test_validate_only_mode(self, mock_datagen, mock_validate, _dataload):
+        """Test that validate_only mode validates without loading data"""
+        from cumulusci.tasks.bulkdata.mapping_parser import ValidationResult
+
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        # Mock ValidationResult
+        validation_result = ValidationResult()
+        mock_validate.return_value = validation_result
+
+        task = _make_task(
+            GenerateAndLoadData,
+            {
+                "options": {
+                    "num_records": 12,
+                    "mapping": mapping_file,
+                    "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                    "validate_only": True,
+                }
+            },
+        )
+
+        task()
+
+        # Verify data generation was called (to create mapping)
+        mock_datagen.assert_called_once()
+
+        # Verify validation was called
+        mock_validate.assert_called_once()
+
+        # Verify load was NOT called
+        _dataload.assert_not_called()
+
+        # Verify return values contain validation_result
+        assert "validation_result" in task.return_values
+        assert task.return_values["validation_result"] == validation_result
+
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    @mock.patch(
+        "cumulusci.tasks.bulkdata.generate_and_load_data.validate_and_inject_mapping"
+    )
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._datagen")
+    def test_validate_only_with_errors(self, mock_datagen, mock_validate, _dataload):
+        """Test that validate_only mode returns errors without raising exception"""
+        from cumulusci.tasks.bulkdata.mapping_parser import ValidationResult
+
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        # Mock ValidationResult with errors
+        validation_result = ValidationResult()
+        validation_result.add_error("Test error: Field does not exist")
+        validation_result.add_warning("Test warning: Field has no permissions")
+        mock_validate.return_value = validation_result
+
+        task = _make_task(
+            GenerateAndLoadData,
+            {
+                "options": {
+                    "num_records": 12,
+                    "mapping": mapping_file,
+                    "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                    "validate_only": True,
+                }
+            },
+        )
+
+        # Should not raise exception even with errors
+        task()
+
+        # Verify data generation was called
+        mock_datagen.assert_called_once()
+
+        # Verify validation was called
+        mock_validate.assert_called_once()
+
+        # Verify load was NOT called
+        _dataload.assert_not_called()
+
+        # Verify return values contain validation_result with errors
+        assert "validation_result" in task.return_values
+        assert task.return_values["validation_result"].has_errors()
+        assert len(task.return_values["validation_result"].errors) == 1
+        assert len(task.return_values["validation_result"].warnings) == 1
+
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    def test_validate_only_false_loads_data(self, _dataload):
+        """Test that validate_only=False performs normal data loading"""
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        task = _make_task(
+            GenerateAndLoadData,
+            {
+                "options": {
+                    "num_records": 12,
+                    "mapping": mapping_file,
+                    "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                    "validate_only": False,
+                }
+            },
+        )
+
+        task()
+
+        # Verify load WAS called
+        _dataload.assert_called_once()
+
+        # Verify return values contain load_results, not validation_result
+        assert "load_results" in task.return_values
+        assert "validation_result" not in task.return_values
+
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._dataload")
+    @mock.patch(
+        "cumulusci.tasks.bulkdata.generate_and_load_data.validate_and_inject_mapping"
+    )
+    @mock.patch("cumulusci.tasks.bulkdata.GenerateAndLoadData._datagen")
+    def test_validate_only_with_working_directory(
+        self, mock_datagen, mock_validate, _dataload
+    ):
+        """Test that validate_only respects working_directory option"""
+        from cumulusci.tasks.bulkdata.mapping_parser import ValidationResult
+
+        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")
+
+        validation_result = ValidationResult()
+        mock_validate.return_value = validation_result
+
+        with TemporaryDirectory() as t:
+            task = _make_task(
+                GenerateAndLoadData,
+                {
+                    "options": {
+                        "num_records": 12,
+                        "mapping": mapping_file,
+                        "data_generation_task": "cumulusci.tasks.bulkdata.tests.dummy_data_factory.GenerateDummyData",
+                        "validate_only": True,
+                        "working_directory": t,
+                    }
+                },
+            )
+
+            task()
+
+            # Verify data generation was called
+            mock_datagen.assert_called_once()
+
+            # Verify validation was called
+            mock_validate.assert_called_once()
+
+            # Verify load was NOT called
+            _dataload.assert_not_called()
+
+            # Verify working directory was used (should have generated files)
+            assert list(Path(t).glob("*"))

cumulusci/tasks/bulkdata/tests/test_load.py

@@ -2715,8 +2715,6 @@ class TestLoadData:
         chunks_index = 0
 
         def fetchmany(batch_size):
-            nonlocal chunks_index
-
             assert 200 == batch_size
 
             # _generate_contact_id_map_for_person_accounts should break if fetchmany returns falsy.