cumulusci-plus 5.0.21__py3-none-any.whl → 5.0.43__py3-none-any.whl

cumulusci/tasks/sfdmu/sfdmu.py

@@ -0,0 +1,376 @@
+"""SFDmu task for CumulusCI."""
+
+import os
+import shutil
+
+import sarge
+
+from cumulusci.core.exceptions import TaskOptionsError
+from cumulusci.core.sfdx import sfdx
+from cumulusci.core.tasks import BaseSalesforceTask
+from cumulusci.core.utils import determine_managed_mode
+from cumulusci.tasks.command import Command
+
+
+class SfdmuTask(BaseSalesforceTask, Command):
+    """Execute SFDmu data migration with namespace injection support."""
+
+    salesforce_task = (
+        False  # Override to False since we manage our own org requirements
+    )
+
+    task_options = {
+        "source": {
+            "description": "Source org name (CCI org name like dev, beta, qa, etc.) or 'csvfile'",
+            "required": True,
+        },
+        "target": {
+            "description": "Target org name (CCI org name like dev, beta, qa, etc.) or 'csvfile'",
+            "required": True,
+        },
+        "path": {
+            "description": "Path to folder containing export.json and other CSV files",
+            "required": True,
+        },
+        "additional_params": {
+            "description": "Additional parameters to append to the sf sfdmu command (e.g., '--simulation --noprompt --nowarnings')",
+            "required": False,
+        },
+        "return_always_success": {
+            "description": "If True, the task will return success (exit code 0) even if SFDMU fails. A warning will be logged instead of raising an error.",
+            "required": False,
+            "default": False,
+        },
+    }
+
+    def _init_options(self, kwargs):
+        super()._init_options(kwargs)
+
+        # Convert path to absolute path
+        if "path" in self.options and self.options["path"]:
+            if os.path.isabs(self.options["path"]):
+                # Path is already absolute, normalize it
+                self.options["path"] = os.path.abspath(self.options["path"])
+            else:
+                # Path is relative, join with repo_root
+                repo_root = self.project_config.repo_root
+                if not repo_root:
+                    raise TaskOptionsError(
+                        "Cannot resolve relative path: no repository root found"
+                    )
+                self.options["path"] = os.path.abspath(
+                    os.path.join(repo_root, self.options["path"])
+                )
+
+        # Validate that the path exists and contains export.json
+        if not os.path.exists(self.options["path"]):
+            raise TaskOptionsError(f"Path {self.options['path']} does not exist")
+
+        export_json_path = os.path.join(self.options["path"], "export.json")
+        if not os.path.exists(export_json_path):
+            raise TaskOptionsError(f"export.json not found in {self.options['path']}")
+
+    def _validate_org(self, org_name):
+        """Validate that a CCI org exists and return the org config."""
+        if org_name == "csvfile":
+            return None
+
+        try:
+            if self.project_config.keychain is None:
+                raise TaskOptionsError("No keychain available")
+            org_config = self.project_config.keychain.get_org(org_name)
+            return org_config
+        except Exception as e:
+            raise TaskOptionsError(f"Org '{org_name}' does not exist: {str(e)}")
+
+    def _get_sf_org_name(self, org_config):
+        """Get the SF org name from org config."""
+        if hasattr(org_config, "sfdx_alias") and org_config.sfdx_alias:
+            return org_config.sfdx_alias
+        elif hasattr(org_config, "username") and org_config.username:
+            return org_config.username
+        else:
+            raise TaskOptionsError("Could not determine SF org name for org config")
+
+    def _create_execute_directory(self, base_path):
+        """Create /execute directory and copy files from base_path."""
+        execute_path = os.path.join(base_path, "execute")
+
+        # Remove existing execute directory if it exists
+        if os.path.exists(execute_path):
+            shutil.rmtree(execute_path)
+
+        # Create execute directory
+        os.makedirs(execute_path, exist_ok=True)
+
+        # Copy only files (not directories) from base_path to execute
+        for item in os.listdir(base_path):
+            item_path = os.path.join(base_path, item)
+            if os.path.isfile(item_path) and item.endswith((".json", ".csv")):
+                shutil.copy2(item_path, execute_path)
+
+        return execute_path
+
+    def _update_credentials(self):
+        """Override to handle cases where org_config might be None."""
+        # Only update credentials if we have an org_config
+        if self.org_config is not None:
+            super()._update_credentials()
+
+    def _inject_namespace_tokens(
+        self, execute_path, source_org_config, target_org_config
+    ):
+        """Inject namespace tokens into files in execute directory using the same mechanism as Deploy task."""
+        # Determine which org config to use for namespace injection
+        # When exporting (source=org, target=csvfile), use source org
+        # When importing (source=csvfile, target=org), use target org
+        # When transferring (source=org, target=org), use target org
+        org_config_for_injection = (
+            target_org_config if target_org_config is not None else source_org_config
+        )
+
+        if (
+            org_config_for_injection is None
+        ):  # both source and target are csvfile (unlikely but handle it)
+            return
+
+        # Get namespace information
+        namespace = self.project_config.project__package__namespace
+        managed = determine_managed_mode(
+            self.options, self.project_config, org_config_for_injection
+        )
+        namespaced_org = bool(namespace) and namespace == getattr(
+            org_config_for_injection, "namespace", None
+        )
+
+        # Create a temporary zipfile with all files from execute directory
+        import tempfile
+        import zipfile
+
+        from cumulusci.core.dependencies.utils import TaskContext
+        from cumulusci.core.source_transforms.transforms import (
+            NamespaceInjectionOptions,
+            NamespaceInjectionTransform,
+        )
+
+        with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as temp_zip:
+            temp_zip_path = temp_zip.name
+
+        try:
+            # Create zipfile with all files from execute directory
+            with zipfile.ZipFile(temp_zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+                for root, dirs, files in os.walk(execute_path):
+                    for file in files:
+                        if file.endswith((".json", ".csv")):
+                            file_path = os.path.join(root, file)
+                            # Calculate relative path from execute_path
+                            rel_path = os.path.relpath(file_path, execute_path)
+                            zf.write(file_path, rel_path)
+
+            # Apply namespace injection using the same mechanism as Deploy task
+            with zipfile.ZipFile(temp_zip_path, "r") as zf:
+                # Create namespace injection options
+                options = NamespaceInjectionOptions(
+                    namespace_tokenize=None,
+                    namespace_inject=namespace,
+                    namespace_strip=None,
+                    unmanaged=not managed,
+                    namespaced_org=namespaced_org,
+                )
+
+                # Create transform
+                transform = NamespaceInjectionTransform(options)
+
+                # Create task context
+                context = TaskContext(
+                    org_config_for_injection, self.project_config, self.logger
+                )
+
+                # Apply namespace injection
+                new_zf = transform.process(zf, context)
+
+                # Extract processed files back to execute directory
+                # First, remove all existing files
+                for root, dirs, files in os.walk(execute_path):
+                    for file in files:
+                        if file.endswith((".json", ".csv")):
+                            os.remove(os.path.join(root, file))
+
+                # Extract processed files
+                for file_info in new_zf.infolist():
+                    if file_info.filename.endswith((".json", ".csv")):
+                        # Extract to execute directory
+                        target_path = os.path.join(execute_path, file_info.filename)
+                        # Ensure directory exists
+                        os.makedirs(os.path.dirname(target_path), exist_ok=True)
+                        with new_zf.open(file_info) as source:
+                            with open(target_path, "wb") as target:
+                                target.write(source.read())
+
+                        self.logger.info(
+                            f"Applied namespace injection to {file_info.filename}"
+                        )
+
+        finally:
+            # Clean up temporary zipfile
+            if os.path.exists(temp_zip_path):
+                os.unlink(temp_zip_path)
+
+    def _process_csv_exports(self, execute_path, base_path):
+        """Process CSV files when target is csvfile.
+
+        This method performs the following operations:
+        1. Replace namespace prefix with %%%MANAGED_OR_NAMESPACED_ORG%%% in CSV file contents
+        2. Rename CSV files replacing namespace prefix with ___MANAGED_OR_NAMESPACED_ORG___
+        3. Copy all CSV files from execute folder to base path, replacing existing files
+        """
+        namespace = self.project_config.project__package__namespace
+        if not namespace:
+            self.logger.info("No namespace configured, skipping CSV post-processing")
+            return
+
+        namespace_prefix = namespace + "__"
+        content_token = "%%%MANAGED_OR_NAMESPACED_ORG%%%"
+        filename_token = "___MANAGED_OR_NAMESPACED_ORG___"
+
+        # Get all CSV files in execute directory
+        csv_files = [f for f in os.listdir(execute_path) if f.endswith(".csv")]
+
+        if not csv_files:
+            self.logger.info("No CSV files found in execute directory")
+            return
+
+        self.logger.info(f"Processing {len(csv_files)} CSV file(s) for export")
+
+        # Process each CSV file
+        processed_files = []
+        for filename in csv_files:
+            file_path = os.path.join(execute_path, filename)
+
+            # Step 1: Replace namespace prefix in file contents
+            with open(file_path, "r", encoding="utf-8") as f:
+                content = f.read()
+
+            if namespace_prefix in content:
+                content = content.replace(namespace_prefix, content_token)
+                with open(file_path, "w", encoding="utf-8") as f:
+                    f.write(content)
+                self.logger.debug(f"Replaced namespace prefix in content of {filename}")
+
+            # Step 2: Rename file if it contains namespace prefix
+            new_filename = filename.replace(namespace_prefix, filename_token)
+            if new_filename != filename:
+                new_file_path = os.path.join(execute_path, new_filename)
+                os.rename(file_path, new_file_path)
+                self.logger.debug(f"Renamed file: {filename} -> {new_filename}")
+                file_path = new_file_path
+                filename = new_filename
+
+            processed_files.append((file_path, filename))
+
+        # Step 3: Delete all CSV files in base_path and copy processed files
+        self.logger.debug(f"Copying processed CSV files to {base_path}")
+
+        # Remove existing CSV files in base_path
+        for item in os.listdir(base_path):
+            if item.endswith(".csv"):
+                item_path = os.path.join(base_path, item)
+                if os.path.isfile(item_path):
+                    os.remove(item_path)
+                    self.logger.debug(f"Removed existing file: {item}")
+
+        # Copy processed files to base_path
+        for file_path, filename in processed_files:
+            target_path = os.path.join(base_path, filename)
+            shutil.copy2(file_path, target_path)
+            self.logger.debug(f"Copied {filename} to {base_path}")
+
+        self.logger.info("CSV post-processing completed successfully")
+
+    def _run_task(self):
+        """Execute the SFDmu task."""
+        # Validate source and target orgs
+        source_org_config = self._validate_org(self.options["source"])
+        target_org_config = self._validate_org(self.options["target"])
+
+        # Get SF org names
+        if source_org_config:
+            source_sf_org = self._get_sf_org_name(source_org_config)
+        else:
+            source_sf_org = "csvfile"
+
+        if target_org_config:
+            target_sf_org = self._get_sf_org_name(target_org_config)
+        else:
+            target_sf_org = "csvfile"
+
+        # Create execute directory and copy files
+        execute_path = self._create_execute_directory(self.options["path"])
+        self.logger.info(f"Created execute directory at {execute_path}")
+
+        # Apply namespace injection
+        self._inject_namespace_tokens(
+            execute_path, source_org_config, target_org_config
+        )
+
+        # Build and execute SFDmu command
+        # Use shell_quote to properly handle paths with spaces on Windows
+        command_parts = [
+            "-s",
+            source_sf_org,
+            "-u",
+            target_sf_org,
+            "-p",
+            execute_path,
+        ]
+
+        # Append additional parameters if provided
+        if self.options.get("additional_params"):
+            # Split the additional_params string into individual arguments
+            # This handles cases like "-no-warnings -m -t error" -> ["-no-warnings", "-m", "-t", "error"]
+            additional_args = self.options["additional_params"].split()
+            # Quote each argument to handle spaces properly
+            command_parts.extend(additional_args)
+
+        # Join command parts into a single string for sarge (which uses shell=True)
+        command = "sf sfdmu run " + " ".join(command_parts)
+        self.logger.info(f"Executing: {command}")
+
+        # Determine if we should fail on error or just warn
+        return_always_success = self.options.get("return_always_success", False)
+
+        try:
+            p: sarge.Command = sfdx(
+                "sfdmu run",
+                log_note="Running SFDmu",
+                args=command_parts,
+                check_return=not return_always_success,  # Don't check return if return_always_success is True
+                username=None,
+            )
+
+            for line in p.stdout_text:
+                self.logger.info(line)
+
+            for line in p.stderr_text:
+                self.logger.error(line)
+
+            # Check if command failed when return_always_success is True
+            if return_always_success and p.returncode != 0:
+                self.logger.warning(
+                    f"SFDmu command failed with exit code {p.returncode}, but return_always_success is True. "
+                    "Task will continue and return success."
+                )
+            else:
+                self.logger.info("SFDmu task completed successfully")
+        except Exception as e:
+            if return_always_success:
+                self.logger.warning(
+                    f"SFDmu command failed with error: {str(e)}, but return_always_success is True. "
+                    "Task will continue and return success."
+                )
+            else:
+                raise
+
+        # Post-process CSV files if target is csvfile
+        if self.options["target"] == "csvfile":
+            self._process_csv_exports(execute_path, self.options["path"])

cumulusci/tasks/sfdmu/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for SFDmu tasks."""

cumulusci/tasks/sfdmu/tests/test_runner.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""Simple test runner for SFDmu tests."""
+
+import os
+import sys
+import tempfile
+from unittest import mock
+
+# Add the project root to the path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
+
+from cumulusci.tasks.salesforce.tests.util import create_task
+from cumulusci.tasks.sfdmu.sfdmu import SfdmuTask
+
+
+def test_create_execute_directory():
+    """Test _create_execute_directory creates directory and copies files."""
+    with tempfile.TemporaryDirectory() as base_dir:
+        # Create test files
+        export_json = os.path.join(base_dir, "export.json")
+        test_csv = os.path.join(base_dir, "test.csv")
+        test_txt = os.path.join(base_dir, "test.txt")  # Should not be copied
+
+        with open(export_json, "w") as f:
+            f.write('{"test": "data"}')
+        with open(test_csv, "w") as f:
+            f.write("col1,col2\nval1,val2")
+        with open(test_txt, "w") as f:
+            f.write("text file")
+
+        # Create subdirectory (should not be copied)
+        subdir = os.path.join(base_dir, "subdir")
+        os.makedirs(subdir)
+        with open(os.path.join(subdir, "file.txt"), "w") as f:
+            f.write("subdir file")
+
+        task = create_task(
+            SfdmuTask, {"source": "dev", "target": "qa", "path": base_dir}
+        )
+
+        execute_path = task._create_execute_directory(base_dir)
+
+        # Check that execute directory was created
+        assert os.path.exists(execute_path)
+        assert execute_path == os.path.join(base_dir, "execute")
+
+        # Check that files were copied
+        assert os.path.exists(os.path.join(execute_path, "export.json"))
+        assert os.path.exists(os.path.join(execute_path, "test.csv"))
+        assert not os.path.exists(
+            os.path.join(execute_path, "test.txt")
+        )  # Not a valid file type
+        assert not os.path.exists(os.path.join(execute_path, "subdir"))  # Not a file
+
+        # Check file contents
+        with open(os.path.join(execute_path, "export.json"), "r") as f:
+            assert f.read() == '{"test": "data"}'
+        with open(os.path.join(execute_path, "test.csv"), "r") as f:
+            assert f.read() == "col1,col2\nval1,val2"
+
+        print("✅ test_create_execute_directory passed")
+
+
+def test_inject_namespace_tokens_csvfile_target():
+    """Test that namespace injection is skipped when both source and target are csvfile."""
+    with tempfile.TemporaryDirectory() as execute_dir:
+        # Create test files
+        test_json = os.path.join(execute_dir, "test.json")
+        with open(test_json, "w") as f:
+            f.write('{"field": "%%%NAMESPACE%%%Test"}')
+
+        # Create export.json file
+        export_json_path = os.path.join(execute_dir, "export.json")
+        with open(export_json_path, "w") as f:
+            f.write('{"test": "data"}')
+
+        task = create_task(
+            SfdmuTask, {"source": "csvfile", "target": "csvfile", "path": execute_dir}
+        )
+
+        # Should not raise any errors and files should remain unchanged
+        task._inject_namespace_tokens(execute_dir, None, None)
+
+        # Check that file content was not changed
+        with open(test_json, "r") as f:
+            assert f.read() == '{"field": "%%%NAMESPACE%%%Test"}'
+
+        print("✅ test_inject_namespace_tokens_csvfile_target passed")
+
+
+def test_inject_namespace_tokens_managed_mode():
+    """Test namespace injection in managed mode."""
+    with tempfile.TemporaryDirectory() as execute_dir:
+        # Create test files with namespace tokens
+        test_json = os.path.join(execute_dir, "test.json")
+        test_csv = os.path.join(execute_dir, "test.csv")
+
+        with open(test_json, "w") as f:
+            f.write(
+                '{"field": "%%%NAMESPACE%%%Test", "org": "%%%NAMESPACED_ORG%%%Value"}'
+            )
+        with open(test_csv, "w") as f:
+            f.write("Name,%%%NAMESPACE%%%Field\nTest,Value")
+
+        # Create filename with namespace token
+        filename_with_token = os.path.join(execute_dir, "___NAMESPACE___test.json")
+        with open(filename_with_token, "w") as f:
+            f.write('{"test": "data"}')
+
+        # Create export.json file
+        export_json_path = os.path.join(execute_dir, "export.json")
+        with open(export_json_path, "w") as f:
+            f.write('{"test": "data"}')
+
+        task = create_task(
+            SfdmuTask, {"source": "dev", "target": "qa", "path": execute_dir}
+        )
+
+        # Mock the project config namespace
+        task.project_config.project__package__namespace = "testns"
+
+        mock_org_config = mock.Mock()
+        mock_org_config.namespace = "testns"
+
+        # Mock determine_managed_mode to return True
+        with mock.patch(
+            "cumulusci.tasks.sfdmu.sfdmu.determine_managed_mode", return_value=True
+        ):
+            task._inject_namespace_tokens(execute_dir, None, mock_org_config)
+
+        # Check that namespace tokens were replaced in content
+        with open(test_json, "r") as f:
+            content = f.read()
+            assert "testns__Test" in content
+            assert "testns__Value" in content
+
+        with open(test_csv, "r") as f:
+            content = f.read()
+            assert "testns__Field" in content
+
+        # Check that filename token was replaced
+        expected_filename = os.path.join(execute_dir, "testns__test.json")
+        assert os.path.exists(expected_filename)
+        assert not os.path.exists(filename_with_token)
+
+        print("✅ test_inject_namespace_tokens_managed_mode passed")
+
+
+def test_inject_namespace_tokens_unmanaged_mode():
+    """Test namespace injection in unmanaged mode."""
+    with tempfile.TemporaryDirectory() as execute_dir:
+        # Create test files with namespace tokens
+        test_json = os.path.join(execute_dir, "test.json")
+        with open(test_json, "w") as f:
+            f.write(
+                '{"field": "%%%NAMESPACE%%%Test", "org": "%%%NAMESPACED_ORG%%%Value"}'
+            )
+
+        # Create export.json file
+        export_json_path = os.path.join(execute_dir, "export.json")
+        with open(export_json_path, "w") as f:
+            f.write('{"test": "data"}')
+
+        task = create_task(
+            SfdmuTask, {"source": "dev", "target": "qa", "path": execute_dir}
+        )
+
+        # Mock the project config namespace
+        task.project_config.project__package__namespace = "testns"
+
+        mock_org_config = mock.Mock()
+        mock_org_config.namespace = "testns"
+
+        # Mock determine_managed_mode to return False
+        with mock.patch(
+            "cumulusci.tasks.sfdmu.sfdmu.determine_managed_mode", return_value=False
+        ):
+            task._inject_namespace_tokens(execute_dir, None, mock_org_config)
+
+        # Check that namespace tokens were replaced with empty strings
+        with open(test_json, "r") as f:
+            content = f.read()
+            assert "Test" in content  # %%NAMESPACE%% removed
+            assert "Value" in content  # %%NAMESPACED_ORG%% removed
+            assert "%%%NAMESPACE%%%" not in content
+            assert "%%%NAMESPACED_ORG%%%" not in content
+
+        print("✅ test_inject_namespace_tokens_unmanaged_mode passed")
+
+
+def main():
+    """Run all tests."""
+    print("Running SFDmu tests...")
+
+    try:
+        test_create_execute_directory()
+        test_inject_namespace_tokens_csvfile_target()
+        test_inject_namespace_tokens_managed_mode()
+        test_inject_namespace_tokens_unmanaged_mode()
+
+        print("\n🎉 All tests passed!")
+        return 0
+    except Exception as e:
+        print(f"\n❌ Test failed: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())