cumulusci-plus 5.0.21__py3-none-any.whl → 5.0.43__py3-none-any.whl

cumulusci/tasks/utility/credentialManager.py

@@ -0,0 +1,302 @@
+import logging
+import os
+from abc import ABC, abstractmethod
+from typing import Any, Optional, Union
+
+
+# Abstract Base Class for Credential Providers
+class CredentialProvider(ABC):
+    """
+    Abstract Base Class that defines the interface for all credential providers.
+    """
+
+    # A class-level dictionary to store a mapping of provider names to provider classes
+    _registry = {}
+    logger: logging.Logger
+
+    def __init__(self, **kwargs):
+        self.key_prefix = kwargs.get(
+            "key_prefix", os.getenv("CUMULUSCI_PREFIX_SECRETS", "").upper()
+        )
+        self.logger = logging.getLogger(__name__)
+
+    def __init_subclass__(cls, **kwargs):
+        """
+        This method is called automatically when a new class inherits from CredentialProvider.
+        It's used to register the new class in our registry.
+        """
+        super().__init_subclass__(**kwargs)
+        # We'll use an attribute on each class to define its provider type.
+        # This will be the key in our registry.
+        if hasattr(cls, "provider_type"):
+            CredentialProvider._registry[cls.provider_type] = cls
+
+    @abstractmethod
+    def get_credentials(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> Any:
+        """
+        Retrieves and returns credentials.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    @abstractmethod
+    def get_all_credentials(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> Any:
+        """
+        Retrieves and returns all credentials in a group, for example all secrets in AWS secret.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def get_key(self, key: str) -> str:
+        return f"{self.key_prefix}{key}"
+
+
+# Concrete Credential Provider for Local Development
+class DevEnvironmentVariableProvider(CredentialProvider):
+    """
+    Retrieves secrets from environment variables.
+    This is suitable for local development.
+    """
+
+    provider_type = "local"
+
+    def get_credentials(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> Any:
+        value = options.get("value", None)
+        if isinstance(value, dict):
+            value = next(iter(value.values()))
+
+        self.logger.info(f"Credentials for {key} from local environment is {value}.")
+        return value
+
+    def get_all_credentials(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> Any:
+        """Local provider doesn't support retrieving all credentials."""
+        raise NotImplementedError("Local provider doesn't support get_all_credentials")
+
+
+# Concrete Credential Provider for Local Development
+class EnvironmentVariableProvider(CredentialProvider):
+    """
+    Retrieves secrets from environment variables.
+    This is suitable for local development.
+    """
+
+    provider_type = "environment"
+
+    def get_credentials(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> Any:
+        value = options.get("value", None)
+        if isinstance(value, dict):
+            value = next(iter(value.values()))
+
+        ret_value = os.getenv(self.get_key(key))
+        if ret_value is None and value is not None:
+            ret_value = os.getenv(self.get_key(value))
+        if ret_value is None:
+            self.logger.info(f"Credentials for {key} from environment is {value}.")
+        return ret_value or value
+
+    def get_all_credentials(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> Any:
+        """Environment provider doesn't support retrieving all credentials."""
+        raise NotImplementedError(
+            "Environment provider doesn't support get_all_credentials"
+        )
+
+
+# Concrete Credential Provider for Azure Pipelines
+class AwsSecretsManagerProvider(CredentialProvider):
+    """
+    Retrieves secrets from AWS Secrets Manager.
+    This is designed for use in a CI/CD environment like Azure Pipelines,
+    where a Service Connection provides a role to assume.
+    """
+
+    provider_type = "aws_secrets"
+    secrets_cache: dict[str, dict[str, Any]]
+    aws_region: str
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.secrets_cache = kwargs.get("secrets_cache", {})
+
+        self.aws_region = kwargs.get("aws_region", os.getenv("AWS_REGION", None))
+        if self.aws_region is None:
+            raise ValueError(
+                "AWS_REGION environment variable or aws_region option is required for AWS Secrets Manager."
+            )
+
+    def get_credentials(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> Any:
+        """
+        Connects to AWS Secrets Manager to retrieve a secret.
+        The boto3 client automatically uses the credentials provided by the
+        Azure DevOps AWS Service Connection (e.g., through OIDC or static keys).
+        """
+        return self.aws_creds(key, options).get(key, None)
+
+    def get_all_credentials(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> Any:
+        return self.aws_creds(key, options)
+
+    def aws_creds(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> dict[str, Any]:
+        secret_name = options.get("secret_name", None)
+        if secret_name is None:
+            raise ValueError("Secret name is required for AWS Secrets Manager.")
+
+        try:
+            import json
+
+            import boto3
+            from botocore.exceptions import ClientError
+        except ImportError:
+            raise RuntimeError(
+                "boto3 is not installed. Please install it using 'pip install boto3' or 'pipx inject cumulusci-plus-azure-devops boto3'."
+            )
+
+        try:
+            if secret_name in self.secrets_cache:
+                return self.secrets_cache[secret_name]
+
+            # Boto3 automatically handles credential lookup. In an Azure Pipeline,
+            # it will find the temporary credentials provided by the AWS Service Connection.
+            # Create a Secrets Manager client
+            session = boto3.session.Session()
+            client = session.client(
+                service_name="secretsmanager", region_name=self.aws_region
+            )
+
+            get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+
+            if "SecretString" in get_secret_value_response:
+                secret = json.loads(get_secret_value_response["SecretString"])
+            elif "SecretBinary" in get_secret_value_response:
+                file_path = self.create_binary_file(
+                    secret_name, get_secret_value_response["SecretBinary"]
+                )
+                secret_key = key if key and key != "*" else os.path.basename(file_path)
+                secret = {secret_key: file_path}
+            else:
+                raise ValueError(f"Secret {secret_name} is not a valid secret.")
+
+            # Assuming the secret is a JSON string with the credentials
+            # We need to check the binary type of the secret at later development
+            self.secrets_cache[secret_name] = secret
+
+            return self.secrets_cache[secret_name]
+        except ClientError as e:
+            # For a list of exceptions thrown, see
+            # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
+            raise e
+        except ValueError:
+            # Re-raise ValueError as-is (e.g., invalid secret format)
+            raise
+        except Exception as e:
+            raise RuntimeError(f"Failed to retrieve secret '{key}': {e}")
+
+    def create_binary_file(self, secret_name: str, content: Union[str, bytes]) -> str:
+        """
+        Write the binary content to a file.
+        secret_name is relative path to the root of the project.
+        create the directory if it doesn't exist.
+        return the absolute path to the file with windows or linux path separator.
+        """
+        absolute_path = os.path.abspath(os.path.join(".cci", secret_name))
+
+        try:
+            os.makedirs(os.path.dirname(absolute_path), exist_ok=True)
+            with open(absolute_path, "wb") as f:
+                f.write(content)
+            return absolute_path
+        except Exception as e:
+            raise RuntimeError(f"Failed to create binary file {absolute_path}: {e}")
+
+
+# Concrete Credential Provider for Azure Variable Groups
+class AzureVariableGroupProvider(CredentialProvider):
+    """
+    Retrieves secrets from Azure Pipeline Variable Groups.
+    When a variable group is linked, its variables are exposed as
+    environment variables in the pipeline job.
+    """
+
+    provider_type = "ado_variables"
+
+    def get_credentials(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> Any:
+        """
+        Looks for AWS credentials in environment variables exposed by
+        an Azure variable group.
+        """
+        value = options.get("value", None)
+        if isinstance(value, dict):
+            value = next(iter(value.values()))
+
+        # Azure pipelines convert variable names to uppercase and replace dots with underscores.
+        ret_value = os.getenv(self.get_key(key).upper().replace(".", "_"))
+
+        if ret_value is None and value is not None:
+            ret_value = os.getenv(self.get_key(value).upper().replace(".", "_"))
+
+        return ret_value
+
+    def get_all_credentials(
+        self, key: str, options: Optional[dict[str, Any]] = None
+    ) -> Any:
+        """Azure variable group provider doesn't support retrieving all credentials."""
+        raise NotImplementedError(
+            "Azure variable group provider doesn't support get_all_credentials"
+        )
+
+
+# The CredentialManager to select the right provider
+class CredentialManager:
+    """
+    Factory class to determine and return the correct CredentialProvider based on the environment.
+    """
+
+    env_secrets_type = "CUMULUSCI_SECRETS_TYPE"
+
+    @staticmethod
+    def load_secrets_type_from_environment() -> str:
+        """Load any secrets specified by environment variables"""
+        provider_type = os.getenv(CredentialManager.env_secrets_type)
+
+        # If no provider type is found, use the dev provider
+        return (provider_type or "local").lower()
+
+    @staticmethod
+    def get_provider(
+        provider_type: Optional[str] = None, **kwargs
+    ) -> CredentialProvider:
+        """
+        Looks up the provider class in the registry and returns an instance.
+        """
+        if not provider_type:
+            # If no config is provided, load the secrets from the environment
+            provider_type = CredentialManager.load_secrets_type_from_environment()
+
+        # Check if the requested provider type exists in our registry
+        if provider_type not in CredentialProvider._registry:
+            raise ValueError(f"Unknown provider type specified: '{provider_type}'")
+
+        # Get the class from the registry
+        ProviderClass = CredentialProvider._registry[provider_type]
+        provider = ProviderClass(**kwargs)
+        provider.logger.info(
+            f'Using "{provider.provider_type}" provider for credentials.'
+        )
+
+        return provider

cumulusci/tasks/utility/directoryRecreator.py

@@ -0,0 +1,30 @@
+import os
+import shutil
+from pathlib import Path
+
+from cumulusci.core.exceptions import TaskOptionsError
+from cumulusci.core.tasks import BaseTask
+from cumulusci.utils.options import CCIOptions, Field
+
+
+class DirectoryRecreator(BaseTask):
+    class Options(CCIOptions):
+        path: Path = Field(..., description="Path to the directory to recreate")
+
+    parsed_options: Options
+
+    def _init_options(self, kwargs):
+        super()._init_options(kwargs)
+
+        if os.path.isfile(self.parsed_options.path):
+            raise TaskOptionsError(f"Path {self.parsed_options.path} is a file")
+
+    def _run_task(self):
+        created = "created"
+        if os.path.exists(self.parsed_options.path):
+            shutil.rmtree(self.parsed_options.path)
+            created = "removed and created"
+
+        os.makedirs(self.parsed_options.path, exist_ok=True)
+
+        self.logger.info(f"Directory {self.parsed_options.path} {created}.")

cumulusci/tasks/utility/env_management.py

@@ -5,7 +5,7 @@ from datetime import date
 from pathlib import Path
 from typing import Any, List, Optional
 
-from pydantic import BaseModel, validator
+from pydantic.v1 import BaseModel, validator
 
 from cumulusci.core.config import BaseProjectConfig, OrgConfig, TaskConfig
 from cumulusci.core.tasks import BaseTask

cumulusci/tasks/utility/secretsToEnv.py

@@ -0,0 +1,135 @@
+import os
+from pathlib import Path
+from typing import Union
+
+from dotenv import dotenv_values
+
+from cumulusci.core.tasks import BaseTask
+from cumulusci.utils.options import (
+    CCIOptions,
+    Field,
+    ListOfStringsOption,
+    MappingOption,
+)
+
+from .credentialManager import CredentialManager
+
+
+class GenericOptions(CCIOptions):
+    env_path: Path = Field("./.env", description="Path to the .env file")
+    secrets_provider: str = Field(
+        None,
+        description='Secrets provider type i.e local, ado_variables, aws_secrets. Default value is None, which will use the secrets type from the environment variable CUMULUSCI_SECRETS_TYPE if it is set, otherwise it will use the "local" provider.',
+    )
+    provider_options: MappingOption = Field({}, description="Provider options")
+
+
+class SecretsToEnv(BaseTask):
+    class Options(GenericOptions):
+        secrets: Union[ListOfStringsOption, MappingOption] = Field(
+            ...,
+            description="List of secret keys to retrieve be it with a list of keys or a mapping of key to secret name.",
+        )
+
+    parsed_options: Options
+
+    def _init_options(self, kwargs):
+        super()._init_options(kwargs)
+        self.provider = CredentialManager.get_provider(
+            self.parsed_options.secrets_provider
+            or CredentialManager.load_secrets_type_from_environment(),
+            **self.parsed_options.provider_options,
+        )
+        self.env_values = dotenv_values(self.parsed_options.env_path)
+
+    def _init_secrets(self):
+        if (
+            isinstance(self.parsed_options.secrets, list)
+            and self.parsed_options.secrets
+        ):
+            try:
+                self.secrets = MappingOption.from_str(
+                    ",".join(self.parsed_options.secrets)
+                )
+            except Exception:
+                self.secrets = {
+                    secret: secret for secret in self.parsed_options.secrets
+                }
+        elif isinstance(self.parsed_options.secrets, dict):
+            self.secrets = self.parsed_options.secrets
+        else:
+            self.secrets = {}
+
+    def _run_task(self):
+
+        self._init_secrets()
+
+        output_file = self.parsed_options.env_path
+
+        for secret_key, secret_name in self.secrets.items():
+            if secret_key == "*":
+                self.env_values.update(
+                    self._get_all_credentials(secret_key, secret_name=secret_name)
+                )
+            else:
+                safe_value, _ = self._get_credential(
+                    secret_key, secret_key, secret_name=secret_name
+                )
+                self.env_values[secret_key] = safe_value
+
+        # Ensure output directory exists
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+
+        # Write env file with UTF-8 encoding to support Unicode characters
+        with open(output_file, "w", encoding="utf-8") as env_file:
+            for key, value in self.env_values.items():
+                env_file.write(f'{key}="{value}"\n')
+
+    def _get_credential(
+        self,
+        credential_key,
+        value,
+        env_key=None,
+        display_value="*****",
+        secret_name=None,
+    ):
+        if env_key is None:
+            env_key = credential_key
+
+        cred_secret_value = self.provider.get_credentials(
+            credential_key, {"value": value, "secret_name": secret_name}
+        )
+
+        if cred_secret_value is None:
+            raise ValueError(
+                f"Failed to retrieve secret {credential_key} from {self.provider.provider_type}"
+            )
+
+        self.logger.info(
+            f"Set secrets env var from {self.provider.provider_type}: {env_key}={display_value}"
+        )
+        safe_value = cred_secret_value.replace('"', '\\"').replace("\n", "\\n")
+        return safe_value, cred_secret_value
+
+    def _get_all_credentials(
+        self, credential_key, display_value="*****", secret_name=None
+    ):
+
+        cred_secret_values = self.provider.get_all_credentials(
+            credential_key, {"secret_name": secret_name}
+        )
+
+        if cred_secret_values is None:
+            raise ValueError(
+                f"Failed to retrieve secret {credential_key}({secret_name}) from {self.provider.provider_type}"
+            )
+
+        dict_values = {}
+        for key, value in cred_secret_values.items():
+            self.logger.info(
+                f"Set secrets env var from {self.provider.provider_type}: {key}={display_value}"
+            )
+            safe_value = value.replace('"', '\\"').replace("\n", "\\n")
+            dict_values[key] = safe_value
+
+        return dict_values