dataflow-core 2.1.14__py3-none-any.whl → 2.1.15rc1__py3-none-any.whl

dataflow/secrets_manager/factory.py

@@ -3,6 +3,7 @@ import os
 from .interface import SecretManager
 from .providers.aws_manager import AWSSecretsManager
 from .providers.azure_manager import AzureKeyVault
+from .providers.gcp_manager import GCPSecretsManager
 from ..configuration import ConfigurationManager
 
 # A custom exception for clear error messages
@@ -17,10 +18,6 @@ def get_secret_manager() -> SecretManager:
     to determine which cloud provider's secret manager to instantiate.
     """
     try:
-        # dataflow_config = None
-        # if os.getenv('HOSTNAME'):
-        #     dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
-        # else:
         dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
     except Exception as e:
         raise SecretProviderError(
@@ -49,11 +46,20 @@ def get_secret_manager() -> SecretManager:
             )
         return AzureKeyVault(vault_url=vault_url)
 
-    # You can easily add more providers here in the future
-    # elif provider == "gcp":
-    #     return GCPSecretManager()
+    elif provider == "gcp":
+        project_id = dataflow_config.get_config_value('cloudProvider', 'gcp_project_id')
+        region = dataflow_config.get_config_value('cloudProvider', 'gcp_region')
+        if not project_id:
+            raise SecretProviderError(
+                "GCP_PROJECT_ID must be set when using the GCP provider."
+            )
+        if not region:
+            raise SecretProviderError(
+                "GCP_REGION must be set when using the GCP provider."
+            )
+        return GCPSecretsManager(project_id=project_id, region=region)
 
     else:
         raise SecretProviderError(
-            f"Unsupported secret provider: '{provider}'. Supported providers are: aws, azure."
+            f"Unsupported secret provider: '{provider}'. Supported providers are: aws, azure and gcp"
         )

dataflow/secrets_manager/providers/gcp_manager.py

@@ -0,0 +1,332 @@
+import os, base64, json, atexit
+from pathlib import Path
+from google.cloud import secretmanager
+from google.cloud.secretmanager_v1 import SecretManagerServiceClient
+from google.cloud.secretmanager_v1.types import Secret, SecretPayload
+from google.api_core.exceptions import (
+    AlreadyExists,
+    NotFound,
+    PermissionDenied,
+    Forbidden,
+    ResourceExhausted,
+    InvalidArgument,
+    FailedPrecondition
+)
+import json
+from ..interface import SecretManager
+from ...utils.exceptions import (
+    SecretNotFoundException,
+    SecretAlreadyExistsException,
+    SecretManagerAuthException,
+    SecretManagerServiceException
+)
+
+def _setup_gcp_credentials():
+    """Setup GCP credentials from base64 encoded JSON environment variable"""
+    
+    # Only run if GOOGLE_APPLICATION_CREDENTIALS is not already set
+    if os.getenv('GOOGLE_APPLICATION_CREDENTIALS'):
+        return
+    
+    # Get base64 encoded JSON credentials
+    encoded_json = os.getenv('GOOGLE_APPLICATION_CREDENTIALS_JSON')
+    
+    if encoded_json:
+        try:
+            # Decode base64 to JSON string
+            json_credentials = base64.b64decode(encoded_json).decode('utf-8')
+            
+            # Validate it's valid JSON
+            json.loads(json_credentials)  # Just to validate
+            
+            # Create credentials file in home directory
+            home_dir = Path.home()
+            credentials_dir = home_dir / '.gcp'
+            credentials_dir.mkdir(exist_ok=True)  # Create .gcp directory if it doesn't exist
+            
+            credentials_path = credentials_dir / 'credentials.json'
+            
+            # Write JSON string to credentials file
+            with open(credentials_path, 'w') as f:
+                f.write(json_credentials)
+            
+            # Set the standard Google environment variable that the SDK looks for
+            os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = str(credentials_path)
+            
+            # Clean up file on exit
+            atexit.register(lambda: credentials_path.unlink() if credentials_path.exists() else None)
+            
+            print(f"GCP credentials decoded and configured at {credentials_path}")
+            
+        except Exception as e:
+            print(f"Error setting up GCP credentials: {e}")
+
+
+class GCPSecretsManager(SecretManager):
+    """Google Cloud Platform Secrets Manager implementation."""
+
+    def __init__(self, project_id: str, region: str):
+        """Initialize the GCP Secret Manager client.
+        
+        Args:
+            project_id: The GCP project ID where secrets will be stored.
+            region: The GCP region where secrets will be stored.
+        """
+        self.project_id = project_id
+        self.region = region
+        try:
+            _setup_gcp_credentials()
+            self.client = secretmanager.SecretManagerServiceClient()
+        except PermissionDenied as e:
+            raise SecretManagerAuthException("initialize_gcp_client", original_error=str(e))
+        except Exception as e:
+            raise SecretManagerServiceException("initialize_gcp_client", original_error=str(e))
+    
+    def _get_secret_path(self, vault_path: str) -> str:
+        """Get the full secret path in GCP format.
+        
+        Args:
+            vault_path: The path/name of the secret.
+            
+        Returns:
+            The full path to the secret in GCP format.
+        """
+        return f"projects/{self.project_id}/secrets/{vault_path}"
+    
+    def _get_secret_version_path(self, vault_path: str, version: str = "latest") -> str:
+        """Get the full path to a specific secret version.
+        
+        Args:
+            vault_path: The path/name of the secret.
+            version: The version of the secret (default is "latest").
+            
+        Returns:
+            The full path to the secret version in GCP format.
+        """
+        return f"{self._get_secret_path(vault_path)}/versions/{version}"
+    
+    def create_secret(self, vault_path: str, secret_data: dict) -> str:
+        """Create a new secret.
+        
+        Args:
+            vault_path: The path/name of the secret.
+            region: The region where the secret will be stored.
+            secret_data: The data to store in the secret.
+            
+        Returns:
+            A success message.
+            
+        Raises:
+            SecretAlreadyExistsException: If the secret already exists.
+            SecretManagerAuthException: If there are permission issues.
+            SecretManagerServiceException: For other service errors.
+        """
+        try:
+            # Convert dictionary to JSON string before saving
+            secret_string = json.dumps(secret_data)
+            
+            # First create the secret
+            parent = f"projects/{self.project_id}"
+            secret = Secret(
+                replication={
+                    "user_managed": {
+                        "replicas": [
+                            {"location": self.region }
+                        ]
+                    }
+                }
+            )
+            
+            self.client.create_secret(
+                request={
+                    "parent": parent,
+                    "secret_id": vault_path,
+                    "secret": secret
+                }
+            )
+            
+            # Then add the secret version with the data
+            secret_path = self._get_secret_path(vault_path)
+            self.client.add_secret_version(
+                request={
+                    "parent": secret_path,
+                    "payload": {"data": secret_string.encode("UTF-8")}
+                }
+            )
+            
+            return "Secret created successfully"
+        except AlreadyExists as e:
+            raise SecretAlreadyExistsException("secret", vault_path, original_error=str(e))
+        except PermissionDenied as e:
+            raise SecretManagerAuthException("create_secret", original_error=str(e))
+        except Forbidden as e:
+            raise SecretManagerAuthException("create_secret", original_error=str(e))
+        except FailedPrecondition as e:
+            # Handle case where secret might be in recovery/deleted state
+            if "pending deletion" in str(e).lower() or "scheduled for deletion" in str(e).lower():
+                raise SecretAlreadyExistsException("secret", vault_path, original_error=str(e), is_scheduled_for_deletion=True)
+            else:
+                raise SecretManagerServiceException("create_secret", original_error=str(e))
+        except ResourceExhausted as e:
+            raise SecretManagerServiceException("create_secret", original_error=str(e))
+        except Exception as e:
+            raise SecretManagerServiceException("create_secret", original_error=str(e))
+    
+    def get_secret_by_key(self, vault_path: str) -> dict:
+        """Get a secret by its key.
+        
+        Args:
+            vault_path: The path/name of the secret.
+            
+        Returns:
+            The secret data as a dictionary.
+            
+        Raises:
+            SecretNotFoundException: If the secret doesn't exist.
+            SecretManagerAuthException: If there are permission issues.
+            SecretManagerServiceException: For other service errors.
+        """
+        try:
+            # Get the latest version of the secret
+            name = self._get_secret_version_path(vault_path)
+            response = self.client.access_secret_version(request={"name": name})
+            
+            # Decode the payload and convert JSON string back to dictionary
+            secret_string = response.payload.data.decode("UTF-8")
+            secret_data = json.loads(secret_string)
+            return secret_data
+        except NotFound as e:
+            raise SecretNotFoundException("secret", vault_path, original_error=str(e))
+        except PermissionDenied as e:
+            raise SecretManagerAuthException("get_secret", original_error=str(e))
+        except Forbidden as e:
+            raise SecretManagerAuthException("get_secret", original_error=str(e))
+        except InvalidArgument as e:
+            raise SecretManagerServiceException("get_secret", original_error=str(e))
+        except json.JSONDecodeError as e:
+            raise SecretManagerServiceException("get_secret", original_error=str(e))
+        except Exception as e:
+            raise SecretManagerServiceException("get_secret", original_error=str(e))
+    
+    def update_secret(self, vault_path: str, update_data: dict) -> str:
+        """Update an existing secret.
+        
+        Args:
+            vault_path: The path/name of the secret.
+            update_data: The data to update in the secret.
+            
+        Returns:
+            A success message.
+            
+        Raises:
+            SecretNotFoundException: If the secret doesn't exist.
+            SecretManagerAuthException: If there are permission issues.
+            SecretManagerServiceException: For other service errors.
+        """
+        try:
+            # Get current secret data
+            current_data = self.get_secret_by_key(vault_path)
+            
+            # Update with new data
+            current_data.update(update_data)
+            
+            # Convert updated dictionary to JSON string
+            updated_string = json.dumps(current_data)
+            
+            # Add a new version of the secret with updated data
+            secret_path = self._get_secret_path(vault_path)
+            self.client.add_secret_version(
+                request={
+                    "parent": secret_path,
+                    "payload": {"data": updated_string.encode("UTF-8")}
+                }
+            )
+            
+            return "Secret updated successfully"
+        except SecretNotFoundException:
+            raise
+        except PermissionDenied as e:
+            raise SecretManagerAuthException("update_secret", original_error=str(e))
+        except Forbidden as e:
+            raise SecretManagerAuthException("update_secret", original_error=str(e))
+        except NotFound as e:
+            raise SecretNotFoundException("secret", vault_path, original_error=str(e))
+        except InvalidArgument as e:
+            raise SecretManagerServiceException("update_secret", original_error=str(e))
+        except json.JSONDecodeError as e:
+            raise SecretManagerServiceException("update_secret", original_error=str(e))
+        except Exception as e:
+            raise SecretManagerServiceException("update_secret", original_error=str(e))
+    
+    def delete_secret(self, vault_path: str) -> str:
+        """Delete a secret.
+        
+        Args:
+            vault_path: The path/name of the secret.
+            
+        Returns:
+            A success message.
+            
+        Raises:
+            SecretNotFoundException: If the secret doesn't exist.
+            SecretManagerAuthException: If there are permission issues.
+            SecretManagerServiceException: For other service errors.
+        """
+        try:
+            # Get the full path to the secret
+            name = self._get_secret_path(vault_path)
+            
+            # For git-ssh secrets, destroy without recovery
+            if "git-ssh" in vault_path:
+                # Get all versions to destroy them permanently
+                versions = self.client.list_secret_versions(request={"parent": name})
+                for version in versions:
+                    if version.state == secretmanager.SecretVersion.State.ENABLED:
+                        version_name = f"{name}/versions/{version.name.split('/')[-1]}"
+                        self.client.destroy_secret_version(request={"name": version_name})
+                
+                # Delete the secret itself
+                self.client.delete_secret(request={"name": name})
+            else:
+                # For regular secrets, use the default 7-day recovery window
+                self.client.delete_secret(request={
+                    "name": name,
+                    # In GCP, the recovery window is configured at the service level, 
+                    # not per API call, so we don't specify it here
+                })
+            
+            return "Secret deleted successfully"
+        except NotFound as e:
+            raise SecretNotFoundException("secret", vault_path, original_error=str(e))
+        except PermissionDenied as e:
+            raise SecretManagerAuthException("delete_secret", original_error=str(e))
+        except Forbidden as e:
+            raise SecretManagerAuthException("delete_secret", original_error=str(e))
+        except InvalidArgument as e:
+            raise SecretManagerServiceException("delete_secret", original_error=str(e))
+        except Exception as e:
+            raise SecretManagerServiceException("delete_secret", original_error=str(e))
+    
+    def test_connection(self, vault_path: str) -> str:
+        """Test the connection to the secret manager by attempting to access a secret.
+        
+        Args:
+            vault_path: The path/name of the secret to test.
+            
+        Returns:
+            The status of the secret.
+            
+        Raises:
+            SecretNotFoundException: If the secret doesn't exist.
+            SecretManagerAuthException: If there are permission issues.
+            SecretManagerServiceException: For other service errors.
+        """
+        try:
+            secret = self.get_secret_by_key(vault_path)
+            return secret.get('status', 'Unknown')
+        except SecretNotFoundException:
+            raise
+        except (SecretManagerAuthException, SecretManagerServiceException):
+            raise
+        except Exception as e:
+            raise SecretManagerServiceException("test_connection", original_error=str(e))

{dataflow_core-2.1.14.dist-info → dataflow_core-2.1.15rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.14
+Version: 2.1.15rc1
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 
@@ -11,6 +11,8 @@ Requires-Dist: pymysql
 Requires-Dist: requests
 Requires-Dist: azure-identity
 Requires-Dist: azure-keyvault-secrets
+Requires-Dist: google-auth
+Requires-Dist: google-cloud-secret-manager
 Dynamic: author
 Dynamic: requires-dist
 Dynamic: summary

{dataflow_core-2.1.14.dist-info → dataflow_core-2.1.15rc1.dist-info}/RECORD

@@ -40,18 +40,19 @@ dataflow/scripts/clone_environment.sh,sha256=Qy0GylsA3kUVUL_L1MirxIWujOFhT1tikKq
 dataflow/scripts/create_environment.sh,sha256=3FHgNplJuEZvyTsLqlCJNX9oyfXgsfqn80VZk2xtvso,828
 dataflow/scripts/update_environment.sh,sha256=2dtn2xlNi6frpig-sqlGE1_IKRbbkqYOCpf_qyMKKII,992
 dataflow/secrets_manager/__init__.py,sha256=idGqIDtYl0De2WIK9Obl-N7SDPSYtVM0D-wXfZjCiy4,559
-dataflow/secrets_manager/factory.py,sha256=k1sIyXBKtas1upWJpq8Mks2d8kjLAHU7CFvjeuMXXxs,2160
+dataflow/secrets_manager/factory.py,sha256=LblshkGG9q2C3RHYp0QykianUtpOOQz7sBdlerutyWY,2479
 dataflow/secrets_manager/interface.py,sha256=HhrKpQrprWIbDsVfU_qc59OXmSIuHXv106OXv6-Epqc,506
 dataflow/secrets_manager/service.py,sha256=SSWgTXJTAwVPqMIc76cB2hR6nghNVOoMpIN9M0i7Su0,7241
 dataflow/secrets_manager/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataflow/secrets_manager/providers/aws_manager.py,sha256=16peXyKeuAjv2RVTMUjrzArPYENK9Zu7jREWVgMfScA,8671
 dataflow/secrets_manager/providers/azure_manager.py,sha256=sWOz-7ALnLt6vyM3lt14GBpzpmDnlH3hkdqtuApqkgU,9430
+dataflow/secrets_manager/providers/gcp_manager.py,sha256=AJgotHZRQraxtmfJX1Z8u2Gcr7KJLRJTN_qbth3A5Xk,13738
 dataflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dataflow/utils/exceptions.py,sha256=8GRFoYZ5dPGQckVm2znaHpPi0ZAs69fK-RGKukEsapk,4432
 dataflow/utils/get_current_user.py,sha256=4nSO3SPVMZhW-MsIgxR3f9ZzrFaIZIuyrM6hvfyE7PQ,1202
 dataflow/utils/logger.py,sha256=7BFrOq5Oiqn8P4XZbgJzMP5O07d2fpdECbbfsjrUuHw,1213
-dataflow_core-2.1.14.dist-info/METADATA,sha256=er2J0BBiZuIgOJJH5yG0mLhVQ81D6qrhHZOad_dYPLE,370
-dataflow_core-2.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dataflow_core-2.1.14.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
-dataflow_core-2.1.14.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
-dataflow_core-2.1.14.dist-info/RECORD,,
+dataflow_core-2.1.15rc1.dist-info/METADATA,sha256=aRoxdVVfP0yY6qPtPuSH_RGHqxAQ13HCTjLq0UxRnR0,443
+dataflow_core-2.1.15rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dataflow_core-2.1.15rc1.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
+dataflow_core-2.1.15rc1.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
+dataflow_core-2.1.15rc1.dist-info/RECORD,,