data-sourcerer 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

sourcerer/infrastructure/access_credentials/services.py

@@ -9,20 +9,28 @@ import json
 from abc import ABC
 
 import boto3
+from azure.identity import ClientSecretCredential
 from dependency_injector.wiring import Provide
 from google.cloud import storage
 
-from sourcerer.domain.access_credentials.entities import Credentials, Boto3Credentials
+from sourcerer.domain.access_credentials.entities import (
+    AzureCredentials,
+    Boto3Credentials,
+    Credentials,
+)
 from sourcerer.domain.access_credentials.repositories import BaseCredentialsRepository
 from sourcerer.domain.access_credentials.services import (
-    BaseAccessCredentialsService,
     AuthField,
+    BaseAccessCredentialsService,
 )
 from sourcerer.domain.shared.entities import StorageProvider
-from sourcerer.infrastructure.access_credentials.exceptions import CredentialsAuthError
+from sourcerer.infrastructure.access_credentials.exceptions import (
+    CredentialsAuthError,
+    MissingAuthFieldsError,
+)
 from sourcerer.infrastructure.access_credentials.registry import (
-    access_credentials_method,
     AccessCredentialsMethod,
+    access_credentials_method,
 )
 from sourcerer.infrastructure.utils import generate_uuid
 from sourcerer.presentation.di_container import DiContainer
@@ -116,6 +124,22 @@ class AccessCredentialsService(BaseAccessCredentialsService, ABC):
         """
         super().__init__(credentials_repo)
 
+    @classmethod
+    def validate_auth_fields_values(cls, auth_fields: dict) -> None:
+        """
+        Validate authentication fields.
+
+        Args:
+            auth_fields (dict): Dictionary containing authentication field,
+                                where keys are field names and values are field values
+
+        Raises:
+            MissingAuthFieldsError: If any required authentication fields are missing
+        """
+        for field in cls.auth_fields():
+            if field.required and field.key not in auth_fields:
+                raise MissingAuthFieldsError(f"Missing required field: {field.key}")
+
 
 class S3AccessCredentialsService(AccessCredentialsService, ABC):
     """
@@ -176,23 +200,26 @@ class S3AccessKeySecretKeyPair(S3AccessCredentialsService):
         Returns:
             boto3.Session: Authenticated boto3 session
         """
-        credentials: dict = json.loads(credentials)
+        try:
+            credentials: dict = json.loads(credentials)
 
-        session_args = {
-            "aws_access_key_id": credentials.get("aws_access_key_id"),
-            "aws_secret_access_key": credentials.get("aws_secret_access_key"),
-        }
+            session_args = {
+                "aws_access_key_id": credentials.get("aws_access_key_id"),
+                "aws_secret_access_key": credentials.get("aws_secret_access_key"),
+            }
 
-        if region := credentials.get("region"):
-            session_args["region_name"] = region
+            if region := credentials.get("region"):
+                session_args["region_name"] = region
 
-        session = boto3.Session(**session_args)
+            session = boto3.Session(**session_args)
 
-        return Boto3Credentials(
-            session=session,
-            endpoint_url=credentials.get("endpoint_url", None),
-            signature_version=credentials.get("signature_version", None),
-        )
+            return Boto3Credentials(
+                session=session,
+                endpoint_url=credentials.get("endpoint_url", None),
+                signature_version=credentials.get("signature_version", None),
+            )
+        except Exception as e:
+            raise CredentialsAuthError("Failed to authenticate") from e
 
     @classmethod
     def auth_fields(cls):
@@ -260,15 +287,18 @@ class S3ProfileName(S3AccessCredentialsService):
         Returns:
             boto3.Session: Authenticated boto3 session
         """
-        credentials: dict = json.loads(credentials)
-        session = boto3.Session(profile_name=credentials.get("profile_name"))
-        return Boto3Credentials(
-            session=session,
-            endpoint_url=credentials.get("endpoint_url"),
-        )
+        try:
+            credentials: dict = json.loads(credentials)
+            session = boto3.Session(profile_name=credentials.get("profile_name"))
+            return Boto3Credentials(
+                session=session,
+                endpoint_url=credentials.get("endpoint_url"),
+            )
+        except Exception as e:
+            raise CredentialsAuthError("Failed to authenticate") from e
 
     @classmethod
-    def auth_fields(cls):
+    def auth_fields(cls) -> list[AuthField]:
         """
         Get list of authentication fields.
 
@@ -286,15 +316,15 @@ class S3ProfileName(S3AccessCredentialsService):
 )
 class GCPCredentialsService(AccessCredentialsService):
     """
-    Google Cloud Platform test credentials service.
+    Google Cloud Platform json credentials service.
 
     This class provides methods for storing, retrieving, and authenticating
-    with GCP using test credentials.
+    with GCP using credentials.
     """
 
     def store(self, name, credentials: dict):
         """
-        Store GCP test credentials.
+        Store GCP json credentials.
 
         Args:
             name (str): Name identifier for the credentials
@@ -357,22 +387,6 @@ class GCPCredentialsService(AccessCredentialsService):
             except json.JSONDecodeError as e:
                 raise ValueError("Invalid service account JSON format") from e
 
-            # Validate required fields for service account
-            required_fields = [
-                "type",
-                "project_id",
-                "private_key_id",
-                "private_key",
-                "client_email",
-            ]
-            missing_fields = [
-                field for field in required_fields if field not in service_acc_info
-            ]
-            if missing_fields:
-                raise ValueError(
-                    f"Service account missing required fields: {', '.join(missing_fields)}"
-                )
-
             # Create and return the authenticated client
             return storage.Client.from_service_account_info(service_acc_info)
 
@@ -384,7 +398,7 @@ class GCPCredentialsService(AccessCredentialsService):
             ) from e
 
     @classmethod
-    def auth_fields(cls):
+    def auth_fields(cls) -> list[AuthField]:
         """
         Get list of authentication fields.
 
@@ -394,3 +408,86 @@ class GCPCredentialsService(AccessCredentialsService):
         return [
             AuthField("service_acc", "Service acc", True, True),
         ]
+
+
+@access_credentials_method(
+    AccessCredentialsMethod(StorageProvider.AzureStorage, "Client Secret Credentials")
+)
+class AzureClientSecretCredentialsService(AccessCredentialsService):
+
+    def store(self, name, credentials: dict):
+        """
+        Store Azure client_id and secret_key_pair credentials.
+
+        Args:
+            name (str): Name identifier for the credentials
+            credentials (dict): Dictionary containing Azure application credential information
+        """
+        self.credentials_repo.create(
+            Credentials(
+                uuid=generate_uuid(),
+                name=name,
+                provider=StorageProvider.AzureStorage,
+                credentials_type="Client Secret Credentials",
+                credentials=json.dumps(credentials),
+                active=True,
+            )
+        )
+
+    def extract(self, uuid):
+        """
+        Extract credentials by UUID.
+
+        Args:
+            uuid (str): UUID of the credentials to extract
+
+        Returns:
+            Credentials: The credentials object
+        """
+        return self.credentials_repo.get(uuid)
+
+    def authenticate(self, credentials: str):  # type: ignore
+        try:
+            # Parse the outer JSON structure
+            parsed_credentials = json.loads(credentials)
+            subscription_id = parsed_credentials.get("subscription_id")
+            cloud_suffix = (
+                parsed_credentials.get("cloud_suffix") or "blob.core.windows.net"
+            )
+
+            client_credentials = ClientSecretCredential(
+                tenant_id=parsed_credentials.get("tenant_id"),
+                client_id=parsed_credentials.get("client_id"),
+                client_secret=parsed_credentials.get("client_secret"),
+            )
+
+            return AzureCredentials(
+                credentials=client_credentials,
+                subscription_id=subscription_id,
+                cloud_suffix=cloud_suffix,
+            )
+
+        except json.JSONDecodeError as e:
+            raise CredentialsAuthError(f"Invalid credentials format: {str(e)}") from e
+        except Exception as e:
+            raise CredentialsAuthError(
+                f"Failed to authenticate with Azure: {str(e)}"
+            ) from e
+
+    @classmethod
+    def auth_fields(cls) -> list[AuthField]:
+        """
+        Get list of authentication fields.
+
+        Returns:
+            List[AuthField]: List of authentication field definitions
+        """
+        return [
+            AuthField("subscription_id", "Subscription Id", True),
+            AuthField("tenant_id", "Tenant Id", True),
+            AuthField("client_id", "Client Id", True),
+            AuthField("client_secret", "Client Secret", True),
+            AuthField(
+                "cloud_suffix", "Cloud Suffix (default blob.core.windows.net)", False
+            ),
+        ]

sourcerer/infrastructure/db/models.py

@@ -7,7 +7,7 @@ and their relationships.
 
 from datetime import datetime
 
-from sqlalchemy import Column, Integer, Boolean, DateTime, String
+from sqlalchemy import Boolean, Column, DateTime, Integer, String
 from sqlalchemy.orm import declarative_base
 from sqlalchemy_utils.types.encrypted.encrypted_type import EncryptedType
 

sourcerer/infrastructure/file_system/exceptions.py

@@ -5,10 +5,10 @@ This module defines exception classes for handling errors that occur
 during interactions with the local file system.
 """
 
-from sourcerer.domain.file_system.exceptions import BaseFileSystemException
+from sourcerer.domain.file_system.exceptions import BaseFileSystemError
 
 
-class FileSystemGrepException(BaseFileSystemException):
+class FileSystemGrepError(BaseFileSystemError):
     """
     Custom exception for errors occurring during a file system search operation.
 
@@ -18,7 +18,7 @@ class FileSystemGrepException(BaseFileSystemException):
     """
 
 
-class ReadFileException(BaseFileSystemException):
+class ReadFileError(BaseFileSystemError):
     """
     Custom exception for errors occurring during a file system read operation.
 
@@ -28,7 +28,7 @@ class ReadFileException(BaseFileSystemException):
     """
 
 
-class CreateFileException(BaseFileSystemException):
+class CreateFileError(BaseFileSystemError):
     """
     Custom exception for errors occurring during a file system create file operation.
 
@@ -38,7 +38,7 @@ class CreateFileException(BaseFileSystemException):
     """
 
 
-class CreateDirException(BaseFileSystemException):
+class CreateDirError(BaseFileSystemError):
     """
     Custom exception for errors occurring during a file system create directory operation.
 
@@ -48,7 +48,7 @@ class CreateDirException(BaseFileSystemException):
     """
 
 
-class MoveFileException(BaseFileSystemException):
+class MoveFileError(BaseFileSystemError):
     """
     Exception raised for errors during file move operation.
 
@@ -59,7 +59,7 @@ class MoveFileException(BaseFileSystemException):
     """
 
 
-class ListDirException(BaseFileSystemException):
+class ListDirError(BaseFileSystemError):
     """
     Custom exception for errors occurring during a file system list directory operation.
 
@@ -69,7 +69,7 @@ class ListDirException(BaseFileSystemException):
     """
 
 
-class DeleteFileException(BaseFileSystemException):
+class DeleteFileError(BaseFileSystemError):
     """
     Custom exception for errors occurring during a file system delete file operation.
 
@@ -79,7 +79,7 @@ class DeleteFileException(BaseFileSystemException):
     """
 
 
-class DeleteDirException(BaseFileSystemException):
+class DeleteDirError(BaseFileSystemError):
     """
     Custom exception for errors occurring during a file system delete directory operation.
 

sourcerer/infrastructure/file_system/services.py

@@ -10,8 +10,8 @@ from pathlib import Path
 from sourcerer.domain.file_system.entities import ListDirOutput
 from sourcerer.domain.file_system.services import BaseFileSystemService
 from sourcerer.infrastructure.file_system.exceptions import (
-    ReadFileException,
-    ListDirException,
+    ListDirError,
+    ReadFileError,
 )
 from sourcerer.infrastructure.utils import custom_sort_key
 
@@ -56,18 +56,18 @@ class FileSystemService(BaseFileSystemService):
             str: The processed data extracted from the file.
 
         Raises:
-            ReadFileException: An error occurred during the file read operation.
+            ReadFileError: An error occurred during the file read operation.
         """
 
         if not isinstance(path, Path):
-            raise ReadFileException("file_path must be a Path object")
+            raise ReadFileError("file_path must be a Path object")
         if not path.exists():
-            raise ReadFileException(f"File not found: {path}")
-        self.validate_path_within_work_dir(path, ReadFileException)
+            raise ReadFileError(f"File not found: {path}")
+        self.validate_path_within_work_dir(path, ReadFileError)
         try:
             return path.read_text()
         except Exception as e:
-            raise ReadFileException(f"Error reading file: {e}") from e
+            raise ReadFileError(f"Error reading file: {e}") from e
 
     def list_dir(
         self,
@@ -91,19 +91,19 @@ class FileSystemService(BaseFileSystemService):
                 - directories (list[Path]): Sorted list of directory paths
 
         Raises:
-            ListDirException: If the path is invalid, directory doesn't exist, or path is not a directory.
+            ListDirError: If the path is invalid, directory doesn't exist, or path is not a directory.
 
         """
         if not isinstance(path, Path):
-            raise ListDirException("Path must be a Path object")
+            raise ListDirError("Path must be a Path object")
         if not path.exists():
-            raise ListDirException(f"Directory not found: {path}")
+            raise ListDirError(f"Directory not found: {path}")
         if not path.is_dir():
-            raise ListDirException(f"Path is not a directory: {path}")
+            raise ListDirError(f"Path is not a directory: {path}")
         try:
             path.relative_to(self.work_dir)
         except ValueError as e:
-            raise ListDirException("Access denied: Path outside work directory") from e
+            raise ListDirError("Access denied: Path outside work directory") from e
         try:
             files = []
             directories = []
@@ -112,10 +112,10 @@ class FileSystemService(BaseFileSystemService):
             items = 0
             for entry in entries:
                 if max_items and items > max_items:
-                    raise ListDirException(
+                    raise ListDirError(
                         f"Too many items, max processable dir size: {max_items}"
                     )
-                items += 1
+                items += 1  # noqa SIM103
                 target = files if entry.is_file() else directories
                 target.append(entry.relative_to(path) if relative_paths else entry)
             return ListDirOutput(
@@ -123,9 +123,9 @@ class FileSystemService(BaseFileSystemService):
                 directories=sorted(directories, key=custom_sort_key),
             )
         except PermissionError as e:
-            raise ListDirException(f"Access denied: {e}") from e
+            raise ListDirError(f"Access denied: {e}") from e
         except Exception as e:
-            raise ListDirException(f"Error listing directory: {e}") from e
+            raise ListDirError(f"Error listing directory: {e}") from e
 
     def validate_path_within_work_dir(self, path: Path, exception_class: type) -> None:
         """

sourcerer/infrastructure/storage_provider/exceptions.py

@@ -5,10 +5,10 @@ This module defines exception classes for handling errors that occur
 during interactions with cloud storage providers.
 """
 
-from sourcerer.domain.storage_provider.exceptions import BaseStorageProviderException
+from sourcerer.domain.storage_provider.exceptions import BaseStorageProviderError
 
 
-class CredentialsNotFoundException(BaseStorageProviderException):
+class CredentialsNotFoundError(BaseStorageProviderError):
     """
     Exception raised when credentials are not found for a cloud storage provider.
 
@@ -18,7 +18,7 @@ class CredentialsNotFoundException(BaseStorageProviderException):
     """
 
 
-class ListStoragesException(BaseStorageProviderException):
+class ListStoragesError(BaseStorageProviderError):
     """
     Exception raised for errors that occur during the listing of cloud storage services.
 
@@ -28,7 +28,7 @@ class ListStoragesException(BaseStorageProviderException):
     """
 
 
-class StoragePermissionException(BaseStorageProviderException):
+class StoragePermissionError(BaseStorageProviderError):
     """
     Exception raised for permission-related errors in cloud storage operations.
 
@@ -37,7 +37,7 @@ class StoragePermissionException(BaseStorageProviderException):
     """
 
 
-class ListStorageItemsException(BaseStorageProviderException):
+class ListStorageItemsError(BaseStorageProviderError):
     """
     Exception raised when an error occurs while listing items in cloud storage.
 
@@ -47,7 +47,17 @@ class ListStorageItemsException(BaseStorageProviderException):
     """
 
 
-class ReadStorageItemsException(BaseStorageProviderException):
+class BlobNotFoundError(BaseStorageProviderError):
+    """
+    Exception raised when a blob (file) is not found in cloud storage.
+
+    This exception is a specific case of BaseStorageProviderException
+    and should be used to indicate that the requested blob could not
+    be located in the specified cloud storage service.
+    """
+
+
+class ReadStorageItemsError(BaseStorageProviderError):
     """
     Exception raised for errors encountered while reading items from cloud storage.
 
@@ -57,7 +67,7 @@ class ReadStorageItemsException(BaseStorageProviderException):
     """
 
 
-class DeleteStorageItemsException(BaseStorageProviderException):
+class DeleteStorageItemsError(BaseStorageProviderError):
     """
     Exception raised for errors that occur during the deletion of storage items.
 
@@ -67,7 +77,7 @@ class DeleteStorageItemsException(BaseStorageProviderException):
     """
 
 
-class UploadStorageItemsException(BaseStorageProviderException):
+class UploadStorageItemsError(BaseStorageProviderError):
     """
     Exception raised for errors that occur during the upload of items
     to cloud storage.
@@ -76,3 +86,13 @@ class UploadStorageItemsException(BaseStorageProviderException):
     and is used to signal issues encountered while uploading data to
     cloud storage services.
     """
+
+
+class AzureMissingContainerError(BaseStorageProviderError):
+    """
+    Exception raised when a container is not found in Azure cloud storage.
+
+    This exception is a specific case of BaseStorageProviderException
+    and should be used to indicate that the requested container could not
+    be located in the specified Azure cloud storage service.
+    """

sourcerer/infrastructure/storage_provider/registry.py

@@ -6,7 +6,6 @@ storage provider services.
 """
 
 import functools
-from typing import Type, Dict
 
 from sourcerer.domain.storage_provider.services import BaseStorageProviderService
 from sourcerer.infrastructure.utils import Singleton
@@ -23,9 +22,9 @@ class StorageProviderRegistry(metaclass=Singleton):
     def __init__(self, *args, **kwargs):
         """Initialize the registry with an empty dictionary."""
         super().__init__(*args, **kwargs)
-        self._items_: Dict[str, type] = {}
+        self._items_: dict[str, type] = {}
 
-    def register(self, storage_provider: str, cls: Type[BaseStorageProviderService]):
+    def register(self, storage_provider: str, cls: type[BaseStorageProviderService]):
         """
         Register a storage provider service implementation.