daedaluspy 1.0.0__py3-none-any.whl

daedaluspy/__init__.py

@@ -0,0 +1,6 @@
+"""
+DaedalusPy - Data Engineering Library Code Generator
+"""
+
+__version__ = "1.0.0"
+__author__ = "Golden Valley Consulting"

daedaluspy/__main__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+"""
+DaedalusPy - Toolkit de geração de código para engenharia de dados
+Entry point para execução via python -m daedaluspy
+"""
+
+from .cli import main
+
+if __name__ == "__main__":
+    main()

daedaluspy/cli.py

@@ -0,0 +1,172 @@
+from daedaluspy.data_pipeline.template.complete_templates import get_template_set
+from daedaluspy.data_pipeline.generator.pipeline_generator_oop import PipelineGenerator, PipelineTier, CloudProvider as PipelineCloudProvider
+import argparse
+import sys
+from enum import Enum
+from daedaluspy.data_lib.generator.library_generator import LibraryGenerator
+from daedaluspy.data_lib.generator.data_entity_oop import DataEntityGenerator, CloudProvider
+from daedaluspy.data_lib.generator.service_generator_oop import ServiceGenerator
+
+class Tier(Enum):
+    RAW = "raw"
+    CLEAR = "clear"
+    MODEL = "model"
+
+class ServiceType(Enum):
+    API = "api"
+    DATABASE = "database"
+
+class Command:
+    def execute(self, args):
+        raise NotImplementedError
+
+class CreateLibCommand(Command):
+    def execute(self, args):
+        generator = LibraryGenerator(
+            name=args.name,
+            data_name=args.dataname,
+            system_name=args.systemname
+        )
+        generator.generate()
+        print(f"Estrutura da biblioteca '{args.name}' criada com sucesso!")
+
+class CreateDataCommand(Command):
+    def execute(self, args):
+        if not args.tier:
+            print("Erro: --tier é obrigatório (raw, clear, model)")
+            sys.exit(1)
+        tier = Tier(args.tier)
+        generator = DataEntityGenerator(
+            classname=args.name,
+            tier=tier,
+            cloud_provider=CloudProvider(args.cloud),
+            columns=args.columns or [],
+            output_path=args.output or f"./{args.name.lower()}/{tier.value}"
+        )
+        generator.generate()
+        print(f"Entidade '{args.name}' criada com sucesso na camada {tier.value}.")
+
+class CreateServiceCommand(Command):
+    def execute(self, args):
+        service_type = ServiceType(args.type)
+        # Permite argumentos opcionais futuros
+        extra_kwargs = {}
+        if hasattr(args, 'output') and args.output:
+            extra_kwargs['output_path'] = args.output
+        # Converter models de lista de strings JSON para lista de dicts
+        import json
+        models = []
+        if args.models:
+            for m in args.models:
+                if isinstance(m, dict):
+                    models.append(m)
+                else:
+                    try:
+                        models.append(json.loads(m))
+                    except Exception:
+                        pass
+        generator = ServiceGenerator(
+            service_name=args.name,
+            service_type=service_type,
+            models=models,
+            **extra_kwargs
+        )
+        generator.generate()
+        print(f"Serviço '{args.name}' ({service_type.value}) criado com sucesso.")
+
+class CreatePipelineCommand(Command):
+    def execute(self, args):
+        tier = PipelineTier(args.tier)
+        # Permitir todos os parâmetros opcionais igual ao create-pipeline-project
+        def enum_from_value(enum_cls, value):
+            for e in enum_cls:
+                if value == e.value or value == e.name or value.upper() == e.name:
+                    return e
+            raise ValueError(f"{value} não é válido para {enum_cls.__name__}")
+
+        generator = PipelineGenerator(
+            system_name=args.system_name,
+            dataname=args.dataname,
+            tier=tier,
+            output_path=args.output if args.output else ".",
+            template_type=getattr(args, "template_type", "all"),
+            lib_name=getattr(args, "lib_name", "atlaspy"),
+            cloud_provider=enum_from_value(PipelineCloudProvider, getattr(args, "cloud_provider", "cloud_azure")),
+            entity_target=getattr(args, "entity_target", None),
+            entity_target_class=getattr(args, "entity_target_class", None)
+        )
+        generator.generate()
+        print(f"Pipeline '{args.system_name}/{args.dataname}/{tier.value}' criado com sucesso.")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="DaedalusPy CLI")
+    subparsers = parser.add_subparsers(dest="command")
+    # create pipeline project (completo)
+    parser_pipeline_project = subparsers.add_parser("create-pipeline-project")
+    parser_pipeline_project.add_argument("system_name")
+    parser_pipeline_project.add_argument("dataname")
+    parser_pipeline_project.add_argument("tier", choices=["raw", "clear", "model"])
+    parser_pipeline_project.add_argument("--output", "-o", default=".")
+    parser_pipeline_project.add_argument("--template-type", "-t", choices=["essential", "dev", "all"], default="all")
+    parser_pipeline_project.add_argument("--lib-name", default="atlaspy")
+    parser_pipeline_project.add_argument("--cloud-provider", default="cloud_azure")
+    parser_pipeline_project.add_argument("--entity-target", default="azure_sql_database")
+    parser_pipeline_project.add_argument("--entity-target-class", default="AzureSQLDatabase")
+
+    # create lib
+    parser_lib = subparsers.add_parser("create-lib")
+    parser_lib.add_argument("name")
+    parser_lib.add_argument("--dataname")
+    parser_lib.add_argument("--systemname")
+
+    # create data
+    parser_data = subparsers.add_parser("create-data")
+    parser_data.add_argument("name")
+    parser_data.add_argument("--tier", required=True, choices=[t.value for t in Tier])
+    parser_data.add_argument("--cloud", required=True)
+    parser_data.add_argument("--extension")
+    parser_data.add_argument("--columns", nargs="*")
+    parser_data.add_argument("--imports")
+    parser_data.add_argument("--read_code")
+    parser_data.add_argument("--write_code")
+    parser_data.add_argument("--output")
+    parser_data.add_argument("--lib")
+
+
+    # create service
+    parser_service = subparsers.add_parser("create-service")
+    parser_service.add_argument("name")
+    parser_service.add_argument("--type", required=True, choices=[t.value for t in ServiceType])
+    parser_service.add_argument("--models", nargs="*")
+    parser_service.add_argument("--output")
+
+    # create pipeline
+    parser_pipeline = subparsers.add_parser("create-pipeline")
+    parser_pipeline.add_argument("system_name")
+    parser_pipeline.add_argument("dataname")
+    parser_pipeline.add_argument("--tier", required=True, choices=[t.value for t in PipelineTier])
+    parser_pipeline.add_argument("--output", default=".")
+    parser_pipeline.add_argument("--template_type", default="all")
+    parser_pipeline.add_argument("--lib_name", default="atlaspy")
+    parser_pipeline.add_argument("--cloud_provider", default="cloud_azure")
+    parser_pipeline.add_argument("--entity_target", default="azure_sql_database")
+    parser_pipeline.add_argument("--entity_target_class", default="AzureSQLDatabase")
+
+    args = parser.parse_args()
+
+
+    commands = {
+        "create-lib": CreateLibCommand(),
+        "create-data": CreateDataCommand(),
+        "create-service": CreateServiceCommand(),
+        "create-pipeline": CreatePipelineCommand(),
+    }
+
+    if args.command in commands:
+        commands[args.command].execute(args)
+    else:
+        parser.print_help()
+
+if __name__ == "__main__":
+    main()

daedaluspy/data_lib/__init__.py

@@ -0,0 +1,38 @@
+"""
+DaedalusPy - Framework para Engenharia de Dados Multi-Cloud
+
+Este módulo fornece ferramentas para automatizar e padronizar o desenvolvimento
+de soluções de engenharia de dados em múltiplas plataformas cloud.
+
+Módulos principais:
+- data: Biblioteca para templates e estruturas de dados
+- generator: Biblioteca para geração de entidades e integrações  
+- services: Biblioteca para templates de serviços
+
+Exemplo de uso:
+    from data_lib.generator.data_entity import generate_entity_file
+    
+    generate_entity_file(
+        cloud_provider="azure",
+        classname="MinhaEntidade",
+        file_extension="parquet",
+        columns=['col1', 'col2'],
+        imports="import pandas as pd",
+        read_code="return pd.read_parquet(buffer)",
+        write_code="data.to_parquet(buffer, index=False)",
+        output_path="./output"
+    )
+
+Autor: Golden Valley Consulting LTDA
+Licença: MIT
+"""
+
+__version__ = "1.0.0"
+__author__ = "Golden Valley Consulting LTDA"
+
+# Importações principais
+from . import data
+from . import generator
+from . import services
+
+__all__ = ['data', 'generator', 'services']

daedaluspy/data_lib/data/base_template.py

@@ -0,0 +1,370 @@
+"""
+Template para base.py - classes base para entidades de dados
+"""
+BASE_DATA_TEMPLATE = '''"""
+Base classes para entidades de dados
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, Type
+from datetime import datetime
+from io import BytesIO
+from enum import StrEnum
+import pandas as pd
+import json
+
+
+class FileExtensionHandler(ABC):
+    """Classe abstrata para handlers de extensões de arquivo"""
+    
+    @property
+    @abstractmethod
+    def extension(self) -> str:
+        """Retorna a extensão do arquivo"""
+        pass
+    
+    @abstractmethod
+    def read_data(self, buffer: BytesIO) -> Any:
+        """Lê dados do buffer"""
+        pass
+    
+    @abstractmethod
+    def write_data(self, data: Any, buffer: BytesIO) -> None:
+        """Escreve dados no buffer"""
+        pass
+
+
+class StorageProvider(ABC):
+    """Classe abstrata para providers de storage"""
+    
+    @property
+    @abstractmethod
+    def provider_name(self) -> str:
+        """Retorna o nome do provider"""
+        pass
+    
+    @abstractmethod
+    def get_data_buffer(self, path: str, storage_client) -> BytesIO:
+        """Obtém buffer de dados do storage pelo caminho"""
+        pass
+    
+    @abstractmethod
+    def save_data_buffer(self, buffer: BytesIO, path: str, storage_client) -> bool:
+        """Salva buffer de dados no storage pelo caminho"""
+        pass
+    
+    @abstractmethod
+    def validate_client(self, storage_client) -> bool:
+        """Valida se o cliente de storage é válido para este provider"""
+        pass
+
+
+class CSVHandler(FileExtensionHandler):
+    """Handler para arquivos CSV"""
+    
+    @property
+    def extension(self) -> str:
+        return "csv"
+    
+    def read_data(self, buffer: BytesIO) -> pd.DataFrame:
+        return pd.read_csv(buffer)
+    
+    def write_data(self, data: pd.DataFrame, buffer: BytesIO) -> None:
+        data.to_csv(buffer, index=False)
+
+
+class JSONHandler(FileExtensionHandler):
+    """Handler para arquivos JSON"""
+    
+    @property
+    def extension(self) -> str:
+        return "json"
+    
+    def read_data(self, buffer: BytesIO) -> Dict[str, Any]:
+        return json.load(buffer)
+    
+    def write_data(self, data: Dict[str, Any], buffer: BytesIO) -> None:
+        json.dump(data, buffer, ensure_ascii=False, indent=2)
+
+
+class ParquetHandler(FileExtensionHandler):
+    """Handler para arquivos Parquet"""
+    
+    @property
+    def extension(self) -> str:
+        return "parquet"
+    
+    def read_data(self, buffer: BytesIO) -> pd.DataFrame:
+        return pd.read_parquet(buffer)
+    
+    def write_data(self, data: pd.DataFrame, buffer: BytesIO) -> None:
+        data.to_parquet(buffer, index=False)
+
+
+class ExcelHandler(FileExtensionHandler):
+    """Handler para arquivos Excel"""
+    
+    @property
+    def extension(self) -> str:
+        return "xlsx"
+    
+    def read_data(self, buffer: BytesIO) -> pd.DataFrame:
+        return pd.read_excel(buffer)
+    
+    def write_data(self, data: pd.DataFrame, buffer: BytesIO) -> None:
+        data.to_excel(buffer, index=False)
+
+
+class AzureStorageProvider(StorageProvider):
+    """Provider para Azure Blob Storage"""
+    
+    @property
+    def provider_name(self) -> str:
+        return "azure"
+    
+    def get_data_buffer(self, path: str, storage_client) -> BytesIO:
+        """Obtém dados do Azure Blob Storage"""
+        blob = storage_client.get_blob_client(blob=path)
+        return BytesIO(blob.download_blob().readall())
+    
+    def save_data_buffer(self, buffer: BytesIO, path: str, storage_client) -> bool:
+        """Salva dados no Azure Blob Storage"""
+        try:
+            blob = storage_client.get_blob_client(blob=path)
+            buffer.seek(0)
+            blob.upload_blob(buffer.read(), overwrite=True)
+            return True
+        except Exception as e:
+            print(f"Erro ao salvar no Azure: {e}")
+            return False
+    
+    def validate_client(self, storage_client) -> bool:
+        """Valida se é um cliente Azure válido"""
+        return hasattr(storage_client, 'get_blob_client')
+
+
+class AWSStorageProvider(StorageProvider):
+    """Provider para AWS S3"""
+    
+    @property
+    def provider_name(self) -> str:
+        return "aws"
+    
+    def get_data_buffer(self, path: str, storage_client) -> BytesIO:
+        """Obtém dados do AWS S3"""
+        bucket, key = self._parse_s3_path(path)
+        response = storage_client.get_object(Bucket=bucket, Key=key)
+        return BytesIO(response['Body'].read())
+    
+    def save_data_buffer(self, buffer: BytesIO, path: str, storage_client) -> bool:
+        """Salva dados no AWS S3"""
+        try:
+            bucket, key = self._parse_s3_path(path)
+            buffer.seek(0)
+            storage_client.put_object(Bucket=bucket, Key=key, Body=buffer.read())
+            return True
+        except Exception as e:
+            print(f"Erro ao salvar no AWS: {e}")
+            return False
+    
+    def validate_client(self, storage_client) -> bool:
+        """Valida se é um cliente AWS S3 válido"""
+        return hasattr(storage_client, 'get_object') and hasattr(storage_client, 'put_object')
+    
+    def _parse_s3_path(self, path: str) -> tuple:
+        """Parse do caminho S3 para bucket e key"""
+        # Implementar lógica de parsing do path S3
+        # Por exemplo: s3://bucket/path/file.csv -> (bucket, path/file.csv)
+        if path.startswith('s3://'):
+            path = path[5:]
+        parts = path.split('/', 1)
+        return parts[0], parts[1] if len(parts) > 1 else ''
+
+
+class GoogleStorageProvider(StorageProvider):
+    """Provider para Google Cloud Storage"""
+    
+    @property
+    def provider_name(self) -> str:
+        return "google"
+    
+    def get_data_buffer(self, path: str, storage_client) -> BytesIO:
+        """Obtém dados do Google Cloud Storage"""
+        bucket_name, blob_name = self._parse_gcs_path(path)
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(blob_name)
+        return BytesIO(blob.download_as_bytes())
+    
+    def save_data_buffer(self, buffer: BytesIO, path: str, storage_client) -> bool:
+        """Salva dados no Google Cloud Storage"""
+        try:
+            bucket_name, blob_name = self._parse_gcs_path(path)
+            bucket = storage_client.bucket(bucket_name)
+            blob = bucket.blob(blob_name)
+            buffer.seek(0)
+            blob.upload_from_file(buffer)
+            return True
+        except Exception as e:
+            print(f"Erro ao salvar no Google: {e}")
+            return False
+    
+    def validate_client(self, storage_client) -> bool:
+        """Valida se é um cliente Google Cloud válido"""
+        return hasattr(storage_client, 'bucket')
+    
+    def _parse_gcs_path(self, path: str) -> tuple:
+        """Parse do caminho GCS para bucket e blob"""
+        # Implementar lógica de parsing do path GCS
+        # Por exemplo: gs://bucket/path/file.csv -> (bucket, path/file.csv)
+        if path.startswith('gs://'):
+            path = path[5:]
+        parts = path.split('/', 1)
+        return parts[0], parts[1] if len(parts) > 1 else ''
+
+        
+class DataLayerOptions(StrEnum):
+    """Opções de camada de dados"""
+    RAW = "raw"
+    CLEAR = "clear"
+    MODEL = "model"
+
+@dataclass
+class BaseEntity(ABC):
+    """Classe base para todas as entidades de dados"""
+    
+    # Propriedades base que devem ser definidas nas classes filhas
+    data_layer: DataLayerOptions
+    path_prefix: str
+    filename: str
+    file_handler: FileExtensionHandler
+    storage_provider: StorageProvider
+    date_reference: Optional[datetime] = None  # data de referência
+    path_sufix: Optional[str] = None  # sufixo opcional para o caminho
+    
+    class Columns:
+        """Classe para definir colunas de entidades"""
+        pass
+    
+    def _build_path_by_date(self) -> str:
+        """Constrói caminho baseado na data de referência"""
+        return "/".join([
+            self.data_layer.value,
+            self.path_prefix,
+            f"{self.date_reference.year:04d}",
+            f"{self.date_reference.month:02d}",
+            f"{self.date_reference.year:04d}.{self.date_reference.month:02d}.{self.date_reference.day:02d}",
+            f"{self.filename}.{self.file_handler.extension}"
+        ])
+
+    def _build_path_by_sufix(self) -> str:
+        """Constrói caminho baseado no sufixo"""
+        return "/".join([
+            self.data_layer.value,
+            self.path_prefix,
+            self.path_sufix,
+            f"{self.filename}.{self.file_handler.extension}"
+        ])
+
+    def read_data_by_date(self, storage_client=None) -> Any:
+        """Lê dados baseado na data de referência"""
+        if not self.date_reference:
+            raise ValueError("date_reference deve ser definida para leitura por data")
+        
+        if not self.storage_provider.validate_client(storage_client):
+            raise ValueError(f"Cliente de storage inválido para provider {self.storage_provider.provider_name}")
+        
+        # Constrói o caminho e obtém os dados do storage
+        path = self._build_path_by_date()
+        buffer = self.storage_provider.get_data_buffer(path, storage_client)
+        
+        # Usa o file handler para processar os dados
+        return self.file_handler.read_data(buffer)
+    
+    def write_data_by_date(self, data: Any, storage_client=None) -> bool:
+        """Escreve dados baseado na data de referência"""
+        if not self.date_reference:
+            raise ValueError("date_reference deve ser definida para escrita por data")
+        
+        if not self.storage_provider.validate_client(storage_client):
+            raise ValueError(f"Cliente de storage inválido para provider {self.storage_provider.provider_name}")
+        
+        try:
+            # Cria buffer e usa o file handler para escrever os dados
+            buffer = BytesIO()
+            self.file_handler.write_data(data, buffer)
+            
+            # Constrói o caminho e salva no storage
+            path = self._build_path_by_date()
+            return self.storage_provider.save_data_buffer(buffer, path, storage_client)
+        except Exception as e:
+            print(f"Erro ao escrever dados por data: {e}")
+            return False
+
+    def read_data_by_sufix(self, storage_client=None) -> Any:
+        """Lê dados baseado no sufixo"""
+        if not self.path_sufix:
+            raise ValueError("path_sufix deve ser definido para leitura por sufixo")
+        
+        if not self.storage_provider.validate_client(storage_client):
+            raise ValueError(f"Cliente de storage inválido para provider {self.storage_provider.provider_name}")
+        
+        # Constrói o caminho e obtém os dados do storage
+        path = self._build_path_by_sufix()
+        buffer = self.storage_provider.get_data_buffer(path, storage_client)
+        
+        # Usa o file handler para processar os dados
+        return self.file_handler.read_data(buffer)
+    
+    def write_data_by_sufix(self, data: Any, storage_client=None) -> bool:
+        """Escreve dados baseado no sufixo"""
+        if not self.path_sufix:
+            raise ValueError("path_sufix deve ser definido para escrita por sufixo")
+        
+        if not self.storage_provider.validate_client(storage_client):
+            raise ValueError(f"Cliente de storage inválido para provider {self.storage_provider.provider_name}")
+        
+        try:
+            # Cria buffer e usa o file handler para escrever os dados
+            buffer = BytesIO()
+            self.file_handler.write_data(data, buffer)
+            
+            # Constrói o caminho e salva no storage
+            path = self._build_path_by_sufix()
+            return self.storage_provider.save_data_buffer(buffer, path, storage_client)
+        except Exception as e:
+            print(f"Erro ao escrever dados por sufixo: {e}")
+            return False
+
+class HandlerFactory:
+    """Factory para criar handlers e storage providers"""
+    
+    _file_handlers = {
+        "csv": CSVHandler,
+        "json": JSONHandler,
+        "parquet": ParquetHandler,
+        "xlsx": ExcelHandler
+    }
+    
+    _storage_providers = {
+        "aws": AWSStorageProvider,
+        "azure": AzureStorageProvider,
+        "google": GoogleStorageProvider
+    }
+    
+    @classmethod
+    def get_file_handler(cls, extension: str) -> FileExtensionHandler:
+        """Cria handler baseado na extensão"""
+        handler_class = cls._file_handlers.get(extension.lower())
+        if not handler_class:
+            raise ValueError(f"Handler não encontrado para extensão: {extension}")
+        return handler_class()
+    
+    @classmethod
+    def get_storage_provider(cls, provider_name: str) -> StorageProvider:
+        """Cria storage provider baseado no nome"""
+        provider_class = cls._storage_providers.get(provider_name.lower())
+        if not provider_class:
+            raise ValueError(f"Storage provider não encontrado: {provider_name}")
+        return provider_class()
+'''

daedaluspy/data_lib/data/cloud_aws/template/container_files/entity_template.py

@@ -0,0 +1,20 @@
+TEMPLATE_ENTITY_AWS = '''from datetime import datetime
+from {lib_name}.data.base import BaseEntity, DataLayerOptions, HandlerFactory, AWSStorageProvider
+{imports}
+
+class {classname}(BaseEntity):
+    """Entidade de dados para {classname}"""
+
+    class Columns:
+{columns}
+
+    def __init__(self):
+        """Configurações da entidade"""
+        self.data_layer = DataLayerOptions.{data_layer_upper}
+        self.path_prefix = "{path_prefix}"
+        self.filename = "{filename}"
+        self.file_handler = HandlerFactory.get_file_handler("{file_extension}")
+        self.storage_provider = AWSStorageProvider()    
+        self.date_reference = datetime.now()
+        self.path_sufix = None
+'''

daedaluspy/data_lib/data/cloud_azure/template/container_files/entity_template.py

@@ -0,0 +1,21 @@
+TEMPLATE_ENTITY_AZURE = '''from datetime import datetime
+from azure.storage.blob import BlobClient
+from {lib_name}.data.base import BaseEntity, DataLayerOptions, HandlerFactory, AzureStorageProvider
+{imports}
+
+class {classname}(BaseEntity):
+    """Entidade de dados para {classname}"""
+
+    class Columns:
+{columns}
+
+    def __init__(self):
+        """Configurações da entidade"""
+        self.data_layer = DataLayerOptions.{data_layer_upper}
+        self.path_prefix = "{path_prefix}"
+        self.filename = "{filename}"
+        self.file_handler = HandlerFactory.get_file_handler("{file_extension}")
+        self.storage_provider = AzureStorageProvider()    
+        self.date_reference = datetime.now()
+        self.path_sufix = None
+'''