nia-etl-utils 0.1.0__tar.gz → 0.2.0__tar.gz

{nia_etl_utils-0.1.0 → nia_etl_utils-0.2.0}/PKG-INFO

@@ -1,10 +1,25 @@
 Metadata-Version: 2.4
 Name: nia-etl-utils
-Version: 0.1.0
+Version: 0.2.0
 Summary: Utilitários compartilhados para pipelines ETL do NIA/MPRJ
 Author-email: Nícolas Esmael <nicolas.esmael@mprj.mp.br>
 License: MIT
 Project-URL: Repository, https://gitlab-dti.mprj.mp.br/nia/etl-nia/nia-etl-utils
+Project-URL: Documentation, https://gitlab-dti.mprj.mp.br/nia/etl-nia/nia-etl-utils/-/blob/main/README.md
+Project-URL: Changelog, https://gitlab-dti.mprj.mp.br/nia/etl-nia/nia-etl-utils/-/blob/main/CHANGELOG.md
+Keywords: etl,data-engineering,pipeline,mprj,nia
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: loguru>=0.7.0
@@ -17,6 +32,12 @@ Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
 Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: pandas-stubs>=2.0.0; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.5.0; extra == "docs"
+Requires-Dist: mkdocs-material>=9.0.0; extra == "docs"
+Requires-Dist: mkdocstrings[python]>=0.24.0; extra == "docs"
 
 # nia-etl-utils
 

nia_etl_utils-0.2.0/pyproject.toml

@@ -0,0 +1,122 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "nia-etl-utils"
+version = "0.2.0"
+description = "Utilitários compartilhados para pipelines ETL do NIA/MPRJ"
+authors = [
+    {name = "Nícolas Esmael", email = "nicolas.esmael@mprj.mp.br"}
+]
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+keywords = ["etl", "data-engineering", "pipeline", "mprj", "nia"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Database",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Typing :: Typed",
+]
+dependencies = [
+    "loguru>=0.7.0",
+    "python-dotenv>=1.0.0",
+    "cx-Oracle>=8.3.0",
+    "psycopg2-binary>=2.9.0",
+    "sqlalchemy>=2.0.0",
+    "pandas>=2.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "ruff>=0.1.0",
+    "mypy>=1.0.0",
+    "pandas-stubs>=2.0.0",
+]
+docs = [
+    "mkdocs>=1.5.0",
+    "mkdocs-material>=9.0.0",
+    "mkdocstrings[python]>=0.24.0",
+]
+
+[project.urls]
+Repository = "https://gitlab-dti.mprj.mp.br/nia/etl-nia/nia-etl-utils"
+Documentation = "https://gitlab-dti.mprj.mp.br/nia/etl-nia/nia-etl-utils/-/blob/main/README.md"
+Changelog = "https://gitlab-dti.mprj.mp.br/nia/etl-nia/nia-etl-utils/-/blob/main/CHANGELOG.md"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+
+[tool.ruff.lint]
+select = [
+    "E",      # pycodestyle errors
+    "W",      # pycodestyle warnings
+    "F",      # Pyflakes
+    "I",      # isort
+    "B",      # flake8-bugbear
+    "C4",     # flake8-comprehensions
+    "UP",     # pyupgrade
+    "ARG",    # flake8-unused-arguments
+    "SIM",    # flake8-simplify
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+    "B008",   # do not perform function calls in argument defaults
+    "C901",   # too complex
+    "ARG001", # unused function argument
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["nia_etl_utils"]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = "--verbose --cov=src/nia_etl_utils --cov-report=term-missing"
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "integration: marks tests as integration tests",
+]
+
+[tool.coverage.run]
+source = ["src/nia_etl_utils"]
+branch = true
+omit = ["*/tests/*", "*/__init__.py"]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise NotImplementedError",
+    "if TYPE_CHECKING:",
+    "if __name__ == .__main__.:",
+]

nia_etl_utils-0.2.0/src/nia_etl_utils/__init__.py

@@ -0,0 +1,236 @@
+"""Utilitários compartilhados para pipelines ETL do NIA/MPRJ.
+
+Este pacote fornece funções reutilizáveis para:
+- Configuração de ambiente (env_config)
+- Configurações de conexão e email (config)
+- Exceções customizadas (exceptions)
+- Resultados estruturados (results)
+- Envio de emails via SMTP (email_smtp)
+- Conexões com bancos de dados Oracle e PostgreSQL (database)
+- Configuração de logging padronizado (logger_config)
+- Processamento e exportação de CSV (processa_csv)
+- Processamento paralelo de CSV grandes (processa_csv_paralelo)
+- Manipulação de arquivos e diretórios (limpeza_pastas)
+
+Exemplo de uso:
+
+    from nia_etl_utils import (
+        configurar_logger_padrao_nia,
+        conectar_postgresql_nia,
+        exportar_para_csv,
+        PostgresConfig,
+    )
+    from nia_etl_utils.exceptions import ConexaoError, ExtracaoVaziaError
+
+    # Configuração
+    configurar_logger_padrao_nia("meu_pipeline")
+
+    # Conexão com context manager
+    try:
+        with conectar_postgresql_nia() as conn:
+            conn.cursor.execute("SELECT * FROM tabela")
+            dados = conn.cursor.fetchall()
+    except ConexaoError as e:
+        logger.error(f"Falha na conexão: {e}")
+        sys.exit(1)  # decisão do CHAMADOR
+
+    # Ou com configuração explícita (para testes)
+    config = PostgresConfig(
+        host="localhost",
+        port="5432",
+        database="teste",
+        user="user",
+        password="pass"
+    )
+    with conectar_postgresql(config) as conn:
+        # ...
+"""
+
+__version__ = "0.2.0"
+__author__ = "Nícolas Galdino Esmael"
+
+# =============================================================================
+# EXCEÇÕES - Importar primeiro para uso em type hints
+# =============================================================================
+
+# =============================================================================
+# CONFIGURAÇÕES (Dataclasses)
+# =============================================================================
+from .config import (
+    LogConfig,
+    OracleConfig,
+    PostgresConfig,
+    SmtpConfig,
+)
+
+# Database - Funções core
+# Database - Wrappers de conveniência
+from .database import (
+    conectar_oracle,
+    conectar_oracle_ouvidorias,
+    conectar_postgresql,
+    conectar_postgresql_nia,
+    conectar_postgresql_opengeo,
+    obter_engine_postgresql,
+    obter_engine_postgresql_nia,
+    obter_engine_postgresql_opengeo,
+)
+
+# Email
+from .email_smtp import (
+    enviar_email,
+    enviar_email_smtp,
+    obter_destinatarios_padrao,
+)
+
+# =============================================================================
+# FUNÇÕES UTILITÁRIAS
+# =============================================================================
+# Configuração de ambiente
+from .env_config import (
+    obter_variavel_env,
+    obter_variavel_env_bool,
+    obter_variavel_env_int,
+    obter_variavel_env_lista,
+)
+from .exceptions import (
+    # Arquivos
+    ArquivoError,
+    ConexaoError,
+    # Configuração
+    ConfiguracaoError,
+    # Database
+    DatabaseError,
+    DestinatarioError,
+    DiretorioError,
+    # Email
+    EmailError,
+    EscritaArquivoError,
+    # Extração
+    ExtracaoError,
+    ExtracaoVaziaError,
+    LeituraArquivoError,
+    # Base
+    NiaEtlError,
+    ProcessamentoError,
+    SmtpError,
+    # Validação
+    ValidacaoError,
+    VariavelAmbienteError,
+)
+
+# Manipulação de arquivos
+from .limpeza_pastas import (
+    criar_pasta_se_nao_existir,
+    limpar_pasta,
+    listar_arquivos,
+    remover_pasta_recursivamente,
+)
+
+# Logging
+from .logger_config import (
+    configurar_logger,
+    configurar_logger_padrao_nia,
+    remover_handlers_existentes,
+)
+
+# Processamento CSV
+from .processa_csv import (
+    exportar_multiplos_csv,
+    exportar_para_csv,
+    extrair_e_exportar_csv,
+)
+
+# Processamento CSV Paralelo
+from .processa_csv_paralelo import (
+    calcular_chunksize,
+    processar_csv_paralelo,
+)
+
+# =============================================================================
+# RESULTADOS (Dataclasses)
+# =============================================================================
+from .results import (
+    Conexao,
+    ResultadoEmail,
+    ResultadoExtracao,
+    ResultadoLote,
+)
+
+# =============================================================================
+# __all__ - Exportações públicas
+# =============================================================================
+
+__all__ = [
+    # Metadata
+    "__version__",
+    "__author__",
+    # Exceções - Base
+    "NiaEtlError",
+    # Exceções - Configuração
+    "ConfiguracaoError",
+    "VariavelAmbienteError",
+    # Exceções - Database
+    "DatabaseError",
+    "ConexaoError",
+    # Exceções - Arquivos
+    "ArquivoError",
+    "DiretorioError",
+    "EscritaArquivoError",
+    "LeituraArquivoError",
+    # Exceções - Extração
+    "ExtracaoError",
+    "ExtracaoVaziaError",
+    "ProcessamentoError",
+    # Exceções - Email
+    "EmailError",
+    "DestinatarioError",
+    "SmtpError",
+    # Exceções - Validação
+    "ValidacaoError",
+    # Configurações
+    "PostgresConfig",
+    "OracleConfig",
+    "SmtpConfig",
+    "LogConfig",
+    # Resultados
+    "Conexao",
+    "ResultadoExtracao",
+    "ResultadoLote",
+    "ResultadoEmail",
+    # Env config
+    "obter_variavel_env",
+    "obter_variavel_env_int",
+    "obter_variavel_env_bool",
+    "obter_variavel_env_lista",
+    # Email
+    "enviar_email",
+    "enviar_email_smtp",
+    "obter_destinatarios_padrao",
+    # Database - Core
+    "conectar_postgresql",
+    "conectar_oracle",
+    "obter_engine_postgresql",
+    # Database - Wrappers
+    "conectar_postgresql_nia",
+    "conectar_postgresql_opengeo",
+    "conectar_oracle_ouvidorias",
+    "obter_engine_postgresql_nia",
+    "obter_engine_postgresql_opengeo",
+    # Logging
+    "configurar_logger",
+    "configurar_logger_padrao_nia",
+    "remover_handlers_existentes",
+    # CSV
+    "exportar_para_csv",
+    "extrair_e_exportar_csv",
+    "exportar_multiplos_csv",
+    # CSV Paralelo
+    "processar_csv_paralelo",
+    "calcular_chunksize",
+    # Arquivos
+    "limpar_pasta",
+    "remover_pasta_recursivamente",
+    "criar_pasta_se_nao_existir",
+    "listar_arquivos",
+]