jcpm-data 0.1.0__py3-none-any.whl

jcpm_data/__init__.py

@@ -0,0 +1,9 @@
+"""jcpm_data — framework de carga de dados do Grupo JCPM.
+
+A API de pipelines vive em ``jcpm_data.pipelines`` e é reexportada aqui por
+conveniência (``from jcpm_data import Pipeline, ScopeConfig, ...``). O CLI de
+scaffolding (``jcpm-data startproject``) está em ``jcpm_data.cli``.
+"""
+
+from .pipelines import *  # noqa: F401,F403
+from .pipelines import __all__, __version__  # noqa: F401

jcpm_data/cli.py

@@ -0,0 +1,283 @@
+"""CLI do framework — ``jcpm-data startproject`` (scaffolding via copier)."""
+
+from __future__ import annotations
+
+import importlib.util
+import re
+import subprocess
+import sys
+import unicodedata
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator, List, Optional, Sequence
+
+import click
+
+TEMPLATE_DIR = Path(__file__).resolve().parent / "templates" / "project"
+
+_SKIP_DIRS = {".venv", "venv", "__pycache__", ".git", "node_modules", "dist", "build"}
+
+SA_ACCOUNT_ID_MAX = 30
+DEVOPS_PREFIX_RESERVED = 3
+PROJECT_NAME_MAX = SA_ACCOUNT_ID_MAX - DEVOPS_PREFIX_RESERVED  # 27
+
+
+def _project_slug(name: str) -> str:
+    """Slug snake_case do projeto: minúsculo, sem acento, só ``[a-z0-9_]``.
+
+    Ex.: ``"Vendas Diárias"`` -> ``"vendas_diarias"``. É o nome usado no
+    ``deploy.yml`` e no ``pyproject.toml`` do projeto gerado.
+    """
+    nfkd = unicodedata.normalize("NFKD", name)
+    without_accents = "".join(c for c in nfkd if not unicodedata.combining(c))
+    slug = re.sub(r"[^a-z0-9]+", "_", without_accents.lower())
+    return slug.strip("_")
+
+
+def _project_name_error(deploy_name: str) -> Optional[str]:
+    """Erro se o nome não couber no limite da SA após o prefixo da devops."""
+    if len(deploy_name) > PROJECT_NAME_MAX:
+        return (
+            f"nome do projeto '{deploy_name}' tem {len(deploy_name)} caracteres; "
+            f"o máximo é {PROJECT_NAME_MAX} (a plataforma devops adiciona um prefixo "
+            f"e o account_id da service account no GCP tem {SA_ACCOUNT_ID_MAX} chars)."
+        )
+    return None
+
+
+def scaffold(
+    name: str,
+    location: Optional[str] = ".",
+    *,
+    scopes: Optional[Sequence[str]] = None,
+    interactive: bool = False,
+) -> Path:
+    """Cria um novo projeto a partir do template copier.
+
+    Args:
+        name: nome do projeto (obrigatório).
+        location: diretório de destino. ``.``/vazio/``None`` usa o diretório atual.
+        scopes: escopos do projeto. O template **não** tem default — no modo
+            interativo o copier pergunta (multiselect, validando a exclusividade
+            de jcpm/external); no modo programático (padrão), sem ``scopes`` usa-se
+            um escopo mínimo (``['jcpm']``) para não exigir prompt.
+        interactive: se ``True``, o copier pergunta os campos sem resposta (os
+            escopos). Usado pelo comando ``startproject``.
+
+    Returns:
+        O ``Path`` do diretório onde o projeto foi criado.
+    """
+    if not name:
+        raise ValueError("Nome do projeto é obrigatório.")
+
+    slug = _project_slug(name)
+    name_error = _project_name_error(slug)
+    if name_error:
+        raise ValueError(name_error)
+
+    try:
+        from copier import run_copy
+    except ImportError as exc:  # pragma: no cover - depende do ambiente
+        raise SystemExit(
+            "O comando 'startproject' requer o copier. "
+            "Instale com: uv add 'jcpm-data[cli]'  (ou pip install 'jcpm-data[cli]')."
+        ) from exc
+
+    dest = Path.cwd() if location in (".", "", None) else Path(location)
+    dest.mkdir(parents=True, exist_ok=True)
+
+    # Passa o slug (snake_case, sem acento) computado em Python — o Jinja do
+    # copier não remove acentos.
+    data: dict = {"project_name": name, "project_slug": slug}
+    if scopes:
+        data["scopes"] = list(scopes)
+    elif not interactive:
+        data["scopes"] = ["jcpm"]
+
+    run_copy(
+        str(TEMPLATE_DIR),
+        str(dest),
+        data=data,
+        defaults=not interactive,
+        quiet=not interactive,
+    )
+    return dest
+
+
+@click.group()
+@click.version_option(package_name="jcpm-data", prog_name="jcpm-data")
+def main() -> None:
+    """CLI do framework jcpm-data."""
+
+
+@main.command(name="startproject")
+@click.argument("name", required=False)
+@click.argument("location", required=False)
+def startproject(name: Optional[str], location: Optional[str]) -> None:
+    """Cria um novo projeto a partir do template.
+
+    NAME é o nome do projeto; LOCATION o destino (. = diretório atual).
+    """
+    if not name:
+        name = click.prompt("Nome do projeto")
+    if location is None:
+        location = click.prompt("Local do projeto", default=".")
+
+    # Interativo: o copier pergunta os escopos (multiselect), respeitando a
+    # exclusividade de jcpm/external. Sem default — o usuário escolhe.
+    dest = scaffold(name, location, interactive=True)
+    click.secho(f"Projeto '{name}' criado em {dest}", fg="green")
+
+
+# --------------------------------------------------------------------------- #
+# check
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class CheckResult:
+    name: str
+    status: str  # "ok" | "fail" | "skip"
+    detail: str = ""
+
+
+def _py_files(root: Path) -> Iterator[Path]:
+    for path in root.rglob("*.py"):
+        rel = path.relative_to(root)
+        if any(part in _SKIP_DIRS or part.startswith(".") for part in rel.parts):
+            continue
+        yield path
+
+
+def _module_name(root: Path, path: Path) -> str:
+    parts = list(path.relative_to(root).with_suffix("").parts)
+    if parts and parts[-1] == "__init__":
+        parts = parts[:-1]
+    return ".".join(parts)
+
+
+def _check_deploy_yaml(root: Path) -> CheckResult:
+    deploy = root / "deploy.yml"
+    if not deploy.exists():
+        return CheckResult("deploy.yml", "fail", "arquivo não encontrado")
+    if importlib.util.find_spec("yaml") is None:  # pragma: no cover
+        return CheckResult("deploy.yml", "skip", "pyyaml indisponível")
+    import yaml
+
+    try:
+        data = yaml.safe_load(deploy.read_text()) or {}
+    except yaml.YAMLError as exc:
+        return CheckResult("deploy.yml", "fail", f"YAML inválido: {exc}")
+    missing = [k for k in ("apiVersion", "name") if not data.get(k)]
+    if missing:
+        return CheckResult("deploy.yml", "fail", f"campos ausentes: {missing}")
+    name_error = _project_name_error(str(data["name"]))
+    if name_error:
+        return CheckResult("deploy.yml", "fail", name_error)
+    return CheckResult(
+        "deploy.yml", "ok", f"name={data['name']}, apiVersion={data['apiVersion']}"
+    )
+
+
+def _check_syntax(root: Path) -> CheckResult:
+    errors = []
+    for path in _py_files(root):
+        try:
+            compile(path.read_text(), str(path), "exec")
+        except SyntaxError as exc:
+            errors.append(f"{path.relative_to(root)}:{exc.lineno}: {exc.msg}")
+    if errors:
+        return CheckResult("sintaxe", "fail", "; ".join(errors))
+    return CheckResult("sintaxe", "ok")
+
+
+def _check_imports(root: Path) -> CheckResult:
+    pkg = root / "project"
+    if not pkg.is_dir():
+        return CheckResult("imports", "skip", "pasta project/ não encontrada")
+    modules = sorted({_module_name(root, p) for p in _py_files(pkg)})
+    modules = [m for m in modules if m]
+    if not modules:
+        return CheckResult("imports", "skip", "nenhum módulo em project/")
+    code = "import importlib\n" + "\n".join(
+        f"importlib.import_module({m!r})" for m in modules
+    )
+    proc = subprocess.run(
+        [sys.executable, "-c", code],
+        cwd=str(root),
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    if proc.returncode != 0:
+        detail = (proc.stderr.strip().splitlines() or ["erro de import"])[-1]
+        return CheckResult("imports", "fail", detail)
+    return CheckResult("imports", "ok", f"{len(modules)} módulo(s)")
+
+
+def _check_lint(root: Path) -> CheckResult:
+    if importlib.util.find_spec("ruff") is None:
+        return CheckResult("lint (ruff)", "skip", "ruff não instalado")
+    proc = subprocess.run(
+        [sys.executable, "-m", "ruff", "check", str(root)],
+        cwd=str(root),
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    if proc.returncode != 0:
+        return CheckResult("lint (ruff)", "fail", proc.stdout.strip() or proc.stderr.strip())
+    return CheckResult("lint (ruff)", "ok")
+
+
+def run_checks(path: str = ".") -> List[CheckResult]:
+    """Roda todas as verificações sobre o projeto em ``path``."""
+    root = Path(path).resolve()
+    return [
+        _check_deploy_yaml(root),
+        _check_syntax(root),
+        _check_imports(root),
+        _check_lint(root),
+    ]
+
+
+def run_project(path: str = ".") -> int:
+    """Executa ``project/main.py`` do projeto (via ``python -m project.main``)."""
+    root = Path(path).resolve()
+    main_file = root / "project" / "main.py"
+    if not main_file.exists():
+        raise SystemExit(
+            f"'{main_file}' não encontrado — rode dentro de um projeto jcpm-data."
+        )
+    proc = subprocess.run([sys.executable, "-m", "project.main"], cwd=str(root))
+    return proc.returncode
+
+
+@main.command(name="run")
+@click.option("--path", default=".", help="Diretório do projeto (default: atual).")
+def run(path: str) -> None:
+    """Executa o pipeline do projeto (`project/main.py`)."""
+    raise SystemExit(run_project(path))
+
+
+@main.command()
+@click.option("--path", default=".", help="Diretório do projeto (default: atual).")
+def check(path: str) -> None:
+    """Verifica o projeto: linter, estrutura e imports quebrados."""
+    colors = {"ok": "green", "fail": "red", "skip": "yellow"}
+    failed = False
+    for r in run_checks(path):
+        line = f"{r.name}: {r.status}"
+        if r.detail:
+            line += f" — {r.detail}"
+        click.secho(line, fg=colors[r.status])
+        failed = failed or r.status == "fail"
+
+    if failed:
+        click.secho("\nFalhas encontradas.", fg="red", bold=True)
+        raise SystemExit(1)
+    click.secho("\nTudo certo.", fg="green", bold=True)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

jcpm_data/pipelines/__init__.py

@@ -0,0 +1,130 @@
+from .bigquery import (
+    PARTITION,
+    SCHEMA_POLICY,
+    WRITE_MODE,
+    BigQueryConfig,
+    Column,
+    DataSource,
+    Partition,
+    SchemaPolicy,
+    WriteMode,
+    data,
+    infer_schema,
+)
+from .config import (
+    ScopeConfig,
+    ScopeName,
+    build_table_name,
+    ensure_scope,
+    is_scope,
+)
+from .context import Context, get_scope_from_context
+from .error_codes import ErrorCatalog, ErrorCode, PipelineError
+from .quality import PrimaryKeyCheck, check_primary_key
+from .errors import (
+    ColumnNameValidationError,
+    DatePartitionValidationError,
+    EnvVarNotFoundError,
+    IdentifierValidationError,
+    JcpmDataError,
+    PrimaryKeyValidationError,
+    SchemaValidationError,
+    ScopeValidationError,
+    TaskIndexError,
+    ValidationError,
+)
+from .logger import log, logger
+from .observability import (
+    Event,
+    EventType,
+    PipelineRun,
+    RunSummary,
+    Severity,
+    Status,
+)
+from .pipeline import Pipeline
+from .storage import DataLake, StorageConfig
+from .utils import (
+    env_var_name,
+    get_env,
+    get_object_by_task_id,
+    get_scope_by_task_id,
+    load_env,
+    range_dates,
+    resolve_task_index,
+    stand_names,
+)
+from .validators import validate_identifier
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # config
+    "ScopeConfig",
+    "ScopeName",
+    "BigQueryConfig",
+    "build_table_name",
+    "ensure_scope",
+    "is_scope",
+    # env
+    "get_env",
+    "env_var_name",
+    "load_env",
+    # context
+    "Context",
+    "get_scope_from_context",
+    # tasks (Cloud Run Jobs)
+    "get_object_by_task_id",
+    "get_scope_by_task_id",
+    "resolve_task_index",
+    # log
+    "logger",
+    "log",
+    # data API (@data / BigQuery)
+    "data",
+    "DataSource",
+    "Column",
+    "WriteMode",
+    "WRITE_MODE",
+    "Partition",
+    "PARTITION",
+    "SchemaPolicy",
+    "SCHEMA_POLICY",
+    # dev / qualidade
+    "infer_schema",
+    "check_primary_key",
+    "PrimaryKeyCheck",
+    # utils extras
+    "range_dates",
+    "stand_names",
+    # storage (GCS)
+    "StorageConfig",
+    "DataLake",
+    # pipeline
+    "Pipeline",
+    # observabilidade (eventos padronizados)
+    "PipelineRun",
+    "EventType",
+    "Status",
+    "Severity",
+    "Event",
+    "RunSummary",
+    # códigos de erro do projeto
+    "ErrorCatalog",
+    "ErrorCode",
+    "PipelineError",
+    # validators
+    "validate_identifier",
+    # errors
+    "JcpmDataError",
+    "ValidationError",
+    "EnvVarNotFoundError",
+    "TaskIndexError",
+    "ScopeValidationError",
+    "IdentifierValidationError",
+    "SchemaValidationError",
+    "ColumnNameValidationError",
+    "DatePartitionValidationError",
+    "PrimaryKeyValidationError",
+    "__version__",
+]

jcpm_data/pipelines/bigquery/__init__.py

@@ -0,0 +1,51 @@
+"""Pacote BigQuery — config do destino, schema e a API ``@data``."""
+
+from .config import BigQueryConfig
+from .data import DataSource, data, validate_datasource
+from .destination import (
+    build_create_table_ddl,
+    build_data_resource,
+    build_destination,
+    build_dlt_pipeline,
+    ensure_partitioned_table,
+)
+from .infer import infer_schema
+from .options import (
+    DATE_PARTITION_COLUMN,
+    PARTITION,
+    SCHEMA_POLICY,
+    WRITE_MODE,
+    Partition,
+    SchemaPolicy,
+    WriteMode,
+    partition_sql,
+)
+from .schema import Column, cast_to_schema, columns_to_dlt
+
+__all__ = [
+    # config / destino
+    "BigQueryConfig",
+    "build_destination",
+    "build_dlt_pipeline",
+    "build_data_resource",
+    # API @data
+    "data",
+    "DataSource",
+    "validate_datasource",
+    "Column",
+    "columns_to_dlt",
+    "cast_to_schema",
+    # dev helper
+    "infer_schema",
+    # opções
+    "WriteMode",
+    "WRITE_MODE",
+    "Partition",
+    "PARTITION",
+    "SchemaPolicy",
+    "SCHEMA_POLICY",
+    "DATE_PARTITION_COLUMN",
+    "partition_sql",
+    "build_create_table_ddl",
+    "ensure_partitioned_table",
+]

jcpm_data/pipelines/bigquery/config.py

@@ -0,0 +1,50 @@
+"""Configuração do destino BigQuery."""
+
+from __future__ import annotations
+
+import os
+from typing import Optional
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class BigQueryConfig(BaseModel):
+    """Config do destino BigQuery.
+
+    Os defaults são derivados do ambiente de execução (``.env`` / variáveis de
+    ambiente / ADC) em vez de fixados no código.
+    """
+
+    @model_validator(mode="before")
+    @classmethod
+    def _load_dotenv(cls, data):
+        # Carrega o .env mais próximo para que os defaults (e os.environ do
+        # consumidor) enxerguem as variáveis — idempotente.
+        from ..utils.env import load_env
+
+        load_env()
+        return data
+
+    project_id: Optional[str] = Field(
+        default_factory=lambda: os.getenv("JCPM_BQ_PROJECT_ID")
+        or os.getenv("GOOGLE_CLOUD_PROJECT"),
+        description="ID do projeto GCP. Se nulo, usa credenciais implícitas (ADC).",
+    )
+    location: str = Field(
+        default_factory=lambda: os.getenv("JCPM_BQ_LOCATION", "us-central1"),
+        description="Região do dataset no BigQuery",
+    )
+    dataset_name: str = Field(
+        default_factory=lambda: os.getenv("JCPM_BQ_DATASET", "raw"),
+        description="Dataset de destino (camada). Default: raw",
+    )
+    truncate_staging: bool = Field(
+        default_factory=lambda: os.getenv("JCPM_BQ_TRUNCATE_STAGING", "true").lower()
+        != "false",
+        description="Limpa (trunca) o dataset de staging após a carga. Default: True.",
+    )
+    add_load_id: bool = Field(
+        default_factory=lambda: os.getenv("JCPM_BQ_ADD_LOAD_ID", "true").lower()
+        != "false",
+        description="Adiciona a coluna '_dlt_load_id' (id da carga) na tabela final. Default: True.",
+    )

jcpm_data/pipelines/bigquery/data.py

@@ -0,0 +1,143 @@
+"""Decorator ``@data`` e o objeto ``DataSource`` que carrega a config de carga.
+
+Diferente do ``@source`` clássico (handler retorna ``Data``), aqui toda a
+configuração de gravação vive no decorator e o handler apenas produz os registros
+brutos (``yield`` de lotes). Ex.:
+
+    @data(source="raw_crm", resource="usuarios", write_mode=WRITE_MODE.MERGE,
+          primary_key=["id"], partition_by=PARTITION.DAY, schema=USERS_SCHEMA)
+    def extract_crm_users(context):
+        yield api.get_users()
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Callable, List, Optional, Union
+
+from ..errors import (
+    DatePartitionValidationError,
+    PrimaryKeyValidationError,
+    SchemaValidationError,
+)
+from ..validators import validate_identifier
+from .options import (
+    DATE_PARTITION_COLUMN,
+    Partition,
+    SchemaPolicy,
+    WriteMode,
+    needs_custom_partitioning,
+)
+from .schema import Column, schema_column_names
+
+
+def _as_list(value: Optional[Union[str, List[str]]]) -> Optional[List[str]]:
+    if value is None:
+        return None
+    return [value] if isinstance(value, str) else list(value)
+
+
+@dataclass
+class DataSource:
+    """Fonte de dados + configuração de carga no BigQuery."""
+
+    handler: Callable
+    source: str
+    resource: str
+    write_mode: WriteMode = WriteMode.APPEND
+    primary_key: Optional[List[str]] = None
+    partition_by: Optional[Partition] = None
+    cluster_by: Optional[List[str]] = None
+    cursor_field: Optional[str] = None
+    schema: Optional[List[Column]] = None
+    schema_policy: Optional[SchemaPolicy] = None
+
+    def __post_init__(self) -> None:
+        self.primary_key = _as_list(self.primary_key)
+        self.cluster_by = _as_list(self.cluster_by)
+
+    def __call__(self, context):
+        """Executa o handler — DataSource é usável como o próprio callable."""
+        return self.handler(context)
+
+
+def data(
+    *,
+    source: str,
+    resource: str,
+    write_mode: WriteMode = WriteMode.APPEND,
+    primary_key: Optional[Union[str, List[str]]] = None,
+    partition_by: Optional[Partition] = None,
+    cluster_by: Optional[Union[str, List[str]]] = None,
+    cursor_field: Optional[str] = None,
+    schema: Optional[List[Column]] = None,
+    schema_policy: Optional[SchemaPolicy] = None,
+) -> Callable[[Callable], DataSource]:
+    """Decorator que transforma um handler em um :class:`DataSource`."""
+
+    def decorator(handler: Callable) -> DataSource:
+        return DataSource(
+            handler=handler,
+            source=source,
+            resource=resource,
+            write_mode=write_mode,
+            primary_key=primary_key,
+            partition_by=partition_by,
+            cluster_by=cluster_by,
+            cursor_field=cursor_field,
+            schema=schema,
+            schema_policy=schema_policy,
+        )
+
+    return decorator
+
+
+def validate_datasource(ds: DataSource) -> None:
+    """Valida a configuração de um ``DataSource`` (fail-fast antes da carga)."""
+    cols = schema_column_names(ds.schema) if ds.schema else None
+
+    # Nomes de coluna do schema devem ser identificadores válidos (snake_case).
+    for name in cols or []:
+        validate_identifier(name, field="coluna")
+
+    # MERGE exige primary_key.
+    if ds.write_mode is WriteMode.MERGE and not ds.primary_key:
+        raise PrimaryKeyValidationError(
+            "write_mode=MERGE exige 'primary_key'."
+        )
+
+    # REPLACE_PARTITION substitui a partição via particao_data — exige partition_by.
+    if ds.write_mode is WriteMode.REPLACE_PARTITION and ds.partition_by is None:
+        raise DatePartitionValidationError(
+            "write_mode=REPLACE_PARTITION exige 'partition_by' "
+            "(a partição é substituída pela coluna particao_data)."
+        )
+
+    # Partição: a coluna é sempre 'particao_data' e deve existir no schema.
+    if ds.partition_by is not None and cols is not None:
+        if DATE_PARTITION_COLUMN not in cols:
+            raise DatePartitionValidationError(
+                f"a coluna de partição '{DATE_PARTITION_COLUMN}' não está no schema. "
+                f"Colunas: {cols}"
+            )
+
+    # Granularidade não-diária (MONTH/YEAR/HOUR) exige schema para pré-criar a
+    # tabela com DATE_TRUNC (o dlt só particiona por DAY nativamente).
+    if needs_custom_partitioning(ds.partition_by) and not ds.schema:
+        raise DatePartitionValidationError(
+            f"particionamento por {ds.partition_by.name} exige 'schema' "
+            f"(para criar a tabela particionada corretamente no BigQuery)."
+        )
+
+    # Colunas referenciadas devem existir no schema (quando schema informado).
+    if cols is not None:
+        for field_name, ref in (("primary_key", ds.primary_key), ("cluster_by", ds.cluster_by)):
+            for c in ref or []:
+                if c not in cols:
+                    raise SchemaValidationError(
+                        f"{field_name}: coluna '{c}' não está no schema. Colunas: {cols}"
+                    )
+        if ds.cursor_field and ds.cursor_field not in cols:
+            raise SchemaValidationError(
+                f"cursor_field '{ds.cursor_field}' não está no schema. Colunas: {cols}"
+            )