sidra-sql 1.3.0__py3-none-any.whl

sidra_sql/utils.py

@@ -0,0 +1,106 @@
+import itertools
+from collections.abc import Iterable
+
+from sidra_fetcher.agregados import (
+    Categoria,
+    Classificacao,
+    Variavel,
+)
+
+
+def unnest_dimensoes(
+    variaveis: list[Variavel],
+    classificacoes: list[Classificacao],
+) -> Iterable[dict]:
+    """Expand variables × classification categories into flat Dimensao rows.
+
+    For each variable, computes the cartesian product of all categories
+    across every classification (up to 6, mapped to d4–d9).  The unit of
+    measure (``mc``/``mn``) is resolved with the following precedence:
+
+    1. The category's own ``unidade`` field (when not ``None``).
+    2. The variable's ``unidade`` field as a fallback.
+
+    Args:
+        variaveis: Iterable of :class:`~sidra_fetcher.agregados.Variavel`.
+        classificacoes: Iterable of
+            :class:`~sidra_fetcher.agregados.Classificacao`.
+
+    Returns:
+        A list of :class:`~sidra_sql.models.Dimensao` instances,
+        one per (variavel, combination-of-categories) tuple.
+    """
+
+    # Pre-build a list of (categoria_list,) per classificacao so that
+    # itertools.product can expand them correctly.
+    cats_per_classificacao: list[list[Categoria]] = [
+        classificacao.categorias for classificacao in classificacoes
+    ]
+
+    # Pad slots d4–d9: the model supports up to 6 classifications.
+    MAX_CLASSIFICACOES = 6
+
+    for variavel in variaveis:
+        variavel_id = str(variavel.id)
+        variavel_nome = variavel.nome
+        unidade_id = None
+        unidade_nome = variavel.unidade
+
+        if not cats_per_classificacao:
+            # No classifications: yield one row per variable with null d4–d9.
+            yield dict(
+                mc=unidade_id,
+                mn=unidade_nome,
+                d2c=variavel_id,
+                d2n=variavel_nome,
+                d4c=None,
+                d4n=None,
+                d5c=None,
+                d5n=None,
+                d6c=None,
+                d6n=None,
+                d7c=None,
+                d7n=None,
+                d8c=None,
+                d8n=None,
+                d9c=None,
+                d9n=None,
+            )
+            continue
+
+        # Cartesian product across all classifications.
+        for combo in itertools.product(*cats_per_classificacao):
+            # Resolve unit: first category that provides one wins;
+            # fall back to the variable's own unit.
+            for cat in combo:
+                if cat.unidade is not None:
+                    unidade_nome = cat.unidade
+                    break
+
+            # Map combo slots → d4…d9 (pad with None when fewer than 6).
+            padded = list(combo) + [None] * (MAX_CLASSIFICACOES - len(combo))
+
+            def _id(cat):
+                return str(cat.id) if cat is not None else None
+
+            def _nome(cat):
+                return cat.nome if cat is not None else None
+
+            yield dict(
+                mc=unidade_id,
+                mn=unidade_nome,
+                d2c=variavel_id,
+                d2n=variavel_nome,
+                d4c=_id(padded[0]),
+                d4n=_nome(padded[0]),
+                d5c=_id(padded[1]),
+                d5n=_nome(padded[1]),
+                d6c=_id(padded[2]),
+                d6n=_nome(padded[2]),
+                d7c=_id(padded[3]),
+                d7n=_nome(padded[3]),
+                d8c=_id(padded[4]),
+                d8n=_nome(padded[4]),
+                d9c=_id(padded[5]),
+                d9n=_nome(padded[5]),
+            )

sidra_sql/validator.py

@@ -0,0 +1,222 @@
+import tomllib
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+
+
+class Severity(Enum):
+    OK = "ok"
+    WARN = "warn"
+    ERROR = "error"
+
+
+@dataclass
+class Issue:
+    severity: Severity
+    message: str
+
+
+@dataclass
+class SectionReport:
+    title: str
+    issues: list[Issue] = field(default_factory=list)
+
+    def ok(self, msg: str) -> None:
+        self.issues.append(Issue(Severity.OK, msg))
+
+    def warn(self, msg: str) -> None:
+        self.issues.append(Issue(Severity.WARN, msg))
+
+    def error(self, msg: str) -> None:
+        self.issues.append(Issue(Severity.ERROR, msg))
+
+    @property
+    def errors(self) -> list[Issue]:
+        return [i for i in self.issues if i.severity == Severity.ERROR]
+
+    @property
+    def warnings(self) -> list[Issue]:
+        return [i for i in self.issues if i.severity == Severity.WARN]
+
+
+@dataclass
+class ValidationReport:
+    sections: list[SectionReport] = field(default_factory=list)
+
+    @property
+    def error_count(self) -> int:
+        return sum(len(s.errors) for s in self.sections)
+
+    @property
+    def warning_count(self) -> int:
+        return sum(len(s.warnings) for s in self.sections)
+
+    @property
+    def is_valid(self) -> bool:
+        return self.error_count == 0
+
+
+class PluginValidator:
+    def __init__(self, plugin_dir: Path):
+        self.plugin_dir = plugin_dir
+
+    def validate(self) -> ValidationReport:
+        report = ValidationReport()
+
+        manifest_section = SectionReport("manifest.toml")
+        report.sections.append(manifest_section)
+
+        manifest_path = self.plugin_dir / "manifest.toml"
+        if not manifest_path.exists():
+            manifest_section.error("manifest.toml não encontrado")
+            return report
+
+        try:
+            with open(manifest_path, "rb") as f:
+                manifest = tomllib.load(f)
+        except tomllib.TOMLDecodeError as e:
+            manifest_section.error(f"TOML inválido: {e}")
+            return report
+
+        manifest_section.ok("TOML válido")
+
+        if "name" not in manifest:
+            manifest_section.warn("Campo 'name' ausente")
+        if "version" not in manifest:
+            manifest_section.warn("Campo 'version' ausente")
+
+        pipelines = manifest.get("pipeline", [])
+        if not pipelines:
+            manifest_section.warn("Nenhum [[pipeline]] declarado")
+        else:
+            manifest_section.ok(f"{len(pipelines)} pipeline(s) declarado(s)")
+
+        ids_seen: set[str] = set()
+        valid_pipelines: list[dict] = []
+
+        for i, p in enumerate(pipelines):
+            entry = f"pipeline[{i}]"
+            pid = p.get("id")
+            ppath = p.get("path")
+
+            if not pid:
+                manifest_section.error(f"{entry}: campo 'id' ausente")
+                continue
+            if not ppath:
+                manifest_section.error(f"pipeline '{pid}': campo 'path' ausente")
+                continue
+            if pid in ids_seen:
+                manifest_section.error(f"ID duplicado: '{pid}'")
+                continue
+
+            ids_seen.add(pid)
+            valid_pipelines.append(p)
+
+        for p in valid_pipelines:
+            section = SectionReport(p["path"])
+            report.sections.append(section)
+            self._validate_pipeline(p["id"], p["path"], section)
+
+        return report
+
+    def _validate_pipeline(
+        self, pid: str, rel_path: str, section: SectionReport
+    ) -> None:
+        pipeline_dir = self.plugin_dir / rel_path
+
+        if not pipeline_dir.exists():
+            section.error(f"Diretório não encontrado: '{pipeline_dir}'")
+            return
+
+        has_fetch = (pipeline_dir / "fetch.toml").exists()
+        has_transform = (pipeline_dir / "transform.toml").exists()
+
+        if not has_fetch and not has_transform:
+            section.error("Nenhum fetch.toml ou transform.toml encontrado")
+            return
+
+        if has_fetch:
+            self._validate_fetch_toml(pipeline_dir, section)
+
+        if has_transform:
+            self._validate_transform_toml(pipeline_dir, section)
+
+    def _validate_fetch_toml(self, pipeline_dir: Path, section: SectionReport) -> None:
+        fetch_path = pipeline_dir / "fetch.toml"
+        try:
+            with open(fetch_path, "rb") as f:
+                data = tomllib.load(f)
+        except tomllib.TOMLDecodeError as e:
+            section.error(f"fetch.toml: TOML inválido: {e}")
+            return
+
+        tabelas = data.get("tabelas", [])
+        if not tabelas:
+            section.error("fetch.toml: nenhuma [[tabelas]] declarada")
+            return
+
+        for i, t in enumerate(tabelas):
+            if "tabela_sidra" not in t:
+                section.error(f"fetch.toml: tabelas[{i}] sem campo 'tabela_sidra'")
+
+        section.ok(f"fetch.toml válido ({len(tabelas)} tabela(s))")
+
+    def _validate_transform_toml(
+        self, pipeline_dir: Path, section: SectionReport
+    ) -> None:
+        transform_path = pipeline_dir / "transform.toml"
+        try:
+            with open(transform_path, "rb") as f:
+                data = tomllib.load(f)
+        except tomllib.TOMLDecodeError as e:
+            section.error(f"transform.toml: TOML inválido: {e}")
+            return
+
+        tables = data.get("table")
+        if isinstance(tables, dict):
+            section.error(
+                "transform.toml: schema [table] singular foi removido — "
+                "migre para [[table]] (array) com campo 'sql' por entrada"
+            )
+            return
+        if not isinstance(tables, list) or not tables:
+            section.error("transform.toml: nenhum [[table]] declarado")
+            return
+
+        seen: set[tuple[str, str]] = set()
+        any_error = False
+        for i, t in enumerate(tables):
+            entry = f"[[table]][{i}]"
+            missing = [f for f in ("name", "schema", "strategy", "sql") if f not in t]
+            if missing:
+                section.error(
+                    f"transform.toml: {entry} sem campo(s) obrigatório(s): "
+                    f"{', '.join(missing)}"
+                )
+                any_error = True
+                continue
+
+            strategy = t["strategy"]
+            if strategy not in ("replace", "view"):
+                section.error(
+                    f"transform.toml: {entry} strategy inválido: {strategy!r} "
+                    "(esperado 'replace' ou 'view')"
+                )
+                any_error = True
+
+            key = (t["schema"], t["name"])
+            if key in seen:
+                section.error(f"transform.toml: saída duplicada {key[0]}.{key[1]}")
+                any_error = True
+            seen.add(key)
+
+            sql_path = pipeline_dir / t["sql"]
+            if not sql_path.exists():
+                section.error(
+                    f"transform.toml: {entry} aponta para sql='{t['sql']}' "
+                    "mas o arquivo não existe"
+                )
+                any_error = True
+
+        if not any_error:
+            section.ok(f"transform.toml válido ({len(tables)} saída(s))")