sidra-sql 1.3.0__py3-none-any.whl

sidra_sql/exporter.py

@@ -0,0 +1,171 @@
+"""Export materialized analytics outputs (and as-of vintage snapshots) to CSV.
+
+Reads a pipeline's ``transform.toml`` to discover its ``[[table]]`` outputs and
+streams each to a CSV file using the PostgreSQL ``COPY ... TO STDOUT`` protocol
+(no DataFrame/Polars dependency).
+
+Two modes:
+
+* **current** — ``COPY (SELECT * FROM "<schema>"."<name>")`` for each
+  materialized analytics table/view (the current truth).
+* **as-of** — reconstruct the outputs as IBGE published them on a date. A temp
+  view ``dados`` shadows the real fact table (``search_path`` puts ``pg_temp``
+  first), exposing, per logical key, the row with the greatest
+  ``modificacao <= date`` flagged ``ativo``. The pipeline's own transform
+  ``.sql`` runs against it unchanged — its ``WHERE ativo = true`` passes exactly
+  that vintage — and the result is streamed to CSV. Persisted ``analytics.*``
+  tables are left untouched.
+"""
+
+import datetime as dt
+import tomllib
+from pathlib import Path
+
+from rich.console import Console
+
+from . import database
+from .config import Config
+
+
+def read_outputs(toml_path: Path) -> list[dict]:
+    """Return the ``[[table]]`` entries declared in a ``transform.toml``."""
+    with open(toml_path, "rb") as f:
+        data = tomllib.load(f)
+    tables = data.get("table")
+    if not isinstance(tables, list) or not tables:
+        raise ValueError(f"{toml_path}: esperado um ou mais [[table]] (array).")
+    for entry in tables:
+        missing = [k for k in ("name", "schema", "sql") if k not in entry]
+        if missing:
+            raise ValueError(
+                f"{toml_path}: [[table]] sem campo(s) obrigatório(s): "
+                f"{', '.join(missing)}"
+            )
+    return tables
+
+
+def _clean_query(sql: str) -> str:
+    """Strip trailing whitespace/semicolons so the SELECT can be embedded."""
+    return sql.strip().rstrip(";").strip()
+
+
+def copy_table_sql(schema: str, name: str) -> str:
+    """COPY that dumps a materialized table/view to CSV on STDOUT."""
+    return (
+        f'COPY (SELECT * FROM "{schema}"."{name}") TO STDOUT WITH (FORMAT csv, HEADER)'
+    )
+
+
+def copy_query_sql(query: str) -> str:
+    """COPY that dumps an arbitrary SELECT to CSV on STDOUT."""
+    return f"COPY ({_clean_query(query)}) TO STDOUT WITH (FORMAT csv, HEADER)"
+
+
+def asof_view_sql(schema: str, asof: dt.date) -> str:
+    """``CREATE TEMP VIEW dados`` exposing the as-of vintage as the active rows.
+
+    Per logical key, picks the row with the greatest ``modificacao <= asof`` and
+    flags it ``ativo``, so transforms filtering ``WHERE ativo = true`` see the
+    data exactly as published on that date. The inner source is
+    schema-qualified so the temp view does not reference itself. The date is a
+    validated ``date`` object, inlined as a literal (no parameters allowed in a
+    view definition).
+    """
+    return (
+        "CREATE TEMP VIEW dados AS"
+        " SELECT id, tabela_sidra_id, dimensao_id, localidade_id, periodo_id,"
+        " modificacao, TRUE AS ativo, v FROM ("
+        " SELECT DISTINCT ON"
+        " (tabela_sidra_id, localidade_id, dimensao_id, periodo_id)"
+        " id, tabela_sidra_id, dimensao_id, localidade_id, periodo_id,"
+        " modificacao, v"
+        f' FROM "{schema}".dados'
+        f" WHERE modificacao <= DATE '{asof.isoformat()}'"
+        " ORDER BY tabela_sidra_id, localidade_id, dimensao_id, periodo_id,"
+        " modificacao DESC"
+        " ) AS _asof"
+    )
+
+
+def stamped_name(name: str, asof: dt.date | None) -> str:
+    """``name.csv`` for the current truth, ``name@YYYYMMDD.csv`` for as-of."""
+    if asof is None:
+        return f"{name}.csv"
+    return f"{name}@{asof:%Y%m%d}.csv"
+
+
+class Exporter:
+    """Export a pipeline's analytics outputs to CSV files."""
+
+    def __init__(
+        self,
+        config: Config,
+        toml_path: Path,
+        console: Console | None = None,
+    ):
+        self.config = config
+        self.toml_path = toml_path
+        self.console = console
+
+    def export(self, output_dir: Path, asof: dt.date | None = None) -> list[Path]:
+        """Export every declared output; return the paths actually written."""
+        outputs = read_outputs(self.toml_path)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        engine = database.get_engine(self.config)
+        written: list[Path] = []
+
+        with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
+            pg_conn = conn.connection.driver_connection
+            cur = pg_conn.cursor()
+
+            if asof is not None:
+                # pg_temp first so the temp view shadows the real `dados`.
+                cur.execute(f"SET search_path = pg_temp, {self.config.db_schema}")
+                cur.execute(asof_view_sql(self.config.db_schema, asof))
+
+            for entry in outputs:
+                name = entry["name"]
+                schema = entry["schema"]
+                out_path = output_dir / stamped_name(name, asof)
+
+                if asof is None:
+                    sql = copy_table_sql(schema, name)
+                else:
+                    query = (self.toml_path.parent / entry["sql"]).read_text(
+                        encoding="utf-8"
+                    )
+                    sql = copy_query_sql(query)
+
+                try:
+                    self._copy_to_file(cur, sql, out_path)
+                except Exception as exc:  # noqa: BLE001
+                    self._warn(name, schema, asof, exc)
+                    continue
+
+                written.append(out_path)
+                if self.console is not None:
+                    self.console.print(f"  [green]✓[/green] {out_path}")
+
+        return written
+
+    @staticmethod
+    def _copy_to_file(cur, sql: str, out_path: Path) -> None:
+        with cur.copy(sql) as copy:
+            with open(out_path, "wb") as fh:
+                for chunk in copy:
+                    fh.write(chunk)
+
+    def _warn(
+        self, name: str, schema: str, asof: dt.date | None, exc: Exception
+    ) -> None:
+        if self.console is None:
+            return
+        if asof is None:
+            self.console.print(
+                f"  [yellow]![/yellow] {schema}.{name} não encontrada — "
+                "rode 'sidra-sql run' para materializar. (pulando)"
+            )
+        else:
+            self.console.print(
+                f"  [yellow]![/yellow] falha ao exportar {name} as-of: {exc}"
+            )

sidra_sql/models.py

@@ -0,0 +1,230 @@
+import datetime as dt
+
+import sqlalchemy as sa
+from sqlalchemy import (
+    BigInteger,
+    Boolean,
+    CheckConstraint,
+    Date,
+    DateTime,
+    ForeignKey,
+    Identity,
+    Integer,
+    SmallInteger,
+    Text,
+    UniqueConstraint,
+    func,
+)
+from sqlalchemy.dialects.postgresql import ARRAY, JSONB
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class TabelaSidra(Base):
+    __tablename__ = "tabela_sidra"
+    id: Mapped[str] = mapped_column(Text, primary_key=True)
+    nome: Mapped[str] = mapped_column(Text, nullable=False)
+    periodicidade: Mapped[str] = mapped_column(Text, nullable=False)
+    ultima_atualizacao: Mapped[Date] = mapped_column(
+        Date,
+        nullable=False,
+        default=func.now(),
+        onupdate=func.now(),
+    )
+    metadados: Mapped[JSONB] = mapped_column(JSONB, nullable=True)
+    dados = relationship("Dados", back_populates="tabela_sidra")
+
+
+class Localidade(Base):
+    __tablename__ = "localidade"
+    __table_args__ = (
+        UniqueConstraint(
+            "nc",
+            "d1c",
+            name="uq_localidade",
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(
+        BigInteger,
+        Identity(always=True),
+        primary_key=True,
+    )
+    dados = relationship("Dados", back_populates="localidade")
+    # NC = NIVEL TERRITORIAL ID
+    nc: Mapped[str] = mapped_column(Text, nullable=False)
+    # NN = NIVEL TERRITORIAL NOME
+    nn: Mapped[str] = mapped_column(Text, nullable=False)
+    # D1C = UNIDADE TERRITORIAL ID
+    d1c: Mapped[str] = mapped_column(Text, nullable=False)
+    # D1N = UNIDADE TERRITORIAL NOME
+    d1n: Mapped[str] = mapped_column(Text, nullable=False)
+
+
+class Periodo(Base):
+    __tablename__ = "periodo"
+    __table_args__ = (
+        UniqueConstraint(
+            "codigo",
+            "literals",
+            name="uq_periodo",
+            postgresql_nulls_not_distinct=True,
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, Identity(always=True), primary_key=True)
+    codigo: Mapped[str] = mapped_column(Text, nullable=False)
+    frequencia: Mapped[str | None] = mapped_column(Text)
+    literals: Mapped[list[str] | None] = mapped_column(ARRAY(Text))
+    data_inicio: Mapped[dt.date | None] = mapped_column(Date)
+    data_fim: Mapped[dt.date | None] = mapped_column(Date)
+    ano: Mapped[int | None] = mapped_column(Integer)
+    ano_fim: Mapped[int | None] = mapped_column(Integer)
+    semestre: Mapped[int | None] = mapped_column(
+        SmallInteger,
+        CheckConstraint("semestre IN (1, 2)"),
+    )
+    trimestre: Mapped[int | None] = mapped_column(
+        SmallInteger,
+        CheckConstraint("trimestre IN (1, 2, 3, 4)"),
+    )
+    mes: Mapped[int | None] = mapped_column(
+        SmallInteger,
+        CheckConstraint("mes IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)"),
+    )
+    dados = relationship("Dados", back_populates="periodo")
+
+
+class Dimensao(Base):
+    __tablename__ = "dimensao"
+    __table_args__ = (
+        UniqueConstraint(
+            "mc",
+            "d2c",
+            "d4c",
+            "d5c",
+            "d6c",
+            "d7c",
+            "d8c",
+            "d9c",
+            name="uq_dimensao",
+            postgresql_nulls_not_distinct=True,
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(
+        BigInteger,
+        Identity(always=True),
+        primary_key=True,
+    )
+    dados = relationship("Dados", back_populates="dimensao")
+    # MC = UNIDADE ID
+    mc: Mapped[str] = mapped_column(Text, nullable=True)
+    # MN = UNIDADE NOME
+    mn: Mapped[str] = mapped_column(Text, nullable=False)
+    # D2C = VARIAVEL ID
+    d2c: Mapped[str] = mapped_column(Text, nullable=False)
+    # D2N = VARIAVEL NOME
+    d2n: Mapped[str] = mapped_column(Text, nullable=False)
+    # D4C = CATEGORIA ID da Classificação 1
+    d4c: Mapped[str] = mapped_column(Text, nullable=True)
+    # D4N = CATEGORIA NOME da Classificação 1
+    d4n: Mapped[str] = mapped_column(Text, nullable=True)
+    # D5C = CATEGORIA ID da Classificação 2
+    d5c: Mapped[str] = mapped_column(Text, nullable=True)
+    # D5N = CATEGORIA NOME da Classificação 2
+    d5n: Mapped[str] = mapped_column(Text, nullable=True)
+    # D6C = CATEGORIA ID da Classificação 3
+    d6c: Mapped[str] = mapped_column(Text, nullable=True)
+    # D6N = CATEGORIA NOME da Classificação 3
+    d6n: Mapped[str] = mapped_column(Text, nullable=True)
+    # D7C = CATEGORIA ID da Classificação 4
+    d7c: Mapped[str] = mapped_column(Text, nullable=True)
+    # D7N = CATEGORIA NOME da Classificação 4
+    d7n: Mapped[str] = mapped_column(Text, nullable=True)
+    # D8C = CATEGORIA ID da Classificação 5
+    d8c: Mapped[str] = mapped_column(Text, nullable=True)
+    # D8N = CATEGORIA NOME da Classificação 5
+    d8n: Mapped[str] = mapped_column(Text, nullable=True)
+    # D9C = CATEGORIA ID da Classificação 6
+    d9c: Mapped[str] = mapped_column(Text, nullable=True)
+    # D9N = CATEGORIA NOME da Classificação 6
+    d9n: Mapped[str] = mapped_column(Text, nullable=True)
+
+
+class ArquivoCarregado(Base):
+    __tablename__ = "arquivo_carregado"
+
+    arquivo: Mapped[str] = mapped_column(Text, primary_key=True)
+    tabela_sidra_id: Mapped[str] = mapped_column(
+        ForeignKey("tabela_sidra.id"), nullable=False, index=True
+    )
+    carregado_em: Mapped[dt.datetime] = mapped_column(
+        DateTime(timezone=True),
+        nullable=False,
+        server_default=func.now(),
+    )
+
+
+class Dados(Base):
+    __tablename__ = "dados"
+    __table_args__ = (
+        sa.Index("ix_dados_periodo", "tabela_sidra_id", "periodo_id"),
+        # Vintage storage: várias revisões (modificacao) coexistem por chave,
+        # no máximo uma ativa. O índice único parcial WHERE ativo garante essa
+        # invariante; as 4 colunas são NOT NULL, então não precisa de
+        # NULLS NOT DISTINCT (sem requisito de PostgreSQL >= 15).
+        sa.Index(
+            "uq_dados_ativo",
+            "tabela_sidra_id",
+            "localidade_id",
+            "dimensao_id",
+            "periodo_id",
+            unique=True,
+            postgresql_where=sa.text("ativo"),
+        ),
+        # Apoio à reconstrução as-of: a linha de maior modificacao <= data.
+        sa.Index(
+            "ix_dados_asof",
+            "tabela_sidra_id",
+            "periodo_id",
+            "modificacao",
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(
+        BigInteger,
+        Identity(always=True),
+        primary_key=True,
+    )
+    tabela_sidra_id: Mapped[str] = mapped_column(
+        ForeignKey("tabela_sidra.id"),
+        nullable=False,
+        index=True,
+    )
+    tabela_sidra = relationship("TabelaSidra", back_populates="dados")
+    dimensao_id: Mapped[int] = mapped_column(
+        ForeignKey("dimensao.id"),
+        nullable=False,
+        index=True,
+    )
+    dimensao = relationship("Dimensao", back_populates="dados")
+    localidade_id: Mapped[int] = mapped_column(
+        ForeignKey("localidade.id"),
+        nullable=False,
+        index=True,
+    )
+    localidade = relationship("Localidade", back_populates="dados")
+    periodo_id: Mapped[int] = mapped_column(
+        ForeignKey("periodo.id"),
+        nullable=False,
+        index=True,
+    )
+    periodo = relationship("Periodo", back_populates="dados")
+    modificacao: Mapped[Date] = mapped_column(Date, nullable=False)
+    ativo: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    # VALOR
+    v: Mapped[str] = mapped_column(Text, nullable=False)

sidra_sql/plugin_manager.py

@@ -0,0 +1,200 @@
+import json
+import logging
+import shutil
+import subprocess
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+import platformdirs
+
+logger = logging.getLogger(__name__)
+
+APP_NAME = "sidra-sql"
+DEFAULT_PLUGIN_URL = "https://github.com/Quantilica/sidra-pipelines.git"
+DEFAULT_PLUGIN_ALIAS = "std"
+
+
+@dataclass
+class PipelineDef:
+    id: str
+    description: str
+    path: Path
+
+
+@dataclass
+class PluginManifest:
+    name: str
+    description: str
+    version: str
+    pipelines: list[PipelineDef]
+
+
+class PluginRegistry:
+    def __init__(self):
+        self.config_dir = Path(platformdirs.user_config_dir(APP_NAME, appauthor=False))
+        self.data_dir = Path(platformdirs.user_data_dir(APP_NAME, appauthor=False))
+        self.plugins_dir = self.data_dir / "plugins"
+        self.registry_file = self.config_dir / "registry.json"
+
+        self.config_dir.mkdir(parents=True, exist_ok=True)
+        self.plugins_dir.mkdir(parents=True, exist_ok=True)
+
+        if not self.registry_file.exists():
+            self._save_registry({})
+
+    def _load_registry(self) -> dict:
+        with open(self.registry_file, encoding="utf-8") as f:
+            return json.load(f)
+
+    def _save_registry(self, data: dict):
+        with open(self.registry_file, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=4)
+
+    def get_plugins(self) -> dict:
+        return self._load_registry()
+
+    def get_plugin_path(self, alias: str) -> Path:
+        return self.plugins_dir / alias
+
+    def register_plugin(self, alias: str, url: str):
+        registry = self._load_registry()
+        registry[alias] = {"url": url}
+        self._save_registry(registry)
+
+    def remove_plugin(self, alias: str):
+        registry = self._load_registry()
+        if alias in registry:
+            del registry[alias]
+            self._save_registry(registry)
+
+
+class PluginManager:
+    def __init__(self):
+        self.registry = PluginRegistry()
+
+    def _check_git(self):
+        """Verifica se o Git está instalado."""
+        if shutil.which("git") is None:
+            raise RuntimeError(
+                "Git não encontrado. O Git é necessário para gerenciar e "
+                "baixar plugins. Por favor, instale o Git "
+                "(https://git-scm.com/) e tente novamente."
+            )
+
+    def install(self, url: str, alias: str | None = None):
+        self._check_git()
+        if not alias:
+            # simple alias extraction from url
+            alias = url.rstrip("/").split("/")[-1]
+            if alias.endswith(".git"):
+                alias = alias[:-4]
+
+        plugin_path = self.registry.get_plugin_path(alias)
+        if plugin_path.exists():
+            raise ValueError(f"Plugin with alias '{alias}' is already installed.")
+
+        logger.info("Cloning %s into %s", url, plugin_path)
+        subprocess.run(["git", "clone", url, str(plugin_path)], check=True)
+        self.registry.register_plugin(alias, url)
+        logger.info("Plugin '%s' installed successfully.", alias)
+
+    def update(self, alias: str | None = None):
+        self._check_git()
+        plugins = self.registry.get_plugins()
+        target_aliases = [alias] if alias else list(plugins.keys())
+
+        for target in target_aliases:
+            if target not in plugins:
+                logger.warning("Plugin '%s' not found in registry.", target)
+                continue
+
+            plugin_path = self.registry.get_plugin_path(target)
+            if not plugin_path.exists():
+                logger.warning("Plugin directory for '%s' is missing.", target)
+                continue
+
+            logger.info("Updating plugin '%s'", target)
+            subprocess.run(["git", "pull"], cwd=plugin_path, check=True)
+
+    def remove(self, alias: str):
+        plugin_path = self.registry.get_plugin_path(alias)
+        if plugin_path.exists():
+            logger.info("Removing directory %s", plugin_path)
+
+            # Use shutil on windows/linux to deal with read-only files
+            # sometimes created by git
+            def handle_remove_readonly(func, path, exc):
+                import os
+                import stat
+
+                os.chmod(path, stat.S_IWRITE)
+                func(path)
+
+            shutil.rmtree(plugin_path, onerror=handle_remove_readonly)
+
+        self.registry.remove_plugin(alias)
+        logger.info("Plugin '%s' removed successfully.", alias)
+
+    def ensure_defaults(self):
+        """Garante que o plugin padrão esteja instalado."""
+        plugins = self.registry.get_plugins()
+        if DEFAULT_PLUGIN_ALIAS not in plugins:
+            try:
+                self._check_git()
+                logger.info("Instalando pipelines padrão...")
+                self.install(DEFAULT_PLUGIN_URL, alias=DEFAULT_PLUGIN_ALIAS)
+            except Exception as e:
+                # Silenciosamente falha se não houver internet no bootstrap,
+                # permitindo que o usuário use o CLI de qualquer forma.
+                logger.debug(f"Falha ao instalar pipelines padrão: {e}")
+
+    def read_manifest(self, alias: str) -> PluginManifest:
+        plugin_path = self.registry.get_plugin_path(alias)
+        manifest_path = plugin_path / "manifest.toml"
+
+        if not manifest_path.exists():
+            raise FileNotFoundError(
+                f"Manifest not found for plugin '{alias}' at {manifest_path}"
+            )
+
+        with open(manifest_path, "rb") as f:
+            data = tomllib.load(f)
+
+        pipelines = []
+        for p in data.get("pipeline", []):
+            pipelines.append(
+                PipelineDef(
+                    id=p["id"],
+                    description=p.get("description", ""),
+                    path=plugin_path / p["path"],
+                )
+            )
+
+        return PluginManifest(
+            name=data.get("name", alias),
+            description=data.get("description", ""),
+            version=data.get("version", "unknown"),
+            pipelines=pipelines,
+        )
+
+    def list_pipelines(self) -> list[tuple[str, str, PipelineDef]]:
+        plugins = self.registry.get_plugins()
+        all_pipelines = []
+
+        for alias in plugins:
+            try:
+                manifest = self.read_manifest(alias)
+                for p in manifest.pipelines:
+                    all_pipelines.append((alias, manifest.name, p))
+            except Exception as e:
+                logger.warning("Could not load manifest for plugin '%s': %s", alias, e)
+
+        return all_pipelines
+
+    def get_pipeline(self, alias: str, pipeline_id: str) -> PipelineDef:
+        manifest = self.read_manifest(alias)
+        for p in manifest.pipelines:
+            if p.id == pipeline_id:
+                return p
+        raise ValueError(f"Pipeline '{pipeline_id}' not found in plugin '{alias}'")

sidra_sql/runner.py

@@ -0,0 +1,76 @@
+"""Hierarchical pipeline runner.
+
+Walks a pipeline directory depth-first, post-order:
+each child subtree (any subdirectory containing ``fetch.toml`` or
+``transform.toml``) runs to completion before the parent's own
+``fetch.toml`` / ``transform.toml`` execute. This lets a parent's
+SQL transform consume the materialized outputs of its children.
+"""
+
+import logging
+import time
+from pathlib import Path
+
+from rich.console import Console
+
+from .config import Config
+from .toml_runner import TomlScript
+from .transform_runner import TransformRunner
+
+logger = logging.getLogger(__name__)
+
+
+def _is_pipeline_dir(p: Path) -> bool:
+    return p.is_dir() and (
+        (p / "fetch.toml").exists() or (p / "transform.toml").exists()
+    )
+
+
+def run_subtree(
+    config: Config,
+    path: Path,
+    force_metadata: bool = False,
+    force_load: bool = False,
+    console: Console | None = None,
+):
+    """Run all sub-pipelines under ``path`` post-order, then ``path`` itself."""
+    if not path.exists() or not path.is_dir():
+        raise FileNotFoundError(f"Pipeline directory not found: {path}")
+
+    for child in sorted(path.iterdir()):
+        if _is_pipeline_dir(child):
+            run_subtree(config, child, force_metadata, force_load, console)
+
+    fetch_path = path / "fetch.toml"
+    transform_path = path / "transform.toml"
+
+    if fetch_path.exists():
+        if console:
+            console.rule(f"[bold cyan]fetch[/bold cyan]  {path.name}", style="cyan dim")
+        t0 = time.monotonic()
+        TomlScript(
+            config,
+            fetch_path,
+            force_metadata=force_metadata,
+            force_load=force_load,
+            console=console,
+        ).run()
+        if console:
+            elapsed = time.monotonic() - t0
+            console.print(
+                f"  [green]✓[/green] fetch concluído em [bold]{elapsed:.1f}s[/bold]"
+            )
+
+    if transform_path.exists():
+        if console:
+            console.rule(
+                f"[bold magenta]transform[/bold magenta]  {path.name}",
+                style="magenta dim",
+            )
+        t0 = time.monotonic()
+        TransformRunner(config, transform_path, console=console).run()
+        if console:
+            elapsed = time.monotonic() - t0
+            console.print(
+                f"  [green]✓[/green] transform concluído em [bold]{elapsed:.1f}s[/bold]"
+            )