andes-open-data 2.0.0__py3-none-any.whl

andes/__init__.py

@@ -0,0 +1,15 @@
+"""ANDES — Open Data Platform.
+
+Plataforma de acceso, integración, exploración y consulta de estadísticas y
+datos abiertos. El paquete conserva el nombre histórico ``andes``
+por compatibilidad de importación; la identidad pública es ANDES.
+"""
+
+from __future__ import annotations
+
+APP_NAME = "ANDES"
+APP_TAGLINE = "Open Data Platform"
+APP_VERSION = "2.0.0"
+__version__ = APP_VERSION
+
+__all__ = ["APP_NAME", "APP_TAGLINE", "APP_VERSION", "__version__"]

andes/cli/__init__.py

@@ -0,0 +1 @@
+"""Command line interface."""

andes/cli/main.py

@@ -0,0 +1,166 @@
+"""Command line interface built with Typer."""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import typer
+
+from .. import APP_NAME, APP_VERSION
+from ..core.api import (
+    EXPORT_FORMATS,
+    download_many,
+    download_series_data,
+    export_series,
+    export_table,
+)
+from ..core.metadata import KNOWN_CODE_COL, KNOWN_NAME_COL, filter_metadata, load_metadata
+
+app = typer.Typer(
+    add_completion=False,
+    no_args_is_help=True,
+    help=f"{APP_NAME} {APP_VERSION}: estadísticas y datos abiertos.",
+)
+
+
+def _filtered(query: str, frequency: str | None, category: str | None):
+    df = load_metadata()
+    mask = (
+        df[KNOWN_CODE_COL].str.contains(query.lower(), regex=False)
+        | df[KNOWN_NAME_COL].str.contains(query.lower(), regex=False)
+    )
+    df = df[mask]
+    filters = {}
+    if frequency:
+        filters["frecuencia"] = frequency
+    if category:
+        filters["categoria"] = category
+    if filters:
+        df = filter_metadata(df, filters)
+    return df
+
+
+@app.command()
+def search(
+    query: str = typer.Argument(..., help="Texto a buscar en código o nombre."),
+    frequency: str | None = typer.Option(None, "--frequency", "-f", help="Diaria, Mensual, Trimestral o Anual."),
+    category: str | None = typer.Option(None, "--category", "-c", help="Categoría exacta."),
+    limit: int = typer.Option(20, "--limit", "-n", help="Máximo de resultados."),
+) -> None:
+    """Search series by code or name."""
+    df = _filtered(query, frequency, category)
+    if df.empty:
+        typer.echo("Sin resultados.")
+        raise typer.Exit(code=1)
+    for _, row in df.head(limit).iterrows():
+        typer.echo(f"{row[KNOWN_CODE_COL].upper():<14} {row[KNOWN_NAME_COL][:90]}")
+    total = len(df)
+    if total > limit:
+        typer.echo(f"... {total - limit} más ({total} en total)")
+
+
+@app.command()
+def download(
+    series_code: str = typer.Argument(..., help="Código de la serie, p. ej. PN00001MM."),
+    start: str = typer.Option(..., "--start", "-s", help="Periodo inicial (ej. 2020-1)."),
+    end: str = typer.Option(..., "--end", "-e", help="Periodo final (ej. 2024-12)."),
+    out: Path | None = typer.Option(None, "--out", "-o", help="Ruta de salida."),
+    fmt: str = typer.Option("csv", "--format", help="csv|json|xls|xml|txt|html."),
+) -> None:
+    """Download one series to a file."""
+    fmt = fmt.lower()
+    if fmt not in EXPORT_FORMATS.values():
+        typer.echo(f"Formato no soportado: {fmt}", err=True)
+        raise typer.Exit(code=2)
+    df = download_series_data(series_code, start, end)
+    if df.empty:
+        typer.echo("No hay datos para el rango consultado.", err=True)
+        raise typer.Exit(code=1)
+    out_dir = str(out.parent) if out else None
+    generated = export_series(df, series_code, start, end, fmt, out_dir=out_dir)
+    if out and Path(generated) != out:
+        shutil.move(generated, out)
+        generated = str(out)
+    typer.echo(f"Guardado: {generated} ({len(df)} periodos)")
+
+
+@app.command("download-many")
+def download_many_cmd(
+    codes: list[str] = typer.Argument(..., help="Dos o más códigos de serie."),
+    start: str = typer.Option(..., "--start", "-s", help="Periodo inicial (ej. 2024-1)."),
+    end: str = typer.Option(..., "--end", "-e", help="Periodo final (ej. 2024-12)."),
+    frequency: str | None = typer.Option(
+        None, "--frequency", "-f", help="Diaria, Mensual, Trimestral o Anual (se deduce si se omite)."
+    ),
+    out: Path | None = typer.Option(None, "--out", "-o", help="Ruta de salida (solo sin --split)."),
+    fmt: str = typer.Option("csv", "--format", help="csv|json|xml|txt|html."),
+    split: bool = typer.Option(False, "--split", help="Un archivo por serie."),
+) -> None:
+    """Download several series in batches (up to 10 per request)."""
+    fmt = fmt.lower()
+    if fmt not in EXPORT_FORMATS.values() or fmt == "xls":
+        typer.echo(f"Formato no soportado para lotes: {fmt}", err=True)
+        raise typer.Exit(code=2)
+    frame = download_many(codes, start, end, frequency)
+    if frame.empty:
+        typer.echo("No hay datos para el rango consultado.", err=True)
+        raise typer.Exit(code=1)
+
+    out_dir = str(out.parent) if out else None
+    if split:
+        generated = []
+        for code in [column for column in frame.columns if column != "fecha"]:
+            subset = frame[["fecha", code]].dropna(subset=[code])
+            if subset.empty:
+                continue
+            generated.append(
+                export_table(
+                    subset,
+                    f"bcrp_serie_{code}_{start}_to_{end}",
+                    fmt,
+                    out_dir=out_dir,
+                )
+            )
+        typer.echo(f"Guardados: {len(generated)} archivos ({len(frame)} periodos)")
+        return
+
+    path = export_table(frame, f"bcrp_lote_{start}_to_{end}", fmt, out_dir=out_dir)
+    if out and Path(path) != out:
+        shutil.move(path, out)
+        path = str(out)
+    typer.echo(f"Guardado: {path} ({len(frame)} periodos, {frame.shape[1] - 1} series)")
+
+
+@app.command()
+def gui(
+    web: bool = typer.Option(False, "--web", help="Abrir en el navegador."),
+    port: int = typer.Option(8550, "--port", help="Puerto para el modo web."),
+) -> None:
+    """Launch the Flet desktop interface."""
+    import flet as ft
+
+    from ..core.paths import ASSETS_DIR
+    from ..gui.views import main as gui_main
+
+    if web:
+        ft.run(
+            gui_main,
+            port=port,
+            view=ft.AppView.WEB_BROWSER,
+            assets_dir=ASSETS_DIR,
+        )
+    else:
+        ft.run(gui_main, assets_dir=ASSETS_DIR)
+
+
+@app.command()
+def mcp() -> None:
+    """Start the MCP server over stdio."""
+    from ..mcp.server import run
+
+    run()
+
+
+if __name__ == "__main__":
+    app()

andes/core/__init__.py

@@ -0,0 +1,27 @@
+"""Core backend: metadata catalogue, HTTP client and exporters."""
+
+from .api import (
+    download_series_data,
+    download_series_file,
+    export_series,
+    export_table,
+    find_pdf_url,
+    periods_to_dataframe,
+)
+from .metadata import filter_metadata, load_metadata
+from .paths import DATA_DIR, METADATA_PATH, METADATA_URL, OUTPUT_DIR
+
+__all__ = [
+    "DATA_DIR",
+    "METADATA_PATH",
+    "METADATA_URL",
+    "OUTPUT_DIR",
+    "download_series_data",
+    "download_series_file",
+    "export_series",
+    "export_table",
+    "filter_metadata",
+    "find_pdf_url",
+    "load_metadata",
+    "periods_to_dataframe",
+]

andes/core/adapters/__init__.py

@@ -0,0 +1,33 @@
+"""Registro de adaptadores nativos por institución."""
+
+from __future__ import annotations
+
+from .bcrp import BcrpAdapter
+from .gobpe import GobpeAdapter
+from .inei import IneiAdapter
+from .inei_tematico import IneiTematicoAdapter
+from .mef import MefAdapter
+
+_ADAPTERS = {
+    "bcrp": BcrpAdapter(),
+    "mef": MefAdapter(),
+    "inei": IneiAdapter(),
+    "inei_tematico": IneiTematicoAdapter(),
+    "gobpe": GobpeAdapter(),
+}
+
+
+def get_adapter(source_id: str):
+    """Devuelve el adaptador de una institución registrada."""
+    key = source_id.strip().lower()
+    if key.startswith("gobpe:"):
+        key = "gobpe"
+    try:
+        return _ADAPTERS[key]
+    except (KeyError, AttributeError) as exc:
+        raise ValueError(f"Sin adaptador para: {source_id}") from exc
+
+
+def registered() -> list[str]:
+    """Ids de portales con adaptador."""
+    return sorted(_ADAPTERS)

andes/core/adapters/base.py

@@ -0,0 +1,23 @@
+"""Contrato común de adaptadores de portal."""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+import pandas as pd
+
+from ..portals import PortalSpec
+
+
+class PortalAdapter(Protocol):
+    """Interfaz nativa que implementa cada portal."""
+
+    spec: PortalSpec
+
+    def search(self, query: str, *, filters: dict | None = None,
+               limit: int | None = 100, offset: int = 0) -> pd.DataFrame:
+        """Busca series en el catálogo local."""
+
+    def fetch(self, codes: list[str], start: str, end: str,
+              frequency: str | None = None) -> pd.DataFrame:
+        """Descarga series; debe hablar directo con el portal."""

andes/core/adapters/bcrp.py

@@ -0,0 +1,24 @@
+"""Adaptador nativo del BCRP (catálogo SQLite + API JSON)."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from .. import api, metadata
+from ..portals import spec_for
+
+
+class BcrpAdapter:
+    spec = spec_for("bcrp")
+
+    def search(self, query: str, *, filters: dict | None = None,
+               limit: int | None = 100, offset: int = 0) -> pd.DataFrame:
+        scoped = dict(filters or {})
+        scoped["institucion"] = "bcrp"
+        return metadata.search_catalog(
+            query, filters=scoped, limit=limit, offset=offset
+        )
+
+    def fetch(self, codes: list[str], start: str, end: str,
+              frequency: str | None = None) -> pd.DataFrame:
+        return api.download_many(codes, start, end, frequency)

andes/core/adapters/gobpe.py

@@ -0,0 +1,185 @@
+"""Adaptador de portales estadísticos de Gob.pe.
+
+Indexa el registro descubierto por ``core.gobpe`` como un catálogo de portales.
+Solo los destinos tabulares (CSV, Excel, API/JSON) se pueden descargar; los
+tableros y páginas web se ofrecen como enlace externo.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+
+import pandas as pd
+import requests
+
+from .. import catalog, gobpe
+from ..paths import DATA_DIR
+
+CACHE_DIR = os.path.join(DATA_DIR, "gobpe", "files")
+INDEX_STAMP = os.path.join(DATA_DIR, "gobpe", "indexed.json")
+HEADERS = {"User-Agent": "Mozilla/5.0 (ANDES/2.0)"}
+_FILE_TTL = 7 * 86400
+
+
+class GobpeAdapter:
+    """Portales publicados por las entidades del Estado peruano."""
+
+    institution = "gobpe"
+
+    def __init__(self) -> None:
+        self._spec = None
+
+    # ------------------------------------------------------------------ #
+    @staticmethod
+    def _digest(portals: list[dict]) -> str:
+        payload = json.dumps(
+            [[portal.get("id"), portal.get("institution_id")] for portal in portals],
+            sort_keys=True,
+        )
+        return hashlib.sha1(payload.encode()).hexdigest()
+
+    def _ensure_indexed(self, force: bool = False) -> int:
+        portals = gobpe.load_registry()
+        if not portals:
+            return 0
+        digest = self._digest(portals)
+        stamped = ""
+        if os.path.exists(INDEX_STAMP):
+            try:
+                with open(INDEX_STAMP, encoding="utf-8") as handle:
+                    stamped = json.load(handle).get("digest", "")
+            except (ValueError, OSError):
+                stamped = ""
+        if (
+            not force
+            and stamped == digest
+            and catalog.count_prefix("GOBPE-") == len(portals)
+        ):
+            return 0
+        catalog.clear_prefix("GOBPE-")
+        rows: list[dict] = []
+        for portal in portals:
+            rows.append(
+                {
+                    "institution": f"gobpe:{portal['institution_id']}",
+                    "code": portal["id"],
+                    "category": portal.get("platform", "web"),
+                    "group_name": portal.get("kind", "") or "Portal",
+                    "name": portal.get("name", ""),
+                    "description": portal.get("url", ""),
+                    "unit": "",
+                    "source": "Gob.pe",
+                    "frequency": "Portal",
+                    "pub_group": portal.get("scope", ""),
+                    "area": portal.get("institution", ""),
+                    "start": "",
+                    "end": "",
+                    "memo": json.dumps(
+                        {
+                            "kind": "gobpe",
+                            "url": portal.get("url", ""),
+                            "platform": portal.get("platform", "web"),
+                        },
+                        ensure_ascii=False,
+                    ),
+                }
+            )
+        written = catalog.upsert(rows)
+        os.makedirs(os.path.dirname(INDEX_STAMP), exist_ok=True)
+        with open(INDEX_STAMP, "w", encoding="utf-8") as handle:
+            json.dump({"digest": digest, "count": len(rows)}, handle)
+        return written
+
+    # ------------------------------------------------------------------ #
+    def search(
+        self,
+        query: str,
+        *,
+        filters: dict | None = None,
+        limit: int | None = 300,
+    ) -> pd.DataFrame:
+        from .. import metadata
+
+        self._ensure_indexed()
+        scoped = dict(filters or {})
+        return metadata.search_catalog(query, filters=scoped, limit=limit)
+
+    def _download(self, url: str) -> tuple[str, str]:
+        os.makedirs(CACHE_DIR, exist_ok=True)
+        digest = hashlib.sha1(url.encode()).hexdigest()[:16]
+        extension = gobpe.classify(url)
+        suffix = {"csv": "csv", "excel": "xlsx", "api": "json", "ckan": "json"}.get(
+            extension, "bin"
+        )
+        path = os.path.join(CACHE_DIR, f"{digest}.{suffix}")
+        if os.path.exists(path) and time.time() - os.path.getmtime(path) < _FILE_TTL:
+            return path, suffix
+        response = requests.get(url, headers=HEADERS, timeout=180)
+        response.raise_for_status()
+        with open(path, "wb") as handle:
+            handle.write(response.content)
+        return path, suffix
+
+    @staticmethod
+    def _parse(path: str, suffix: str) -> pd.DataFrame:
+        from .inei_tematico import IneiTematicoAdapter
+
+        if suffix in ("csv", "xlsx"):
+            return IneiTematicoAdapter._table(path, suffix)
+        with open(path, encoding="utf-8") as handle:
+            payload = json.load(handle)
+        records = payload
+        if isinstance(payload, dict):
+            for key in ("records", "results", "data", "datos"):
+                if isinstance(payload.get(key), list):
+                    records = payload[key]
+                    break
+        if isinstance(records, list) and records:
+            return pd.DataFrame(records)
+        return pd.DataFrame()
+
+    def fetch(
+        self,
+        codes: list[str],
+        start: str = "archivo",
+        end: str = "unico",
+        frequency: str | None = None,
+    ) -> pd.DataFrame:
+        frames: list[pd.DataFrame] = []
+        for code in codes:
+            row = catalog.get(code)
+            if not row or not row.get("memo"):
+                continue
+            try:
+                memo = json.loads(row["memo"])
+            except (TypeError, ValueError):
+                continue
+            if memo.get("platform") not in gobpe.TABULAR:
+                continue
+            try:
+                path, suffix = self._download(memo["url"])
+                frame = self._parse(path, suffix)
+            except Exception:
+                continue
+            if frame.empty:
+                continue
+            frames.append(frame)
+        if not frames:
+            return pd.DataFrame()
+        if len(frames) == 1:
+            return frames[0]
+        return pd.concat(frames, ignore_index=True)
+
+    def info(self, code: str) -> dict:
+        row = catalog.get(code) or {}
+        return {
+            "institution": "gobpe",
+            "code": code,
+            "source": "Gob.pe",
+            "name": row.get("name", ""),
+            "url": row.get("description", ""),
+            "updated": time.strftime("%Y-%m-%d"),
+        }