agrobr 0.1.0__py3-none-any.whl

agrobr/sync.py

@@ -0,0 +1,147 @@
+"""Wrappers síncronos para APIs async do agrobr."""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+from collections.abc import Awaitable, Callable
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+
+
+def _get_or_create_event_loop() -> asyncio.AbstractEventLoop:
+    """
+    Obtém event loop existente ou cria novo.
+
+    Trata casos especiais:
+    - Jupyter notebooks (loop já rodando)
+    - Threads secundárias (sem loop default)
+    """
+    try:
+        loop = asyncio.get_running_loop()
+        try:
+            import nest_asyncio
+
+            nest_asyncio.apply()
+            return loop
+        except ImportError:
+            raise RuntimeError(
+                "Event loop already running. Install nest_asyncio for Jupyter support: "
+                "pip install nest_asyncio"
+            ) from None
+    except RuntimeError:
+        try:
+            return asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            return loop
+
+
+def run_sync(coro: Awaitable[T]) -> T:
+    """
+    Executa coroutine de forma síncrona.
+
+    Args:
+        coro: Coroutine a executar
+
+    Returns:
+        Resultado da coroutine
+    """
+    loop = _get_or_create_event_loop()
+
+    if loop.is_running():
+        import nest_asyncio
+
+        nest_asyncio.apply()
+        return loop.run_until_complete(coro)
+    else:
+        return asyncio.run(coro)  # type: ignore[arg-type]
+
+
+def sync_wrapper(async_func: Callable[..., Awaitable[T]]) -> Callable[..., T]:
+    """
+    Decorator que cria versão síncrona de função async.
+
+    Usage:
+        @sync_wrapper
+        async def fetch_data():
+            ...
+
+        # Agora pode chamar:
+        fetch_data()  # Síncrono
+    """
+
+    @functools.wraps(async_func)
+    def wrapper(*args: Any, **kwargs: Any) -> T:
+        return run_sync(async_func(*args, **kwargs))
+
+    if wrapper.__doc__:
+        wrapper.__doc__ = f"[SYNC] {wrapper.__doc__}"
+
+    return wrapper
+
+
+class _SyncModule:
+    """Módulo que expõe versões síncronas da API."""
+
+    def __init__(self, async_module: Any) -> None:
+        self._async_module = async_module
+
+    def __getattr__(self, name: str) -> Any:
+        attr = getattr(self._async_module, name)
+
+        if asyncio.iscoroutinefunction(attr):
+            return sync_wrapper(attr)
+
+        return attr
+
+
+class _SyncCepea(_SyncModule):
+    """API síncrona do CEPEA."""
+
+    pass
+
+
+class _SyncConab(_SyncModule):
+    """API síncrona da CONAB."""
+
+    pass
+
+
+class _SyncIbge(_SyncModule):
+    """API síncrona do IBGE."""
+
+    pass
+
+
+_cepea: _SyncCepea | None = None
+_conab: _SyncConab | None = None
+_ibge: _SyncIbge | None = None
+
+
+def __getattr__(name: str) -> Any:
+    """Lazy loading para evitar imports circulares."""
+    global _cepea, _conab, _ibge
+
+    if name == "cepea":
+        if _cepea is None:
+            from agrobr import cepea as async_cepea
+
+            _cepea = _SyncCepea(async_cepea)
+        return _cepea
+    elif name == "conab":
+        if _conab is None:
+            from agrobr import conab as async_conab
+
+            _conab = _SyncConab(async_conab)
+        return _conab
+    elif name == "ibge":
+        if _ibge is None:
+            from agrobr import ibge as async_ibge
+
+            _ibge = _SyncIbge(async_ibge)
+        return _ibge
+
+    raise AttributeError(f"module 'agrobr.sync' has no attribute '{name}'")

agrobr/telemetry/__init__.py

@@ -0,0 +1,17 @@
+"""Telemetria opt-in."""
+
+from __future__ import annotations
+
+from .collector import (
+    TelemetryCollector,
+    track_cache_operation,
+    track_fetch,
+    track_parse_error,
+)
+
+__all__: list[str] = [
+    "TelemetryCollector",
+    "track_cache_operation",
+    "track_fetch",
+    "track_parse_error",
+]

agrobr/telemetry/collector.py

@@ -0,0 +1,153 @@
+"""Coleta de telemetria opt-in."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import platform
+import uuid
+from datetime import datetime
+from typing import Any
+
+import httpx
+import structlog
+
+from agrobr import __version__
+from agrobr.constants import TelemetrySettings
+
+logger = structlog.get_logger()
+
+
+class TelemetryCollector:
+    """
+    Coleta telemetria opt-in de forma não-intrusiva.
+
+    Características:
+    - Fire-and-forget (nunca bloqueia)
+    - Batching para reduzir requests
+    - Falhas silenciosas
+    """
+
+    _instance_id: str | None = None
+    _buffer: list[dict[str, Any]] = []
+    _lock = asyncio.Lock()
+
+    @classmethod
+    def get_instance_id(cls) -> str:
+        """
+        Gera ID único por instalação.
+
+        Não é rastreável para o usuário específico,
+        apenas identifica a instalação para deduplicação.
+        """
+        if cls._instance_id is None:
+            machine_id = uuid.getnode().to_bytes(6, "big")
+            cls._instance_id = hashlib.sha256(machine_id).hexdigest()[:16]
+        return cls._instance_id
+
+    @classmethod
+    def get_context(cls) -> dict[str, Any]:
+        """Contexto comum para todos os eventos."""
+        return {
+            "instance_id": cls.get_instance_id(),
+            "package_version": __version__,
+            "python_version": platform.python_version(),
+            "os": platform.system(),
+            "os_version": platform.release(),
+            "timestamp": datetime.utcnow().isoformat(),
+        }
+
+    @classmethod
+    async def track(
+        cls,
+        event: str,
+        properties: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Registra evento de telemetria.
+
+        Args:
+            event: Nome do evento (ex: "fetch", "parse_error", "cache_hit")
+            properties: Propriedades do evento
+        """
+        settings = TelemetrySettings()
+
+        if not settings.enabled:
+            return
+
+        payload = {
+            "event": event,
+            "context": cls.get_context(),
+            "properties": properties or {},
+        }
+
+        async with cls._lock:
+            cls._buffer.append(payload)
+
+            if len(cls._buffer) >= settings.batch_size:
+                asyncio.create_task(cls._flush())
+
+    @classmethod
+    async def _flush(cls) -> None:
+        """Envia buffer para o servidor."""
+        settings = TelemetrySettings()
+
+        async with cls._lock:
+            if not cls._buffer:
+                return
+
+            events = cls._buffer.copy()
+            cls._buffer.clear()
+
+        try:
+            async with httpx.AsyncClient() as client:
+                await client.post(
+                    settings.endpoint,
+                    json={"events": events},
+                    timeout=5.0,
+                )
+            logger.debug("telemetry_flushed", count=len(events))
+        except Exception as e:
+            logger.debug("telemetry_flush_failed", error=str(e))
+
+    @classmethod
+    def reset(cls) -> None:
+        """Reseta estado do coletor (para testes)."""
+        cls._buffer.clear()
+        cls._instance_id = None
+
+
+async def track_fetch(source: str, produto: str, latency_ms: float, from_cache: bool) -> None:
+    """Registra evento de fetch."""
+    await TelemetryCollector.track(
+        "fetch",
+        {
+            "source": source,
+            "produto": produto,
+            "latency_ms": latency_ms,
+            "from_cache": from_cache,
+        },
+    )
+
+
+async def track_parse_error(source: str, parser_version: int, error_type: str) -> None:
+    """Registra erro de parsing."""
+    await TelemetryCollector.track(
+        "parse_error",
+        {
+            "source": source,
+            "parser_version": parser_version,
+            "error_type": error_type,
+        },
+    )
+
+
+async def track_cache_operation(operation: str, hit: bool) -> None:
+    """Registra operação de cache."""
+    await TelemetryCollector.track(
+        "cache",
+        {
+            "operation": operation,
+            "hit": hit,
+        },
+    )

agrobr/utils/__init__.py

@@ -0,0 +1,5 @@
+"""Utilitários gerais."""
+
+from __future__ import annotations
+
+__all__: list[str] = []

agrobr/utils/logging.py

@@ -0,0 +1,59 @@
+"""Configuracao de logging estruturado com structlog."""
+
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+
+import structlog
+from structlog.types import Processor
+
+
+def configure_logging(
+    level: str = "INFO",
+    json_format: bool = True,
+    log_file: Path | str | None = None,
+) -> None:
+    """Configura structlog para logging estruturado."""
+    processors: list[Processor] = [
+        structlog.contextvars.merge_contextvars,
+        structlog.processors.add_log_level,
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.UnicodeDecoder(),
+    ]
+
+    if json_format:
+        processors.append(structlog.processors.JSONRenderer())
+    else:
+        processors.append(structlog.dev.ConsoleRenderer(colors=True))
+
+    structlog.configure(
+        processors=processors,
+        wrapper_class=structlog.stdlib.BoundLogger,
+        context_class=dict,
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        cache_logger_on_first_use=True,
+    )
+
+    logging.basicConfig(
+        format="%(message)s",
+        stream=sys.stdout,
+        level=getattr(logging, level.upper()),
+    )
+
+    if log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setLevel(getattr(logging, level.upper()))
+        logging.getLogger().addHandler(file_handler)
+
+
+def get_logger(name: str | None = None) -> structlog.stdlib.BoundLogger:
+    """Obtem logger configurado."""
+    logger: structlog.stdlib.BoundLogger = structlog.get_logger(name)
+    return logger
+
+
+configure_logging(level="INFO", json_format=True)

agrobr/validators/__init__.py

@@ -0,0 +1,35 @@
+"""Validadores - sanity checks e validação estrutural."""
+
+from __future__ import annotations
+
+from .sanity import (
+    AnomalyReport,
+    SanityRule,
+    validate_batch,
+    validate_indicador,
+    validate_safra,
+)
+from .structural import (
+    StructuralMonitor,
+    StructuralValidationResult,
+    compare_fingerprints,
+    load_baseline,
+    save_baseline,
+    validate_against_baseline,
+    validate_structure,
+)
+
+__all__: list[str] = [
+    "AnomalyReport",
+    "SanityRule",
+    "validate_batch",
+    "validate_indicador",
+    "validate_safra",
+    "StructuralValidationResult",
+    "validate_structure",
+    "validate_against_baseline",
+    "compare_fingerprints",
+    "load_baseline",
+    "save_baseline",
+    "StructuralMonitor",
+]

agrobr/validators/sanity.py

@@ -0,0 +1,286 @@
+"""Validação estatística de sanidade para dados agrícolas."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from decimal import Decimal
+from typing import Any
+
+import structlog
+
+from agrobr.exceptions import ValidationError
+from agrobr.models import Indicador, Safra
+
+logger = structlog.get_logger()
+
+
+@dataclass
+class SanityRule:
+    """Regra de validação estatística."""
+
+    field: str
+    min_value: Decimal | None
+    max_value: Decimal | None
+    max_daily_change_pct: Decimal | None = None
+    description: str = ""
+
+
+PRICE_RULES: dict[str, SanityRule] = {
+    "soja": SanityRule(
+        field="valor",
+        min_value=Decimal("30"),
+        max_value=Decimal("300"),
+        max_daily_change_pct=Decimal("15"),
+        description="Soja (BRL/sc60kg)",
+    ),
+    "milho": SanityRule(
+        field="valor",
+        min_value=Decimal("15"),
+        max_value=Decimal("150"),
+        max_daily_change_pct=Decimal("15"),
+        description="Milho (BRL/sc60kg)",
+    ),
+    "cafe": SanityRule(
+        field="valor",
+        min_value=Decimal("200"),
+        max_value=Decimal("3000"),
+        max_daily_change_pct=Decimal("10"),
+        description="Café Arábica (BRL/sc60kg)",
+    ),
+    "boi": SanityRule(
+        field="valor",
+        min_value=Decimal("100"),
+        max_value=Decimal("500"),
+        max_daily_change_pct=Decimal("10"),
+        description="Boi Gordo (BRL/@)",
+    ),
+    "boi_gordo": SanityRule(
+        field="valor",
+        min_value=Decimal("100"),
+        max_value=Decimal("500"),
+        max_daily_change_pct=Decimal("10"),
+        description="Boi Gordo (BRL/@)",
+    ),
+    "trigo": SanityRule(
+        field="valor",
+        min_value=Decimal("20"),
+        max_value=Decimal("150"),
+        max_daily_change_pct=Decimal("15"),
+        description="Trigo (BRL/sc60kg)",
+    ),
+    "algodao": SanityRule(
+        field="valor",
+        min_value=Decimal("50"),
+        max_value=Decimal("250"),
+        max_daily_change_pct=Decimal("10"),
+        description="Algodão (BRL/@)",
+    ),
+}
+
+SAFRA_RULES: dict[str, dict[str, SanityRule]] = {
+    "soja": {
+        "area_plantada": SanityRule(
+            field="area_plantada",
+            min_value=Decimal("20000"),
+            max_value=Decimal("50000"),
+            description="Área plantada soja Brasil (mil ha)",
+        ),
+        "producao": SanityRule(
+            field="producao",
+            min_value=Decimal("50000"),
+            max_value=Decimal("200000"),
+            description="Produção soja Brasil (mil ton)",
+        ),
+    },
+    "milho": {
+        "area_plantada": SanityRule(
+            field="area_plantada",
+            min_value=Decimal("10000"),
+            max_value=Decimal("30000"),
+            description="Área plantada milho Brasil (mil ha)",
+        ),
+        "producao": SanityRule(
+            field="producao",
+            min_value=Decimal("50000"),
+            max_value=Decimal("150000"),
+            description="Produção milho Brasil (mil ton)",
+        ),
+    },
+}
+
+
+@dataclass
+class AnomalyReport:
+    """Relatório de anomalia detectada."""
+
+    field: str
+    value: Any
+    expected_range: str
+    anomaly_type: str
+    severity: str
+    details: dict[str, Any]
+
+
+def validate_indicador(
+    indicador: Indicador,
+    valor_anterior: Decimal | None = None,
+) -> list[AnomalyReport]:
+    """
+    Valida indicador contra regras estatísticas.
+
+    Args:
+        indicador: Indicador a validar
+        valor_anterior: Valor do dia anterior (para validar variação)
+
+    Returns:
+        Lista de anomalias detectadas (vazia se OK)
+    """
+    anomalies: list[AnomalyReport] = []
+    rule = PRICE_RULES.get(indicador.produto.lower())
+
+    if not rule:
+        logger.debug("sanity_no_rules", produto=indicador.produto)
+        return anomalies
+
+    if rule.min_value and indicador.valor < rule.min_value:
+        anomalies.append(
+            AnomalyReport(
+                field="valor",
+                value=indicador.valor,
+                expected_range=f"[{rule.min_value}, {rule.max_value}]",
+                anomaly_type="out_of_range",
+                severity="critical",
+                details={
+                    "produto": indicador.produto,
+                    "rule": rule.description,
+                    "below_min_by": float(rule.min_value - indicador.valor),
+                },
+            )
+        )
+
+    if rule.max_value and indicador.valor > rule.max_value:
+        anomalies.append(
+            AnomalyReport(
+                field="valor",
+                value=indicador.valor,
+                expected_range=f"[{rule.min_value}, {rule.max_value}]",
+                anomaly_type="out_of_range",
+                severity="critical",
+                details={
+                    "produto": indicador.produto,
+                    "rule": rule.description,
+                    "above_max_by": float(indicador.valor - rule.max_value),
+                },
+            )
+        )
+
+    if valor_anterior and rule.max_daily_change_pct:
+        change_pct = abs((indicador.valor - valor_anterior) / valor_anterior) * 100
+
+        if change_pct > rule.max_daily_change_pct:
+            severity = "critical" if change_pct > rule.max_daily_change_pct * 2 else "warning"
+            anomalies.append(
+                AnomalyReport(
+                    field="valor",
+                    value=indicador.valor,
+                    expected_range=f"±{rule.max_daily_change_pct}% do dia anterior",
+                    anomaly_type="excessive_change",
+                    severity=severity,
+                    details={
+                        "produto": indicador.produto,
+                        "valor_anterior": float(valor_anterior),
+                        "change_pct": float(change_pct),
+                        "max_allowed_pct": float(rule.max_daily_change_pct),
+                    },
+                )
+            )
+
+    if anomalies:
+        logger.warning(
+            "sanity_anomalies_detected",
+            produto=indicador.produto,
+            count=len(anomalies),
+            types=[a.anomaly_type for a in anomalies],
+        )
+    else:
+        logger.debug("sanity_check_passed", produto=indicador.produto)
+
+    return anomalies
+
+
+def validate_safra(safra: Safra) -> list[AnomalyReport]:
+    """Valida dados de safra contra regras estatísticas."""
+    anomalies: list[AnomalyReport] = []
+    rules = SAFRA_RULES.get(safra.produto.lower(), {})
+
+    for field_name, rule in rules.items():
+        value = getattr(safra, field_name)
+
+        if value is None:
+            continue
+
+        if rule.min_value and value < rule.min_value:
+            anomalies.append(
+                AnomalyReport(
+                    field=field_name,
+                    value=value,
+                    expected_range=f"[{rule.min_value}, {rule.max_value}]",
+                    anomaly_type="out_of_range",
+                    severity="critical",
+                    details={"rule": rule.description},
+                )
+            )
+
+        if rule.max_value and value > rule.max_value:
+            anomalies.append(
+                AnomalyReport(
+                    field=field_name,
+                    value=value,
+                    expected_range=f"[{rule.min_value}, {rule.max_value}]",
+                    anomaly_type="out_of_range",
+                    severity="critical",
+                    details={"rule": rule.description},
+                )
+            )
+
+    return anomalies
+
+
+async def validate_batch(
+    indicadores: list[Indicador],
+    strict: bool = False,
+) -> tuple[list[Indicador], list[AnomalyReport]]:
+    """
+    Valida batch de indicadores.
+
+    Args:
+        indicadores: Lista de indicadores a validar
+        strict: Se True, levanta exceção em anomalias críticas
+
+    Returns:
+        tuple: (indicadores com anomalies preenchidas, todas as anomalias)
+    """
+    all_anomalies: list[AnomalyReport] = []
+
+    sorted_indicadores = sorted(indicadores, key=lambda x: x.data)
+
+    for i, ind in enumerate(sorted_indicadores):
+        valor_anterior = None
+        if i > 0 and sorted_indicadores[i - 1].produto == ind.produto:
+            valor_anterior = sorted_indicadores[i - 1].valor
+
+        anomalies = validate_indicador(ind, valor_anterior)
+
+        if anomalies:
+            ind.anomalies = [f"{a.anomaly_type}: {a.field}" for a in anomalies]
+            all_anomalies.extend(anomalies)
+
+            if strict and any(a.severity == "critical" for a in anomalies):
+                raise ValidationError(
+                    source=ind.fonte.value,
+                    field=anomalies[0].field,
+                    value=anomalies[0].value,
+                    reason=anomalies[0].anomaly_type,
+                )
+
+    return sorted_indicadores, all_anomalies