agrobr 0.1.0__py3-none-any.whl

agrobr/http/browser.py

@@ -0,0 +1,214 @@
+"""Browser automation com Playwright para sites com proteção anti-bot."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+
+import structlog
+from playwright.async_api import Browser, Page, Playwright, async_playwright
+
+from agrobr import constants
+from agrobr.exceptions import SourceUnavailableError
+from agrobr.http.user_agents import UserAgentRotator
+
+logger = structlog.get_logger()
+
+# Singleton para reutilizar browser
+_playwright: Playwright | None = None
+_browser: Browser | None = None
+_lock = asyncio.Lock()
+
+
+async def _get_browser() -> Browser:
+    """Obtém ou cria instância do browser (singleton)."""
+    global _playwright, _browser
+
+    async with _lock:
+        if _browser is None or not _browser.is_connected():
+            logger.info("browser_starting", browser="chromium")
+
+            _playwright = await async_playwright().start()
+            _browser = await _playwright.chromium.launch(
+                headless=True,
+                args=[
+                    "--disable-blink-features=AutomationControlled",
+                    "--disable-dev-shm-usage",
+                    "--no-sandbox",
+                ],
+            )
+
+            logger.info("browser_started")
+
+        return _browser
+
+
+async def close_browser() -> None:
+    """Fecha o browser e libera recursos."""
+    global _playwright, _browser
+
+    async with _lock:
+        if _browser is not None:
+            await _browser.close()
+            _browser = None
+            logger.info("browser_closed")
+
+        if _playwright is not None:
+            await _playwright.stop()
+            _playwright = None
+
+
+@asynccontextmanager
+async def get_page() -> AsyncGenerator[Page, None]:
+    """Context manager para obter uma página do browser."""
+    browser = await _get_browser()
+
+    # Cria contexto com fingerprint realista
+    ua = UserAgentRotator.get_random()
+    context = await browser.new_context(
+        user_agent=ua,
+        viewport={"width": 1920, "height": 1080},
+        locale="pt-BR",
+        timezone_id="America/Sao_Paulo",
+        extra_http_headers={
+            "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7",
+        },
+    )
+
+    page = await context.new_page()
+
+    # Esconde sinais de automação
+    await page.add_init_script(
+        """
+        Object.defineProperty(navigator, 'webdriver', {
+            get: () => undefined
+        });
+    """
+    )
+
+    try:
+        yield page
+    finally:
+        await context.close()
+
+
+async def fetch_with_browser(
+    url: str,
+    source: str = "unknown",
+    wait_selector: str | None = None,
+    wait_timeout: float = 30000,
+) -> str:
+    """
+    Busca página usando browser headless.
+
+    Contorna proteções anti-bot como Cloudflare.
+
+    Args:
+        url: URL para buscar
+        source: Nome da fonte (para logging)
+        wait_selector: Seletor CSS para aguardar antes de retornar
+        wait_timeout: Timeout em ms para aguardar
+
+    Returns:
+        HTML da página
+
+    Raises:
+        SourceUnavailableError: Se não conseguir carregar a página
+    """
+    logger.info(
+        "browser_fetch_start",
+        source=source,
+        url=url,
+    )
+
+    try:
+        async with get_page() as page:
+            # Navega para a URL
+            response = await page.goto(
+                url,
+                wait_until="domcontentloaded",
+                timeout=wait_timeout,
+            )
+
+            if response is None:
+                raise SourceUnavailableError(
+                    source=source,
+                    url=url,
+                    last_error="No response received",
+                )
+
+            # Aguarda seletor específico se fornecido
+            if wait_selector:
+                try:
+                    await page.wait_for_selector(
+                        wait_selector,
+                        timeout=wait_timeout,
+                    )
+                except Exception as e:
+                    logger.warning(
+                        "browser_wait_selector_timeout",
+                        selector=wait_selector,
+                        error=str(e),
+                    )
+
+            # Aguarda Cloudflare resolver e JS terminar
+            await page.wait_for_timeout(5000)
+
+            # Verifica se foi bloqueado pelo Cloudflare
+            if response.status in (403, 503):
+                check_html: str = await page.content()
+                # Detecta página de challenge do Cloudflare
+                if "cloudflare" in check_html.lower() or "challenge" in check_html.lower():
+                    raise SourceUnavailableError(
+                        source=source,
+                        url=url,
+                        last_error=f"Cloudflare block detected (status {response.status})",
+                    )
+
+            # Obtém HTML
+            html: str = await page.content()
+
+            logger.info(
+                "browser_fetch_success",
+                source=source,
+                url=url,
+                content_length=len(html),
+                status=response.status,
+            )
+
+            return html
+
+    except Exception as e:
+        logger.error(
+            "browser_fetch_failed",
+            source=source,
+            url=url,
+            error=str(e),
+        )
+        raise SourceUnavailableError(
+            source=source,
+            url=url,
+            last_error=str(e),
+        ) from e
+
+
+async def fetch_cepea_indicador(produto: str) -> str:
+    """
+    Busca página de indicador do CEPEA usando browser.
+
+    Args:
+        produto: Nome do produto (soja, milho, etc)
+
+    Returns:
+        HTML da página
+    """
+    produto_key = constants.CEPEA_PRODUTOS.get(produto.lower(), produto.lower())
+    url = f"{constants.URLS[constants.Fonte.CEPEA]['indicadores']}/{produto_key}.aspx"
+
+    return await fetch_with_browser(
+        url=url,
+        source="cepea",
+        wait_selector="table",
+        wait_timeout=90000,
+    )

agrobr/http/rate_limiter.py

@@ -0,0 +1,69 @@
+"""Rate limiter por fonte usando semáforos."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+import structlog
+
+from agrobr import constants
+
+logger = structlog.get_logger()
+
+
+class RateLimiter:
+    """Garante intervalo mínimo entre requests para cada fonte."""
+
+    _semaphores: dict[str, asyncio.Semaphore] = {}
+    _last_request: dict[str, float] = {}
+    _lock = asyncio.Lock()
+
+    @classmethod
+    def _get_delay(cls, source: constants.Fonte) -> float:
+        settings = constants.HTTPSettings()
+        delays = {
+            constants.Fonte.CEPEA: settings.rate_limit_cepea,
+            constants.Fonte.CONAB: settings.rate_limit_conab,
+            constants.Fonte.IBGE: settings.rate_limit_ibge,
+            constants.Fonte.NOTICIAS_AGRICOLAS: settings.rate_limit_noticias_agricolas,
+        }
+        return delays.get(source, 1.0)
+
+    @classmethod
+    @asynccontextmanager
+    async def acquire(cls, source: constants.Fonte) -> AsyncIterator[None]:
+        """Context manager que garante rate limiting."""
+        source_key = source.value
+
+        async with cls._lock:
+            if source_key not in cls._semaphores:
+                cls._semaphores[source_key] = asyncio.Semaphore(1)
+
+        async with cls._semaphores[source_key]:
+            now = time.monotonic()
+            last = cls._last_request.get(source_key, 0)
+            delay = cls._get_delay(source)
+            elapsed = now - last
+
+            if elapsed < delay:
+                wait_time = delay - elapsed
+                logger.debug(
+                    "rate_limit_wait",
+                    source=source_key,
+                    wait_seconds=wait_time,
+                )
+                await asyncio.sleep(wait_time)
+
+            try:
+                yield
+            finally:
+                cls._last_request[source_key] = time.monotonic()
+
+    @classmethod
+    def reset(cls) -> None:
+        """Reseta estado do rate limiter."""
+        cls._semaphores.clear()
+        cls._last_request.clear()

agrobr/http/retry.py

@@ -0,0 +1,93 @@
+"""Retry com exponential backoff."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable, Sequence
+from functools import wraps
+from typing import Any, TypeVar
+
+import httpx
+import structlog
+
+from agrobr import constants
+
+logger = structlog.get_logger()
+T = TypeVar("T")
+
+RETRIABLE_EXCEPTIONS: tuple[type[Exception], ...] = (
+    httpx.TimeoutException,
+    httpx.NetworkError,
+    httpx.RemoteProtocolError,
+)
+
+
+async def retry_async(
+    func: Callable[[], Awaitable[T]],
+    max_attempts: int | None = None,
+    base_delay: float | None = None,
+    max_delay: float | None = None,
+    retriable_exceptions: Sequence[type[Exception]] = RETRIABLE_EXCEPTIONS,
+) -> T:
+    """Executa função async com retry exponential backoff."""
+    settings = constants.HTTPSettings()
+    max_attempts = max_attempts or settings.max_retries
+    base_delay = base_delay or settings.retry_base_delay
+    max_delay = max_delay or settings.retry_max_delay
+
+    last_exception: Exception | None = None
+
+    for attempt in range(max_attempts):
+        try:
+            return await func()
+
+        except tuple(retriable_exceptions) as e:
+            last_exception = e
+            if attempt < max_attempts - 1:
+                delay = min(
+                    base_delay * (settings.retry_exponential_base**attempt),
+                    max_delay,
+                )
+                logger.warning(
+                    "retry_scheduled",
+                    attempt=attempt + 1,
+                    max_attempts=max_attempts,
+                    delay_seconds=delay,
+                    error=str(e),
+                )
+                await asyncio.sleep(delay)
+            else:
+                logger.error(
+                    "retry_exhausted",
+                    attempts=max_attempts,
+                    last_error=str(e),
+                )
+
+    if last_exception:
+        raise last_exception
+    raise RuntimeError("Retry logic error: no exception captured")
+
+
+def with_retry(
+    max_attempts: int | None = None,
+    base_delay: float | None = None,
+) -> Callable[[Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]]:
+    """Decorator para retry automático."""
+
+    def decorator(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
+        @wraps(func)
+        async def wrapper(*args: Any, **kwargs: Any) -> T:
+            return await retry_async(
+                lambda: func(*args, **kwargs),
+                max_attempts=max_attempts,
+                base_delay=base_delay,
+            )
+
+        return wrapper
+
+    return decorator
+
+
+def should_retry_status(status_code: int) -> bool:
+    """Verifica se o status code permite retry."""
+    return status_code in constants.RETRIABLE_STATUS_CODES

agrobr/http/user_agents.py

@@ -0,0 +1,67 @@
+"""Pool rotativo de User-Agents."""
+
+from __future__ import annotations
+
+import random
+from collections.abc import Sequence
+
+USER_AGENT_POOL: Sequence[str] = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
+)
+
+DEFAULT_HEADERS: dict[str, str] = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+}
+
+
+class UserAgentRotator:
+    """Rotaciona User-Agents de forma determinística por fonte."""
+
+    _counters: dict[str, int] = {}
+
+    @classmethod
+    def get(cls, source: str | None = None) -> str:
+        """Retorna próximo User-Agent do pool."""
+        key = source or "default"
+
+        if key not in cls._counters:
+            cls._counters[key] = random.randint(0, len(USER_AGENT_POOL) - 1)
+
+        ua = USER_AGENT_POOL[cls._counters[key] % len(USER_AGENT_POOL)]
+        cls._counters[key] += 1
+
+        return ua
+
+    @classmethod
+    def get_random(cls) -> str:
+        """Retorna User-Agent aleatório."""
+        return random.choice(USER_AGENT_POOL)
+
+    @classmethod
+    def get_headers(cls, source: str | None = None) -> dict[str, str]:
+        """Retorna headers completos incluindo User-Agent."""
+        headers = DEFAULT_HEADERS.copy()
+        headers["User-Agent"] = cls.get(source)
+        return headers
+
+    @classmethod
+    def reset(cls) -> None:
+        """Reseta contadores."""
+        cls._counters.clear()

agrobr/ibge/__init__.py

@@ -0,0 +1,19 @@
+"""Modulo IBGE - Dados PAM e LSPA."""
+
+from __future__ import annotations
+
+from agrobr.ibge.api import (
+    lspa,
+    pam,
+    produtos_lspa,
+    produtos_pam,
+    ufs,
+)
+
+__all__ = [
+    "pam",
+    "lspa",
+    "produtos_pam",
+    "produtos_lspa",
+    "ufs",
+]