agrobr 0.1.0__py3-none-any.whl

agrobr/cepea/parsers/consensus.py

@@ -0,0 +1,300 @@
+"""
+Multi-parser consensus para validação cruzada.
+
+Executa múltiplos parsers e compara resultados para detectar
+problemas de parsing.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import structlog
+
+from ...alerts.notifier import AlertLevel, send_alert
+from ...exceptions import ParseError
+from ...models import Indicador
+from .base import BaseParser
+from .v1 import CepeaParserV1
+
+logger = structlog.get_logger()
+
+CONSENSUS_PARSERS: list[type[BaseParser]] = [
+    CepeaParserV1,
+]
+
+DIVERGENCE_THRESHOLD_COUNT = 0.1
+DIVERGENCE_THRESHOLD_VALUE = 0.01
+
+
+@dataclass
+class ConsensusResult:
+    """Resultado de parsing com consensus."""
+
+    indicadores: list[Indicador]
+    parser_used: BaseParser
+    all_results: dict[int, list[Indicador]]
+    has_consensus: bool
+    divergences: list[dict[str, Any]]
+    report: dict[str, Any]
+
+
+@dataclass
+class ParserDivergence:
+    """Divergência entre parsers."""
+
+    divergence_type: str
+    versions: list[int]
+    details: dict[str, Any]
+
+
+async def parse_with_consensus(
+    html: str,
+    produto: str,
+    require_consensus: bool = False,
+) -> ConsensusResult:
+    """
+    Executa múltiplos parsers e compara resultados.
+
+    Args:
+        html: Conteúdo HTML
+        produto: Produto a parsear
+        require_consensus: Se True, levanta erro se parsers divergem
+
+    Returns:
+        ConsensusResult
+
+    Raises:
+        ParseError: Se require_consensus e parsers divergem
+    """
+    results: dict[int, list[Indicador]] = {}
+    errors: dict[int, str] = {}
+
+    for parser_cls in CONSENSUS_PARSERS:
+        parser = parser_cls()
+        try:
+            can_parse, confidence = parser.can_parse(html)
+            if can_parse and confidence > 0.5:
+                parsed = parser.parse(html, produto)
+                results[parser.version] = parsed
+                logger.debug(
+                    "consensus_parser_success",
+                    version=parser.version,
+                    count=len(parsed),
+                )
+        except Exception as e:
+            errors[parser.version] = str(e)
+            logger.warning(
+                "consensus_parser_failed",
+                version=parser.version,
+                error=str(e),
+            )
+
+    divergences, report = analyze_consensus(results, errors)
+
+    has_consensus = len(divergences) == 0
+
+    if not has_consensus:
+        logger.warning(
+            "consensus_divergence_detected",
+            divergence_count=len(divergences),
+        )
+
+        if require_consensus:
+            await send_alert(
+                level=AlertLevel.WARNING,
+                title="Parser consensus failed",
+                details=report,
+            )
+            raise ParseError(
+                source="cepea",
+                parser_version=0,
+                reason=f"Parsers diverged: {len(divergences)} differences",
+            )
+
+    latest_version = max(results.keys()) if results else 0
+    best_results = results.get(latest_version, [])
+
+    parser_used: BaseParser = CepeaParserV1()
+    for parser_cls in CONSENSUS_PARSERS:
+        if parser_cls().version == latest_version:
+            parser_used = parser_cls()
+            break
+
+    return ConsensusResult(
+        indicadores=best_results,
+        parser_used=parser_used,
+        all_results=results,
+        has_consensus=has_consensus,
+        divergences=divergences,
+        report=report,
+    )
+
+
+def analyze_consensus(
+    results: dict[int, list[Indicador]],
+    errors: dict[int, str],
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    """
+    Analisa resultados de múltiplos parsers.
+
+    Args:
+        results: Resultados por versão de parser
+        errors: Erros por versão de parser
+
+    Returns:
+        Tupla (lista de divergências, relatório completo)
+    """
+    report = {
+        "parser_count": len(CONSENSUS_PARSERS),
+        "successful": list(results.keys()),
+        "failed": list(errors.keys()),
+        "errors": errors,
+    }
+
+    divergences: list[dict[str, Any]] = []
+
+    if len(results) < 2:
+        return divergences, report
+
+    counts = {v: len(r) for v, r in results.items()}
+    unique_counts = set(counts.values())
+
+    if len(unique_counts) > 1:
+        divergences.append(
+            {
+                "type": "count_mismatch",
+                "versions": list(counts.keys()),
+                "counts": counts,
+                "description": f"Different record counts: {counts}",
+            }
+        )
+
+    versions = list(results.keys())
+    base_version = versions[0]
+    base_results = results[base_version]
+
+    for other_version in versions[1:]:
+        other_results = results[other_version]
+
+        if base_results and other_results:
+            if base_results[0].data != other_results[0].data:
+                divergences.append(
+                    {
+                        "type": "first_date_mismatch",
+                        "versions": [base_version, other_version],
+                        "values": [str(base_results[0].data), str(other_results[0].data)],
+                    }
+                )
+
+            first_diff = abs(float(base_results[0].valor) - float(other_results[0].valor))
+            if first_diff > DIVERGENCE_THRESHOLD_VALUE:
+                divergences.append(
+                    {
+                        "type": "first_value_mismatch",
+                        "versions": [base_version, other_version],
+                        "values": [str(base_results[0].valor), str(other_results[0].valor)],
+                        "difference": first_diff,
+                    }
+                )
+
+            if base_results[-1].data != other_results[-1].data:
+                divergences.append(
+                    {
+                        "type": "last_date_mismatch",
+                        "versions": [base_version, other_version],
+                        "values": [str(base_results[-1].data), str(other_results[-1].data)],
+                    }
+                )
+
+            last_diff = abs(float(base_results[-1].valor) - float(other_results[-1].valor))
+            if last_diff > DIVERGENCE_THRESHOLD_VALUE:
+                divergences.append(
+                    {
+                        "type": "last_value_mismatch",
+                        "versions": [base_version, other_version],
+                        "values": [str(base_results[-1].valor), str(other_results[-1].valor)],
+                        "difference": last_diff,
+                    }
+                )
+
+    report["divergences"] = divergences
+    report["has_divergence"] = len(divergences) > 0
+
+    return divergences, report
+
+
+def select_best_result(
+    results: dict[int, list[Indicador]],
+    divergences: list[dict[str, Any]],
+) -> tuple[int, list[Indicador]]:
+    """
+    Seleciona melhor resultado quando há divergência.
+
+    Estratégia:
+    1. Prefere parser mais recente
+    2. Se contagens diferem, prefere quem tem mais dados
+    3. Em último caso, usa mais recente
+
+    Args:
+        results: Resultados por versão
+        divergences: Divergências detectadas
+
+    Returns:
+        Tupla (versão selecionada, indicadores)
+    """
+    if not results:
+        return 0, []
+
+    has_count_mismatch = any(d["type"] == "count_mismatch" for d in divergences)
+
+    if has_count_mismatch:
+        best_version = max(results.keys(), key=lambda v: len(results[v]))
+    else:
+        best_version = max(results.keys())
+
+    return best_version, results[best_version]
+
+
+class ConsensusValidator:
+    """Validador de consensus para uso contínuo."""
+
+    def __init__(self) -> None:
+        self.history: list[ConsensusResult] = []
+        self.divergence_count = 0
+
+    async def validate(self, html: str, produto: str) -> ConsensusResult:
+        """
+        Executa validação com tracking de histórico.
+
+        Args:
+            html: Conteúdo HTML
+            produto: Produto
+
+        Returns:
+            ConsensusResult
+        """
+        result = await parse_with_consensus(html, produto, require_consensus=False)
+
+        self.history.append(result)
+        if not result.has_consensus:
+            self.divergence_count += 1
+
+        return result
+
+    @property
+    def divergence_rate(self) -> float:
+        """Taxa de divergência no histórico."""
+        if not self.history:
+            return 0.0
+        return self.divergence_count / len(self.history)
+
+    def get_statistics(self) -> dict[str, Any]:
+        """Retorna estatísticas do validador."""
+        return {
+            "total_validations": len(self.history),
+            "divergence_count": self.divergence_count,
+            "divergence_rate": self.divergence_rate,
+            "consensus_rate": 1 - self.divergence_rate,
+        }

agrobr/cepea/parsers/detector.py

@@ -0,0 +1,108 @@
+"""Detector e seletor de parser com fallback em cascata."""
+
+from __future__ import annotations
+
+from datetime import date
+from typing import TYPE_CHECKING
+
+import structlog
+
+from agrobr import constants, exceptions
+from agrobr.cepea.parsers import base
+from agrobr.cepea.parsers.v1 import CepeaParserV1
+
+if TYPE_CHECKING:
+    from agrobr import models
+
+logger = structlog.get_logger()
+
+PARSERS: list[type[base.BaseParser]] = [
+    CepeaParserV1,
+]
+
+
+async def get_parser_with_fallback(
+    html: str,
+    produto: str,
+    data_referencia: date | None = None,
+    strict: bool = False,
+) -> tuple[base.BaseParser, list[models.Indicador]]:
+    """Seleciona parser e executa com fallback em cascata."""
+    if not PARSERS:
+        raise exceptions.ParseError(
+            source="cepea",
+            parser_version=0,
+            reason="No parsers registered. CEPEA parser will be implemented in WEEK 3.",
+            html_snippet=html[:200],
+        )
+
+    errors: list[tuple[str, str]] = []
+    warnings: list[str] = []
+
+    for parser_cls in reversed(PARSERS):
+        parser = parser_cls()
+
+        if data_referencia:
+            if parser.valid_from > data_referencia:
+                continue
+            if parser.valid_until and data_referencia > parser.valid_until:
+                continue
+
+        can_parse, confidence = parser.can_parse(html)
+
+        logger.debug(
+            "parser_check",
+            parser_version=parser.version,
+            can_parse=can_parse,
+            confidence=confidence,
+        )
+
+        if not can_parse:
+            continue
+
+        if confidence < constants.CONFIDENCE_LOW and strict:
+            raise exceptions.FingerprintMismatchError(
+                source=parser.source,
+                similarity=confidence,
+                threshold=constants.CONFIDENCE_LOW,
+            )
+
+        if confidence < constants.CONFIDENCE_HIGH:
+            warnings.append(
+                f"Parser v{parser.version} confidence {confidence:.1%} "
+                f"(expected >= {constants.CONFIDENCE_HIGH:.1%})"
+            )
+
+        try:
+            result = parser.parse(html, produto)
+
+            if not result:
+                errors.append((f"v{parser.version}", "No data extracted"))
+                continue
+
+            if warnings:
+                logger.warning(
+                    "parser_low_confidence",
+                    parser_version=parser.version,
+                    confidence=confidence,
+                    warnings=warnings,
+                )
+
+            return parser, result
+
+        except Exception as e:
+            errors.append((f"v{parser.version}", str(e)))
+            logger.warning(
+                "parser_failed",
+                parser_version=parser.version,
+                error=str(e),
+            )
+            continue
+
+    error_summary = "; ".join(f"{v}: {e}" for v, e in errors)
+    raise exceptions.ParseError(
+        source=PARSERS[0]().source if PARSERS else "cepea",
+        parser_version=0,
+        reason=f"All parsers failed: {error_summary}",
+        html_snippet=html[:500],
+    )

agrobr/cepea/parsers/fingerprint.py

@@ -0,0 +1,226 @@
+"""Fingerprinting de layout para detecção de mudanças."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import structlog
+from bs4 import BeautifulSoup
+
+from agrobr.constants import Fonte
+from agrobr.models import Fingerprint
+
+logger = structlog.get_logger()
+
+
+def extract_fingerprint(
+    html: str,
+    source: Fonte,
+    url: str,
+) -> Fingerprint:
+    """
+    Extrai assinatura estrutural do HTML.
+
+    Args:
+        html: Conteúdo HTML
+        source: Fonte de dados
+        url: URL original
+
+    Returns:
+        Fingerprint com assinatura estrutural
+    """
+    soup = BeautifulSoup(html, "lxml")
+
+    table_classes: list[list[str]] = []
+    for table in soup.find_all("table")[:10]:
+        classes_raw = table.get("class")
+        if classes_raw is None:
+            classes: list[str] = []
+        elif isinstance(classes_raw, str):
+            classes = [classes_raw]
+        else:
+            classes = list(classes_raw)
+        table_classes.append(sorted(classes))
+
+    keywords = ["preco", "indicador", "cotacao", "valor", "tabela", "dados"]
+    key_ids: list[str] = []
+    for elem in soup.find_all(id=True):
+        elem_id_raw = elem.get("id")
+        if elem_id_raw is None or not isinstance(elem_id_raw, str):
+            continue
+        elem_id = elem_id_raw.lower()
+        if any(kw in elem_id for kw in keywords):
+            key_ids.append(elem_id_raw)
+    key_ids = sorted(set(key_ids))[:20]
+
+    table_headers: list[list[str]] = []
+    for table in soup.find_all("table")[:5]:
+        headers: list[str] = []
+        for th in table.find_all("th"):
+            text = th.get_text(strip=True)[:50]
+            if text:
+                headers.append(text)
+        if headers:
+            table_headers.append(headers)
+
+    element_counts = {
+        "tables": len(soup.find_all("table")),
+        "forms": len(soup.find_all("form")),
+        "divs_with_id": len(soup.find_all("div", id=True)),
+        "inputs": len(soup.find_all("input")),
+        "selects": len(soup.find_all("select")),
+        "links": len(soup.find_all("a")),
+        "scripts": len(soup.find_all("script")),
+    }
+
+    structure_elements: list[tuple[str, int, tuple[str, ...]]] = []
+    for tag in soup.find_all(["table", "div", "form", "section", "article"])[:30]:
+        tag_classes_raw = tag.get("class")
+        if tag_classes_raw is None:
+            tag_classes: list[str] = []
+        elif isinstance(tag_classes_raw, str):
+            tag_classes = [tag_classes_raw]
+        else:
+            tag_classes = list(tag_classes_raw)
+        structure_elements.append(
+            (
+                tag.name or "",
+                len(tag.find_all(recursive=False)),
+                tuple(sorted(tag_classes))[:3] if tag_classes else (),
+            )
+        )
+
+    structure_hash = hashlib.md5(str(structure_elements).encode()).hexdigest()[:12]
+
+    return Fingerprint(
+        source=source,
+        url=url,
+        collected_at=datetime.utcnow(),
+        table_classes=table_classes,
+        key_ids=key_ids,
+        structure_hash=structure_hash,
+        table_headers=table_headers,
+        element_counts=element_counts,
+    )
+
+
+def compare_fingerprints(
+    current: Fingerprint,
+    reference: Fingerprint,
+) -> tuple[float, dict[str, Any]]:
+    """
+    Compara duas fingerprints e retorna similaridade.
+
+    Args:
+        current: Fingerprint atual
+        reference: Fingerprint de referência (baseline)
+
+    Returns:
+        tuple[float, dict]: (similaridade 0-1, detalhes das diferenças)
+    """
+    scores: dict[str, float] = {}
+    details: dict[str, Any] = {}
+
+    scores["structure"] = 1.0 if current.structure_hash == reference.structure_hash else 0.0
+    if scores["structure"] == 0:
+        details["structure_changed"] = {
+            "current": current.structure_hash,
+            "reference": reference.structure_hash,
+        }
+
+    if reference.table_classes:
+        matches = sum(1 for tc in current.table_classes if tc in reference.table_classes)
+        scores["table_classes"] = matches / len(reference.table_classes)
+        if scores["table_classes"] < 1.0:
+            details["table_classes_diff"] = {
+                "missing": [
+                    tc for tc in reference.table_classes if tc not in current.table_classes
+                ],
+                "new": [tc for tc in current.table_classes if tc not in reference.table_classes],
+            }
+    else:
+        scores["table_classes"] = 1.0
+
+    if reference.key_ids:
+        matches = sum(1 for kid in reference.key_ids if kid in current.key_ids)
+        scores["key_ids"] = matches / len(reference.key_ids)
+        if scores["key_ids"] < 1.0:
+            details["key_ids_diff"] = {
+                "missing": [kid for kid in reference.key_ids if kid not in current.key_ids],
+                "new": [kid for kid in current.key_ids if kid not in reference.key_ids],
+            }
+    else:
+        scores["key_ids"] = 1.0
+
+    if reference.table_headers:
+        header_score = 0.0
+        for ref_headers in reference.table_headers:
+            for cur_headers in current.table_headers:
+                ref_set = set(ref_headers)
+                cur_set = set(cur_headers)
+                if ref_set or cur_set:
+                    jaccard = len(ref_set & cur_set) / len(ref_set | cur_set)
+                    header_score = max(header_score, jaccard)
+        scores["table_headers"] = header_score
+        if scores["table_headers"] < 0.9:
+            details["table_headers_diff"] = {
+                "reference": reference.table_headers,
+                "current": current.table_headers,
+            }
+    else:
+        scores["table_headers"] = 1.0
+
+    count_diffs: dict[str, dict[str, int]] = {}
+    for key in reference.element_counts:
+        ref_count = reference.element_counts.get(key, 0)
+        cur_count = current.element_counts.get(key, 0)
+        if ref_count > 0:
+            diff_ratio = abs(cur_count - ref_count) / ref_count
+            if diff_ratio > 0.5:
+                count_diffs[key] = {"reference": ref_count, "current": cur_count}
+
+    if count_diffs:
+        scores["element_counts"] = max(0, 1 - len(count_diffs) * 0.2)
+        details["element_counts_diff"] = count_diffs
+    else:
+        scores["element_counts"] = 1.0
+
+    weights = {
+        "structure": 0.25,
+        "table_classes": 0.20,
+        "key_ids": 0.15,
+        "table_headers": 0.30,
+        "element_counts": 0.10,
+    }
+
+    final_score = sum(scores[k] * weights[k] for k in weights)
+
+    logger.debug(
+        "fingerprint_comparison",
+        scores=scores,
+        final_score=final_score,
+        has_changes=bool(details),
+    )
+
+    return final_score, details
+
+
+def save_baseline_fingerprint(fingerprint: Fingerprint, path: str) -> None:
+    """Salva fingerprint como baseline de referência."""
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(fingerprint.model_dump(mode="json"), f, indent=2, default=str)
+
+
+def load_baseline_fingerprint(path: str) -> Fingerprint | None:
+    """Carrega fingerprint de referência."""
+    if not Path(path).exists():
+        return None
+
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+        return Fingerprint.model_validate(data)