get-sinapi 3.1.0__py3-none-any.whl

getsinapi/collectors/caixa.py

@@ -0,0 +1,386 @@
+"""Coleta resiliente dos pacotes mensais XLSX publicados pela CAIXA."""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+import time
+import zipfile
+from dataclasses import asdict, dataclass
+from datetime import UTC, datetime
+from io import BytesIO
+from pathlib import Path
+from urllib.parse import urljoin, urlparse
+
+import requests
+from bs4 import BeautifulSoup
+from curl_cffi import requests as browser_requests
+
+from getsinapi.config import Settings
+
+REFERENCE_PATTERN = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
+BASE_URL = (
+    "https://www.caixa.gov.br/Downloads/sinapi-relatorios-mensais/"
+    "SINAPI-{reference}-formato-xlsx.zip"
+)
+DOWNLOADS_PAGE_URL = "https://www.caixa.gov.br/site/paginas/downloads.aspx"
+DOWNLOAD_LINK_PATTERN = re.compile(
+    r"SINAPI[-_](20\d{2})[-_](0[1-9]|1[0-2])[-_ ]formato[-_ ]xlsx",
+    re.IGNORECASE,
+)
+log = logging.getLogger(__name__)
+RETRYABLE_STATUS = {408, 425, 429, 500, 502, 503, 504}
+
+
+@dataclass(frozen=True, slots=True)
+class DownloadArtifact:
+    reference: str
+    url: str
+    path: Path
+    sha256: str
+    size_bytes: int
+    downloaded_at: str
+
+
+class CaixaDownloadError(RuntimeError):
+    """Falha ao obter ou validar um pacote oficial da CAIXA."""
+
+
+def report_url(reference: str) -> str:
+    if not REFERENCE_PATTERN.fullmatch(reference):
+        raise ValueError("Competência inválida. Informe AAAA-MM, por exemplo 2026-09.")
+    return BASE_URL.format(reference=reference)
+
+
+def _retry_delay(response: requests.Response | None, attempt: int, base: float) -> float:
+    if response is not None:
+        retry_after = response.headers.get("Retry-After")
+        if retry_after:
+            try:
+                return min(max(float(retry_after), 0.0), 120.0)
+            except ValueError:
+                pass
+    return min(base * (2 ** (attempt - 1)), 60.0)
+
+
+
+def _http_get(url: str, *, timeout: int, headers: dict, session=None):
+    """Acessa a CAIXA com fingerprint de navegador para contornar bloqueios anti-bot."""
+    if session is not None:
+        return session.get(url, timeout=timeout, headers=headers)
+    return browser_requests.get(
+        url, timeout=timeout, headers=headers, impersonate="chrome",
+        allow_redirects=True,
+    )
+
+
+def _discover_by_monthly_url(settings: Settings, session=None, months_back: int = 12) -> tuple[str, str] | None:
+    """Probe the official monthly ZIP URL used by the CAIXA publication pattern.
+
+    The CAIXA HTML/SharePoint indexes are not consistently populated. PulseSINAPI
+    uses this stable URL template directly, so probe recent references newest-first.
+    A URL is accepted only when the response is a valid ZIP archive.
+    """
+    now = datetime.now(UTC)
+    headers = {
+        "Accept": "application/zip,application/octet-stream,*/*",
+        "Referer": DOWNLOADS_PAGE_URL,
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        ),
+    }
+    year, month = now.year, now.month
+    for _ in range(months_back):
+        reference = f"{year:04d}-{month:02d}"
+        url = report_url(reference)
+        for attempt in range(1, settings.http_max_retries + 2):
+            response = None
+            try:
+                response = _http_get(
+                    url, timeout=settings.http_timeout, headers=headers, session=session,
+                )
+                status = response.status_code
+                if status == 200:
+                    payload = response.content
+                    if payload and zipfile.is_zipfile(BytesIO(payload)):
+                        log.info("Pacote mensal oficial identificado por URL direta: %s", reference)
+                        return reference, url
+                    log.warning("URL mensal %s respondeu 200, mas o conteúdo não é ZIP.", reference)
+                    break
+                if status in RETRYABLE_STATUS and attempt <= settings.http_max_retries:
+                    delay = _retry_delay(response, attempt, settings.http_backoff_seconds)
+                    response.close()
+                    response = None
+                    time.sleep(delay)
+                    continue
+                if status not in {403, 404, 410}:
+                    log.warning("Sondagem da competência %s retornou HTTP %s.", reference, status)
+                break
+            except (requests.RequestException, browser_requests.RequestsError) as exc:
+                if attempt > settings.http_max_retries:
+                    log.warning("Falha ao sondar a competência %s: %s", reference, exc)
+                    break
+                time.sleep(_retry_delay(None, attempt, settings.http_backoff_seconds))
+            finally:
+                if response is not None:
+                    response.close()
+        month -= 1
+        if month == 0:
+            year -= 1
+            month = 12
+    return None
+
+
+def _discover_via_sharepoint(settings: Settings, session=None) -> tuple[str, str]:
+    """Descobre pacotes mensais pelo índice SharePoint oficial quando o catálogo HTML falha."""
+    endpoint = (
+        "https://www.caixa.gov.br/_api/web/lists/getbytitle('Downloads')/Items"
+        "?$select=Title,FileLeafRef,EncodedAbsUrl&$filter=FSObjType%20eq%200"
+        "%20and%20substringof('SINAPI',FileLeafRef)&$top=5000"
+    )
+    headers = {
+        "Accept": "application/json;odata=verbose",
+        "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
+        "Referer": "https://www.caixa.gov.br/",
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        ),
+    }
+    candidates: dict[str, str] = {}
+    next_url: str | None = endpoint
+    pages = 0
+    while next_url and pages < 4:
+        response = None
+        try:
+            response = _http_get(
+                next_url, timeout=settings.http_timeout, headers=headers, session=session,
+            )
+            if response.status_code != 200:
+                raise CaixaDownloadError(
+                    "Índice SharePoint oficial da CAIXA indisponível "
+                    f"(HTTP {response.status_code}); nenhuma fonte alternativa validada."
+                )
+            payload = response.json()
+            data = payload.get("d", payload)
+            items = data.get("results", data.get("value", []))
+            for item in items:
+                filename = str(item.get("FileLeafRef") or item.get("Title") or "")
+                target = str(item.get("EncodedAbsUrl") or "")
+                match = DOWNLOAD_LINK_PATTERN.search(filename)
+                if not match or not filename.lower().endswith(".zip") or not target:
+                    continue
+                reference = f"{match.group(1)}-{match.group(2)}"
+                parsed = urlparse(target)
+                hostname = (parsed.hostname or "").lower()
+                if parsed.scheme == "https" and (
+                    hostname == "caixa.gov.br" or hostname.endswith(".caixa.gov.br")
+                ):
+                    candidates[reference] = target
+            next_url = data.get("__next") or payload.get("@odata.nextLink")
+            if next_url:
+                next_url = urljoin("https://www.caixa.gov.br", next_url)
+            pages += 1
+        except (requests.RequestException, browser_requests.RequestsError, ValueError) as exc:
+            raise CaixaDownloadError(
+                f"Falha ao consultar o índice SharePoint oficial da CAIXA: {exc}"
+            ) from exc
+        finally:
+            if response is not None:
+                response.close()
+    if not candidates:
+        raise CaixaDownloadError(
+            "Catálogo HTML e índice SharePoint da CAIXA não forneceram "
+            "um pacote mensal XLSX oficial identificável."
+        )
+    return max(candidates.items(), key=lambda item: item[0])
+
+
+def discover_latest_monthly_xlsx(
+    settings: Settings | None = None,
+    session=None,
+) -> tuple[str, str]:
+    """Descobre a última competência XLSX pelos links publicados no catálogo oficial."""
+    settings = settings or Settings.from_env()
+    headers = {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
+        "Referer": "https://www.caixa.gov.br/",
+    }
+    response = None
+    try:
+        response = _http_get(
+            DOWNLOADS_PAGE_URL, timeout=settings.http_timeout,
+            headers=headers, session=session,
+        )
+        if response.status_code != 200:
+            catalog_status = response.status_code
+            response.close()
+            response = None
+            log.warning(
+                "Catálogo HTML da CAIXA indisponível (HTTP %s); tentando o índice oficial SharePoint.",
+                catalog_status,
+            )
+            direct = _discover_by_monthly_url(settings=settings, session=session)
+            if direct:
+                return direct
+            return _discover_via_sharepoint(settings=settings, session=session)
+        soup = BeautifulSoup(response.text, "html.parser")
+        candidates: dict[str, str] = {}
+        for anchor in soup.find_all("a", href=True):
+            href = str(anchor.get("href", "")).strip()
+            label = f"{anchor.get_text(' ', strip=True)} {href}"
+            match = DOWNLOAD_LINK_PATTERN.search(label)
+            if not match:
+                continue
+            reference = f"{match.group(1)}-{match.group(2)}"
+            target = urljoin(DOWNLOADS_PAGE_URL, href)
+            parsed = urlparse(target)
+            hostname = (parsed.hostname or "").lower()
+            if parsed.scheme != "https" or not (
+                hostname == "caixa.gov.br" or hostname.endswith(".caixa.gov.br")
+            ):
+                continue
+            candidates[reference] = target
+        if not candidates:
+            log.warning(
+                "O catálogo HTML respondeu, mas não expôs links mensais XLSX; "
+                "acionando a descoberta pelo índice SharePoint oficial."
+            )
+            direct = _discover_by_monthly_url(settings=settings, session=session)
+            if direct:
+                return direct
+            return _discover_via_sharepoint(settings=settings, session=session)
+        return max(candidates.items(), key=lambda item: item[0])
+    except (requests.RequestException, browser_requests.RequestsError) as exc:
+        raise CaixaDownloadError(
+            f"Falha de rede ao consultar o catálogo oficial da CAIXA: {exc}"
+        ) from exc
+    finally:
+        if response is not None:
+            response.close()
+
+
+def download_monthly_xlsx(
+    reference: str,
+    settings: Settings | None = None,
+    session: requests.Session | None = None,
+    source_url: str | None = None,
+) -> DownloadArtifact:
+    """Baixa, valida e registra o ZIP XLSX mensal; nunca substitui um arquivo válido."""
+    url = source_url or report_url(reference)
+    settings = settings or Settings.from_env()
+    settings.create_data_directories()
+    headers = {
+        "Accept": "application/zip,application/octet-stream,*/*",
+        "Referer": DOWNLOADS_PAGE_URL,
+    }
+    response = None
+
+    for attempt in range(1, settings.http_max_retries + 2):
+        try:
+            response = _http_get(
+                url, timeout=settings.http_timeout, headers=headers, session=session,
+            )
+        except (requests.RequestException, browser_requests.RequestsError) as exc:
+            if attempt > settings.http_max_retries:
+                raise CaixaDownloadError(f"Falha de rede ao baixar {url}: {exc}") from exc
+            time.sleep(_retry_delay(None, attempt, settings.http_backoff_seconds))
+            continue
+
+        if response.status_code == 200:
+            break
+        if response.status_code not in RETRYABLE_STATUS or attempt > settings.http_max_retries:
+            status = response.status_code
+            response.close()
+            if status == 403:
+                raise CaixaDownloadError(
+                    f"CAIXA negou o download da competência {reference} (HTTP 403), "
+                    "mesmo após a descoberta pelo catálogo oficial. Nenhum arquivo foi aceito."
+                )
+            raise CaixaDownloadError(
+                f"CAIXA respondeu HTTP {status} para a competência {reference}."
+            )
+        delay = _retry_delay(response, attempt, settings.http_backoff_seconds)
+        response.close()
+        time.sleep(delay)
+
+    if response is None or response.status_code != 200:
+        raise CaixaDownloadError(f"Não foi possível obter a competência {reference}.")
+
+    payload = response.content
+    response.close()
+    if not payload or not zipfile.is_zipfile(BytesIO(payload)):
+        raise CaixaDownloadError(
+            "A resposta não é um ZIP válido. O pacote não será salvo nem tratado como dado SINAPI."
+        )
+
+    digest = hashlib.sha256(payload).hexdigest()
+    filename = f"SINAPI-{reference}-formato-xlsx-{digest[:12]}.zip"
+    target = settings.raw_dir / filename
+    target.parent.mkdir(parents=True, exist_ok=True)
+
+    if not target.exists():
+        temporary = target.with_suffix(target.suffix + ".part")
+        temporary.write_bytes(payload)
+        temporary.replace(target)
+
+    artifact = DownloadArtifact(
+        reference=reference,
+        url=url,
+        path=target,
+        sha256=digest,
+        size_bytes=len(payload),
+        downloaded_at=datetime.now(UTC).isoformat(),
+    )
+    manifest_dir = settings.manifest_dir
+    manifest_dir.mkdir(parents=True, exist_ok=True)
+    manifest_path = manifest_dir / f"{reference}-{digest[:12]}.json"
+    manifest_tmp = manifest_path.with_suffix(".json.part")
+    manifest_tmp.write_text(
+        json.dumps({**asdict(artifact), "path": str(target)}, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    manifest_tmp.replace(manifest_path)
+    return artifact
+
+
+def safe_extract_zip(
+    archive_path: Path,
+    destination: Path,
+    max_total_uncompressed_bytes: int = 2_000_000_000,
+) -> list[Path]:
+    """Extrai ZIP após bloquear traversal, caminhos absolutos e links simbólicos."""
+    destination = destination.resolve()
+    destination.mkdir(parents=True, exist_ok=True)
+    extracted: list[Path] = []
+
+    with zipfile.ZipFile(archive_path) as archive:
+        members = archive.infolist()
+        total_size = sum(member.file_size for member in members)
+        if total_size > max_total_uncompressed_bytes:
+            raise CaixaDownloadError("ZIP excede o limite de extração configurado.")
+
+        for member in members:
+            name = member.filename.replace("\\", "/")
+            parsed = Path(name)
+            mode = (member.external_attr >> 16) & 0o170000
+            if parsed.is_absolute() or ".." in parsed.parts or mode == 0o120000:
+                raise CaixaDownloadError(f"Caminho inseguro encontrado no ZIP: {member.filename}")
+            target = (destination / parsed).resolve()
+            if target != destination and destination not in target.parents:
+                raise CaixaDownloadError(f"Destino de extração inválido: {member.filename}")
+
+        for member in members:
+            if member.is_dir():
+                continue
+            target = (destination / member.filename.replace("\\", "/")).resolve()
+            target.parent.mkdir(parents=True, exist_ok=True)
+            with archive.open(member) as source, target.open("wb") as output:
+                while chunk := source.read(1024 * 1024):
+                    output.write(chunk)
+            extracted.append(target)
+
+    return extracted

getsinapi/collectors/history.py

@@ -0,0 +1,238 @@
+"""Descoberta do catálogo histórico de pacotes XLSX mensais da CAIXA.
+
+A estrutura XLSX unificada publicada a partir de 2025 contém todas as UFs.
+Os relatórios até 2024 seguem o layout legado por UF e exigem parser próprio;
+não devem ser declarados como cobertos por este coletor.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+from urllib.parse import urljoin, urlparse
+
+from bs4 import BeautifulSoup
+from curl_cffi import requests as browser_requests
+
+from getsinapi.collectors.caixa import (
+    DOWNLOAD_LINK_PATTERN,
+    DOWNLOADS_PAGE_URL,
+    CaixaDownloadError,
+    _http_get,
+    report_url,
+)
+from getsinapi.config import Settings
+
+log = logging.getLogger("getsinapi.collectors.history")
+LEGACY_LAYOUT_END = "2024-12"
+UNIFIED_LAYOUT_START = "2025-01"
+
+
+def _trusted_caixa_url(value: str, base_url: str) -> str | None:
+    target = urljoin(base_url, value.strip())
+    parsed = urlparse(target)
+    host = (parsed.hostname or "").lower()
+    if parsed.scheme != "https" or not (
+        host == "caixa.gov.br" or host.endswith(".caixa.gov.br")
+    ):
+        return None
+    return target
+
+
+def _reference_from_filename(filename: str) -> str | None:
+    match = DOWNLOAD_LINK_PATTERN.search(filename)
+    if not match:
+        return None
+    return f"{match.group(1)}-{match.group(2)}"
+
+
+def _from_html(html: str) -> dict[str, str]:
+    soup = BeautifulSoup(html, "html.parser")
+    found: dict[str, str] = {}
+    for anchor in soup.find_all("a", href=True):
+        href = str(anchor.get("href", "")).strip()
+        label = f"{anchor.get_text(' ', strip=True)} {href}"
+        reference = _reference_from_filename(label)
+        if not reference or reference < UNIFIED_LAYOUT_START:
+            continue
+        target = _trusted_caixa_url(href, DOWNLOADS_PAGE_URL)
+        if target and "formato-xlsx" in target.lower():
+            found[reference] = target
+    return found
+
+
+def _http_head(url: str, *, timeout: int, headers: dict, session=None):
+    """Consulta metadados do pacote sem transferir dezenas de megabytes."""
+    if session is not None:
+        return session.head(url, timeout=timeout, headers=headers, allow_redirects=True)
+    return browser_requests.head(
+        url, timeout=timeout, headers=headers, impersonate="chrome",
+        allow_redirects=True,
+    )
+
+
+def _from_monthly_urls(settings: Settings, session=None) -> dict[str, str]:
+    """Descobre cada publicação pelo padrão oficial de URL, validando HTTP HEAD."""
+    now = datetime.now(UTC)
+    current = now.year * 12 + now.month - 1
+    first = 2025 * 12
+    headers = {
+        "Accept": "application/zip,application/octet-stream,*/*",
+        "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
+        "Referer": "https://www.caixa.gov.br/poder-publico/modernizacao-gestao/sinapi/Paginas/default.aspx",
+    }
+    found: dict[str, str] = {}
+    for absolute_month in range(first, current + 1):
+        year, zero_month = divmod(absolute_month, 12)
+        reference = f"{year:04d}-{zero_month + 1:02d}"
+        url = report_url(reference)
+        response = None
+        try:
+            response = _http_head(
+                url, timeout=settings.http_timeout, headers=headers, session=session
+            )
+            if response.status_code != 200:
+                continue
+            content_type = str(response.headers.get("Content-Type", "")).lower()
+            if "text/html" in content_type:
+                continue
+            content_length = response.headers.get("Content-Length")
+            if content_length is not None and int(content_length) <= 0:
+                continue
+            found[reference] = url
+        except Exception as exc:  # noqa: BLE001 - falha isolada por competência não deve abortar o inventário
+            log.warning("Não foi possível sondar a publicação %s por URL direta: %s", reference, exc)
+        finally:
+            if response is not None:
+                response.close()
+    log.info(
+        "Sondagem mensal direta da CAIXA: %s competências XLSX identificadas entre 2025-01 e %s.",
+        len(found), f"{now.year:04d}-{now.month:02d}",
+    )
+    return found
+
+
+def _from_sharepoint(settings: Settings, session=None) -> dict[str, str]:
+    endpoint = (
+        "https://www.caixa.gov.br/_api/web/lists/getbytitle('Downloads')/Items"
+        "?$select=Title,FileLeafRef,EncodedAbsUrl,FSObjType&$top=5000"
+    )
+    headers = {
+        "Accept": "application/json;odata=verbose",
+        "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
+        "Referer": "https://www.caixa.gov.br/",
+    }
+    found: dict[str, str] = {}
+    items_seen = 0
+    next_url: str | None = endpoint
+    pages = 0
+    while next_url and pages < 20:
+        response = _http_get(
+            next_url, timeout=settings.http_timeout, headers=headers, session=session
+        )
+        try:
+            if response.status_code != 200:
+                raise CaixaDownloadError(
+                    "Índice SharePoint da CAIXA indisponível "
+                    f"(HTTP {response.status_code}) durante descoberta histórica."
+                )
+            payload = response.json()
+            data = payload.get("d", payload)
+            items = data.get("results", data.get("value", []))
+            items_seen += len(items)
+            for item in items:
+                filename = str(item.get("FileLeafRef") or item.get("Title") or "")
+                if "SINAPI" not in filename.upper() or not filename.lower().endswith(".zip"):
+                    continue
+                reference = _reference_from_filename(filename)
+                target = str(item.get("EncodedAbsUrl") or "")
+                if not reference or reference < UNIFIED_LAYOUT_START:
+                    continue
+                trusted = _trusted_caixa_url(target, "https://www.caixa.gov.br/")
+                if trusted and "formato-xlsx" in trusted.lower():
+                    found[reference] = trusted
+            next_url = data.get("__next") or payload.get("@odata.nextLink")
+            if next_url:
+                next_url = urljoin("https://www.caixa.gov.br", next_url)
+            pages += 1
+        finally:
+            response.close()
+    if next_url:
+        raise CaixaDownloadError(
+            "Catálogo histórico da CAIXA excedeu o limite de paginação; "
+            "a descoberta foi interrompida para não declarar cobertura parcial como completa."
+        )
+    log.info(
+        "Índice SharePoint consultado: %s itens examinados, %s competências XLSX identificadas.",
+        items_seen, len(found),
+    )
+    return found
+
+
+def discover_historical_xlsx_catalog(
+    settings: Settings | None = None,
+    session=None,
+) -> dict[str, str]:
+    """Retorna o inventário consolidado das competências XLSX publicadas pela CAIXA.
+
+    O catálogo HTML, o índice SharePoint e as URLs mensais oficiais são
+    consultados como superfícies complementares. O inventário final inclui
+    somente competências encontradas por pelo menos uma fonte oficial; o pacote
+    ZIP completo é validado durante o download antes de qualquer carga.
+    """
+    config = settings or Settings.from_env()
+    response = None
+    try:
+        response = _http_get(
+            DOWNLOADS_PAGE_URL,
+            timeout=config.http_timeout,
+            headers={
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
+                "Referer": "https://www.caixa.gov.br/",
+            },
+            session=session,
+        )
+        if response.status_code == 200:
+            catalog = _from_html(response.text)
+        else:
+            log.warning(
+                "Catálogo HTML da CAIXA indisponível (HTTP %s); "
+                "tentando o índice SharePoint oficial.",
+                response.status_code,
+            )
+            catalog = {}
+    except Exception as exc:  # noqa: BLE001 - fontes alternativas devem ser tentadas mesmo após falha de rede
+        log.warning("Falha ao consultar o catálogo HTML oficial da CAIXA: %s", exc)
+        catalog = {}
+    finally:
+        if response is not None:
+            response.close()
+
+    # A página pública de Downloads é a fonte primária. O índice SharePoint
+    # complementa o catálogo, mas não pode invalidar links oficiais já extraídos
+    # do HTML quando a API interna da CAIXA estiver indisponível ou incompatível.
+    try:
+        sharepoint_catalog = _from_sharepoint(config, session=session)
+    except Exception as exc:  # noqa: BLE001 - falha do índice não deve impedir a sondagem direta
+        log.warning(
+            "Índice SharePoint indisponível (%s); prosseguindo com sondagem "
+            "das URLs mensais oficiais. Catálogo HTML: %s competências.",
+            exc, len(catalog),
+        )
+        sharepoint_catalog = {}
+
+    catalog.update(sharepoint_catalog)
+
+    # A listagem do portal pode ser renderizada dinamicamente e o índice
+    # SharePoint pode não expor todos os itens. A URL mensal oficial é sondada
+    # competência a competência; somente HTTP 200 com resposta não-HTML entra
+    # no inventário. O download completo e a validação ZIP ocorrem na ingestão.
+    monthly_catalog = _from_monthly_urls(config, session=session)
+    catalog.update(monthly_catalog)
+
+    if not catalog:
+        raise CaixaDownloadError(
+            "Nenhuma competência XLSX unificada foi encontrada nas fontes oficiais. "
+            "A ingestão histórica não será marcada como concluída."
+        )
+    return dict(sorted(catalog.items()))

getsinapi/config.py

@@ -0,0 +1,60 @@
+"""Configuração centralizada e validada por variáveis de ambiente."""
+from __future__ import annotations
+
+import math
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True, slots=True)
+class Settings:
+    environment: str
+    log_level: str
+    data_dir: Path
+    raw_dir: Path
+    staging_dir: Path
+    processed_dir: Path
+    manifest_dir: Path
+    http_timeout: int
+    http_max_retries: int
+    http_backoff_seconds: float
+
+    @classmethod
+    def from_env(cls) -> Settings:
+        data_dir = Path(os.getenv("GETSINAPI_DATA_DIR", "data"))
+        settings = cls(
+            environment=os.getenv("GETSINAPI_ENV", "development").strip().lower(),
+            log_level=os.getenv("GETSINAPI_LOG_LEVEL", "INFO").strip().upper(),
+            data_dir=data_dir,
+            raw_dir=Path(os.getenv("GETSINAPI_RAW_DIR", str(data_dir / "raw"))),
+            staging_dir=Path(os.getenv("GETSINAPI_STAGING_DIR", str(data_dir / "staging"))),
+            processed_dir=Path(os.getenv("GETSINAPI_PROCESSED_DIR", str(data_dir / "processed"))),
+            manifest_dir=Path(os.getenv("GETSINAPI_MANIFEST_DIR", str(data_dir / "manifests"))),
+            http_timeout=int(os.getenv("GETSINAPI_HTTP_TIMEOUT", "45")),
+            http_max_retries=int(os.getenv("GETSINAPI_HTTP_MAX_RETRIES", "4")),
+            http_backoff_seconds=float(os.getenv("GETSINAPI_HTTP_BACKOFF_SECONDS", "2")),
+        )
+        settings.validate()
+        return settings
+
+    def validate(self) -> None:
+        if self.environment not in {"development", "test", "production"}:
+            raise ValueError("GETSINAPI_ENV deve ser development, test ou production.")
+        if self.http_timeout <= 0:
+            raise ValueError("GETSINAPI_HTTP_TIMEOUT deve ser maior que zero.")
+        if self.http_max_retries < 0:
+            raise ValueError("GETSINAPI_HTTP_MAX_RETRIES não pode ser negativo.")
+        if (
+            isinstance(self.http_backoff_seconds, bool)
+            or not isinstance(self.http_backoff_seconds, (int, float))
+            or not math.isfinite(self.http_backoff_seconds)
+            or self.http_backoff_seconds < 0
+        ):
+            raise ValueError(
+                "GETSINAPI_HTTP_BACKOFF_SECONDS deve ser finito e não negativo."
+            )
+
+    def create_data_directories(self) -> None:
+        for directory in (self.raw_dir, self.staging_dir, self.processed_dir, self.manifest_dir):
+            directory.mkdir(parents=True, exist_ok=True)