quantilica-cli 0.11.0__tar.gz → 0.12.0__tar.gz

{quantilica_cli-0.11.0 → quantilica_cli-0.12.0}/CHANGELOG.md

@@ -5,6 +5,17 @@ Todas as mudanças notáveis deste projeto serão documentadas neste arquivo.
 O formato segue [Keep a Changelog](https://keepachangelog.com/pt-BR/1.1.0/),
 e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
 
+## [0.12.0] - 2026-10-19
+
+### Adicionado
+
+- `FetcherApp(source_id=...)`: `source_id` explícito, gravado nos manifests de download. Sem ele, continua derivado de `name` sem o sufixo `-fetcher`, e o `default_output` padrão passa a ser `/data/{source_id}`.
+
+### Corrigido
+
+- `check`: o probe `GET Range` tenta de novo (com backoff) em 429/5xx e em erros de transporte (`DEFAULT_RETRY_EXCEPTIONS`: timeout, reset de conexão) antes de desistir. Antes, uma falha transitória virava 404 e a URL saía como indisponível. Quando o probe desiste, a causa é logada em `WARNING` (`[check] probe Range falhou para <url> após N tentativa(s): ...`), e não mais em `DEBUG`.
+- `check`: uma exceção inesperada em uma entrada (por exemplo, `OSError` ao montar o caminho local) não aborta mais o `check` inteiro com código 1. A entrada é verificada de novo uma vez e, se falhar outra vez, vira `download` com `reason=check-error: ...`, e o plano segue cobrindo as demais.
+
 ## [0.11.0] - 2026-10-14
 
 ### Adicionado

{quantilica_cli-0.11.0 → quantilica_cli-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quantilica-cli
-Version: 0.11.0
+Version: 0.12.0
 Summary: Unified CLI for Quantilica open data fetchers
 Author-email: "Komesu, D.K." <daniel@dkko.me>
 License-Expression: MIT

{quantilica_cli-0.11.0 → quantilica_cli-0.12.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "quantilica-cli"
-version = "0.11.0"
+version = "0.12.0"
 description = "Unified CLI for Quantilica open data fetchers"
 readme = "README.md"
 authors = [{ name = "Komesu, D.K.", email = "daniel@dkko.me" }]

{quantilica_cli-0.11.0 → quantilica_cli-0.12.0}/src/quantilica/cli/sdk.py

@@ -12,6 +12,7 @@ import email.utils
 import json
 import os
 import threading
+import time
 from collections.abc import Callable, Iterable
 from dataclasses import dataclass, field
 from importlib.metadata import entry_points
@@ -22,6 +23,7 @@ import typer
 from quantilica.core.exceptions import FetchError
 from quantilica.core.ftp import FtpClient
 from quantilica.core.http import (
+    DEFAULT_RETRY_EXCEPTIONS,
     HttpClient,
     HttpStatusError,
     ProgressCallback,
@@ -165,6 +167,13 @@ def _short_remote(item: CheckPlanItem) -> str:
     return remote
 
 
+# Probe Range: novas tentativas em status ou erro de transporte transitório
+# (``DEFAULT_RETRY_EXCEPTIONS``) antes de desistir.
+_PROBE_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
+_PROBE_ATTEMPTS = 3
+_PROBE_BACKOFF_S = 1.0
+
+
 class _ProbeNotFoundError(Exception):
     """O GET de probe Range respondeu 404 (URL removida no servidor)."""
 
@@ -513,6 +522,9 @@ class FetcherApp:
             :func:`default_path_builder` (canonical ``DataRepository`` layout).
         default_output: Default output directory path.
         client: HTTP or FTP client instance. Defaults to default_client().
+        source_id: Source ID recorded in download manifests. Defaults to
+            ``name`` without the ``-fetcher`` suffix. When ``default_output``
+            is omitted it is derived as ``/data/{source_id}``.
         build_default_commands: If True (default), registers the built-in
             ``sync`` and ``list`` commands. Set to False to register only
             custom commands via :meth:`attach_command`.
@@ -530,6 +542,7 @@ class FetcherApp:
         path_builder: Callable[[Path, dict[str, Any], dt.date | None], Path]
         | None = None,
         default_output: Path | None = None,
+        source_id: str | None = None,
         client: HttpClient | FtpClient | None = None,
         build_default_commands: bool = True,
     ):
@@ -539,9 +552,10 @@ class FetcherApp:
         self.aliases = aliases_dict
         self.list_datasets = list_datasets
         self.path_builder = path_builder or default_path_builder
-        self.default_output = default_output or Path(
-            f"/data/{name.replace('-fetcher', '')}"
-        )
+        # source_id explícito vence; senão deriva de `name` sem o sufixo
+        # "-fetcher" (comportamento histórico).
+        self.source_id = source_id or name.replace("-fetcher", "")
+        self.default_output = default_output or Path(f"/data/{self.source_id}")
         self.client = client or default_client()
 
         self.all_group_keys = list(self.groups.keys())
@@ -819,7 +833,7 @@ class FetcherApp:
         return self.client.download_with_manifest(
             url,
             output,
-            source_id=self.name.replace("-fetcher", ""),
+            source_id=self.source_id,
             dataset_id=dataset_id,
             producer=self.name,
             progress=progress,
@@ -1146,24 +1160,41 @@ class FetcherApp:
         stream = getattr(self.client, "stream", None)
         if not callable(stream):
             return None
-        try:
-            with stream(
-                "GET",
+        for attempt in range(1, _PROBE_ATTEMPTS + 1):
+            try:
+                with stream(
+                    "GET",
+                    url,
+                    # identity: Content-Length/Content-Range devem refletir o
+                    # tamanho real, não o comprimido.
+                    headers={"Range": "bytes=0-0", "Accept-Encoding": "identity"},
+                ) as response:
+                    headers = response.headers
+                    status = response.status_code
+                    final_url = str(response.url)
+                break
+            except HttpStatusError as exc:
+                if exc.status_code == 404:
+                    raise _ProbeNotFoundError(url) from exc
+                failure: Exception = exc
+                retryable = exc.status_code in _PROBE_RETRY_STATUSES
+            except DEFAULT_RETRY_EXCEPTIONS as exc:
+                failure, retryable = exc, True
+            except Exception as exc:  # noqa: BLE001 - probe é best effort
+                failure, retryable = exc, False
+            # 429/5xx ou erro de transporte transitório (ex.: 503 intermitente
+            # do gov.br): sem nova tentativa, uma URL removida viraria
+            # "download" e falharia com 404 no sync.
+            if retryable and attempt < _PROBE_ATTEMPTS:
+                time.sleep(_PROBE_BACKOFF_S * attempt)
+                continue
+            # WARNING: o cron não roda com --verbose e precisa registrar a causa.
+            logger.warning(
+                "[check] probe Range falhou para %s após %d tentativa(s): %r",
                 url,
-                # identity: Content-Length/Content-Range devem refletir o
-                # tamanho real, não o comprimido.
-                headers={"Range": "bytes=0-0", "Accept-Encoding": "identity"},
-            ) as response:
-                headers = response.headers
-                status = response.status_code
-                final_url = str(response.url)
-        except HttpStatusError as exc:
-            if exc.status_code == 404:
-                raise _ProbeNotFoundError(url) from exc
-            logger.debug("[check] probe Range falhou para %s: %r", url, exc)
-            return None
-        except Exception as exc:  # noqa: BLE001 - probe é best effort
-            logger.debug("[check] probe Range falhou para %s: %r", url, exc)
+                attempt,
+                failure,
+            )
             return None
 
         total: int | None = None
@@ -1208,6 +1239,45 @@ class FetcherApp:
             entry=dict(entry),
         )
 
+    def _check_entry_isolated(
+        self, entry: dict[str, Any], output_dir: Path
+    ) -> CheckPlanItem:
+        """Run :meth:`check_entry` without letting one entry abort the check.
+
+        An unexpected error (e.g. ``OSError`` while building the local path)
+        is retried once; if it persists the entry becomes a conservative
+        ``download`` verdict, so the plan still covers every other entry
+        instead of the whole ``check`` exiting 1.
+        """
+        for attempt in (1, 2):
+            try:
+                return self.check_entry(entry, output_dir)
+            except Exception as exc:  # noqa: BLE001 - isolate per entry
+                failure = exc
+                if attempt == 1:
+                    time.sleep(1.0)
+        eid = str(entry.get("id", "unknown"))
+        logger.warning(
+            "[check] falha ao verificar %s (%s: %s); marcada para download",
+            eid,
+            type(failure).__name__,
+            failure,
+        )
+        return CheckPlanItem(
+            dataset=str(entry.get("group") or entry.get("id") or "datasets"),
+            id=eid,
+            url=str(entry.get("url", "")),
+            partition=_partition_label(entry),
+            local_path="",
+            remote_etag=None,
+            remote_last_modified=None,
+            remote_size=None,
+            local_exists=False,
+            action="download",
+            reason=f"check-error: {type(failure).__name__}: {failure}",
+            entry=dict(entry),
+        )
+
     def check_datasets(
         self,
         entries: list[dict[str, Any]],
@@ -1227,7 +1297,7 @@ class FetcherApp:
         results: list[CheckPlanItem | None] = [None] * len(entries)
 
         def _worker(index: int, entry: dict[str, Any]) -> None:
-            results[index] = self.check_entry(entry, output_dir)
+            results[index] = self._check_entry_isolated(entry, output_dir)
             if (index + 1) % 100 == 0 or index + 1 == len(entries):
                 logger.info("[check] %d/%d verificados", index + 1, len(entries))
 

{quantilica_cli-0.11.0 → quantilica_cli-0.12.0}/tests/test_sdk.py

@@ -497,3 +497,86 @@ def test_check_entry_ephemeral_stale_downloads(tmp_path: Path):
     item = app.check_entry(entry, tmp_path)
     assert item.action == "download"
     assert item.reason == "remote-newer"
+
+
+def test_source_id_defaults_to_name_without_suffix():
+    app = make_app()
+    assert app.source_id == "comex"
+    assert app.default_output == Path("/data/comex")
+
+
+def test_explicit_source_id_derives_default_output():
+    app = make_app(source_id="custom")
+    assert app.source_id == "custom"
+    assert app.default_output == Path("/data/custom")
+
+
+def test_explicit_default_output_wins_over_source_id(tmp_path: Path):
+    app = make_app(source_id="custom", default_output=tmp_path)
+    assert app.default_output == tmp_path
+
+
+def test_check_datasets_isolates_entry_failure(tmp_path: Path, monkeypatch):
+    """Uma entrada que levanta (ex.: OSError) não derruba o check inteiro."""
+    monkeypatch.setattr("quantilica.cli.sdk.time.sleep", lambda _s: None)
+    app = make_app(
+        client=HttpClient(
+            attempts=1,
+            transport=httpx2.MockTransport(lambda r: httpx2.Response(200)),
+        )
+    )
+    entries = sample_list_datasets("exp")
+    calls: dict[str, int] = {}
+    real = app.check_entry
+
+    def flaky(entry, output_dir):
+        key = str(entry["id"])
+        calls[key] = calls.get(key, 0) + 1
+        if key == str(entries[0]["id"]):
+            raise FileExistsError(17, "File exists", "/data")
+        return real(entry, output_dir)
+
+    monkeypatch.setattr(app, "check_entry", flaky)
+    plan = app.check_datasets(entries, tmp_path)
+    assert len(plan.items) == len(entries)
+    bad = next(i for i in plan.items if i.id == str(entries[0]["id"]))
+    assert bad.action == "download"
+    assert bad.reason.startswith("check-error: FileExistsError")
+    assert calls[str(entries[0]["id"])] == 2  # uma nova tentativa
+
+
+def test_check_datasets_retry_recovers_transient_failure(tmp_path: Path, monkeypatch):
+    monkeypatch.setattr("quantilica.cli.sdk.time.sleep", lambda _s: None)
+    app = make_app(
+        client=HttpClient(
+            attempts=1,
+            transport=httpx2.MockTransport(lambda r: httpx2.Response(200)),
+        )
+    )
+    entries = sample_list_datasets("exp")[:1]
+    real = app.check_entry
+    state = {"n": 0}
+
+    def once(entry, output_dir):
+        state["n"] += 1
+        if state["n"] == 1:
+            raise OSError("transiente")
+        return real(entry, output_dir)
+
+    monkeypatch.setattr(app, "check_entry", once)
+    plan = app.check_datasets(entries, tmp_path)
+    assert plan.items[0].reason != "" and not plan.items[0].reason.startswith(
+        "check-error"
+    )
+
+
+def test_check_entry_403_head_without_probe_support_downloads(tmp_path: Path):
+    """HEAD 403 e probe Range 403 → download (nunca exceção/exit 1)."""
+    app = make_app(
+        client=HttpClient(
+            attempts=1,
+            transport=httpx2.MockTransport(lambda r: httpx2.Response(403)),
+        )
+    )
+    item = app.check_entry(sample_list_datasets("exp")[0], tmp_path)
+    assert item.action == "download"

{quantilica_cli-0.11.0 → quantilica_cli-0.12.0}/tests/test_sdk_force_probe.py

@@ -485,3 +485,92 @@ def test_head_403_probe_500_stays_download(tmp_path: Path):
     item = _probe_app(handler).check_entry(ENTRY, tmp_path)
     assert item.action == "download"
     assert item.reason.startswith("metadata-unavailable")
+
+
+@pytest.mark.parametrize(
+    ("statuses", "action", "reason"),
+    [
+        # 503 transitório e depois 404 → removida de fato
+        ([503, 404], "skip-unavailable", "remote-404"),
+        # 429 e depois 206 → metadados pelo Range
+        ([429, 206], "download", "range-probe"),
+        # 503 persistente → desiste após as tentativas
+        ([503, 503, 503], "download", "metadata-unavailable"),
+        # 403 no GET não é transitório → sem nova tentativa
+        ([403], "download", "metadata-unavailable"),
+    ],
+)
+def test_probe_retries_transient_status(
+    tmp_path: Path, monkeypatch, statuses, action, reason
+):
+    from quantilica.cli import sdk
+
+    monkeypatch.setattr(sdk, "_PROBE_BACKOFF_S", 0.0)
+    gets: list[int] = []
+
+    def handler(request):
+        if request.method == "HEAD":
+            return httpx2.Response(403)
+        status = statuses[min(len(gets), len(statuses) - 1)]
+        gets.append(status)
+        headers = {"Content-Range": "bytes 0-0/10"} if status == 206 else {}
+        return httpx2.Response(status, headers=headers)
+
+    item = _probe_app(handler).check_entry(ENTRY, tmp_path)
+    assert item.action == action
+    assert item.reason.startswith(reason)
+    assert gets == statuses
+
+
+@pytest.mark.parametrize(
+    ("failures", "action", "reason"),
+    [
+        # timeout transitório e depois 404 → removida de fato
+        ([httpx2.ReadTimeout], "skip-unavailable", "remote-404"),
+        # conexão derrubada e depois 206 → metadados pelo Range
+        ([httpx2.RemoteProtocolError], "download", "range-probe"),
+        # erro de transporte persistente → desiste após as tentativas
+        ([httpx2.ConnectError] * 3, "download", "metadata-unavailable"),
+    ],
+)
+def test_probe_retries_transport_errors(
+    tmp_path: Path, monkeypatch, failures, action, reason
+):
+    from quantilica.cli import sdk
+
+    monkeypatch.setattr(sdk, "_PROBE_BACKOFF_S", 0.0)
+    gets: list[int] = []
+
+    def handler(request):
+        if request.method == "HEAD":
+            return httpx2.Response(403)
+        gets.append(1)
+        if len(gets) <= len(failures):
+            raise failures[len(gets) - 1]("transitório", request=request)
+        if reason == "remote-404":
+            return httpx2.Response(404)
+        return httpx2.Response(206, headers={"Content-Range": "bytes 0-0/10"})
+
+    item = _probe_app(handler).check_entry(ENTRY, tmp_path)
+    assert item.action == action
+    assert item.reason.startswith(reason)
+    assert len(gets) == min(len(failures) + 1, 3)
+
+
+def test_probe_failure_logs_warning_with_cause(tmp_path: Path, monkeypatch):
+    from quantilica.cli import sdk
+
+    monkeypatch.setattr(sdk, "_PROBE_BACKOFF_S", 0.0)
+    warnings: list[str] = []
+    monkeypatch.setattr(
+        sdk.logger, "warning", lambda msg, *args: warnings.append(msg % args)
+    )
+
+    def handler(request):
+        return httpx2.Response(403 if request.method == "HEAD" else 503)
+
+    item = _probe_app(handler).check_entry(ENTRY, tmp_path)
+    assert item.reason.startswith("metadata-unavailable")
+    assert len(warnings) == 1
+    assert "após 3 tentativa(s)" in warnings[0]
+    assert "503" in warnings[0]