quantilica-cli 0.10.1__tar.gz → 0.11.0__tar.gz

{quantilica_cli-0.10.1 → quantilica_cli-0.11.0}/CHANGELOG.md

@@ -5,6 +5,31 @@ Todas as mudanças notáveis deste projeto serão documentadas neste arquivo.
 O formato segue [Keep a Changelog](https://keepachangelog.com/pt-BR/1.1.0/),
 e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
 
+## [0.11.0] - 2026-10-14
+
+### Adicionado
+
+- `sync --force` (e `force` keyword-only em `download_datasets`, `download_entry` e `_download_file`, repassado ao cliente só quando `True`): baixa mesmo com o local atualizado.
+- `check --out PATH`: grava o plano JSON de forma atômica e imprime só um resumo; `OSError` sai com código 1.
+- `check`: `HEAD` com 403 ou 405 passa a um probe `GET` com `Range: bytes=0-0` e `Accept-Encoding: identity` (tamanho via `Content-Range`, corpo nunca lido); URL que dá 404 no `HEAD` e no probe vira a ação `skip-unavailable`, aceita por `CheckPlan.from_dict`.
+- `FetcherApp.entry_from_plan`: gancho para fetchers que serializam objetos ricos na entrada do plano.
+- `default_client` lê `QUANTILICA_HTTP_MAX_RESUMES` (opcional), repassado como `max_resumes` ao `HttpClient`.
+- `quantilica install <fonte> --extra <nome>` (`-e`, repetível, aceita lista separada por vírgula): instala `<dist>[<extra>]` no ambiente da CLI, por exemplo `quantilica install pdet --extra analysis` para habilitar `convert` e `pipeline`. Funciona também em fonte já instalada. O estado continua registrando só a distribuição, e o re-exec pós-instalação não repete `--extra`, para não reinstalar em laço. Nomes de extra inválidos saem com código 2.
+
+### Alterado
+
+- Piso de `quantilica-core` elevado para `>=0.11.0` (símbolo usado: `max_resumes` em `HttpClient`).
+
+### Removido
+
+- `quantilica.cli.progress` (`file_progress`, `batch_progress`): módulo morto que nunca importou. Fazia `from .http import ProgressCallback` (não existe em `quantilica.cli`; o tipo vive em `quantilica.core.http`) e usava `tqdm`, que não é dependência do pacote. Nenhum fetcher o importava; o progresso real é Rich (`quantilica.cli.ui.make_download_progress`, `make_batch_progress`, `ProgressPool`).
+
+### Corrigido
+
+- `install_package` imprimia `Instalando pdet-fetcher...` sem o `[analysis]`: o Rich tratava o colchete como marcação. Agora o nome é escapado.
+- `sdk`: a dica de extra ausente em `convert`, `archive` e `pipeline` mandava `pip install <fetcher>[analysis]`, que não funciona para fetchers do Fluxo B (fora do PyPI). Agora imprime `quantilica install <chave> --extra analysis`, com a chave de registro resolvida pelo entry point (por exemplo `td` para `tesouro-direto-fetcher`).
+- `sdk`: `quantilica <fonte> list` (e o rótulo de partição do `check`/`sync`) terminava com `ValueError: Unknown format code 'd' for object of type 'str'` quando o catálogo traz `month` como string, caso do PDET (grupo capturado de regex). O mês e o semestre agora passam por `int()` antes do `:02d`, em `_partition_label`, `default_path_builder` e `list`; o `list` reutiliza `_partition_label` em vez de duplicar a lógica.
+
 ## [0.10.1] - 2026-10-08
 
 ### Corrigido

{quantilica_cli-0.10.1 → quantilica_cli-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quantilica-cli
-Version: 0.10.1
+Version: 0.11.0
 Summary: Unified CLI for Quantilica open data fetchers
 Author-email: "Komesu, D.K." <daniel@dkko.me>
 License-Expression: MIT
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
 Requires-Python: >=3.12
-Requires-Dist: quantilica-core>=0.10.1
+Requires-Dist: quantilica-core>=0.11.0
 Requires-Dist: rich>=13.0.0
 Requires-Dist: typer>=0.15.0
 Description-Content-Type: text/markdown

{quantilica_cli-0.10.1 → quantilica_cli-0.11.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "quantilica-cli"
-version = "0.10.1"
+version = "0.11.0"
 description = "Unified CLI for Quantilica open data fetchers"
 readme = "README.md"
 authors = [{ name = "Komesu, D.K.", email = "daniel@dkko.me" }]
@@ -20,7 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "quantilica-core>=0.10.1",
+    "quantilica-core>=0.11.0",
     "rich>=13.0.0",
     "typer>=0.15.0",
 ]

{quantilica_cli-0.10.1 → quantilica_cli-0.11.0}/src/quantilica/cli/sdk.py

@@ -14,6 +14,7 @@ import os
 import threading
 from collections.abc import Callable, Iterable
 from dataclasses import dataclass, field
+from importlib.metadata import entry_points
 from pathlib import Path
 from typing import Annotated, Any
 
@@ -125,12 +126,22 @@ class SyncPlan:
         )
 
 
+def _two_digits(value: Any) -> str:
+    """Format a month/semester as two digits, accepting ``int`` or numeric ``str``.
+
+    Catalogs built from filename regexes (e.g. PDET) carry the month as the
+    captured string (``"03"``), while others use ``int``; ``f"{x:02d}"`` only
+    works for the latter.
+    """
+    return f"{int(value):02d}"
+
+
 def _partition_label(entry: dict[str, Any]) -> str | None:
     """Human-readable partition label for a dataset entry."""
     if entry.get("semester") is not None:
         return f"{entry['year']}-S{entry['semester']}"
     if entry.get("month") is not None:
-        return f"{entry['year']}-{entry['month']:02d}"
+        return f"{entry['year']}-{_two_digits(entry['month'])}"
     if entry.get("year") is not None:
         return str(entry["year"])
     return None
@@ -154,6 +165,10 @@ def _short_remote(item: CheckPlanItem) -> str:
     return remote
 
 
+class _ProbeNotFoundError(Exception):
+    """O GET de probe Range respondeu 404 (URL removida no servidor)."""
+
+
 @dataclass(frozen=True)
 class CheckPlanItem:
     """Freshness verdict for a single dataset entry (no download performed).
@@ -170,7 +185,8 @@ class CheckPlanItem:
         local_exists: Whether the artifact already exists locally — this
             covers either the raw file itself or its ephemeral ``.manifest.json``
             sidecar marker (file deleted for retention, sidecar still fresh).
-        action: ``"download"`` or ``"skip-up-to-date"``.
+        action: ``"download"``, ``"skip-up-to-date"`` or ``"skip-unavailable"``
+            (todas as URLs da entrada responderam 404 no HEAD).
         reason: Machine-readable reason for the action.
         entry: Original dataset entry (allows ``sync --from-plan`` round-trip).
     """
@@ -225,11 +241,7 @@ class CheckPlan:
                 _short_remote(item),
             )
         con.print(table)
-        counts: dict[str, int] = {}
-        for item in self.items:
-            counts[item.action] = counts.get(item.action, 0) + 1
-        summary = ", ".join(f"{v} {k}" for k, v in sorted(counts.items()))
-        con.print(f"[bold]{len(self.items)}[/bold] verificado(s): {summary}.")
+        con.print(self.summary_line())
 
     def to_dict(self) -> dict[str, Any]:
         """Return the JSON-serializable representation of the plan."""
@@ -256,6 +268,14 @@ class CheckPlan:
             ],
         }
 
+    def summary_line(self) -> str:
+        """Return a one-line summary with the total and per-action counts."""
+        counts: dict[str, int] = {}
+        for item in self.items:
+            counts[item.action] = counts.get(item.action, 0) + 1
+        summary = ", ".join(f"{v} {k}" for k, v in sorted(counts.items()))
+        return f"[bold]{len(self.items)}[/bold] verificado(s): {summary}."
+
     def to_json(self) -> str:
         """Serialize the plan to a JSON string (for ``sync --from-plan``)."""
         return json.dumps(self.to_dict(), ensure_ascii=False, indent=2)
@@ -282,6 +302,7 @@ class CheckPlan:
             if not isinstance(entry, dict) or action not in (
                 "download",
                 "skip-up-to-date",
+                "skip-unavailable",
             ):
                 raise ValueError("not a check plan: invalid item")
             items.append(
@@ -332,6 +353,8 @@ def default_client() -> HttpClient:
       a path uses it as CA bundle.
     - ``QUANTILICA_HTTP_ATTEMPTS`` / ``QUANTILICA_HTTP_TIMEOUT`` /
       ``QUANTILICA_HTTP_RETRY_DELAY``: retry budget overrides.
+    - ``QUANTILICA_HTTP_MAX_RESUMES``: teto de retomadas com progresso de
+      downloads interrompidos (default do ``HttpClient`` quando ausente).
 
     Returns:
         A pre-configured HttpClient instance (browser-like WAF headers + pooling).
@@ -345,12 +368,18 @@ def default_client() -> HttpClient:
     attempts = int(os.environ.get("QUANTILICA_HTTP_ATTEMPTS", "5"))
     timeout = float(os.environ.get("QUANTILICA_HTTP_TIMEOUT", "180.0"))
     retry_base_delay = float(os.environ.get("QUANTILICA_HTTP_RETRY_DELAY", "2.0"))
+    # Só repassado quando definido: preserva o default do core.
+    extra: dict[str, Any] = {}
+    max_resumes = os.environ.get("QUANTILICA_HTTP_MAX_RESUMES", "").strip()
+    if max_resumes:
+        extra["max_resumes"] = int(max_resumes)
     return HttpClient(
         timeout=timeout,
         verify=verify,
         attempts=attempts,
         retry_base_delay=retry_base_delay,
         emulate_browser=True,
+        **extra,
     )
 
 
@@ -396,14 +425,14 @@ class DataRepository(StampedDataRepository):
         if year is not None and semester is not None:
             filename = build_stamped_filename(
                 slug,
-                f"{year}-{semester:02d}",
+                f"{year}-{_two_digits(semester)}",
                 ext=ext,
                 timestamp=last_modified,
             )
         elif year is not None and month is not None:
             filename = build_stamped_filename(
                 slug,
-                f"{year}-{month:02d}",
+                f"{year}-{_two_digits(month)}",
                 ext=ext,
                 timestamp=last_modified,
             )
@@ -543,6 +572,28 @@ class FetcherApp:
         """
         self.app.command(name=name, **kwargs)(cmd_func)
 
+    def _analysis_hint(self) -> str:
+        """Comando que instala o extra ``analysis`` no ambiente desta CLI.
+
+        Usa a chave de registro do fetcher (a que ``quantilica install`` e o
+        entry point conhecem, ex.: ``td``) e, se não a encontrar, o nome da
+        distribuição, que ``quantilica install`` também aceita.
+
+        Returns:
+            Texto sem colchetes, seguro para ``console.print``.
+        """
+        key = self.name
+        try:
+            for ep in entry_points(group="quantilica.fetchers"):
+                if ep.dist is not None and ep.dist.name == self.name:
+                    key = ep.name
+                    break
+        except (
+            Exception
+        ):  # pragma: no cover - metadados quebrados não podem mascarar o erro
+            pass
+        return f"quantilica install {key} --extra analysis"
+
     def command_convert(
         self, func: Callable[[Path, Path], Any]
     ) -> Callable[[Path, Path], Any]:
@@ -586,7 +637,7 @@ class FetcherApp:
             except ImportError:
                 console.print(
                     f"[red]Erro:[/red] convert requer extras de análise: "
-                    rf"pip install {self.name}\[analysis]"
+                    f"{self._analysis_hint()}"
                 )
                 raise typer.Exit(1) from None
             console.print(
@@ -639,7 +690,7 @@ class FetcherApp:
             except ImportError:
                 console.print(
                     f"[red]Erro:[/red] archive requer extras de análise: "
-                    rf"pip install {self.name}\[analysis]"
+                    f"{self._analysis_hint()}"
                 )
                 raise typer.Exit(1) from None
             console.print(f"[green]✓[/green] Arquivo criado em [dim]{output}[/dim].")
@@ -738,7 +789,7 @@ class FetcherApp:
             except ImportError:
                 console.print(
                     f"[red]Erro:[/red] pipeline (conversão) requer extras de "
-                    rf"análise: pip install {self.name}\[analysis]"
+                    f"análise: {self._analysis_hint()}"
                 )
                 raise typer.Exit(1) from None
             console.print(
@@ -754,9 +805,17 @@ class FetcherApp:
         return None
 
     def _download_file(
-        self, url: str, output: Path, progress: ProgressCallback | None = None
+        self,
+        url: str,
+        output: Path,
+        progress: ProgressCallback | None = None,
+        *,
+        force: bool = False,
     ) -> Path:
         dataset_id = output.parent.name
+        # `force` só é repassado quando True: clients/subclasses antigos não
+        # precisam aceitar o kwarg.
+        extra: dict[str, Any] = {"force": True} if force else {}
         return self.client.download_with_manifest(
             url,
             output,
@@ -764,6 +823,7 @@ class FetcherApp:
             dataset_id=dataset_id,
             producer=self.name,
             progress=progress,
+            **extra,
         )
 
     def download_entry(
@@ -773,6 +833,7 @@ class FetcherApp:
         *,
         dry_run: bool = False,
         progress: ProgressCallback | None = None,
+        force: bool = False,
     ) -> Path:
         """Download one dataset entry and return the destination path.
 
@@ -781,6 +842,7 @@ class FetcherApp:
             output_dir: Destination directory.
             dry_run: If True, computes the destination path without downloading.
             progress: Optional callback to track download progress.
+            force: Se True, ignora a checagem de frescor local e baixa sempre.
 
         Returns:
             The local path where the file was (or would be) saved.
@@ -811,6 +873,10 @@ class FetcherApp:
 
                 if dry_run:
                     return output
+                if force:
+                    return self._download_file(
+                        url, output, progress=progress, force=True
+                    )
                 return self._download_file(url, output, progress=progress)
             except HttpStatusError as exc:
                 if exc.status_code == 404:
@@ -834,6 +900,8 @@ class FetcherApp:
         entries: list[dict[str, Any]],
         output_dir: Path,
         workers: int = 4,
+        *,
+        force: bool = False,
     ) -> tuple[int, int, list[tuple[str, str]]]:
         """Executes parallel download for a list of dataset entries.
 
@@ -841,6 +909,7 @@ class FetcherApp:
             entries: List of dataset entries to download.
             output_dir: The target base directory for downloads.
             workers: Maximum number of parallel downloads.
+            force: Se True, ignora a checagem de frescor local e baixa tudo.
 
         Returns:
             A tuple containing:
@@ -891,7 +960,10 @@ class FetcherApp:
             try:
                 eid = entry.get("id", "unknown")
                 with pool.acquire(description=f"[cyan]{eid}[/cyan]") as cb:
-                    self.download_entry(entry, output_dir, progress=cb)
+                    if force:
+                        self.download_entry(entry, output_dir, progress=cb, force=True)
+                    else:
+                        self.download_entry(entry, output_dir, progress=cb)
                     return True
             except Exception as exc:
                 errors.append((entry.get("id", "unknown"), str(exc)))
@@ -946,14 +1018,24 @@ class FetcherApp:
 
         head = None
         head_error: Exception | None = None
+        not_found = 0
         for url in urls_to_try:
             try:
                 candidate = self.client.head(url)
             except HttpStatusError as exc:
+                head_error = exc
                 if exc.status_code == 404:
-                    head_error = exc
+                    not_found += 1
                     continue
-                head_error = exc
+                if exc.status_code in (403, 405):
+                    try:
+                        probed = self._range_probe(entry, output_dir, url, partition)
+                    except _ProbeNotFoundError:
+                        # HEAD recusado, mas o GET de probe prova que sumiu.
+                        not_found += 1
+                        continue
+                    if probed is not None:
+                        return probed
                 break
             except Exception as exc:  # noqa: BLE001 - metadata best effort
                 head_error = exc
@@ -963,6 +1045,21 @@ class FetcherApp:
 
         if head is None:
             local = self.path_builder(output_dir, entry, None)
+            if not_found == len(urls_to_try):
+                return CheckPlanItem(
+                    dataset=dataset,
+                    id=eid,
+                    url=str(entry["url"]),
+                    partition=partition,
+                    local_path=str(local),
+                    remote_etag=None,
+                    remote_last_modified=None,
+                    remote_size=None,
+                    local_exists=local.exists(),
+                    action="skip-unavailable",
+                    reason="remote-404",
+                    entry=dict(entry),
+                )
             return CheckPlanItem(
                 dataset=dataset,
                 id=eid,
@@ -1011,6 +1108,106 @@ class FetcherApp:
             entry=dict(entry),
         )
 
+    def entry_from_plan(self, data: dict[str, Any]) -> dict[str, Any]:
+        """Restaura uma entry desserializada de um :class:`CheckPlan`.
+
+        O padrão retorna uma cópia rasa de ``data``, sem nenhuma
+        transformação. Sobrescreva (subclasse ou
+        atribuição de função na instância, que recebe só ``data``) quando
+        ``check_entry`` serializa objetos ricos (ex.: ``datetime``) na entry.
+
+        Args:
+            data: Entry como lida do plano JSON.
+
+        Returns:
+            A entry pronta para ``download_entry``/``path_builder``.
+        """
+        return dict(data)
+
+    def _range_probe(
+        self,
+        entry: dict[str, Any],
+        output_dir: Path,
+        url: str,
+        partition: str | None,
+    ) -> CheckPlanItem | None:
+        """Probe ``GET`` com ``Range: bytes=0-0`` quando o ``HEAD`` é recusado.
+
+        Nunca lê o corpo: se o servidor ignorar o Range e responder 200, a
+        resposta é fechada sem consumir o body.
+
+        Returns:
+            O veredito da entrada, ou ``None`` se o client não suporta
+            streaming ou o probe falhou (mantém o comportamento padrão).
+
+        Raises:
+            _ProbeNotFoundError: Se o GET de probe responder 404.
+        """
+        stream = getattr(self.client, "stream", None)
+        if not callable(stream):
+            return None
+        try:
+            with stream(
+                "GET",
+                url,
+                # identity: Content-Length/Content-Range devem refletir o
+                # tamanho real, não o comprimido.
+                headers={"Range": "bytes=0-0", "Accept-Encoding": "identity"},
+            ) as response:
+                headers = response.headers
+                status = response.status_code
+                final_url = str(response.url)
+        except HttpStatusError as exc:
+            if exc.status_code == 404:
+                raise _ProbeNotFoundError(url) from exc
+            logger.debug("[check] probe Range falhou para %s: %r", url, exc)
+            return None
+        except Exception as exc:  # noqa: BLE001 - probe é best effort
+            logger.debug("[check] probe Range falhou para %s: %r", url, exc)
+            return None
+
+        total: int | None = None
+        content_range = headers.get("Content-Range")
+        if status == 206 and content_range and "/" in content_range:
+            raw = content_range.rsplit("/", 1)[1].strip()
+            total = int(raw) if raw.isdigit() else None
+        elif status == 200:
+            encoding = (headers.get("Content-Encoding") or "identity").strip()
+            if encoding.lower() != "identity":
+                return None  # Content-Length seria o tamanho comprimido
+            raw = (headers.get("Content-Length") or "").strip()
+            total = int(raw) if raw.isdigit() else None
+        if total is None:
+            return None
+
+        last_modified = _parse_head_date(headers.get("Last-Modified"))
+        local = self.path_builder(output_dir, entry, last_modified)
+        sidecar = manifest_sidecar_path(local)
+        local_exists = local.exists()
+        exists = local_exists or sidecar.exists()
+        action, reason = "download", "range-probe"
+        if local_exists and local.stat().st_size == total:
+            action, reason = "skip-up-to-date", "size-match-range-probe"
+        elif sidecar.exists():
+            with contextlib.suppress(OSError, ValueError, AttributeError):
+                data = json.loads(sidecar.read_text(encoding="utf-8"))
+                if data.get("size_bytes") == total:
+                    action, reason = "skip-up-to-date", "size-match-range-probe"
+        return CheckPlanItem(
+            dataset=str(entry.get("group") or entry.get("id") or "datasets"),
+            id=str(entry.get("id", "unknown")),
+            url=final_url,
+            partition=partition,
+            local_path=str(local),
+            remote_etag=headers.get("ETag"),
+            remote_last_modified=headers.get("Last-Modified"),
+            remote_size=total,
+            local_exists=exists,
+            action=action,
+            reason=reason,
+            entry=dict(entry),
+        )
+
     def check_datasets(
         self,
         entries: list[dict[str, Any]],
@@ -1083,6 +1280,14 @@ class FetcherApp:
                     "de grupos).",
                 ),
             ] = None,
+            force: Annotated[
+                bool,
+                typer.Option(
+                    "--force",
+                    help="Ignora a checagem de frescor local (arquivo/sidecar) "
+                    "e baixa todas as entradas selecionadas (útil com --from-plan)",
+                ),
+            ] = False,
             verbose: Annotated[
                 bool, typer.Option("--verbose", help="Logs detalhados")
             ] = False,
@@ -1097,7 +1302,9 @@ class FetcherApp:
                     console.print(f"[red]Plano inválido: {exc}[/red]")
                     raise typer.Exit(1) from None
                 entries = [
-                    dict(item.entry) for item in plan.items if item.action == "download"
+                    self.entry_from_plan(dict(item.entry))
+                    for item in plan.items
+                    if item.action == "download"
                 ]
                 console.print(
                     f"[dim]Plano {from_plan}: {len(entries)} entrada(s) "
@@ -1121,9 +1328,14 @@ class FetcherApp:
                 console.print(f"\n[bold]{len(entries)}[/bold] arquivo(s) listado(s).")
                 return
 
-            downloaded, total, errors = self.download_datasets(
-                entries, actual_output, workers=workers
-            )
+            if force:
+                downloaded, total, errors = self.download_datasets(
+                    entries, actual_output, workers=workers, force=True
+                )
+            else:
+                downloaded, total, errors = self.download_datasets(
+                    entries, actual_output, workers=workers
+                )
 
             console.print(
                 f"\n[green]Concluído:[/green] {downloaded}/{total} arquivo(s) baixado(s)."
@@ -1154,6 +1366,14 @@ class FetcherApp:
                     help="Imprime o plano em JSON (para 'sync --from-plan')",
                 ),
             ] = False,
+            out: Annotated[
+                Path | None,
+                typer.Option(
+                    "--out",
+                    help="Grava o plano JSON neste arquivo (atômico) e imprime "
+                    "só um resumo; prevalece sobre --json",
+                ),
+            ] = None,
             verbose: Annotated[
                 bool, typer.Option("--verbose", help="Logs detalhados")
             ] = False,
@@ -1171,7 +1391,20 @@ class FetcherApp:
             entries = [e for g in target_groups for e in self.list_datasets(g)]
             plan = self.check_datasets(entries, actual_output, workers=workers)
 
-            if as_json:
+            if out is not None:
+                tmp = out.with_name(f".{out.name}.{os.getpid()}.tmp")
+                try:
+                    out.parent.mkdir(parents=True, exist_ok=True)
+                    try:
+                        tmp.write_text(plan.to_json(), encoding="utf-8")
+                        os.replace(tmp, out)
+                    finally:
+                        tmp.unlink(missing_ok=True)
+                except OSError as exc:
+                    console.print(f"[red]Falha ao gravar o plano em {out}: {exc}[/red]")
+                    raise typer.Exit(1) from None
+                console.print(f"{plan.summary_line()} Plano gravado em {out}")
+            elif as_json:
                 print(plan.to_json())
             else:
                 plan.render_table()
@@ -1193,17 +1426,9 @@ class FetcherApp:
                     title=f"[bold]{group_id}[/bold] — {group_info.get('name', '')}",
                 )
                 for entry in self.list_datasets(group_id):
-                    if entry.get("semester") is not None:
-                        partition = f"{entry['year']}-S{entry['semester']}"
-                    elif entry.get("month") is not None:
-                        partition = f"{entry['year']}-{entry['month']:02d}"
-                    elif entry.get("year") is not None:
-                        partition = str(entry["year"])
-                    else:
-                        partition = "—"
                     table.add_row(
                         entry.get("id", ""),
-                        partition,
+                        _partition_label(entry) or "—",
                         entry.get("ext", ""),
                         entry.get("url", ""),
                     )