quantilica-cli 0.10.0__tar.gz → 0.11.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/CHANGELOG.md +33 -0
  2. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/PKG-INFO +2 -2
  3. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/pyproject.toml +2 -2
  4. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/sdk.py +265 -33
  5. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/sources.py +54 -9
  6. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_package_hygiene.py +21 -0
  7. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_sdk.py +62 -0
  8. quantilica_cli-0.11.0/tests/test_sdk_force_probe.py +487 -0
  9. quantilica_cli-0.11.0/tests/test_sdk_mes_string.py +51 -0
  10. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_sdk_onda_a2.py +12 -4
  11. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_sources.py +91 -0
  12. quantilica_cli-0.10.0/src/quantilica/cli/progress.py +0 -77
  13. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/.githooks/pre-commit +0 -0
  14. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/.githooks/pre-push +0 -0
  15. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/.github/workflows/publish.yml +0 -0
  16. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/.github/workflows/test.yml +0 -0
  17. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/.gitignore +0 -0
  18. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/LICENSE +0 -0
  19. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/README.md +0 -0
  20. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/__init__.py +0 -0
  21. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/cli.py +0 -0
  22. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/health.py +0 -0
  23. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/manifests.py +0 -0
  24. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/ui.py +0 -0
  25. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/py.typed +0 -0
  26. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/__init__.py +0 -0
  27. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_health.py +0 -0
  28. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_manifests.py +0 -0
  29. {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_ui.py +0 -0
@@ -5,6 +5,39 @@ Todas as mudanças notáveis deste projeto serão documentadas neste arquivo.
5
5
  O formato segue [Keep a Changelog](https://keepachangelog.com/pt-BR/1.1.0/),
6
6
  e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
7
7
 
8
+ ## [0.11.0] - 2026-10-14
9
+
10
+ ### Adicionado
11
+
12
+ - `sync --force` (e `force` keyword-only em `download_datasets`, `download_entry` e `_download_file`, repassado ao cliente só quando `True`): baixa mesmo com o local atualizado.
13
+ - `check --out PATH`: grava o plano JSON de forma atômica e imprime só um resumo; `OSError` sai com código 1.
14
+ - `check`: `HEAD` com 403 ou 405 passa a um probe `GET` com `Range: bytes=0-0` e `Accept-Encoding: identity` (tamanho via `Content-Range`, corpo nunca lido); URL que dá 404 no `HEAD` e no probe vira a ação `skip-unavailable`, aceita por `CheckPlan.from_dict`.
15
+ - `FetcherApp.entry_from_plan`: gancho para fetchers que serializam objetos ricos na entrada do plano.
16
+ - `default_client` lê `QUANTILICA_HTTP_MAX_RESUMES` (opcional), repassado como `max_resumes` ao `HttpClient`.
17
+ - `quantilica install <fonte> --extra <nome>` (`-e`, repetível, aceita lista separada por vírgula): instala `<dist>[<extra>]` no ambiente da CLI, por exemplo `quantilica install pdet --extra analysis` para habilitar `convert` e `pipeline`. Funciona também em fonte já instalada. O estado continua registrando só a distribuição, e o re-exec pós-instalação não repete `--extra`, para não reinstalar em laço. Nomes de extra inválidos saem com código 2.
18
+
19
+ ### Alterado
20
+
21
+ - Piso de `quantilica-core` elevado para `>=0.11.0` (símbolo usado: `max_resumes` em `HttpClient`).
22
+
23
+ ### Removido
24
+
25
+ - `quantilica.cli.progress` (`file_progress`, `batch_progress`): módulo morto que nunca importou. Fazia `from .http import ProgressCallback` (não existe em `quantilica.cli`; o tipo vive em `quantilica.core.http`) e usava `tqdm`, que não é dependência do pacote. Nenhum fetcher o importava; o progresso real é Rich (`quantilica.cli.ui.make_download_progress`, `make_batch_progress`, `ProgressPool`).
26
+
27
+ ### Corrigido
28
+
29
+ - `install_package` imprimia `Instalando pdet-fetcher...` sem o `[analysis]`: o Rich tratava o colchete como marcação. Agora o nome é escapado.
30
+ - `sdk`: a dica de extra ausente em `convert`, `archive` e `pipeline` mandava `pip install <fetcher>[analysis]`, que não funciona para fetchers do Fluxo B (fora do PyPI). Agora imprime `quantilica install <chave> --extra analysis`, com a chave de registro resolvida pelo entry point (por exemplo `td` para `tesouro-direto-fetcher`).
31
+ - `sdk`: `quantilica <fonte> list` (e o rótulo de partição do `check`/`sync`) terminava com `ValueError: Unknown format code 'd' for object of type 'str'` quando o catálogo traz `month` como string, caso do PDET (grupo capturado de regex). O mês e o semestre agora passam por `int()` antes do `:02d`, em `_partition_label`, `default_path_builder` e `list`; o `list` reutiliza `_partition_label` em vez de duplicar a lógica.
32
+
33
+ ## [0.10.1] - 2026-10-08
34
+
35
+ ### Corrigido
36
+
37
+ - `sdk`: `check_entry` considera `local.exists() or sidecar.exists()`, preservando
38
+ o status de `manifest-fresh` em ambientes de ingestão efêmera sem forçar
39
+ re-download redundante quando o dado bruto já foi espelhado.
40
+
8
41
  ## [0.10.0] - 2026-10-07
9
42
 
10
43
  ### Adicionado
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: quantilica-cli
3
- Version: 0.10.0
3
+ Version: 0.11.0
4
4
  Summary: Unified CLI for Quantilica open data fetchers
5
5
  Author-email: "Komesu, D.K." <daniel@dkko.me>
6
6
  License-Expression: MIT
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.12
16
16
  Classifier: Programming Language :: Python :: 3.13
17
17
  Classifier: Typing :: Typed
18
18
  Requires-Python: >=3.12
19
- Requires-Dist: quantilica-core>=0.10.0
19
+ Requires-Dist: quantilica-core>=0.11.0
20
20
  Requires-Dist: rich>=13.0.0
21
21
  Requires-Dist: typer>=0.15.0
22
22
  Description-Content-Type: text/markdown
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "quantilica-cli"
3
- version = "0.10.0"
3
+ version = "0.11.0"
4
4
  description = "Unified CLI for Quantilica open data fetchers"
5
5
  readme = "README.md"
6
6
  authors = [{ name = "Komesu, D.K.", email = "daniel@dkko.me" }]
@@ -20,7 +20,7 @@ classifiers = [
20
20
  "Programming Language :: Python :: 3.13",
21
21
  ]
22
22
  dependencies = [
23
- "quantilica-core>=0.10.0",
23
+ "quantilica-core>=0.11.0",
24
24
  "rich>=13.0.0",
25
25
  "typer>=0.15.0",
26
26
  ]
@@ -14,6 +14,7 @@ import os
14
14
  import threading
15
15
  from collections.abc import Callable, Iterable
16
16
  from dataclasses import dataclass, field
17
+ from importlib.metadata import entry_points
17
18
  from pathlib import Path
18
19
  from typing import Annotated, Any
19
20
 
@@ -28,6 +29,7 @@ from quantilica.core.http import (
28
29
  resolve_verify_from_env,
29
30
  )
30
31
  from quantilica.core.logging import get_logger
32
+ from quantilica.core.manifests import manifest_sidecar_path
31
33
  from quantilica.core.storage import (
32
34
  StampedDataRepository,
33
35
  build_stamped_filename,
@@ -124,12 +126,22 @@ class SyncPlan:
124
126
  )
125
127
 
126
128
 
129
+ def _two_digits(value: Any) -> str:
130
+ """Format a month/semester as two digits, accepting ``int`` or numeric ``str``.
131
+
132
+ Catalogs built from filename regexes (e.g. PDET) carry the month as the
133
+ captured string (``"03"``), while others use ``int``; ``f"{x:02d}"`` only
134
+ works for the latter.
135
+ """
136
+ return f"{int(value):02d}"
137
+
138
+
127
139
  def _partition_label(entry: dict[str, Any]) -> str | None:
128
140
  """Human-readable partition label for a dataset entry."""
129
141
  if entry.get("semester") is not None:
130
142
  return f"{entry['year']}-S{entry['semester']}"
131
143
  if entry.get("month") is not None:
132
- return f"{entry['year']}-{entry['month']:02d}"
144
+ return f"{entry['year']}-{_two_digits(entry['month'])}"
133
145
  if entry.get("year") is not None:
134
146
  return str(entry["year"])
135
147
  return None
@@ -153,6 +165,10 @@ def _short_remote(item: CheckPlanItem) -> str:
153
165
  return remote
154
166
 
155
167
 
168
+ class _ProbeNotFoundError(Exception):
169
+ """O GET de probe Range respondeu 404 (URL removida no servidor)."""
170
+
171
+
156
172
  @dataclass(frozen=True)
157
173
  class CheckPlanItem:
158
174
  """Freshness verdict for a single dataset entry (no download performed).
@@ -166,8 +182,11 @@ class CheckPlanItem:
166
182
  remote_etag: Remote ETag header, or None.
167
183
  remote_last_modified: Remote Last-Modified header, or None.
168
184
  remote_size: Remote Content-Length, or None.
169
- local_exists: Whether the local file already exists.
170
- action: ``"download"`` or ``"skip-up-to-date"``.
185
+ local_exists: Whether the artifact already exists locally — this
186
+ covers either the raw file itself or its ephemeral ``.manifest.json``
187
+ sidecar marker (file deleted for retention, sidecar still fresh).
188
+ action: ``"download"``, ``"skip-up-to-date"`` or ``"skip-unavailable"``
189
+ (todas as URLs da entrada responderam 404 no HEAD).
171
190
  reason: Machine-readable reason for the action.
172
191
  entry: Original dataset entry (allows ``sync --from-plan`` round-trip).
173
192
  """
@@ -222,11 +241,7 @@ class CheckPlan:
222
241
  _short_remote(item),
223
242
  )
224
243
  con.print(table)
225
- counts: dict[str, int] = {}
226
- for item in self.items:
227
- counts[item.action] = counts.get(item.action, 0) + 1
228
- summary = ", ".join(f"{v} {k}" for k, v in sorted(counts.items()))
229
- con.print(f"[bold]{len(self.items)}[/bold] verificado(s): {summary}.")
244
+ con.print(self.summary_line())
230
245
 
231
246
  def to_dict(self) -> dict[str, Any]:
232
247
  """Return the JSON-serializable representation of the plan."""
@@ -253,6 +268,14 @@ class CheckPlan:
253
268
  ],
254
269
  }
255
270
 
271
+ def summary_line(self) -> str:
272
+ """Return a one-line summary with the total and per-action counts."""
273
+ counts: dict[str, int] = {}
274
+ for item in self.items:
275
+ counts[item.action] = counts.get(item.action, 0) + 1
276
+ summary = ", ".join(f"{v} {k}" for k, v in sorted(counts.items()))
277
+ return f"[bold]{len(self.items)}[/bold] verificado(s): {summary}."
278
+
256
279
  def to_json(self) -> str:
257
280
  """Serialize the plan to a JSON string (for ``sync --from-plan``)."""
258
281
  return json.dumps(self.to_dict(), ensure_ascii=False, indent=2)
@@ -279,6 +302,7 @@ class CheckPlan:
279
302
  if not isinstance(entry, dict) or action not in (
280
303
  "download",
281
304
  "skip-up-to-date",
305
+ "skip-unavailable",
282
306
  ):
283
307
  raise ValueError("not a check plan: invalid item")
284
308
  items.append(
@@ -329,6 +353,8 @@ def default_client() -> HttpClient:
329
353
  a path uses it as CA bundle.
330
354
  - ``QUANTILICA_HTTP_ATTEMPTS`` / ``QUANTILICA_HTTP_TIMEOUT`` /
331
355
  ``QUANTILICA_HTTP_RETRY_DELAY``: retry budget overrides.
356
+ - ``QUANTILICA_HTTP_MAX_RESUMES``: teto de retomadas com progresso de
357
+ downloads interrompidos (default do ``HttpClient`` quando ausente).
332
358
 
333
359
  Returns:
334
360
  A pre-configured HttpClient instance (browser-like WAF headers + pooling).
@@ -342,12 +368,18 @@ def default_client() -> HttpClient:
342
368
  attempts = int(os.environ.get("QUANTILICA_HTTP_ATTEMPTS", "5"))
343
369
  timeout = float(os.environ.get("QUANTILICA_HTTP_TIMEOUT", "180.0"))
344
370
  retry_base_delay = float(os.environ.get("QUANTILICA_HTTP_RETRY_DELAY", "2.0"))
371
+ # Só repassado quando definido: preserva o default do core.
372
+ extra: dict[str, Any] = {}
373
+ max_resumes = os.environ.get("QUANTILICA_HTTP_MAX_RESUMES", "").strip()
374
+ if max_resumes:
375
+ extra["max_resumes"] = int(max_resumes)
345
376
  return HttpClient(
346
377
  timeout=timeout,
347
378
  verify=verify,
348
379
  attempts=attempts,
349
380
  retry_base_delay=retry_base_delay,
350
381
  emulate_browser=True,
382
+ **extra,
351
383
  )
352
384
 
353
385
 
@@ -393,14 +425,14 @@ class DataRepository(StampedDataRepository):
393
425
  if year is not None and semester is not None:
394
426
  filename = build_stamped_filename(
395
427
  slug,
396
- f"{year}-{semester:02d}",
428
+ f"{year}-{_two_digits(semester)}",
397
429
  ext=ext,
398
430
  timestamp=last_modified,
399
431
  )
400
432
  elif year is not None and month is not None:
401
433
  filename = build_stamped_filename(
402
434
  slug,
403
- f"{year}-{month:02d}",
435
+ f"{year}-{_two_digits(month)}",
404
436
  ext=ext,
405
437
  timestamp=last_modified,
406
438
  )
@@ -540,6 +572,28 @@ class FetcherApp:
540
572
  """
541
573
  self.app.command(name=name, **kwargs)(cmd_func)
542
574
 
575
+ def _analysis_hint(self) -> str:
576
+ """Comando que instala o extra ``analysis`` no ambiente desta CLI.
577
+
578
+ Usa a chave de registro do fetcher (a que ``quantilica install`` e o
579
+ entry point conhecem, ex.: ``td``) e, se não a encontrar, o nome da
580
+ distribuição, que ``quantilica install`` também aceita.
581
+
582
+ Returns:
583
+ Texto sem colchetes, seguro para ``console.print``.
584
+ """
585
+ key = self.name
586
+ try:
587
+ for ep in entry_points(group="quantilica.fetchers"):
588
+ if ep.dist is not None and ep.dist.name == self.name:
589
+ key = ep.name
590
+ break
591
+ except (
592
+ Exception
593
+ ): # pragma: no cover - metadados quebrados não podem mascarar o erro
594
+ pass
595
+ return f"quantilica install {key} --extra analysis"
596
+
543
597
  def command_convert(
544
598
  self, func: Callable[[Path, Path], Any]
545
599
  ) -> Callable[[Path, Path], Any]:
@@ -583,7 +637,7 @@ class FetcherApp:
583
637
  except ImportError:
584
638
  console.print(
585
639
  f"[red]Erro:[/red] convert requer extras de análise: "
586
- rf"pip install {self.name}\[analysis]"
640
+ f"{self._analysis_hint()}"
587
641
  )
588
642
  raise typer.Exit(1) from None
589
643
  console.print(
@@ -636,7 +690,7 @@ class FetcherApp:
636
690
  except ImportError:
637
691
  console.print(
638
692
  f"[red]Erro:[/red] archive requer extras de análise: "
639
- rf"pip install {self.name}\[analysis]"
693
+ f"{self._analysis_hint()}"
640
694
  )
641
695
  raise typer.Exit(1) from None
642
696
  console.print(f"[green]✓[/green] Arquivo criado em [dim]{output}[/dim].")
@@ -735,7 +789,7 @@ class FetcherApp:
735
789
  except ImportError:
736
790
  console.print(
737
791
  f"[red]Erro:[/red] pipeline (conversão) requer extras de "
738
- rf"análise: pip install {self.name}\[analysis]"
792
+ f"análise: {self._analysis_hint()}"
739
793
  )
740
794
  raise typer.Exit(1) from None
741
795
  console.print(
@@ -751,9 +805,17 @@ class FetcherApp:
751
805
  return None
752
806
 
753
807
  def _download_file(
754
- self, url: str, output: Path, progress: ProgressCallback | None = None
808
+ self,
809
+ url: str,
810
+ output: Path,
811
+ progress: ProgressCallback | None = None,
812
+ *,
813
+ force: bool = False,
755
814
  ) -> Path:
756
815
  dataset_id = output.parent.name
816
+ # `force` só é repassado quando True: clients/subclasses antigos não
817
+ # precisam aceitar o kwarg.
818
+ extra: dict[str, Any] = {"force": True} if force else {}
757
819
  return self.client.download_with_manifest(
758
820
  url,
759
821
  output,
@@ -761,6 +823,7 @@ class FetcherApp:
761
823
  dataset_id=dataset_id,
762
824
  producer=self.name,
763
825
  progress=progress,
826
+ **extra,
764
827
  )
765
828
 
766
829
  def download_entry(
@@ -770,6 +833,7 @@ class FetcherApp:
770
833
  *,
771
834
  dry_run: bool = False,
772
835
  progress: ProgressCallback | None = None,
836
+ force: bool = False,
773
837
  ) -> Path:
774
838
  """Download one dataset entry and return the destination path.
775
839
 
@@ -778,6 +842,7 @@ class FetcherApp:
778
842
  output_dir: Destination directory.
779
843
  dry_run: If True, computes the destination path without downloading.
780
844
  progress: Optional callback to track download progress.
845
+ force: Se True, ignora a checagem de frescor local e baixa sempre.
781
846
 
782
847
  Returns:
783
848
  The local path where the file was (or would be) saved.
@@ -808,6 +873,10 @@ class FetcherApp:
808
873
 
809
874
  if dry_run:
810
875
  return output
876
+ if force:
877
+ return self._download_file(
878
+ url, output, progress=progress, force=True
879
+ )
811
880
  return self._download_file(url, output, progress=progress)
812
881
  except HttpStatusError as exc:
813
882
  if exc.status_code == 404:
@@ -831,6 +900,8 @@ class FetcherApp:
831
900
  entries: list[dict[str, Any]],
832
901
  output_dir: Path,
833
902
  workers: int = 4,
903
+ *,
904
+ force: bool = False,
834
905
  ) -> tuple[int, int, list[tuple[str, str]]]:
835
906
  """Executes parallel download for a list of dataset entries.
836
907
 
@@ -838,6 +909,7 @@ class FetcherApp:
838
909
  entries: List of dataset entries to download.
839
910
  output_dir: The target base directory for downloads.
840
911
  workers: Maximum number of parallel downloads.
912
+ force: Se True, ignora a checagem de frescor local e baixa tudo.
841
913
 
842
914
  Returns:
843
915
  A tuple containing:
@@ -888,7 +960,10 @@ class FetcherApp:
888
960
  try:
889
961
  eid = entry.get("id", "unknown")
890
962
  with pool.acquire(description=f"[cyan]{eid}[/cyan]") as cb:
891
- self.download_entry(entry, output_dir, progress=cb)
963
+ if force:
964
+ self.download_entry(entry, output_dir, progress=cb, force=True)
965
+ else:
966
+ self.download_entry(entry, output_dir, progress=cb)
892
967
  return True
893
968
  except Exception as exc:
894
969
  errors.append((entry.get("id", "unknown"), str(exc)))
@@ -943,14 +1018,24 @@ class FetcherApp:
943
1018
 
944
1019
  head = None
945
1020
  head_error: Exception | None = None
1021
+ not_found = 0
946
1022
  for url in urls_to_try:
947
1023
  try:
948
1024
  candidate = self.client.head(url)
949
1025
  except HttpStatusError as exc:
1026
+ head_error = exc
950
1027
  if exc.status_code == 404:
951
- head_error = exc
1028
+ not_found += 1
952
1029
  continue
953
- head_error = exc
1030
+ if exc.status_code in (403, 405):
1031
+ try:
1032
+ probed = self._range_probe(entry, output_dir, url, partition)
1033
+ except _ProbeNotFoundError:
1034
+ # HEAD recusado, mas o GET de probe prova que sumiu.
1035
+ not_found += 1
1036
+ continue
1037
+ if probed is not None:
1038
+ return probed
954
1039
  break
955
1040
  except Exception as exc: # noqa: BLE001 - metadata best effort
956
1041
  head_error = exc
@@ -960,6 +1045,21 @@ class FetcherApp:
960
1045
 
961
1046
  if head is None:
962
1047
  local = self.path_builder(output_dir, entry, None)
1048
+ if not_found == len(urls_to_try):
1049
+ return CheckPlanItem(
1050
+ dataset=dataset,
1051
+ id=eid,
1052
+ url=str(entry["url"]),
1053
+ partition=partition,
1054
+ local_path=str(local),
1055
+ remote_etag=None,
1056
+ remote_last_modified=None,
1057
+ remote_size=None,
1058
+ local_exists=local.exists(),
1059
+ action="skip-unavailable",
1060
+ reason="remote-404",
1061
+ entry=dict(entry),
1062
+ )
963
1063
  return CheckPlanItem(
964
1064
  dataset=dataset,
965
1065
  id=eid,
@@ -983,9 +1083,13 @@ class FetcherApp:
983
1083
  size = None
984
1084
  last_modified = _parse_head_date(head.headers.get("Last-Modified"))
985
1085
  local = self.path_builder(output_dir, entry, last_modified)
986
- exists = local.exists()
1086
+ sidecar = manifest_sidecar_path(local)
1087
+ exists = local.exists() or sidecar.exists()
987
1088
  if exists and not is_remote_more_recent(head, local):
988
- action, reason = "skip-up-to-date", "local-fresh"
1089
+ if local.exists():
1090
+ action, reason = "skip-up-to-date", "local-fresh"
1091
+ else:
1092
+ action, reason = "skip-up-to-date", "manifest-fresh"
989
1093
  else:
990
1094
  action = "download"
991
1095
  reason = "not-present" if not exists else "remote-newer"
@@ -1004,6 +1108,106 @@ class FetcherApp:
1004
1108
  entry=dict(entry),
1005
1109
  )
1006
1110
 
1111
+ def entry_from_plan(self, data: dict[str, Any]) -> dict[str, Any]:
1112
+ """Restaura uma entry desserializada de um :class:`CheckPlan`.
1113
+
1114
+ O padrão retorna uma cópia rasa de ``data``, sem nenhuma
1115
+ transformação. Sobrescreva (subclasse ou
1116
+ atribuição de função na instância, que recebe só ``data``) quando
1117
+ ``check_entry`` serializa objetos ricos (ex.: ``datetime``) na entry.
1118
+
1119
+ Args:
1120
+ data: Entry como lida do plano JSON.
1121
+
1122
+ Returns:
1123
+ A entry pronta para ``download_entry``/``path_builder``.
1124
+ """
1125
+ return dict(data)
1126
+
1127
+ def _range_probe(
1128
+ self,
1129
+ entry: dict[str, Any],
1130
+ output_dir: Path,
1131
+ url: str,
1132
+ partition: str | None,
1133
+ ) -> CheckPlanItem | None:
1134
+ """Probe ``GET`` com ``Range: bytes=0-0`` quando o ``HEAD`` é recusado.
1135
+
1136
+ Nunca lê o corpo: se o servidor ignorar o Range e responder 200, a
1137
+ resposta é fechada sem consumir o body.
1138
+
1139
+ Returns:
1140
+ O veredito da entrada, ou ``None`` se o client não suporta
1141
+ streaming ou o probe falhou (mantém o comportamento padrão).
1142
+
1143
+ Raises:
1144
+ _ProbeNotFoundError: Se o GET de probe responder 404.
1145
+ """
1146
+ stream = getattr(self.client, "stream", None)
1147
+ if not callable(stream):
1148
+ return None
1149
+ try:
1150
+ with stream(
1151
+ "GET",
1152
+ url,
1153
+ # identity: Content-Length/Content-Range devem refletir o
1154
+ # tamanho real, não o comprimido.
1155
+ headers={"Range": "bytes=0-0", "Accept-Encoding": "identity"},
1156
+ ) as response:
1157
+ headers = response.headers
1158
+ status = response.status_code
1159
+ final_url = str(response.url)
1160
+ except HttpStatusError as exc:
1161
+ if exc.status_code == 404:
1162
+ raise _ProbeNotFoundError(url) from exc
1163
+ logger.debug("[check] probe Range falhou para %s: %r", url, exc)
1164
+ return None
1165
+ except Exception as exc: # noqa: BLE001 - probe é best effort
1166
+ logger.debug("[check] probe Range falhou para %s: %r", url, exc)
1167
+ return None
1168
+
1169
+ total: int | None = None
1170
+ content_range = headers.get("Content-Range")
1171
+ if status == 206 and content_range and "/" in content_range:
1172
+ raw = content_range.rsplit("/", 1)[1].strip()
1173
+ total = int(raw) if raw.isdigit() else None
1174
+ elif status == 200:
1175
+ encoding = (headers.get("Content-Encoding") or "identity").strip()
1176
+ if encoding.lower() != "identity":
1177
+ return None # Content-Length seria o tamanho comprimido
1178
+ raw = (headers.get("Content-Length") or "").strip()
1179
+ total = int(raw) if raw.isdigit() else None
1180
+ if total is None:
1181
+ return None
1182
+
1183
+ last_modified = _parse_head_date(headers.get("Last-Modified"))
1184
+ local = self.path_builder(output_dir, entry, last_modified)
1185
+ sidecar = manifest_sidecar_path(local)
1186
+ local_exists = local.exists()
1187
+ exists = local_exists or sidecar.exists()
1188
+ action, reason = "download", "range-probe"
1189
+ if local_exists and local.stat().st_size == total:
1190
+ action, reason = "skip-up-to-date", "size-match-range-probe"
1191
+ elif sidecar.exists():
1192
+ with contextlib.suppress(OSError, ValueError, AttributeError):
1193
+ data = json.loads(sidecar.read_text(encoding="utf-8"))
1194
+ if data.get("size_bytes") == total:
1195
+ action, reason = "skip-up-to-date", "size-match-range-probe"
1196
+ return CheckPlanItem(
1197
+ dataset=str(entry.get("group") or entry.get("id") or "datasets"),
1198
+ id=str(entry.get("id", "unknown")),
1199
+ url=final_url,
1200
+ partition=partition,
1201
+ local_path=str(local),
1202
+ remote_etag=headers.get("ETag"),
1203
+ remote_last_modified=headers.get("Last-Modified"),
1204
+ remote_size=total,
1205
+ local_exists=exists,
1206
+ action=action,
1207
+ reason=reason,
1208
+ entry=dict(entry),
1209
+ )
1210
+
1007
1211
  def check_datasets(
1008
1212
  self,
1009
1213
  entries: list[dict[str, Any]],
@@ -1076,6 +1280,14 @@ class FetcherApp:
1076
1280
  "de grupos).",
1077
1281
  ),
1078
1282
  ] = None,
1283
+ force: Annotated[
1284
+ bool,
1285
+ typer.Option(
1286
+ "--force",
1287
+ help="Ignora a checagem de frescor local (arquivo/sidecar) "
1288
+ "e baixa todas as entradas selecionadas (útil com --from-plan)",
1289
+ ),
1290
+ ] = False,
1079
1291
  verbose: Annotated[
1080
1292
  bool, typer.Option("--verbose", help="Logs detalhados")
1081
1293
  ] = False,
@@ -1090,7 +1302,9 @@ class FetcherApp:
1090
1302
  console.print(f"[red]Plano inválido: {exc}[/red]")
1091
1303
  raise typer.Exit(1) from None
1092
1304
  entries = [
1093
- dict(item.entry) for item in plan.items if item.action == "download"
1305
+ self.entry_from_plan(dict(item.entry))
1306
+ for item in plan.items
1307
+ if item.action == "download"
1094
1308
  ]
1095
1309
  console.print(
1096
1310
  f"[dim]Plano {from_plan}: {len(entries)} entrada(s) "
@@ -1114,9 +1328,14 @@ class FetcherApp:
1114
1328
  console.print(f"\n[bold]{len(entries)}[/bold] arquivo(s) listado(s).")
1115
1329
  return
1116
1330
 
1117
- downloaded, total, errors = self.download_datasets(
1118
- entries, actual_output, workers=workers
1119
- )
1331
+ if force:
1332
+ downloaded, total, errors = self.download_datasets(
1333
+ entries, actual_output, workers=workers, force=True
1334
+ )
1335
+ else:
1336
+ downloaded, total, errors = self.download_datasets(
1337
+ entries, actual_output, workers=workers
1338
+ )
1120
1339
 
1121
1340
  console.print(
1122
1341
  f"\n[green]Concluído:[/green] {downloaded}/{total} arquivo(s) baixado(s)."
@@ -1147,6 +1366,14 @@ class FetcherApp:
1147
1366
  help="Imprime o plano em JSON (para 'sync --from-plan')",
1148
1367
  ),
1149
1368
  ] = False,
1369
+ out: Annotated[
1370
+ Path | None,
1371
+ typer.Option(
1372
+ "--out",
1373
+ help="Grava o plano JSON neste arquivo (atômico) e imprime "
1374
+ "só um resumo; prevalece sobre --json",
1375
+ ),
1376
+ ] = None,
1150
1377
  verbose: Annotated[
1151
1378
  bool, typer.Option("--verbose", help="Logs detalhados")
1152
1379
  ] = False,
@@ -1164,7 +1391,20 @@ class FetcherApp:
1164
1391
  entries = [e for g in target_groups for e in self.list_datasets(g)]
1165
1392
  plan = self.check_datasets(entries, actual_output, workers=workers)
1166
1393
 
1167
- if as_json:
1394
+ if out is not None:
1395
+ tmp = out.with_name(f".{out.name}.{os.getpid()}.tmp")
1396
+ try:
1397
+ out.parent.mkdir(parents=True, exist_ok=True)
1398
+ try:
1399
+ tmp.write_text(plan.to_json(), encoding="utf-8")
1400
+ os.replace(tmp, out)
1401
+ finally:
1402
+ tmp.unlink(missing_ok=True)
1403
+ except OSError as exc:
1404
+ console.print(f"[red]Falha ao gravar o plano em {out}: {exc}[/red]")
1405
+ raise typer.Exit(1) from None
1406
+ console.print(f"{plan.summary_line()} Plano gravado em {out}")
1407
+ elif as_json:
1168
1408
  print(plan.to_json())
1169
1409
  else:
1170
1410
  plan.render_table()
@@ -1186,17 +1426,9 @@ class FetcherApp:
1186
1426
  title=f"[bold]{group_id}[/bold] — {group_info.get('name', '')}",
1187
1427
  )
1188
1428
  for entry in self.list_datasets(group_id):
1189
- if entry.get("semester") is not None:
1190
- partition = f"{entry['year']}-S{entry['semester']}"
1191
- elif entry.get("month") is not None:
1192
- partition = f"{entry['year']}-{entry['month']:02d}"
1193
- elif entry.get("year") is not None:
1194
- partition = str(entry["year"])
1195
- else:
1196
- partition = "—"
1197
1429
  table.add_row(
1198
1430
  entry.get("id", ""),
1199
- partition,
1431
+ _partition_label(entry) or "—",
1200
1432
  entry.get("ext", ""),
1201
1433
  entry.get("url", ""),
1202
1434
  )