quantilica-cli 0.10.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/CHANGELOG.md +33 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/PKG-INFO +2 -2
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/pyproject.toml +2 -2
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/sdk.py +265 -33
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/sources.py +54 -9
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_package_hygiene.py +21 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_sdk.py +62 -0
- quantilica_cli-0.11.0/tests/test_sdk_force_probe.py +487 -0
- quantilica_cli-0.11.0/tests/test_sdk_mes_string.py +51 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_sdk_onda_a2.py +12 -4
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_sources.py +91 -0
- quantilica_cli-0.10.0/src/quantilica/cli/progress.py +0 -77
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/.githooks/pre-commit +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/.githooks/pre-push +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/.github/workflows/publish.yml +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/.github/workflows/test.yml +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/.gitignore +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/LICENSE +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/README.md +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/__init__.py +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/cli.py +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/health.py +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/manifests.py +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/cli/ui.py +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/src/quantilica/py.typed +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/__init__.py +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_health.py +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_manifests.py +0 -0
- {quantilica_cli-0.10.0 → quantilica_cli-0.11.0}/tests/test_ui.py +0 -0
|
@@ -5,6 +5,39 @@ Todas as mudanças notáveis deste projeto serão documentadas neste arquivo.
|
|
|
5
5
|
O formato segue [Keep a Changelog](https://keepachangelog.com/pt-BR/1.1.0/),
|
|
6
6
|
e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
|
|
7
7
|
|
|
8
|
+
## [0.11.0] - 2026-10-14
|
|
9
|
+
|
|
10
|
+
### Adicionado
|
|
11
|
+
|
|
12
|
+
- `sync --force` (e `force` keyword-only em `download_datasets`, `download_entry` e `_download_file`, repassado ao cliente só quando `True`): baixa mesmo com o local atualizado.
|
|
13
|
+
- `check --out PATH`: grava o plano JSON de forma atômica e imprime só um resumo; `OSError` sai com código 1.
|
|
14
|
+
- `check`: `HEAD` com 403 ou 405 passa a um probe `GET` com `Range: bytes=0-0` e `Accept-Encoding: identity` (tamanho via `Content-Range`, corpo nunca lido); URL que dá 404 no `HEAD` e no probe vira a ação `skip-unavailable`, aceita por `CheckPlan.from_dict`.
|
|
15
|
+
- `FetcherApp.entry_from_plan`: gancho para fetchers que serializam objetos ricos na entrada do plano.
|
|
16
|
+
- `default_client` lê `QUANTILICA_HTTP_MAX_RESUMES` (opcional), repassado como `max_resumes` ao `HttpClient`.
|
|
17
|
+
- `quantilica install <fonte> --extra <nome>` (`-e`, repetível, aceita lista separada por vírgula): instala `<dist>[<extra>]` no ambiente da CLI, por exemplo `quantilica install pdet --extra analysis` para habilitar `convert` e `pipeline`. Funciona também em fonte já instalada. O estado continua registrando só a distribuição, e o re-exec pós-instalação não repete `--extra`, para não reinstalar em laço. Nomes de extra inválidos saem com código 2.
|
|
18
|
+
|
|
19
|
+
### Alterado
|
|
20
|
+
|
|
21
|
+
- Piso de `quantilica-core` elevado para `>=0.11.0` (símbolo usado: `max_resumes` em `HttpClient`).
|
|
22
|
+
|
|
23
|
+
### Removido
|
|
24
|
+
|
|
25
|
+
- `quantilica.cli.progress` (`file_progress`, `batch_progress`): módulo morto que nunca importou. Fazia `from .http import ProgressCallback` (não existe em `quantilica.cli`; o tipo vive em `quantilica.core.http`) e usava `tqdm`, que não é dependência do pacote. Nenhum fetcher o importava; o progresso real é Rich (`quantilica.cli.ui.make_download_progress`, `make_batch_progress`, `ProgressPool`).
|
|
26
|
+
|
|
27
|
+
### Corrigido
|
|
28
|
+
|
|
29
|
+
- `install_package` imprimia `Instalando pdet-fetcher...` sem o `[analysis]`: o Rich tratava o colchete como marcação. Agora o nome é escapado.
|
|
30
|
+
- `sdk`: a dica de extra ausente em `convert`, `archive` e `pipeline` mandava `pip install <fetcher>[analysis]`, que não funciona para fetchers do Fluxo B (fora do PyPI). Agora imprime `quantilica install <chave> --extra analysis`, com a chave de registro resolvida pelo entry point (por exemplo `td` para `tesouro-direto-fetcher`).
|
|
31
|
+
- `sdk`: `quantilica <fonte> list` (e o rótulo de partição do `check`/`sync`) terminava com `ValueError: Unknown format code 'd' for object of type 'str'` quando o catálogo traz `month` como string, caso do PDET (grupo capturado de regex). O mês e o semestre agora passam por `int()` antes do `:02d`, em `_partition_label`, `default_path_builder` e `list`; o `list` reutiliza `_partition_label` em vez de duplicar a lógica.
|
|
32
|
+
|
|
33
|
+
## [0.10.1] - 2026-10-08
|
|
34
|
+
|
|
35
|
+
### Corrigido
|
|
36
|
+
|
|
37
|
+
- `sdk`: `check_entry` considera `local.exists() or sidecar.exists()`, preservando
|
|
38
|
+
o status de `manifest-fresh` em ambientes de ingestão efêmera sem forçar
|
|
39
|
+
re-download redundante quando o dado bruto já foi espelhado.
|
|
40
|
+
|
|
8
41
|
## [0.10.0] - 2026-10-07
|
|
9
42
|
|
|
10
43
|
### Adicionado
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: quantilica-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Unified CLI for Quantilica open data fetchers
|
|
5
5
|
Author-email: "Komesu, D.K." <daniel@dkko.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.13
|
|
17
17
|
Classifier: Typing :: Typed
|
|
18
18
|
Requires-Python: >=3.12
|
|
19
|
-
Requires-Dist: quantilica-core>=0.
|
|
19
|
+
Requires-Dist: quantilica-core>=0.11.0
|
|
20
20
|
Requires-Dist: rich>=13.0.0
|
|
21
21
|
Requires-Dist: typer>=0.15.0
|
|
22
22
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "quantilica-cli"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.11.0"
|
|
4
4
|
description = "Unified CLI for Quantilica open data fetchers"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [{ name = "Komesu, D.K.", email = "daniel@dkko.me" }]
|
|
@@ -20,7 +20,7 @@ classifiers = [
|
|
|
20
20
|
"Programming Language :: Python :: 3.13",
|
|
21
21
|
]
|
|
22
22
|
dependencies = [
|
|
23
|
-
"quantilica-core>=0.
|
|
23
|
+
"quantilica-core>=0.11.0",
|
|
24
24
|
"rich>=13.0.0",
|
|
25
25
|
"typer>=0.15.0",
|
|
26
26
|
]
|
|
@@ -14,6 +14,7 @@ import os
|
|
|
14
14
|
import threading
|
|
15
15
|
from collections.abc import Callable, Iterable
|
|
16
16
|
from dataclasses import dataclass, field
|
|
17
|
+
from importlib.metadata import entry_points
|
|
17
18
|
from pathlib import Path
|
|
18
19
|
from typing import Annotated, Any
|
|
19
20
|
|
|
@@ -28,6 +29,7 @@ from quantilica.core.http import (
|
|
|
28
29
|
resolve_verify_from_env,
|
|
29
30
|
)
|
|
30
31
|
from quantilica.core.logging import get_logger
|
|
32
|
+
from quantilica.core.manifests import manifest_sidecar_path
|
|
31
33
|
from quantilica.core.storage import (
|
|
32
34
|
StampedDataRepository,
|
|
33
35
|
build_stamped_filename,
|
|
@@ -124,12 +126,22 @@ class SyncPlan:
|
|
|
124
126
|
)
|
|
125
127
|
|
|
126
128
|
|
|
129
|
+
def _two_digits(value: Any) -> str:
|
|
130
|
+
"""Format a month/semester as two digits, accepting ``int`` or numeric ``str``.
|
|
131
|
+
|
|
132
|
+
Catalogs built from filename regexes (e.g. PDET) carry the month as the
|
|
133
|
+
captured string (``"03"``), while others use ``int``; ``f"{x:02d}"`` only
|
|
134
|
+
works for the latter.
|
|
135
|
+
"""
|
|
136
|
+
return f"{int(value):02d}"
|
|
137
|
+
|
|
138
|
+
|
|
127
139
|
def _partition_label(entry: dict[str, Any]) -> str | None:
|
|
128
140
|
"""Human-readable partition label for a dataset entry."""
|
|
129
141
|
if entry.get("semester") is not None:
|
|
130
142
|
return f"{entry['year']}-S{entry['semester']}"
|
|
131
143
|
if entry.get("month") is not None:
|
|
132
|
-
return f"{entry['year']}-{entry['month']
|
|
144
|
+
return f"{entry['year']}-{_two_digits(entry['month'])}"
|
|
133
145
|
if entry.get("year") is not None:
|
|
134
146
|
return str(entry["year"])
|
|
135
147
|
return None
|
|
@@ -153,6 +165,10 @@ def _short_remote(item: CheckPlanItem) -> str:
|
|
|
153
165
|
return remote
|
|
154
166
|
|
|
155
167
|
|
|
168
|
+
class _ProbeNotFoundError(Exception):
|
|
169
|
+
"""O GET de probe Range respondeu 404 (URL removida no servidor)."""
|
|
170
|
+
|
|
171
|
+
|
|
156
172
|
@dataclass(frozen=True)
|
|
157
173
|
class CheckPlanItem:
|
|
158
174
|
"""Freshness verdict for a single dataset entry (no download performed).
|
|
@@ -166,8 +182,11 @@ class CheckPlanItem:
|
|
|
166
182
|
remote_etag: Remote ETag header, or None.
|
|
167
183
|
remote_last_modified: Remote Last-Modified header, or None.
|
|
168
184
|
remote_size: Remote Content-Length, or None.
|
|
169
|
-
local_exists: Whether the
|
|
170
|
-
|
|
185
|
+
local_exists: Whether the artifact already exists locally — this
|
|
186
|
+
covers either the raw file itself or its ephemeral ``.manifest.json``
|
|
187
|
+
sidecar marker (file deleted for retention, sidecar still fresh).
|
|
188
|
+
action: ``"download"``, ``"skip-up-to-date"`` or ``"skip-unavailable"``
|
|
189
|
+
(todas as URLs da entrada responderam 404 no HEAD).
|
|
171
190
|
reason: Machine-readable reason for the action.
|
|
172
191
|
entry: Original dataset entry (allows ``sync --from-plan`` round-trip).
|
|
173
192
|
"""
|
|
@@ -222,11 +241,7 @@ class CheckPlan:
|
|
|
222
241
|
_short_remote(item),
|
|
223
242
|
)
|
|
224
243
|
con.print(table)
|
|
225
|
-
|
|
226
|
-
for item in self.items:
|
|
227
|
-
counts[item.action] = counts.get(item.action, 0) + 1
|
|
228
|
-
summary = ", ".join(f"{v} {k}" for k, v in sorted(counts.items()))
|
|
229
|
-
con.print(f"[bold]{len(self.items)}[/bold] verificado(s): {summary}.")
|
|
244
|
+
con.print(self.summary_line())
|
|
230
245
|
|
|
231
246
|
def to_dict(self) -> dict[str, Any]:
|
|
232
247
|
"""Return the JSON-serializable representation of the plan."""
|
|
@@ -253,6 +268,14 @@ class CheckPlan:
|
|
|
253
268
|
],
|
|
254
269
|
}
|
|
255
270
|
|
|
271
|
+
def summary_line(self) -> str:
|
|
272
|
+
"""Return a one-line summary with the total and per-action counts."""
|
|
273
|
+
counts: dict[str, int] = {}
|
|
274
|
+
for item in self.items:
|
|
275
|
+
counts[item.action] = counts.get(item.action, 0) + 1
|
|
276
|
+
summary = ", ".join(f"{v} {k}" for k, v in sorted(counts.items()))
|
|
277
|
+
return f"[bold]{len(self.items)}[/bold] verificado(s): {summary}."
|
|
278
|
+
|
|
256
279
|
def to_json(self) -> str:
|
|
257
280
|
"""Serialize the plan to a JSON string (for ``sync --from-plan``)."""
|
|
258
281
|
return json.dumps(self.to_dict(), ensure_ascii=False, indent=2)
|
|
@@ -279,6 +302,7 @@ class CheckPlan:
|
|
|
279
302
|
if not isinstance(entry, dict) or action not in (
|
|
280
303
|
"download",
|
|
281
304
|
"skip-up-to-date",
|
|
305
|
+
"skip-unavailable",
|
|
282
306
|
):
|
|
283
307
|
raise ValueError("not a check plan: invalid item")
|
|
284
308
|
items.append(
|
|
@@ -329,6 +353,8 @@ def default_client() -> HttpClient:
|
|
|
329
353
|
a path uses it as CA bundle.
|
|
330
354
|
- ``QUANTILICA_HTTP_ATTEMPTS`` / ``QUANTILICA_HTTP_TIMEOUT`` /
|
|
331
355
|
``QUANTILICA_HTTP_RETRY_DELAY``: retry budget overrides.
|
|
356
|
+
- ``QUANTILICA_HTTP_MAX_RESUMES``: teto de retomadas com progresso de
|
|
357
|
+
downloads interrompidos (default do ``HttpClient`` quando ausente).
|
|
332
358
|
|
|
333
359
|
Returns:
|
|
334
360
|
A pre-configured HttpClient instance (browser-like WAF headers + pooling).
|
|
@@ -342,12 +368,18 @@ def default_client() -> HttpClient:
|
|
|
342
368
|
attempts = int(os.environ.get("QUANTILICA_HTTP_ATTEMPTS", "5"))
|
|
343
369
|
timeout = float(os.environ.get("QUANTILICA_HTTP_TIMEOUT", "180.0"))
|
|
344
370
|
retry_base_delay = float(os.environ.get("QUANTILICA_HTTP_RETRY_DELAY", "2.0"))
|
|
371
|
+
# Só repassado quando definido: preserva o default do core.
|
|
372
|
+
extra: dict[str, Any] = {}
|
|
373
|
+
max_resumes = os.environ.get("QUANTILICA_HTTP_MAX_RESUMES", "").strip()
|
|
374
|
+
if max_resumes:
|
|
375
|
+
extra["max_resumes"] = int(max_resumes)
|
|
345
376
|
return HttpClient(
|
|
346
377
|
timeout=timeout,
|
|
347
378
|
verify=verify,
|
|
348
379
|
attempts=attempts,
|
|
349
380
|
retry_base_delay=retry_base_delay,
|
|
350
381
|
emulate_browser=True,
|
|
382
|
+
**extra,
|
|
351
383
|
)
|
|
352
384
|
|
|
353
385
|
|
|
@@ -393,14 +425,14 @@ class DataRepository(StampedDataRepository):
|
|
|
393
425
|
if year is not None and semester is not None:
|
|
394
426
|
filename = build_stamped_filename(
|
|
395
427
|
slug,
|
|
396
|
-
f"{year}-{semester
|
|
428
|
+
f"{year}-{_two_digits(semester)}",
|
|
397
429
|
ext=ext,
|
|
398
430
|
timestamp=last_modified,
|
|
399
431
|
)
|
|
400
432
|
elif year is not None and month is not None:
|
|
401
433
|
filename = build_stamped_filename(
|
|
402
434
|
slug,
|
|
403
|
-
f"{year}-{month
|
|
435
|
+
f"{year}-{_two_digits(month)}",
|
|
404
436
|
ext=ext,
|
|
405
437
|
timestamp=last_modified,
|
|
406
438
|
)
|
|
@@ -540,6 +572,28 @@ class FetcherApp:
|
|
|
540
572
|
"""
|
|
541
573
|
self.app.command(name=name, **kwargs)(cmd_func)
|
|
542
574
|
|
|
575
|
+
def _analysis_hint(self) -> str:
|
|
576
|
+
"""Comando que instala o extra ``analysis`` no ambiente desta CLI.
|
|
577
|
+
|
|
578
|
+
Usa a chave de registro do fetcher (a que ``quantilica install`` e o
|
|
579
|
+
entry point conhecem, ex.: ``td``) e, se não a encontrar, o nome da
|
|
580
|
+
distribuição, que ``quantilica install`` também aceita.
|
|
581
|
+
|
|
582
|
+
Returns:
|
|
583
|
+
Texto sem colchetes, seguro para ``console.print``.
|
|
584
|
+
"""
|
|
585
|
+
key = self.name
|
|
586
|
+
try:
|
|
587
|
+
for ep in entry_points(group="quantilica.fetchers"):
|
|
588
|
+
if ep.dist is not None and ep.dist.name == self.name:
|
|
589
|
+
key = ep.name
|
|
590
|
+
break
|
|
591
|
+
except (
|
|
592
|
+
Exception
|
|
593
|
+
): # pragma: no cover - metadados quebrados não podem mascarar o erro
|
|
594
|
+
pass
|
|
595
|
+
return f"quantilica install {key} --extra analysis"
|
|
596
|
+
|
|
543
597
|
def command_convert(
|
|
544
598
|
self, func: Callable[[Path, Path], Any]
|
|
545
599
|
) -> Callable[[Path, Path], Any]:
|
|
@@ -583,7 +637,7 @@ class FetcherApp:
|
|
|
583
637
|
except ImportError:
|
|
584
638
|
console.print(
|
|
585
639
|
f"[red]Erro:[/red] convert requer extras de análise: "
|
|
586
|
-
|
|
640
|
+
f"{self._analysis_hint()}"
|
|
587
641
|
)
|
|
588
642
|
raise typer.Exit(1) from None
|
|
589
643
|
console.print(
|
|
@@ -636,7 +690,7 @@ class FetcherApp:
|
|
|
636
690
|
except ImportError:
|
|
637
691
|
console.print(
|
|
638
692
|
f"[red]Erro:[/red] archive requer extras de análise: "
|
|
639
|
-
|
|
693
|
+
f"{self._analysis_hint()}"
|
|
640
694
|
)
|
|
641
695
|
raise typer.Exit(1) from None
|
|
642
696
|
console.print(f"[green]✓[/green] Arquivo criado em [dim]{output}[/dim].")
|
|
@@ -735,7 +789,7 @@ class FetcherApp:
|
|
|
735
789
|
except ImportError:
|
|
736
790
|
console.print(
|
|
737
791
|
f"[red]Erro:[/red] pipeline (conversão) requer extras de "
|
|
738
|
-
|
|
792
|
+
f"análise: {self._analysis_hint()}"
|
|
739
793
|
)
|
|
740
794
|
raise typer.Exit(1) from None
|
|
741
795
|
console.print(
|
|
@@ -751,9 +805,17 @@ class FetcherApp:
|
|
|
751
805
|
return None
|
|
752
806
|
|
|
753
807
|
def _download_file(
|
|
754
|
-
self,
|
|
808
|
+
self,
|
|
809
|
+
url: str,
|
|
810
|
+
output: Path,
|
|
811
|
+
progress: ProgressCallback | None = None,
|
|
812
|
+
*,
|
|
813
|
+
force: bool = False,
|
|
755
814
|
) -> Path:
|
|
756
815
|
dataset_id = output.parent.name
|
|
816
|
+
# `force` só é repassado quando True: clients/subclasses antigos não
|
|
817
|
+
# precisam aceitar o kwarg.
|
|
818
|
+
extra: dict[str, Any] = {"force": True} if force else {}
|
|
757
819
|
return self.client.download_with_manifest(
|
|
758
820
|
url,
|
|
759
821
|
output,
|
|
@@ -761,6 +823,7 @@ class FetcherApp:
|
|
|
761
823
|
dataset_id=dataset_id,
|
|
762
824
|
producer=self.name,
|
|
763
825
|
progress=progress,
|
|
826
|
+
**extra,
|
|
764
827
|
)
|
|
765
828
|
|
|
766
829
|
def download_entry(
|
|
@@ -770,6 +833,7 @@ class FetcherApp:
|
|
|
770
833
|
*,
|
|
771
834
|
dry_run: bool = False,
|
|
772
835
|
progress: ProgressCallback | None = None,
|
|
836
|
+
force: bool = False,
|
|
773
837
|
) -> Path:
|
|
774
838
|
"""Download one dataset entry and return the destination path.
|
|
775
839
|
|
|
@@ -778,6 +842,7 @@ class FetcherApp:
|
|
|
778
842
|
output_dir: Destination directory.
|
|
779
843
|
dry_run: If True, computes the destination path without downloading.
|
|
780
844
|
progress: Optional callback to track download progress.
|
|
845
|
+
force: Se True, ignora a checagem de frescor local e baixa sempre.
|
|
781
846
|
|
|
782
847
|
Returns:
|
|
783
848
|
The local path where the file was (or would be) saved.
|
|
@@ -808,6 +873,10 @@ class FetcherApp:
|
|
|
808
873
|
|
|
809
874
|
if dry_run:
|
|
810
875
|
return output
|
|
876
|
+
if force:
|
|
877
|
+
return self._download_file(
|
|
878
|
+
url, output, progress=progress, force=True
|
|
879
|
+
)
|
|
811
880
|
return self._download_file(url, output, progress=progress)
|
|
812
881
|
except HttpStatusError as exc:
|
|
813
882
|
if exc.status_code == 404:
|
|
@@ -831,6 +900,8 @@ class FetcherApp:
|
|
|
831
900
|
entries: list[dict[str, Any]],
|
|
832
901
|
output_dir: Path,
|
|
833
902
|
workers: int = 4,
|
|
903
|
+
*,
|
|
904
|
+
force: bool = False,
|
|
834
905
|
) -> tuple[int, int, list[tuple[str, str]]]:
|
|
835
906
|
"""Executes parallel download for a list of dataset entries.
|
|
836
907
|
|
|
@@ -838,6 +909,7 @@ class FetcherApp:
|
|
|
838
909
|
entries: List of dataset entries to download.
|
|
839
910
|
output_dir: The target base directory for downloads.
|
|
840
911
|
workers: Maximum number of parallel downloads.
|
|
912
|
+
force: Se True, ignora a checagem de frescor local e baixa tudo.
|
|
841
913
|
|
|
842
914
|
Returns:
|
|
843
915
|
A tuple containing:
|
|
@@ -888,7 +960,10 @@ class FetcherApp:
|
|
|
888
960
|
try:
|
|
889
961
|
eid = entry.get("id", "unknown")
|
|
890
962
|
with pool.acquire(description=f"[cyan]{eid}[/cyan]") as cb:
|
|
891
|
-
|
|
963
|
+
if force:
|
|
964
|
+
self.download_entry(entry, output_dir, progress=cb, force=True)
|
|
965
|
+
else:
|
|
966
|
+
self.download_entry(entry, output_dir, progress=cb)
|
|
892
967
|
return True
|
|
893
968
|
except Exception as exc:
|
|
894
969
|
errors.append((entry.get("id", "unknown"), str(exc)))
|
|
@@ -943,14 +1018,24 @@ class FetcherApp:
|
|
|
943
1018
|
|
|
944
1019
|
head = None
|
|
945
1020
|
head_error: Exception | None = None
|
|
1021
|
+
not_found = 0
|
|
946
1022
|
for url in urls_to_try:
|
|
947
1023
|
try:
|
|
948
1024
|
candidate = self.client.head(url)
|
|
949
1025
|
except HttpStatusError as exc:
|
|
1026
|
+
head_error = exc
|
|
950
1027
|
if exc.status_code == 404:
|
|
951
|
-
|
|
1028
|
+
not_found += 1
|
|
952
1029
|
continue
|
|
953
|
-
|
|
1030
|
+
if exc.status_code in (403, 405):
|
|
1031
|
+
try:
|
|
1032
|
+
probed = self._range_probe(entry, output_dir, url, partition)
|
|
1033
|
+
except _ProbeNotFoundError:
|
|
1034
|
+
# HEAD recusado, mas o GET de probe prova que sumiu.
|
|
1035
|
+
not_found += 1
|
|
1036
|
+
continue
|
|
1037
|
+
if probed is not None:
|
|
1038
|
+
return probed
|
|
954
1039
|
break
|
|
955
1040
|
except Exception as exc: # noqa: BLE001 - metadata best effort
|
|
956
1041
|
head_error = exc
|
|
@@ -960,6 +1045,21 @@ class FetcherApp:
|
|
|
960
1045
|
|
|
961
1046
|
if head is None:
|
|
962
1047
|
local = self.path_builder(output_dir, entry, None)
|
|
1048
|
+
if not_found == len(urls_to_try):
|
|
1049
|
+
return CheckPlanItem(
|
|
1050
|
+
dataset=dataset,
|
|
1051
|
+
id=eid,
|
|
1052
|
+
url=str(entry["url"]),
|
|
1053
|
+
partition=partition,
|
|
1054
|
+
local_path=str(local),
|
|
1055
|
+
remote_etag=None,
|
|
1056
|
+
remote_last_modified=None,
|
|
1057
|
+
remote_size=None,
|
|
1058
|
+
local_exists=local.exists(),
|
|
1059
|
+
action="skip-unavailable",
|
|
1060
|
+
reason="remote-404",
|
|
1061
|
+
entry=dict(entry),
|
|
1062
|
+
)
|
|
963
1063
|
return CheckPlanItem(
|
|
964
1064
|
dataset=dataset,
|
|
965
1065
|
id=eid,
|
|
@@ -983,9 +1083,13 @@ class FetcherApp:
|
|
|
983
1083
|
size = None
|
|
984
1084
|
last_modified = _parse_head_date(head.headers.get("Last-Modified"))
|
|
985
1085
|
local = self.path_builder(output_dir, entry, last_modified)
|
|
986
|
-
|
|
1086
|
+
sidecar = manifest_sidecar_path(local)
|
|
1087
|
+
exists = local.exists() or sidecar.exists()
|
|
987
1088
|
if exists and not is_remote_more_recent(head, local):
|
|
988
|
-
|
|
1089
|
+
if local.exists():
|
|
1090
|
+
action, reason = "skip-up-to-date", "local-fresh"
|
|
1091
|
+
else:
|
|
1092
|
+
action, reason = "skip-up-to-date", "manifest-fresh"
|
|
989
1093
|
else:
|
|
990
1094
|
action = "download"
|
|
991
1095
|
reason = "not-present" if not exists else "remote-newer"
|
|
@@ -1004,6 +1108,106 @@ class FetcherApp:
|
|
|
1004
1108
|
entry=dict(entry),
|
|
1005
1109
|
)
|
|
1006
1110
|
|
|
1111
|
+
def entry_from_plan(self, data: dict[str, Any]) -> dict[str, Any]:
|
|
1112
|
+
"""Restaura uma entry desserializada de um :class:`CheckPlan`.
|
|
1113
|
+
|
|
1114
|
+
O padrão retorna uma cópia rasa de ``data``, sem nenhuma
|
|
1115
|
+
transformação. Sobrescreva (subclasse ou
|
|
1116
|
+
atribuição de função na instância, que recebe só ``data``) quando
|
|
1117
|
+
``check_entry`` serializa objetos ricos (ex.: ``datetime``) na entry.
|
|
1118
|
+
|
|
1119
|
+
Args:
|
|
1120
|
+
data: Entry como lida do plano JSON.
|
|
1121
|
+
|
|
1122
|
+
Returns:
|
|
1123
|
+
A entry pronta para ``download_entry``/``path_builder``.
|
|
1124
|
+
"""
|
|
1125
|
+
return dict(data)
|
|
1126
|
+
|
|
1127
|
+
def _range_probe(
|
|
1128
|
+
self,
|
|
1129
|
+
entry: dict[str, Any],
|
|
1130
|
+
output_dir: Path,
|
|
1131
|
+
url: str,
|
|
1132
|
+
partition: str | None,
|
|
1133
|
+
) -> CheckPlanItem | None:
|
|
1134
|
+
"""Probe ``GET`` com ``Range: bytes=0-0`` quando o ``HEAD`` é recusado.
|
|
1135
|
+
|
|
1136
|
+
Nunca lê o corpo: se o servidor ignorar o Range e responder 200, a
|
|
1137
|
+
resposta é fechada sem consumir o body.
|
|
1138
|
+
|
|
1139
|
+
Returns:
|
|
1140
|
+
O veredito da entrada, ou ``None`` se o client não suporta
|
|
1141
|
+
streaming ou o probe falhou (mantém o comportamento padrão).
|
|
1142
|
+
|
|
1143
|
+
Raises:
|
|
1144
|
+
_ProbeNotFoundError: Se o GET de probe responder 404.
|
|
1145
|
+
"""
|
|
1146
|
+
stream = getattr(self.client, "stream", None)
|
|
1147
|
+
if not callable(stream):
|
|
1148
|
+
return None
|
|
1149
|
+
try:
|
|
1150
|
+
with stream(
|
|
1151
|
+
"GET",
|
|
1152
|
+
url,
|
|
1153
|
+
# identity: Content-Length/Content-Range devem refletir o
|
|
1154
|
+
# tamanho real, não o comprimido.
|
|
1155
|
+
headers={"Range": "bytes=0-0", "Accept-Encoding": "identity"},
|
|
1156
|
+
) as response:
|
|
1157
|
+
headers = response.headers
|
|
1158
|
+
status = response.status_code
|
|
1159
|
+
final_url = str(response.url)
|
|
1160
|
+
except HttpStatusError as exc:
|
|
1161
|
+
if exc.status_code == 404:
|
|
1162
|
+
raise _ProbeNotFoundError(url) from exc
|
|
1163
|
+
logger.debug("[check] probe Range falhou para %s: %r", url, exc)
|
|
1164
|
+
return None
|
|
1165
|
+
except Exception as exc: # noqa: BLE001 - probe é best effort
|
|
1166
|
+
logger.debug("[check] probe Range falhou para %s: %r", url, exc)
|
|
1167
|
+
return None
|
|
1168
|
+
|
|
1169
|
+
total: int | None = None
|
|
1170
|
+
content_range = headers.get("Content-Range")
|
|
1171
|
+
if status == 206 and content_range and "/" in content_range:
|
|
1172
|
+
raw = content_range.rsplit("/", 1)[1].strip()
|
|
1173
|
+
total = int(raw) if raw.isdigit() else None
|
|
1174
|
+
elif status == 200:
|
|
1175
|
+
encoding = (headers.get("Content-Encoding") or "identity").strip()
|
|
1176
|
+
if encoding.lower() != "identity":
|
|
1177
|
+
return None # Content-Length seria o tamanho comprimido
|
|
1178
|
+
raw = (headers.get("Content-Length") or "").strip()
|
|
1179
|
+
total = int(raw) if raw.isdigit() else None
|
|
1180
|
+
if total is None:
|
|
1181
|
+
return None
|
|
1182
|
+
|
|
1183
|
+
last_modified = _parse_head_date(headers.get("Last-Modified"))
|
|
1184
|
+
local = self.path_builder(output_dir, entry, last_modified)
|
|
1185
|
+
sidecar = manifest_sidecar_path(local)
|
|
1186
|
+
local_exists = local.exists()
|
|
1187
|
+
exists = local_exists or sidecar.exists()
|
|
1188
|
+
action, reason = "download", "range-probe"
|
|
1189
|
+
if local_exists and local.stat().st_size == total:
|
|
1190
|
+
action, reason = "skip-up-to-date", "size-match-range-probe"
|
|
1191
|
+
elif sidecar.exists():
|
|
1192
|
+
with contextlib.suppress(OSError, ValueError, AttributeError):
|
|
1193
|
+
data = json.loads(sidecar.read_text(encoding="utf-8"))
|
|
1194
|
+
if data.get("size_bytes") == total:
|
|
1195
|
+
action, reason = "skip-up-to-date", "size-match-range-probe"
|
|
1196
|
+
return CheckPlanItem(
|
|
1197
|
+
dataset=str(entry.get("group") or entry.get("id") or "datasets"),
|
|
1198
|
+
id=str(entry.get("id", "unknown")),
|
|
1199
|
+
url=final_url,
|
|
1200
|
+
partition=partition,
|
|
1201
|
+
local_path=str(local),
|
|
1202
|
+
remote_etag=headers.get("ETag"),
|
|
1203
|
+
remote_last_modified=headers.get("Last-Modified"),
|
|
1204
|
+
remote_size=total,
|
|
1205
|
+
local_exists=exists,
|
|
1206
|
+
action=action,
|
|
1207
|
+
reason=reason,
|
|
1208
|
+
entry=dict(entry),
|
|
1209
|
+
)
|
|
1210
|
+
|
|
1007
1211
|
def check_datasets(
|
|
1008
1212
|
self,
|
|
1009
1213
|
entries: list[dict[str, Any]],
|
|
@@ -1076,6 +1280,14 @@ class FetcherApp:
|
|
|
1076
1280
|
"de grupos).",
|
|
1077
1281
|
),
|
|
1078
1282
|
] = None,
|
|
1283
|
+
force: Annotated[
|
|
1284
|
+
bool,
|
|
1285
|
+
typer.Option(
|
|
1286
|
+
"--force",
|
|
1287
|
+
help="Ignora a checagem de frescor local (arquivo/sidecar) "
|
|
1288
|
+
"e baixa todas as entradas selecionadas (útil com --from-plan)",
|
|
1289
|
+
),
|
|
1290
|
+
] = False,
|
|
1079
1291
|
verbose: Annotated[
|
|
1080
1292
|
bool, typer.Option("--verbose", help="Logs detalhados")
|
|
1081
1293
|
] = False,
|
|
@@ -1090,7 +1302,9 @@ class FetcherApp:
|
|
|
1090
1302
|
console.print(f"[red]Plano inválido: {exc}[/red]")
|
|
1091
1303
|
raise typer.Exit(1) from None
|
|
1092
1304
|
entries = [
|
|
1093
|
-
dict(item.entry)
|
|
1305
|
+
self.entry_from_plan(dict(item.entry))
|
|
1306
|
+
for item in plan.items
|
|
1307
|
+
if item.action == "download"
|
|
1094
1308
|
]
|
|
1095
1309
|
console.print(
|
|
1096
1310
|
f"[dim]Plano {from_plan}: {len(entries)} entrada(s) "
|
|
@@ -1114,9 +1328,14 @@ class FetcherApp:
|
|
|
1114
1328
|
console.print(f"\n[bold]{len(entries)}[/bold] arquivo(s) listado(s).")
|
|
1115
1329
|
return
|
|
1116
1330
|
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1331
|
+
if force:
|
|
1332
|
+
downloaded, total, errors = self.download_datasets(
|
|
1333
|
+
entries, actual_output, workers=workers, force=True
|
|
1334
|
+
)
|
|
1335
|
+
else:
|
|
1336
|
+
downloaded, total, errors = self.download_datasets(
|
|
1337
|
+
entries, actual_output, workers=workers
|
|
1338
|
+
)
|
|
1120
1339
|
|
|
1121
1340
|
console.print(
|
|
1122
1341
|
f"\n[green]Concluído:[/green] {downloaded}/{total} arquivo(s) baixado(s)."
|
|
@@ -1147,6 +1366,14 @@ class FetcherApp:
|
|
|
1147
1366
|
help="Imprime o plano em JSON (para 'sync --from-plan')",
|
|
1148
1367
|
),
|
|
1149
1368
|
] = False,
|
|
1369
|
+
out: Annotated[
|
|
1370
|
+
Path | None,
|
|
1371
|
+
typer.Option(
|
|
1372
|
+
"--out",
|
|
1373
|
+
help="Grava o plano JSON neste arquivo (atômico) e imprime "
|
|
1374
|
+
"só um resumo; prevalece sobre --json",
|
|
1375
|
+
),
|
|
1376
|
+
] = None,
|
|
1150
1377
|
verbose: Annotated[
|
|
1151
1378
|
bool, typer.Option("--verbose", help="Logs detalhados")
|
|
1152
1379
|
] = False,
|
|
@@ -1164,7 +1391,20 @@ class FetcherApp:
|
|
|
1164
1391
|
entries = [e for g in target_groups for e in self.list_datasets(g)]
|
|
1165
1392
|
plan = self.check_datasets(entries, actual_output, workers=workers)
|
|
1166
1393
|
|
|
1167
|
-
if
|
|
1394
|
+
if out is not None:
|
|
1395
|
+
tmp = out.with_name(f".{out.name}.{os.getpid()}.tmp")
|
|
1396
|
+
try:
|
|
1397
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
1398
|
+
try:
|
|
1399
|
+
tmp.write_text(plan.to_json(), encoding="utf-8")
|
|
1400
|
+
os.replace(tmp, out)
|
|
1401
|
+
finally:
|
|
1402
|
+
tmp.unlink(missing_ok=True)
|
|
1403
|
+
except OSError as exc:
|
|
1404
|
+
console.print(f"[red]Falha ao gravar o plano em {out}: {exc}[/red]")
|
|
1405
|
+
raise typer.Exit(1) from None
|
|
1406
|
+
console.print(f"{plan.summary_line()} Plano gravado em {out}")
|
|
1407
|
+
elif as_json:
|
|
1168
1408
|
print(plan.to_json())
|
|
1169
1409
|
else:
|
|
1170
1410
|
plan.render_table()
|
|
@@ -1186,17 +1426,9 @@ class FetcherApp:
|
|
|
1186
1426
|
title=f"[bold]{group_id}[/bold] — {group_info.get('name', '')}",
|
|
1187
1427
|
)
|
|
1188
1428
|
for entry in self.list_datasets(group_id):
|
|
1189
|
-
if entry.get("semester") is not None:
|
|
1190
|
-
partition = f"{entry['year']}-S{entry['semester']}"
|
|
1191
|
-
elif entry.get("month") is not None:
|
|
1192
|
-
partition = f"{entry['year']}-{entry['month']:02d}"
|
|
1193
|
-
elif entry.get("year") is not None:
|
|
1194
|
-
partition = str(entry["year"])
|
|
1195
|
-
else:
|
|
1196
|
-
partition = "—"
|
|
1197
1429
|
table.add_row(
|
|
1198
1430
|
entry.get("id", ""),
|
|
1199
|
-
|
|
1431
|
+
_partition_label(entry) or "—",
|
|
1200
1432
|
entry.get("ext", ""),
|
|
1201
1433
|
entry.get("url", ""),
|
|
1202
1434
|
)
|