quantilica-cli 0.9.1__tar.gz → 0.10.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/CHANGELOG.md +28 -0
  2. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/PKG-INFO +2 -2
  3. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/pyproject.toml +2 -2
  4. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/src/quantilica/cli/sdk.py +436 -12
  5. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/src/quantilica/cli/sources.py +1 -0
  6. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/tests/test_sdk.py +222 -0
  7. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/.githooks/pre-commit +0 -0
  8. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/.githooks/pre-push +0 -0
  9. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/.github/workflows/publish.yml +0 -0
  10. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/.github/workflows/test.yml +0 -0
  11. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/.gitignore +0 -0
  12. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/LICENSE +0 -0
  13. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/README.md +0 -0
  14. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/src/quantilica/cli/__init__.py +0 -0
  15. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/src/quantilica/cli/cli.py +0 -0
  16. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/src/quantilica/cli/health.py +0 -0
  17. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/src/quantilica/cli/manifests.py +0 -0
  18. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/src/quantilica/cli/progress.py +0 -0
  19. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/src/quantilica/cli/ui.py +0 -0
  20. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/src/quantilica/py.typed +0 -0
  21. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/tests/__init__.py +0 -0
  22. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/tests/test_health.py +0 -0
  23. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/tests/test_manifests.py +0 -0
  24. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/tests/test_package_hygiene.py +0 -0
  25. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/tests/test_sdk_onda_a2.py +0 -0
  26. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/tests/test_sources.py +0 -0
  27. {quantilica_cli-0.9.1 → quantilica_cli-0.10.1}/tests/test_ui.py +0 -0
@@ -5,6 +5,34 @@ Todas as mudanças notáveis deste projeto serão documentadas neste arquivo.
5
5
  O formato segue [Keep a Changelog](https://keepachangelog.com/pt-BR/1.1.0/),
6
6
  e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
7
7
 
8
+ ## [0.10.1] - 2026-10-08
9
+
10
+ ### Corrigido
11
+
12
+ - `sdk`: `check_entry` considera `local.exists() or sidecar.exists()`, preservando
13
+ o status de `manifest-fresh` em ambientes de ingestão efêmera sem forçar
14
+ re-download redundante quando o dado bruto já foi espelhado.
15
+
16
+ ## [0.10.0] - 2026-10-07
17
+
18
+ ### Adicionado
19
+
20
+ - `FetcherApp`: comando `check` — verificação remoto × local (HEAD por
21
+ entrada) com plano de freshness, sem baixar; `--json` emite plano
22
+ consumível por `sync --from-plan`.
23
+ - `FetcherApp.sync`: opção `--from-plan <arquivo.json>` — baixa somente as
24
+ entradas com ação `download` do plano.
25
+ - SDK: `CheckPlan`/`CheckPlanItem` (tabela Rich + round-trip JSON) e
26
+ `default_client()` com overrides via ambiente (`QUANTILICA_CA_BUNDLE`,
27
+ `QUANTILICA_SSL_VERIFY`, `QUANTILICA_HTTP_ATTEMPTS/TIMEOUT/RETRY_DELAY`).
28
+
29
+ ### Alterado
30
+
31
+ - Piso de `quantilica-core` elevado para `>=0.10.0` (símbolos usados:
32
+ `is_remote_more_recent`, `resolve_verify_from_env`).
33
+ - Workers internos do `download_datasets` propagam o `transport` customizado
34
+ do client canônico.
35
+
8
36
  ## [0.9.1] - 2026-10-04
9
37
 
10
38
  Correção de escape inválido introduzido na v0.8.0 (Onda A.2 do SDK).
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: quantilica-cli
3
- Version: 0.9.1
3
+ Version: 0.10.1
4
4
  Summary: Unified CLI for Quantilica open data fetchers
5
5
  Author-email: "Komesu, D.K." <daniel@dkko.me>
6
6
  License-Expression: MIT
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.12
16
16
  Classifier: Programming Language :: Python :: 3.13
17
17
  Classifier: Typing :: Typed
18
18
  Requires-Python: >=3.12
19
- Requires-Dist: quantilica-core>=0.7.0
19
+ Requires-Dist: quantilica-core>=0.10.1
20
20
  Requires-Dist: rich>=13.0.0
21
21
  Requires-Dist: typer>=0.15.0
22
22
  Description-Content-Type: text/markdown
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "quantilica-cli"
3
- version = "0.9.1"
3
+ version = "0.10.1"
4
4
  description = "Unified CLI for Quantilica open data fetchers"
5
5
  readme = "README.md"
6
6
  authors = [{ name = "Komesu, D.K.", email = "daniel@dkko.me" }]
@@ -20,7 +20,7 @@ classifiers = [
20
20
  "Programming Language :: Python :: 3.13",
21
21
  ]
22
22
  dependencies = [
23
- "quantilica-core>=0.7.0",
23
+ "quantilica-core>=0.10.1",
24
24
  "rich>=13.0.0",
25
25
  "typer>=0.15.0",
26
26
  ]
@@ -8,6 +8,9 @@ from __future__ import annotations
8
8
  import concurrent.futures
9
9
  import contextlib
10
10
  import datetime as dt
11
+ import email.utils
12
+ import json
13
+ import os
11
14
  import threading
12
15
  from collections.abc import Callable, Iterable
13
16
  from dataclasses import dataclass, field
@@ -17,8 +20,15 @@ from typing import Annotated, Any
17
20
  import typer
18
21
  from quantilica.core.exceptions import FetchError
19
22
  from quantilica.core.ftp import FtpClient
20
- from quantilica.core.http import HttpClient, HttpStatusError, ProgressCallback
23
+ from quantilica.core.http import (
24
+ HttpClient,
25
+ HttpStatusError,
26
+ ProgressCallback,
27
+ is_remote_more_recent,
28
+ resolve_verify_from_env,
29
+ )
21
30
  from quantilica.core.logging import get_logger
31
+ from quantilica.core.manifests import manifest_sidecar_path
22
32
  from quantilica.core.storage import (
23
33
  StampedDataRepository,
24
34
  build_stamped_filename,
@@ -41,6 +51,8 @@ from quantilica.cli.ui import (
41
51
  logger = get_logger(__name__)
42
52
 
43
53
  __all__ = [
54
+ "CheckPlan",
55
+ "CheckPlanItem",
44
56
  "DataRepository",
45
57
  "FetcherApp",
46
58
  "SyncPlan",
@@ -113,17 +125,231 @@ class SyncPlan:
113
125
  )
114
126
 
115
127
 
128
+ def _partition_label(entry: dict[str, Any]) -> str | None:
129
+ """Human-readable partition label for a dataset entry."""
130
+ if entry.get("semester") is not None:
131
+ return f"{entry['year']}-S{entry['semester']}"
132
+ if entry.get("month") is not None:
133
+ return f"{entry['year']}-{entry['month']:02d}"
134
+ if entry.get("year") is not None:
135
+ return str(entry["year"])
136
+ return None
137
+
138
+
139
+ def _parse_head_date(value: str | None) -> dt.date | None:
140
+ """Parse an HTTP Last-Modified header value into a date, or None."""
141
+ if not value:
142
+ return None
143
+ with contextlib.suppress(Exception):
144
+ return email.utils.parsedate_to_datetime(value).date()
145
+ return None
146
+
147
+
148
+ def _short_remote(item: CheckPlanItem) -> str:
149
+ """Compact remote description for table display (narrow terminals)."""
150
+ date = _parse_head_date(item.remote_last_modified)
151
+ remote = date.isoformat() if date else (item.remote_last_modified or "—")
152
+ if item.remote_size:
153
+ remote += f" ({item.remote_size} B)"
154
+ return remote
155
+
156
+
157
+ @dataclass(frozen=True)
158
+ class CheckPlanItem:
159
+ """Freshness verdict for a single dataset entry (no download performed).
160
+
161
+ Args:
162
+ dataset: Canonical dataset (group) identifier.
163
+ id: Dataset entry identifier.
164
+ url: Remote URL that was checked.
165
+ partition: Human-readable partition label, or None.
166
+ local_path: Local destination path the entry maps to.
167
+ remote_etag: Remote ETag header, or None.
168
+ remote_last_modified: Remote Last-Modified header, or None.
169
+ remote_size: Remote Content-Length, or None.
170
+ local_exists: Whether the artifact already exists locally — this
171
+ covers either the raw file itself or its ephemeral ``.manifest.json``
172
+ sidecar marker (file deleted for retention, sidecar still fresh).
173
+ action: ``"download"`` or ``"skip-up-to-date"``.
174
+ reason: Machine-readable reason for the action.
175
+ entry: Original dataset entry (allows ``sync --from-plan`` round-trip).
176
+ """
177
+
178
+ dataset: str
179
+ id: str
180
+ url: str
181
+ partition: str | None
182
+ local_path: str
183
+ remote_etag: str | None
184
+ remote_last_modified: str | None
185
+ remote_size: int | None
186
+ local_exists: bool
187
+ action: str
188
+ reason: str
189
+ entry: dict[str, Any]
190
+
191
+
192
+ @dataclass(frozen=True)
193
+ class CheckPlan:
194
+ """Freshness plan for a verification run (``check`` command).
195
+
196
+ Args:
197
+ fetcher: Name of the fetcher that produced the plan.
198
+ output_dir: Output directory the plan was computed against.
199
+ generated_at: ISO-8601 UTC timestamp of plan generation.
200
+ items: Per-entry verdicts.
201
+ """
202
+
203
+ fetcher: str
204
+ output_dir: str
205
+ generated_at: str
206
+ items: list[CheckPlanItem] = field(default_factory=list)
207
+
208
+ def render_table(self, console: Console | None = None) -> None:
209
+ """Render the plan as a Rich table with a summary footer."""
210
+ con = console or get_console()
211
+ table = Table(
212
+ "Dataset",
213
+ "ID",
214
+ "Ação",
215
+ "Motivo",
216
+ "Remoto",
217
+ title="Verificação remoto × local (check)",
218
+ )
219
+ for item in self.items:
220
+ table.add_row(
221
+ item.dataset,
222
+ item.id,
223
+ item.action,
224
+ item.reason,
225
+ _short_remote(item),
226
+ )
227
+ con.print(table)
228
+ counts: dict[str, int] = {}
229
+ for item in self.items:
230
+ counts[item.action] = counts.get(item.action, 0) + 1
231
+ summary = ", ".join(f"{v} {k}" for k, v in sorted(counts.items()))
232
+ con.print(f"[bold]{len(self.items)}[/bold] verificado(s): {summary}.")
233
+
234
+ def to_dict(self) -> dict[str, Any]:
235
+ """Return the JSON-serializable representation of the plan."""
236
+ return {
237
+ "fetcher": self.fetcher,
238
+ "output_dir": self.output_dir,
239
+ "generated_at": self.generated_at,
240
+ "items": [
241
+ {
242
+ "dataset": it.dataset,
243
+ "id": it.id,
244
+ "url": it.url,
245
+ "partition": it.partition,
246
+ "local_path": it.local_path,
247
+ "remote_etag": it.remote_etag,
248
+ "remote_last_modified": it.remote_last_modified,
249
+ "remote_size": it.remote_size,
250
+ "local_exists": it.local_exists,
251
+ "action": it.action,
252
+ "reason": it.reason,
253
+ "entry": it.entry,
254
+ }
255
+ for it in self.items
256
+ ],
257
+ }
258
+
259
+ def to_json(self) -> str:
260
+ """Serialize the plan to a JSON string (for ``sync --from-plan``)."""
261
+ return json.dumps(self.to_dict(), ensure_ascii=False, indent=2)
262
+
263
+ @classmethod
264
+ def from_dict(cls, data: dict[str, Any]) -> CheckPlan:
265
+ """Rebuild a plan from :meth:`to_dict` output.
266
+
267
+ Raises:
268
+ ValueError: If the payload is not a valid check plan.
269
+ """
270
+ if not isinstance(data, dict) or not isinstance(data.get("items"), list):
271
+ raise ValueError("not a check plan: missing 'items' list")
272
+ items = []
273
+ for raw in data["items"]:
274
+ if not isinstance(raw, dict):
275
+ raise ValueError("not a check plan: invalid item")
276
+ try:
277
+ entry = raw["entry"]
278
+ action = raw["action"]
279
+ eid = raw["id"]
280
+ except KeyError as exc:
281
+ raise ValueError(f"not a check plan: item missing {exc}") from exc
282
+ if not isinstance(entry, dict) or action not in (
283
+ "download",
284
+ "skip-up-to-date",
285
+ ):
286
+ raise ValueError("not a check plan: invalid item")
287
+ items.append(
288
+ CheckPlanItem(
289
+ dataset=str(raw.get("dataset", "")),
290
+ id=str(eid),
291
+ url=str(raw.get("url", "")),
292
+ partition=raw.get("partition"),
293
+ local_path=str(raw.get("local_path", "")),
294
+ remote_etag=raw.get("remote_etag"),
295
+ remote_last_modified=raw.get("remote_last_modified"),
296
+ remote_size=raw.get("remote_size"),
297
+ local_exists=bool(raw.get("local_exists", False)),
298
+ action=action,
299
+ reason=str(raw.get("reason", "")),
300
+ entry=entry,
301
+ )
302
+ )
303
+ return cls(
304
+ fetcher=str(data.get("fetcher", "")),
305
+ output_dir=str(data.get("output_dir", "")),
306
+ generated_at=str(data.get("generated_at", "")),
307
+ items=items,
308
+ )
309
+
310
+ @classmethod
311
+ def from_json(cls, payload: str) -> CheckPlan:
312
+ """Parse a plan serialized with :meth:`to_json`.
313
+
314
+ Raises:
315
+ ValueError: If the payload is not valid JSON or not a plan.
316
+ """
317
+ try:
318
+ data = json.loads(payload)
319
+ except json.JSONDecodeError as exc:
320
+ raise ValueError(f"not a check plan: invalid JSON ({exc})") from exc
321
+ return cls.from_dict(data)
322
+
323
+
116
324
  def default_client() -> HttpClient:
117
325
  """Create a default HttpClient with standard configuration.
118
326
 
327
+ TLS verification and retry budget can be overridden via environment:
328
+
329
+ - ``QUANTILICA_CA_BUNDLE``: path to a custom CA bundle.
330
+ - ``QUANTILICA_SSL_VERIFY``: ``0``/``false`` disables verification
331
+ (ops escape hatch for hosts with broken chains — never default);
332
+ a path uses it as CA bundle.
333
+ - ``QUANTILICA_HTTP_ATTEMPTS`` / ``QUANTILICA_HTTP_TIMEOUT`` /
334
+ ``QUANTILICA_HTTP_RETRY_DELAY``: retry budget overrides.
335
+
119
336
  Returns:
120
337
  A pre-configured HttpClient instance (browser-like WAF headers + pooling).
121
338
  """
339
+ verify = resolve_verify_from_env()
340
+ if verify is False:
341
+ logger.warning(
342
+ "TLS verification disabled via QUANTILICA_SSL_VERIFY — "
343
+ "use only for hosts with broken chains"
344
+ )
345
+ attempts = int(os.environ.get("QUANTILICA_HTTP_ATTEMPTS", "5"))
346
+ timeout = float(os.environ.get("QUANTILICA_HTTP_TIMEOUT", "180.0"))
347
+ retry_base_delay = float(os.environ.get("QUANTILICA_HTTP_RETRY_DELAY", "2.0"))
122
348
  return HttpClient(
123
- timeout=180.0,
124
- verify=True,
125
- attempts=5,
126
- retry_base_delay=2.0,
349
+ timeout=timeout,
350
+ verify=verify,
351
+ attempts=attempts,
352
+ retry_base_delay=retry_base_delay,
127
353
  emulate_browser=True,
128
354
  )
129
355
 
@@ -648,6 +874,7 @@ class FetcherApp:
648
874
  attempts=self.client.attempts,
649
875
  retry_base_delay=self.client.retry_base_delay,
650
876
  verify=self.client.verify,
877
+ transport=self.client.transport,
651
878
  limits=self.client.limits,
652
879
  emulate_browser=True,
653
880
  )
@@ -696,6 +923,137 @@ class FetcherApp:
696
923
 
697
924
  return downloaded, total, errors
698
925
 
926
+ def check_entry(self, entry: dict[str, Any], output_dir: Path) -> CheckPlanItem:
927
+ """Check one dataset entry against the remote without downloading.
928
+
929
+ Performs HEAD requests (same URLs ``download_entry`` would try,
930
+ including ``fallback_urls``) and compares against the local stamped
931
+ file using the same freshness predicate as the download path.
932
+
933
+ Args:
934
+ entry: Dictionary containing dataset metadata (url, id, etc).
935
+ output_dir: Destination directory used for path computation.
936
+
937
+ Returns:
938
+ The freshness verdict for the entry.
939
+ """
940
+ dataset = str(entry.get("group") or entry.get("id") or "datasets")
941
+ eid = str(entry.get("id", "unknown"))
942
+ partition = _partition_label(entry)
943
+ urls_to_try = [entry["url"]]
944
+ if "fallback_urls" in entry and entry["fallback_urls"]:
945
+ urls_to_try.extend(entry["fallback_urls"])
946
+
947
+ head = None
948
+ head_error: Exception | None = None
949
+ for url in urls_to_try:
950
+ try:
951
+ candidate = self.client.head(url)
952
+ except HttpStatusError as exc:
953
+ if exc.status_code == 404:
954
+ head_error = exc
955
+ continue
956
+ head_error = exc
957
+ break
958
+ except Exception as exc: # noqa: BLE001 - metadata best effort
959
+ head_error = exc
960
+ break
961
+ head = candidate
962
+ break
963
+
964
+ if head is None:
965
+ local = self.path_builder(output_dir, entry, None)
966
+ return CheckPlanItem(
967
+ dataset=dataset,
968
+ id=eid,
969
+ url=str(entry["url"]),
970
+ partition=partition,
971
+ local_path=str(local),
972
+ remote_etag=None,
973
+ remote_last_modified=None,
974
+ remote_size=None,
975
+ local_exists=local.exists(),
976
+ action="download",
977
+ reason=f"metadata-unavailable: {head_error}",
978
+ entry=dict(entry),
979
+ )
980
+
981
+ etag = head.headers.get("ETag")
982
+ size_raw = head.headers.get("Content-Length")
983
+ try:
984
+ size = int(size_raw) if size_raw else None
985
+ except (TypeError, ValueError):
986
+ size = None
987
+ last_modified = _parse_head_date(head.headers.get("Last-Modified"))
988
+ local = self.path_builder(output_dir, entry, last_modified)
989
+ sidecar = manifest_sidecar_path(local)
990
+ exists = local.exists() or sidecar.exists()
991
+ if exists and not is_remote_more_recent(head, local):
992
+ if local.exists():
993
+ action, reason = "skip-up-to-date", "local-fresh"
994
+ else:
995
+ action, reason = "skip-up-to-date", "manifest-fresh"
996
+ else:
997
+ action = "download"
998
+ reason = "not-present" if not exists else "remote-newer"
999
+ return CheckPlanItem(
1000
+ dataset=dataset,
1001
+ id=eid,
1002
+ url=str(head.url),
1003
+ partition=partition,
1004
+ local_path=str(local),
1005
+ remote_etag=etag,
1006
+ remote_last_modified=head.headers.get("Last-Modified"),
1007
+ remote_size=size,
1008
+ local_exists=exists,
1009
+ action=action,
1010
+ reason=reason,
1011
+ entry=dict(entry),
1012
+ )
1013
+
1014
+ def check_datasets(
1015
+ self,
1016
+ entries: list[dict[str, Any]],
1017
+ output_dir: Path,
1018
+ workers: int = 4,
1019
+ ) -> CheckPlan:
1020
+ """Checks freshness for a list of dataset entries (no downloads).
1021
+
1022
+ Args:
1023
+ entries: List of dataset entries to check.
1024
+ output_dir: The base directory used for path computation.
1025
+ workers: Maximum number of parallel checks.
1026
+
1027
+ Returns:
1028
+ A :class:`CheckPlan` with one verdict per entry.
1029
+ """
1030
+ results: list[CheckPlanItem | None] = [None] * len(entries)
1031
+
1032
+ def _worker(index: int, entry: dict[str, Any]) -> None:
1033
+ results[index] = self.check_entry(entry, output_dir)
1034
+ if (index + 1) % 100 == 0 or index + 1 == len(entries):
1035
+ logger.info("[check] %d/%d verificados", index + 1, len(entries))
1036
+
1037
+ with graceful_executor(max_workers=workers) as executor:
1038
+ try:
1039
+ futures = {
1040
+ executor.submit(_worker, index, entry): index
1041
+ for index, entry in enumerate(entries)
1042
+ }
1043
+ for future in concurrent.futures.as_completed(futures):
1044
+ future.result()
1045
+ except KeyboardInterrupt:
1046
+ get_console().print("\n[yellow]Interrompido.[/yellow]")
1047
+ raise typer.Exit(130) from None
1048
+
1049
+ generated = dt.datetime.now(dt.UTC).isoformat(timespec="seconds")
1050
+ return CheckPlan(
1051
+ fetcher=self.name,
1052
+ output_dir=str(output_dir),
1053
+ generated_at=generated,
1054
+ items=[item for item in results if item is not None],
1055
+ )
1056
+
699
1057
  def _build_commands(self) -> None:
700
1058
  console = get_console()
701
1059
 
@@ -716,6 +1074,15 @@ class FetcherApp:
716
1074
  workers: Annotated[
717
1075
  int, typer.Option("--workers", help="Downloads paralelos")
718
1076
  ] = 4,
1077
+ from_plan: Annotated[
1078
+ Path | None,
1079
+ typer.Option(
1080
+ "--from-plan",
1081
+ help="Baixar somente as entradas com ação 'download' "
1082
+ "de um plano gerado por 'check' (ignora a seleção "
1083
+ "de grupos).",
1084
+ ),
1085
+ ] = None,
719
1086
  verbose: Annotated[
720
1087
  bool, typer.Option("--verbose", help="Logs detalhados")
721
1088
  ] = False,
@@ -723,14 +1090,28 @@ class FetcherApp:
723
1090
  setup_rich_logging(verbose, console=console)
724
1091
  actual_output = output or self.default_output
725
1092
 
726
- try:
727
- target_groups = self.resolve_groups(groups)
728
- except ValueError as exc:
729
- console.print(f"[red]{exc}[/red]")
730
- console.print(f"Grupos válidos: {', '.join(self.all_keys)}")
731
- raise typer.Exit(1) from None
1093
+ if from_plan is not None:
1094
+ try:
1095
+ plan = CheckPlan.from_json(from_plan.read_text(encoding="utf-8"))
1096
+ except (OSError, ValueError) as exc:
1097
+ console.print(f"[red]Plano inválido: {exc}[/red]")
1098
+ raise typer.Exit(1) from None
1099
+ entries = [
1100
+ dict(item.entry) for item in plan.items if item.action == "download"
1101
+ ]
1102
+ console.print(
1103
+ f"[dim]Plano {from_plan}: {len(entries)} entrada(s) "
1104
+ f"com ação 'download' de {len(plan.items)} verificada(s).[/dim]"
1105
+ )
1106
+ else:
1107
+ try:
1108
+ target_groups = self.resolve_groups(groups)
1109
+ except ValueError as exc:
1110
+ console.print(f"[red]{exc}[/red]")
1111
+ console.print(f"Grupos válidos: {', '.join(self.all_keys)}")
1112
+ raise typer.Exit(1) from None
732
1113
 
733
- entries = [e for g in target_groups for e in self.list_datasets(g)]
1114
+ entries = [e for g in target_groups for e in self.list_datasets(g)]
734
1115
 
735
1116
  if dry_run:
736
1117
  table = Table("Grupo", "ID", "URL", title="Arquivos a baixar (dry-run)")
@@ -752,6 +1133,49 @@ class FetcherApp:
752
1133
  for eid, emsg in errors:
753
1134
  console.print(f" {eid}: {emsg}")
754
1135
 
1136
+ @self.app.command("check")
1137
+ def check(
1138
+ groups: Annotated[
1139
+ list[str] | None,
1140
+ typer.Argument(
1141
+ help="Grupos a verificar. Use 'list' para ver grupos disponíveis. Padrão: todos."
1142
+ ),
1143
+ ] = None,
1144
+ output: Annotated[
1145
+ Path | None, typer.Option("-o", "--output", help="Diretório de saída")
1146
+ ] = None,
1147
+ workers: Annotated[
1148
+ int, typer.Option("--workers", help="Verificações paralelas")
1149
+ ] = 4,
1150
+ as_json: Annotated[
1151
+ bool,
1152
+ typer.Option(
1153
+ "--json",
1154
+ help="Imprime o plano em JSON (para 'sync --from-plan')",
1155
+ ),
1156
+ ] = False,
1157
+ verbose: Annotated[
1158
+ bool, typer.Option("--verbose", help="Logs detalhados")
1159
+ ] = False,
1160
+ ) -> None:
1161
+ setup_rich_logging(verbose, console=console)
1162
+ actual_output = output or self.default_output
1163
+
1164
+ try:
1165
+ target_groups = self.resolve_groups(groups)
1166
+ except ValueError as exc:
1167
+ console.print(f"[red]{exc}[/red]")
1168
+ console.print(f"Grupos válidos: {', '.join(self.all_keys)}")
1169
+ raise typer.Exit(1) from None
1170
+
1171
+ entries = [e for g in target_groups for e in self.list_datasets(g)]
1172
+ plan = self.check_datasets(entries, actual_output, workers=workers)
1173
+
1174
+ if as_json:
1175
+ print(plan.to_json())
1176
+ else:
1177
+ plan.render_table()
1178
+
755
1179
  @self.app.command("list")
756
1180
  def cmd_list(
757
1181
  verbose: Annotated[
@@ -37,6 +37,7 @@ SOURCES_REGISTRY: dict[str, str] = {
37
37
  "rtn": "rtn-fetcher",
38
38
  "sidra": "sidra-fetcher",
39
39
  "td": "tesouro-direto-fetcher",
40
+ "tse": "tse-fetcher",
40
41
  }
41
42
 
42
43
 
@@ -3,16 +3,21 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  import datetime as dt
6
+ import json
6
7
  from pathlib import Path
7
8
  from typing import Annotated
8
9
 
10
+ import httpx2
9
11
  import pytest
10
12
  import typer
13
+ from quantilica.core.http import HttpClient
11
14
  from typer.testing import CliRunner
12
15
 
13
16
  from quantilica.cli.sdk import (
17
+ CheckPlan,
14
18
  DataRepository,
15
19
  FetcherApp,
20
+ default_client,
16
21
  default_path_builder,
17
22
  make_resolve_groups,
18
23
  )
@@ -275,3 +280,220 @@ def test_builtin_list():
275
280
  result = runner.invoke(app.app, ["list"])
276
281
  assert result.exit_code == 0, result.output
277
282
  assert "2 dataset(s) no catálogo." in result.output
283
+
284
+
285
+ # ---------------------------------------------------------------
286
+ # check (verificação remoto × local, sem download) + sync --from-plan
287
+ # ---------------------------------------------------------------
288
+
289
+
290
+ def _head_handler(payload: bytes = b"0123456789"):
291
+ def handler(request):
292
+ if request.method == "HEAD":
293
+ return httpx2.Response(
294
+ 200,
295
+ headers={
296
+ "Content-Length": str(len(payload)),
297
+ "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT",
298
+ "ETag": '"abc123"',
299
+ },
300
+ )
301
+ return httpx2.Response(200, content=payload)
302
+
303
+ return handler
304
+
305
+
306
+ def _mock_client(payload: bytes = b"0123456789") -> HttpClient:
307
+ return HttpClient(
308
+ attempts=1, transport=httpx2.MockTransport(_head_handler(payload))
309
+ )
310
+
311
+
312
+ def test_check_entry_missing_downloads(tmp_path: Path):
313
+ app = make_app(client=_mock_client())
314
+ entry = sample_list_datasets("exp")[0]
315
+ item = app.check_entry(entry, tmp_path)
316
+ assert item.action == "download"
317
+ assert item.reason == "not-present"
318
+ assert item.remote_etag == '"abc123"'
319
+ assert item.remote_size == 10
320
+ assert item.remote_last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
321
+ assert item.local_exists is False
322
+
323
+
324
+ def test_check_entry_fresh_skips(tmp_path: Path):
325
+ from quantilica.cli.sdk import default_path_builder
326
+
327
+ app = make_app(client=_mock_client())
328
+ entry = sample_list_datasets("exp")[0]
329
+ local = default_path_builder(tmp_path, entry, dt.date(2024, 1, 1))
330
+ local.parent.mkdir(parents=True, exist_ok=True)
331
+ local.write_bytes(b"0123456789")
332
+ item = app.check_entry(entry, tmp_path)
333
+ assert item.action == "skip-up-to-date"
334
+ assert item.reason == "local-fresh"
335
+ assert item.local_exists is True
336
+
337
+
338
+ def test_check_entry_head_error_downloads(tmp_path: Path):
339
+ def handler(request):
340
+ raise httpx2.ConnectError("down")
341
+
342
+ app = make_app(
343
+ client=HttpClient(attempts=1, transport=httpx2.MockTransport(handler))
344
+ )
345
+ entry = sample_list_datasets("exp")[0]
346
+ item = app.check_entry(entry, tmp_path)
347
+ assert item.action == "download"
348
+ assert item.reason.startswith("metadata-unavailable")
349
+
350
+
351
+ def test_check_plan_json_roundtrip(tmp_path: Path):
352
+ app = make_app(client=_mock_client())
353
+ entries = sample_list_datasets("exp")
354
+ plan = app.check_datasets(entries, tmp_path, workers=1)
355
+ restored = CheckPlan.from_json(plan.to_json())
356
+ assert restored.fetcher == plan.fetcher
357
+ assert [it.id for it in restored.items] == [it.id for it in plan.items]
358
+ assert restored.items[0].entry["url"] == entries[0]["url"]
359
+
360
+
361
+ def test_check_plan_from_json_rejects_garbage():
362
+ with pytest.raises(ValueError, match="not a check plan"):
363
+ CheckPlan.from_json('{"foo": 1}')
364
+ with pytest.raises(ValueError, match="not a check plan"):
365
+ CheckPlan.from_json("not json")
366
+
367
+
368
+ def test_builtin_check_command(tmp_path: Path):
369
+ app = make_app(client=_mock_client())
370
+ result = runner.invoke(app.app, ["check", "exp", "-o", str(tmp_path)])
371
+ assert result.exit_code == 0, result.output
372
+ assert "download" in result.output
373
+
374
+
375
+ def test_builtin_check_json_output(tmp_path: Path):
376
+ app = make_app(client=_mock_client())
377
+ result = runner.invoke(app.app, ["check", "exp", "-o", str(tmp_path), "--json"])
378
+ assert result.exit_code == 0, result.output
379
+ data = json.loads(result.output)
380
+ assert data["items"][0]["action"] == "download"
381
+
382
+
383
+ def test_builtin_sync_from_plan_downloads_only_planned(tmp_path: Path):
384
+ from quantilica.cli.sdk import default_path_builder
385
+
386
+ payload = b"0123456789"
387
+ app = make_app(client=_mock_client(payload))
388
+ fresh_entry = sample_list_datasets("exp")[0]
389
+ stale_entry = dict(sample_list_datasets("imp")[0])
390
+ local = default_path_builder(tmp_path, fresh_entry, dt.date(2024, 1, 1))
391
+ local.parent.mkdir(parents=True, exist_ok=True)
392
+ local.write_bytes(payload)
393
+
394
+ plan = app.check_datasets([fresh_entry, stale_entry], tmp_path, workers=1)
395
+ actions = {it.id: it.action for it in plan.items}
396
+ assert actions == {"exp-monthly": "skip-up-to-date", "imp-monthly": "download"}
397
+
398
+ plan_file = tmp_path / "plan.json"
399
+ plan_file.write_text(plan.to_json(), encoding="utf-8")
400
+ result = runner.invoke(
401
+ app.app, ["sync", "--from-plan", str(plan_file), "-o", str(tmp_path)]
402
+ )
403
+ assert result.exit_code == 0, result.output
404
+ assert "1/1 arquivo(s) baixado(s)" in result.output
405
+
406
+
407
+ def test_builtin_sync_from_plan_invalid_file(tmp_path: Path):
408
+ app = make_app(client=_mock_client())
409
+ bad = tmp_path / "bad.json"
410
+ bad.write_text('{"nope": true}', encoding="utf-8")
411
+ result = runner.invoke(app.app, ["sync", "--from-plan", str(bad)])
412
+ assert result.exit_code == 1
413
+ assert "Plano inválido" in result.output
414
+
415
+
416
+ # ---------------------------------------------------------------
417
+ # default_client: overrides via ambiente
418
+ # ---------------------------------------------------------------
419
+
420
+
421
+ def test_default_client_verify_defaults_true(monkeypatch):
422
+ monkeypatch.delenv("QUANTILICA_SSL_VERIFY", raising=False)
423
+ monkeypatch.delenv("QUANTILICA_CA_BUNDLE", raising=False)
424
+ assert default_client().verify is True
425
+
426
+
427
+ def test_default_client_verify_env_disable(monkeypatch):
428
+ monkeypatch.delenv("QUANTILICA_CA_BUNDLE", raising=False)
429
+ monkeypatch.setenv("QUANTILICA_SSL_VERIFY", "0")
430
+ assert default_client().verify is False
431
+
432
+
433
+ def test_default_client_attempts_env_override(monkeypatch):
434
+ monkeypatch.delenv("QUANTILICA_SSL_VERIFY", raising=False)
435
+ monkeypatch.delenv("QUANTILICA_CA_BUNDLE", raising=False)
436
+ monkeypatch.setenv("QUANTILICA_HTTP_ATTEMPTS", "9")
437
+ assert default_client().attempts == 9
438
+
439
+
440
+ def test_check_entry_ephemeral_manifest_fresh(tmp_path: Path):
441
+ """Raw file deleted but sidecar manifest survives -> manifest-fresh skip."""
442
+ from quantilica.core.manifests import (
443
+ DownloadManifest,
444
+ SourceMetadata,
445
+ write_manifest_sidecar,
446
+ )
447
+
448
+ app = make_app(client=_mock_client())
449
+ entry = sample_list_datasets("exp")[0]
450
+ local = default_path_builder(tmp_path, entry, dt.date(2024, 1, 1))
451
+ local.parent.mkdir(parents=True, exist_ok=True)
452
+ write_manifest_sidecar(
453
+ local,
454
+ DownloadManifest(
455
+ source_id="src",
456
+ dataset_id="ds",
457
+ url=entry["url"],
458
+ fetched_at="2024-01-02T00:00:00+00:00",
459
+ sha256="0" * 64,
460
+ size_bytes=10,
461
+ source_meta=SourceMetadata(
462
+ etag='"abc123"',
463
+ last_modified="Mon, 01 Jan 2024 00:00:00 GMT",
464
+ ),
465
+ ),
466
+ )
467
+ item = app.check_entry(entry, tmp_path)
468
+ assert item.action == "skip-up-to-date"
469
+ assert item.reason == "manifest-fresh"
470
+ assert item.local_exists is True
471
+
472
+
473
+ def test_check_entry_ephemeral_stale_downloads(tmp_path: Path):
474
+ """Sidecar with stale metadata (remote ETag differs) -> download."""
475
+ from quantilica.core.manifests import (
476
+ DownloadManifest,
477
+ SourceMetadata,
478
+ write_manifest_sidecar,
479
+ )
480
+
481
+ app = make_app(client=_mock_client())
482
+ entry = sample_list_datasets("exp")[0]
483
+ local = default_path_builder(tmp_path, entry, dt.date(2024, 1, 1))
484
+ local.parent.mkdir(parents=True, exist_ok=True)
485
+ write_manifest_sidecar(
486
+ local,
487
+ DownloadManifest(
488
+ source_id="src",
489
+ dataset_id="ds",
490
+ url=entry["url"],
491
+ fetched_at="2023-12-31T00:00:00+00:00",
492
+ sha256="0" * 64,
493
+ size_bytes=10,
494
+ source_meta=SourceMetadata(etag='"outdated"', last_modified=None),
495
+ ),
496
+ )
497
+ item = app.check_entry(entry, tmp_path)
498
+ assert item.action == "download"
499
+ assert item.reason == "remote-newer"
File without changes