quantilica-cli 0.7.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/CHANGELOG.md +40 -0
  2. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/PKG-INFO +1 -1
  3. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/pyproject.toml +1 -1
  4. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/cli.py +3 -1
  5. quantilica_cli-0.9.0/src/quantilica/cli/health.py +188 -0
  6. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/sdk.py +280 -1
  7. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/sources.py +1 -0
  8. quantilica_cli-0.9.0/tests/test_health.py +208 -0
  9. quantilica_cli-0.9.0/tests/test_sdk_onda_a2.py +295 -0
  10. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/.githooks/pre-commit +0 -0
  11. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/.githooks/pre-push +0 -0
  12. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/.github/workflows/publish.yml +0 -0
  13. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/.github/workflows/test.yml +0 -0
  14. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/.gitignore +0 -0
  15. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/LICENSE +0 -0
  16. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/README.md +0 -0
  17. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/__init__.py +0 -0
  18. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/manifests.py +0 -0
  19. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/progress.py +0 -0
  20. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/ui.py +0 -0
  21. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/py.typed +0 -0
  22. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/tests/__init__.py +0 -0
  23. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/tests/test_manifests.py +0 -0
  24. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/tests/test_sdk.py +0 -0
  25. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/tests/test_sources.py +0 -0
  26. {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/tests/test_ui.py +0 -0
@@ -5,6 +5,46 @@ Todas as mudanças notáveis deste projeto serão documentadas neste arquivo.
5
5
  O formato segue [Keep a Changelog](https://keepachangelog.com/pt-BR/1.1.0/),
6
6
  e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
7
7
 
8
+ ## [0.9.0] - 2026-10-04
9
+
10
+ Registra o `cvm-fetcher` como fonte instalável sob demanda, após a v0.1.0 do
11
+ fetcher (wheel no GitHub Releases, entrada no índice PEP 503 do ecossistema).
12
+
13
+ ### Adicionado
14
+ - Fonte `cvm` em `SOURCES_REGISTRY` (`quantilica install cvm`), apontando
15
+ para a distribuição `cvm-fetcher`.
16
+
17
+ ## [0.8.0] - 2026-10-03
18
+
19
+ Onda A.2 do plano `2026-10-03-padronizacao-core-e-consolidacao-fetchers` —
20
+ decorators de ciclo de vida no SDK (`quantilica.cli.sdk`) e abstração de
21
+ pré-visualização tabular de sincronização.
22
+
23
+ ### Adicionado
24
+ - `FetcherApp.command_convert(func)`: decorator que registra o subcomando
25
+ `convert` com flags canônicas (`-i/--input`, `-o/--output`, `--verbose`,
26
+ padrões derivados de `default_output`). Configura logging Rich, trata
27
+ `ImportError` graciosamente (sugerindo `pip install {nome}[analysis]`,
28
+ saída com código 1) e exibe confirmação com check verde.
29
+ - `FetcherApp.command_pipeline(func)`: decorator que registra o subcomando
30
+ `pipeline` encadeando sincronização (passo 1/2, via `sync`) e conversão
31
+ analítica (passo 2/2, via `func`). Opções: grupos, `--output`,
32
+ `--parquet-dir`, `--workers`, `--dry-run` (interrompe antes do passo 2) e
33
+ `--verbose`.
34
+ - `FetcherApp.command_archive(func)`: decorator que registra o subcomando
35
+ `archive` para arquivamento histórico, com o mesmo padrão de flags e
36
+ tratamento gracioso de `ImportError`.
37
+ - `SyncPlanItem`/`SyncPlan`: dataclasses (imutáveis) para planejamento de
38
+ sincronização, com `SyncPlan.render_table(console=None)` que renderiza uma
39
+ tabela Rich (`Dataset | Partição | Arquivo | URL`) e o sumário `Total: X
40
+ arquivos planejados. Y ignorados fora de cobertura.` Importados do
41
+ `quantilica.cli.sdk`.
42
+ - `quantilica health`: subcomando de diagnóstico que faz sondas HTTP leves
43
+ (HEAD com fallback para GET, timeout padrão 5s) contra as fontes canônicas
44
+ do ecossistema — BCB SGS, SIDRA, Tesouro Direto, Comex e INMET — em
45
+ paralelo, com saída em tabela Rich (`Fonte | Estado | Latência | HTTP`) ou
46
+ em JSON estruturado via `--json`. Usa `quantilica.core.http` para o probe.
47
+
8
48
  ## [0.7.0] - 2026-10-02
9
49
 
10
50
  Onda 2 — extensões do SDK (`quantilica.cli.sdk`) para eliminação de
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: quantilica-cli
3
- Version: 0.7.0
3
+ Version: 0.9.0
4
4
  Summary: Unified CLI for Quantilica open data fetchers
5
5
  Author-email: "Komesu, D.K." <daniel@dkko.me>
6
6
  License-Expression: MIT
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "quantilica-cli"
3
- version = "0.7.0"
3
+ version = "0.9.0"
4
4
  description = "Unified CLI for Quantilica open data fetchers"
5
5
  readme = "README.md"
6
6
  authors = [{ name = "Komesu, D.K.", email = "daniel@dkko.me" }]
@@ -13,6 +13,7 @@ from rich.logging import RichHandler
13
13
  from rich.table import Table
14
14
 
15
15
  from quantilica.cli import __version__
16
+ from quantilica.cli.health import cmd_health
16
17
  from quantilica.cli.manifests import app as manifests_app
17
18
  from quantilica.cli.sources import (
18
19
  app as sources_app,
@@ -35,10 +36,11 @@ app = typer.Typer(
35
36
  app.add_typer(manifests_app, name="manifests")
36
37
  app.add_typer(sources_app, name="sources")
37
38
 
38
- # Adiciona comandos top-level install, uninstall e doctor
39
+ # Adiciona comandos top-level install, uninstall, doctor e health
39
40
  app.command("install")(cmd_install)
40
41
  app.command("uninstall")(cmd_uninstall)
41
42
  app.command("doctor")(cmd_doctor)
43
+ app.command("health")(cmd_health)
42
44
 
43
45
  console = Console()
44
46
 
@@ -0,0 +1,188 @@
1
+ """Comando `quantilica health` — sondas de disponibilidade das fontes.
2
+
3
+ Realiza sondas HTTP leves (HEAD com fallback para GET) contra as fontes
4
+ canônicas de dados do ecossistema Quantilica e reporta o resultado em
5
+ tabela Rich ou JSON estruturado.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import concurrent.futures
11
+ import json
12
+ import time
13
+ from typing import Annotated
14
+
15
+ import httpx2
16
+ import typer
17
+ from rich.console import Console
18
+ from rich.table import Table
19
+
20
+ console = Console()
21
+
22
+ DEFAULT_TIMEOUT = 5.0
23
+ USER_AGENT = "quantilica-cli (health)"
24
+
25
+ # Lista canônica de fontes sondadas (nome, URL de sonda leve).
26
+ HEALTH_SOURCES: list[tuple[str, str]] = [
27
+ ("bcb", "https://api.bcb.gov.br/dados/serie/bcdata.sgs.1/dados?formato=json"),
28
+ ("sidra", "https://servicodados.ibge.gov.br/api/v3/calendario"),
29
+ (
30
+ "tesouro_direto",
31
+ "https://www.tesourotransparente.gov.br/ckan/api/3/action/package_search?rows=1",
32
+ ),
33
+ ("comex", "https://balanca.economia.gov.br"),
34
+ ("inmet", "https://apitempo.inmet.gov.br/estacoes"),
35
+ ]
36
+
37
+
38
+ def _make_client(timeout: float) -> httpx2.Client:
39
+ """Cria o cliente HTTP usado pelas sondas.
40
+
41
+ Args:
42
+ timeout: Timeout (segundos) aplicado a cada requisição.
43
+
44
+ Returns:
45
+ Um httpx2.Client configurado com redirects e User-Agent da CLI.
46
+ """
47
+ return httpx2.Client(
48
+ timeout=timeout,
49
+ follow_redirects=True,
50
+ headers={"User-Agent": USER_AGENT},
51
+ )
52
+
53
+
54
+ def _probe(client: httpx2.Client, name: str, url: str) -> dict[str, object]:
55
+ """Executa uma sonda HEAD (com fallback GET) contra uma fonte.
56
+
57
+ Args:
58
+ client: O cliente HTTP a ser usado.
59
+ name: Nome canônico da fonte.
60
+ url: URL alvo da sonda.
61
+
62
+ Returns:
63
+ Dicioário com name, url, status, http_status, latency_ms e error.
64
+ """
65
+ start = time.perf_counter()
66
+ try:
67
+ response = client.head(url)
68
+ # Muitos endpoints não implementam HEAD — fallback para GET leve.
69
+ if response.status_code >= 400:
70
+ response = client.get(url)
71
+ latency_ms = (time.perf_counter() - start) * 1000.0
72
+ if response.status_code < 400:
73
+ return {
74
+ "name": name,
75
+ "url": url,
76
+ "status": "ok",
77
+ "http_status": response.status_code,
78
+ "latency_ms": round(latency_ms, 1),
79
+ "error": None,
80
+ }
81
+ return {
82
+ "name": name,
83
+ "url": url,
84
+ "status": "falha",
85
+ "http_status": response.status_code,
86
+ "latency_ms": round(latency_ms, 1),
87
+ "error": f"HTTP {response.status_code}",
88
+ }
89
+ except Exception as exc:
90
+ latency_ms = (time.perf_counter() - start) * 1000.0
91
+ return {
92
+ "name": name,
93
+ "url": url,
94
+ "status": "falha",
95
+ "http_status": None,
96
+ "latency_ms": round(latency_ms, 1),
97
+ "error": f"{type(exc).__name__}: {exc}",
98
+ }
99
+
100
+
101
+ def _run_probes(timeout: float, fail_fast: bool) -> list[dict[str, object]]:
102
+ """Sonda todas as fontes, em sequência (fail-fast) ou em paralelo.
103
+
104
+ Args:
105
+ timeout: Timeout por requisição, em segundos.
106
+ fail_fast: Se True, interrompe as sondas na primeira falha.
107
+
108
+ Returns:
109
+ A lista de resultados por fonte.
110
+ """
111
+ with _make_client(timeout=timeout) as client:
112
+ if fail_fast:
113
+ results = []
114
+ for name, url in HEALTH_SOURCES:
115
+ result = _probe(client, name, url)
116
+ results.append(result)
117
+ if result["status"] != "ok":
118
+ break
119
+ return results
120
+
121
+ with concurrent.futures.ThreadPoolExecutor(
122
+ max_workers=len(HEALTH_SOURCES)
123
+ ) as pool:
124
+ futures = [
125
+ (name, pool.submit(_probe, client, name, url))
126
+ for name, url in HEALTH_SOURCES
127
+ ]
128
+ return [future.result() for _, future in futures]
129
+
130
+
131
+ def _render_table(results: list[dict[str, object]]) -> None:
132
+ """Renderiza o relatório de health como tabela Rich.
133
+
134
+ Args:
135
+ results: Resultados das sondas.
136
+ """
137
+ table = Table(title="Health — fontes de dados", show_header=True)
138
+ table.add_column("Fonte", style="cyan")
139
+ table.add_column("Status")
140
+ table.add_column("Código", justify="right")
141
+ table.add_column("Latência (ms)", justify="right")
142
+
143
+ for r in results:
144
+ if r["status"] == "ok":
145
+ status = "[green]OK[/green]"
146
+ else:
147
+ status = "[red]FALHA[/red]"
148
+ code = str(r["http_status"]) if r["http_status"] is not None else "-"
149
+ latency = str(r["latency_ms"]) if r["latency_ms"] is not None else "-"
150
+ table.add_row(str(r["name"]), status, code, latency)
151
+
152
+ console.print(table)
153
+
154
+
155
+ def cmd_health(
156
+ json_output: Annotated[
157
+ bool,
158
+ typer.Option(
159
+ "--json",
160
+ help="Saída em JSON estruturado em vez de tabela.",
161
+ ),
162
+ ] = False,
163
+ fail_fast: Annotated[
164
+ bool,
165
+ typer.Option(
166
+ "--fail-fast",
167
+ help="Interrompe as sondas na primeira falha e encerra com código 1.",
168
+ ),
169
+ ] = False,
170
+ timeout: Annotated[
171
+ float,
172
+ typer.Option(
173
+ "--timeout",
174
+ help="Timeout (s) aplicado a cada sonda HTTP.",
175
+ ),
176
+ ] = DEFAULT_TIMEOUT,
177
+ ) -> None:
178
+ """Verifica a disponibilidade das fontes de dados (sondas HTTP leves)."""
179
+ results = _run_probes(timeout, fail_fast)
180
+
181
+ if json_output:
182
+ status = "ok" if all(r["status"] == "ok" for r in results) else "degraded"
183
+ print(json.dumps({"status": status, "sources": results}))
184
+ else:
185
+ _render_table(results)
186
+
187
+ if fail_fast and any(r["status"] != "ok" for r in results):
188
+ raise typer.Exit(code=1)
@@ -10,6 +10,7 @@ import contextlib
10
10
  import datetime as dt
11
11
  import threading
12
12
  from collections.abc import Callable, Iterable
13
+ from dataclasses import dataclass, field
13
14
  from pathlib import Path
14
15
  from typing import Annotated, Any
15
16
 
@@ -23,8 +24,9 @@ from quantilica.core.storage import (
23
24
  build_stamped_filename,
24
25
  stamp_filename,
25
26
  )
26
- from rich.console import Group
27
+ from rich.console import Console, Group
27
28
  from rich.live import Live
29
+ from rich.rule import Rule
28
30
  from rich.table import Table
29
31
 
30
32
  from quantilica.cli.ui import (
@@ -38,6 +40,78 @@ from quantilica.cli.ui import (
38
40
 
39
41
  logger = get_logger(__name__)
40
42
 
43
+ __all__ = [
44
+ "DataRepository",
45
+ "FetcherApp",
46
+ "SyncPlan",
47
+ "SyncPlanItem",
48
+ "default_client",
49
+ "default_path_builder",
50
+ "make_resolve_groups",
51
+ ]
52
+
53
+
54
+ @dataclass(frozen=True)
55
+ class SyncPlanItem:
56
+ """A single dataset file planned for synchronization.
57
+
58
+ Args:
59
+ dataset: Canonical dataset (group) identifier.
60
+ partition: Human-readable partition label (e.g. ``2023-05``), or None.
61
+ filename: The stamped file name for the entry.
62
+ url: Remote URL for the resource.
63
+ target: Local destination path.
64
+ dataset_name: Optional human-readable dataset name.
65
+ """
66
+
67
+ dataset: str
68
+ partition: str | None
69
+ filename: str
70
+ url: str
71
+ target: Path
72
+ dataset_name: str = ""
73
+
74
+
75
+ @dataclass(frozen=True)
76
+ class SyncPlan:
77
+ """Preview plan for a synchronization run (``--dry-run``).
78
+
79
+ Args:
80
+ items: Planned :class:`SyncPlanItem` entries.
81
+ skipped: Number of entries ignored (outside of coverage).
82
+ """
83
+
84
+ items: list[SyncPlanItem] = field(default_factory=list)
85
+ skipped: int = 0
86
+
87
+ def render_table(self, console: Console | None = None) -> None:
88
+ """Render the plan as a Rich table with a summary footer.
89
+
90
+ Args:
91
+ console: Optional Rich Console to print to (defaults to the shared
92
+ console from :mod:`quantilica.cli.ui`).
93
+ """
94
+ con = console or get_console()
95
+ table = Table(
96
+ "Dataset",
97
+ "Partição",
98
+ "Arquivo",
99
+ "URL",
100
+ title="Arquivos a baixar (dry-run)",
101
+ )
102
+ for item in self.items:
103
+ table.add_row(
104
+ item.dataset,
105
+ item.partition or "—",
106
+ item.filename,
107
+ item.url,
108
+ )
109
+ con.print(table)
110
+ con.print(
111
+ f"Total: {len(self.items)} arquivos planejados. "
112
+ f"{self.skipped} ignorados fora de cobertura."
113
+ )
114
+
41
115
 
42
116
  def default_client() -> HttpClient:
43
117
  """Create a default HttpClient with standard configuration.
@@ -243,6 +317,211 @@ class FetcherApp:
243
317
  """
244
318
  self.app.command(name=name, **kwargs)(cmd_func)
245
319
 
320
+ def command_convert(
321
+ self, func: Callable[[Path, Path], Any]
322
+ ) -> Callable[[Path, Path], Any]:
323
+ """Register the standard ``convert`` subcommand.
324
+
325
+ The generated command exposes the canonical flags (``-i/--input``,
326
+ ``-o/--output``, ``--verbose``) and delegates to ``func(input, output)``.
327
+ An ``ImportError`` propagating from ``func`` is treated as missing
328
+ analytical extras and exits gracefully with code 1.
329
+
330
+ Args:
331
+ func: Callable receiving ``(input, output)`` paths and performing
332
+ the conversion.
333
+
334
+ Returns:
335
+ The original ``func`` (the method can be used as a decorator).
336
+ """
337
+ console = get_console()
338
+
339
+ @self.app.command("convert")
340
+ def convert(
341
+ input: Annotated[
342
+ Path,
343
+ typer.Option(
344
+ "-i",
345
+ "--input",
346
+ help="Diretório de origem com arquivos brutos",
347
+ ),
348
+ ] = self.default_output,
349
+ output: Annotated[
350
+ Path,
351
+ typer.Option("-o", "--output", help="Diretório de destino"),
352
+ ] = self.default_output,
353
+ verbose: Annotated[
354
+ bool, typer.Option("--verbose", help="Logs detalhados")
355
+ ] = False,
356
+ ) -> None:
357
+ setup_rich_logging(verbose, console=console)
358
+ try:
359
+ func(input, output)
360
+ except ImportError:
361
+ console.print(
362
+ f"[red]Erro:[/red] convert requer extras de análise: "
363
+ f"pip install {self.name}\[analysis]"
364
+ )
365
+ raise typer.Exit(1) from None
366
+ console.print(
367
+ f"[green]✓[/green] Conversão concluída em [dim]{output}[/dim]."
368
+ )
369
+
370
+ return func
371
+
372
+ def command_archive(
373
+ self, func: Callable[[Path, Path], Any]
374
+ ) -> Callable[[Path, Path], Any]:
375
+ """Register the standard ``archive`` subcommand.
376
+
377
+ Mirrors :meth:`command_convert` with the canonical flags
378
+ (``-i/--input``, ``-o/--output``, ``--verbose``) and delegates to
379
+ ``func(input, output)``. An ``ImportError`` propagating from ``func``
380
+ is treated as missing analytical extras and exits gracefully with
381
+ code 1.
382
+
383
+ Args:
384
+ func: Callable receiving ``(input, output)`` paths and creating
385
+ the historical archive.
386
+
387
+ Returns:
388
+ The original ``func`` (the method can be used as a decorator).
389
+ """
390
+ console = get_console()
391
+
392
+ @self.app.command("archive")
393
+ def archive(
394
+ input: Annotated[
395
+ Path,
396
+ typer.Option(
397
+ "-i",
398
+ "--input",
399
+ help="Diretório de dados a arquivar",
400
+ ),
401
+ ] = self.default_output,
402
+ output: Annotated[
403
+ Path,
404
+ typer.Option("-o", "--output", help="Diretório do arquivo histórico"),
405
+ ] = self.default_output,
406
+ verbose: Annotated[
407
+ bool, typer.Option("--verbose", help="Logs detalhados")
408
+ ] = False,
409
+ ) -> None:
410
+ setup_rich_logging(verbose, console=console)
411
+ try:
412
+ func(input, output)
413
+ except ImportError:
414
+ console.print(
415
+ f"[red]Erro:[/red] archive requer extras de análise: "
416
+ f"pip install {self.name}\[analysis]"
417
+ )
418
+ raise typer.Exit(1) from None
419
+ console.print(f"[green]✓[/green] Arquivo criado em [dim]{output}[/dim].")
420
+
421
+ return func
422
+
423
+ def command_pipeline(
424
+ self, convert_func: Callable[[Path, Path], Any]
425
+ ) -> Callable[[Path, Path], Any]:
426
+ """Register the standard ``pipeline`` subcommand (sync → convert).
427
+
428
+ Step 1/2 invokes the built-in ``sync`` command (same flags: groups,
429
+ ``-o/--output``, ``--parquet-dir``, ``--workers``, ``--dry-run``,
430
+ ``--verbose``). Step 2/2 invokes ``convert_func(output, parquet_dir)``.
431
+ With ``--dry-run`` the plan is only listed and the conversion step is
432
+ skipped. An ``ImportError`` from ``convert_func`` is treated as
433
+ missing analytical extras.
434
+
435
+ Args:
436
+ convert_func: Callable receiving ``(output, parquet_dir)`` paths
437
+ and performing the conversion.
438
+
439
+ Returns:
440
+ The original ``convert_func`` (usable as a decorator).
441
+ """
442
+ console = get_console()
443
+
444
+ @self.app.command("pipeline")
445
+ def pipeline(
446
+ ctx: typer.Context,
447
+ groups: Annotated[
448
+ list[str] | None,
449
+ typer.Argument(
450
+ help="Grupos a baixar. Use 'list' para ver grupos "
451
+ "disponíveis. Padrão: todos."
452
+ ),
453
+ ] = None,
454
+ output: Annotated[
455
+ Path | None,
456
+ typer.Option("-o", "--output", help="Diretório de saída"),
457
+ ] = None,
458
+ parquet_dir: Annotated[
459
+ Path | None,
460
+ typer.Option(
461
+ "--parquet-dir",
462
+ help="Diretório para os Parquet (padrão: igual a --output)",
463
+ ),
464
+ ] = None,
465
+ workers: Annotated[
466
+ int, typer.Option("--workers", help="Downloads paralelos")
467
+ ] = 4,
468
+ dry_run: Annotated[
469
+ bool, typer.Option("--dry-run", help="Listar arquivos sem baixar")
470
+ ] = False,
471
+ verbose: Annotated[
472
+ bool, typer.Option("--verbose", help="Logs detalhados")
473
+ ] = False,
474
+ ) -> None:
475
+ setup_rich_logging(verbose, console=console)
476
+ actual_output = output or self.default_output
477
+ parquet_out = parquet_dir or actual_output
478
+
479
+ console.print(Rule("[bold]Passo 1/2: Download[/bold]"))
480
+ sync_cmd = next(
481
+ (
482
+ cmd.callback
483
+ for cmd in self.app.registered_commands
484
+ if cmd.name == "sync"
485
+ ),
486
+ None,
487
+ )
488
+ if sync_cmd is None:
489
+ console.print(
490
+ "[red]Erro:[/red] pipeline requer o comando 'sync' "
491
+ "(build_default_commands=True)."
492
+ )
493
+ raise typer.Exit(1) from None
494
+ ctx.invoke(
495
+ sync_cmd,
496
+ groups=groups,
497
+ output=actual_output,
498
+ dry_run=dry_run,
499
+ workers=workers,
500
+ verbose=verbose,
501
+ )
502
+
503
+ if dry_run:
504
+ console.print(
505
+ "\n[yellow]Dry-run:[/yellow] conversão (passo 2/2) não executada."
506
+ )
507
+ return
508
+
509
+ console.print(Rule("[bold]Passo 2/2: Conversão[/bold]"))
510
+ try:
511
+ convert_func(actual_output, parquet_out)
512
+ except ImportError:
513
+ console.print(
514
+ f"[red]Erro:[/red] pipeline (conversão) requer extras de "
515
+ f"análise: pip install {self.name}\[analysis]"
516
+ )
517
+ raise typer.Exit(1) from None
518
+ console.print(
519
+ f"[green]✓[/green] Pipeline concluído: Parquet em "
520
+ f"[dim]{parquet_out}[/dim]."
521
+ )
522
+
523
+ return convert_func
524
+
246
525
  def _safe_head_date(self, url: str) -> dt.date | None:
247
526
  with contextlib.suppress(Exception):
248
527
  return self.client.head_last_modified_date(url)
@@ -30,6 +30,7 @@ SOURCES_REGISTRY: dict[str, str] = {
30
30
  "anp": "anp-fetcher",
31
31
  "bcb-sgs": "bcb-sgs-fetcher",
32
32
  "comex": "comex-fetcher",
33
+ "cvm": "cvm-fetcher",
33
34
  "datasus": "datasus-fetcher",
34
35
  "inmet": "inmet-fetcher",
35
36
  "pdet": "pdet-fetcher",
@@ -0,0 +1,208 @@
1
+ """Testes para o comando `quantilica health` (sondas de disponibilidade)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from typing import Any
7
+ from unittest.mock import patch
8
+
9
+ import httpx2
10
+ from typer.testing import CliRunner
11
+
12
+ from quantilica.cli import health
13
+ from quantilica.cli.cli import app
14
+ from quantilica.cli.health import HEALTH_SOURCES
15
+
16
+ runner = CliRunner()
17
+
18
+ SOURCE_NAMES = [name for name, _ in HEALTH_SOURCES]
19
+ URLS = {url: name for name, url in HEALTH_SOURCES}
20
+
21
+
22
+ def _handler(status_by_url: dict[str, int], head_405: bool = False):
23
+ """Cria um handler de MockTransport mapeando URL -> status HTTP."""
24
+
25
+ def handle(request: httpx2.Request) -> httpx2.Response:
26
+ url = str(request.url).split("?")[0]
27
+ status = status_by_url.get(url, 200)
28
+ if request.method == "HEAD" and head_405:
29
+ return httpx2.Response(405)
30
+ return httpx2.Response(status)
31
+
32
+ return handle
33
+
34
+
35
+ def _patch_client(
36
+ status_by_url: dict[str, int],
37
+ head_405: bool = False,
38
+ ):
39
+ """Parchea o cliente de sondas com um MockTransport determinístico."""
40
+ transport = httpx2.MockTransport(_handler(status_by_url, head_405))
41
+ fake_client = httpx2.Client(transport=transport, follow_redirects=True)
42
+ return patch.object(health, "_make_client", return_value=fake_client)
43
+
44
+
45
+ def _all_ok() -> dict[str, int]:
46
+ return {url: 200 for url in URLS}
47
+
48
+
49
+ def _parse_json(output: str) -> dict[str, Any]:
50
+ lines = [line for line in output.splitlines() if line.startswith("{")]
51
+ assert lines, f"nenhuma linha JSON na saída: {output!r}"
52
+ return json.loads(lines[-1])
53
+
54
+
55
+ # --- modo tabela -----------------------------------------------------------
56
+
57
+
58
+ def test_health_table_all_ok() -> None:
59
+ with _patch_client(_all_ok()):
60
+ result = runner.invoke(app, ["health"])
61
+ assert result.exit_code == 0
62
+ for name in SOURCE_NAMES:
63
+ assert name in result.output
64
+ assert "OK" in result.output
65
+ assert "FALHA" not in result.output
66
+
67
+
68
+ def test_health_table_with_failure() -> None:
69
+ statuses = _all_ok()
70
+ statuses["https://balanca.economia.gov.br"] = 500
71
+ with _patch_client(statuses):
72
+ result = runner.invoke(app, ["health"])
73
+ assert result.exit_code == 0
74
+ assert "FALHA" in result.output
75
+ assert "OK" in result.output
76
+
77
+
78
+ def test_health_head_fallback_to_get() -> None:
79
+ # HEAD retorna 405 e o CLI deve refazer com GET (levemente diferente).
80
+ handlers: list[str] = []
81
+
82
+ def handle(request: httpx2.Request) -> httpx2.Response:
83
+ handlers.append(request.method)
84
+ if request.method == "HEAD":
85
+ return httpx2.Response(405)
86
+ return httpx2.Response(200)
87
+
88
+ transport = httpx2.MockTransport(handle)
89
+ with patch.object(
90
+ health, "_make_client", return_value=httpx2.Client(transport=transport)
91
+ ):
92
+ result = runner.invoke(app, ["health"])
93
+ assert result.exit_code == 0
94
+ assert "OK" in result.output
95
+ assert "HEAD" in handlers
96
+ assert "GET" in handlers
97
+
98
+
99
+ # --- modo --json -------------------------------------------------------------
100
+
101
+
102
+ def test_health_json_all_ok() -> None:
103
+ with _patch_client(_all_ok()):
104
+ result = runner.invoke(app, ["health", "--json"])
105
+ assert result.exit_code == 0
106
+ payload = _parse_json(result.output)
107
+ assert payload["status"] == "ok"
108
+ source_names = [s["name"] for s in payload["sources"]]
109
+ assert source_names == SOURCE_NAMES
110
+ for source in payload["sources"]:
111
+ assert source["status"] == "ok"
112
+ assert source["http_status"] == 200
113
+ assert isinstance(source["latency_ms"], (int, float))
114
+ assert source["error"] is None
115
+
116
+
117
+ def test_health_json_degraded() -> None:
118
+ statuses = _all_ok()
119
+ statuses["https://balanca.economia.gov.br"] = 503
120
+ with _patch_client(statuses):
121
+ result = runner.invoke(app, ["health", "--json"])
122
+ assert result.exit_code == 0
123
+ payload = _parse_json(result.output)
124
+ assert payload["status"] == "degraded"
125
+ comex = next(s for s in payload["sources"] if s["name"] == "comex")
126
+ assert comex["status"] == "falha"
127
+ assert comex["http_status"] == 503
128
+ assert comex["error"] == "HTTP 503"
129
+
130
+
131
+ def test_health_json_network_error() -> None:
132
+ def handler(request: httpx2.Request) -> httpx2.Response:
133
+ if "inmet" in str(request.url):
134
+ raise httpx2.ConnectError("boom")
135
+ return httpx2.Response(200)
136
+
137
+ transport = httpx2.MockTransport(handler)
138
+ with patch.object(
139
+ health, "_make_client", return_value=httpx2.Client(transport=transport)
140
+ ):
141
+ result = runner.invoke(app, ["health", "--json"])
142
+ assert result.exit_code == 0
143
+ payload = _parse_json(result.output)
144
+ assert payload["status"] == "degraded"
145
+ inmet = next(s for s in payload["sources"] if s["name"] == "inmet")
146
+ assert inmet["status"] == "falha"
147
+ assert inmet["http_status"] is None
148
+ assert "ConnectError" in inmet["error"]
149
+
150
+
151
+ # --- --fail-fast ------------------------------------------------------------
152
+
153
+
154
+ def test_health_fail_fast_exit_code_ok() -> None:
155
+ with _patch_client(_all_ok()):
156
+ result = runner.invoke(app, ["health", "--fail-fast", "--json"])
157
+ assert result.exit_code == 0
158
+ payload = _parse_json(result.output)
159
+ assert payload["status"] == "ok"
160
+
161
+
162
+ def test_health_fail_fast_exit_code_on_failure() -> None:
163
+ statuses = _all_ok()
164
+ statuses["https://balanca.economia.gov.br"] = 500
165
+ with _patch_client(statuses):
166
+ result = runner.invoke(app, ["health", "--fail-fast"])
167
+ assert result.exit_code == 1
168
+
169
+
170
+ def test_health_fail_fast_stops_probes() -> None:
171
+ # com a primeira fonte falhando em fail-fast, as demais não são sondadas.
172
+ probes: list[str] = []
173
+
174
+ def fake_probe(client: httpx2.Client, name: str, url: str) -> dict[str, object]:
175
+ probes.append(name)
176
+ status = "falha" if name == "bcb" else "ok"
177
+ return {
178
+ "name": name,
179
+ "url": url,
180
+ "status": status,
181
+ "http_status": 500 if status == "falha" else 200,
182
+ "latency_ms": 1.0,
183
+ "error": None if status == "ok" else "HTTP 500",
184
+ }
185
+
186
+ with (
187
+ _patch_client(_all_ok()),
188
+ patch.object(health, "_probe", side_effect=fake_probe),
189
+ ):
190
+ result = runner.invoke(app, ["health", "--fail-fast"])
191
+ assert result.exit_code == 1
192
+ assert probes == ["bcb"]
193
+
194
+
195
+ # --- --timeout ------------------------------------------------------------
196
+
197
+
198
+ def test_health_passes_timeout_to_client() -> None:
199
+ with patch.object(
200
+ health, "_make_client", return_value=httpx2.Client()
201
+ ) as mock_client:
202
+ result = runner.invoke(app, ["health", "--timeout", "2.5", "--json"])
203
+ assert result.exit_code == 0
204
+ mock_client.assert_called_once_with(timeout=2.5)
205
+
206
+
207
+ def test_health_sources_canonical_list() -> None:
208
+ assert SOURCE_NAMES == ["bcb", "sidra", "tesouro_direto", "comex", "inmet"]
@@ -0,0 +1,295 @@
1
+ """Testes unitários para Onda A.2 do SDK (command_convert/pipeline/archive, SyncPlan)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+
7
+ import pytest
8
+ import typer
9
+ from rich.console import Console
10
+ from typer.testing import CliRunner
11
+
12
+ from quantilica.cli.sdk import SyncPlan, SyncPlanItem
13
+ from tests.test_sdk import _grouped, make_app
14
+
15
+ runner = CliRunner()
16
+
17
+
18
+ # ---------------------------------------------------------------
19
+ # SyncPlan / SyncPlanItem
20
+ # ---------------------------------------------------------------
21
+
22
+
23
+ def _sample_item(
24
+ dataset: str = "exp", partition: str | None = "2023-05"
25
+ ) -> SyncPlanItem:
26
+ return SyncPlanItem(
27
+ dataset=dataset,
28
+ partition=partition,
29
+ filename=f"{dataset}-monthly_2023-05.csv",
30
+ url=f"https://example.test/{dataset}-monthly.csv",
31
+ target=Path(f"/data/output/{dataset}/{dataset}-monthly_2023-05.csv"),
32
+ dataset_name="Exportações",
33
+ )
34
+
35
+
36
+ def test_sync_plan_item_is_frozen():
37
+ item = _sample_item()
38
+ with pytest.raises(Exception): # noqa: B017 (FrozenInstanceError levantada dinamicamente)
39
+ item.dataset = "imp" # type: ignore[misc]
40
+
41
+
42
+ def test_sync_plan_is_frozen():
43
+ plan = SyncPlan(items=[_sample_item()])
44
+ with pytest.raises(Exception): # noqa: B017 (FrozenInstanceError levantada dinamicamente)
45
+ plan.skipped = 5 # type: ignore[misc]
46
+
47
+
48
+ def test_sync_plan_item_defaults():
49
+ item = _sample_item()
50
+ assert item.dataset_name == "Exportações"
51
+ plain = SyncPlanItem(
52
+ dataset="d", partition=None, filename="f", url="u", target=Path("t")
53
+ )
54
+ assert plain.dataset_name == ""
55
+
56
+
57
+ def test_sync_plan_defaults():
58
+ plan = SyncPlan()
59
+ assert plan.items == []
60
+ assert plan.skipped == 0
61
+
62
+
63
+ def test_render_table_prints_columns_and_rows(capsys: pytest.CaptureFixture[str]):
64
+ plan = SyncPlan(
65
+ items=[_sample_item("exp", "2023-05"), _sample_item("imp", None)], skipped=3
66
+ )
67
+ console = Console(record=True, width=120, legacy_windows=False)
68
+ plan.render_table(console=console)
69
+ output = console.export_text()
70
+ assert "exp" in output
71
+ assert "exp-monthly_2023-05.csv" in output
72
+ assert "https://example.test/exp-monthly.csv" in output
73
+ # partição ausente é exibida como "—"
74
+ assert "—" in output
75
+ assert "Total: 2 arquivos planejados. 3 ignorados fora de cobertura." in output
76
+
77
+
78
+ def test_render_table_uses_shared_console_when_none(capsys: pytest.CaptureFixture[str]):
79
+ plan = SyncPlan(items=[_sample_item()], skipped=1)
80
+ plan.render_table()
81
+ output = capsys.readouterr().out
82
+ assert "Total: 1 arquivos planejados. 1 ignorados fora de cobertura." in output
83
+
84
+
85
+ # ---------------------------------------------------------------
86
+ # command_convert
87
+ # ---------------------------------------------------------------
88
+
89
+
90
+ def test_command_convert_success(tmp_path: Path):
91
+ app = make_app()
92
+ calls: list[tuple[Path, Path]] = []
93
+
94
+ @app.command_convert
95
+ def conv(input: Path, output: Path) -> None:
96
+ calls.append((input, output))
97
+
98
+ result = runner.invoke(
99
+ _grouped(app.app), ["fd", "convert", "-i", str(tmp_path), "-o", str(tmp_path)]
100
+ )
101
+ assert result.exit_code == 0, result.output
102
+ assert calls == [(tmp_path, tmp_path)]
103
+ assert "Conversão concluída" in result.output
104
+ assert "✓" in result.output
105
+
106
+
107
+ def test_command_convert_defaults_to_default_output(tmp_path: Path):
108
+ app = make_app()
109
+ calls: list[tuple[Path, Path]] = []
110
+
111
+ @app.command_convert
112
+ def conv(input: Path, output: Path) -> None:
113
+ calls.append((input, output))
114
+
115
+ default = app.default_output
116
+ result = runner.invoke(_grouped(app.app), ["fd", "convert"])
117
+ assert result.exit_code == 0, result.output
118
+ assert calls == [(default, default)]
119
+
120
+
121
+ def test_command_convert_graceful_import_error(tmp_path: Path):
122
+ app = make_app()
123
+
124
+ @app.command_convert
125
+ def conv(input: Path, output: Path) -> None:
126
+ raise ImportError("No module named 'polars'")
127
+
128
+ result = runner.invoke(_grouped(app.app), ["fd", "convert", "-i", str(tmp_path)])
129
+ assert result.exit_code == 1
130
+ assert "convert requer extras de análise" in result.output
131
+ assert "pip install comex-fetcher[analysis]" in result.output
132
+
133
+
134
+ def test_command_convert_verbose_sets_logging(tmp_path: Path):
135
+ app = make_app()
136
+
137
+ @app.command_convert
138
+ def conv(input: Path, output: Path) -> None:
139
+ raise typer.Exit(3)
140
+
141
+ result = runner.invoke(_grouped(app.app), ["fd", "convert", "--verbose"])
142
+ assert result.exit_code == 3
143
+
144
+
145
+ # ---------------------------------------------------------------
146
+ # command_pipeline
147
+ # ---------------------------------------------------------------
148
+
149
+
150
+ def test_command_pipeline_runs_both_steps(
151
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
152
+ ):
153
+ app = make_app()
154
+ calls: list[tuple[Path, Path]] = []
155
+ sync_calls: list[tuple[list, Path, int]] = []
156
+
157
+ def fake_download(entries, output_dir, workers=4):
158
+ sync_calls.append((entries, output_dir, workers))
159
+ return len(entries), len(entries), []
160
+
161
+ monkeypatch.setattr(app, "download_datasets", fake_download)
162
+
163
+ @app.command_pipeline
164
+ def conv(data_dir: Path, parquet_dir: Path) -> None:
165
+ calls.append((data_dir, parquet_dir))
166
+
167
+ raw = tmp_path / "raw"
168
+ parquet = tmp_path / "parquet"
169
+ result = runner.invoke(
170
+ _grouped(app.app),
171
+ ["fd", "pipeline", "exp", "-o", str(raw), "--parquet-dir", str(parquet)],
172
+ )
173
+ assert result.exit_code == 0, result.output
174
+ assert "Passo 1/2: Download" in result.output
175
+ assert "Passo 2/2: Conversão" in result.output
176
+ # Passo 1 invocou o sync (download_datasets) uma vez
177
+ assert len(sync_calls) == 1
178
+ entries, out_dir, workers = sync_calls[0]
179
+ assert out_dir == raw
180
+ assert [e["id"] for e in entries] == ["exp-monthly"]
181
+ # Passo 2 invocou a convert_func com (output, parquet_dir)
182
+ assert calls == [(raw, parquet)]
183
+ assert "✓" in result.output
184
+
185
+
186
+ def test_command_pipeline_dry_run_skips_conversion(
187
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
188
+ ):
189
+ app = make_app()
190
+ downloads: list[tuple] = []
191
+ monkeypatch.setattr(
192
+ app,
193
+ "download_datasets",
194
+ lambda entries, output_dir, workers=4: (
195
+ downloads.append(output_dir) or (0, 0, [])
196
+ ),
197
+ )
198
+ converted: list[Path] = []
199
+
200
+ @app.command_pipeline
201
+ def conv(data_dir: Path, parquet_dir: Path) -> None:
202
+ converted.append(parquet_dir)
203
+
204
+ result = runner.invoke(_grouped(app.app), ["fd", "pipeline", "exp", "--dry-run"])
205
+ assert result.exit_code == 0, result.output
206
+ assert "Passo 1/2: Download" in result.output
207
+ assert "Passo 2/2: Conversão" not in result.output
208
+ assert converted == []
209
+ # dry-run: nenhum download executado (download_datasets não foi invocado)
210
+ assert downloads == []
211
+
212
+
213
+ def test_command_pipeline_graceful_import_error(
214
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
215
+ ):
216
+ app = make_app()
217
+ monkeypatch.setattr(
218
+ app, "download_datasets", lambda entries, output_dir, workers=4: (0, 0, [])
219
+ )
220
+
221
+ @app.command_pipeline
222
+ def conv(data_dir: Path, parquet_dir: Path) -> None:
223
+ raise ImportError("No module named 'polars'")
224
+
225
+ result = runner.invoke(
226
+ _grouped(app.app), ["fd", "pipeline", "exp", "-o", str(tmp_path)]
227
+ )
228
+ assert result.exit_code == 1
229
+ assert "pipeline (conversão) requer extras de análise" in result.output
230
+ assert "comex-fetcher[analysis]" in result.output
231
+
232
+
233
+ def test_command_pipeline_parquet_dir_defaults_to_output(
234
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
235
+ ):
236
+ app = make_app()
237
+ monkeypatch.setattr(
238
+ app, "download_datasets", lambda entries, output_dir, workers=4: (0, 0, [])
239
+ )
240
+ calls: list[tuple[Path, Path]] = []
241
+
242
+ @app.command_pipeline
243
+ def conv(data_dir: Path, parquet_dir: Path) -> None:
244
+ calls.append((data_dir, parquet_dir))
245
+
246
+ raw = tmp_path / "raw"
247
+ result = runner.invoke(_grouped(app.app), ["fd", "pipeline", "exp", "-o", str(raw)])
248
+ assert result.exit_code == 0, result.output
249
+ assert calls == [(raw, raw)]
250
+
251
+
252
+ def test_command_pipeline_requires_sync_command():
253
+ app = make_app(build_default_commands=False)
254
+
255
+ @app.command_pipeline
256
+ def conv(data_dir: Path, parquet_dir: Path) -> None:
257
+ raise AssertionError("não deveria ser chamado")
258
+
259
+ result = runner.invoke(_grouped(app.app), ["fd", "pipeline", "exp"])
260
+ assert result.exit_code == 1
261
+ assert "pipeline requer o comando 'sync'" in result.output
262
+
263
+
264
+ # ---------------------------------------------------------------
265
+ # command_archive
266
+ # ---------------------------------------------------------------
267
+
268
+
269
+ def test_command_archive_success(tmp_path: Path):
270
+ app = make_app()
271
+ calls: list[tuple[Path, Path]] = []
272
+
273
+ @app.command_archive
274
+ def arch(input: Path, output: Path) -> None:
275
+ calls.append((input, output))
276
+
277
+ result = runner.invoke(
278
+ _grouped(app.app), ["fd", "archive", "-i", str(tmp_path), "-o", str(tmp_path)]
279
+ )
280
+ assert result.exit_code == 0, result.output
281
+ assert calls == [(tmp_path, tmp_path)]
282
+ assert "Arquivo criado" in result.output
283
+
284
+
285
+ def test_command_archive_graceful_import_error(tmp_path: Path):
286
+ app = make_app()
287
+
288
+ @app.command_archive
289
+ def arch(input: Path, output: Path) -> None:
290
+ raise ImportError("No module named 'some_extra'")
291
+
292
+ result = runner.invoke(_grouped(app.app), ["fd", "archive", "-o", str(tmp_path)])
293
+ assert result.exit_code == 1
294
+ assert "archive requer extras de análise" in result.output
295
+ assert "pip install comex-fetcher[analysis]" in result.output
File without changes
File without changes