quantilica-cli 0.7.0__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/CHANGELOG.md +40 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/PKG-INFO +1 -1
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/pyproject.toml +1 -1
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/cli.py +3 -1
- quantilica_cli-0.9.0/src/quantilica/cli/health.py +188 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/sdk.py +280 -1
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/sources.py +1 -0
- quantilica_cli-0.9.0/tests/test_health.py +208 -0
- quantilica_cli-0.9.0/tests/test_sdk_onda_a2.py +295 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/.githooks/pre-commit +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/.githooks/pre-push +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/.github/workflows/publish.yml +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/.github/workflows/test.yml +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/.gitignore +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/LICENSE +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/README.md +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/__init__.py +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/manifests.py +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/progress.py +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/cli/ui.py +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/src/quantilica/py.typed +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/tests/__init__.py +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/tests/test_manifests.py +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/tests/test_sdk.py +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/tests/test_sources.py +0 -0
- {quantilica_cli-0.7.0 → quantilica_cli-0.9.0}/tests/test_ui.py +0 -0
|
@@ -5,6 +5,46 @@ Todas as mudanças notáveis deste projeto serão documentadas neste arquivo.
|
|
|
5
5
|
O formato segue [Keep a Changelog](https://keepachangelog.com/pt-BR/1.1.0/),
|
|
6
6
|
e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
|
|
7
7
|
|
|
8
|
+
## [0.9.0] - 2026-10-04
|
|
9
|
+
|
|
10
|
+
Registra o `cvm-fetcher` como fonte instalável sob demanda, após a v0.1.0 do
|
|
11
|
+
fetcher (wheel no GitHub Releases, entrada no índice PEP 503 do ecossistema).
|
|
12
|
+
|
|
13
|
+
### Adicionado
|
|
14
|
+
- Fonte `cvm` em `SOURCES_REGISTRY` (`quantilica install cvm`), apontando
|
|
15
|
+
para a distribuição `cvm-fetcher`.
|
|
16
|
+
|
|
17
|
+
## [0.8.0] - 2026-10-03
|
|
18
|
+
|
|
19
|
+
Onda A.2 do plano `2026-10-03-padronizacao-core-e-consolidacao-fetchers` —
|
|
20
|
+
decorators de ciclo de vida no SDK (`quantilica.cli.sdk`) e abstração de
|
|
21
|
+
pré-visualização tabular de sincronização.
|
|
22
|
+
|
|
23
|
+
### Adicionado
|
|
24
|
+
- `FetcherApp.command_convert(func)`: decorator que registra o subcomando
|
|
25
|
+
`convert` com flags canônicas (`-i/--input`, `-o/--output`, `--verbose`,
|
|
26
|
+
padrões derivados de `default_output`). Configura logging Rich, trata
|
|
27
|
+
`ImportError` graciosamente (sugerindo `pip install {nome}[analysis]`,
|
|
28
|
+
saída com código 1) e exibe confirmação com check verde.
|
|
29
|
+
- `FetcherApp.command_pipeline(func)`: decorator que registra o subcomando
|
|
30
|
+
`pipeline` encadeando sincronização (passo 1/2, via `sync`) e conversão
|
|
31
|
+
analítica (passo 2/2, via `func`). Opções: grupos, `--output`,
|
|
32
|
+
`--parquet-dir`, `--workers`, `--dry-run` (interrompe antes do passo 2) e
|
|
33
|
+
`--verbose`.
|
|
34
|
+
- `FetcherApp.command_archive(func)`: decorator que registra o subcomando
|
|
35
|
+
`archive` para arquivamento histórico, com o mesmo padrão de flags e
|
|
36
|
+
tratamento gracioso de `ImportError`.
|
|
37
|
+
- `SyncPlanItem`/`SyncPlan`: dataclasses (imutáveis) para planejamento de
|
|
38
|
+
sincronização, com `SyncPlan.render_table(console=None)` que renderiza uma
|
|
39
|
+
tabela Rich (`Dataset | Partição | Arquivo | URL`) e o sumário `Total: X
|
|
40
|
+
arquivos planejados. Y ignorados fora de cobertura.` Importados do
|
|
41
|
+
`quantilica.cli.sdk`.
|
|
42
|
+
- `quantilica health`: subcomando de diagnóstico que faz sondas HTTP leves
|
|
43
|
+
(HEAD com fallback para GET, timeout padrão 5s) contra as fontes canônicas
|
|
44
|
+
do ecossistema — BCB SGS, SIDRA, Tesouro Direto, Comex e INMET — em
|
|
45
|
+
paralelo, com saída em tabela Rich (`Fonte | Estado | Latência | HTTP`) ou
|
|
46
|
+
em JSON estruturado via `--json`. Usa `quantilica.core.http` para o probe.
|
|
47
|
+
|
|
8
48
|
## [0.7.0] - 2026-10-02
|
|
9
49
|
|
|
10
50
|
Onda 2 — extensões do SDK (`quantilica.cli.sdk`) para eliminação de
|
|
@@ -13,6 +13,7 @@ from rich.logging import RichHandler
|
|
|
13
13
|
from rich.table import Table
|
|
14
14
|
|
|
15
15
|
from quantilica.cli import __version__
|
|
16
|
+
from quantilica.cli.health import cmd_health
|
|
16
17
|
from quantilica.cli.manifests import app as manifests_app
|
|
17
18
|
from quantilica.cli.sources import (
|
|
18
19
|
app as sources_app,
|
|
@@ -35,10 +36,11 @@ app = typer.Typer(
|
|
|
35
36
|
app.add_typer(manifests_app, name="manifests")
|
|
36
37
|
app.add_typer(sources_app, name="sources")
|
|
37
38
|
|
|
38
|
-
# Adiciona comandos top-level install, uninstall e
|
|
39
|
+
# Adiciona comandos top-level install, uninstall, doctor e health
|
|
39
40
|
app.command("install")(cmd_install)
|
|
40
41
|
app.command("uninstall")(cmd_uninstall)
|
|
41
42
|
app.command("doctor")(cmd_doctor)
|
|
43
|
+
app.command("health")(cmd_health)
|
|
42
44
|
|
|
43
45
|
console = Console()
|
|
44
46
|
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Comando `quantilica health` — sondas de disponibilidade das fontes.
|
|
2
|
+
|
|
3
|
+
Realiza sondas HTTP leves (HEAD com fallback para GET) contra as fontes
|
|
4
|
+
canônicas de dados do ecossistema Quantilica e reporta o resultado em
|
|
5
|
+
tabela Rich ou JSON estruturado.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import concurrent.futures
|
|
11
|
+
import json
|
|
12
|
+
import time
|
|
13
|
+
from typing import Annotated
|
|
14
|
+
|
|
15
|
+
import httpx2
|
|
16
|
+
import typer
|
|
17
|
+
from rich.console import Console
|
|
18
|
+
from rich.table import Table
|
|
19
|
+
|
|
20
|
+
console = Console()
|
|
21
|
+
|
|
22
|
+
DEFAULT_TIMEOUT = 5.0
|
|
23
|
+
USER_AGENT = "quantilica-cli (health)"
|
|
24
|
+
|
|
25
|
+
# Lista canônica de fontes sondadas (nome, URL de sonda leve).
|
|
26
|
+
HEALTH_SOURCES: list[tuple[str, str]] = [
|
|
27
|
+
("bcb", "https://api.bcb.gov.br/dados/serie/bcdata.sgs.1/dados?formato=json"),
|
|
28
|
+
("sidra", "https://servicodados.ibge.gov.br/api/v3/calendario"),
|
|
29
|
+
(
|
|
30
|
+
"tesouro_direto",
|
|
31
|
+
"https://www.tesourotransparente.gov.br/ckan/api/3/action/package_search?rows=1",
|
|
32
|
+
),
|
|
33
|
+
("comex", "https://balanca.economia.gov.br"),
|
|
34
|
+
("inmet", "https://apitempo.inmet.gov.br/estacoes"),
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _make_client(timeout: float) -> httpx2.Client:
|
|
39
|
+
"""Cria o cliente HTTP usado pelas sondas.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
timeout: Timeout (segundos) aplicado a cada requisição.
|
|
43
|
+
|
|
44
|
+
Returns:
|
|
45
|
+
Um httpx2.Client configurado com redirects e User-Agent da CLI.
|
|
46
|
+
"""
|
|
47
|
+
return httpx2.Client(
|
|
48
|
+
timeout=timeout,
|
|
49
|
+
follow_redirects=True,
|
|
50
|
+
headers={"User-Agent": USER_AGENT},
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _probe(client: httpx2.Client, name: str, url: str) -> dict[str, object]:
|
|
55
|
+
"""Executa uma sonda HEAD (com fallback GET) contra uma fonte.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
client: O cliente HTTP a ser usado.
|
|
59
|
+
name: Nome canônico da fonte.
|
|
60
|
+
url: URL alvo da sonda.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
Dicioário com name, url, status, http_status, latency_ms e error.
|
|
64
|
+
"""
|
|
65
|
+
start = time.perf_counter()
|
|
66
|
+
try:
|
|
67
|
+
response = client.head(url)
|
|
68
|
+
# Muitos endpoints não implementam HEAD — fallback para GET leve.
|
|
69
|
+
if response.status_code >= 400:
|
|
70
|
+
response = client.get(url)
|
|
71
|
+
latency_ms = (time.perf_counter() - start) * 1000.0
|
|
72
|
+
if response.status_code < 400:
|
|
73
|
+
return {
|
|
74
|
+
"name": name,
|
|
75
|
+
"url": url,
|
|
76
|
+
"status": "ok",
|
|
77
|
+
"http_status": response.status_code,
|
|
78
|
+
"latency_ms": round(latency_ms, 1),
|
|
79
|
+
"error": None,
|
|
80
|
+
}
|
|
81
|
+
return {
|
|
82
|
+
"name": name,
|
|
83
|
+
"url": url,
|
|
84
|
+
"status": "falha",
|
|
85
|
+
"http_status": response.status_code,
|
|
86
|
+
"latency_ms": round(latency_ms, 1),
|
|
87
|
+
"error": f"HTTP {response.status_code}",
|
|
88
|
+
}
|
|
89
|
+
except Exception as exc:
|
|
90
|
+
latency_ms = (time.perf_counter() - start) * 1000.0
|
|
91
|
+
return {
|
|
92
|
+
"name": name,
|
|
93
|
+
"url": url,
|
|
94
|
+
"status": "falha",
|
|
95
|
+
"http_status": None,
|
|
96
|
+
"latency_ms": round(latency_ms, 1),
|
|
97
|
+
"error": f"{type(exc).__name__}: {exc}",
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _run_probes(timeout: float, fail_fast: bool) -> list[dict[str, object]]:
|
|
102
|
+
"""Sonda todas as fontes, em sequência (fail-fast) ou em paralelo.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
timeout: Timeout por requisição, em segundos.
|
|
106
|
+
fail_fast: Se True, interrompe as sondas na primeira falha.
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
A lista de resultados por fonte.
|
|
110
|
+
"""
|
|
111
|
+
with _make_client(timeout=timeout) as client:
|
|
112
|
+
if fail_fast:
|
|
113
|
+
results = []
|
|
114
|
+
for name, url in HEALTH_SOURCES:
|
|
115
|
+
result = _probe(client, name, url)
|
|
116
|
+
results.append(result)
|
|
117
|
+
if result["status"] != "ok":
|
|
118
|
+
break
|
|
119
|
+
return results
|
|
120
|
+
|
|
121
|
+
with concurrent.futures.ThreadPoolExecutor(
|
|
122
|
+
max_workers=len(HEALTH_SOURCES)
|
|
123
|
+
) as pool:
|
|
124
|
+
futures = [
|
|
125
|
+
(name, pool.submit(_probe, client, name, url))
|
|
126
|
+
for name, url in HEALTH_SOURCES
|
|
127
|
+
]
|
|
128
|
+
return [future.result() for _, future in futures]
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _render_table(results: list[dict[str, object]]) -> None:
|
|
132
|
+
"""Renderiza o relatório de health como tabela Rich.
|
|
133
|
+
|
|
134
|
+
Args:
|
|
135
|
+
results: Resultados das sondas.
|
|
136
|
+
"""
|
|
137
|
+
table = Table(title="Health — fontes de dados", show_header=True)
|
|
138
|
+
table.add_column("Fonte", style="cyan")
|
|
139
|
+
table.add_column("Status")
|
|
140
|
+
table.add_column("Código", justify="right")
|
|
141
|
+
table.add_column("Latência (ms)", justify="right")
|
|
142
|
+
|
|
143
|
+
for r in results:
|
|
144
|
+
if r["status"] == "ok":
|
|
145
|
+
status = "[green]OK[/green]"
|
|
146
|
+
else:
|
|
147
|
+
status = "[red]FALHA[/red]"
|
|
148
|
+
code = str(r["http_status"]) if r["http_status"] is not None else "-"
|
|
149
|
+
latency = str(r["latency_ms"]) if r["latency_ms"] is not None else "-"
|
|
150
|
+
table.add_row(str(r["name"]), status, code, latency)
|
|
151
|
+
|
|
152
|
+
console.print(table)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def cmd_health(
|
|
156
|
+
json_output: Annotated[
|
|
157
|
+
bool,
|
|
158
|
+
typer.Option(
|
|
159
|
+
"--json",
|
|
160
|
+
help="Saída em JSON estruturado em vez de tabela.",
|
|
161
|
+
),
|
|
162
|
+
] = False,
|
|
163
|
+
fail_fast: Annotated[
|
|
164
|
+
bool,
|
|
165
|
+
typer.Option(
|
|
166
|
+
"--fail-fast",
|
|
167
|
+
help="Interrompe as sondas na primeira falha e encerra com código 1.",
|
|
168
|
+
),
|
|
169
|
+
] = False,
|
|
170
|
+
timeout: Annotated[
|
|
171
|
+
float,
|
|
172
|
+
typer.Option(
|
|
173
|
+
"--timeout",
|
|
174
|
+
help="Timeout (s) aplicado a cada sonda HTTP.",
|
|
175
|
+
),
|
|
176
|
+
] = DEFAULT_TIMEOUT,
|
|
177
|
+
) -> None:
|
|
178
|
+
"""Verifica a disponibilidade das fontes de dados (sondas HTTP leves)."""
|
|
179
|
+
results = _run_probes(timeout, fail_fast)
|
|
180
|
+
|
|
181
|
+
if json_output:
|
|
182
|
+
status = "ok" if all(r["status"] == "ok" for r in results) else "degraded"
|
|
183
|
+
print(json.dumps({"status": status, "sources": results}))
|
|
184
|
+
else:
|
|
185
|
+
_render_table(results)
|
|
186
|
+
|
|
187
|
+
if fail_fast and any(r["status"] != "ok" for r in results):
|
|
188
|
+
raise typer.Exit(code=1)
|
|
@@ -10,6 +10,7 @@ import contextlib
|
|
|
10
10
|
import datetime as dt
|
|
11
11
|
import threading
|
|
12
12
|
from collections.abc import Callable, Iterable
|
|
13
|
+
from dataclasses import dataclass, field
|
|
13
14
|
from pathlib import Path
|
|
14
15
|
from typing import Annotated, Any
|
|
15
16
|
|
|
@@ -23,8 +24,9 @@ from quantilica.core.storage import (
|
|
|
23
24
|
build_stamped_filename,
|
|
24
25
|
stamp_filename,
|
|
25
26
|
)
|
|
26
|
-
from rich.console import Group
|
|
27
|
+
from rich.console import Console, Group
|
|
27
28
|
from rich.live import Live
|
|
29
|
+
from rich.rule import Rule
|
|
28
30
|
from rich.table import Table
|
|
29
31
|
|
|
30
32
|
from quantilica.cli.ui import (
|
|
@@ -38,6 +40,78 @@ from quantilica.cli.ui import (
|
|
|
38
40
|
|
|
39
41
|
logger = get_logger(__name__)
|
|
40
42
|
|
|
43
|
+
__all__ = [
|
|
44
|
+
"DataRepository",
|
|
45
|
+
"FetcherApp",
|
|
46
|
+
"SyncPlan",
|
|
47
|
+
"SyncPlanItem",
|
|
48
|
+
"default_client",
|
|
49
|
+
"default_path_builder",
|
|
50
|
+
"make_resolve_groups",
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class SyncPlanItem:
|
|
56
|
+
"""A single dataset file planned for synchronization.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
dataset: Canonical dataset (group) identifier.
|
|
60
|
+
partition: Human-readable partition label (e.g. ``2023-05``), or None.
|
|
61
|
+
filename: The stamped file name for the entry.
|
|
62
|
+
url: Remote URL for the resource.
|
|
63
|
+
target: Local destination path.
|
|
64
|
+
dataset_name: Optional human-readable dataset name.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
dataset: str
|
|
68
|
+
partition: str | None
|
|
69
|
+
filename: str
|
|
70
|
+
url: str
|
|
71
|
+
target: Path
|
|
72
|
+
dataset_name: str = ""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass(frozen=True)
|
|
76
|
+
class SyncPlan:
|
|
77
|
+
"""Preview plan for a synchronization run (``--dry-run``).
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
items: Planned :class:`SyncPlanItem` entries.
|
|
81
|
+
skipped: Number of entries ignored (outside of coverage).
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
items: list[SyncPlanItem] = field(default_factory=list)
|
|
85
|
+
skipped: int = 0
|
|
86
|
+
|
|
87
|
+
def render_table(self, console: Console | None = None) -> None:
|
|
88
|
+
"""Render the plan as a Rich table with a summary footer.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
console: Optional Rich Console to print to (defaults to the shared
|
|
92
|
+
console from :mod:`quantilica.cli.ui`).
|
|
93
|
+
"""
|
|
94
|
+
con = console or get_console()
|
|
95
|
+
table = Table(
|
|
96
|
+
"Dataset",
|
|
97
|
+
"Partição",
|
|
98
|
+
"Arquivo",
|
|
99
|
+
"URL",
|
|
100
|
+
title="Arquivos a baixar (dry-run)",
|
|
101
|
+
)
|
|
102
|
+
for item in self.items:
|
|
103
|
+
table.add_row(
|
|
104
|
+
item.dataset,
|
|
105
|
+
item.partition or "—",
|
|
106
|
+
item.filename,
|
|
107
|
+
item.url,
|
|
108
|
+
)
|
|
109
|
+
con.print(table)
|
|
110
|
+
con.print(
|
|
111
|
+
f"Total: {len(self.items)} arquivos planejados. "
|
|
112
|
+
f"{self.skipped} ignorados fora de cobertura."
|
|
113
|
+
)
|
|
114
|
+
|
|
41
115
|
|
|
42
116
|
def default_client() -> HttpClient:
|
|
43
117
|
"""Create a default HttpClient with standard configuration.
|
|
@@ -243,6 +317,211 @@ class FetcherApp:
|
|
|
243
317
|
"""
|
|
244
318
|
self.app.command(name=name, **kwargs)(cmd_func)
|
|
245
319
|
|
|
320
|
+
def command_convert(
|
|
321
|
+
self, func: Callable[[Path, Path], Any]
|
|
322
|
+
) -> Callable[[Path, Path], Any]:
|
|
323
|
+
"""Register the standard ``convert`` subcommand.
|
|
324
|
+
|
|
325
|
+
The generated command exposes the canonical flags (``-i/--input``,
|
|
326
|
+
``-o/--output``, ``--verbose``) and delegates to ``func(input, output)``.
|
|
327
|
+
An ``ImportError`` propagating from ``func`` is treated as missing
|
|
328
|
+
analytical extras and exits gracefully with code 1.
|
|
329
|
+
|
|
330
|
+
Args:
|
|
331
|
+
func: Callable receiving ``(input, output)`` paths and performing
|
|
332
|
+
the conversion.
|
|
333
|
+
|
|
334
|
+
Returns:
|
|
335
|
+
The original ``func`` (the method can be used as a decorator).
|
|
336
|
+
"""
|
|
337
|
+
console = get_console()
|
|
338
|
+
|
|
339
|
+
@self.app.command("convert")
|
|
340
|
+
def convert(
|
|
341
|
+
input: Annotated[
|
|
342
|
+
Path,
|
|
343
|
+
typer.Option(
|
|
344
|
+
"-i",
|
|
345
|
+
"--input",
|
|
346
|
+
help="Diretório de origem com arquivos brutos",
|
|
347
|
+
),
|
|
348
|
+
] = self.default_output,
|
|
349
|
+
output: Annotated[
|
|
350
|
+
Path,
|
|
351
|
+
typer.Option("-o", "--output", help="Diretório de destino"),
|
|
352
|
+
] = self.default_output,
|
|
353
|
+
verbose: Annotated[
|
|
354
|
+
bool, typer.Option("--verbose", help="Logs detalhados")
|
|
355
|
+
] = False,
|
|
356
|
+
) -> None:
|
|
357
|
+
setup_rich_logging(verbose, console=console)
|
|
358
|
+
try:
|
|
359
|
+
func(input, output)
|
|
360
|
+
except ImportError:
|
|
361
|
+
console.print(
|
|
362
|
+
f"[red]Erro:[/red] convert requer extras de análise: "
|
|
363
|
+
f"pip install {self.name}\[analysis]"
|
|
364
|
+
)
|
|
365
|
+
raise typer.Exit(1) from None
|
|
366
|
+
console.print(
|
|
367
|
+
f"[green]✓[/green] Conversão concluída em [dim]{output}[/dim]."
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
return func
|
|
371
|
+
|
|
372
|
+
def command_archive(
|
|
373
|
+
self, func: Callable[[Path, Path], Any]
|
|
374
|
+
) -> Callable[[Path, Path], Any]:
|
|
375
|
+
"""Register the standard ``archive`` subcommand.
|
|
376
|
+
|
|
377
|
+
Mirrors :meth:`command_convert` with the canonical flags
|
|
378
|
+
(``-i/--input``, ``-o/--output``, ``--verbose``) and delegates to
|
|
379
|
+
``func(input, output)``. An ``ImportError`` propagating from ``func``
|
|
380
|
+
is treated as missing analytical extras and exits gracefully with
|
|
381
|
+
code 1.
|
|
382
|
+
|
|
383
|
+
Args:
|
|
384
|
+
func: Callable receiving ``(input, output)`` paths and creating
|
|
385
|
+
the historical archive.
|
|
386
|
+
|
|
387
|
+
Returns:
|
|
388
|
+
The original ``func`` (the method can be used as a decorator).
|
|
389
|
+
"""
|
|
390
|
+
console = get_console()
|
|
391
|
+
|
|
392
|
+
@self.app.command("archive")
|
|
393
|
+
def archive(
|
|
394
|
+
input: Annotated[
|
|
395
|
+
Path,
|
|
396
|
+
typer.Option(
|
|
397
|
+
"-i",
|
|
398
|
+
"--input",
|
|
399
|
+
help="Diretório de dados a arquivar",
|
|
400
|
+
),
|
|
401
|
+
] = self.default_output,
|
|
402
|
+
output: Annotated[
|
|
403
|
+
Path,
|
|
404
|
+
typer.Option("-o", "--output", help="Diretório do arquivo histórico"),
|
|
405
|
+
] = self.default_output,
|
|
406
|
+
verbose: Annotated[
|
|
407
|
+
bool, typer.Option("--verbose", help="Logs detalhados")
|
|
408
|
+
] = False,
|
|
409
|
+
) -> None:
|
|
410
|
+
setup_rich_logging(verbose, console=console)
|
|
411
|
+
try:
|
|
412
|
+
func(input, output)
|
|
413
|
+
except ImportError:
|
|
414
|
+
console.print(
|
|
415
|
+
f"[red]Erro:[/red] archive requer extras de análise: "
|
|
416
|
+
f"pip install {self.name}\[analysis]"
|
|
417
|
+
)
|
|
418
|
+
raise typer.Exit(1) from None
|
|
419
|
+
console.print(f"[green]✓[/green] Arquivo criado em [dim]{output}[/dim].")
|
|
420
|
+
|
|
421
|
+
return func
|
|
422
|
+
|
|
423
|
+
def command_pipeline(
|
|
424
|
+
self, convert_func: Callable[[Path, Path], Any]
|
|
425
|
+
) -> Callable[[Path, Path], Any]:
|
|
426
|
+
"""Register the standard ``pipeline`` subcommand (sync → convert).
|
|
427
|
+
|
|
428
|
+
Step 1/2 invokes the built-in ``sync`` command (same flags: groups,
|
|
429
|
+
``-o/--output``, ``--parquet-dir``, ``--workers``, ``--dry-run``,
|
|
430
|
+
``--verbose``). Step 2/2 invokes ``convert_func(output, parquet_dir)``.
|
|
431
|
+
With ``--dry-run`` the plan is only listed and the conversion step is
|
|
432
|
+
skipped. An ``ImportError`` from ``convert_func`` is treated as
|
|
433
|
+
missing analytical extras.
|
|
434
|
+
|
|
435
|
+
Args:
|
|
436
|
+
convert_func: Callable receiving ``(output, parquet_dir)`` paths
|
|
437
|
+
and performing the conversion.
|
|
438
|
+
|
|
439
|
+
Returns:
|
|
440
|
+
The original ``convert_func`` (usable as a decorator).
|
|
441
|
+
"""
|
|
442
|
+
console = get_console()
|
|
443
|
+
|
|
444
|
+
@self.app.command("pipeline")
|
|
445
|
+
def pipeline(
|
|
446
|
+
ctx: typer.Context,
|
|
447
|
+
groups: Annotated[
|
|
448
|
+
list[str] | None,
|
|
449
|
+
typer.Argument(
|
|
450
|
+
help="Grupos a baixar. Use 'list' para ver grupos "
|
|
451
|
+
"disponíveis. Padrão: todos."
|
|
452
|
+
),
|
|
453
|
+
] = None,
|
|
454
|
+
output: Annotated[
|
|
455
|
+
Path | None,
|
|
456
|
+
typer.Option("-o", "--output", help="Diretório de saída"),
|
|
457
|
+
] = None,
|
|
458
|
+
parquet_dir: Annotated[
|
|
459
|
+
Path | None,
|
|
460
|
+
typer.Option(
|
|
461
|
+
"--parquet-dir",
|
|
462
|
+
help="Diretório para os Parquet (padrão: igual a --output)",
|
|
463
|
+
),
|
|
464
|
+
] = None,
|
|
465
|
+
workers: Annotated[
|
|
466
|
+
int, typer.Option("--workers", help="Downloads paralelos")
|
|
467
|
+
] = 4,
|
|
468
|
+
dry_run: Annotated[
|
|
469
|
+
bool, typer.Option("--dry-run", help="Listar arquivos sem baixar")
|
|
470
|
+
] = False,
|
|
471
|
+
verbose: Annotated[
|
|
472
|
+
bool, typer.Option("--verbose", help="Logs detalhados")
|
|
473
|
+
] = False,
|
|
474
|
+
) -> None:
|
|
475
|
+
setup_rich_logging(verbose, console=console)
|
|
476
|
+
actual_output = output or self.default_output
|
|
477
|
+
parquet_out = parquet_dir or actual_output
|
|
478
|
+
|
|
479
|
+
console.print(Rule("[bold]Passo 1/2: Download[/bold]"))
|
|
480
|
+
sync_cmd = next(
|
|
481
|
+
(
|
|
482
|
+
cmd.callback
|
|
483
|
+
for cmd in self.app.registered_commands
|
|
484
|
+
if cmd.name == "sync"
|
|
485
|
+
),
|
|
486
|
+
None,
|
|
487
|
+
)
|
|
488
|
+
if sync_cmd is None:
|
|
489
|
+
console.print(
|
|
490
|
+
"[red]Erro:[/red] pipeline requer o comando 'sync' "
|
|
491
|
+
"(build_default_commands=True)."
|
|
492
|
+
)
|
|
493
|
+
raise typer.Exit(1) from None
|
|
494
|
+
ctx.invoke(
|
|
495
|
+
sync_cmd,
|
|
496
|
+
groups=groups,
|
|
497
|
+
output=actual_output,
|
|
498
|
+
dry_run=dry_run,
|
|
499
|
+
workers=workers,
|
|
500
|
+
verbose=verbose,
|
|
501
|
+
)
|
|
502
|
+
|
|
503
|
+
if dry_run:
|
|
504
|
+
console.print(
|
|
505
|
+
"\n[yellow]Dry-run:[/yellow] conversão (passo 2/2) não executada."
|
|
506
|
+
)
|
|
507
|
+
return
|
|
508
|
+
|
|
509
|
+
console.print(Rule("[bold]Passo 2/2: Conversão[/bold]"))
|
|
510
|
+
try:
|
|
511
|
+
convert_func(actual_output, parquet_out)
|
|
512
|
+
except ImportError:
|
|
513
|
+
console.print(
|
|
514
|
+
f"[red]Erro:[/red] pipeline (conversão) requer extras de "
|
|
515
|
+
f"análise: pip install {self.name}\[analysis]"
|
|
516
|
+
)
|
|
517
|
+
raise typer.Exit(1) from None
|
|
518
|
+
console.print(
|
|
519
|
+
f"[green]✓[/green] Pipeline concluído: Parquet em "
|
|
520
|
+
f"[dim]{parquet_out}[/dim]."
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
return convert_func
|
|
524
|
+
|
|
246
525
|
def _safe_head_date(self, url: str) -> dt.date | None:
|
|
247
526
|
with contextlib.suppress(Exception):
|
|
248
527
|
return self.client.head_last_modified_date(url)
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Testes para o comando `quantilica health` (sondas de disponibilidade)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from typing import Any
|
|
7
|
+
from unittest.mock import patch
|
|
8
|
+
|
|
9
|
+
import httpx2
|
|
10
|
+
from typer.testing import CliRunner
|
|
11
|
+
|
|
12
|
+
from quantilica.cli import health
|
|
13
|
+
from quantilica.cli.cli import app
|
|
14
|
+
from quantilica.cli.health import HEALTH_SOURCES
|
|
15
|
+
|
|
16
|
+
runner = CliRunner()
|
|
17
|
+
|
|
18
|
+
SOURCE_NAMES = [name for name, _ in HEALTH_SOURCES]
|
|
19
|
+
URLS = {url: name for name, url in HEALTH_SOURCES}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _handler(status_by_url: dict[str, int], head_405: bool = False):
|
|
23
|
+
"""Cria um handler de MockTransport mapeando URL -> status HTTP."""
|
|
24
|
+
|
|
25
|
+
def handle(request: httpx2.Request) -> httpx2.Response:
|
|
26
|
+
url = str(request.url).split("?")[0]
|
|
27
|
+
status = status_by_url.get(url, 200)
|
|
28
|
+
if request.method == "HEAD" and head_405:
|
|
29
|
+
return httpx2.Response(405)
|
|
30
|
+
return httpx2.Response(status)
|
|
31
|
+
|
|
32
|
+
return handle
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _patch_client(
|
|
36
|
+
status_by_url: dict[str, int],
|
|
37
|
+
head_405: bool = False,
|
|
38
|
+
):
|
|
39
|
+
"""Parchea o cliente de sondas com um MockTransport determinístico."""
|
|
40
|
+
transport = httpx2.MockTransport(_handler(status_by_url, head_405))
|
|
41
|
+
fake_client = httpx2.Client(transport=transport, follow_redirects=True)
|
|
42
|
+
return patch.object(health, "_make_client", return_value=fake_client)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _all_ok() -> dict[str, int]:
|
|
46
|
+
return {url: 200 for url in URLS}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _parse_json(output: str) -> dict[str, Any]:
|
|
50
|
+
lines = [line for line in output.splitlines() if line.startswith("{")]
|
|
51
|
+
assert lines, f"nenhuma linha JSON na saída: {output!r}"
|
|
52
|
+
return json.loads(lines[-1])
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# --- modo tabela -----------------------------------------------------------
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_health_table_all_ok() -> None:
|
|
59
|
+
with _patch_client(_all_ok()):
|
|
60
|
+
result = runner.invoke(app, ["health"])
|
|
61
|
+
assert result.exit_code == 0
|
|
62
|
+
for name in SOURCE_NAMES:
|
|
63
|
+
assert name in result.output
|
|
64
|
+
assert "OK" in result.output
|
|
65
|
+
assert "FALHA" not in result.output
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_health_table_with_failure() -> None:
|
|
69
|
+
statuses = _all_ok()
|
|
70
|
+
statuses["https://balanca.economia.gov.br"] = 500
|
|
71
|
+
with _patch_client(statuses):
|
|
72
|
+
result = runner.invoke(app, ["health"])
|
|
73
|
+
assert result.exit_code == 0
|
|
74
|
+
assert "FALHA" in result.output
|
|
75
|
+
assert "OK" in result.output
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_health_head_fallback_to_get() -> None:
|
|
79
|
+
# HEAD retorna 405 e o CLI deve refazer com GET (levemente diferente).
|
|
80
|
+
handlers: list[str] = []
|
|
81
|
+
|
|
82
|
+
def handle(request: httpx2.Request) -> httpx2.Response:
|
|
83
|
+
handlers.append(request.method)
|
|
84
|
+
if request.method == "HEAD":
|
|
85
|
+
return httpx2.Response(405)
|
|
86
|
+
return httpx2.Response(200)
|
|
87
|
+
|
|
88
|
+
transport = httpx2.MockTransport(handle)
|
|
89
|
+
with patch.object(
|
|
90
|
+
health, "_make_client", return_value=httpx2.Client(transport=transport)
|
|
91
|
+
):
|
|
92
|
+
result = runner.invoke(app, ["health"])
|
|
93
|
+
assert result.exit_code == 0
|
|
94
|
+
assert "OK" in result.output
|
|
95
|
+
assert "HEAD" in handlers
|
|
96
|
+
assert "GET" in handlers
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
# --- modo --json -------------------------------------------------------------
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_health_json_all_ok() -> None:
|
|
103
|
+
with _patch_client(_all_ok()):
|
|
104
|
+
result = runner.invoke(app, ["health", "--json"])
|
|
105
|
+
assert result.exit_code == 0
|
|
106
|
+
payload = _parse_json(result.output)
|
|
107
|
+
assert payload["status"] == "ok"
|
|
108
|
+
source_names = [s["name"] for s in payload["sources"]]
|
|
109
|
+
assert source_names == SOURCE_NAMES
|
|
110
|
+
for source in payload["sources"]:
|
|
111
|
+
assert source["status"] == "ok"
|
|
112
|
+
assert source["http_status"] == 200
|
|
113
|
+
assert isinstance(source["latency_ms"], (int, float))
|
|
114
|
+
assert source["error"] is None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_health_json_degraded() -> None:
|
|
118
|
+
statuses = _all_ok()
|
|
119
|
+
statuses["https://balanca.economia.gov.br"] = 503
|
|
120
|
+
with _patch_client(statuses):
|
|
121
|
+
result = runner.invoke(app, ["health", "--json"])
|
|
122
|
+
assert result.exit_code == 0
|
|
123
|
+
payload = _parse_json(result.output)
|
|
124
|
+
assert payload["status"] == "degraded"
|
|
125
|
+
comex = next(s for s in payload["sources"] if s["name"] == "comex")
|
|
126
|
+
assert comex["status"] == "falha"
|
|
127
|
+
assert comex["http_status"] == 503
|
|
128
|
+
assert comex["error"] == "HTTP 503"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def test_health_json_network_error() -> None:
|
|
132
|
+
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
133
|
+
if "inmet" in str(request.url):
|
|
134
|
+
raise httpx2.ConnectError("boom")
|
|
135
|
+
return httpx2.Response(200)
|
|
136
|
+
|
|
137
|
+
transport = httpx2.MockTransport(handler)
|
|
138
|
+
with patch.object(
|
|
139
|
+
health, "_make_client", return_value=httpx2.Client(transport=transport)
|
|
140
|
+
):
|
|
141
|
+
result = runner.invoke(app, ["health", "--json"])
|
|
142
|
+
assert result.exit_code == 0
|
|
143
|
+
payload = _parse_json(result.output)
|
|
144
|
+
assert payload["status"] == "degraded"
|
|
145
|
+
inmet = next(s for s in payload["sources"] if s["name"] == "inmet")
|
|
146
|
+
assert inmet["status"] == "falha"
|
|
147
|
+
assert inmet["http_status"] is None
|
|
148
|
+
assert "ConnectError" in inmet["error"]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
# --- --fail-fast ------------------------------------------------------------
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def test_health_fail_fast_exit_code_ok() -> None:
|
|
155
|
+
with _patch_client(_all_ok()):
|
|
156
|
+
result = runner.invoke(app, ["health", "--fail-fast", "--json"])
|
|
157
|
+
assert result.exit_code == 0
|
|
158
|
+
payload = _parse_json(result.output)
|
|
159
|
+
assert payload["status"] == "ok"
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_health_fail_fast_exit_code_on_failure() -> None:
|
|
163
|
+
statuses = _all_ok()
|
|
164
|
+
statuses["https://balanca.economia.gov.br"] = 500
|
|
165
|
+
with _patch_client(statuses):
|
|
166
|
+
result = runner.invoke(app, ["health", "--fail-fast"])
|
|
167
|
+
assert result.exit_code == 1
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def test_health_fail_fast_stops_probes() -> None:
|
|
171
|
+
# com a primeira fonte falhando em fail-fast, as demais não são sondadas.
|
|
172
|
+
probes: list[str] = []
|
|
173
|
+
|
|
174
|
+
def fake_probe(client: httpx2.Client, name: str, url: str) -> dict[str, object]:
|
|
175
|
+
probes.append(name)
|
|
176
|
+
status = "falha" if name == "bcb" else "ok"
|
|
177
|
+
return {
|
|
178
|
+
"name": name,
|
|
179
|
+
"url": url,
|
|
180
|
+
"status": status,
|
|
181
|
+
"http_status": 500 if status == "falha" else 200,
|
|
182
|
+
"latency_ms": 1.0,
|
|
183
|
+
"error": None if status == "ok" else "HTTP 500",
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
with (
|
|
187
|
+
_patch_client(_all_ok()),
|
|
188
|
+
patch.object(health, "_probe", side_effect=fake_probe),
|
|
189
|
+
):
|
|
190
|
+
result = runner.invoke(app, ["health", "--fail-fast"])
|
|
191
|
+
assert result.exit_code == 1
|
|
192
|
+
assert probes == ["bcb"]
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# --- --timeout ------------------------------------------------------------
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def test_health_passes_timeout_to_client() -> None:
|
|
199
|
+
with patch.object(
|
|
200
|
+
health, "_make_client", return_value=httpx2.Client()
|
|
201
|
+
) as mock_client:
|
|
202
|
+
result = runner.invoke(app, ["health", "--timeout", "2.5", "--json"])
|
|
203
|
+
assert result.exit_code == 0
|
|
204
|
+
mock_client.assert_called_once_with(timeout=2.5)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def test_health_sources_canonical_list() -> None:
|
|
208
|
+
assert SOURCE_NAMES == ["bcb", "sidra", "tesouro_direto", "comex", "inmet"]
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
"""Testes unitários para Onda A.2 do SDK (command_convert/pipeline/archive, SyncPlan)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
import typer
|
|
9
|
+
from rich.console import Console
|
|
10
|
+
from typer.testing import CliRunner
|
|
11
|
+
|
|
12
|
+
from quantilica.cli.sdk import SyncPlan, SyncPlanItem
|
|
13
|
+
from tests.test_sdk import _grouped, make_app
|
|
14
|
+
|
|
15
|
+
runner = CliRunner()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# ---------------------------------------------------------------
|
|
19
|
+
# SyncPlan / SyncPlanItem
|
|
20
|
+
# ---------------------------------------------------------------
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _sample_item(
|
|
24
|
+
dataset: str = "exp", partition: str | None = "2023-05"
|
|
25
|
+
) -> SyncPlanItem:
|
|
26
|
+
return SyncPlanItem(
|
|
27
|
+
dataset=dataset,
|
|
28
|
+
partition=partition,
|
|
29
|
+
filename=f"{dataset}-monthly_2023-05.csv",
|
|
30
|
+
url=f"https://example.test/{dataset}-monthly.csv",
|
|
31
|
+
target=Path(f"/data/output/{dataset}/{dataset}-monthly_2023-05.csv"),
|
|
32
|
+
dataset_name="Exportações",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_sync_plan_item_is_frozen():
|
|
37
|
+
item = _sample_item()
|
|
38
|
+
with pytest.raises(Exception): # noqa: B017 (FrozenInstanceError levantada dinamicamente)
|
|
39
|
+
item.dataset = "imp" # type: ignore[misc]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_sync_plan_is_frozen():
|
|
43
|
+
plan = SyncPlan(items=[_sample_item()])
|
|
44
|
+
with pytest.raises(Exception): # noqa: B017 (FrozenInstanceError levantada dinamicamente)
|
|
45
|
+
plan.skipped = 5 # type: ignore[misc]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_sync_plan_item_defaults():
|
|
49
|
+
item = _sample_item()
|
|
50
|
+
assert item.dataset_name == "Exportações"
|
|
51
|
+
plain = SyncPlanItem(
|
|
52
|
+
dataset="d", partition=None, filename="f", url="u", target=Path("t")
|
|
53
|
+
)
|
|
54
|
+
assert plain.dataset_name == ""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_sync_plan_defaults():
|
|
58
|
+
plan = SyncPlan()
|
|
59
|
+
assert plan.items == []
|
|
60
|
+
assert plan.skipped == 0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_render_table_prints_columns_and_rows(capsys: pytest.CaptureFixture[str]):
|
|
64
|
+
plan = SyncPlan(
|
|
65
|
+
items=[_sample_item("exp", "2023-05"), _sample_item("imp", None)], skipped=3
|
|
66
|
+
)
|
|
67
|
+
console = Console(record=True, width=120, legacy_windows=False)
|
|
68
|
+
plan.render_table(console=console)
|
|
69
|
+
output = console.export_text()
|
|
70
|
+
assert "exp" in output
|
|
71
|
+
assert "exp-monthly_2023-05.csv" in output
|
|
72
|
+
assert "https://example.test/exp-monthly.csv" in output
|
|
73
|
+
# partição ausente é exibida como "—"
|
|
74
|
+
assert "—" in output
|
|
75
|
+
assert "Total: 2 arquivos planejados. 3 ignorados fora de cobertura." in output
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_render_table_uses_shared_console_when_none(capsys: pytest.CaptureFixture[str]):
|
|
79
|
+
plan = SyncPlan(items=[_sample_item()], skipped=1)
|
|
80
|
+
plan.render_table()
|
|
81
|
+
output = capsys.readouterr().out
|
|
82
|
+
assert "Total: 1 arquivos planejados. 1 ignorados fora de cobertura." in output
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# ---------------------------------------------------------------
|
|
86
|
+
# command_convert
|
|
87
|
+
# ---------------------------------------------------------------
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_command_convert_success(tmp_path: Path):
|
|
91
|
+
app = make_app()
|
|
92
|
+
calls: list[tuple[Path, Path]] = []
|
|
93
|
+
|
|
94
|
+
@app.command_convert
|
|
95
|
+
def conv(input: Path, output: Path) -> None:
|
|
96
|
+
calls.append((input, output))
|
|
97
|
+
|
|
98
|
+
result = runner.invoke(
|
|
99
|
+
_grouped(app.app), ["fd", "convert", "-i", str(tmp_path), "-o", str(tmp_path)]
|
|
100
|
+
)
|
|
101
|
+
assert result.exit_code == 0, result.output
|
|
102
|
+
assert calls == [(tmp_path, tmp_path)]
|
|
103
|
+
assert "Conversão concluída" in result.output
|
|
104
|
+
assert "✓" in result.output
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_command_convert_defaults_to_default_output(tmp_path: Path):
|
|
108
|
+
app = make_app()
|
|
109
|
+
calls: list[tuple[Path, Path]] = []
|
|
110
|
+
|
|
111
|
+
@app.command_convert
|
|
112
|
+
def conv(input: Path, output: Path) -> None:
|
|
113
|
+
calls.append((input, output))
|
|
114
|
+
|
|
115
|
+
default = app.default_output
|
|
116
|
+
result = runner.invoke(_grouped(app.app), ["fd", "convert"])
|
|
117
|
+
assert result.exit_code == 0, result.output
|
|
118
|
+
assert calls == [(default, default)]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def test_command_convert_graceful_import_error(tmp_path: Path):
|
|
122
|
+
app = make_app()
|
|
123
|
+
|
|
124
|
+
@app.command_convert
|
|
125
|
+
def conv(input: Path, output: Path) -> None:
|
|
126
|
+
raise ImportError("No module named 'polars'")
|
|
127
|
+
|
|
128
|
+
result = runner.invoke(_grouped(app.app), ["fd", "convert", "-i", str(tmp_path)])
|
|
129
|
+
assert result.exit_code == 1
|
|
130
|
+
assert "convert requer extras de análise" in result.output
|
|
131
|
+
assert "pip install comex-fetcher[analysis]" in result.output
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_command_convert_verbose_sets_logging(tmp_path: Path):
|
|
135
|
+
app = make_app()
|
|
136
|
+
|
|
137
|
+
@app.command_convert
|
|
138
|
+
def conv(input: Path, output: Path) -> None:
|
|
139
|
+
raise typer.Exit(3)
|
|
140
|
+
|
|
141
|
+
result = runner.invoke(_grouped(app.app), ["fd", "convert", "--verbose"])
|
|
142
|
+
assert result.exit_code == 3
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# ---------------------------------------------------------------
|
|
146
|
+
# command_pipeline
|
|
147
|
+
# ---------------------------------------------------------------
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def test_command_pipeline_runs_both_steps(
|
|
151
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
152
|
+
):
|
|
153
|
+
app = make_app()
|
|
154
|
+
calls: list[tuple[Path, Path]] = []
|
|
155
|
+
sync_calls: list[tuple[list, Path, int]] = []
|
|
156
|
+
|
|
157
|
+
def fake_download(entries, output_dir, workers=4):
|
|
158
|
+
sync_calls.append((entries, output_dir, workers))
|
|
159
|
+
return len(entries), len(entries), []
|
|
160
|
+
|
|
161
|
+
monkeypatch.setattr(app, "download_datasets", fake_download)
|
|
162
|
+
|
|
163
|
+
@app.command_pipeline
|
|
164
|
+
def conv(data_dir: Path, parquet_dir: Path) -> None:
|
|
165
|
+
calls.append((data_dir, parquet_dir))
|
|
166
|
+
|
|
167
|
+
raw = tmp_path / "raw"
|
|
168
|
+
parquet = tmp_path / "parquet"
|
|
169
|
+
result = runner.invoke(
|
|
170
|
+
_grouped(app.app),
|
|
171
|
+
["fd", "pipeline", "exp", "-o", str(raw), "--parquet-dir", str(parquet)],
|
|
172
|
+
)
|
|
173
|
+
assert result.exit_code == 0, result.output
|
|
174
|
+
assert "Passo 1/2: Download" in result.output
|
|
175
|
+
assert "Passo 2/2: Conversão" in result.output
|
|
176
|
+
# Passo 1 invocou o sync (download_datasets) uma vez
|
|
177
|
+
assert len(sync_calls) == 1
|
|
178
|
+
entries, out_dir, workers = sync_calls[0]
|
|
179
|
+
assert out_dir == raw
|
|
180
|
+
assert [e["id"] for e in entries] == ["exp-monthly"]
|
|
181
|
+
# Passo 2 invocou a convert_func com (output, parquet_dir)
|
|
182
|
+
assert calls == [(raw, parquet)]
|
|
183
|
+
assert "✓" in result.output
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_command_pipeline_dry_run_skips_conversion(
|
|
187
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
188
|
+
):
|
|
189
|
+
app = make_app()
|
|
190
|
+
downloads: list[tuple] = []
|
|
191
|
+
monkeypatch.setattr(
|
|
192
|
+
app,
|
|
193
|
+
"download_datasets",
|
|
194
|
+
lambda entries, output_dir, workers=4: (
|
|
195
|
+
downloads.append(output_dir) or (0, 0, [])
|
|
196
|
+
),
|
|
197
|
+
)
|
|
198
|
+
converted: list[Path] = []
|
|
199
|
+
|
|
200
|
+
@app.command_pipeline
|
|
201
|
+
def conv(data_dir: Path, parquet_dir: Path) -> None:
|
|
202
|
+
converted.append(parquet_dir)
|
|
203
|
+
|
|
204
|
+
result = runner.invoke(_grouped(app.app), ["fd", "pipeline", "exp", "--dry-run"])
|
|
205
|
+
assert result.exit_code == 0, result.output
|
|
206
|
+
assert "Passo 1/2: Download" in result.output
|
|
207
|
+
assert "Passo 2/2: Conversão" not in result.output
|
|
208
|
+
assert converted == []
|
|
209
|
+
# dry-run: nenhum download executado (download_datasets não foi invocado)
|
|
210
|
+
assert downloads == []
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def test_command_pipeline_graceful_import_error(
|
|
214
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
215
|
+
):
|
|
216
|
+
app = make_app()
|
|
217
|
+
monkeypatch.setattr(
|
|
218
|
+
app, "download_datasets", lambda entries, output_dir, workers=4: (0, 0, [])
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
@app.command_pipeline
|
|
222
|
+
def conv(data_dir: Path, parquet_dir: Path) -> None:
|
|
223
|
+
raise ImportError("No module named 'polars'")
|
|
224
|
+
|
|
225
|
+
result = runner.invoke(
|
|
226
|
+
_grouped(app.app), ["fd", "pipeline", "exp", "-o", str(tmp_path)]
|
|
227
|
+
)
|
|
228
|
+
assert result.exit_code == 1
|
|
229
|
+
assert "pipeline (conversão) requer extras de análise" in result.output
|
|
230
|
+
assert "comex-fetcher[analysis]" in result.output
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def test_command_pipeline_parquet_dir_defaults_to_output(
|
|
234
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
235
|
+
):
|
|
236
|
+
app = make_app()
|
|
237
|
+
monkeypatch.setattr(
|
|
238
|
+
app, "download_datasets", lambda entries, output_dir, workers=4: (0, 0, [])
|
|
239
|
+
)
|
|
240
|
+
calls: list[tuple[Path, Path]] = []
|
|
241
|
+
|
|
242
|
+
@app.command_pipeline
|
|
243
|
+
def conv(data_dir: Path, parquet_dir: Path) -> None:
|
|
244
|
+
calls.append((data_dir, parquet_dir))
|
|
245
|
+
|
|
246
|
+
raw = tmp_path / "raw"
|
|
247
|
+
result = runner.invoke(_grouped(app.app), ["fd", "pipeline", "exp", "-o", str(raw)])
|
|
248
|
+
assert result.exit_code == 0, result.output
|
|
249
|
+
assert calls == [(raw, raw)]
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def test_command_pipeline_requires_sync_command():
|
|
253
|
+
app = make_app(build_default_commands=False)
|
|
254
|
+
|
|
255
|
+
@app.command_pipeline
|
|
256
|
+
def conv(data_dir: Path, parquet_dir: Path) -> None:
|
|
257
|
+
raise AssertionError("não deveria ser chamado")
|
|
258
|
+
|
|
259
|
+
result = runner.invoke(_grouped(app.app), ["fd", "pipeline", "exp"])
|
|
260
|
+
assert result.exit_code == 1
|
|
261
|
+
assert "pipeline requer o comando 'sync'" in result.output
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
# ---------------------------------------------------------------
|
|
265
|
+
# command_archive
|
|
266
|
+
# ---------------------------------------------------------------
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def test_command_archive_success(tmp_path: Path):
|
|
270
|
+
app = make_app()
|
|
271
|
+
calls: list[tuple[Path, Path]] = []
|
|
272
|
+
|
|
273
|
+
@app.command_archive
|
|
274
|
+
def arch(input: Path, output: Path) -> None:
|
|
275
|
+
calls.append((input, output))
|
|
276
|
+
|
|
277
|
+
result = runner.invoke(
|
|
278
|
+
_grouped(app.app), ["fd", "archive", "-i", str(tmp_path), "-o", str(tmp_path)]
|
|
279
|
+
)
|
|
280
|
+
assert result.exit_code == 0, result.output
|
|
281
|
+
assert calls == [(tmp_path, tmp_path)]
|
|
282
|
+
assert "Arquivo criado" in result.output
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def test_command_archive_graceful_import_error(tmp_path: Path):
|
|
286
|
+
app = make_app()
|
|
287
|
+
|
|
288
|
+
@app.command_archive
|
|
289
|
+
def arch(input: Path, output: Path) -> None:
|
|
290
|
+
raise ImportError("No module named 'some_extra'")
|
|
291
|
+
|
|
292
|
+
result = runner.invoke(_grouped(app.app), ["fd", "archive", "-o", str(tmp_path)])
|
|
293
|
+
assert result.exit_code == 1
|
|
294
|
+
assert "archive requer extras de análise" in result.output
|
|
295
|
+
assert "pip install comex-fetcher[analysis]" in result.output
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|