quantilica-cli 0.15.1__tar.gz → 0.16.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/CHANGELOG.md +7 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/PKG-INFO +3 -1
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/pyproject.toml +4 -2
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/src/quantilica/cli/cli.py +2 -0
- quantilica_cli-0.16.0/src/quantilica/cli/docs.py +271 -0
- quantilica_cli-0.16.0/tests/test_docs_cli.py +215 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/.githooks/pre-commit +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/.githooks/pre-push +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/.github/workflows/public-hygiene.yml +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/.github/workflows/publish.yml +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/.github/workflows/test.yml +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/.gitignore +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/LICENSE +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/README.md +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/src/quantilica/cli/__init__.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/src/quantilica/cli/catalog.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/src/quantilica/cli/health.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/src/quantilica/cli/lake.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/src/quantilica/cli/manifests.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/src/quantilica/cli/sdk.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/src/quantilica/cli/sources.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/src/quantilica/cli/ui.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/src/quantilica/py.typed +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/__init__.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/conftest.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/lake_build_fakes.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/lake_fixtures.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_catalog_cli.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_health.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_lake_cli.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_manifests.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_package_hygiene.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_sdk.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_sdk_backfill.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_sdk_force_probe.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_sdk_mes_string.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_sdk_onda_a2.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_sources.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.16.0}/tests/test_ui.py +0 -0
|
@@ -7,6 +7,13 @@ e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
|
|
|
7
7
|
|
|
8
8
|
## [Não lançado]
|
|
9
9
|
|
|
10
|
+
## [0.16.0] - 2026-11-08
|
|
11
|
+
|
|
12
|
+
### Adicionado
|
|
13
|
+
|
|
14
|
+
- `quantilica docs`: ferramentas da norma de documentação dos dados dos fetchers. `quantilica docs check <repo> [--source]` verifica a pasta `docs/` (a fonte e os datasets vêm do conversor `quantilica.lake` instalado ou, sem ele, das páginas em `docs/datasets/`) e sai com 1 se houver erro; `quantilica docs schema <fonte> --lake <dir> --out <docs/datasets> [--dataset ...]` grava os esquemas de referência lendo o lake; `quantilica docs update <repo>` reescreve a ficha técnica das páginas.
|
|
15
|
+
- Os comandos `quantilica docs` exigem o `quantilica-core` com o extra `docs` (`pip install "quantilica-core[docs]"`); sem ele, falham com a instrução de instalação.
|
|
16
|
+
|
|
10
17
|
## [0.15.1] - 2026-11-06
|
|
11
18
|
|
|
12
19
|
### Alterado
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: quantilica-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.16.0
|
|
4
4
|
Summary: Unified CLI for Quantilica open data fetchers
|
|
5
5
|
Author-email: "Komesu, D.K." <daniel@dkko.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -19,6 +19,8 @@ Requires-Python: >=3.12
|
|
|
19
19
|
Requires-Dist: quantilica-core>=0.16.0
|
|
20
20
|
Requires-Dist: rich>=13.0.0
|
|
21
21
|
Requires-Dist: typer>=0.15.0
|
|
22
|
+
Provides-Extra: docs
|
|
23
|
+
Requires-Dist: quantilica-core[docs]>=0.18.0; extra == 'docs'
|
|
22
24
|
Provides-Extra: lake
|
|
23
25
|
Requires-Dist: quantilica-analytics>=0.11.0; extra == 'lake'
|
|
24
26
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "quantilica-cli"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.16.0"
|
|
4
4
|
description = "Unified CLI for Quantilica open data fetchers"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [{ name = "Komesu, D.K.", email = "daniel@dkko.me" }]
|
|
@@ -28,12 +28,14 @@ dependencies = [
|
|
|
28
28
|
[project.optional-dependencies]
|
|
29
29
|
# Motor de `quantilica lake build` e `quantilica lake audit`.
|
|
30
30
|
lake = ["quantilica-analytics>=0.11.0"]
|
|
31
|
+
# `quantilica docs`: verificação da documentação dos dados dos fetchers.
|
|
32
|
+
docs = ["quantilica-core[docs]>=0.18.0"]
|
|
31
33
|
|
|
32
34
|
[project.scripts]
|
|
33
35
|
quantilica = "quantilica.cli.cli:main"
|
|
34
36
|
|
|
35
37
|
[dependency-groups]
|
|
36
|
-
dev = ["ruff>=0.9.0", "pytest>=8.0.0", "pyarrow>=15.0.0"]
|
|
38
|
+
dev = ["ruff>=0.9.0", "pytest>=8.0.0", "pyarrow>=15.0.0", "pyyaml>=6.0", "polars>=1.0"]
|
|
37
39
|
|
|
38
40
|
# Os fetchers são instalados independentemente pelo usuário — quantilica-cli
|
|
39
41
|
# os descobre via entry points (grupo "quantilica.fetchers") sem dependência direta.
|
|
@@ -14,6 +14,7 @@ from rich.table import Table
|
|
|
14
14
|
|
|
15
15
|
from quantilica.cli import __version__
|
|
16
16
|
from quantilica.cli.catalog import app as catalog_app
|
|
17
|
+
from quantilica.cli.docs import app as docs_app
|
|
17
18
|
from quantilica.cli.health import cmd_health
|
|
18
19
|
from quantilica.cli.lake import app as lake_app
|
|
19
20
|
from quantilica.cli.manifests import app as manifests_app
|
|
@@ -36,6 +37,7 @@ app = typer.Typer(
|
|
|
36
37
|
no_args_is_help=True,
|
|
37
38
|
)
|
|
38
39
|
app.add_typer(catalog_app, name="catalog")
|
|
40
|
+
app.add_typer(docs_app, name="docs")
|
|
39
41
|
app.add_typer(lake_app, name="lake")
|
|
40
42
|
app.add_typer(manifests_app, name="manifests")
|
|
41
43
|
app.add_typer(sources_app, name="sources")
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""Comandos `quantilica docs` — a documentação dos dados dos fetchers.
|
|
2
|
+
|
|
3
|
+
Ferramentas da norma "Documentação dos dados": ``check`` confere a pasta
|
|
4
|
+
``docs/`` de um fetcher, ``schema`` grava os esquemas de referência a partir do
|
|
5
|
+
lake e ``update`` reescreve a ficha técnica das páginas.
|
|
6
|
+
|
|
7
|
+
Dependem do ``quantilica-core`` com o extra ``docs`` (``pyyaml`` e
|
|
8
|
+
``pyarrow``), importado só quando o comando roda, de modo que ``--help``
|
|
9
|
+
funciona sem ele.
|
|
10
|
+
|
|
11
|
+
Como o ``check`` descobre o que verificar:
|
|
12
|
+
|
|
13
|
+
- fonte: ``--source``; senão o entry point ``quantilica.lake`` declarado no
|
|
14
|
+
``pyproject.toml`` do repositório; senão o campo ``fonte`` das páginas;
|
|
15
|
+
- datasets: ``datasets()`` do conversor, se o fetcher estiver instalado
|
|
16
|
+
(entry point ``quantilica.lake``); senão os arquivos de ``docs/datasets/``
|
|
17
|
+
(e então a verificação "dataset sem página" não pode falhar).
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import tomllib
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Annotated
|
|
25
|
+
|
|
26
|
+
import typer
|
|
27
|
+
from rich.console import Console
|
|
28
|
+
from rich.markup import escape
|
|
29
|
+
|
|
30
|
+
app = typer.Typer(
|
|
31
|
+
help="Documentação dos dados dos fetchers: verificar, gerar esquemas e fichas.",
|
|
32
|
+
no_args_is_help=True,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
console = Console()
|
|
36
|
+
err_console = Console(stderr=True)
|
|
37
|
+
|
|
38
|
+
DOCS_HINT = 'uv add "quantilica-core[docs]" (ou: pip install "quantilica-core[docs]")'
|
|
39
|
+
LAKE_ENTRY_POINT_GROUP = "quantilica.lake"
|
|
40
|
+
|
|
41
|
+
EXIT_FAILURE = 1
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _fail(message: str, code: int = EXIT_FAILURE) -> typer.Exit:
|
|
45
|
+
err_console.print(f"[red]Erro:[/red] {escape(message)}", highlight=False)
|
|
46
|
+
return typer.Exit(code)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _import_docs():
|
|
50
|
+
try:
|
|
51
|
+
from quantilica.core import docs
|
|
52
|
+
except ImportError as exc:
|
|
53
|
+
raise _fail(
|
|
54
|
+
f"'quantilica docs' requer o quantilica-core com o extra docs "
|
|
55
|
+
f"({exc}). Instale com: {DOCS_HINT}"
|
|
56
|
+
) from None
|
|
57
|
+
return docs
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _source_from_pyproject(repo_dir: Path) -> str | None:
|
|
61
|
+
"""Nome do entry point ``quantilica.lake`` do ``pyproject.toml`` do repo."""
|
|
62
|
+
path = repo_dir / "pyproject.toml"
|
|
63
|
+
if not path.is_file():
|
|
64
|
+
return None
|
|
65
|
+
try:
|
|
66
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
67
|
+
except (OSError, tomllib.TOMLDecodeError):
|
|
68
|
+
return None
|
|
69
|
+
eps = data.get("project", {}).get("entry-points", {}).get(LAKE_ENTRY_POINT_GROUP)
|
|
70
|
+
if isinstance(eps, dict) and len(eps) == 1:
|
|
71
|
+
return next(iter(eps))
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _datasets_dir(repo_dir: Path) -> Path:
|
|
76
|
+
return repo_dir / "docs" / "datasets"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _page_stems(repo_dir: Path) -> list[str]:
|
|
80
|
+
folder = _datasets_dir(repo_dir)
|
|
81
|
+
return sorted(p.stem for p in folder.glob("*.md")) if folder.is_dir() else []
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _sources_in_pages(docs, repo_dir: Path) -> set[str]:
|
|
85
|
+
found: set[str] = set()
|
|
86
|
+
for stem in _page_stems(repo_dir):
|
|
87
|
+
try:
|
|
88
|
+
header, _ = docs.parse_dataset_page(_datasets_dir(repo_dir) / f"{stem}.md")
|
|
89
|
+
except docs.DocFormatError:
|
|
90
|
+
continue
|
|
91
|
+
found.add(header.fonte)
|
|
92
|
+
return found
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@app.command("check")
|
|
96
|
+
def check(
|
|
97
|
+
repo_dir: Annotated[
|
|
98
|
+
Path,
|
|
99
|
+
typer.Argument(
|
|
100
|
+
help="Raiz do repositório do fetcher (a pasta que contém docs/)",
|
|
101
|
+
exists=True,
|
|
102
|
+
file_okay=False,
|
|
103
|
+
),
|
|
104
|
+
],
|
|
105
|
+
source: Annotated[
|
|
106
|
+
str | None,
|
|
107
|
+
typer.Option(
|
|
108
|
+
"--source",
|
|
109
|
+
help="Id da fonte; padrão: entry point do pyproject.toml ou o campo "
|
|
110
|
+
"'fonte' das páginas",
|
|
111
|
+
),
|
|
112
|
+
] = None,
|
|
113
|
+
) -> None:
|
|
114
|
+
"""Verifica a documentação dos dados de um fetcher (sai com 1 se houver erro)."""
|
|
115
|
+
docs = _import_docs()
|
|
116
|
+
docs_dir = repo_dir / "docs"
|
|
117
|
+
if not docs_dir.is_dir():
|
|
118
|
+
raise _fail(f"{repo_dir} não tem a pasta docs/.")
|
|
119
|
+
|
|
120
|
+
source_id = source or _source_from_pyproject(repo_dir)
|
|
121
|
+
if source_id is None:
|
|
122
|
+
sources = _sources_in_pages(docs, repo_dir)
|
|
123
|
+
if len(sources) == 1:
|
|
124
|
+
source_id = next(iter(sources))
|
|
125
|
+
else:
|
|
126
|
+
raise _fail(
|
|
127
|
+
"não consegui descobrir a fonte (pyproject.toml sem entry point "
|
|
128
|
+
f"'{LAKE_ENTRY_POINT_GROUP}' e páginas com fontes {sorted(sources)}); "
|
|
129
|
+
"informe --source."
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
datasets: list[str] | None = None
|
|
133
|
+
origin = ""
|
|
134
|
+
try:
|
|
135
|
+
from quantilica.core.lake import load_lake_converter
|
|
136
|
+
|
|
137
|
+
converter = load_lake_converter(source_id)
|
|
138
|
+
if converter is not None:
|
|
139
|
+
datasets = list(converter.datasets())
|
|
140
|
+
origin = f"conversor '{source_id}'"
|
|
141
|
+
except Exception as exc:
|
|
142
|
+
err_console.print(
|
|
143
|
+
f"[yellow]Aviso:[/yellow] não carreguei o conversor '{escape(source_id)}' "
|
|
144
|
+
f"({escape(str(exc))}); uso os arquivos de docs/datasets/.",
|
|
145
|
+
highlight=False,
|
|
146
|
+
)
|
|
147
|
+
if datasets is None:
|
|
148
|
+
datasets = _page_stems(repo_dir)
|
|
149
|
+
origin = "arquivos de docs/datasets/"
|
|
150
|
+
err_console.print(
|
|
151
|
+
f"[dim]Datasets descobertos pelos {origin} (fetcher não instalado): "
|
|
152
|
+
"dataset sem página não é detectado.[/dim]",
|
|
153
|
+
highlight=False,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
problems = docs.check_fetcher_docs(docs_dir, datasets, source_id)
|
|
157
|
+
errors = [p for p in problems if p.severity == "error"]
|
|
158
|
+
warnings = [p for p in problems if p.severity == "warning"]
|
|
159
|
+
for problem in problems:
|
|
160
|
+
color = "yellow" if problem.severity == "warning" else "red"
|
|
161
|
+
label = "aviso" if problem.severity == "warning" else "erro"
|
|
162
|
+
console.print(
|
|
163
|
+
f"[{color}]{label}[/{color}] {escape(problem.path)}: "
|
|
164
|
+
f"{escape(problem.message)}",
|
|
165
|
+
highlight=False,
|
|
166
|
+
)
|
|
167
|
+
if errors:
|
|
168
|
+
console.print(
|
|
169
|
+
f"[red]✗[/red] {len(errors)} erro(s), {len(warnings)} aviso(s) em "
|
|
170
|
+
f"{len(datasets)} dataset(s) de {escape(source_id)}.",
|
|
171
|
+
highlight=False,
|
|
172
|
+
)
|
|
173
|
+
raise typer.Exit(EXIT_FAILURE)
|
|
174
|
+
console.print(
|
|
175
|
+
f"[green]✓[/green] {len(datasets)} dataset(s) de {escape(source_id)} "
|
|
176
|
+
f"documentado(s); {len(warnings)} aviso(s).",
|
|
177
|
+
highlight=False,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@app.command("schema")
|
|
182
|
+
def schema(
|
|
183
|
+
source_id: Annotated[str, typer.Argument(help="Id da fonte (pasta no lake)")],
|
|
184
|
+
lake: Annotated[
|
|
185
|
+
Path,
|
|
186
|
+
typer.Option("--lake", help="Raiz do lake", exists=True, file_okay=False),
|
|
187
|
+
],
|
|
188
|
+
out: Annotated[
|
|
189
|
+
Path,
|
|
190
|
+
typer.Option(
|
|
191
|
+
"--out",
|
|
192
|
+
help="Pasta docs/datasets do fetcher onde gravar os <id>.schema.json",
|
|
193
|
+
file_okay=False,
|
|
194
|
+
),
|
|
195
|
+
],
|
|
196
|
+
dataset: Annotated[
|
|
197
|
+
list[str] | None,
|
|
198
|
+
typer.Option(
|
|
199
|
+
"--dataset",
|
|
200
|
+
help="Dataset a atualizar (repetível); padrão: todos os da fonte no lake",
|
|
201
|
+
),
|
|
202
|
+
] = None,
|
|
203
|
+
) -> None:
|
|
204
|
+
"""Grava os esquemas de referência (<id>.schema.json) lendo o lake."""
|
|
205
|
+
docs = _import_docs()
|
|
206
|
+
source_dir = lake / source_id
|
|
207
|
+
if not source_dir.is_dir():
|
|
208
|
+
raise _fail(f"a fonte '{source_id}' não está no lake: {source_dir} não existe.")
|
|
209
|
+
wanted = dataset or sorted(
|
|
210
|
+
p.name
|
|
211
|
+
for p in source_dir.iterdir()
|
|
212
|
+
if p.is_dir() and not p.name.startswith(("_", "."))
|
|
213
|
+
)
|
|
214
|
+
if not wanted:
|
|
215
|
+
raise _fail(f"nenhum dataset em {source_dir}.")
|
|
216
|
+
|
|
217
|
+
failed = 0
|
|
218
|
+
for dataset_id in wanted:
|
|
219
|
+
try:
|
|
220
|
+
data = docs.schema_from_lake(lake, source_id, dataset_id)
|
|
221
|
+
except (FileNotFoundError, OSError) as exc:
|
|
222
|
+
err_console.print(
|
|
223
|
+
f"[red]erro[/red] {escape(dataset_id)}: {escape(str(exc))}",
|
|
224
|
+
highlight=False,
|
|
225
|
+
)
|
|
226
|
+
failed += 1
|
|
227
|
+
continue
|
|
228
|
+
path = out / f"{dataset_id}.schema.json"
|
|
229
|
+
changed = docs.write_schema(path, data)
|
|
230
|
+
state = "gravado" if changed else "sem mudança"
|
|
231
|
+
console.print(
|
|
232
|
+
f"[green]✓[/green] {escape(dataset_id)}: {state} "
|
|
233
|
+
f"({len(data['colunas'])} colunas, {data['arquivos']} arquivos)",
|
|
234
|
+
highlight=False,
|
|
235
|
+
)
|
|
236
|
+
if failed:
|
|
237
|
+
raise typer.Exit(EXIT_FAILURE)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
@app.command("update")
|
|
241
|
+
def update(
|
|
242
|
+
repo_dir: Annotated[
|
|
243
|
+
Path,
|
|
244
|
+
typer.Argument(
|
|
245
|
+
help="Raiz do repositório do fetcher (a pasta que contém docs/)",
|
|
246
|
+
exists=True,
|
|
247
|
+
file_okay=False,
|
|
248
|
+
),
|
|
249
|
+
],
|
|
250
|
+
) -> None:
|
|
251
|
+
"""Reescreve a ficha técnica das páginas a partir do cabeçalho."""
|
|
252
|
+
docs = _import_docs()
|
|
253
|
+
folder = _datasets_dir(repo_dir)
|
|
254
|
+
pages = sorted(folder.glob("*.md")) if folder.is_dir() else []
|
|
255
|
+
if not pages:
|
|
256
|
+
raise _fail(f"nenhuma página em {folder}.")
|
|
257
|
+
failed = 0
|
|
258
|
+
for page in pages:
|
|
259
|
+
try:
|
|
260
|
+
changed = docs.update_ficha(page)
|
|
261
|
+
except docs.DocFormatError as exc:
|
|
262
|
+
err_console.print(
|
|
263
|
+
f"[red]erro[/red] {escape(page.name)}: {escape(str(exc))}",
|
|
264
|
+
highlight=False,
|
|
265
|
+
)
|
|
266
|
+
failed += 1
|
|
267
|
+
continue
|
|
268
|
+
state = "atualizada" if changed else "sem mudança"
|
|
269
|
+
console.print(f"[green]✓[/green] {escape(page.name)}: {state}", highlight=False)
|
|
270
|
+
if failed:
|
|
271
|
+
raise typer.Exit(EXIT_FAILURE)
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Testes dos comandos `quantilica docs` (check, schema, update)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from types import SimpleNamespace
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
from typer.testing import CliRunner
|
|
11
|
+
|
|
12
|
+
pytest.importorskip("yaml")
|
|
13
|
+
pytest.importorskip("pyarrow")
|
|
14
|
+
pl = pytest.importorskip("polars")
|
|
15
|
+
pytest.importorskip("quantilica.core.docs")
|
|
16
|
+
|
|
17
|
+
from quantilica.cli.cli import app # noqa: E402
|
|
18
|
+
|
|
19
|
+
runner = CliRunner()
|
|
20
|
+
|
|
21
|
+
SECTIONS = [
|
|
22
|
+
"Resumo",
|
|
23
|
+
"Ficha técnica",
|
|
24
|
+
"Origem e coleta",
|
|
25
|
+
"Conteúdo",
|
|
26
|
+
"Variáveis",
|
|
27
|
+
"No lake",
|
|
28
|
+
"Cuidados de uso",
|
|
29
|
+
"Exemplos",
|
|
30
|
+
"Referências",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
HEADER = """\
|
|
34
|
+
---
|
|
35
|
+
dataset:
|
|
36
|
+
id: vendas
|
|
37
|
+
fonte: alfa
|
|
38
|
+
titulo: Vendas
|
|
39
|
+
resumo: Vendas mensais.
|
|
40
|
+
orgao: Órgão X
|
|
41
|
+
pagina_oficial: https://example.org
|
|
42
|
+
licenca: CC-BY 4.0
|
|
43
|
+
periodicidade: mensal
|
|
44
|
+
cobertura: {inicio: 2020-01, fim: em-curso}
|
|
45
|
+
unidade: uma venda
|
|
46
|
+
granularidade: {espacial: Brasil, temporal: mês}
|
|
47
|
+
formato_bruto: CSV
|
|
48
|
+
lake: {estado: convertido, particoes: [ano], chave: [id_venda], colunas: 2}
|
|
49
|
+
unidades_de_medida: {valor: R$}
|
|
50
|
+
dados_pessoais: nenhum
|
|
51
|
+
eras: []
|
|
52
|
+
revisado_em: 2026-11-07
|
|
53
|
+
versao_fetcher: 1.0.0
|
|
54
|
+
---
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
VARS = (
|
|
58
|
+
"| Coluna no lake | Na fonte | Tipo | Descrição | Unidade / códigos | Período |\n"
|
|
59
|
+
"|---|---|---|---|---|---|\n"
|
|
60
|
+
"| `id_venda` | ID | int | id | | |\n"
|
|
61
|
+
"| `valor` | VLR | double | valor | R$ | |\n"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def page_text(skip: str | None = None) -> str:
|
|
66
|
+
text = HEADER + "\n# Vendas\n\n"
|
|
67
|
+
for title in SECTIONS:
|
|
68
|
+
if title == skip:
|
|
69
|
+
continue
|
|
70
|
+
text += f"## {title}\n\n"
|
|
71
|
+
if title == "Ficha técnica":
|
|
72
|
+
text += "<!-- gerado:ficha -->\n\n"
|
|
73
|
+
if title == "Variáveis":
|
|
74
|
+
text += VARS + "\n"
|
|
75
|
+
return text
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def make_repo(root: Path, *, skip: str | None = None, entry_point: bool = True) -> Path:
|
|
79
|
+
datasets = root / "docs" / "datasets"
|
|
80
|
+
datasets.mkdir(parents=True)
|
|
81
|
+
(root / "docs" / "README.md").write_text("# Alfa\n", encoding="utf-8")
|
|
82
|
+
(datasets / "vendas.md").write_text(page_text(skip), encoding="utf-8")
|
|
83
|
+
schema = {
|
|
84
|
+
"fonte": "alfa",
|
|
85
|
+
"dataset": "vendas",
|
|
86
|
+
"gerado_em": "2026-11-07T00:00:00+00:00",
|
|
87
|
+
"particoes": ["ano"],
|
|
88
|
+
"colunas": [
|
|
89
|
+
{"nome": "id_venda", "tipos": ["int64"]},
|
|
90
|
+
{"nome": "valor", "tipos": ["double"]},
|
|
91
|
+
],
|
|
92
|
+
"arquivos": 1,
|
|
93
|
+
}
|
|
94
|
+
(datasets / "vendas.schema.json").write_text(json.dumps(schema), encoding="utf-8")
|
|
95
|
+
if entry_point:
|
|
96
|
+
(root / "pyproject.toml").write_text(
|
|
97
|
+
'[project]\nname = "alfa-fetcher"\n'
|
|
98
|
+
'[project.entry-points."quantilica.lake"]\nalfa = "alfa.lake:converter"\n',
|
|
99
|
+
encoding="utf-8",
|
|
100
|
+
)
|
|
101
|
+
return root
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def no_converter(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
105
|
+
monkeypatch.setattr("quantilica.core.lake.load_lake_converter", lambda _s: None)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def test_docs_help_lists_commands() -> None:
|
|
109
|
+
result = runner.invoke(app, ["docs", "--help"])
|
|
110
|
+
assert result.exit_code == 0
|
|
111
|
+
for name in ("check", "schema", "update"):
|
|
112
|
+
assert name in result.output
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_check_ok_from_files(tmp_path: Path, monkeypatch) -> None:
|
|
116
|
+
no_converter(monkeypatch)
|
|
117
|
+
repo = make_repo(tmp_path)
|
|
118
|
+
result = runner.invoke(app, ["docs", "check", str(repo)])
|
|
119
|
+
assert result.exit_code == 0, result.output
|
|
120
|
+
assert "1 dataset(s) de alfa" in result.output
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_check_reports_problems_and_exits_1(tmp_path: Path, monkeypatch) -> None:
|
|
124
|
+
no_converter(monkeypatch)
|
|
125
|
+
repo = make_repo(tmp_path, skip="Exemplos")
|
|
126
|
+
result = runner.invoke(app, ["docs", "check", str(repo)])
|
|
127
|
+
assert result.exit_code == 1
|
|
128
|
+
assert "seções ausentes: Exemplos" in result.output
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def test_check_uses_installed_converter_datasets(tmp_path: Path, monkeypatch) -> None:
|
|
132
|
+
converter = SimpleNamespace(source_id="alfa", datasets=lambda: ["vendas", "extra"])
|
|
133
|
+
monkeypatch.setattr(
|
|
134
|
+
"quantilica.core.lake.load_lake_converter", lambda _s: converter
|
|
135
|
+
)
|
|
136
|
+
repo = make_repo(tmp_path)
|
|
137
|
+
result = runner.invoke(app, ["docs", "check", str(repo)])
|
|
138
|
+
assert result.exit_code == 1
|
|
139
|
+
assert "dataset declarado sem página" in result.output
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_check_source_from_pages_and_option(tmp_path: Path, monkeypatch) -> None:
|
|
143
|
+
no_converter(monkeypatch)
|
|
144
|
+
repo = make_repo(tmp_path, entry_point=False)
|
|
145
|
+
assert runner.invoke(app, ["docs", "check", str(repo)]).exit_code == 0
|
|
146
|
+
wrong = runner.invoke(app, ["docs", "check", str(repo), "--source", "beta"])
|
|
147
|
+
assert wrong.exit_code == 1
|
|
148
|
+
assert "fonte 'alfa' diferente" in wrong.output
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def test_check_without_docs_dir(tmp_path: Path) -> None:
|
|
152
|
+
result = runner.invoke(app, ["docs", "check", str(tmp_path)])
|
|
153
|
+
assert result.exit_code == 1
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_update_rewrites_ficha_and_is_idempotent(tmp_path: Path) -> None:
|
|
157
|
+
repo = make_repo(tmp_path)
|
|
158
|
+
page = repo / "docs" / "datasets" / "vendas.md"
|
|
159
|
+
first = runner.invoke(app, ["docs", "update", str(repo)])
|
|
160
|
+
assert first.exit_code == 0, first.output
|
|
161
|
+
assert "atualizada" in first.output
|
|
162
|
+
assert "| Dataset | `alfa/vendas` |" in page.read_text(encoding="utf-8")
|
|
163
|
+
second = runner.invoke(app, ["docs", "update", str(repo)])
|
|
164
|
+
assert second.exit_code == 0
|
|
165
|
+
assert "sem mudança" in second.output
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def test_update_fails_on_invalid_page(tmp_path: Path) -> None:
|
|
169
|
+
repo = make_repo(tmp_path)
|
|
170
|
+
(repo / "docs" / "datasets" / "vendas.md").write_text("sem cabeçalho", "utf-8")
|
|
171
|
+
result = runner.invoke(app, ["docs", "update", str(repo)])
|
|
172
|
+
assert result.exit_code == 1
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def make_lake(root: Path) -> Path:
|
|
176
|
+
for mes in (1, 2):
|
|
177
|
+
folder = root / "alfa" / "vendas" / "ano=2024" / f"mes={mes}"
|
|
178
|
+
folder.mkdir(parents=True)
|
|
179
|
+
pl.DataFrame({"id_venda": [1], "valor": [1.5]}).write_parquet(
|
|
180
|
+
folder / "p.parquet"
|
|
181
|
+
)
|
|
182
|
+
(root / "alfa" / "vendas" / "_dataset.json").write_text("{}", encoding="utf-8")
|
|
183
|
+
return root
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_schema_writes_files(tmp_path: Path) -> None:
|
|
187
|
+
lake = make_lake(tmp_path / "lake")
|
|
188
|
+
out = tmp_path / "out"
|
|
189
|
+
result = runner.invoke(
|
|
190
|
+
app, ["docs", "schema", "alfa", "--lake", str(lake), "--out", str(out)]
|
|
191
|
+
)
|
|
192
|
+
assert result.exit_code == 0, result.output
|
|
193
|
+
data = json.loads((out / "vendas.schema.json").read_text(encoding="utf-8"))
|
|
194
|
+
assert data["particoes"] == ["ano", "mes"]
|
|
195
|
+
assert data["arquivos"] == 2
|
|
196
|
+
assert {c["nome"] for c in data["colunas"]} == {"id_venda", "valor"}
|
|
197
|
+
again = runner.invoke(
|
|
198
|
+
app, ["docs", "schema", "alfa", "--lake", str(lake), "--out", str(out)]
|
|
199
|
+
)
|
|
200
|
+
assert "sem mudança" in again.output
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def test_schema_dataset_filter_and_errors(tmp_path: Path) -> None:
|
|
204
|
+
lake = make_lake(tmp_path / "lake")
|
|
205
|
+
out = tmp_path / "out"
|
|
206
|
+
bad = runner.invoke(
|
|
207
|
+
app,
|
|
208
|
+
["docs", "schema", "alfa", "--lake", str(lake), "--out", str(out),
|
|
209
|
+
"--dataset", "nada"],
|
|
210
|
+
) # fmt: skip
|
|
211
|
+
assert bad.exit_code == 1
|
|
212
|
+
missing = runner.invoke(
|
|
213
|
+
app, ["docs", "schema", "zeta", "--lake", str(lake), "--out", str(out)]
|
|
214
|
+
)
|
|
215
|
+
assert missing.exit_code == 1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|