quantilica-cli 0.15.1__tar.gz → 0.17.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/CHANGELOG.md +27 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/PKG-INFO +4 -2
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/pyproject.toml +5 -3
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/src/quantilica/cli/cli.py +4 -0
- quantilica_cli-0.17.0/src/quantilica/cli/docs.py +271 -0
- quantilica_cli-0.17.0/src/quantilica/cli/registry.py +223 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/src/quantilica/cli/sdk.py +24 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/src/quantilica/cli/sources.py +24 -2
- quantilica_cli-0.17.0/tests/test_docs_cli.py +215 -0
- quantilica_cli-0.17.0/tests/test_registry_cli.py +165 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/.githooks/pre-commit +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/.githooks/pre-push +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/.github/workflows/public-hygiene.yml +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/.github/workflows/publish.yml +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/.github/workflows/test.yml +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/.gitignore +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/LICENSE +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/README.md +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/src/quantilica/cli/__init__.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/src/quantilica/cli/catalog.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/src/quantilica/cli/health.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/src/quantilica/cli/lake.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/src/quantilica/cli/manifests.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/src/quantilica/cli/ui.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/src/quantilica/py.typed +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/__init__.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/conftest.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/lake_build_fakes.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/lake_fixtures.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_catalog_cli.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_health.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_lake_cli.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_manifests.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_package_hygiene.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_sdk.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_sdk_backfill.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_sdk_force_probe.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_sdk_mes_string.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_sdk_onda_a2.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_sources.py +0 -0
- {quantilica_cli-0.15.1 → quantilica_cli-0.17.0}/tests/test_ui.py +0 -0
|
@@ -7,6 +7,33 @@ e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
|
|
|
7
7
|
|
|
8
8
|
## [Não lançado]
|
|
9
9
|
|
|
10
|
+
## [0.17.0] - 2026-11-10
|
|
11
|
+
|
|
12
|
+
### Adicionado
|
|
13
|
+
|
|
14
|
+
- `FetcherApp(info=FetcherInfo(...))`: o fetcher declara os fatos da fonte
|
|
15
|
+
(id canônico, apelidos, órgão, licença, página, cadência). Sem `info`, um
|
|
16
|
+
`FetcherInfo` mínimo é derivado de `name` e `info_declared` fica `False`;
|
|
17
|
+
`source_id` segue como antes.
|
|
18
|
+
- `quantilica registry export [--json] [--strict]`: descobre os entry points
|
|
19
|
+
`quantilica.fetchers` e `quantilica.lake` e monta o registro (info,
|
|
20
|
+
`freshness_policy`, `default_output`, grupos, `lake` e `lake_source_id`).
|
|
21
|
+
`--strict` falha se algum fetcher não declarar `info`, não carregar ou houver
|
|
22
|
+
id/apelido duplicado.
|
|
23
|
+
|
|
24
|
+
### Alterado
|
|
25
|
+
|
|
26
|
+
- `quantilica install` e afins: para fetchers instalados que declaram `info`,
|
|
27
|
+
id e apelidos vêm do `FetcherInfo`; `SOURCES_REGISTRY` fica como reserva
|
|
28
|
+
para instalar fontes ausentes. Teste garante que os dois não se contradizem.
|
|
29
|
+
|
|
30
|
+
## [0.16.0] - 2026-11-08
|
|
31
|
+
|
|
32
|
+
### Adicionado
|
|
33
|
+
|
|
34
|
+
- `quantilica docs`: ferramentas da norma de documentação dos dados dos fetchers. `quantilica docs check <repo> [--source]` verifica a pasta `docs/` (a fonte e os datasets vêm do conversor `quantilica.lake` instalado ou, sem ele, das páginas em `docs/datasets/`) e sai com 1 se houver erro; `quantilica docs schema <fonte> --lake <dir> --out <docs/datasets> [--dataset ...]` grava os esquemas de referência lendo o lake; `quantilica docs update <repo>` reescreve a ficha técnica das páginas.
|
|
35
|
+
- Os comandos `quantilica docs` exigem o `quantilica-core` com o extra `docs` (`pip install "quantilica-core[docs]"`); sem ele, falham com a instrução de instalação.
|
|
36
|
+
|
|
10
37
|
## [0.15.1] - 2026-11-06
|
|
11
38
|
|
|
12
39
|
### Alterado
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: quantilica-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.17.0
|
|
4
4
|
Summary: Unified CLI for Quantilica open data fetchers
|
|
5
5
|
Author-email: "Komesu, D.K." <daniel@dkko.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -16,9 +16,11 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.13
|
|
17
17
|
Classifier: Typing :: Typed
|
|
18
18
|
Requires-Python: >=3.12
|
|
19
|
-
Requires-Dist: quantilica-core>=0.
|
|
19
|
+
Requires-Dist: quantilica-core>=0.19.0
|
|
20
20
|
Requires-Dist: rich>=13.0.0
|
|
21
21
|
Requires-Dist: typer>=0.15.0
|
|
22
|
+
Provides-Extra: docs
|
|
23
|
+
Requires-Dist: quantilica-core[docs]>=0.18.0; extra == 'docs'
|
|
22
24
|
Provides-Extra: lake
|
|
23
25
|
Requires-Dist: quantilica-analytics>=0.11.0; extra == 'lake'
|
|
24
26
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "quantilica-cli"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.17.0"
|
|
4
4
|
description = "Unified CLI for Quantilica open data fetchers"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [{ name = "Komesu, D.K.", email = "daniel@dkko.me" }]
|
|
@@ -20,7 +20,7 @@ classifiers = [
|
|
|
20
20
|
"Programming Language :: Python :: 3.13",
|
|
21
21
|
]
|
|
22
22
|
dependencies = [
|
|
23
|
-
"quantilica-core>=0.
|
|
23
|
+
"quantilica-core>=0.19.0",
|
|
24
24
|
"rich>=13.0.0",
|
|
25
25
|
"typer>=0.15.0",
|
|
26
26
|
]
|
|
@@ -28,12 +28,14 @@ dependencies = [
|
|
|
28
28
|
[project.optional-dependencies]
|
|
29
29
|
# Motor de `quantilica lake build` e `quantilica lake audit`.
|
|
30
30
|
lake = ["quantilica-analytics>=0.11.0"]
|
|
31
|
+
# `quantilica docs`: verificação da documentação dos dados dos fetchers.
|
|
32
|
+
docs = ["quantilica-core[docs]>=0.18.0"]
|
|
31
33
|
|
|
32
34
|
[project.scripts]
|
|
33
35
|
quantilica = "quantilica.cli.cli:main"
|
|
34
36
|
|
|
35
37
|
[dependency-groups]
|
|
36
|
-
dev = ["ruff>=0.9.0", "pytest>=8.0.0", "pyarrow>=15.0.0"]
|
|
38
|
+
dev = ["ruff>=0.9.0", "pytest>=8.0.0", "pyarrow>=15.0.0", "pyyaml>=6.0", "polars>=1.0"]
|
|
37
39
|
|
|
38
40
|
# Os fetchers são instalados independentemente pelo usuário — quantilica-cli
|
|
39
41
|
# os descobre via entry points (grupo "quantilica.fetchers") sem dependência direta.
|
|
@@ -14,9 +14,11 @@ from rich.table import Table
|
|
|
14
14
|
|
|
15
15
|
from quantilica.cli import __version__
|
|
16
16
|
from quantilica.cli.catalog import app as catalog_app
|
|
17
|
+
from quantilica.cli.docs import app as docs_app
|
|
17
18
|
from quantilica.cli.health import cmd_health
|
|
18
19
|
from quantilica.cli.lake import app as lake_app
|
|
19
20
|
from quantilica.cli.manifests import app as manifests_app
|
|
21
|
+
from quantilica.cli.registry import app as registry_app
|
|
20
22
|
from quantilica.cli.sources import (
|
|
21
23
|
app as sources_app,
|
|
22
24
|
)
|
|
@@ -36,8 +38,10 @@ app = typer.Typer(
|
|
|
36
38
|
no_args_is_help=True,
|
|
37
39
|
)
|
|
38
40
|
app.add_typer(catalog_app, name="catalog")
|
|
41
|
+
app.add_typer(docs_app, name="docs")
|
|
39
42
|
app.add_typer(lake_app, name="lake")
|
|
40
43
|
app.add_typer(manifests_app, name="manifests")
|
|
44
|
+
app.add_typer(registry_app, name="registry")
|
|
41
45
|
app.add_typer(sources_app, name="sources")
|
|
42
46
|
|
|
43
47
|
# Adiciona comandos top-level install, uninstall, doctor e health
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""Comandos `quantilica docs` — a documentação dos dados dos fetchers.
|
|
2
|
+
|
|
3
|
+
Ferramentas da norma "Documentação dos dados": ``check`` confere a pasta
|
|
4
|
+
``docs/`` de um fetcher, ``schema`` grava os esquemas de referência a partir do
|
|
5
|
+
lake e ``update`` reescreve a ficha técnica das páginas.
|
|
6
|
+
|
|
7
|
+
Dependem do ``quantilica-core`` com o extra ``docs`` (``pyyaml`` e
|
|
8
|
+
``pyarrow``), importado só quando o comando roda, de modo que ``--help``
|
|
9
|
+
funciona sem ele.
|
|
10
|
+
|
|
11
|
+
Como o ``check`` descobre o que verificar:
|
|
12
|
+
|
|
13
|
+
- fonte: ``--source``; senão o entry point ``quantilica.lake`` declarado no
|
|
14
|
+
``pyproject.toml`` do repositório; senão o campo ``fonte`` das páginas;
|
|
15
|
+
- datasets: ``datasets()`` do conversor, se o fetcher estiver instalado
|
|
16
|
+
(entry point ``quantilica.lake``); senão os arquivos de ``docs/datasets/``
|
|
17
|
+
(e então a verificação "dataset sem página" não pode falhar).
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import tomllib
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Annotated
|
|
25
|
+
|
|
26
|
+
import typer
|
|
27
|
+
from rich.console import Console
|
|
28
|
+
from rich.markup import escape
|
|
29
|
+
|
|
30
|
+
app = typer.Typer(
|
|
31
|
+
help="Documentação dos dados dos fetchers: verificar, gerar esquemas e fichas.",
|
|
32
|
+
no_args_is_help=True,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
console = Console()
|
|
36
|
+
err_console = Console(stderr=True)
|
|
37
|
+
|
|
38
|
+
DOCS_HINT = 'uv add "quantilica-core[docs]" (ou: pip install "quantilica-core[docs]")'
|
|
39
|
+
LAKE_ENTRY_POINT_GROUP = "quantilica.lake"
|
|
40
|
+
|
|
41
|
+
EXIT_FAILURE = 1
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _fail(message: str, code: int = EXIT_FAILURE) -> typer.Exit:
|
|
45
|
+
err_console.print(f"[red]Erro:[/red] {escape(message)}", highlight=False)
|
|
46
|
+
return typer.Exit(code)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _import_docs():
|
|
50
|
+
try:
|
|
51
|
+
from quantilica.core import docs
|
|
52
|
+
except ImportError as exc:
|
|
53
|
+
raise _fail(
|
|
54
|
+
f"'quantilica docs' requer o quantilica-core com o extra docs "
|
|
55
|
+
f"({exc}). Instale com: {DOCS_HINT}"
|
|
56
|
+
) from None
|
|
57
|
+
return docs
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _source_from_pyproject(repo_dir: Path) -> str | None:
|
|
61
|
+
"""Nome do entry point ``quantilica.lake`` do ``pyproject.toml`` do repo."""
|
|
62
|
+
path = repo_dir / "pyproject.toml"
|
|
63
|
+
if not path.is_file():
|
|
64
|
+
return None
|
|
65
|
+
try:
|
|
66
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
67
|
+
except (OSError, tomllib.TOMLDecodeError):
|
|
68
|
+
return None
|
|
69
|
+
eps = data.get("project", {}).get("entry-points", {}).get(LAKE_ENTRY_POINT_GROUP)
|
|
70
|
+
if isinstance(eps, dict) and len(eps) == 1:
|
|
71
|
+
return next(iter(eps))
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _datasets_dir(repo_dir: Path) -> Path:
|
|
76
|
+
return repo_dir / "docs" / "datasets"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _page_stems(repo_dir: Path) -> list[str]:
|
|
80
|
+
folder = _datasets_dir(repo_dir)
|
|
81
|
+
return sorted(p.stem for p in folder.glob("*.md")) if folder.is_dir() else []
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _sources_in_pages(docs, repo_dir: Path) -> set[str]:
|
|
85
|
+
found: set[str] = set()
|
|
86
|
+
for stem in _page_stems(repo_dir):
|
|
87
|
+
try:
|
|
88
|
+
header, _ = docs.parse_dataset_page(_datasets_dir(repo_dir) / f"{stem}.md")
|
|
89
|
+
except docs.DocFormatError:
|
|
90
|
+
continue
|
|
91
|
+
found.add(header.fonte)
|
|
92
|
+
return found
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@app.command("check")
|
|
96
|
+
def check(
|
|
97
|
+
repo_dir: Annotated[
|
|
98
|
+
Path,
|
|
99
|
+
typer.Argument(
|
|
100
|
+
help="Raiz do repositório do fetcher (a pasta que contém docs/)",
|
|
101
|
+
exists=True,
|
|
102
|
+
file_okay=False,
|
|
103
|
+
),
|
|
104
|
+
],
|
|
105
|
+
source: Annotated[
|
|
106
|
+
str | None,
|
|
107
|
+
typer.Option(
|
|
108
|
+
"--source",
|
|
109
|
+
help="Id da fonte; padrão: entry point do pyproject.toml ou o campo "
|
|
110
|
+
"'fonte' das páginas",
|
|
111
|
+
),
|
|
112
|
+
] = None,
|
|
113
|
+
) -> None:
|
|
114
|
+
"""Verifica a documentação dos dados de um fetcher (sai com 1 se houver erro)."""
|
|
115
|
+
docs = _import_docs()
|
|
116
|
+
docs_dir = repo_dir / "docs"
|
|
117
|
+
if not docs_dir.is_dir():
|
|
118
|
+
raise _fail(f"{repo_dir} não tem a pasta docs/.")
|
|
119
|
+
|
|
120
|
+
source_id = source or _source_from_pyproject(repo_dir)
|
|
121
|
+
if source_id is None:
|
|
122
|
+
sources = _sources_in_pages(docs, repo_dir)
|
|
123
|
+
if len(sources) == 1:
|
|
124
|
+
source_id = next(iter(sources))
|
|
125
|
+
else:
|
|
126
|
+
raise _fail(
|
|
127
|
+
"não consegui descobrir a fonte (pyproject.toml sem entry point "
|
|
128
|
+
f"'{LAKE_ENTRY_POINT_GROUP}' e páginas com fontes {sorted(sources)}); "
|
|
129
|
+
"informe --source."
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
datasets: list[str] | None = None
|
|
133
|
+
origin = ""
|
|
134
|
+
try:
|
|
135
|
+
from quantilica.core.lake import load_lake_converter
|
|
136
|
+
|
|
137
|
+
converter = load_lake_converter(source_id)
|
|
138
|
+
if converter is not None:
|
|
139
|
+
datasets = list(converter.datasets())
|
|
140
|
+
origin = f"conversor '{source_id}'"
|
|
141
|
+
except Exception as exc:
|
|
142
|
+
err_console.print(
|
|
143
|
+
f"[yellow]Aviso:[/yellow] não carreguei o conversor '{escape(source_id)}' "
|
|
144
|
+
f"({escape(str(exc))}); uso os arquivos de docs/datasets/.",
|
|
145
|
+
highlight=False,
|
|
146
|
+
)
|
|
147
|
+
if datasets is None:
|
|
148
|
+
datasets = _page_stems(repo_dir)
|
|
149
|
+
origin = "arquivos de docs/datasets/"
|
|
150
|
+
err_console.print(
|
|
151
|
+
f"[dim]Datasets descobertos pelos {origin} (fetcher não instalado): "
|
|
152
|
+
"dataset sem página não é detectado.[/dim]",
|
|
153
|
+
highlight=False,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
problems = docs.check_fetcher_docs(docs_dir, datasets, source_id)
|
|
157
|
+
errors = [p for p in problems if p.severity == "error"]
|
|
158
|
+
warnings = [p for p in problems if p.severity == "warning"]
|
|
159
|
+
for problem in problems:
|
|
160
|
+
color = "yellow" if problem.severity == "warning" else "red"
|
|
161
|
+
label = "aviso" if problem.severity == "warning" else "erro"
|
|
162
|
+
console.print(
|
|
163
|
+
f"[{color}]{label}[/{color}] {escape(problem.path)}: "
|
|
164
|
+
f"{escape(problem.message)}",
|
|
165
|
+
highlight=False,
|
|
166
|
+
)
|
|
167
|
+
if errors:
|
|
168
|
+
console.print(
|
|
169
|
+
f"[red]✗[/red] {len(errors)} erro(s), {len(warnings)} aviso(s) em "
|
|
170
|
+
f"{len(datasets)} dataset(s) de {escape(source_id)}.",
|
|
171
|
+
highlight=False,
|
|
172
|
+
)
|
|
173
|
+
raise typer.Exit(EXIT_FAILURE)
|
|
174
|
+
console.print(
|
|
175
|
+
f"[green]✓[/green] {len(datasets)} dataset(s) de {escape(source_id)} "
|
|
176
|
+
f"documentado(s); {len(warnings)} aviso(s).",
|
|
177
|
+
highlight=False,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@app.command("schema")
|
|
182
|
+
def schema(
|
|
183
|
+
source_id: Annotated[str, typer.Argument(help="Id da fonte (pasta no lake)")],
|
|
184
|
+
lake: Annotated[
|
|
185
|
+
Path,
|
|
186
|
+
typer.Option("--lake", help="Raiz do lake", exists=True, file_okay=False),
|
|
187
|
+
],
|
|
188
|
+
out: Annotated[
|
|
189
|
+
Path,
|
|
190
|
+
typer.Option(
|
|
191
|
+
"--out",
|
|
192
|
+
help="Pasta docs/datasets do fetcher onde gravar os <id>.schema.json",
|
|
193
|
+
file_okay=False,
|
|
194
|
+
),
|
|
195
|
+
],
|
|
196
|
+
dataset: Annotated[
|
|
197
|
+
list[str] | None,
|
|
198
|
+
typer.Option(
|
|
199
|
+
"--dataset",
|
|
200
|
+
help="Dataset a atualizar (repetível); padrão: todos os da fonte no lake",
|
|
201
|
+
),
|
|
202
|
+
] = None,
|
|
203
|
+
) -> None:
|
|
204
|
+
"""Grava os esquemas de referência (<id>.schema.json) lendo o lake."""
|
|
205
|
+
docs = _import_docs()
|
|
206
|
+
source_dir = lake / source_id
|
|
207
|
+
if not source_dir.is_dir():
|
|
208
|
+
raise _fail(f"a fonte '{source_id}' não está no lake: {source_dir} não existe.")
|
|
209
|
+
wanted = dataset or sorted(
|
|
210
|
+
p.name
|
|
211
|
+
for p in source_dir.iterdir()
|
|
212
|
+
if p.is_dir() and not p.name.startswith(("_", "."))
|
|
213
|
+
)
|
|
214
|
+
if not wanted:
|
|
215
|
+
raise _fail(f"nenhum dataset em {source_dir}.")
|
|
216
|
+
|
|
217
|
+
failed = 0
|
|
218
|
+
for dataset_id in wanted:
|
|
219
|
+
try:
|
|
220
|
+
data = docs.schema_from_lake(lake, source_id, dataset_id)
|
|
221
|
+
except (FileNotFoundError, OSError) as exc:
|
|
222
|
+
err_console.print(
|
|
223
|
+
f"[red]erro[/red] {escape(dataset_id)}: {escape(str(exc))}",
|
|
224
|
+
highlight=False,
|
|
225
|
+
)
|
|
226
|
+
failed += 1
|
|
227
|
+
continue
|
|
228
|
+
path = out / f"{dataset_id}.schema.json"
|
|
229
|
+
changed = docs.write_schema(path, data)
|
|
230
|
+
state = "gravado" if changed else "sem mudança"
|
|
231
|
+
console.print(
|
|
232
|
+
f"[green]✓[/green] {escape(dataset_id)}: {state} "
|
|
233
|
+
f"({len(data['colunas'])} colunas, {data['arquivos']} arquivos)",
|
|
234
|
+
highlight=False,
|
|
235
|
+
)
|
|
236
|
+
if failed:
|
|
237
|
+
raise typer.Exit(EXIT_FAILURE)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
@app.command("update")
|
|
241
|
+
def update(
|
|
242
|
+
repo_dir: Annotated[
|
|
243
|
+
Path,
|
|
244
|
+
typer.Argument(
|
|
245
|
+
help="Raiz do repositório do fetcher (a pasta que contém docs/)",
|
|
246
|
+
exists=True,
|
|
247
|
+
file_okay=False,
|
|
248
|
+
),
|
|
249
|
+
],
|
|
250
|
+
) -> None:
|
|
251
|
+
"""Reescreve a ficha técnica das páginas a partir do cabeçalho."""
|
|
252
|
+
docs = _import_docs()
|
|
253
|
+
folder = _datasets_dir(repo_dir)
|
|
254
|
+
pages = sorted(folder.glob("*.md")) if folder.is_dir() else []
|
|
255
|
+
if not pages:
|
|
256
|
+
raise _fail(f"nenhuma página em {folder}.")
|
|
257
|
+
failed = 0
|
|
258
|
+
for page in pages:
|
|
259
|
+
try:
|
|
260
|
+
changed = docs.update_ficha(page)
|
|
261
|
+
except docs.DocFormatError as exc:
|
|
262
|
+
err_console.print(
|
|
263
|
+
f"[red]erro[/red] {escape(page.name)}: {escape(str(exc))}",
|
|
264
|
+
highlight=False,
|
|
265
|
+
)
|
|
266
|
+
failed += 1
|
|
267
|
+
continue
|
|
268
|
+
state = "atualizada" if changed else "sem mudança"
|
|
269
|
+
console.print(f"[green]✓[/green] {escape(page.name)}: {state}", highlight=False)
|
|
270
|
+
if failed:
|
|
271
|
+
raise typer.Exit(EXIT_FAILURE)
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""Comando ``quantilica registry`` — o registro dos fetchers instalados.
|
|
2
|
+
|
|
3
|
+
Descobre os entry points ``quantilica.fetchers`` (e ``quantilica.lake``) e monta
|
|
4
|
+
o ``registry.json`` com :func:`quantilica.core.registry.build_registry`.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextlib
|
|
10
|
+
import json
|
|
11
|
+
import sys
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from importlib.metadata import entry_points
|
|
14
|
+
from typing import Annotated, Any
|
|
15
|
+
|
|
16
|
+
import typer
|
|
17
|
+
from quantilica.core.registry import (
|
|
18
|
+
FetcherInfo,
|
|
19
|
+
RegistryError,
|
|
20
|
+
build_registry,
|
|
21
|
+
check_unique,
|
|
22
|
+
)
|
|
23
|
+
from rich.console import Console
|
|
24
|
+
from rich.table import Table
|
|
25
|
+
|
|
26
|
+
FETCHER_GROUP = "quantilica.fetchers"
|
|
27
|
+
LAKE_GROUP = "quantilica.lake"
|
|
28
|
+
|
|
29
|
+
app = typer.Typer(
|
|
30
|
+
help="Registro dos fetchers instalados (ids, apelidos, fatos da fonte).",
|
|
31
|
+
no_args_is_help=True,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@app.callback()
|
|
36
|
+
def _registry_group() -> None:
|
|
37
|
+
"""Registro dos fetchers instalados."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class DiscoveredFetcher:
|
|
42
|
+
"""Um fetcher instalado, com o que o registro precisa saber dele."""
|
|
43
|
+
|
|
44
|
+
info: FetcherInfo
|
|
45
|
+
info_declared: bool
|
|
46
|
+
entry_point: str
|
|
47
|
+
distribution: str | None
|
|
48
|
+
source_id: str | None = None
|
|
49
|
+
freshness_policy: str | None = None
|
|
50
|
+
default_output: str | None = None
|
|
51
|
+
groups: list[str] | None = None
|
|
52
|
+
lake: bool = False
|
|
53
|
+
lake_source_id: str | None = None
|
|
54
|
+
error: str | None = None
|
|
55
|
+
|
|
56
|
+
def to_dict(self) -> dict[str, Any]:
|
|
57
|
+
"""Info do fetcher mais os campos de execução."""
|
|
58
|
+
return {
|
|
59
|
+
**self.info.to_dict(),
|
|
60
|
+
"info_declared": self.info_declared,
|
|
61
|
+
"entry_point": self.entry_point,
|
|
62
|
+
"distribution": self.distribution,
|
|
63
|
+
"source_id": self.source_id,
|
|
64
|
+
"freshness_policy": self.freshness_policy,
|
|
65
|
+
"default_output": self.default_output,
|
|
66
|
+
"groups": self.groups,
|
|
67
|
+
"lake": self.lake,
|
|
68
|
+
"lake_source_id": self.lake_source_id,
|
|
69
|
+
"error": self.error,
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _fallback_info(ep_name: str, dist: str | None) -> FetcherInfo:
|
|
74
|
+
from quantilica.cli.sdk import derive_fetcher_info
|
|
75
|
+
|
|
76
|
+
return derive_fetcher_info(dist or ep_name)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _find_fetcher_app(obj: Any) -> Any | None:
|
|
80
|
+
"""Acha o ``FetcherApp`` a partir do objeto carregado do entry point."""
|
|
81
|
+
from quantilica.cli.sdk import FetcherApp
|
|
82
|
+
|
|
83
|
+
if isinstance(obj, FetcherApp):
|
|
84
|
+
return obj
|
|
85
|
+
candidate = getattr(obj, "fetcher_app", None)
|
|
86
|
+
return candidate if isinstance(candidate, FetcherApp) else None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def discover_fetchers() -> list[DiscoveredFetcher]:
|
|
90
|
+
"""Carrega os entry points de fetchers e devolve o que cada um declara.
|
|
91
|
+
|
|
92
|
+
Nunca levanta por causa de um plugin quebrado: o erro vai em ``error``.
|
|
93
|
+
A saída padrão durante o carregamento é desviada para o stderr (módulos de
|
|
94
|
+
fetchers podem imprimir ao importar).
|
|
95
|
+
"""
|
|
96
|
+
lake_eps = list(entry_points(group=LAKE_GROUP))
|
|
97
|
+
found: list[DiscoveredFetcher] = []
|
|
98
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
99
|
+
for ep in entry_points(group=FETCHER_GROUP):
|
|
100
|
+
dist = ep.dist.name if ep.dist is not None else None
|
|
101
|
+
try:
|
|
102
|
+
fetcher = _find_fetcher_app(ep.load())
|
|
103
|
+
error = None
|
|
104
|
+
except Exception as exc:
|
|
105
|
+
fetcher, error = None, f"{type(exc).__name__}: {exc}"
|
|
106
|
+
if fetcher is not None:
|
|
107
|
+
policy = fetcher.freshness_policy
|
|
108
|
+
item = DiscoveredFetcher(
|
|
109
|
+
info=fetcher.info,
|
|
110
|
+
info_declared=fetcher.info_declared,
|
|
111
|
+
entry_point=ep.name,
|
|
112
|
+
distribution=dist,
|
|
113
|
+
source_id=fetcher.source_id,
|
|
114
|
+
freshness_policy=(
|
|
115
|
+
None
|
|
116
|
+
if policy is None
|
|
117
|
+
else str(getattr(policy, "value", policy))
|
|
118
|
+
),
|
|
119
|
+
default_output=str(fetcher.default_output),
|
|
120
|
+
groups=sorted(fetcher.groups),
|
|
121
|
+
)
|
|
122
|
+
else:
|
|
123
|
+
item = DiscoveredFetcher(
|
|
124
|
+
info=_fallback_info(ep.name, dist),
|
|
125
|
+
info_declared=False,
|
|
126
|
+
entry_point=ep.name,
|
|
127
|
+
distribution=dist,
|
|
128
|
+
error=error,
|
|
129
|
+
)
|
|
130
|
+
for lep in lake_eps:
|
|
131
|
+
ldist = lep.dist.name if lep.dist is not None else None
|
|
132
|
+
if (dist is not None and ldist == dist) or lep.name in (
|
|
133
|
+
item.info.names
|
|
134
|
+
):
|
|
135
|
+
item.lake = True
|
|
136
|
+
item.lake_source_id = lep.name
|
|
137
|
+
break
|
|
138
|
+
found.append(item)
|
|
139
|
+
return found
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def build_export(found: list[DiscoveredFetcher]) -> tuple[dict[str, Any], str | None]:
|
|
143
|
+
"""Monta o registro; devolve também a mensagem de duplicidade, se houver."""
|
|
144
|
+
infos = [d.info for d in found]
|
|
145
|
+
problem: str | None = None
|
|
146
|
+
try:
|
|
147
|
+
registry = build_registry(infos)
|
|
148
|
+
except RegistryError as exc:
|
|
149
|
+
problem = str(exc)
|
|
150
|
+
registry = {"version": 1, "fetchers": []}
|
|
151
|
+
extras = sorted(found, key=lambda d: (d.info.id, d.entry_point))
|
|
152
|
+
registry["fetchers"] = [d.to_dict() for d in extras]
|
|
153
|
+
return registry, problem
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@app.command("export")
|
|
157
|
+
def cmd_export(
|
|
158
|
+
as_json: Annotated[
|
|
159
|
+
bool, typer.Option("--json", help="Saída em JSON puro no stdout.")
|
|
160
|
+
] = False,
|
|
161
|
+
strict: Annotated[
|
|
162
|
+
bool,
|
|
163
|
+
typer.Option(
|
|
164
|
+
"--strict",
|
|
165
|
+
help="Falha se algum fetcher não declarar info, não carregar "
|
|
166
|
+
"ou houver id/apelido duplicado.",
|
|
167
|
+
),
|
|
168
|
+
] = False,
|
|
169
|
+
) -> None:
|
|
170
|
+
"""Exporta o registro dos fetchers instalados (entry points)."""
|
|
171
|
+
found = discover_fetchers()
|
|
172
|
+
registry, duplicate = build_export(found)
|
|
173
|
+
if duplicate is None:
|
|
174
|
+
with contextlib.suppress(RegistryError):
|
|
175
|
+
check_unique(d.info for d in found)
|
|
176
|
+
undeclared = [d.info.id for d in found if not d.info_declared]
|
|
177
|
+
errors = [f"{d.entry_point}: {d.error}" for d in found if d.error]
|
|
178
|
+
|
|
179
|
+
err = Console(stderr=True)
|
|
180
|
+
if as_json:
|
|
181
|
+
sys.stdout.write(json.dumps(registry, ensure_ascii=False, indent=2) + "\n")
|
|
182
|
+
else:
|
|
183
|
+
table = Table(title=f"Registro de fetchers ({len(found)})")
|
|
184
|
+
for col in ("Id", "Apelidos", "Órgão", "Cadência", "Frescor", "Info", "Lake"):
|
|
185
|
+
table.add_column(col)
|
|
186
|
+
for d, row in zip(
|
|
187
|
+
sorted(found, key=lambda d: (d.info.id, d.entry_point)),
|
|
188
|
+
registry["fetchers"],
|
|
189
|
+
strict=True,
|
|
190
|
+
): # mesma ordenação de build_export
|
|
191
|
+
table.add_row(
|
|
192
|
+
d.info.id,
|
|
193
|
+
", ".join(d.info.aliases) or "-",
|
|
194
|
+
d.info.orgao or "-",
|
|
195
|
+
d.info.cadencia or "-",
|
|
196
|
+
d.freshness_policy or "-",
|
|
197
|
+
"[green]sim[/green]" if d.info_declared else "[yellow]não[/yellow]",
|
|
198
|
+
"sim" if row["lake"] else "-",
|
|
199
|
+
)
|
|
200
|
+
Console().print(table)
|
|
201
|
+
if duplicate:
|
|
202
|
+
err.print(f"[red]Duplicidade:[/red] {duplicate}", markup=True)
|
|
203
|
+
for line in errors:
|
|
204
|
+
err.print(f"[red]Falha ao carregar[/red] {line}", markup=True, highlight=False)
|
|
205
|
+
if undeclared:
|
|
206
|
+
err.print(
|
|
207
|
+
f"[yellow]{len(undeclared)} fetcher(s) sem info declarada:[/yellow] "
|
|
208
|
+
+ ", ".join(sorted(undeclared)),
|
|
209
|
+
markup=True,
|
|
210
|
+
highlight=False,
|
|
211
|
+
)
|
|
212
|
+
if strict and (duplicate or undeclared or errors):
|
|
213
|
+
raise typer.Exit(1)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def declared_source_names() -> dict[str, str]:
|
|
217
|
+
"""Mapa ``id/apelido -> distribuição`` dos fetchers instalados que declaram info."""
|
|
218
|
+
names: dict[str, str] = {}
|
|
219
|
+
for d in discover_fetchers():
|
|
220
|
+
if d.info_declared and d.distribution:
|
|
221
|
+
for name in d.info.names:
|
|
222
|
+
names[name] = d.distribution
|
|
223
|
+
return names
|
|
@@ -12,6 +12,7 @@ import email.utils
|
|
|
12
12
|
import inspect
|
|
13
13
|
import json
|
|
14
14
|
import os
|
|
15
|
+
import re
|
|
15
16
|
import threading
|
|
16
17
|
import time
|
|
17
18
|
from collections.abc import Callable, Iterable
|
|
@@ -33,6 +34,7 @@ from quantilica.core.http import (
|
|
|
33
34
|
)
|
|
34
35
|
from quantilica.core.logging import get_logger
|
|
35
36
|
from quantilica.core.manifests import manifest_sidecar_path
|
|
37
|
+
from quantilica.core.registry import FetcherInfo
|
|
36
38
|
from quantilica.core.storage import (
|
|
37
39
|
StampedDataRepository,
|
|
38
40
|
expected_format_for_path,
|
|
@@ -55,7 +57,20 @@ from quantilica.cli.ui import (
|
|
|
55
57
|
|
|
56
58
|
logger = get_logger(__name__)
|
|
57
59
|
|
|
60
|
+
|
|
61
|
+
def derive_fetcher_info(name: str) -> FetcherInfo:
|
|
62
|
+
"""Deriva um :class:`FetcherInfo` mínimo do nome do fetcher.
|
|
63
|
+
|
|
64
|
+
Usado quando o fetcher ainda não declara ``info``: o id é o ``name`` sem o
|
|
65
|
+
sufixo ``-fetcher``, normalizado (``bcb_sgs-fetcher`` -> ``bcb-sgs``).
|
|
66
|
+
"""
|
|
67
|
+
slug = re.sub(r"[^a-z0-9]+", "-", name.lower().removesuffix("-fetcher"))
|
|
68
|
+
slug = slug.strip("-") or "fetcher"
|
|
69
|
+
return FetcherInfo(id=slug, nome=slug)
|
|
70
|
+
|
|
71
|
+
|
|
58
72
|
__all__ = [
|
|
73
|
+
"derive_fetcher_info",
|
|
59
74
|
"CheckPlan",
|
|
60
75
|
"CheckPlanItem",
|
|
61
76
|
"DataRepository",
|
|
@@ -469,6 +484,9 @@ class FetcherApp:
|
|
|
469
484
|
source_id: Source ID recorded in download manifests. Defaults to
|
|
470
485
|
``name`` without the ``-fetcher`` suffix. When ``default_output``
|
|
471
486
|
is omitted it is derived as ``/data/{source_id}``.
|
|
487
|
+
info: Fatos da fonte (:class:`~quantilica.core.registry.FetcherInfo`).
|
|
488
|
+
Sem ele, um ``FetcherInfo`` mínimo é derivado de ``name`` e
|
|
489
|
+
``info_declared`` fica ``False`` (compatibilidade com fetchers antigos).
|
|
472
490
|
build_default_commands: If True (default), registers the built-in
|
|
473
491
|
``sync`` and ``list`` commands. Set to False to register only
|
|
474
492
|
custom commands via :meth:`attach_command`.
|
|
@@ -489,6 +507,7 @@ class FetcherApp:
|
|
|
489
507
|
source_id: str | None = None,
|
|
490
508
|
client: HttpClient | FtpClient | None = None,
|
|
491
509
|
freshness_policy: Any = None,
|
|
510
|
+
info: FetcherInfo | None = None,
|
|
492
511
|
build_default_commands: bool = True,
|
|
493
512
|
):
|
|
494
513
|
self.name = name
|
|
@@ -503,6 +522,8 @@ class FetcherApp:
|
|
|
503
522
|
self.default_output = default_output or Path(f"/data/{self.source_id}")
|
|
504
523
|
self.client = client or default_client()
|
|
505
524
|
self.freshness_policy = freshness_policy
|
|
525
|
+
self.info_declared = info is not None
|
|
526
|
+
self.info: FetcherInfo = info or derive_fetcher_info(name)
|
|
506
527
|
|
|
507
528
|
self.all_group_keys = list(self.groups.keys())
|
|
508
529
|
self.all_keys = self.all_group_keys + list(self.aliases.keys())
|
|
@@ -510,6 +531,9 @@ class FetcherApp:
|
|
|
510
531
|
|
|
511
532
|
# O objeto typer principal
|
|
512
533
|
self.app = typer.Typer(help=help)
|
|
534
|
+
# O entry point `quantilica.fetchers` aponta para o Typer; este atributo
|
|
535
|
+
# deixa `quantilica registry export` chegar ao FetcherApp.
|
|
536
|
+
self.app.fetcher_app = self # type: ignore[attr-defined]
|
|
513
537
|
if build_default_commands:
|
|
514
538
|
self._build_commands()
|
|
515
539
|
|
|
@@ -287,9 +287,31 @@ def reexec_cli(args: list[str]) -> None:
|
|
|
287
287
|
os.execv(sys.executable, [sys.executable, "-m", "quantilica.cli.cli", *args])
|
|
288
288
|
|
|
289
289
|
|
|
290
|
+
def installed_declared_names() -> dict[str, str]:
|
|
291
|
+
"""Ids e apelidos dos fetchers instalados que declaram ``FetcherInfo``.
|
|
292
|
+
|
|
293
|
+
Mapeia ``id/apelido -> distribuição``. Falhas na descoberta não impedem o
|
|
294
|
+
uso da tabela manual.
|
|
295
|
+
"""
|
|
296
|
+
try:
|
|
297
|
+
from quantilica.cli.registry import declared_source_names
|
|
298
|
+
|
|
299
|
+
return declared_source_names()
|
|
300
|
+
except Exception: # pragma: no cover - a reserva manual cobre
|
|
301
|
+
return {}
|
|
302
|
+
|
|
303
|
+
|
|
290
304
|
def _merged_registry() -> dict[str, str]:
|
|
291
|
-
"""Combina o registro remoto
|
|
292
|
-
|
|
305
|
+
"""Combina o registro remoto, a tabela manual e o ``FetcherInfo`` instalado.
|
|
306
|
+
|
|
307
|
+
A tabela manual (``SOURCES_REGISTRY``) é a reserva para instalar fontes
|
|
308
|
+
ausentes; para o que está instalado, id e apelidos do ``FetcherInfo`` mandam.
|
|
309
|
+
"""
|
|
310
|
+
return {
|
|
311
|
+
**fetch_remote_sources(),
|
|
312
|
+
**SOURCES_REGISTRY,
|
|
313
|
+
**installed_declared_names(),
|
|
314
|
+
}
|
|
293
315
|
|
|
294
316
|
|
|
295
317
|
_EXTRA_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Testes dos comandos `quantilica docs` (check, schema, update)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from types import SimpleNamespace
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
from typer.testing import CliRunner
|
|
11
|
+
|
|
12
|
+
pytest.importorskip("yaml")
|
|
13
|
+
pytest.importorskip("pyarrow")
|
|
14
|
+
pl = pytest.importorskip("polars")
|
|
15
|
+
pytest.importorskip("quantilica.core.docs")
|
|
16
|
+
|
|
17
|
+
from quantilica.cli.cli import app # noqa: E402
|
|
18
|
+
|
|
19
|
+
runner = CliRunner()
|
|
20
|
+
|
|
21
|
+
SECTIONS = [
|
|
22
|
+
"Resumo",
|
|
23
|
+
"Ficha técnica",
|
|
24
|
+
"Origem e coleta",
|
|
25
|
+
"Conteúdo",
|
|
26
|
+
"Variáveis",
|
|
27
|
+
"No lake",
|
|
28
|
+
"Cuidados de uso",
|
|
29
|
+
"Exemplos",
|
|
30
|
+
"Referências",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
HEADER = """\
|
|
34
|
+
---
|
|
35
|
+
dataset:
|
|
36
|
+
id: vendas
|
|
37
|
+
fonte: alfa
|
|
38
|
+
titulo: Vendas
|
|
39
|
+
resumo: Vendas mensais.
|
|
40
|
+
orgao: Órgão X
|
|
41
|
+
pagina_oficial: https://example.org
|
|
42
|
+
licenca: CC-BY 4.0
|
|
43
|
+
periodicidade: mensal
|
|
44
|
+
cobertura: {inicio: 2020-01, fim: em-curso}
|
|
45
|
+
unidade: uma venda
|
|
46
|
+
granularidade: {espacial: Brasil, temporal: mês}
|
|
47
|
+
formato_bruto: CSV
|
|
48
|
+
lake: {estado: convertido, particoes: [ano], chave: [id_venda], colunas: 2}
|
|
49
|
+
unidades_de_medida: {valor: R$}
|
|
50
|
+
dados_pessoais: nenhum
|
|
51
|
+
eras: []
|
|
52
|
+
revisado_em: 2026-11-07
|
|
53
|
+
versao_fetcher: 1.0.0
|
|
54
|
+
---
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
VARS = (
|
|
58
|
+
"| Coluna no lake | Na fonte | Tipo | Descrição | Unidade / códigos | Período |\n"
|
|
59
|
+
"|---|---|---|---|---|---|\n"
|
|
60
|
+
"| `id_venda` | ID | int | id | | |\n"
|
|
61
|
+
"| `valor` | VLR | double | valor | R$ | |\n"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def page_text(skip: str | None = None) -> str:
|
|
66
|
+
text = HEADER + "\n# Vendas\n\n"
|
|
67
|
+
for title in SECTIONS:
|
|
68
|
+
if title == skip:
|
|
69
|
+
continue
|
|
70
|
+
text += f"## {title}\n\n"
|
|
71
|
+
if title == "Ficha técnica":
|
|
72
|
+
text += "<!-- gerado:ficha -->\n\n"
|
|
73
|
+
if title == "Variáveis":
|
|
74
|
+
text += VARS + "\n"
|
|
75
|
+
return text
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def make_repo(root: Path, *, skip: str | None = None, entry_point: bool = True) -> Path:
|
|
79
|
+
datasets = root / "docs" / "datasets"
|
|
80
|
+
datasets.mkdir(parents=True)
|
|
81
|
+
(root / "docs" / "README.md").write_text("# Alfa\n", encoding="utf-8")
|
|
82
|
+
(datasets / "vendas.md").write_text(page_text(skip), encoding="utf-8")
|
|
83
|
+
schema = {
|
|
84
|
+
"fonte": "alfa",
|
|
85
|
+
"dataset": "vendas",
|
|
86
|
+
"gerado_em": "2026-11-07T00:00:00+00:00",
|
|
87
|
+
"particoes": ["ano"],
|
|
88
|
+
"colunas": [
|
|
89
|
+
{"nome": "id_venda", "tipos": ["int64"]},
|
|
90
|
+
{"nome": "valor", "tipos": ["double"]},
|
|
91
|
+
],
|
|
92
|
+
"arquivos": 1,
|
|
93
|
+
}
|
|
94
|
+
(datasets / "vendas.schema.json").write_text(json.dumps(schema), encoding="utf-8")
|
|
95
|
+
if entry_point:
|
|
96
|
+
(root / "pyproject.toml").write_text(
|
|
97
|
+
'[project]\nname = "alfa-fetcher"\n'
|
|
98
|
+
'[project.entry-points."quantilica.lake"]\nalfa = "alfa.lake:converter"\n',
|
|
99
|
+
encoding="utf-8",
|
|
100
|
+
)
|
|
101
|
+
return root
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def no_converter(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
105
|
+
monkeypatch.setattr("quantilica.core.lake.load_lake_converter", lambda _s: None)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def test_docs_help_lists_commands() -> None:
|
|
109
|
+
result = runner.invoke(app, ["docs", "--help"])
|
|
110
|
+
assert result.exit_code == 0
|
|
111
|
+
for name in ("check", "schema", "update"):
|
|
112
|
+
assert name in result.output
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_check_ok_from_files(tmp_path: Path, monkeypatch) -> None:
|
|
116
|
+
no_converter(monkeypatch)
|
|
117
|
+
repo = make_repo(tmp_path)
|
|
118
|
+
result = runner.invoke(app, ["docs", "check", str(repo)])
|
|
119
|
+
assert result.exit_code == 0, result.output
|
|
120
|
+
assert "1 dataset(s) de alfa" in result.output
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_check_reports_problems_and_exits_1(tmp_path: Path, monkeypatch) -> None:
|
|
124
|
+
no_converter(monkeypatch)
|
|
125
|
+
repo = make_repo(tmp_path, skip="Exemplos")
|
|
126
|
+
result = runner.invoke(app, ["docs", "check", str(repo)])
|
|
127
|
+
assert result.exit_code == 1
|
|
128
|
+
assert "seções ausentes: Exemplos" in result.output
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def test_check_uses_installed_converter_datasets(tmp_path: Path, monkeypatch) -> None:
|
|
132
|
+
converter = SimpleNamespace(source_id="alfa", datasets=lambda: ["vendas", "extra"])
|
|
133
|
+
monkeypatch.setattr(
|
|
134
|
+
"quantilica.core.lake.load_lake_converter", lambda _s: converter
|
|
135
|
+
)
|
|
136
|
+
repo = make_repo(tmp_path)
|
|
137
|
+
result = runner.invoke(app, ["docs", "check", str(repo)])
|
|
138
|
+
assert result.exit_code == 1
|
|
139
|
+
assert "dataset declarado sem página" in result.output
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_check_source_from_pages_and_option(tmp_path: Path, monkeypatch) -> None:
|
|
143
|
+
no_converter(monkeypatch)
|
|
144
|
+
repo = make_repo(tmp_path, entry_point=False)
|
|
145
|
+
assert runner.invoke(app, ["docs", "check", str(repo)]).exit_code == 0
|
|
146
|
+
wrong = runner.invoke(app, ["docs", "check", str(repo), "--source", "beta"])
|
|
147
|
+
assert wrong.exit_code == 1
|
|
148
|
+
assert "fonte 'alfa' diferente" in wrong.output
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def test_check_without_docs_dir(tmp_path: Path) -> None:
|
|
152
|
+
result = runner.invoke(app, ["docs", "check", str(tmp_path)])
|
|
153
|
+
assert result.exit_code == 1
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_update_rewrites_ficha_and_is_idempotent(tmp_path: Path) -> None:
|
|
157
|
+
repo = make_repo(tmp_path)
|
|
158
|
+
page = repo / "docs" / "datasets" / "vendas.md"
|
|
159
|
+
first = runner.invoke(app, ["docs", "update", str(repo)])
|
|
160
|
+
assert first.exit_code == 0, first.output
|
|
161
|
+
assert "atualizada" in first.output
|
|
162
|
+
assert "| Dataset | `alfa/vendas` |" in page.read_text(encoding="utf-8")
|
|
163
|
+
second = runner.invoke(app, ["docs", "update", str(repo)])
|
|
164
|
+
assert second.exit_code == 0
|
|
165
|
+
assert "sem mudança" in second.output
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def test_update_fails_on_invalid_page(tmp_path: Path) -> None:
|
|
169
|
+
repo = make_repo(tmp_path)
|
|
170
|
+
(repo / "docs" / "datasets" / "vendas.md").write_text("sem cabeçalho", "utf-8")
|
|
171
|
+
result = runner.invoke(app, ["docs", "update", str(repo)])
|
|
172
|
+
assert result.exit_code == 1
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def make_lake(root: Path) -> Path:
|
|
176
|
+
for mes in (1, 2):
|
|
177
|
+
folder = root / "alfa" / "vendas" / "ano=2024" / f"mes={mes}"
|
|
178
|
+
folder.mkdir(parents=True)
|
|
179
|
+
pl.DataFrame({"id_venda": [1], "valor": [1.5]}).write_parquet(
|
|
180
|
+
folder / "p.parquet"
|
|
181
|
+
)
|
|
182
|
+
(root / "alfa" / "vendas" / "_dataset.json").write_text("{}", encoding="utf-8")
|
|
183
|
+
return root
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_schema_writes_files(tmp_path: Path) -> None:
|
|
187
|
+
lake = make_lake(tmp_path / "lake")
|
|
188
|
+
out = tmp_path / "out"
|
|
189
|
+
result = runner.invoke(
|
|
190
|
+
app, ["docs", "schema", "alfa", "--lake", str(lake), "--out", str(out)]
|
|
191
|
+
)
|
|
192
|
+
assert result.exit_code == 0, result.output
|
|
193
|
+
data = json.loads((out / "vendas.schema.json").read_text(encoding="utf-8"))
|
|
194
|
+
assert data["particoes"] == ["ano", "mes"]
|
|
195
|
+
assert data["arquivos"] == 2
|
|
196
|
+
assert {c["nome"] for c in data["colunas"]} == {"id_venda", "valor"}
|
|
197
|
+
again = runner.invoke(
|
|
198
|
+
app, ["docs", "schema", "alfa", "--lake", str(lake), "--out", str(out)]
|
|
199
|
+
)
|
|
200
|
+
assert "sem mudança" in again.output
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def test_schema_dataset_filter_and_errors(tmp_path: Path) -> None:
|
|
204
|
+
lake = make_lake(tmp_path / "lake")
|
|
205
|
+
out = tmp_path / "out"
|
|
206
|
+
bad = runner.invoke(
|
|
207
|
+
app,
|
|
208
|
+
["docs", "schema", "alfa", "--lake", str(lake), "--out", str(out),
|
|
209
|
+
"--dataset", "nada"],
|
|
210
|
+
) # fmt: skip
|
|
211
|
+
assert bad.exit_code == 1
|
|
212
|
+
missing = runner.invoke(
|
|
213
|
+
app, ["docs", "schema", "zeta", "--lake", str(lake), "--out", str(out)]
|
|
214
|
+
)
|
|
215
|
+
assert missing.exit_code == 1
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Testes de `quantilica registry export` e da compatibilidade do FetcherApp."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from types import SimpleNamespace
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
from quantilica.core.registry import FetcherInfo
|
|
10
|
+
from typer.testing import CliRunner
|
|
11
|
+
|
|
12
|
+
from quantilica.cli import registry as reg
|
|
13
|
+
from quantilica.cli import sources
|
|
14
|
+
from quantilica.cli.registry import app
|
|
15
|
+
from quantilica.cli.sdk import FetcherApp, derive_fetcher_info
|
|
16
|
+
|
|
17
|
+
runner = CliRunner()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _fetcher(name: str, info: FetcherInfo | None = None, **kw) -> FetcherApp:
|
|
21
|
+
return FetcherApp(
|
|
22
|
+
name,
|
|
23
|
+
groups_dict={"g1": {"name": "g1"}, "g0": {"name": "g0"}},
|
|
24
|
+
aliases_dict={},
|
|
25
|
+
list_datasets=lambda g: [],
|
|
26
|
+
client=object(), # type: ignore[arg-type]
|
|
27
|
+
info=info,
|
|
28
|
+
**kw,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _ep(name: str, obj, dist: str | None = None, boom: bool = False):
|
|
33
|
+
def load():
|
|
34
|
+
if boom:
|
|
35
|
+
raise ImportError("quebrado")
|
|
36
|
+
return obj
|
|
37
|
+
|
|
38
|
+
d = SimpleNamespace(name=dist) if dist else None
|
|
39
|
+
return SimpleNamespace(name=name, dist=d, load=load)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@pytest.fixture
|
|
43
|
+
def fake_eps(monkeypatch: pytest.MonkeyPatch):
|
|
44
|
+
def install(fetchers=(), lake=()):
|
|
45
|
+
def fake(group: str):
|
|
46
|
+
return {
|
|
47
|
+
"quantilica.fetchers": list(fetchers),
|
|
48
|
+
"quantilica.lake": list(lake),
|
|
49
|
+
}[group]
|
|
50
|
+
|
|
51
|
+
monkeypatch.setattr(reg, "entry_points", fake)
|
|
52
|
+
|
|
53
|
+
return install
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_sem_info_deriva_do_name():
|
|
57
|
+
f = _fetcher("tesouro-direto-fetcher")
|
|
58
|
+
assert f.info_declared is False
|
|
59
|
+
assert f.info.id == "tesouro-direto" and f.info.nome == "tesouro-direto"
|
|
60
|
+
assert f.source_id == "tesouro-direto"
|
|
61
|
+
assert derive_fetcher_info("bcb_sgs-fetcher").id == "bcb-sgs"
|
|
62
|
+
assert f.app.fetcher_app is f
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_com_info_declarada_e_source_id_independente():
|
|
66
|
+
info = FetcherInfo("tesouro-direto", nome="Tesouro Direto", aliases=("td",))
|
|
67
|
+
f = _fetcher("tddata-fetcher", info=info, source_id="td")
|
|
68
|
+
assert f.info_declared is True and f.info is info
|
|
69
|
+
assert f.source_id == "td"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_export_json(fake_eps):
|
|
73
|
+
info = FetcherInfo(
|
|
74
|
+
"tesouro-direto", nome="Tesouro Direto", aliases=("td",), cadencia="daily"
|
|
75
|
+
)
|
|
76
|
+
a = _fetcher("tesouro-direto-fetcher", info=info, freshness_policy="remote_only")
|
|
77
|
+
b = _fetcher("anp-fetcher")
|
|
78
|
+
fake_eps(
|
|
79
|
+
fetchers=[
|
|
80
|
+
_ep("td", a.app, "tesouro-direto-fetcher"),
|
|
81
|
+
_ep("anp", b.app, "anp-fetcher"),
|
|
82
|
+
_ep("morto", None, "morto-fetcher", boom=True),
|
|
83
|
+
],
|
|
84
|
+
lake=[_ep("tesouro-direto", object(), "tesouro-direto-fetcher")],
|
|
85
|
+
)
|
|
86
|
+
res = runner.invoke(app, ["export", "--json"])
|
|
87
|
+
assert res.exit_code == 0
|
|
88
|
+
data = json.loads(res.stdout)
|
|
89
|
+
assert data["version"] == 1
|
|
90
|
+
assert [f["id"] for f in data["fetchers"]] == ["anp", "morto", "tesouro-direto"]
|
|
91
|
+
anp, morto, td = data["fetchers"]
|
|
92
|
+
assert td["info_declared"] is True and td["aliases"] == ["td"]
|
|
93
|
+
assert td["lake"] is True and td["lake_source_id"] == "tesouro-direto"
|
|
94
|
+
assert td["entry_point"] == "td" and td["freshness_policy"] == "remote_only"
|
|
95
|
+
assert td["groups"] == ["g0", "g1"] and td["cadencia"] == "daily"
|
|
96
|
+
assert anp["info_declared"] is False and anp["lake"] is False
|
|
97
|
+
assert "ImportError" in morto["error"]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_export_tabela(fake_eps):
|
|
101
|
+
fake_eps(fetchers=[_ep("anp", _fetcher("anp-fetcher").app, "anp-fetcher")])
|
|
102
|
+
res = runner.invoke(app, ["export"])
|
|
103
|
+
assert res.exit_code == 0
|
|
104
|
+
assert "anp" in res.stdout and "Registro de fetchers" in res.stdout
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_strict_falha_sem_info(fake_eps):
|
|
108
|
+
fake_eps(fetchers=[_ep("anp", _fetcher("anp-fetcher").app, "anp-fetcher")])
|
|
109
|
+
assert runner.invoke(app, ["export", "--strict"]).exit_code == 1
|
|
110
|
+
assert runner.invoke(app, ["export"]).exit_code == 0
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_strict_ok_com_info_e_duplicidade_falha(fake_eps):
|
|
114
|
+
def decl(i: str, aliases=()):
|
|
115
|
+
info = FetcherInfo(i, nome=i, aliases=aliases)
|
|
116
|
+
return _fetcher(f"{i}-fetcher", info=info).app
|
|
117
|
+
|
|
118
|
+
fake_eps(fetchers=[_ep("a", decl("a"), "a-fetcher")])
|
|
119
|
+
assert runner.invoke(app, ["export", "--strict", "--json"]).exit_code == 0
|
|
120
|
+
|
|
121
|
+
fake_eps(
|
|
122
|
+
fetchers=[
|
|
123
|
+
_ep("a", decl("a"), "a-fetcher"),
|
|
124
|
+
_ep("b", decl("b", aliases=("a",)), "b-fetcher"),
|
|
125
|
+
]
|
|
126
|
+
)
|
|
127
|
+
res = runner.invoke(app, ["export", "--strict", "--json"])
|
|
128
|
+
assert res.exit_code == 1
|
|
129
|
+
plain = runner.invoke(app, ["export", "--json"])
|
|
130
|
+
assert plain.exit_code == 0 and len(json.loads(plain.stdout)["fetchers"]) == 2
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def test_fonte_instalada_usa_apelidos_do_info(fake_eps, monkeypatch):
|
|
134
|
+
info = FetcherInfo("tesouro-direto", nome="TD", aliases=("td", "tddata"))
|
|
135
|
+
fake_eps(
|
|
136
|
+
fetchers=[
|
|
137
|
+
_ep(
|
|
138
|
+
"td",
|
|
139
|
+
_fetcher("tesouro-direto-fetcher", info=info).app,
|
|
140
|
+
"tesouro-direto-fetcher",
|
|
141
|
+
),
|
|
142
|
+
_ep("anp", _fetcher("anp-fetcher").app, "anp-fetcher"),
|
|
143
|
+
]
|
|
144
|
+
)
|
|
145
|
+
names = reg.declared_source_names()
|
|
146
|
+
assert names == {
|
|
147
|
+
"tesouro-direto": "tesouro-direto-fetcher",
|
|
148
|
+
"td": "tesouro-direto-fetcher",
|
|
149
|
+
"tddata": "tesouro-direto-fetcher",
|
|
150
|
+
}
|
|
151
|
+
monkeypatch.setattr(sources, "fetch_remote_sources", lambda: {})
|
|
152
|
+
merged = sources._merged_registry()
|
|
153
|
+
assert merged["tddata"] == "tesouro-direto-fetcher"
|
|
154
|
+
assert merged["anp"] == "anp-fetcher" # reserva manual
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def test_apelidos_do_info_instalado_nao_contradizem_a_tabela():
|
|
158
|
+
"""Falha se um fetcher instalado apelida algo que a tabela manual aponta para outro pacote."""
|
|
159
|
+
names = reg.declared_source_names()
|
|
160
|
+
conflicts = {
|
|
161
|
+
n: (dist, sources.SOURCES_REGISTRY[n])
|
|
162
|
+
for n, dist in names.items()
|
|
163
|
+
if n in sources.SOURCES_REGISTRY and sources.SOURCES_REGISTRY[n] != dist
|
|
164
|
+
}
|
|
165
|
+
assert not conflicts, f"FetcherInfo contradiz SOURCES_REGISTRY: {conflicts}"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|