quantilica-cli 0.21.1__tar.gz → 0.21.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/CHANGELOG.md +6 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/PKG-INFO +1 -1
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/pyproject.toml +1 -1
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/catalog.py +109 -1
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_catalog_cli.py +61 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/.githooks/pre-commit +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/.githooks/pre-push +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/.github/workflows/public-hygiene.yml +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/.github/workflows/publish.yml +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/.github/workflows/test.yml +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/.gitignore +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/LICENSE +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/README.md +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/__init__.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/cli.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/docs.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/health.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/lake.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/manifests.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/registry.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/sdk.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/sources.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/cli/ui.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/src/quantilica/py.typed +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/__init__.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/conftest.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/lake_build_fakes.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/lake_fixtures.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_catalog_registro.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_catalog_sigla.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_cli_apelidos.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_docs_cli.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_health.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_lake_cli.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_manifests.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_package_hygiene.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_registry_cli.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_sdk.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_sdk_backfill.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_sdk_force_probe.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_sdk_mes_string.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_sdk_onda_a2.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_sdk_sync_exit.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_sources.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_sources_registro.py +0 -0
- {quantilica_cli-0.21.1 → quantilica_cli-0.21.2}/tests/test_ui.py +0 -0
|
@@ -7,6 +7,12 @@ e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
|
|
|
7
7
|
|
|
8
8
|
## [Não lançado]
|
|
9
9
|
|
|
10
|
+
## [0.21.2] - 2026-11-13
|
|
11
|
+
|
|
12
|
+
### Corrigido
|
|
13
|
+
|
|
14
|
+
- `quantilica catalog sync` envia o payload em lotes por fonte (até 4 MB cada, a fonte nunca é partida): o portal recusa corpos acima de 5 MiB (413) e o lake inteiro passa de 8 MB. O sync do portal é upsert, então os lotes não apagam o que veio antes; as respostas são somadas.
|
|
15
|
+
|
|
10
16
|
## [0.21.1] - 2026-11-13
|
|
11
17
|
|
|
12
18
|
### Corrigido
|
|
@@ -493,6 +493,105 @@ def send_catalog_payload(
|
|
|
493
493
|
raise RuntimeError(f"Erro de conexão com o portal em {url}: {exc}") from exc
|
|
494
494
|
|
|
495
495
|
|
|
496
|
+
# O portal recusa corpos acima de 5 MiB em /admin/catalog/sync (413). O lake
|
|
497
|
+
# inteiro passa disso (8,5 MB em 13/11/2026), então o envio vai em lotes, cada
|
|
498
|
+
# fonte inteira num lote só. O sync do portal é upsert: lotes não apagam nada
|
|
499
|
+
# do que veio nos anteriores.
|
|
500
|
+
LOTE_MAX_BYTES = 4_000_000
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _tamanho_json(obj: Any) -> int:
|
|
504
|
+
return len(json.dumps(obj, ensure_ascii=False, default=str).encode("utf-8"))
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def split_catalog_payload(
|
|
508
|
+
payload: dict[str, Any], max_bytes: int = LOTE_MAX_BYTES
|
|
509
|
+
) -> list[dict[str, Any]]:
|
|
510
|
+
"""Divide o payload em lotes por fonte, cada um com até ``max_bytes``.
|
|
511
|
+
|
|
512
|
+
Uma fonte nunca é partida (a fonte, os datasets dela e as colunas e amostras
|
|
513
|
+
desses datasets vão juntos); uma fonte sozinha acima do limite vira um lote
|
|
514
|
+
próprio, e o portal decide. Itens sem ``dataset_id`` conhecido vão no primeiro
|
|
515
|
+
lote. Payload que já cabe volta como está, num lote só.
|
|
516
|
+
"""
|
|
517
|
+
if _tamanho_json(payload) <= max_bytes:
|
|
518
|
+
return [payload]
|
|
519
|
+
|
|
520
|
+
listas = [k for k, v in payload.items() if isinstance(v, list)]
|
|
521
|
+
filhos = [k for k in listas if k not in ("sources", "datasets")]
|
|
522
|
+
fonte_do_dataset = {
|
|
523
|
+
d.get("id"): d.get("source_id") for d in payload.get("datasets", [])
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
grupos: dict[Any, dict[str, list]] = {}
|
|
527
|
+
ordem: list[Any] = []
|
|
528
|
+
|
|
529
|
+
def grupo(fonte: Any) -> dict[str, list]:
|
|
530
|
+
if fonte not in grupos:
|
|
531
|
+
grupos[fonte] = {k: [] for k in listas}
|
|
532
|
+
ordem.append(fonte)
|
|
533
|
+
return grupos[fonte]
|
|
534
|
+
|
|
535
|
+
for src in payload.get("sources", []):
|
|
536
|
+
grupo(src.get("id"))["sources"].append(src)
|
|
537
|
+
for ds in payload.get("datasets", []):
|
|
538
|
+
grupo(ds.get("source_id"))["datasets"].append(ds)
|
|
539
|
+
soltos: dict[str, list] = {k: [] for k in listas}
|
|
540
|
+
for k in filhos:
|
|
541
|
+
for item in payload[k]:
|
|
542
|
+
fonte = fonte_do_dataset.get(item.get("dataset_id"), _SEM_FONTE)
|
|
543
|
+
if fonte is _SEM_FONTE:
|
|
544
|
+
soltos[k].append(item)
|
|
545
|
+
else:
|
|
546
|
+
grupo(fonte)[k].append(item)
|
|
547
|
+
|
|
548
|
+
extras = {k: v for k, v in payload.items() if k not in listas}
|
|
549
|
+
lotes: list[dict[str, Any]] = []
|
|
550
|
+
atual: dict[str, Any] | None = None
|
|
551
|
+
for fonte in ordem:
|
|
552
|
+
parte = grupos[fonte]
|
|
553
|
+
if atual is not None:
|
|
554
|
+
candidato = {k: atual[k] + parte[k] for k in listas}
|
|
555
|
+
if _tamanho_json({**extras, **candidato}) <= max_bytes:
|
|
556
|
+
atual.update(candidato)
|
|
557
|
+
continue
|
|
558
|
+
lotes.append(atual)
|
|
559
|
+
atual = {**extras, **{k: list(parte[k]) for k in listas}}
|
|
560
|
+
if atual is not None:
|
|
561
|
+
lotes.append(atual)
|
|
562
|
+
if not lotes:
|
|
563
|
+
lotes.append({**extras, **{k: [] for k in listas}})
|
|
564
|
+
for k in listas:
|
|
565
|
+
lotes[0][k] = lotes[0][k] + soltos[k]
|
|
566
|
+
return lotes
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
_SEM_FONTE = object()
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def _somar_respostas(respostas: list[dict[str, Any]]) -> dict[str, Any]:
|
|
573
|
+
"""Soma os contadores numéricos das respostas dos lotes."""
|
|
574
|
+
if len(respostas) == 1:
|
|
575
|
+
return respostas[0]
|
|
576
|
+
total: dict[str, Any] = {"lotes": len(respostas)}
|
|
577
|
+
for res in respostas:
|
|
578
|
+
for chave, valor in res.items():
|
|
579
|
+
if isinstance(valor, bool):
|
|
580
|
+
total[chave] = valor
|
|
581
|
+
elif isinstance(valor, (int, float)):
|
|
582
|
+
total[chave] = total.get(chave, 0) + valor
|
|
583
|
+
elif isinstance(valor, dict):
|
|
584
|
+
sub = total.setdefault(chave, {})
|
|
585
|
+
for k, v in valor.items():
|
|
586
|
+
if isinstance(v, (int, float)) and not isinstance(v, bool):
|
|
587
|
+
sub[k] = sub.get(k, 0) + v
|
|
588
|
+
else:
|
|
589
|
+
sub[k] = v
|
|
590
|
+
else:
|
|
591
|
+
total[chave] = valor
|
|
592
|
+
return total
|
|
593
|
+
|
|
594
|
+
|
|
496
595
|
@app.command("sync")
|
|
497
596
|
def cmd_catalog_sync(
|
|
498
597
|
lake_dir: Annotated[
|
|
@@ -635,8 +734,17 @@ def cmd_catalog_sync(
|
|
|
635
734
|
f"[cyan]Enviando payload ({sources_count} fontes, "
|
|
636
735
|
f"{datasets_count} datasets)...[/cyan]"
|
|
637
736
|
)
|
|
737
|
+
lotes = split_catalog_payload(payload)
|
|
738
|
+
respostas: list[dict[str, Any]] = []
|
|
638
739
|
try:
|
|
639
|
-
|
|
740
|
+
for i, lote in enumerate(lotes, 1):
|
|
741
|
+
if len(lotes) > 1:
|
|
742
|
+
console.print(
|
|
743
|
+
f"[cyan]Lote {i}/{len(lotes)}:[/cyan] "
|
|
744
|
+
+ ", ".join(sorted(str(x.get("id")) for x in lote["sources"]))
|
|
745
|
+
)
|
|
746
|
+
respostas.append(send_catalog_payload(portal_url, lote, api_key=api_key))
|
|
747
|
+
res = _somar_respostas(respostas)
|
|
640
748
|
console.print(
|
|
641
749
|
f"[green]✓ Catálogo sincronizado com sucesso![/green] Resposta: {res}"
|
|
642
750
|
)
|
|
@@ -370,3 +370,64 @@ def test_envio_do_catalogo_usa_user_agent_proprio(monkeypatch):
|
|
|
370
370
|
catalog_mod.send_catalog_payload("https://portal.exemplo", {"sources": []}, "k")
|
|
371
371
|
assert capturado["ua"].startswith("quantilica-cli/")
|
|
372
372
|
assert "urllib" not in capturado["ua"].lower()
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _payload_grande():
|
|
376
|
+
fontes = ["a", "b", "c"]
|
|
377
|
+
payload = {"sources": [], "datasets": [], "columns": [], "samples": []}
|
|
378
|
+
for f in fontes:
|
|
379
|
+
payload["sources"].append({"id": f})
|
|
380
|
+
for n in range(3):
|
|
381
|
+
did = f"{f}/d{n}"
|
|
382
|
+
payload["datasets"].append({"id": did, "source_id": f})
|
|
383
|
+
payload["columns"].append({"dataset_id": did, "nome": "x" * 1000})
|
|
384
|
+
payload["samples"].append(
|
|
385
|
+
{"dataset_id": did, "linhas_amostra": ["y" * 1000]}
|
|
386
|
+
)
|
|
387
|
+
return payload
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def test_split_payload_pequeno_volta_inteiro():
|
|
391
|
+
from quantilica.cli.catalog import split_catalog_payload
|
|
392
|
+
|
|
393
|
+
payload = {"sources": [{"id": "a"}], "datasets": [], "columns": [], "samples": []}
|
|
394
|
+
assert split_catalog_payload(payload) == [payload]
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def test_split_payload_por_fonte_sem_perder_nem_partir_fonte():
|
|
398
|
+
import json as _json
|
|
399
|
+
|
|
400
|
+
from quantilica.cli.catalog import split_catalog_payload
|
|
401
|
+
|
|
402
|
+
payload = _payload_grande()
|
|
403
|
+
lotes = split_catalog_payload(payload, max_bytes=8000)
|
|
404
|
+
assert len(lotes) > 1
|
|
405
|
+
for chave in payload:
|
|
406
|
+
assert sorted(map(_json.dumps, payload[chave])) == sorted(
|
|
407
|
+
_json.dumps(x) for lote in lotes for x in lote[chave]
|
|
408
|
+
)
|
|
409
|
+
for lote in lotes:
|
|
410
|
+
fontes = {s["id"] for s in lote["sources"]}
|
|
411
|
+
assert {d["source_id"] for d in lote["datasets"]} <= fontes
|
|
412
|
+
ids = {d["id"] for d in lote["datasets"]}
|
|
413
|
+
assert {c["dataset_id"] for c in lote["columns"]} <= ids
|
|
414
|
+
assert {c["dataset_id"] for c in lote["samples"]} <= ids
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def test_sync_envia_em_lotes_e_soma_as_respostas(monkeypatch):
|
|
418
|
+
from quantilica.cli import catalog as catalog_mod
|
|
419
|
+
|
|
420
|
+
payload = _payload_grande()
|
|
421
|
+
lotes = catalog_mod.split_catalog_payload(payload, max_bytes=8000)
|
|
422
|
+
enviados = []
|
|
423
|
+
|
|
424
|
+
def _send(url, lote, api_key=None):
|
|
425
|
+
enviados.append(lote)
|
|
426
|
+
return {"status": "success", "counts": {"datasets": len(lote["datasets"])}}
|
|
427
|
+
|
|
428
|
+
monkeypatch.setattr(catalog_mod, "send_catalog_payload", _send)
|
|
429
|
+
monkeypatch.setattr(catalog_mod, "LOTE_MAX_BYTES", 8000)
|
|
430
|
+
respostas = [_send("u", lote) for lote in lotes]
|
|
431
|
+
total = catalog_mod._somar_respostas(respostas)
|
|
432
|
+
assert total["counts"]["datasets"] == 9
|
|
433
|
+
assert total["lotes"] == len(lotes)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|