quantilica-cli 0.24.0__tar.gz → 0.25.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/CHANGELOG.md +11 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/PKG-INFO +3 -3
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/pyproject.toml +3 -3
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/sdk.py +41 -13
- quantilica_cli-0.25.0/tests/fake_http_server.py +80 -0
- quantilica_cli-0.25.0/tests/test_sdk_politica_por_host.py +158 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/.githooks/pre-commit +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/.githooks/pre-push +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/.github/dependabot.yml +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/.github/workflows/public-hygiene.yml +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/.github/workflows/publish.yml +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/.github/workflows/test.yml +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/.gitignore +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/LICENSE +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/README.md +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/__init__.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/catalog.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/cli.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/docs.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/health.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/lake.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/manifests.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/registry.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/sources.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/cli/ui.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/src/quantilica/py.typed +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/__init__.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/conftest.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/lake_build_fakes.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/lake_fixtures.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/renomes.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_catalog_cli.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_catalog_perfil.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_catalog_registro.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_catalog_sigla.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_cli_apelidos.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_docs_cli.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_health.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_lake_cli.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_manifests.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_package_hygiene.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_registry_cli.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_registry_v2.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_sdk.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_sdk_backfill.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_sdk_force_probe.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_sdk_mes_string.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_sdk_onda_a2.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_sdk_sync_exit.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_sources.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_sources_registro.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_sources_registro_publico.py +0 -0
- {quantilica_cli-0.24.0 → quantilica_cli-0.25.0}/tests/test_ui.py +0 -0
|
@@ -8,6 +8,17 @@ e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
|
|
|
8
8
|
## [Não lançado]
|
|
9
9
|
|
|
10
10
|
|
|
11
|
+
## [0.25.0] - 2026-11-16
|
|
12
|
+
### Alterado
|
|
13
|
+
|
|
14
|
+
- Piso de `quantilica-core` sobe para `>=0.25.0` (`HttpClient.share_host_state` e `max_concurrency` em `stream()`, de que o `download_datasets` depende para compartilhar a política por host entre as threads).
|
|
15
|
+
|
|
16
|
+
### Corrigido
|
|
17
|
+
|
|
18
|
+
- `FetcherApp.download_datasets` respeita a política por host do client do app (`HttpClient.host_policies`): os clients das threads passam a compartilhar as políticas e o estado do limitador (`min_interval`, pausa de `429` com `Retry-After`, `max_concurrency`) com o client do app, com `HttpClient.share_host_state`. Antes cada thread criava um client sem política e com limitador próprio, e o host recebia N vezes o ritmo configurado com N `--workers`; o fetcher precisava sobrescrever `download_datasets` para contornar. Sem política por host nada muda (clients independentes, mesmo paralelismo). Com a política, o teto de simultâneas vale também no `GET` do arquivo (`stream` do core), então quem declara só `min_interval` fica, pelo padrão `max_concurrency=4`, com no máximo 4 downloads simultâneos por host.
|
|
19
|
+
- Ao fim de `download_datasets` o `self.client` volta sempre ao client do app. Antes, com threads concorrentes, podia ficar um client de thread (já fechado) no lugar dele.
|
|
20
|
+
|
|
21
|
+
|
|
11
22
|
## [0.24.0] - 2026-11-15
|
|
12
23
|
### Alterado
|
|
13
24
|
- Piso de `quantilica-core` sobe para `>=0.24.0` (granularidades semestral, trimestral e diária do perfil de colunas).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: quantilica-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.25.0
|
|
4
4
|
Summary: Unified CLI for Quantilica open data fetchers
|
|
5
5
|
Author-email: "Komesu, D.K." <daniel@dkko.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -16,11 +16,11 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.13
|
|
17
17
|
Classifier: Typing :: Typed
|
|
18
18
|
Requires-Python: >=3.12
|
|
19
|
-
Requires-Dist: quantilica-core>=0.
|
|
19
|
+
Requires-Dist: quantilica-core>=0.25.0
|
|
20
20
|
Requires-Dist: rich>=13.0.0
|
|
21
21
|
Requires-Dist: typer>=0.15.0
|
|
22
22
|
Provides-Extra: docs
|
|
23
|
-
Requires-Dist: quantilica-core[docs]>=0.
|
|
23
|
+
Requires-Dist: quantilica-core[docs]>=0.25.0; extra == 'docs'
|
|
24
24
|
Provides-Extra: lake
|
|
25
25
|
Requires-Dist: pyarrow>=15.0.0; extra == 'lake'
|
|
26
26
|
Requires-Dist: quantilica-analytics>=0.11.0; extra == 'lake'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "quantilica-cli"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.25.0"
|
|
4
4
|
description = "Unified CLI for Quantilica open data fetchers"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [{ name = "Komesu, D.K.", email = "daniel@dkko.me" }]
|
|
@@ -20,7 +20,7 @@ classifiers = [
|
|
|
20
20
|
"Programming Language :: Python :: 3.13",
|
|
21
21
|
]
|
|
22
22
|
dependencies = [
|
|
23
|
-
"quantilica-core>=0.
|
|
23
|
+
"quantilica-core>=0.25.0",
|
|
24
24
|
"rich>=13.0.0",
|
|
25
25
|
"typer>=0.15.0",
|
|
26
26
|
]
|
|
@@ -29,7 +29,7 @@ dependencies = [
|
|
|
29
29
|
# Motor de `quantilica lake build` e `quantilica lake audit`.
|
|
30
30
|
lake = ["quantilica-analytics>=0.11.0", "pyarrow>=15.0.0"]
|
|
31
31
|
# `quantilica docs`: verificação da documentação dos dados dos fetchers.
|
|
32
|
-
docs = ["quantilica-core[docs]>=0.
|
|
32
|
+
docs = ["quantilica-core[docs]>=0.25.0"]
|
|
33
33
|
|
|
34
34
|
[project.scripts]
|
|
35
35
|
quantilica = "quantilica.cli.cli:main"
|
|
@@ -417,6 +417,23 @@ def default_client() -> HttpClient:
|
|
|
417
417
|
)
|
|
418
418
|
|
|
419
419
|
|
|
420
|
+
_aviso_core_antigo = False
|
|
421
|
+
_aviso_core_lock = threading.Lock()
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _warn_core_sem_estado_compartilhado() -> None:
|
|
425
|
+
"""Avisa uma só vez por processo (não por thread) que o core é antigo."""
|
|
426
|
+
global _aviso_core_antigo
|
|
427
|
+
with _aviso_core_lock:
|
|
428
|
+
if _aviso_core_antigo:
|
|
429
|
+
return
|
|
430
|
+
_aviso_core_antigo = True
|
|
431
|
+
logger.warning(
|
|
432
|
+
"quantilica-core sem share_host_state: os downloads paralelos não "
|
|
433
|
+
"compartilham a política por host"
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
|
|
420
437
|
class DataRepository(StampedDataRepository):
|
|
421
438
|
"""Canonical data repository layout shared by Quantilica fetchers.
|
|
422
439
|
|
|
@@ -1320,23 +1337,34 @@ class FetcherApp:
|
|
|
1320
1337
|
# Pooling por worker: cada thread mantém seu HttpClient com keep-alive.
|
|
1321
1338
|
thread_local = threading.local()
|
|
1322
1339
|
_worker_clients: list[HttpClient] = []
|
|
1340
|
+
# O client do app, antes de qualquer troca por um client de thread.
|
|
1341
|
+
base_client = self.client
|
|
1323
1342
|
|
|
1324
1343
|
def _get_worker_client() -> HttpClient | FtpClient:
|
|
1325
|
-
if isinstance(
|
|
1326
|
-
return
|
|
1344
|
+
if isinstance(base_client, FtpClient):
|
|
1345
|
+
return base_client
|
|
1327
1346
|
if not hasattr(thread_local, "client"):
|
|
1328
1347
|
# Reusa config do client canônico, mas com sessão persistente.
|
|
1329
1348
|
c = HttpClient(
|
|
1330
|
-
timeout=
|
|
1331
|
-
headers=dict(
|
|
1332
|
-
follow_redirects=
|
|
1333
|
-
attempts=
|
|
1334
|
-
retry_base_delay=
|
|
1335
|
-
verify=
|
|
1336
|
-
transport=
|
|
1337
|
-
limits=
|
|
1349
|
+
timeout=base_client.timeout,
|
|
1350
|
+
headers=dict(base_client.headers),
|
|
1351
|
+
follow_redirects=base_client.follow_redirects,
|
|
1352
|
+
attempts=base_client.attempts,
|
|
1353
|
+
retry_base_delay=base_client.retry_base_delay,
|
|
1354
|
+
verify=base_client.verify,
|
|
1355
|
+
transport=base_client.transport,
|
|
1356
|
+
limits=base_client.limits,
|
|
1338
1357
|
emulate_browser=True,
|
|
1339
1358
|
)
|
|
1359
|
+
if base_client.host_policies:
|
|
1360
|
+
# Política por host: o ritmo, a pausa do 429 e o teto de
|
|
1361
|
+
# simultâneas são do host, não da thread. Sem política o
|
|
1362
|
+
# worker segue independente, como sempre foi.
|
|
1363
|
+
share = getattr(c, "share_host_state", None)
|
|
1364
|
+
if share is not None:
|
|
1365
|
+
share(base_client)
|
|
1366
|
+
else: # quantilica-core anterior à política compartilhada
|
|
1367
|
+
_warn_core_sem_estado_compartilhado()
|
|
1340
1368
|
c.__enter__()
|
|
1341
1369
|
thread_local.client = c
|
|
1342
1370
|
_worker_clients.append(c)
|
|
@@ -1345,7 +1373,6 @@ class FetcherApp:
|
|
|
1345
1373
|
def _worker(entry: dict[str, Any]) -> bool:
|
|
1346
1374
|
# Troca temporária do client para o da thread (com pooling).
|
|
1347
1375
|
worker_client = _get_worker_client()
|
|
1348
|
-
prev = self.client
|
|
1349
1376
|
self.client = worker_client # type: ignore[assignment]
|
|
1350
1377
|
try:
|
|
1351
1378
|
eid = entry.get("id", "unknown")
|
|
@@ -1358,8 +1385,6 @@ class FetcherApp:
|
|
|
1358
1385
|
except Exception as exc:
|
|
1359
1386
|
errors.append((entry.get("id", "unknown"), str(exc)))
|
|
1360
1387
|
return False
|
|
1361
|
-
finally:
|
|
1362
|
-
self.client = prev
|
|
1363
1388
|
|
|
1364
1389
|
with graceful_executor(max_workers=workers) as executor:
|
|
1365
1390
|
try:
|
|
@@ -1379,6 +1404,9 @@ class FetcherApp:
|
|
|
1379
1404
|
console.print("\n[yellow]Interrompido.[/yellow]")
|
|
1380
1405
|
raise typer.Exit(130) from None
|
|
1381
1406
|
finally:
|
|
1407
|
+
# Devolve o client do app: as threads o trocam pelo seu e, com
|
|
1408
|
+
# a troca por thread, o "anterior" de uma pode ser o de outra.
|
|
1409
|
+
self.client = base_client
|
|
1382
1410
|
for c in _worker_clients:
|
|
1383
1411
|
with contextlib.suppress(Exception):
|
|
1384
1412
|
c.close()
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Servidor HTTP local para testes de tráfego: mede simultaneidade e chegadas."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import Callable
|
|
8
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
9
|
+
|
|
10
|
+
HOST = "127.0.0.1"
|
|
11
|
+
BODY = b"a;b\n" + b"1;2\n" * 1024
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class FakeServer:
|
|
15
|
+
"""Registra o instante de cada requisição e o máximo de simultâneas."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, *, body_delay: float = 0.15) -> None:
|
|
18
|
+
self.body_delay = body_delay
|
|
19
|
+
self.arrivals: list[tuple[float, str, str]] = []
|
|
20
|
+
self.active = 0
|
|
21
|
+
self.max_active = 0
|
|
22
|
+
self.lock = threading.Lock()
|
|
23
|
+
self.responder: Callable[[BaseHTTPRequestHandler, str, str], bool] | None = None
|
|
24
|
+
outer = self
|
|
25
|
+
|
|
26
|
+
class Handler(BaseHTTPRequestHandler):
|
|
27
|
+
protocol_version = "HTTP/1.1"
|
|
28
|
+
|
|
29
|
+
def log_message(self, *args: object) -> None:
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
def _handle(self, method: str) -> None:
|
|
33
|
+
with outer.lock:
|
|
34
|
+
outer.arrivals.append((time.monotonic(), method, self.path))
|
|
35
|
+
outer.active += 1
|
|
36
|
+
outer.max_active = max(outer.max_active, outer.active)
|
|
37
|
+
try:
|
|
38
|
+
if outer.responder is not None and outer.responder(
|
|
39
|
+
self, method, self.path
|
|
40
|
+
):
|
|
41
|
+
return
|
|
42
|
+
self.send_response(200)
|
|
43
|
+
self.send_header("Content-Length", str(len(BODY)))
|
|
44
|
+
self.end_headers()
|
|
45
|
+
if method == "HEAD":
|
|
46
|
+
return
|
|
47
|
+
half = len(BODY) // 2
|
|
48
|
+
self.wfile.write(BODY[:half])
|
|
49
|
+
self.wfile.flush()
|
|
50
|
+
time.sleep(outer.body_delay)
|
|
51
|
+
self.wfile.write(BODY[half:])
|
|
52
|
+
finally:
|
|
53
|
+
with outer.lock:
|
|
54
|
+
outer.active -= 1
|
|
55
|
+
|
|
56
|
+
def do_GET(self) -> None: # noqa: N802
|
|
57
|
+
self._handle("GET")
|
|
58
|
+
|
|
59
|
+
def do_HEAD(self) -> None: # noqa: N802
|
|
60
|
+
self._handle("HEAD")
|
|
61
|
+
|
|
62
|
+
self.httpd = ThreadingHTTPServer((HOST, 0), Handler)
|
|
63
|
+
self.httpd.daemon_threads = True
|
|
64
|
+
self.port = self.httpd.server_address[1]
|
|
65
|
+
self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
|
|
66
|
+
|
|
67
|
+
def url(self, path: str = "/f") -> str:
|
|
68
|
+
return f"http://{HOST}:{self.port}{path}"
|
|
69
|
+
|
|
70
|
+
def times(self) -> list[float]:
|
|
71
|
+
return sorted(t for t, _, _ in self.arrivals)
|
|
72
|
+
|
|
73
|
+
def __enter__(self) -> FakeServer:
|
|
74
|
+
self.thread.start()
|
|
75
|
+
return self
|
|
76
|
+
|
|
77
|
+
def __exit__(self, *exc: object) -> None:
|
|
78
|
+
self.httpd.shutdown()
|
|
79
|
+
self.httpd.server_close()
|
|
80
|
+
self.thread.join(timeout=5)
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""``download_datasets`` com política por host: o ritmo é do host, não da thread."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from http.server import BaseHTTPRequestHandler
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
from quantilica.core.http import HostPolicy, HttpClient
|
|
12
|
+
|
|
13
|
+
from quantilica.cli.sdk import FetcherApp
|
|
14
|
+
from tests.fake_http_server import BODY, HOST, FakeServer
|
|
15
|
+
from tests.test_sdk import ALIASES, GROUPS, sample_list_datasets
|
|
16
|
+
|
|
17
|
+
WORKERS = 8
|
|
18
|
+
ENTRIES = 12
|
|
19
|
+
# Intervalo longo e tolerância relativa: o relógio do Windows tem 15,6 ms de
|
|
20
|
+
# resolução e a máquina sob carga atrasa as threads.
|
|
21
|
+
INTERVALO = 0.2
|
|
22
|
+
TOLERANCIA = 0.5
|
|
23
|
+
|
|
24
|
+
requer_core_novo = pytest.mark.skipif(
|
|
25
|
+
not hasattr(HttpClient, "share_host_state"),
|
|
26
|
+
reason="exige o quantilica-core com share_host_state",
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def make_entries(server: FakeServer, n: int = ENTRIES) -> list[dict[str, Any]]:
|
|
31
|
+
return [
|
|
32
|
+
{
|
|
33
|
+
"group": "exp",
|
|
34
|
+
"id": f"exp-{i}",
|
|
35
|
+
"ext": "csv",
|
|
36
|
+
"year": 2020 + i,
|
|
37
|
+
"url": server.url(f"/dados{i}.csv"),
|
|
38
|
+
}
|
|
39
|
+
for i in range(n)
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def path_builder(output_dir: Path, entry: dict[str, Any], last_modified: Any) -> Path:
|
|
44
|
+
return output_dir / "exp" / f"{entry['id']}.csv"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def make_app(policy: HostPolicy | None, **client_kwargs: Any) -> FetcherApp:
|
|
48
|
+
client = HttpClient(
|
|
49
|
+
attempts=3,
|
|
50
|
+
retry_base_delay=0.01,
|
|
51
|
+
jitter=0.0,
|
|
52
|
+
host_policies={HOST: policy} if policy is not None else None,
|
|
53
|
+
**client_kwargs,
|
|
54
|
+
)
|
|
55
|
+
return FetcherApp(
|
|
56
|
+
name="politica-fetcher",
|
|
57
|
+
groups_dict=GROUPS,
|
|
58
|
+
aliases_dict=ALIASES,
|
|
59
|
+
list_datasets=sample_list_datasets,
|
|
60
|
+
path_builder=path_builder,
|
|
61
|
+
client=client,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def run(app: FetcherApp, server: FakeServer, out: Path, n: int = ENTRIES):
|
|
66
|
+
return app.download_datasets(make_entries(server, n), out, workers=WORKERS)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@requer_core_novo
|
|
70
|
+
def test_max_concurrency_e_min_interval_valem_para_todas_as_threads(tmp_path: Path):
|
|
71
|
+
n = 6
|
|
72
|
+
with FakeServer(body_delay=0.5) as server:
|
|
73
|
+
app = make_app(HostPolicy(min_interval=INTERVALO, max_concurrency=2))
|
|
74
|
+
done, total, errors = run(app, server, tmp_path, n)
|
|
75
|
+
assert (done, total, errors) == (n, n, [])
|
|
76
|
+
assert len(list((tmp_path / "exp").glob("*.csv"))) == n
|
|
77
|
+
assert server.max_active <= 2
|
|
78
|
+
times = server.times()
|
|
79
|
+
assert len(times) >= n * 2 # HEAD + GET de cada entrada
|
|
80
|
+
gaps = [b - a for a, b in zip(times, times[1:], strict=False)]
|
|
81
|
+
assert min(gaps) >= TOLERANCIA * INTERVALO
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@requer_core_novo
|
|
85
|
+
def test_429_pausa_todas_as_threads(tmp_path: Path):
|
|
86
|
+
n = 24 # três levas de 8 threads: sobra trabalho depois do 429
|
|
87
|
+
with FakeServer(body_delay=0.4) as server:
|
|
88
|
+
t429: list[float] = []
|
|
89
|
+
|
|
90
|
+
def responder(handler: BaseHTTPRequestHandler, method: str, path: str) -> bool:
|
|
91
|
+
# O primeiro GET do servidor leva 429; os demais seguem normais.
|
|
92
|
+
if method == "GET" and not t429:
|
|
93
|
+
t429.append(time.monotonic())
|
|
94
|
+
handler.send_response(429)
|
|
95
|
+
handler.send_header("Retry-After", "1")
|
|
96
|
+
handler.send_header("Content-Length", "0")
|
|
97
|
+
handler.end_headers()
|
|
98
|
+
return True
|
|
99
|
+
return False
|
|
100
|
+
|
|
101
|
+
server.responder = responder
|
|
102
|
+
app = make_app(
|
|
103
|
+
HostPolicy(min_interval=0.02, max_concurrency=8, backoff_base=0.5)
|
|
104
|
+
)
|
|
105
|
+
done, total, errors = run(app, server, tmp_path, n)
|
|
106
|
+
assert (done, total, errors) == (n, n, [])
|
|
107
|
+
assert t429
|
|
108
|
+
# Nenhuma thread chega ao servidor durante a pausa (a folga inicial deixa
|
|
109
|
+
# passar só o que já estava a caminho quando o 429 foi registrado).
|
|
110
|
+
na_pausa = [t for t in server.times() if t429[0] + 0.3 < t < t429[0] + 0.9]
|
|
111
|
+
assert na_pausa == []
|
|
112
|
+
assert max(server.times()) > t429[0] + 0.9 # e o trabalho retoma depois
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_sem_politica_o_paralelismo_segue_o_mesmo(tmp_path: Path):
|
|
116
|
+
with FakeServer(body_delay=0.3) as server:
|
|
117
|
+
app = make_app(None)
|
|
118
|
+
done, total, errors = run(app, server, tmp_path)
|
|
119
|
+
assert (done, total, errors) == (ENTRIES, ENTRIES, [])
|
|
120
|
+
assert server.max_active > 2 # sem teto: as threads baixam juntas
|
|
121
|
+
assert (tmp_path / "exp" / "exp-0.csv").read_bytes() == BODY
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_core_sem_share_host_state_cai_no_comportamento_antigo(
|
|
125
|
+
tmp_path: Path, monkeypatch
|
|
126
|
+
):
|
|
127
|
+
monkeypatch.delattr(HttpClient, "share_host_state", raising=False)
|
|
128
|
+
with FakeServer(body_delay=0.05) as server:
|
|
129
|
+
app = make_app(HostPolicy(min_interval=0.0, max_concurrency=2))
|
|
130
|
+
done, total, errors = run(app, server, tmp_path)
|
|
131
|
+
assert (done, total, errors) == (ENTRIES, ENTRIES, [])
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@requer_core_novo
|
|
135
|
+
def test_o_client_do_app_volta_ao_original(tmp_path: Path):
|
|
136
|
+
with FakeServer(body_delay=0.0) as server:
|
|
137
|
+
app = make_app(HostPolicy(min_interval=0.0, max_concurrency=2))
|
|
138
|
+
original = app.client
|
|
139
|
+
run(app, server, tmp_path)
|
|
140
|
+
assert app.client is original
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def test_aviso_de_core_antigo_sai_uma_vez_por_processo(
|
|
144
|
+
tmp_path: Path, monkeypatch, caplog
|
|
145
|
+
):
|
|
146
|
+
import logging
|
|
147
|
+
|
|
148
|
+
from quantilica.cli import sdk
|
|
149
|
+
|
|
150
|
+
monkeypatch.delattr(HttpClient, "share_host_state", raising=False)
|
|
151
|
+
monkeypatch.setattr(sdk, "_aviso_core_antigo", False)
|
|
152
|
+
with FakeServer(body_delay=0.05) as server:
|
|
153
|
+
app = make_app(HostPolicy(min_interval=0.0, max_concurrency=2))
|
|
154
|
+
with caplog.at_level(logging.WARNING):
|
|
155
|
+
run(app, server, tmp_path)
|
|
156
|
+
run(app, server, tmp_path) # segundo download, mesmas threads novas
|
|
157
|
+
avisos = [r for r in caplog.records if "share_host_state" in r.getMessage()]
|
|
158
|
+
assert len(avisos) == 1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|