quantilica-cli 0.9.1__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/CHANGELOG.md +20 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/PKG-INFO +2 -2
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/pyproject.toml +2 -2
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/src/quantilica/cli/sdk.py +429 -12
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/src/quantilica/cli/sources.py +1 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/tests/test_sdk.py +160 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/.githooks/pre-commit +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/.githooks/pre-push +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/.github/workflows/publish.yml +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/.github/workflows/test.yml +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/.gitignore +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/LICENSE +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/README.md +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/src/quantilica/cli/__init__.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/src/quantilica/cli/cli.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/src/quantilica/cli/health.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/src/quantilica/cli/manifests.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/src/quantilica/cli/progress.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/src/quantilica/cli/ui.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/src/quantilica/py.typed +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/tests/__init__.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/tests/test_health.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/tests/test_manifests.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/tests/test_package_hygiene.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/tests/test_sdk_onda_a2.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/tests/test_sources.py +0 -0
- {quantilica_cli-0.9.1 → quantilica_cli-0.10.0}/tests/test_ui.py +0 -0
|
@@ -5,6 +5,26 @@ Todas as mudanças notáveis deste projeto serão documentadas neste arquivo.
|
|
|
5
5
|
O formato segue [Keep a Changelog](https://keepachangelog.com/pt-BR/1.1.0/),
|
|
6
6
|
e este projeto adere ao [Semantic Versioning](https://semver.org/lang/pt-BR/).
|
|
7
7
|
|
|
8
|
+
## [0.10.0] - 2026-10-07
|
|
9
|
+
|
|
10
|
+
### Adicionado
|
|
11
|
+
|
|
12
|
+
- `FetcherApp`: comando `check` — verificação remoto × local (HEAD por
|
|
13
|
+
entrada) com plano de freshness, sem baixar; `--json` emite plano
|
|
14
|
+
consumível por `sync --from-plan`.
|
|
15
|
+
- `FetcherApp.sync`: opção `--from-plan <arquivo.json>` — baixa somente as
|
|
16
|
+
entradas com ação `download` do plano.
|
|
17
|
+
- SDK: `CheckPlan`/`CheckPlanItem` (tabela Rich + round-trip JSON) e
|
|
18
|
+
`default_client()` com overrides via ambiente (`QUANTILICA_CA_BUNDLE`,
|
|
19
|
+
`QUANTILICA_SSL_VERIFY`, `QUANTILICA_HTTP_ATTEMPTS/TIMEOUT/RETRY_DELAY`).
|
|
20
|
+
|
|
21
|
+
### Alterado
|
|
22
|
+
|
|
23
|
+
- Piso de `quantilica-core` elevado para `>=0.10.0` (símbolos usados:
|
|
24
|
+
`is_remote_more_recent`, `resolve_verify_from_env`).
|
|
25
|
+
- Workers internos do `download_datasets` propagam o `transport` customizado
|
|
26
|
+
do client canônico.
|
|
27
|
+
|
|
8
28
|
## [0.9.1] - 2026-10-04
|
|
9
29
|
|
|
10
30
|
Correção de escape inválido introduzido na v0.8.0 (Onda A.2 do SDK).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: quantilica-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.10.0
|
|
4
4
|
Summary: Unified CLI for Quantilica open data fetchers
|
|
5
5
|
Author-email: "Komesu, D.K." <daniel@dkko.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.13
|
|
17
17
|
Classifier: Typing :: Typed
|
|
18
18
|
Requires-Python: >=3.12
|
|
19
|
-
Requires-Dist: quantilica-core>=0.
|
|
19
|
+
Requires-Dist: quantilica-core>=0.10.0
|
|
20
20
|
Requires-Dist: rich>=13.0.0
|
|
21
21
|
Requires-Dist: typer>=0.15.0
|
|
22
22
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "quantilica-cli"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.10.0"
|
|
4
4
|
description = "Unified CLI for Quantilica open data fetchers"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [{ name = "Komesu, D.K.", email = "daniel@dkko.me" }]
|
|
@@ -20,7 +20,7 @@ classifiers = [
|
|
|
20
20
|
"Programming Language :: Python :: 3.13",
|
|
21
21
|
]
|
|
22
22
|
dependencies = [
|
|
23
|
-
"quantilica-core>=0.
|
|
23
|
+
"quantilica-core>=0.10.0",
|
|
24
24
|
"rich>=13.0.0",
|
|
25
25
|
"typer>=0.15.0",
|
|
26
26
|
]
|
|
@@ -8,6 +8,9 @@ from __future__ import annotations
|
|
|
8
8
|
import concurrent.futures
|
|
9
9
|
import contextlib
|
|
10
10
|
import datetime as dt
|
|
11
|
+
import email.utils
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
11
14
|
import threading
|
|
12
15
|
from collections.abc import Callable, Iterable
|
|
13
16
|
from dataclasses import dataclass, field
|
|
@@ -17,7 +20,13 @@ from typing import Annotated, Any
|
|
|
17
20
|
import typer
|
|
18
21
|
from quantilica.core.exceptions import FetchError
|
|
19
22
|
from quantilica.core.ftp import FtpClient
|
|
20
|
-
from quantilica.core.http import
|
|
23
|
+
from quantilica.core.http import (
|
|
24
|
+
HttpClient,
|
|
25
|
+
HttpStatusError,
|
|
26
|
+
ProgressCallback,
|
|
27
|
+
is_remote_more_recent,
|
|
28
|
+
resolve_verify_from_env,
|
|
29
|
+
)
|
|
21
30
|
from quantilica.core.logging import get_logger
|
|
22
31
|
from quantilica.core.storage import (
|
|
23
32
|
StampedDataRepository,
|
|
@@ -41,6 +50,8 @@ from quantilica.cli.ui import (
|
|
|
41
50
|
logger = get_logger(__name__)
|
|
42
51
|
|
|
43
52
|
__all__ = [
|
|
53
|
+
"CheckPlan",
|
|
54
|
+
"CheckPlanItem",
|
|
44
55
|
"DataRepository",
|
|
45
56
|
"FetcherApp",
|
|
46
57
|
"SyncPlan",
|
|
@@ -113,17 +124,229 @@ class SyncPlan:
|
|
|
113
124
|
)
|
|
114
125
|
|
|
115
126
|
|
|
127
|
+
def _partition_label(entry: dict[str, Any]) -> str | None:
|
|
128
|
+
"""Human-readable partition label for a dataset entry."""
|
|
129
|
+
if entry.get("semester") is not None:
|
|
130
|
+
return f"{entry['year']}-S{entry['semester']}"
|
|
131
|
+
if entry.get("month") is not None:
|
|
132
|
+
return f"{entry['year']}-{entry['month']:02d}"
|
|
133
|
+
if entry.get("year") is not None:
|
|
134
|
+
return str(entry["year"])
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _parse_head_date(value: str | None) -> dt.date | None:
|
|
139
|
+
"""Parse an HTTP Last-Modified header value into a date, or None."""
|
|
140
|
+
if not value:
|
|
141
|
+
return None
|
|
142
|
+
with contextlib.suppress(Exception):
|
|
143
|
+
return email.utils.parsedate_to_datetime(value).date()
|
|
144
|
+
return None
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _short_remote(item: CheckPlanItem) -> str:
|
|
148
|
+
"""Compact remote description for table display (narrow terminals)."""
|
|
149
|
+
date = _parse_head_date(item.remote_last_modified)
|
|
150
|
+
remote = date.isoformat() if date else (item.remote_last_modified or "—")
|
|
151
|
+
if item.remote_size:
|
|
152
|
+
remote += f" ({item.remote_size} B)"
|
|
153
|
+
return remote
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@dataclass(frozen=True)
|
|
157
|
+
class CheckPlanItem:
|
|
158
|
+
"""Freshness verdict for a single dataset entry (no download performed).
|
|
159
|
+
|
|
160
|
+
Args:
|
|
161
|
+
dataset: Canonical dataset (group) identifier.
|
|
162
|
+
id: Dataset entry identifier.
|
|
163
|
+
url: Remote URL that was checked.
|
|
164
|
+
partition: Human-readable partition label, or None.
|
|
165
|
+
local_path: Local destination path the entry maps to.
|
|
166
|
+
remote_etag: Remote ETag header, or None.
|
|
167
|
+
remote_last_modified: Remote Last-Modified header, or None.
|
|
168
|
+
remote_size: Remote Content-Length, or None.
|
|
169
|
+
local_exists: Whether the local file already exists.
|
|
170
|
+
action: ``"download"`` or ``"skip-up-to-date"``.
|
|
171
|
+
reason: Machine-readable reason for the action.
|
|
172
|
+
entry: Original dataset entry (allows ``sync --from-plan`` round-trip).
|
|
173
|
+
"""
|
|
174
|
+
|
|
175
|
+
dataset: str
|
|
176
|
+
id: str
|
|
177
|
+
url: str
|
|
178
|
+
partition: str | None
|
|
179
|
+
local_path: str
|
|
180
|
+
remote_etag: str | None
|
|
181
|
+
remote_last_modified: str | None
|
|
182
|
+
remote_size: int | None
|
|
183
|
+
local_exists: bool
|
|
184
|
+
action: str
|
|
185
|
+
reason: str
|
|
186
|
+
entry: dict[str, Any]
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@dataclass(frozen=True)
|
|
190
|
+
class CheckPlan:
|
|
191
|
+
"""Freshness plan for a verification run (``check`` command).
|
|
192
|
+
|
|
193
|
+
Args:
|
|
194
|
+
fetcher: Name of the fetcher that produced the plan.
|
|
195
|
+
output_dir: Output directory the plan was computed against.
|
|
196
|
+
generated_at: ISO-8601 UTC timestamp of plan generation.
|
|
197
|
+
items: Per-entry verdicts.
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
fetcher: str
|
|
201
|
+
output_dir: str
|
|
202
|
+
generated_at: str
|
|
203
|
+
items: list[CheckPlanItem] = field(default_factory=list)
|
|
204
|
+
|
|
205
|
+
def render_table(self, console: Console | None = None) -> None:
|
|
206
|
+
"""Render the plan as a Rich table with a summary footer."""
|
|
207
|
+
con = console or get_console()
|
|
208
|
+
table = Table(
|
|
209
|
+
"Dataset",
|
|
210
|
+
"ID",
|
|
211
|
+
"Ação",
|
|
212
|
+
"Motivo",
|
|
213
|
+
"Remoto",
|
|
214
|
+
title="Verificação remoto × local (check)",
|
|
215
|
+
)
|
|
216
|
+
for item in self.items:
|
|
217
|
+
table.add_row(
|
|
218
|
+
item.dataset,
|
|
219
|
+
item.id,
|
|
220
|
+
item.action,
|
|
221
|
+
item.reason,
|
|
222
|
+
_short_remote(item),
|
|
223
|
+
)
|
|
224
|
+
con.print(table)
|
|
225
|
+
counts: dict[str, int] = {}
|
|
226
|
+
for item in self.items:
|
|
227
|
+
counts[item.action] = counts.get(item.action, 0) + 1
|
|
228
|
+
summary = ", ".join(f"{v} {k}" for k, v in sorted(counts.items()))
|
|
229
|
+
con.print(f"[bold]{len(self.items)}[/bold] verificado(s): {summary}.")
|
|
230
|
+
|
|
231
|
+
def to_dict(self) -> dict[str, Any]:
|
|
232
|
+
"""Return the JSON-serializable representation of the plan."""
|
|
233
|
+
return {
|
|
234
|
+
"fetcher": self.fetcher,
|
|
235
|
+
"output_dir": self.output_dir,
|
|
236
|
+
"generated_at": self.generated_at,
|
|
237
|
+
"items": [
|
|
238
|
+
{
|
|
239
|
+
"dataset": it.dataset,
|
|
240
|
+
"id": it.id,
|
|
241
|
+
"url": it.url,
|
|
242
|
+
"partition": it.partition,
|
|
243
|
+
"local_path": it.local_path,
|
|
244
|
+
"remote_etag": it.remote_etag,
|
|
245
|
+
"remote_last_modified": it.remote_last_modified,
|
|
246
|
+
"remote_size": it.remote_size,
|
|
247
|
+
"local_exists": it.local_exists,
|
|
248
|
+
"action": it.action,
|
|
249
|
+
"reason": it.reason,
|
|
250
|
+
"entry": it.entry,
|
|
251
|
+
}
|
|
252
|
+
for it in self.items
|
|
253
|
+
],
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
def to_json(self) -> str:
|
|
257
|
+
"""Serialize the plan to a JSON string (for ``sync --from-plan``)."""
|
|
258
|
+
return json.dumps(self.to_dict(), ensure_ascii=False, indent=2)
|
|
259
|
+
|
|
260
|
+
@classmethod
|
|
261
|
+
def from_dict(cls, data: dict[str, Any]) -> CheckPlan:
|
|
262
|
+
"""Rebuild a plan from :meth:`to_dict` output.
|
|
263
|
+
|
|
264
|
+
Raises:
|
|
265
|
+
ValueError: If the payload is not a valid check plan.
|
|
266
|
+
"""
|
|
267
|
+
if not isinstance(data, dict) or not isinstance(data.get("items"), list):
|
|
268
|
+
raise ValueError("not a check plan: missing 'items' list")
|
|
269
|
+
items = []
|
|
270
|
+
for raw in data["items"]:
|
|
271
|
+
if not isinstance(raw, dict):
|
|
272
|
+
raise ValueError("not a check plan: invalid item")
|
|
273
|
+
try:
|
|
274
|
+
entry = raw["entry"]
|
|
275
|
+
action = raw["action"]
|
|
276
|
+
eid = raw["id"]
|
|
277
|
+
except KeyError as exc:
|
|
278
|
+
raise ValueError(f"not a check plan: item missing {exc}") from exc
|
|
279
|
+
if not isinstance(entry, dict) or action not in (
|
|
280
|
+
"download",
|
|
281
|
+
"skip-up-to-date",
|
|
282
|
+
):
|
|
283
|
+
raise ValueError("not a check plan: invalid item")
|
|
284
|
+
items.append(
|
|
285
|
+
CheckPlanItem(
|
|
286
|
+
dataset=str(raw.get("dataset", "")),
|
|
287
|
+
id=str(eid),
|
|
288
|
+
url=str(raw.get("url", "")),
|
|
289
|
+
partition=raw.get("partition"),
|
|
290
|
+
local_path=str(raw.get("local_path", "")),
|
|
291
|
+
remote_etag=raw.get("remote_etag"),
|
|
292
|
+
remote_last_modified=raw.get("remote_last_modified"),
|
|
293
|
+
remote_size=raw.get("remote_size"),
|
|
294
|
+
local_exists=bool(raw.get("local_exists", False)),
|
|
295
|
+
action=action,
|
|
296
|
+
reason=str(raw.get("reason", "")),
|
|
297
|
+
entry=entry,
|
|
298
|
+
)
|
|
299
|
+
)
|
|
300
|
+
return cls(
|
|
301
|
+
fetcher=str(data.get("fetcher", "")),
|
|
302
|
+
output_dir=str(data.get("output_dir", "")),
|
|
303
|
+
generated_at=str(data.get("generated_at", "")),
|
|
304
|
+
items=items,
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
@classmethod
|
|
308
|
+
def from_json(cls, payload: str) -> CheckPlan:
|
|
309
|
+
"""Parse a plan serialized with :meth:`to_json`.
|
|
310
|
+
|
|
311
|
+
Raises:
|
|
312
|
+
ValueError: If the payload is not valid JSON or not a plan.
|
|
313
|
+
"""
|
|
314
|
+
try:
|
|
315
|
+
data = json.loads(payload)
|
|
316
|
+
except json.JSONDecodeError as exc:
|
|
317
|
+
raise ValueError(f"not a check plan: invalid JSON ({exc})") from exc
|
|
318
|
+
return cls.from_dict(data)
|
|
319
|
+
|
|
320
|
+
|
|
116
321
|
def default_client() -> HttpClient:
|
|
117
322
|
"""Create a default HttpClient with standard configuration.
|
|
118
323
|
|
|
324
|
+
TLS verification and retry budget can be overridden via environment:
|
|
325
|
+
|
|
326
|
+
- ``QUANTILICA_CA_BUNDLE``: path to a custom CA bundle.
|
|
327
|
+
- ``QUANTILICA_SSL_VERIFY``: ``0``/``false`` disables verification
|
|
328
|
+
(ops escape hatch for hosts with broken chains — never default);
|
|
329
|
+
a path uses it as CA bundle.
|
|
330
|
+
- ``QUANTILICA_HTTP_ATTEMPTS`` / ``QUANTILICA_HTTP_TIMEOUT`` /
|
|
331
|
+
``QUANTILICA_HTTP_RETRY_DELAY``: retry budget overrides.
|
|
332
|
+
|
|
119
333
|
Returns:
|
|
120
334
|
A pre-configured HttpClient instance (browser-like WAF headers + pooling).
|
|
121
335
|
"""
|
|
336
|
+
verify = resolve_verify_from_env()
|
|
337
|
+
if verify is False:
|
|
338
|
+
logger.warning(
|
|
339
|
+
"TLS verification disabled via QUANTILICA_SSL_VERIFY — "
|
|
340
|
+
"use only for hosts with broken chains"
|
|
341
|
+
)
|
|
342
|
+
attempts = int(os.environ.get("QUANTILICA_HTTP_ATTEMPTS", "5"))
|
|
343
|
+
timeout = float(os.environ.get("QUANTILICA_HTTP_TIMEOUT", "180.0"))
|
|
344
|
+
retry_base_delay = float(os.environ.get("QUANTILICA_HTTP_RETRY_DELAY", "2.0"))
|
|
122
345
|
return HttpClient(
|
|
123
|
-
timeout=
|
|
124
|
-
verify=
|
|
125
|
-
attempts=
|
|
126
|
-
retry_base_delay=
|
|
346
|
+
timeout=timeout,
|
|
347
|
+
verify=verify,
|
|
348
|
+
attempts=attempts,
|
|
349
|
+
retry_base_delay=retry_base_delay,
|
|
127
350
|
emulate_browser=True,
|
|
128
351
|
)
|
|
129
352
|
|
|
@@ -648,6 +871,7 @@ class FetcherApp:
|
|
|
648
871
|
attempts=self.client.attempts,
|
|
649
872
|
retry_base_delay=self.client.retry_base_delay,
|
|
650
873
|
verify=self.client.verify,
|
|
874
|
+
transport=self.client.transport,
|
|
651
875
|
limits=self.client.limits,
|
|
652
876
|
emulate_browser=True,
|
|
653
877
|
)
|
|
@@ -696,6 +920,133 @@ class FetcherApp:
|
|
|
696
920
|
|
|
697
921
|
return downloaded, total, errors
|
|
698
922
|
|
|
923
|
+
def check_entry(self, entry: dict[str, Any], output_dir: Path) -> CheckPlanItem:
|
|
924
|
+
"""Check one dataset entry against the remote without downloading.
|
|
925
|
+
|
|
926
|
+
Performs HEAD requests (same URLs ``download_entry`` would try,
|
|
927
|
+
including ``fallback_urls``) and compares against the local stamped
|
|
928
|
+
file using the same freshness predicate as the download path.
|
|
929
|
+
|
|
930
|
+
Args:
|
|
931
|
+
entry: Dictionary containing dataset metadata (url, id, etc).
|
|
932
|
+
output_dir: Destination directory used for path computation.
|
|
933
|
+
|
|
934
|
+
Returns:
|
|
935
|
+
The freshness verdict for the entry.
|
|
936
|
+
"""
|
|
937
|
+
dataset = str(entry.get("group") or entry.get("id") or "datasets")
|
|
938
|
+
eid = str(entry.get("id", "unknown"))
|
|
939
|
+
partition = _partition_label(entry)
|
|
940
|
+
urls_to_try = [entry["url"]]
|
|
941
|
+
if "fallback_urls" in entry and entry["fallback_urls"]:
|
|
942
|
+
urls_to_try.extend(entry["fallback_urls"])
|
|
943
|
+
|
|
944
|
+
head = None
|
|
945
|
+
head_error: Exception | None = None
|
|
946
|
+
for url in urls_to_try:
|
|
947
|
+
try:
|
|
948
|
+
candidate = self.client.head(url)
|
|
949
|
+
except HttpStatusError as exc:
|
|
950
|
+
if exc.status_code == 404:
|
|
951
|
+
head_error = exc
|
|
952
|
+
continue
|
|
953
|
+
head_error = exc
|
|
954
|
+
break
|
|
955
|
+
except Exception as exc: # noqa: BLE001 - metadata best effort
|
|
956
|
+
head_error = exc
|
|
957
|
+
break
|
|
958
|
+
head = candidate
|
|
959
|
+
break
|
|
960
|
+
|
|
961
|
+
if head is None:
|
|
962
|
+
local = self.path_builder(output_dir, entry, None)
|
|
963
|
+
return CheckPlanItem(
|
|
964
|
+
dataset=dataset,
|
|
965
|
+
id=eid,
|
|
966
|
+
url=str(entry["url"]),
|
|
967
|
+
partition=partition,
|
|
968
|
+
local_path=str(local),
|
|
969
|
+
remote_etag=None,
|
|
970
|
+
remote_last_modified=None,
|
|
971
|
+
remote_size=None,
|
|
972
|
+
local_exists=local.exists(),
|
|
973
|
+
action="download",
|
|
974
|
+
reason=f"metadata-unavailable: {head_error}",
|
|
975
|
+
entry=dict(entry),
|
|
976
|
+
)
|
|
977
|
+
|
|
978
|
+
etag = head.headers.get("ETag")
|
|
979
|
+
size_raw = head.headers.get("Content-Length")
|
|
980
|
+
try:
|
|
981
|
+
size = int(size_raw) if size_raw else None
|
|
982
|
+
except (TypeError, ValueError):
|
|
983
|
+
size = None
|
|
984
|
+
last_modified = _parse_head_date(head.headers.get("Last-Modified"))
|
|
985
|
+
local = self.path_builder(output_dir, entry, last_modified)
|
|
986
|
+
exists = local.exists()
|
|
987
|
+
if exists and not is_remote_more_recent(head, local):
|
|
988
|
+
action, reason = "skip-up-to-date", "local-fresh"
|
|
989
|
+
else:
|
|
990
|
+
action = "download"
|
|
991
|
+
reason = "not-present" if not exists else "remote-newer"
|
|
992
|
+
return CheckPlanItem(
|
|
993
|
+
dataset=dataset,
|
|
994
|
+
id=eid,
|
|
995
|
+
url=str(head.url),
|
|
996
|
+
partition=partition,
|
|
997
|
+
local_path=str(local),
|
|
998
|
+
remote_etag=etag,
|
|
999
|
+
remote_last_modified=head.headers.get("Last-Modified"),
|
|
1000
|
+
remote_size=size,
|
|
1001
|
+
local_exists=exists,
|
|
1002
|
+
action=action,
|
|
1003
|
+
reason=reason,
|
|
1004
|
+
entry=dict(entry),
|
|
1005
|
+
)
|
|
1006
|
+
|
|
1007
|
+
def check_datasets(
|
|
1008
|
+
self,
|
|
1009
|
+
entries: list[dict[str, Any]],
|
|
1010
|
+
output_dir: Path,
|
|
1011
|
+
workers: int = 4,
|
|
1012
|
+
) -> CheckPlan:
|
|
1013
|
+
"""Checks freshness for a list of dataset entries (no downloads).
|
|
1014
|
+
|
|
1015
|
+
Args:
|
|
1016
|
+
entries: List of dataset entries to check.
|
|
1017
|
+
output_dir: The base directory used for path computation.
|
|
1018
|
+
workers: Maximum number of parallel checks.
|
|
1019
|
+
|
|
1020
|
+
Returns:
|
|
1021
|
+
A :class:`CheckPlan` with one verdict per entry.
|
|
1022
|
+
"""
|
|
1023
|
+
results: list[CheckPlanItem | None] = [None] * len(entries)
|
|
1024
|
+
|
|
1025
|
+
def _worker(index: int, entry: dict[str, Any]) -> None:
|
|
1026
|
+
results[index] = self.check_entry(entry, output_dir)
|
|
1027
|
+
if (index + 1) % 100 == 0 or index + 1 == len(entries):
|
|
1028
|
+
logger.info("[check] %d/%d verificados", index + 1, len(entries))
|
|
1029
|
+
|
|
1030
|
+
with graceful_executor(max_workers=workers) as executor:
|
|
1031
|
+
try:
|
|
1032
|
+
futures = {
|
|
1033
|
+
executor.submit(_worker, index, entry): index
|
|
1034
|
+
for index, entry in enumerate(entries)
|
|
1035
|
+
}
|
|
1036
|
+
for future in concurrent.futures.as_completed(futures):
|
|
1037
|
+
future.result()
|
|
1038
|
+
except KeyboardInterrupt:
|
|
1039
|
+
get_console().print("\n[yellow]Interrompido.[/yellow]")
|
|
1040
|
+
raise typer.Exit(130) from None
|
|
1041
|
+
|
|
1042
|
+
generated = dt.datetime.now(dt.UTC).isoformat(timespec="seconds")
|
|
1043
|
+
return CheckPlan(
|
|
1044
|
+
fetcher=self.name,
|
|
1045
|
+
output_dir=str(output_dir),
|
|
1046
|
+
generated_at=generated,
|
|
1047
|
+
items=[item for item in results if item is not None],
|
|
1048
|
+
)
|
|
1049
|
+
|
|
699
1050
|
def _build_commands(self) -> None:
|
|
700
1051
|
console = get_console()
|
|
701
1052
|
|
|
@@ -716,6 +1067,15 @@ class FetcherApp:
|
|
|
716
1067
|
workers: Annotated[
|
|
717
1068
|
int, typer.Option("--workers", help="Downloads paralelos")
|
|
718
1069
|
] = 4,
|
|
1070
|
+
from_plan: Annotated[
|
|
1071
|
+
Path | None,
|
|
1072
|
+
typer.Option(
|
|
1073
|
+
"--from-plan",
|
|
1074
|
+
help="Baixar somente as entradas com ação 'download' "
|
|
1075
|
+
"de um plano gerado por 'check' (ignora a seleção "
|
|
1076
|
+
"de grupos).",
|
|
1077
|
+
),
|
|
1078
|
+
] = None,
|
|
719
1079
|
verbose: Annotated[
|
|
720
1080
|
bool, typer.Option("--verbose", help="Logs detalhados")
|
|
721
1081
|
] = False,
|
|
@@ -723,14 +1083,28 @@ class FetcherApp:
|
|
|
723
1083
|
setup_rich_logging(verbose, console=console)
|
|
724
1084
|
actual_output = output or self.default_output
|
|
725
1085
|
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
1086
|
+
if from_plan is not None:
|
|
1087
|
+
try:
|
|
1088
|
+
plan = CheckPlan.from_json(from_plan.read_text(encoding="utf-8"))
|
|
1089
|
+
except (OSError, ValueError) as exc:
|
|
1090
|
+
console.print(f"[red]Plano inválido: {exc}[/red]")
|
|
1091
|
+
raise typer.Exit(1) from None
|
|
1092
|
+
entries = [
|
|
1093
|
+
dict(item.entry) for item in plan.items if item.action == "download"
|
|
1094
|
+
]
|
|
1095
|
+
console.print(
|
|
1096
|
+
f"[dim]Plano {from_plan}: {len(entries)} entrada(s) "
|
|
1097
|
+
f"com ação 'download' de {len(plan.items)} verificada(s).[/dim]"
|
|
1098
|
+
)
|
|
1099
|
+
else:
|
|
1100
|
+
try:
|
|
1101
|
+
target_groups = self.resolve_groups(groups)
|
|
1102
|
+
except ValueError as exc:
|
|
1103
|
+
console.print(f"[red]{exc}[/red]")
|
|
1104
|
+
console.print(f"Grupos válidos: {', '.join(self.all_keys)}")
|
|
1105
|
+
raise typer.Exit(1) from None
|
|
732
1106
|
|
|
733
|
-
|
|
1107
|
+
entries = [e for g in target_groups for e in self.list_datasets(g)]
|
|
734
1108
|
|
|
735
1109
|
if dry_run:
|
|
736
1110
|
table = Table("Grupo", "ID", "URL", title="Arquivos a baixar (dry-run)")
|
|
@@ -752,6 +1126,49 @@ class FetcherApp:
|
|
|
752
1126
|
for eid, emsg in errors:
|
|
753
1127
|
console.print(f" {eid}: {emsg}")
|
|
754
1128
|
|
|
1129
|
+
@self.app.command("check")
|
|
1130
|
+
def check(
|
|
1131
|
+
groups: Annotated[
|
|
1132
|
+
list[str] | None,
|
|
1133
|
+
typer.Argument(
|
|
1134
|
+
help="Grupos a verificar. Use 'list' para ver grupos disponíveis. Padrão: todos."
|
|
1135
|
+
),
|
|
1136
|
+
] = None,
|
|
1137
|
+
output: Annotated[
|
|
1138
|
+
Path | None, typer.Option("-o", "--output", help="Diretório de saída")
|
|
1139
|
+
] = None,
|
|
1140
|
+
workers: Annotated[
|
|
1141
|
+
int, typer.Option("--workers", help="Verificações paralelas")
|
|
1142
|
+
] = 4,
|
|
1143
|
+
as_json: Annotated[
|
|
1144
|
+
bool,
|
|
1145
|
+
typer.Option(
|
|
1146
|
+
"--json",
|
|
1147
|
+
help="Imprime o plano em JSON (para 'sync --from-plan')",
|
|
1148
|
+
),
|
|
1149
|
+
] = False,
|
|
1150
|
+
verbose: Annotated[
|
|
1151
|
+
bool, typer.Option("--verbose", help="Logs detalhados")
|
|
1152
|
+
] = False,
|
|
1153
|
+
) -> None:
|
|
1154
|
+
setup_rich_logging(verbose, console=console)
|
|
1155
|
+
actual_output = output or self.default_output
|
|
1156
|
+
|
|
1157
|
+
try:
|
|
1158
|
+
target_groups = self.resolve_groups(groups)
|
|
1159
|
+
except ValueError as exc:
|
|
1160
|
+
console.print(f"[red]{exc}[/red]")
|
|
1161
|
+
console.print(f"Grupos válidos: {', '.join(self.all_keys)}")
|
|
1162
|
+
raise typer.Exit(1) from None
|
|
1163
|
+
|
|
1164
|
+
entries = [e for g in target_groups for e in self.list_datasets(g)]
|
|
1165
|
+
plan = self.check_datasets(entries, actual_output, workers=workers)
|
|
1166
|
+
|
|
1167
|
+
if as_json:
|
|
1168
|
+
print(plan.to_json())
|
|
1169
|
+
else:
|
|
1170
|
+
plan.render_table()
|
|
1171
|
+
|
|
755
1172
|
@self.app.command("list")
|
|
756
1173
|
def cmd_list(
|
|
757
1174
|
verbose: Annotated[
|
|
@@ -3,16 +3,21 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import datetime as dt
|
|
6
|
+
import json
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
from typing import Annotated
|
|
8
9
|
|
|
10
|
+
import httpx2
|
|
9
11
|
import pytest
|
|
10
12
|
import typer
|
|
13
|
+
from quantilica.core.http import HttpClient
|
|
11
14
|
from typer.testing import CliRunner
|
|
12
15
|
|
|
13
16
|
from quantilica.cli.sdk import (
|
|
17
|
+
CheckPlan,
|
|
14
18
|
DataRepository,
|
|
15
19
|
FetcherApp,
|
|
20
|
+
default_client,
|
|
16
21
|
default_path_builder,
|
|
17
22
|
make_resolve_groups,
|
|
18
23
|
)
|
|
@@ -275,3 +280,158 @@ def test_builtin_list():
|
|
|
275
280
|
result = runner.invoke(app.app, ["list"])
|
|
276
281
|
assert result.exit_code == 0, result.output
|
|
277
282
|
assert "2 dataset(s) no catálogo." in result.output
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
# ---------------------------------------------------------------
|
|
286
|
+
# check (verificação remoto × local, sem download) + sync --from-plan
|
|
287
|
+
# ---------------------------------------------------------------
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _head_handler(payload: bytes = b"0123456789"):
|
|
291
|
+
def handler(request):
|
|
292
|
+
if request.method == "HEAD":
|
|
293
|
+
return httpx2.Response(
|
|
294
|
+
200,
|
|
295
|
+
headers={
|
|
296
|
+
"Content-Length": str(len(payload)),
|
|
297
|
+
"Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT",
|
|
298
|
+
"ETag": '"abc123"',
|
|
299
|
+
},
|
|
300
|
+
)
|
|
301
|
+
return httpx2.Response(200, content=payload)
|
|
302
|
+
|
|
303
|
+
return handler
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _mock_client(payload: bytes = b"0123456789") -> HttpClient:
|
|
307
|
+
return HttpClient(
|
|
308
|
+
attempts=1, transport=httpx2.MockTransport(_head_handler(payload))
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def test_check_entry_missing_downloads(tmp_path: Path):
|
|
313
|
+
app = make_app(client=_mock_client())
|
|
314
|
+
entry = sample_list_datasets("exp")[0]
|
|
315
|
+
item = app.check_entry(entry, tmp_path)
|
|
316
|
+
assert item.action == "download"
|
|
317
|
+
assert item.reason == "not-present"
|
|
318
|
+
assert item.remote_etag == '"abc123"'
|
|
319
|
+
assert item.remote_size == 10
|
|
320
|
+
assert item.remote_last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
|
|
321
|
+
assert item.local_exists is False
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def test_check_entry_fresh_skips(tmp_path: Path):
|
|
325
|
+
from quantilica.cli.sdk import default_path_builder
|
|
326
|
+
|
|
327
|
+
app = make_app(client=_mock_client())
|
|
328
|
+
entry = sample_list_datasets("exp")[0]
|
|
329
|
+
local = default_path_builder(tmp_path, entry, dt.date(2024, 1, 1))
|
|
330
|
+
local.parent.mkdir(parents=True, exist_ok=True)
|
|
331
|
+
local.write_bytes(b"0123456789")
|
|
332
|
+
item = app.check_entry(entry, tmp_path)
|
|
333
|
+
assert item.action == "skip-up-to-date"
|
|
334
|
+
assert item.reason == "local-fresh"
|
|
335
|
+
assert item.local_exists is True
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def test_check_entry_head_error_downloads(tmp_path: Path):
|
|
339
|
+
def handler(request):
|
|
340
|
+
raise httpx2.ConnectError("down")
|
|
341
|
+
|
|
342
|
+
app = make_app(
|
|
343
|
+
client=HttpClient(attempts=1, transport=httpx2.MockTransport(handler))
|
|
344
|
+
)
|
|
345
|
+
entry = sample_list_datasets("exp")[0]
|
|
346
|
+
item = app.check_entry(entry, tmp_path)
|
|
347
|
+
assert item.action == "download"
|
|
348
|
+
assert item.reason.startswith("metadata-unavailable")
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def test_check_plan_json_roundtrip(tmp_path: Path):
|
|
352
|
+
app = make_app(client=_mock_client())
|
|
353
|
+
entries = sample_list_datasets("exp")
|
|
354
|
+
plan = app.check_datasets(entries, tmp_path, workers=1)
|
|
355
|
+
restored = CheckPlan.from_json(plan.to_json())
|
|
356
|
+
assert restored.fetcher == plan.fetcher
|
|
357
|
+
assert [it.id for it in restored.items] == [it.id for it in plan.items]
|
|
358
|
+
assert restored.items[0].entry["url"] == entries[0]["url"]
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def test_check_plan_from_json_rejects_garbage():
|
|
362
|
+
with pytest.raises(ValueError, match="not a check plan"):
|
|
363
|
+
CheckPlan.from_json('{"foo": 1}')
|
|
364
|
+
with pytest.raises(ValueError, match="not a check plan"):
|
|
365
|
+
CheckPlan.from_json("not json")
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def test_builtin_check_command(tmp_path: Path):
|
|
369
|
+
app = make_app(client=_mock_client())
|
|
370
|
+
result = runner.invoke(app.app, ["check", "exp", "-o", str(tmp_path)])
|
|
371
|
+
assert result.exit_code == 0, result.output
|
|
372
|
+
assert "download" in result.output
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def test_builtin_check_json_output(tmp_path: Path):
|
|
376
|
+
app = make_app(client=_mock_client())
|
|
377
|
+
result = runner.invoke(app.app, ["check", "exp", "-o", str(tmp_path), "--json"])
|
|
378
|
+
assert result.exit_code == 0, result.output
|
|
379
|
+
data = json.loads(result.output)
|
|
380
|
+
assert data["items"][0]["action"] == "download"
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def test_builtin_sync_from_plan_downloads_only_planned(tmp_path: Path):
|
|
384
|
+
from quantilica.cli.sdk import default_path_builder
|
|
385
|
+
|
|
386
|
+
payload = b"0123456789"
|
|
387
|
+
app = make_app(client=_mock_client(payload))
|
|
388
|
+
fresh_entry = sample_list_datasets("exp")[0]
|
|
389
|
+
stale_entry = dict(sample_list_datasets("imp")[0])
|
|
390
|
+
local = default_path_builder(tmp_path, fresh_entry, dt.date(2024, 1, 1))
|
|
391
|
+
local.parent.mkdir(parents=True, exist_ok=True)
|
|
392
|
+
local.write_bytes(payload)
|
|
393
|
+
|
|
394
|
+
plan = app.check_datasets([fresh_entry, stale_entry], tmp_path, workers=1)
|
|
395
|
+
actions = {it.id: it.action for it in plan.items}
|
|
396
|
+
assert actions == {"exp-monthly": "skip-up-to-date", "imp-monthly": "download"}
|
|
397
|
+
|
|
398
|
+
plan_file = tmp_path / "plan.json"
|
|
399
|
+
plan_file.write_text(plan.to_json(), encoding="utf-8")
|
|
400
|
+
result = runner.invoke(
|
|
401
|
+
app.app, ["sync", "--from-plan", str(plan_file), "-o", str(tmp_path)]
|
|
402
|
+
)
|
|
403
|
+
assert result.exit_code == 0, result.output
|
|
404
|
+
assert "1/1 arquivo(s) baixado(s)" in result.output
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def test_builtin_sync_from_plan_invalid_file(tmp_path: Path):
|
|
408
|
+
app = make_app(client=_mock_client())
|
|
409
|
+
bad = tmp_path / "bad.json"
|
|
410
|
+
bad.write_text('{"nope": true}', encoding="utf-8")
|
|
411
|
+
result = runner.invoke(app.app, ["sync", "--from-plan", str(bad)])
|
|
412
|
+
assert result.exit_code == 1
|
|
413
|
+
assert "Plano inválido" in result.output
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
# ---------------------------------------------------------------
|
|
417
|
+
# default_client: overrides via ambiente
|
|
418
|
+
# ---------------------------------------------------------------
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def test_default_client_verify_defaults_true(monkeypatch):
|
|
422
|
+
monkeypatch.delenv("QUANTILICA_SSL_VERIFY", raising=False)
|
|
423
|
+
monkeypatch.delenv("QUANTILICA_CA_BUNDLE", raising=False)
|
|
424
|
+
assert default_client().verify is True
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def test_default_client_verify_env_disable(monkeypatch):
|
|
428
|
+
monkeypatch.delenv("QUANTILICA_CA_BUNDLE", raising=False)
|
|
429
|
+
monkeypatch.setenv("QUANTILICA_SSL_VERIFY", "0")
|
|
430
|
+
assert default_client().verify is False
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def test_default_client_attempts_env_override(monkeypatch):
|
|
434
|
+
monkeypatch.delenv("QUANTILICA_SSL_VERIFY", raising=False)
|
|
435
|
+
monkeypatch.delenv("QUANTILICA_CA_BUNDLE", raising=False)
|
|
436
|
+
monkeypatch.setenv("QUANTILICA_HTTP_ATTEMPTS", "9")
|
|
437
|
+
assert default_client().attempts == 9
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|