dexverse 0.1.0__py3-none-any.whl

dexverse/__init__.py

@@ -0,0 +1,35 @@
+"""DexVerse Downloader — S3-compatible dataset downloader.
+
+Public API:
+
+    from dexverse import DexVerseClient, download_manifest
+    from dexverse import Manifest, FileSpec
+"""
+from __future__ import annotations
+
+from .client import DexVerseClient, DexVerseError, download_manifest
+from .manifest import (
+    FileSpec,
+    Manifest,
+    Storage,
+    StorageCredentials,
+    DownloadOptions,
+    resolve_dest,
+    resolve_relpath,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "DexVerseClient",
+    "DexVerseError",
+    "download_manifest",
+    "Manifest",
+    "FileSpec",
+    "Storage",
+    "StorageCredentials",
+    "DownloadOptions",
+    "resolve_dest",
+    "resolve_relpath",
+    "__version__",
+]

dexverse/cli.py

@@ -0,0 +1,226 @@
+"""Command-line interface (doc §10).
+
+    dexverse-dl login   --server URL [--token TOKEN]
+    dexverse-dl download --manifest PATH | --task ID  --out DIR
+                         [--parallel N] [--backend s5cmd|python]
+                         [--server URL] [--no-verify] [--deep]
+    dexverse-dl verify  --manifest PATH --out DIR [--deep]
+    dexverse-dl repair  --manifest PATH --out DIR [--backend python] [--deep]
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from .client import DexVerseClient, DexVerseError, download_manifest
+from .config import config_path, save_profile
+from .downloader.manager import DownloadManager, repair as repair_fn
+from .downloader.s5cmd_backend import S5CmdNotAvailable
+from .downloader.verify import verify_files, summarize
+from .manifest import Manifest
+
+app = typer.Typer(
+    name="dexverse-dl",
+    help="DexVerse dataset downloader (S3-compatible).",
+    no_args_is_help=True,
+)
+console = Console()
+
+
+# --------------------------------------------------------------------------- #
+# login
+# --------------------------------------------------------------------------- #
+@app.command(help="Save a server + token to the active profile.")
+def login(
+    server: str = typer.Option(..., "--server", "-s", help="Platform base URL"),
+    token: Optional[str] = typer.Option(
+        None, "--token", "-t", help="Bearer token (dev-token in mock mode)"
+    ),
+    profile: str = typer.Option("default", "--profile", help="Profile name"),
+):
+    save_profile(profile, server, token)
+    console.print(f"[green]✓[/green] Saved profile [bold]{profile}[/bold] → {server}")
+    console.print(f"  config: {config_path()}")
+
+
+# --------------------------------------------------------------------------- #
+# download
+# --------------------------------------------------------------------------- #
+@app.command(help="Download by --manifest (Phase 0) or --task (Phase 1+).")
+def download(
+    out: Path = typer.Option(..., "--out", "-o", help="Local download root"),
+    manifest: Optional[Path] = typer.Option(
+        None, "--manifest", "-m", help="Local manifest.json path"
+    ),
+    task: Optional[str] = typer.Option(None, "--task", help="Platform download taskId"),
+    parallel: Optional[int] = typer.Option(None, "--parallel", "-p"),
+    backend: str = typer.Option("s5cmd", "--backend", "-b",
+                                help="s5cmd | python | http (presigned URLs)"),
+    server: Optional[str] = typer.Option(None, "--server", help="Override profile server"),
+    token: Optional[str] = typer.Option(None, "--token", help="Override profile token"),
+    no_verify: bool = typer.Option(False, "--no-verify", help="Skip post-download verify"),
+    deep: bool = typer.Option(False, "--deep", help="Also verify sha256 where recorded"),
+    insecure: bool = typer.Option(False, "--insecure", "-k",
+                                  help="Skip TLS verify (self-signed/internal endpoints)"),
+):
+    if (manifest is None) == (task is None):
+        console.print("[red]error:[/red] pass exactly one of --manifest or --task")
+        raise typer.Exit(code=2)
+
+    _warn_insecure(insecure, backend)
+
+    try:
+        if manifest is not None:
+            report = download_manifest(
+                manifest, out, parallel=parallel, backend=backend,
+                verify=not no_verify, deep=deep, insecure=insecure,
+            )
+        else:
+            # download_task returns the out-path per §11.1; run the manager here
+            # so the CLI can print a full report.
+            with DexVerseClient(server=server, token=token) as client:
+                man = client.get_manifest(task)
+            report = DownloadManager(
+                out, backend=backend, parallel=parallel,
+                verify=not no_verify, deep=deep, insecure=insecure,
+            ).run(man)
+    except S5CmdNotAvailable as exc:
+        console.print(f"[red]s5cmd not available:[/red] {exc}")
+        console.print("  hint: re-run with [cyan]--backend python[/cyan]")
+        raise typer.Exit(code=2)
+    except DexVerseError as exc:
+        console.print(f"[red]error:[/red] {exc}")
+        raise typer.Exit(code=1)
+    except (FileNotFoundError, ValueError) as exc:
+        console.print(f"[red]error:[/red] {exc}")
+        raise typer.Exit(code=1)
+
+    _print_report(report)
+    if not report.ok:
+        raise typer.Exit(code=1)
+
+
+# --------------------------------------------------------------------------- #
+# verify
+# --------------------------------------------------------------------------- #
+@app.command(help="Verify downloaded files against a manifest (size, or --deep sha256).")
+def verify(
+    manifest: Path = typer.Option(..., "--manifest", "-m"),
+    out: Path = typer.Option(..., "--out", "-o"),
+    deep: bool = typer.Option(False, "--deep", help="Also check sha256"),
+):
+    try:
+        man = Manifest.parse_file(manifest)
+        results = verify_files(man.files, out, deep=deep)
+    except (FileNotFoundError, ValueError) as exc:
+        console.print(f"[red]error:[/red] {exc}")
+        raise typer.Exit(code=1)
+    summary = summarize(results)
+
+    table = Table(title=f"Verify — task {man.effective_task_id()}")
+    table.add_column("status")
+    table.add_column("count", justify="right")
+    table.add_row("[green]ok[/green]", str(summary.ok))
+    table.add_row("missing", str(summary.missing))
+    table.add_row("size mismatch", str(summary.size_mismatch))
+    table.add_row("sha256 mismatch", str(summary.sha256_mismatch))
+    console.print(table)
+
+    for r in summary.bad_results:
+        console.print(f"  [red]✗[/red] {r.relpath}  ({r.reason})")
+
+    console.print(
+        f"\n{summary.ok}/{summary.total} files verified"
+        f"{' [bold green]all good[/bold green]' if summary.all_ok else ''}"
+    )
+    if not summary.all_ok:
+        raise typer.Exit(code=1)
+
+
+# --------------------------------------------------------------------------- #
+# repair
+# --------------------------------------------------------------------------- #
+@app.command(help="Re-download missing / size-mismatched / sha256-mismatched files. "
+                  "Use --task to fetch a fresh manifest (refreshes expired presigned URLs).")
+def repair(
+    out: Path = typer.Option(..., "--out", "-o"),
+    manifest: Optional[Path] = typer.Option(None, "--manifest", "-m"),
+    task: Optional[str] = typer.Option(None, "--task", help="Refresh manifest from backend"),
+    server: Optional[str] = typer.Option(None, "--server"),
+    token: Optional[str] = typer.Option(None, "--token"),
+    backend: str = typer.Option("python", "--backend", "-b",
+                                help="default python (no external binary needed)"),
+    parallel: Optional[int] = typer.Option(None, "--parallel", "-p"),
+    deep: bool = typer.Option(False, "--deep", help="Detect sha256 corruption too"),
+    insecure: bool = typer.Option(False, "--insecure", "-k",
+                                  help="Skip TLS verify (self-signed/internal endpoints)"),
+):
+    if (manifest is None) == (task is None):
+        console.print("[red]error:[/red] pass exactly one of --manifest or --task")
+        raise typer.Exit(code=2)
+
+    _warn_insecure(insecure, backend)
+
+    try:
+        if task is not None:
+            # fresh manifest = fresh presigned URLs (the persisted copy's may be expired)
+            with DexVerseClient(server=server, token=token) as client:
+                man = client.get_manifest(task)
+        else:
+            man = Manifest.parse_file(manifest)
+        report = repair_fn(man, out, backend=backend, parallel=parallel,
+                           deep=deep, insecure=insecure)
+    except S5CmdNotAvailable as exc:
+        console.print(f"[red]s5cmd not available:[/red] {exc}")
+        console.print("  hint: re-run with [cyan]--backend python[/cyan]")
+        raise typer.Exit(code=2)
+    except DexVerseError as exc:
+        console.print(f"[red]error:[/red] {exc}")
+        raise typer.Exit(code=1)
+    except (FileNotFoundError, ValueError) as exc:
+        console.print(f"[red]error:[/red] {exc}")
+        raise typer.Exit(code=1)
+    _print_report(report)
+    if not report.ok:
+        raise typer.Exit(code=1)
+
+
+# --------------------------------------------------------------------------- #
+# helpers
+# --------------------------------------------------------------------------- #
+def _warn_insecure(insecure: bool, backend: str) -> None:
+    if not insecure:
+        return
+    backend = (backend or "").lower()
+    if backend in ("http", "https", "presigned"):
+        console.print(
+            "[yellow]WARNING:[/yellow] --insecure disables TLS verification. For "
+            "self-authenticating presigned URLs, TLS is the only transport-integrity "
+            "check — any certificate will be accepted and redirects followed."
+        )
+    else:
+        console.print(
+            "[yellow]WARNING:[/yellow] --insecure disables TLS verification "
+            "(self-signed/internal endpoint)."
+        )
+
+
+def _print_report(report) -> None:
+    console.print(f"\n[bold]Download report[/bold] — {report.summary_line()}")
+    console.print(f"  out:       {report.out}")
+    console.print(f"  task dir:  {report.task_dir}")
+    console.print(f"  state:     {report.state_path}")
+    if report.failures:
+        console.print(f"  [red]failures:[/red]")
+        for fl in report.failures[:20]:
+            console.print(f"    ✗ {fl['relpath']} — {fl.get('error')}")
+        if len(report.failures) > 20:
+            console.print(f"    ... and {len(report.failures) - 20} more")
+
+
+if __name__ == "__main__":
+    app()

dexverse/client.py

@@ -0,0 +1,107 @@
+"""Python SDK client (doc §11).
+
+``DexVerseClient`` talks to the platform backend to fetch a manifest for a
+taskId, then hands it to the download manager. ``download_manifest`` runs the
+manager directly off a local manifest file (Phase 0, no server).
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional, Union
+
+import httpx
+
+from .config import active_profile
+from .downloader.manager import DownloadManager, DownloadReport, get_backend  # noqa: F401
+from .manifest import Manifest
+
+
+class DexVerseError(RuntimeError):
+    pass
+
+
+class DexVerseClient:
+    def __init__(self, server: Optional[str] = None, token: Optional[str] = None, *,
+                 timeout: float = 30.0):
+        # Fall back to the active profile saved by `dexverse-dl login`.
+        if not server or not token:
+            prof = active_profile()
+            if prof:
+                server = server or prof.server
+                token = token if token is not None else prof.token
+        if not server:
+            raise DexVerseError("no server configured (run `dexverse-dl login` first)")
+        self.server = server.rstrip("/")
+        self.token = token
+        self._client = httpx.Client(timeout=timeout)
+
+    def _headers(self) -> dict:
+        h = {"Accept": "application/json"}
+        if self.token:
+            h["Authorization"] = f"Bearer {self.token}"
+        return h
+
+    def get_manifest(self, task_id: str) -> Manifest:
+        url = f"{self.server}/api/download-tasks/{task_id}/manifest"
+        resp = self._client.get(url, headers=self._headers())
+        if resp.status_code == 404:
+            raise DexVerseError(f"task not found: {task_id}")
+        if resp.status_code in (401, 403):
+            raise DexVerseError(f"auth failed ({resp.status_code}) for task {task_id}")
+        resp.raise_for_status()
+        return Manifest.model_validate(resp.json())
+
+    def download_task(self, task_id: str, out: Union[str, Path], *,
+                      parallel: Optional[int] = None,
+                      backend: str = "s5cmd",
+                      verify: bool = True,
+                      deep: bool = False,
+                      insecure: bool = False) -> Path:
+        """Download a task by id; returns the local out directory (doc §11.1).
+
+        ``local_root = client.download_task(...)`` is the user-supplied ``out``,
+        with files landed at their original relative paths. Raises DexVerseError
+        if the run finishes with failures.
+        """
+        manifest = self.get_manifest(task_id)
+        mgr = DownloadManager(out, backend=backend, parallel=parallel,
+                              verify=verify, deep=deep, insecure=insecure)
+        report = mgr.run(manifest)
+        if not report.ok:
+            raise DexVerseError(
+                f"download completed with {report.failed + report.corrupt} problem(s): "
+                + report.summary_line()
+            )
+        return report.out
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> "DexVerseClient":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+
+def download_manifest(manifest_path: Union[str, Path], out: Union[str, Path], *,
+                      parallel: Optional[int] = None,
+                      backend: str = "s5cmd",
+                      verify: bool = True,
+                      deep: bool = False,
+                      insecure: bool = False) -> DownloadReport:
+    """Run a download straight from a local manifest file (Phase 0).
+
+    Returns the detailed DownloadReport (counts + failures). Note this differs
+    from ``DexVerseClient.download_task``, which returns the out path per §11.1.
+    """
+    manifest = Manifest.parse_file(Path(manifest_path))
+    mgr = DownloadManager(out, backend=backend, parallel=parallel,
+                          verify=verify, deep=deep, insecure=insecure)
+    report = mgr.run(manifest)
+    if not report.ok:
+        raise DexVerseError(
+            f"download completed with {report.failed + report.corrupt} problem(s): "
+            + report.summary_line()
+        )
+    return report

dexverse/config.py

@@ -0,0 +1,85 @@
+"""Login / server configuration.
+
+Stores the active server URL and bearer token per *profile* in a TOML file at
+``~/.dexverse/config.toml``. The file is created with mode 0600 where the OS
+supports it. Secrets are never written into the manifest or into Git
+(see doc §13.1).
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+import tomlkit
+
+
+def config_path() -> Path:
+    """Location of the CLI config file (overridable via DEXVERSE_CONFIG)."""
+    override = os.getenv("DEXVERSE_CONFIG")
+    if override:
+        return Path(override)
+    return Path.home() / ".dexverse" / "config.toml"
+
+
+@dataclass
+class Profile:
+    server: str
+    token: Optional[str] = None
+
+
+def _read() -> tomlkit.TOMLDocument:
+    p = config_path()
+    if not p.exists():
+        return tomlkit.document()
+    try:
+        return tomlkit.parse(p.read_text(encoding="utf-8"))
+    except Exception:
+        return tomlkit.document()
+
+
+def _write(doc: tomlkit.TOMLDocument) -> None:
+    p = config_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    # Best-effort restrictive permissions (POSIX only; no-op on Windows).
+    p.write_text(tomlkit.dumps(doc), encoding="utf-8")
+    try:
+        os.chmod(p, 0o600)
+    except OSError:
+        pass
+
+
+def save_profile(name: str, server: str, token: Optional[str]) -> None:
+    doc = _read()
+    profiles = doc.get("profiles")
+    if not isinstance(profiles, tomlkit.items.Table) or profiles is None:
+        profiles = tomlkit.table()
+        doc["profiles"] = profiles
+    table = tomlkit.table()
+    table["server"] = server
+    if token:
+        table["token"] = token
+    profiles[name] = table  # type: ignore[assignment]
+    doc["active"] = name
+    _write(doc)
+
+
+def load_profile(name: Optional[str] = None) -> Optional[Profile]:
+    """Load the named profile, or the active one if name is None."""
+    doc = _read()
+    profiles = doc.get("profiles")
+    if not isinstance(profiles, dict):
+        return None
+    target = name or doc.get("active")
+    if not target or target not in profiles:
+        return None
+    row = profiles[target]
+    server = row.get("server")
+    if not server:
+        return None
+    return Profile(server=server, token=row.get("token"))
+
+
+def active_profile() -> Optional[Profile]:
+    return load_profile(None)

dexverse/downloader/__init__.py

@@ -0,0 +1,26 @@
+"""Download core: manager, backends, verify, state."""
+from __future__ import annotations
+
+from .manager import DownloadManager, DownloadReport, get_backend, repair
+from .base import Backend, Credentials, FileResult
+from .s5cmd_backend import S5CmdBackend, S5CmdNotAvailable, generate_s5cmd_lines
+from .python_s3_backend import PythonS3Backend
+from .http_backend import HttpBackend
+from .verify import verify_files, summarize
+
+__all__ = [
+    "DownloadManager",
+    "DownloadReport",
+    "get_backend",
+    "repair",
+    "Backend",
+    "Credentials",
+    "FileResult",
+    "S5CmdBackend",
+    "S5CmdNotAvailable",
+    "generate_s5cmd_lines",
+    "PythonS3Backend",
+    "HttpBackend",
+    "verify_files",
+    "summarize",
+]

dexverse/downloader/base.py

@@ -0,0 +1,82 @@
+"""Backend contract + shared credential resolution.
+
+A *backend* takes the manifest, the subset of files that still need fetching,
+the out root, and returns a per-object success/failure outcome. The download
+manager owns skip/resume/verify; backends only move bytes.
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Optional, Protocol
+
+from ..manifest import FileSpec, Manifest
+
+
+@dataclass
+class Credentials:
+    access_key: Optional[str]
+    secret_key: Optional[str]
+    session_token: Optional[str]
+    region: Optional[str]
+    endpoint: Optional[str]
+
+    @classmethod
+    def from_manifest(cls, manifest: Manifest) -> "Credentials":
+        """Prefer manifest credentials; fall back to AWS_* env vars."""
+        storage = manifest.storage
+        endpoint = storage.endpoint if storage else None
+        region = (storage.region if storage else None) or os.getenv("AWS_REGION") \
+            or os.getenv("AWS_DEFAULT_REGION") or "us-east-1"
+
+        ak = sk = st = None
+        if storage and storage.credentials and storage.credentials.is_populated():
+            ak = storage.credentials.accessKey
+            sk = storage.credentials.secretKey
+            st = storage.credentials.sessionToken or None
+        else:
+            ak = os.getenv("AWS_ACCESS_KEY_ID")
+            sk = os.getenv("AWS_SECRET_ACCESS_KEY")
+            st = os.getenv("AWS_SESSION_TOKEN") or None
+
+        return cls(
+            access_key=ak, secret_key=sk, session_token=st,
+            region=region, endpoint=endpoint,
+        )
+
+    def as_env(self) -> Dict[str, str]:
+        """A dict suitable for merging into a subprocess environment."""
+        env: Dict[str, str] = {}
+        if self.access_key:
+            env["AWS_ACCESS_KEY_ID"] = self.access_key
+        if self.secret_key:
+            env["AWS_SECRET_ACCESS_KEY"] = self.secret_key
+        if self.session_token:
+            env["AWS_SESSION_TOKEN"] = self.session_token
+        if self.region:
+            env["AWS_REGION"] = self.region
+            env["AWS_DEFAULT_REGION"] = self.region
+        return env
+
+
+@dataclass
+class FileResult:
+    objectKey: str
+    ok: bool
+    error: Optional[str] = None
+
+
+class Backend(Protocol):
+    name: str
+
+    def download(
+        self,
+        manifest: Manifest,
+        files: List[FileSpec],
+        out: Path,
+        *,
+        parallel: int,
+        log_path: Optional[Path] = None,
+    ) -> Dict[str, FileResult]:
+        ...