vizo 0.0.1__py3-none-any.whl

vizo/__init__.py

@@ -0,0 +1,8 @@
+"""VIZO command-line tool and, later, Python SDK.
+
+The `vizo` command (datasets, download, verify) is in cli.py; the manifest
+format it reads is in manifest.py. "design §N" in comments refers to
+roadmap_v2/cli_design.md in the vizo-platform repository.
+"""
+
+__version__ = "0.0.1"

vizo/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

vizo/api.py

@@ -0,0 +1,221 @@
+"""HTTP client for the platform's /api/v1 (design §2-§4).
+
+Authenticates with the API key from VIZO_API_KEY, sends the CLI version and the
+run id with every request, and turns refusals into ApiError with the code and
+message the platform returned. Standard library only, so the CLI installs
+anywhere Python does.
+"""
+
+import json
+import os
+import re
+import socket
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
+
+from . import __version__
+
+DEFAULT_API_URL = "https://vizo-platform.vercel.app"
+ENV_API_KEY = "VIZO_API_KEY"
+ENV_API_URL = "VIZO_API_URL"
+KEY_PREFIX = "vz_sk_live_"
+# The platform issues the prefix plus base64url characters. Anything else (a
+# line break from `$(cat keyfile)`, a space) must stop here: the HTTP layer
+# would refuse the header with an error that spells the key out.
+KEY_PATTERN = re.compile(re.escape(KEY_PREFIX) + r"[A-Za-z0-9_-]+")
+LOCAL_HOSTS = ("localhost", "127.0.0.1", "::1")
+
+RETRIES = 4
+RETRY_STATUSES = {502, 503, 504}
+
+
+class ApiError(Exception):
+    """A refusal or failure from the platform, with the code it gave."""
+
+    def __init__(self, status: int, code: str, message: str, extra: Optional[Dict[str, Any]] = None):
+        super().__init__(message)
+        self.status = status
+        self.code = code
+        self.message = message
+        self.extra = extra or {}
+
+    def __str__(self) -> str:
+        hint = ""
+        if self.extra.get("license_url"):
+            hint = f"\nRequest a license at {self.extra['license_url']}"
+        elif self.extra.get("api_keys_url"):
+            hint = f"\nManage keys at {self.extra['api_keys_url']}"
+        return f"{self.message} [{self.code}]{hint}"
+
+
+@dataclass
+class Link:
+    file_id: str
+    url: str
+    expires_at: str
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):  # type: ignore[override]
+        return None
+
+
+def api_key_from_env(api_url: Optional[str] = None) -> str:
+    key = os.environ.get(ENV_API_KEY, "").strip()
+    if not key:
+        raise ApiError(0, "missing_key", f"Set {ENV_API_KEY} to your API key.",
+                       {"api_keys_url": resolve_api_url(api_url) + "/api-keys"})
+    if not key.startswith(KEY_PREFIX):
+        raise ApiError(0, "invalid_key", f"{ENV_API_KEY} does not look like a VIZO key (expected {KEY_PREFIX}...).")
+    if not KEY_PATTERN.fullmatch(key):
+        raise ApiError(0, "invalid_key", f"{ENV_API_KEY} has characters a VIZO key never contains "
+                                         "(a space or a line break inside it?); set it to the key alone.")
+    return key
+
+
+def resolve_api_url(api_url: Optional[str] = None) -> str:
+    url = (api_url or os.environ.get(ENV_API_URL) or DEFAULT_API_URL).rstrip("/")
+    u = urllib.parse.urlparse(url)
+    if u.scheme != "https" and not (u.scheme == "http" and u.hostname in LOCAL_HOSTS):
+        raise ValueError(f"the API URL {url} must use https (plain http only for localhost), "
+                         "or your API key would travel in cleartext")
+    return url
+
+
+class PlatformClient:
+    def __init__(self, api_key: str, run_id: str, api_url: Optional[str] = None, timeout: int = 60):
+        self.api_key = api_key
+        self.run_id = run_id
+        self.api_url = resolve_api_url(api_url)
+        self.timeout = timeout
+        self.key_expires_at: Optional[str] = None
+        # Redirects are handled by hand: the manifest endpoint may answer 302 to
+        # a presigned R2 URL, which must be fetched WITHOUT our bearer header.
+        self._opener = urllib.request.build_opener(_NoRedirect)
+
+    # -- plumbing ---------------------------------------------------------
+
+    def _headers(self, extra: Optional[Dict[str, str]] = None) -> Dict[str, str]:
+        h = {
+            "Authorization": f"Bearer {self.api_key}",
+            "X-Vizo-CLI-Version": __version__,
+            "X-Vizo-Run-Id": self.run_id,
+            "User-Agent": f"vizo-cli/{__version__}",
+            "Accept": "application/json, text/csv",
+        }
+        if extra:
+            h.update(extra)
+        return h
+
+    def _request(self, method: str, path: str, body: Optional[Dict[str, Any]] = None) -> Tuple[int, Dict[str, str], bytes]:
+        data = json.dumps(body).encode() if body is not None else None
+        headers = self._headers({"Content-Type": "application/json"} if data else None)
+        return self._open(urllib.request.Request(self.api_url + path, data=data, method=method, headers=headers),
+                          self.api_url)
+
+    def _open(self, req: urllib.request.Request, where: str) -> Tuple[int, Dict[str, str], bytes]:
+        last_error: Optional[Exception] = None
+        for attempt in range(RETRIES):
+            try:
+                with self._opener.open(req, timeout=self.timeout) as resp:
+                    return resp.status, dict(resp.headers), resp.read()
+            except urllib.error.HTTPError as e:
+                payload = e.read()
+                if self._retryable(e.code, payload) and attempt < RETRIES - 1:
+                    last_error = e
+                    time.sleep(2 ** attempt)
+                    continue
+                if 300 <= e.code < 400:
+                    return e.code, dict(e.headers), payload
+                err = self._api_error(e.code, payload)
+                if e.headers.get("Retry-After"):
+                    err.extra["retry_after"] = e.headers["Retry-After"]
+                raise err
+            except (urllib.error.URLError, socket.timeout, ConnectionError) as e:
+                last_error = e
+                if attempt < RETRIES - 1:
+                    time.sleep(2 ** attempt)
+                    continue
+        raise ApiError(0, "unreachable", f"Could not reach {where}: {last_error}")
+
+    @classmethod
+    def _retryable(cls, status: int, payload: bytes) -> bool:
+        if status == 503:
+            # the platform marks a transient 503 `unavailable`; one without a JSON error comes from a proxy
+            return cls._api_error(status, payload).code in ("unavailable", f"http_{status}")
+        return status in RETRY_STATUSES
+
+    @staticmethod
+    def _api_error(status: int, payload: bytes) -> ApiError:
+        try:
+            err = json.loads(payload.decode("utf-8"))["error"]
+            code, message = err.pop("code"), err.pop("message")
+            return ApiError(status, code, message, err)
+        except (ValueError, KeyError, TypeError):
+            return ApiError(status, f"http_{status}", payload.decode("utf-8", "replace")[:300] or f"HTTP {status}")
+
+    def _note_expiry(self, headers: Dict[str, str]) -> None:
+        exp = headers.get("X-Vizo-Key-Expires-At") or headers.get("x-vizo-key-expires-at")
+        if exp:
+            self.key_expires_at = exp
+
+    # -- endpoints --------------------------------------------------------
+
+    @staticmethod
+    def _unexpected(status: int, path: str, e: Exception) -> ApiError:
+        return ApiError(status, "unexpected_response", f"The platform sent an unexpected response to {path} ({e!r}).")
+
+    def datasets(self) -> List[Dict[str, Any]]:
+        path = "/api/v1/datasets"
+        status, headers, body = self._request("GET", path)
+        self._note_expiry(headers)
+        try:
+            data = json.loads(body)
+            datasets = data["datasets"]
+            if not all(isinstance(d, dict) and "id" in d for d in datasets):
+                raise TypeError("datasets entries must be objects with an id")
+            self.key_expires_at = data.get("key_expires_at") or self.key_expires_at
+        except (ValueError, KeyError, TypeError, AttributeError) as e:
+            raise self._unexpected(status, path, e)
+        return datasets
+
+    def manifest(self, dataset: str) -> bytes:
+        """The current release's manifest as bytes (gzip or plain CSV; the reader detects which)."""
+        path = f"/api/v1/datasets/{urllib.parse.quote(dataset)}/manifest"
+        try:
+            status, headers, body = self._request("GET", path)
+        except ApiError as e:
+            if e.code == "manifest_not_published":
+                raise ApiError(e.status, e.code, f"The current {dataset} release has not been published yet; "
+                                                 "try again once it has.", e.extra)
+            raise
+        self._note_expiry(headers)
+        if status in (301, 302, 303, 307, 308):
+            location = headers.get("Location") or headers.get("location")
+            if not location:
+                raise ApiError(status, "bad_redirect", "The platform redirected without a location.")
+            # A presigned URL: no bearer header, or R2 rejects the request.
+            req = urllib.request.Request(location, headers={"User-Agent": f"vizo-cli/{__version__}"})
+            try:
+                status, _, body = self._open(req, "the manifest storage")
+            except ApiError as e:
+                # status 0: storage being unreachable or refusing says nothing about the key or license
+                reason = f"HTTP {e.status}" if e.status else e.message
+                raise ApiError(0, "manifest_fetch_failed", f"Fetching the manifest from storage failed: {reason}")
+        return body
+
+    def links(self, dataset: str, file_ids: List[str]) -> Tuple[List[Link], List[str]]:
+        path = f"/api/v1/datasets/{urllib.parse.quote(dataset)}/links"
+        status, headers, body = self._request("POST", path, {"file_ids": file_ids})
+        self._note_expiry(headers)
+        try:
+            data = json.loads(body)
+            links = [Link(l["file_id"], l["url"], l["expires_at"])
+                     for l in data.get("links", [])]
+            return links, list(data.get("unknown_file_ids", []))
+        except (ValueError, KeyError, TypeError, AttributeError) as e:
+            raise self._unexpected(status, path, e)

vizo/cli.py

@@ -0,0 +1,387 @@
+"""The `vizo` command: datasets, download, verify (design §1, §5).
+
+Exit codes: 0 done · 1 usage or local error · 2 authentication (key) ·
+3 not entitled · 4 download incomplete or platform unreachable.
+"""
+
+import argparse
+import datetime as dt
+import io
+import json
+import os
+import sys
+import time
+from typing import List, Optional
+
+from . import __version__
+from .api import ApiError, PlatformClient, api_key_from_env
+from .download import Downloader, LocalWriteError, Progress
+from .filters import Selection, Shard, select, shard_files, totals, unknown_values
+from .manifest import ManifestError, parse_manifest
+from .state import PathOutsideOutput, Workspace, free_bytes, new_run_id
+from .verify import verify
+
+EXIT_OK, EXIT_USAGE, EXIT_AUTH, EXIT_ENTITLEMENT, EXIT_INCOMPLETE = 0, 1, 2, 3, 4
+KEY_WARNING_DAYS = 7
+LOG_PROGRESS_SECONDS = 60
+REFERENCE_BYTES_PER_SECOND = 10_000_000 / 8   # 10 Mbit/s, for the dry run's finish estimate
+
+
+def human(n: float) -> str:
+    for unit in ("B", "KB", "MB", "GB", "TB", "PB"):
+        if n < 1024 or unit == "PB":
+            return f"{n:.1f} {unit}" if unit != "B" else f"{int(n)} B"
+        n /= 1024
+    return f"{n:.1f} PB"
+
+
+def duration(seconds: float) -> str:
+    s = int(seconds)
+    if s < 60:
+        return f"{s}s"
+    if s < 3600:
+        return f"{s // 60}m {s % 60:02d}s"
+    return f"{s // 3600}h {s % 3600 // 60:02d}m"
+
+
+def exit_code_for(e: ApiError) -> int:
+    if e.status in (401, 426) or e.code in ("missing_key", "invalid_key"):
+        return EXIT_AUTH
+    if e.code == "unknown_dataset":
+        return EXIT_USAGE
+    if e.status == 403:
+        return EXIT_ENTITLEMENT
+    return EXIT_INCOMPLETE
+
+
+def key_expiry(client: PlatformClient) -> Optional[dt.date]:
+    """The key's expiry date, once a platform response has reported it."""
+    if not client.key_expires_at:
+        return None
+    try:
+        # only the date matters; the time part's format varies between deployments
+        return dt.date.fromisoformat(client.key_expires_at[:10])
+    except ValueError:
+        return None
+
+
+def warn_key_expiry(client: PlatformClient) -> bool:
+    """Warn when the key expires soon. Returns whether the expiry was known."""
+    exp = key_expiry(client)
+    if exp is None:
+        return False
+    days = (exp - dt.datetime.now(dt.timezone.utc).date()).days
+    if days < KEY_WARNING_DAYS:
+        print(f"warning: your API key expires in {max(days, 0)} day(s) ({exp}); "
+              f"create a new one on the API Keys page before long downloads.", file=sys.stderr)
+    return True
+
+
+# -- selectors shared by download and verify ---------------------------------
+
+def add_selectors(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--parts", "--datasets", nargs="+", metavar="PART",
+                   help="parts to include (Ego-Exo4D --parts / Ego4D --datasets names); default: everything")
+    p.add_argument("--uids", "--video_uids", "--video-uids", nargs="+", metavar="UID",
+                   help="video / take / capture uids (Ego-Exo4D take names also work)")
+    p.add_argument("--uid-file", "--video_uid_file", metavar="FILE", help="file with one uid per line")
+    p.add_argument("--universities", "-u", nargs="+", metavar="ID")
+    p.add_argument("--benchmarks", nargs="+", metavar="NAME", help="case-insensitive (EM, FHO, NLQ, keystep, ...)")
+    p.add_argument("--splits", nargs="+", metavar="SPLIT", help="train val test (Ego4D also accepts nlq:val style)")
+    p.add_argument("--views", nargs="+", choices=["ego", "exo"])
+    p.add_argument("--metadata", action="store_true", help="also fetch the metadata part (ego4d.json, takes.json, ...)")
+
+
+def selection_from(args: argparse.Namespace) -> Selection:
+    uids = list(args.uids or [])
+    if args.uid_file:
+        with open(args.uid_file, encoding="utf-8") as fp:
+            uids += [line.strip() for line in fp if line.strip() and not line.startswith("#")]
+    return Selection.from_lists(parts=args.parts, uids=uids, universities=args.universities,
+                                benchmarks=args.benchmarks, splits=args.splits, views=args.views,
+                                include_metadata=args.metadata)
+
+
+def check_selection(rows, sel: Selection) -> None:
+    """An unknown part is an error; any other selector value on no row is a warning."""
+    unknown = unknown_values(rows, sel)
+    if "parts" in unknown:
+        raise ValueError(f"unknown part(s): {' '.join(sorted(unknown.pop('parts')))}; "
+                         f"this release has: {' '.join(sorted({f.part for f in rows}))}")
+    for name, values in unknown.items():
+        shown = sorted(values)
+        more = f" (and {len(shown) - 10} more)" if len(shown) > 10 else ""
+        print(f"warning: --{name} {' '.join(shown[:10])}{more}: on no file in the manifest", file=sys.stderr)
+
+
+def print_plan(dataset: str, release: str, chosen, shard: Optional[Shard]) -> None:
+    t = totals(chosen)
+    meta = f" + {t.meta_files} metadata file{'s' if t.meta_files != 1 else ''} ({human(t.meta_bytes)})" if t.meta_files else ""
+    print(f"{dataset} {release}: {t.files} files, {human(t.bytes)}{meta}"
+          + (f"  (shard {shard.index}/{shard.count})" if shard else ""))
+    for part, pt in sorted(t.by_part.items()):
+        extra = f"  + {pt.meta_files} metadata ({human(pt.meta_bytes)})" if pt.meta_files else ""
+        print(f"  {part:28} {pt.files:>8} files  {human(pt.bytes):>10}{extra}")
+    if t.files == 0 and t.meta_files:
+        print("note: no data files match this selection; only the consortium metadata files that "
+              "accompany the chosen part(s) would be fetched.", file=sys.stderr)
+
+
+def report_temp_files(ws: Workspace) -> None:
+    count, size = ws.temp_files()
+    if count:
+        print(f"note: {count} temporary *.part file(s) ({human(size)}) from unfinished downloads are in "
+              f"{ws.out_dir}; they are safe to delete when no other vizo run is using this folder.", file=sys.stderr)
+
+
+# -- commands -----------------------------------------------------------------
+
+def cmd_datasets(args: argparse.Namespace) -> int:
+    client = PlatformClient(api_key_from_env(args.api_url), new_run_id(), args.api_url)
+    datasets = client.datasets()
+    warn_key_expiry(client)
+    if args.json:
+        print(json.dumps({"datasets": datasets, "key_expires_at": client.key_expires_at}, indent=2))
+        return EXIT_OK
+    print(f"{'dataset':12} {'access':14} {'release':8} {'files':>8} {'size':>10}")
+    for d in datasets:
+        access = "download" if d.get("entitled") else (d.get("license_status") or "no license")
+        rel = d.get("release") or {}
+        print(f"{d['id']:12} {access:14} {rel.get('release', '-'):8} {rel.get('file_count', 0):>8} {human(rel.get('total_bytes', 0)):>10}")
+        if not d.get("entitled"):
+            label = {"pending": "under review", "approved": "license"}.get(d.get("license_status"), "request access")
+            print(f"{'':12} {label}: {d.get('license_url')}")
+    return EXIT_OK
+
+
+def load_or_fetch_manifest(client: PlatformClient, ws: Workspace, dataset: str, refresh: bool):
+    if ws.has_manifest():
+        try:
+            header, rows = ws.load_manifest()
+        except ManifestError:
+            if not refresh:
+                raise
+        else:
+            # one dataset per folder, refreshing included
+            if header.dataset != dataset:
+                raise SystemExit(f"{ws.out_dir} already holds a {header.dataset} download; use another output folder.")
+            if not refresh:
+                return header, rows, "saved"
+    data = client.manifest(dataset)
+    try:
+        header, rows = parse_manifest(io.BytesIO(data))
+    except ManifestError as e:
+        raise ApiError(0, "bad_manifest", f"The platform sent a manifest this CLI cannot read ({e}); nothing was saved.")
+    ws.save_manifest(data)
+    return header, rows, "fetched"
+
+
+def cmd_download(args: argparse.Namespace) -> int:
+    run_id = new_run_id()
+    client = PlatformClient(api_key_from_env(args.api_url), run_id, args.api_url)
+    ws = Workspace(args.output, run_id)
+    shard = Shard.parse(args.shard) if args.shard else None
+
+    sel = selection_from(args)
+    header, rows, source = load_or_fetch_manifest(client, ws, args.dataset, args.refresh_manifest)
+    expiry_known = warn_key_expiry(client)
+    check_selection(rows, sel)
+    chosen = select(rows, sel, args.dataset)
+    if shard:
+        chosen = shard_files(chosen, shard)
+    if not chosen:
+        print("Nothing matches the selection.", file=sys.stderr)
+        return EXIT_USAGE
+
+    print(f"manifest: {source} ({header.files} files in release {header.release})")
+    print_plan(args.dataset, header.release, chosen, shard)
+
+    tty = sys.stdout.isatty()
+    last_line = [0.0]
+
+    def on_progress(p: Progress) -> None:
+        if not tty:
+            # a job log gets a line about once a minute, not a redraw every half second
+            if last_line[0] and time.monotonic() - last_line[0] < LOG_PROGRESS_SECONDS:
+                return
+            last_line[0] = time.monotonic()
+        speed = p.speed()
+        left = p.bytes_total - p.bytes_skipped - p.bytes_received
+        eta = duration(left / speed) if speed > 0 else "?"
+        line = (f"{p.files_done + p.files_skipped}/{p.files_total} files  "
+                f"{human(p.bytes_received + p.bytes_skipped)}/{human(p.bytes_total)}  "
+                f"{human(speed)}/s  eta {eta}")
+        if tty:
+            print(f"\r{line}   ", end="", flush=True)
+        else:
+            print(line, flush=True)
+
+    def on_links() -> None:
+        # a resumed run's first request is for links: the first chance to learn the expiry
+        nonlocal expiry_known
+        if not expiry_known:
+            expiry_known = warn_key_expiry(client)
+
+    dl = Downloader(client, args.dataset, ws, chosen, workers=args.workers, on_progress=on_progress, on_links=on_links)
+    todo = dl.pending()
+    remaining = sum(f.size for f in todo)
+    print(f"to fetch: {len(todo)} files, {human(remaining)} ({dl.progress.files_skipped} already complete)")
+    if args.dry_run:
+        if not expiry_known:
+            # no response so far reported it (a saved manifest means no request at all): ask
+            try:
+                client.datasets()
+            except ApiError as e:
+                print(f"key: expiry could not be checked ({e})")
+                return EXIT_OK
+        exp = key_expiry(client)
+        if exp is None:
+            print("key: never expires")
+        else:
+            seconds = remaining / REFERENCE_BYTES_PER_SECOND
+            finish = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=seconds)
+            when = "before" if exp <= finish.date() else "after"
+            print(f"key: expires {exp}, {when} the estimated finish ({duration(seconds)} at 10 Mbit/s)")
+        return EXIT_OK
+
+    free = free_bytes(ws.out_dir)   # exists: the manifest was saved in it
+    if free is not None and free < remaining and not args.ignore_disk_space:
+        print(f"error: {human(free)} free at {ws.out_dir}, {human(remaining)} needed (--ignore-disk-space to proceed anyway)", file=sys.stderr)
+        return EXIT_USAGE
+    if not args.yes and sys.stdin.isatty() and remaining > 0:
+        answer = input("Proceed? [y/N] ").strip().lower()
+        if answer not in ("y", "yes"):
+            return EXIT_USAGE
+
+    try:
+        result = dl.run()
+    except (LocalWriteError, ApiError) as e:
+        if tty:
+            print()
+        print(f"error: {e}", file=sys.stderr)
+        print(f"stopped: {dl.progress.files_done} fetched, {len(dl.failures)} failed. log: {ws.log_path}", file=sys.stderr)
+        report_temp_files(ws)
+        return exit_code_for(e) if isinstance(e, ApiError) else EXIT_INCOMPLETE
+    if tty:
+        print()
+    if result.failures:
+        print(f"{len(result.failures)} file(s) failed:", file=sys.stderr)
+        for fail in result.failures[:20]:
+            print(f"  {fail.path}: {fail.reason}", file=sys.stderr)
+        if len(result.failures) > 20:
+            print(f"  ... and {len(result.failures) - 20} more (see {ws.log_path})", file=sys.stderr)
+    if result.interrupted:
+        print("interrupted; run the same command again to resume", file=sys.stderr)
+    print(f"done: {result.progress.files_done} fetched, {result.progress.files_skipped} already present, "
+          f"{len(result.failures)} failed. log: {ws.log_path}")
+    report_temp_files(ws)
+    return EXIT_OK if result.ok else EXIT_INCOMPLETE
+
+
+def present_or_refused(ws: Workspace, f) -> bool:
+    """In verify's default scope: the file is here, or its path is one verify() reports as refused."""
+    try:
+        return os.path.exists(ws.final_path(f.path))
+    except PathOutsideOutput:
+        return True
+
+
+def cmd_verify(args: argparse.Namespace) -> int:
+    ws = Workspace(args.output, new_run_id())
+    if not ws.has_manifest():
+        print(f"no saved manifest in {ws.root}; this folder was not produced by vizo download", file=sys.stderr)
+        return EXIT_USAGE
+    header, rows = ws.load_manifest()
+    sel = selection_from(args)
+    check_selection(rows, sel)
+    chosen = select(rows, sel, header.dataset)
+    if args.shard:
+        chosen = shard_files(chosen, Shard.parse(args.shard))
+    # The saved manifest lists the whole release, not what this folder asked
+    # for. Without a selection, check the files that are here; with one (or
+    # --all), check the selection and report what is missing from it.
+    scoped = sel.is_everything() and not sel.include_metadata and not args.shard and not args.all
+    if scoped:
+        chosen = [f for f in chosen if present_or_refused(ws, f)]
+    human_out = sys.stderr if args.json else sys.stdout   # --json keeps stdout parseable
+    report = verify(ws, chosen, checksum=args.checksum,
+                    on_file=(lambda f, s: print(f"{s:12} {f.path}", file=human_out)) if args.verbose else None)
+    scope = "files present" if scoped else "files in selection"
+    print(f"{header.dataset} {header.release}: {report.ok}/{report.checked} {scope} ok"
+          + ("" if scoped else f", {len(report.missing)} missing")
+          + f", {len(report.wrong_size)} wrong size, {len(report.bad_checksum)} bad checksum"
+          + (f", {len(report.refused)} refused" if report.refused else ""), file=human_out)
+    if args.json:
+        print(json.dumps({**report.__dict__, "scope": scope}, indent=2))
+    report_temp_files(ws)
+    return EXIT_OK if report.clean else EXIT_INCOMPLETE
+
+
+# -- entry --------------------------------------------------------------------
+
+class Parser(argparse.ArgumentParser):
+    def error(self, message: str):  # type: ignore[override]
+        # argparse exits 2 on a usage error; here 2 means a key problem
+        self.print_usage(sys.stderr)
+        self.exit(EXIT_USAGE, f"{self.prog}: error: {message}\n")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = Parser(prog="vizo", description="Download licensed datasets from the VIZO platform.")
+    p.add_argument("--version", action="version", version=f"vizo {__version__}")
+    p.add_argument("--api-url", help="platform URL (default: VIZO_API_URL or the production platform)")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    d = sub.add_parser("datasets", help="list datasets and your access to them")
+    d.add_argument("--json", action="store_true")
+    d.set_defaults(func=cmd_datasets)
+
+    dl = sub.add_parser("download", help="download all or part of a dataset")
+    dl.add_argument("dataset", help="dataset id, e.g. ego4d or egoexo4d")
+    dl.add_argument("-o", "--output", required=True, help="output folder (files use the official layout)")
+    add_selectors(dl)
+    dl.add_argument("--shard", metavar="N/M",
+                    help="fetch only group N of M: the selection is split into M groups of about equal size, "
+                         "the same way on every machine, so running 1/M to M/M with the same command fetches "
+                         "everything exactly once")
+    dl.add_argument("--workers", type=int, default=8, help="parallel downloads (default 8)")
+    dl.add_argument("--dry-run", action="store_true",
+                    help="show what would be fetched and exit; it still creates the output folder and saves "
+                         "the manifest there, so the real run targets the same release")
+    dl.add_argument("-y", "--yes", action="store_true", help="do not ask for confirmation")
+    dl.add_argument("--refresh-manifest", action="store_true", help="fetch the current manifest even if one is saved")
+    dl.add_argument("--ignore-disk-space", action="store_true")
+    dl.set_defaults(func=cmd_download)
+
+    v = sub.add_parser("verify", help="check a downloaded folder against its saved manifest")
+    v.add_argument("output", help="the folder given to download -o")
+    add_selectors(v)
+    v.add_argument("--shard", metavar="N/M")
+    v.add_argument("--all", action="store_true",
+                   help="check every file of the release and report missing ones (default: only files present, unless selectors are given)")
+    v.add_argument("--checksum", action="store_true", help="also compare MD5 (slow for large downloads)")
+    v.add_argument("--verbose", "-v", action="store_true")
+    v.add_argument("--json", action="store_true")
+    v.set_defaults(func=cmd_verify)
+    return p
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except ApiError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return exit_code_for(e)
+    except (ManifestError, ValueError, OSError) as e:
+        print(f"error: {e}", file=sys.stderr)
+        return EXIT_USAGE
+    except SystemExit:
+        raise
+    except KeyboardInterrupt:
+        print("\ninterrupted", file=sys.stderr)
+        return EXIT_INCOMPLETE
+
+
+if __name__ == "__main__":
+    sys.exit(main())

vizo/datasets/__init__.py

@@ -0,0 +1,28 @@
+"""Per-dataset knowledge: how a file's path maps to the attributes the CLI filters on.
+
+Each dataset module exposes `classify(path) -> Optional[FileAttrs]`, returning
+None for a path that does not belong to the dataset's known layout so the
+manifest builder can report it instead of guessing.
+
+Each dataset module also has an enricher: a class built from the consortium
+metadata laid out as in the bucket (its `INPUTS` are the files it reads) whose
+`enrich(attrs, path)` fills in what the path cannot and returns the row's
+meta_status. The manifest builder uses it; the CLI never does.
+"""
+
+from typing import Callable, Dict, Optional
+
+from . import ego4d, egoexo4d
+from .common import FileAttrs
+
+Classifier = Callable[[str], Optional[FileAttrs]]
+
+CLASSIFIERS: Dict[str, Classifier] = {
+    "ego4d": ego4d.classify,
+    "egoexo4d": egoexo4d.classify,
+}
+
+ENRICHERS = {
+    "ego4d": ego4d.Ego4DMetadata,
+    "egoexo4d": egoexo4d.EgoExo4DMetadata,
+}