vizo 0.0.1__tar.gz

vizo-0.0.1/MANIFEST.in

@@ -0,0 +1,4 @@
+# The source archive on PyPI is public: only the client goes in it. The staff
+# tools and the test fixtures (consortium metadata) stay in the repository.
+prune tests
+prune tools

vizo-0.0.1/PKG-INFO

@@ -0,0 +1,67 @@
+Metadata-Version: 2.4
+Name: vizo
+Version: 0.0.1
+Summary: VIZO command-line tool: download licensed datasets from the VIZO platform
+Author: VIZO
+License-Expression: LicenseRef-Proprietary
+Project-URL: Homepage, https://vizo-platform.vercel.app
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+
+# VIZO CLI
+
+`vizo` is the command-line tool for downloading the datasets you are licensed
+for on the VIZO platform. It needs Python 3.9 or newer and nothing beyond the
+standard library.
+
+## Install
+
+```bash
+python3 -m venv ~/vizo-venv            # Python 3.9 or newer; the one macOS ships is fine
+source ~/vizo-venv/bin/activate
+pip3 install vizo
+```
+
+## Use
+
+```bash
+export VIZO_API_KEY=vz_sk_live_...     # from the API Keys page; the only way to authenticate
+
+vizo datasets                          # what you can download, and where to request access
+vizo download ego4d -o ~/vizo-data/ego4d --parts clips annotations --benchmarks FHO --dry-run
+vizo download ego4d -o ~/vizo-data/ego4d --parts clips annotations --benchmarks FHO -y
+vizo verify ~/vizo-data/ego4d --checksum
+```
+
+`python3 -m vizo` works the same as `vizo`.
+
+`download` accepts the official selectors (`--parts`/`--datasets`, `--uids`/
+`--video_uids`, `--uid-file`, `--universities`, `--benchmarks`, `--splits`,
+`--views`, `--metadata`), plus `--shard N/M` for splitting one download across
+machines, `--workers`, `--dry-run`, `-y`, and `--refresh-manifest`. A dry run
+fetches nothing but does create the output folder and save the manifest in it,
+so the real run that follows targets the same release. Files land
+in the official layout under `-o`; `.vizo/` inside it holds the saved manifest,
+a log and a done-record per run. Rerunning the same command skips completed
+files; a file that was interrupted starts again from its beginning. Interrupted
+or failed downloads leave `*.part` files, which a run and `verify` report and
+never delete (shards may share a folder); delete them when no run is using the
+folder.
+
+Exit codes: 0 done · 1 usage · 2 key problem · 3 not entitled ·
+4 incomplete/unreachable.
+
+The full guide is on the VIZO platform, signed in, under Develop → CLI
+(https://vizo-platform.vercel.app/docs/cli): installation, API keys, choosing
+what to download, each dataset's parts and layout, verifying, large and sharded
+downloads, a command reference and troubleshooting. Its source is `docs/` in
+the source repository; working on the CLI itself is covered in `DEVELOPMENT.md`
+there.

vizo-0.0.1/README.md

@@ -0,0 +1,49 @@
+# VIZO CLI
+
+`vizo` is the command-line tool for downloading the datasets you are licensed
+for on the VIZO platform. It needs Python 3.9 or newer and nothing beyond the
+standard library.
+
+## Install
+
+```bash
+python3 -m venv ~/vizo-venv            # Python 3.9 or newer; the one macOS ships is fine
+source ~/vizo-venv/bin/activate
+pip3 install vizo
+```
+
+## Use
+
+```bash
+export VIZO_API_KEY=vz_sk_live_...     # from the API Keys page; the only way to authenticate
+
+vizo datasets                          # what you can download, and where to request access
+vizo download ego4d -o ~/vizo-data/ego4d --parts clips annotations --benchmarks FHO --dry-run
+vizo download ego4d -o ~/vizo-data/ego4d --parts clips annotations --benchmarks FHO -y
+vizo verify ~/vizo-data/ego4d --checksum
+```
+
+`python3 -m vizo` works the same as `vizo`.
+
+`download` accepts the official selectors (`--parts`/`--datasets`, `--uids`/
+`--video_uids`, `--uid-file`, `--universities`, `--benchmarks`, `--splits`,
+`--views`, `--metadata`), plus `--shard N/M` for splitting one download across
+machines, `--workers`, `--dry-run`, `-y`, and `--refresh-manifest`. A dry run
+fetches nothing but does create the output folder and save the manifest in it,
+so the real run that follows targets the same release. Files land
+in the official layout under `-o`; `.vizo/` inside it holds the saved manifest,
+a log and a done-record per run. Rerunning the same command skips completed
+files; a file that was interrupted starts again from its beginning. Interrupted
+or failed downloads leave `*.part` files, which a run and `verify` report and
+never delete (shards may share a folder); delete them when no run is using the
+folder.
+
+Exit codes: 0 done · 1 usage · 2 key problem · 3 not entitled ·
+4 incomplete/unreachable.
+
+The full guide is on the VIZO platform, signed in, under Develop → CLI
+(https://vizo-platform.vercel.app/docs/cli): installation, API keys, choosing
+what to download, each dataset's parts and layout, verifying, large and sharded
+downloads, a command reference and troubleshooting. Its source is `docs/` in
+the source repository; working on the CLI itself is covered in `DEVELOPMENT.md`
+there.

vizo-0.0.1/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "vizo"
+dynamic = ["version"]
+description = "VIZO command-line tool: download licensed datasets from the VIZO platform"
+readme = "README.md"
+requires-python = ">=3.9"
+license = "LicenseRef-Proprietary"
+authors = [{ name = "VIZO" }]
+dependencies = []
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Scientific/Engineering",
+]
+
+[project.urls]
+Homepage = "https://vizo-platform.vercel.app"
+
+[project.scripts]
+vizo = "vizo.cli:main"
+
+[project.optional-dependencies]
+dev = ["pytest>=7"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.dynamic]
+version = { attr = "vizo.__version__" }
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

vizo-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

vizo-0.0.1/src/vizo/__init__.py

@@ -0,0 +1,8 @@
+"""VIZO command-line tool and, later, Python SDK.
+
+The `vizo` command (datasets, download, verify) is in cli.py; the manifest
+format it reads is in manifest.py. "design §N" in comments refers to
+roadmap_v2/cli_design.md in the vizo-platform repository.
+"""
+
+__version__ = "0.0.1"

vizo-0.0.1/src/vizo/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

vizo-0.0.1/src/vizo/api.py

@@ -0,0 +1,221 @@
+"""HTTP client for the platform's /api/v1 (design §2-§4).
+
+Authenticates with the API key from VIZO_API_KEY, sends the CLI version and the
+run id with every request, and turns refusals into ApiError with the code and
+message the platform returned. Standard library only, so the CLI installs
+anywhere Python does.
+"""
+
+import json
+import os
+import re
+import socket
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
+
+from . import __version__
+
+DEFAULT_API_URL = "https://vizo-platform.vercel.app"
+ENV_API_KEY = "VIZO_API_KEY"
+ENV_API_URL = "VIZO_API_URL"
+KEY_PREFIX = "vz_sk_live_"
+# The platform issues the prefix plus base64url characters. Anything else (a
+# line break from `$(cat keyfile)`, a space) must stop here: the HTTP layer
+# would refuse the header with an error that spells the key out.
+KEY_PATTERN = re.compile(re.escape(KEY_PREFIX) + r"[A-Za-z0-9_-]+")
+LOCAL_HOSTS = ("localhost", "127.0.0.1", "::1")
+
+RETRIES = 4
+RETRY_STATUSES = {502, 503, 504}
+
+
+class ApiError(Exception):
+    """A refusal or failure from the platform, with the code it gave."""
+
+    def __init__(self, status: int, code: str, message: str, extra: Optional[Dict[str, Any]] = None):
+        super().__init__(message)
+        self.status = status
+        self.code = code
+        self.message = message
+        self.extra = extra or {}
+
+    def __str__(self) -> str:
+        hint = ""
+        if self.extra.get("license_url"):
+            hint = f"\nRequest a license at {self.extra['license_url']}"
+        elif self.extra.get("api_keys_url"):
+            hint = f"\nManage keys at {self.extra['api_keys_url']}"
+        return f"{self.message} [{self.code}]{hint}"
+
+
+@dataclass
+class Link:
+    file_id: str
+    url: str
+    expires_at: str
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):  # type: ignore[override]
+        return None
+
+
+def api_key_from_env(api_url: Optional[str] = None) -> str:
+    key = os.environ.get(ENV_API_KEY, "").strip()
+    if not key:
+        raise ApiError(0, "missing_key", f"Set {ENV_API_KEY} to your API key.",
+                       {"api_keys_url": resolve_api_url(api_url) + "/api-keys"})
+    if not key.startswith(KEY_PREFIX):
+        raise ApiError(0, "invalid_key", f"{ENV_API_KEY} does not look like a VIZO key (expected {KEY_PREFIX}...).")
+    if not KEY_PATTERN.fullmatch(key):
+        raise ApiError(0, "invalid_key", f"{ENV_API_KEY} has characters a VIZO key never contains "
+                                         "(a space or a line break inside it?); set it to the key alone.")
+    return key
+
+
+def resolve_api_url(api_url: Optional[str] = None) -> str:
+    url = (api_url or os.environ.get(ENV_API_URL) or DEFAULT_API_URL).rstrip("/")
+    u = urllib.parse.urlparse(url)
+    if u.scheme != "https" and not (u.scheme == "http" and u.hostname in LOCAL_HOSTS):
+        raise ValueError(f"the API URL {url} must use https (plain http only for localhost), "
+                         "or your API key would travel in cleartext")
+    return url
+
+
+class PlatformClient:
+    def __init__(self, api_key: str, run_id: str, api_url: Optional[str] = None, timeout: int = 60):
+        self.api_key = api_key
+        self.run_id = run_id
+        self.api_url = resolve_api_url(api_url)
+        self.timeout = timeout
+        self.key_expires_at: Optional[str] = None
+        # Redirects are handled by hand: the manifest endpoint may answer 302 to
+        # a presigned R2 URL, which must be fetched WITHOUT our bearer header.
+        self._opener = urllib.request.build_opener(_NoRedirect)
+
+    # -- plumbing ---------------------------------------------------------
+
+    def _headers(self, extra: Optional[Dict[str, str]] = None) -> Dict[str, str]:
+        h = {
+            "Authorization": f"Bearer {self.api_key}",
+            "X-Vizo-CLI-Version": __version__,
+            "X-Vizo-Run-Id": self.run_id,
+            "User-Agent": f"vizo-cli/{__version__}",
+            "Accept": "application/json, text/csv",
+        }
+        if extra:
+            h.update(extra)
+        return h
+
+    def _request(self, method: str, path: str, body: Optional[Dict[str, Any]] = None) -> Tuple[int, Dict[str, str], bytes]:
+        data = json.dumps(body).encode() if body is not None else None
+        headers = self._headers({"Content-Type": "application/json"} if data else None)
+        return self._open(urllib.request.Request(self.api_url + path, data=data, method=method, headers=headers),
+                          self.api_url)
+
+    def _open(self, req: urllib.request.Request, where: str) -> Tuple[int, Dict[str, str], bytes]:
+        last_error: Optional[Exception] = None
+        for attempt in range(RETRIES):
+            try:
+                with self._opener.open(req, timeout=self.timeout) as resp:
+                    return resp.status, dict(resp.headers), resp.read()
+            except urllib.error.HTTPError as e:
+                payload = e.read()
+                if self._retryable(e.code, payload) and attempt < RETRIES - 1:
+                    last_error = e
+                    time.sleep(2 ** attempt)
+                    continue
+                if 300 <= e.code < 400:
+                    return e.code, dict(e.headers), payload
+                err = self._api_error(e.code, payload)
+                if e.headers.get("Retry-After"):
+                    err.extra["retry_after"] = e.headers["Retry-After"]
+                raise err
+            except (urllib.error.URLError, socket.timeout, ConnectionError) as e:
+                last_error = e
+                if attempt < RETRIES - 1:
+                    time.sleep(2 ** attempt)
+                    continue
+        raise ApiError(0, "unreachable", f"Could not reach {where}: {last_error}")
+
+    @classmethod
+    def _retryable(cls, status: int, payload: bytes) -> bool:
+        if status == 503:
+            # the platform marks a transient 503 `unavailable`; one without a JSON error comes from a proxy
+            return cls._api_error(status, payload).code in ("unavailable", f"http_{status}")
+        return status in RETRY_STATUSES
+
+    @staticmethod
+    def _api_error(status: int, payload: bytes) -> ApiError:
+        try:
+            err = json.loads(payload.decode("utf-8"))["error"]
+            code, message = err.pop("code"), err.pop("message")
+            return ApiError(status, code, message, err)
+        except (ValueError, KeyError, TypeError):
+            return ApiError(status, f"http_{status}", payload.decode("utf-8", "replace")[:300] or f"HTTP {status}")
+
+    def _note_expiry(self, headers: Dict[str, str]) -> None:
+        exp = headers.get("X-Vizo-Key-Expires-At") or headers.get("x-vizo-key-expires-at")
+        if exp:
+            self.key_expires_at = exp
+
+    # -- endpoints --------------------------------------------------------
+
+    @staticmethod
+    def _unexpected(status: int, path: str, e: Exception) -> ApiError:
+        return ApiError(status, "unexpected_response", f"The platform sent an unexpected response to {path} ({e!r}).")
+
+    def datasets(self) -> List[Dict[str, Any]]:
+        path = "/api/v1/datasets"
+        status, headers, body = self._request("GET", path)
+        self._note_expiry(headers)
+        try:
+            data = json.loads(body)
+            datasets = data["datasets"]
+            if not all(isinstance(d, dict) and "id" in d for d in datasets):
+                raise TypeError("datasets entries must be objects with an id")
+            self.key_expires_at = data.get("key_expires_at") or self.key_expires_at
+        except (ValueError, KeyError, TypeError, AttributeError) as e:
+            raise self._unexpected(status, path, e)
+        return datasets
+
+    def manifest(self, dataset: str) -> bytes:
+        """The current release's manifest as bytes (gzip or plain CSV; the reader detects which)."""
+        path = f"/api/v1/datasets/{urllib.parse.quote(dataset)}/manifest"
+        try:
+            status, headers, body = self._request("GET", path)
+        except ApiError as e:
+            if e.code == "manifest_not_published":
+                raise ApiError(e.status, e.code, f"The current {dataset} release has not been published yet; "
+                                                 "try again once it has.", e.extra)
+            raise
+        self._note_expiry(headers)
+        if status in (301, 302, 303, 307, 308):
+            location = headers.get("Location") or headers.get("location")
+            if not location:
+                raise ApiError(status, "bad_redirect", "The platform redirected without a location.")
+            # A presigned URL: no bearer header, or R2 rejects the request.
+            req = urllib.request.Request(location, headers={"User-Agent": f"vizo-cli/{__version__}"})
+            try:
+                status, _, body = self._open(req, "the manifest storage")
+            except ApiError as e:
+                # status 0: storage being unreachable or refusing says nothing about the key or license
+                reason = f"HTTP {e.status}" if e.status else e.message
+                raise ApiError(0, "manifest_fetch_failed", f"Fetching the manifest from storage failed: {reason}")
+        return body
+
+    def links(self, dataset: str, file_ids: List[str]) -> Tuple[List[Link], List[str]]:
+        path = f"/api/v1/datasets/{urllib.parse.quote(dataset)}/links"
+        status, headers, body = self._request("POST", path, {"file_ids": file_ids})
+        self._note_expiry(headers)
+        try:
+            data = json.loads(body)
+            links = [Link(l["file_id"], l["url"], l["expires_at"])
+                     for l in data.get("links", [])]
+            return links, list(data.get("unknown_file_ids", []))
+        except (ValueError, KeyError, TypeError, AttributeError) as e:
+            raise self._unexpected(status, path, e)