scale-nucleus 0.4.2__tar.gz → 0.6b3__tar.gz

scale-nucleus-0.6b3/PKG-INFO

@@ -0,0 +1,213 @@
+Metadata-Version: 2.1
+Name: scale-nucleus
+Version: 0.6b3
+Summary: The official Python client library for Nucleus, the Data Platform for AI
+Home-page: https://scale.com/nucleus
+License: MIT
+Author: Scale AI Nucleus Team
+Author-email: nucleusapi@scaleapi.com
+Requires-Python: >=3.6.2,<4.0.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Requires-Dist: aiohttp (>=3.7.4,<4.0.0)
+Requires-Dist: click (>=7.1.2,<9.0)
+Requires-Dist: dataclasses (>=0.7,<0.8); python_full_version >= "3.6.1" and python_version < "3.7"
+Requires-Dist: nest-asyncio (>=1.5.1,<2.0.0)
+Requires-Dist: numpy (>=1.19.5,<2.0.0)
+Requires-Dist: pydantic (>=1.8.2,<2.0.0)
+Requires-Dist: requests (>=2.23.0,<3.0.0)
+Requires-Dist: rich (>=10.15.2,<11.0.0)
+Requires-Dist: scikit-learn (>=0.24.0)
+Requires-Dist: scipy (>=1.4.1)
+Requires-Dist: shellingham (>=1.4.0,<2.0.0)
+Requires-Dist: tqdm (>=4.41.0,<5.0.0)
+Project-URL: Documentation, https://dashboard.scale.com/nucleus/docs/api
+Project-URL: Repository, https://github.com/scaleapi/nucleus-python-client
+Description-Content-Type: text/markdown
+
+# Nucleus
+
+https://dashboard.scale.com/nucleus
+
+Aggregate metrics in ML are not good enough. To improve production ML, you need to understand their qualitative failure modes, fix them by gathering more data, and curate diverse scenarios.
+
+Scale Nucleus helps you:
+
+- Visualize your data
+- Curate interesting slices within your dataset
+- Review and manage annotations
+- Measure and debug your model performance
+
+Nucleus is a new way—the right way—to develop ML models, helping us move away from the concept of one dataset and towards a paradigm of collections of scenarios.
+
+## Installation
+
+`$ pip install scale-nucleus`
+
+
+## CLI installation
+We recommend installing the CLI via `pipx` (https://pypa.github.io/pipx/installation/). This makes sure that
+the CLI does not interfere with you system packages and is accessible from your favorite terminal.
+
+For MacOS:
+```bash
+brew install pipx
+pipx ensurepath
+pipx install scale-nucleus
+# Optional installation of shell completion (for bash, zsh or fish)
+nu install-completions
+```
+
+Otherwise, install via pip (requires pip 19.0 or later):
+```bash
+python3 -m pip install --user pipx
+python3 -m pipx ensurepath
+python3 -m pipx install scale-nucleus
+# Optional installation of shell completion (for bash, zsh or fish)
+nu install-completions
+```
+
+## Common issues/FAQ
+
+### Outdated Client
+
+Nucleus is iterating rapidly and as a result we do not always perfectly preserve backwards compatibility with older versions of the client. If you run into any unexpected error, it's a good idea to upgrade your version of the client by running
+```
+pip install --upgrade scale-nucleus
+```
+
+## Usage
+
+For the most up to date documentation, reference: https://dashboard.scale.com/nucleus/docs/api?language=python.
+
+## For Developers
+
+Clone from github and install as editable
+
+```
+git clone git@github.com:scaleapi/nucleus-python-client.git
+cd nucleus-python-client
+pip3 install poetry
+poetry install
+```
+
+Please install the pre-commit hooks by running the following command:
+
+```python
+poetry run pre-commit install
+```
+
+When releasing a new version please add release notes to the changelog in `CHANGELOG.md`.
+
+**Best practices for testing:**
+(1). Please run pytest from the root directory of the repo, i.e.
+
+```
+poetry run pytest tests/test_dataset.py
+```
+
+(2) To skip slow integration tests that have to wait for an async job to start.
+
+```
+poetry run pytest -m "not integration"
+```
+
+## Pydantic Models
+
+Prefer using [Pydantic](https://pydantic-docs.helpmanual.io/usage/models/) models rather than creating raw dictionaries 
+or dataclasses to send or receive over the wire as JSONs. Pydantic is created with data validation in mind and provides very clear error 
+messages when it encounters a problem with the payload.
+
+The Pydantic model(s) should mirror the payload to send. To represent a JSON payload that looks like this:
+```json
+{
+  "example_json_with_info": {
+      "metadata": {
+        "frame": 0
+      },
+      "reference_id": "frame0",
+      "url": "s3://example/scale_nucleus/2021/lidar/0038711321865000.json",
+      "type": "pointcloud"
+    },
+  "example_image_with_info": {
+      "metadata": {
+        "author": "Picasso"
+      },
+      "reference_id": "frame0",
+      "url": "s3://bucket/0038711321865000.jpg",
+      "type": "image"
+    },
+}
+```
+
+Could be represented as the following structure. Note that the field names map to the JSON keys and the usage of field 
+validators (`@validator`).
+
+```python
+import os.path
+from pydantic import BaseModel, validator
+from typing import Literal
+
+
+class JsonWithInfo(BaseModel):
+    metadata: dict  # any dict is valid
+    reference_id: str
+    url: str
+    type: Literal["pointcloud", "recipe"]
+
+    @validator("url")
+    def has_json_extension(cls, v):
+        if not v.endswith(".json"):
+            raise ValueError(f"Expected '.json' extension got {v}")
+        return v
+
+
+class ImageWithInfo(BaseModel):
+    metadata: dict  # any dict is valid
+    reference_id: str
+    url: str
+    type: Literal["image", "mask"]
+
+    @validator("url")
+    def has_valid_extension(cls, v):
+        valid_extensions = {".jpg", ".jpeg", ".png", ".tiff"}
+        _, extension = os.path.splitext(v)
+        if extension not in valid_extensions:
+            raise ValueError(f"Expected extension in {valid_extensions} got {v}")
+        return v
+
+
+class ExampleNestedModel(BaseModel):
+    example_json_with_info: JsonWithInfo
+    example_image_with_info: ImageWithInfo
+
+# Usage:
+import requests
+payload = requests.get("/example")
+parsed_model = ExampleNestedModel.parse_obj(payload.json())
+requests.post("example/post_to", json=parsed_model.dict())
+```
+
+
+### Migrating to Pydantic
+- When migrating an interface from a dictionary use `nucleus.pydantic_base.DictCompatibleModel`. That allows you to get
+the benefits of Pydantic but maintaints backwards compatibility with a Python dictionary by delegating `__getitem__` to 
+fields.
+- When migrating a frozen dataclass use `nucleus.pydantic_base.ImmutableModel`. That is a base class set up to be 
+immutable after initialization.
+
+**Updating documentation:**
+We use [Sphinx](https://www.sphinx-doc.org/en/master/) to autogenerate our API Reference from docstrings.
+
+To test your local docstring changes, run the following commands from the repository's root directory:
+```
+poetry shell
+cd docs
+sphinx-autobuild . ./_build/html --watch ../nucleus
+```
+`sphinx-autobuild` will spin up a server on localhost (port 8000 by default) that will watch for and automatically rebuild a version of the API reference based on your local docstring changes.
+

scale-nucleus-0.6b3/README.md

@@ -0,0 +1,181 @@
+# Nucleus
+
+https://dashboard.scale.com/nucleus
+
+Aggregate metrics in ML are not good enough. To improve production ML, you need to understand their qualitative failure modes, fix them by gathering more data, and curate diverse scenarios.
+
+Scale Nucleus helps you:
+
+- Visualize your data
+- Curate interesting slices within your dataset
+- Review and manage annotations
+- Measure and debug your model performance
+
+Nucleus is a new way—the right way—to develop ML models, helping us move away from the concept of one dataset and towards a paradigm of collections of scenarios.
+
+## Installation
+
+`$ pip install scale-nucleus`
+
+
+## CLI installation
+We recommend installing the CLI via `pipx` (https://pypa.github.io/pipx/installation/). This makes sure that
+the CLI does not interfere with you system packages and is accessible from your favorite terminal.
+
+For MacOS:
+```bash
+brew install pipx
+pipx ensurepath
+pipx install scale-nucleus
+# Optional installation of shell completion (for bash, zsh or fish)
+nu install-completions
+```
+
+Otherwise, install via pip (requires pip 19.0 or later):
+```bash
+python3 -m pip install --user pipx
+python3 -m pipx ensurepath
+python3 -m pipx install scale-nucleus
+# Optional installation of shell completion (for bash, zsh or fish)
+nu install-completions
+```
+
+## Common issues/FAQ
+
+### Outdated Client
+
+Nucleus is iterating rapidly and as a result we do not always perfectly preserve backwards compatibility with older versions of the client. If you run into any unexpected error, it's a good idea to upgrade your version of the client by running
+```
+pip install --upgrade scale-nucleus
+```
+
+## Usage
+
+For the most up to date documentation, reference: https://dashboard.scale.com/nucleus/docs/api?language=python.
+
+## For Developers
+
+Clone from github and install as editable
+
+```
+git clone git@github.com:scaleapi/nucleus-python-client.git
+cd nucleus-python-client
+pip3 install poetry
+poetry install
+```
+
+Please install the pre-commit hooks by running the following command:
+
+```python
+poetry run pre-commit install
+```
+
+When releasing a new version please add release notes to the changelog in `CHANGELOG.md`.
+
+**Best practices for testing:**
+(1). Please run pytest from the root directory of the repo, i.e.
+
+```
+poetry run pytest tests/test_dataset.py
+```
+
+(2) To skip slow integration tests that have to wait for an async job to start.
+
+```
+poetry run pytest -m "not integration"
+```
+
+## Pydantic Models
+
+Prefer using [Pydantic](https://pydantic-docs.helpmanual.io/usage/models/) models rather than creating raw dictionaries 
+or dataclasses to send or receive over the wire as JSONs. Pydantic is created with data validation in mind and provides very clear error 
+messages when it encounters a problem with the payload.
+
+The Pydantic model(s) should mirror the payload to send. To represent a JSON payload that looks like this:
+```json
+{
+  "example_json_with_info": {
+      "metadata": {
+        "frame": 0
+      },
+      "reference_id": "frame0",
+      "url": "s3://example/scale_nucleus/2021/lidar/0038711321865000.json",
+      "type": "pointcloud"
+    },
+  "example_image_with_info": {
+      "metadata": {
+        "author": "Picasso"
+      },
+      "reference_id": "frame0",
+      "url": "s3://bucket/0038711321865000.jpg",
+      "type": "image"
+    },
+}
+```
+
+Could be represented as the following structure. Note that the field names map to the JSON keys and the usage of field 
+validators (`@validator`).
+
+```python
+import os.path
+from pydantic import BaseModel, validator
+from typing import Literal
+
+
+class JsonWithInfo(BaseModel):
+    metadata: dict  # any dict is valid
+    reference_id: str
+    url: str
+    type: Literal["pointcloud", "recipe"]
+
+    @validator("url")
+    def has_json_extension(cls, v):
+        if not v.endswith(".json"):
+            raise ValueError(f"Expected '.json' extension got {v}")
+        return v
+
+
+class ImageWithInfo(BaseModel):
+    metadata: dict  # any dict is valid
+    reference_id: str
+    url: str
+    type: Literal["image", "mask"]
+
+    @validator("url")
+    def has_valid_extension(cls, v):
+        valid_extensions = {".jpg", ".jpeg", ".png", ".tiff"}
+        _, extension = os.path.splitext(v)
+        if extension not in valid_extensions:
+            raise ValueError(f"Expected extension in {valid_extensions} got {v}")
+        return v
+
+
+class ExampleNestedModel(BaseModel):
+    example_json_with_info: JsonWithInfo
+    example_image_with_info: ImageWithInfo
+
+# Usage:
+import requests
+payload = requests.get("/example")
+parsed_model = ExampleNestedModel.parse_obj(payload.json())
+requests.post("example/post_to", json=parsed_model.dict())
+```
+
+
+### Migrating to Pydantic
+- When migrating an interface from a dictionary use `nucleus.pydantic_base.DictCompatibleModel`. That allows you to get
+the benefits of Pydantic but maintaints backwards compatibility with a Python dictionary by delegating `__getitem__` to 
+fields.
+- When migrating a frozen dataclass use `nucleus.pydantic_base.ImmutableModel`. That is a base class set up to be 
+immutable after initialization.
+
+**Updating documentation:**
+We use [Sphinx](https://www.sphinx-doc.org/en/master/) to autogenerate our API Reference from docstrings.
+
+To test your local docstring changes, run the following commands from the repository's root directory:
+```
+poetry shell
+cd docs
+sphinx-autobuild . ./_build/html --watch ../nucleus
+```
+`sphinx-autobuild` will spin up a server on localhost (port 8000 by default) that will watch for and automatically rebuild a version of the API reference based on your local docstring changes.

scale-nucleus-0.6b3/cli/client.py

@@ -0,0 +1,14 @@
+import functools
+import os
+
+import nucleus
+
+
+@functools.lru_cache()
+def init_client():
+    api_key = os.environ.get("NUCLEUS_API_KEY", None)
+    if api_key:
+        client = nucleus.NucleusClient(api_key)
+    else:
+        raise RuntimeError("No NUCLEUS_API_KEY set")
+    return client

scale-nucleus-0.6b3/cli/datasets.py

@@ -0,0 +1,77 @@
+import click
+from rich.console import Console
+from rich.table import Column, Table
+
+from cli.client import init_client
+from cli.helpers.nucleus_url import nucleus_url
+from cli.helpers.web_helper import launch_web_or_invoke
+
+
+@click.group("datasets", invoke_without_command=True)
+@click.option("--web", is_flag=True, help="Launch browser")
+@click.pass_context
+def datasets(ctx, web):
+    """Datasets are the base collections of items in Nucleus
+
+    https://dashboard.scale.com/nucleus/datasets
+    """
+    launch_web_or_invoke(
+        sub_url="datasets", ctx=ctx, launch_browser=web, command=list_datasets
+    )
+
+
+@datasets.command("list")
+@click.option(
+    "-m", "--machine-readable", is_flag=True, help="Removes pretty printing"
+)
+def list_datasets(machine_readable):
+    """List all available Datasets"""
+    console = Console()
+    with console.status("Finding your Datasets!", spinner="dots4"):
+        client = init_client()
+        all_datasets = client.datasets
+        if machine_readable:
+            table_params = {"box": None, "pad_edge": False}
+        else:
+            table_params = {
+                "title": ":fire: Datasets",
+                "title_justify": "left",
+            }
+
+        table = Table(
+            "id", "Name", Column("url", overflow="fold"), **table_params
+        )
+        for ds in all_datasets:
+            table.add_row(ds.id, ds.name, nucleus_url(ds.id))
+    console.print(table)
+
+
+@datasets.command("delete")
+@click.option("--id", prompt=True)
+@click.option(
+    "--no-confirm-deletion",
+    is_flag=True,
+    help="WARNING: No confirmation for deletion",
+)
+@click.pass_context
+def delete_dataset(ctx, id, no_confirm_deletion):
+    """Delete a Dataset"""
+    console = Console()
+    client = init_client()
+    id = id.strip()
+    dataset = client.get_dataset(id)
+    delete_string = ""
+    if not no_confirm_deletion:
+        delete_string = click.prompt(
+            click.style(
+                f"Type 'DELETE' to delete dataset: {dataset}", fg="red"
+            )
+        )
+    if no_confirm_deletion or delete_string == "DELETE":
+        client.delete_dataset(dataset.id)
+        console.print(f":fire: :anguished: Deleted {id}")
+    else:
+        console.print(
+            f":rotating_light: Refusing to delete {id}. Received '{delete_string}' instead of 'DELETE'"
+        )
+        ctx.abort()

scale-nucleus-0.6b3/cli/helpers/nucleus_url.py

@@ -0,0 +1,10 @@
+import os
+from urllib.parse import urljoin
+
+
+def nucleus_url(sub_path: str):
+    nucleus_base = os.environ.get(
+        "NUCLEUS_DASHBOARD", "https://dashboard.scale.com/nucleus/"
+    )
+    extra_params = os.environ.get("NUCLEUS_DASH_PARAMS", "")
+    return urljoin(nucleus_base, sub_path.lstrip("/") + extra_params)

scale-nucleus-0.6b3/cli/helpers/web_helper.py

@@ -0,0 +1,40 @@
+import click
+
+from cli.helpers.nucleus_url import nucleus_url
+
+
+def launch_web_or_show_help(
+    sub_url: str, ctx: click.Context, launch_browser: bool
+):
+    """ Launches the sub_url (composed with nuclues_url(sub_url)) in the browser if requested"""
+    if not ctx.invoked_subcommand:
+        if launch_browser:
+            url = nucleus_url(sub_url)
+            click.launch(url)
+        else:
+            click.echo(ctx.get_help())
+    else:
+        if launch_browser:
+            click.echo(click.style("--web does not work with sub-commands"))
+            ctx.abort()
+
+
+def launch_web_or_invoke(
+    sub_url: str,
+    ctx: click.Context,
+    launch_browser: bool,
+    command: click.BaseCommand,
+):
+    """Launches the sub_url (composed with nuclues_url(sub_url)) in the browser if requested, otherwise invokes
+    the passed command
+    """
+    if not ctx.invoked_subcommand:
+        if launch_browser:
+            url = nucleus_url(sub_url)
+            click.launch(url)
+        else:
+            ctx.invoke(command)
+    else:
+        if launch_browser:
+            click.echo(click.style("--web does not work with sub-commands"))
+            ctx.abort()

scale-nucleus-0.6b3/cli/install_completion.py

@@ -0,0 +1,33 @@
+import os
+import shutil
+
+import click
+from shellingham import detect_shell
+
+
+@click.command("install-completion")
+def install_completion():
+    """Install shell completion script to your rc file"""
+    shell, _ = detect_shell()
+    if shell == "zsh":
+        rc_path = "~/.zshrc"
+        append_to_file = 'eval "$(_NU_COMPLETE=zsh_source nu)"'
+    elif shell == "bash":
+        rc_path = "~/.bashrc"
+        append_to_file = 'eval "$(_NU_COMPLETE=bash_source nu)"'
+    elif shell == "fish":
+        rc_path = "~/.config/fish/completions/foo-bar.fish"
+        append_to_file = "eval (env _NU_COMPLETE=fish_source nu)"
+    else:
+        raise RuntimeError(f"Unsupported shell {shell} for completions")
+
+    rc_path_expanded = os.path.expanduser(rc_path)
+    rc_bak = f"{rc_path_expanded}.bak"
+    shutil.copy(rc_path_expanded, rc_bak)
+    click.echo(f"Backed up {rc_path} to {rc_bak}")
+    with open(rc_path_expanded, mode="a") as rc_file:
+        rc_file.write("\n")
+        rc_file.write("# Shell completion for nu\n")
+        rc_file.write(append_to_file)
+    click.echo(f"Completion script added to {rc_path}")
+    click.echo(f"Don't forget to `source {rc_path}")

scale-nucleus-0.6b3/cli/jobs.py

@@ -0,0 +1,42 @@
+import click
+from rich.live import Live
+from rich.spinner import Spinner
+from rich.table import Column, Table
+
+from cli.client import init_client
+from cli.helpers.web_helper import launch_web_or_invoke
+
+
+@click.group("jobs", invoke_without_command=True)
+@click.option("--web", is_flag=True, help="Launch browser")
+@click.pass_context
+def jobs(ctx, web):
+    """Jobs are a wrapper around various long-running tasks withing Nucleus
+
+    https://dashboard.scale.com/nucleus/jobs
+    """
+    launch_web_or_invoke("jobs", ctx, web, list_jobs)
+
+
+@jobs.command("list")
+def list_jobs():
+    """List all of your Jobs"""
+    client = init_client()
+    table = Table(
+        Column("id", overflow="fold", min_width=24),
+        "status",
+        "type",
+        "created at",
+        title=":satellite: Jobs",
+        title_justify="left",
+    )
+    with Live(Spinner("dots4", text="Finding your Jobs!")) as live:
+        all_jobs = client.jobs
+        for job in all_jobs:
+            table.add_row(
+                job.job_id,
+                job.job_last_known_status,
+                job.job_type,
+                job.job_creation_time,
+            )
+            live.update(table)

scale-nucleus-0.6b3/cli/models.py

@@ -0,0 +1,35 @@
+import click
+from rich.console import Console
+from rich.table import Column, Table
+
+from cli.client import init_client
+from cli.helpers.nucleus_url import nucleus_url
+from cli.helpers.web_helper import launch_web_or_invoke
+
+
+@click.group("models", invoke_without_command=True)
+@click.option("--web", is_flag=True, help="Launch browser")
+@click.pass_context
+def models(ctx, web):
+    """Models help you store and access your ML model data
+
+    https://dashboard.scale.com/nucleus/models
+    """
+    launch_web_or_invoke("models", ctx, web, list_models)
+
+
+@models.command("list")
+def list_models():
+    """List your Models"""
+    console = Console()
+    with console.status("Finding your Models!", spinner="dots4"):
+        client = init_client()
+        table = Table(
+            Column("id", overflow="fold", min_width=24),
+            "name",
+            Column("url", overflow="fold"),
+        )
+        models = client.models
+        for m in models:
+            table.add_row(m.id, m.name, nucleus_url(m.id))
+    console.print(table)