nomic 3.5.2__tar.gz → 3.6.0__tar.gz

{nomic-3.5.2 → nomic-3.6.0}/PKG-INFO

@@ -1,19 +1,51 @@
 Metadata-Version: 2.1
 Name: nomic
-Version: 3.5.2
+Version: 3.6.0
 Summary: The official Nomic python client.
 Home-page: https://github.com/nomic-ai/nomic
 Author: nomic.ai
 Author-email: support@nomic.ai
-License: UNKNOWN
-Platform: UNKNOWN
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Description-Content-Type: text/markdown
+Requires-Dist: click
+Requires-Dist: jsonlines
+Requires-Dist: jsonschema<5,>=4.23.0
+Requires-Dist: loguru
+Requires-Dist: rich
+Requires-Dist: requests
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: pydantic<3,>=2
+Requires-Dist: tqdm
+Requires-Dist: pyarrow
+Requires-Dist: pillow
+Requires-Dist: pyjwt
 Provides-Extra: local
+Requires-Dist: gpt4all<3,>=2.5.0; extra == "local"
 Provides-Extra: aws
+Requires-Dist: boto3; extra == "aws"
+Requires-Dist: sagemaker; extra == "aws"
 Provides-Extra: all
+Requires-Dist: nomic[aws,local]; extra == "all"
 Provides-Extra: dev
+Requires-Dist: nomic[all]; extra == "dev"
+Requires-Dist: black==24.3.0; extra == "dev"
+Requires-Dist: coverage; extra == "dev"
+Requires-Dist: pylint; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: pyright<=1.1.377; extra == "dev"
+Requires-Dist: myst-parser; extra == "dev"
+Requires-Dist: mkdocs-material; extra == "dev"
+Requires-Dist: mkautodoc; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: mkdocstrings[python]; extra == "dev"
+Requires-Dist: mkdocs-jupyter; extra == "dev"
+Requires-Dist: pillow; extra == "dev"
+Requires-Dist: cairosvg; extra == "dev"
+Requires-Dist: pytorch-lightning; extra == "dev"
+Requires-Dist: pandas; extra == "dev"
 
 <h1 align="center">Nomic Atlas Python Client</h1>
 <h3 align="center">Explore, label, search and share massive datasets in your web browser.</h3>
@@ -243,5 +275,3 @@ Join the discussion on our [:hut: Discord](https://discord.gg/myY5YDR8z8) to ask
 <br>
 
 [Go to top](#)
-
-

{nomic-3.5.2 → nomic-3.6.0}/nomic/__init__.py

@@ -1,2 +1,3 @@
 from .cli import login
+from .client import NomicClient as NomicClient
 from .dataset import AtlasDataset, AtlasUser

{nomic-3.5.2 → nomic-3.6.0}/nomic/cli.py

@@ -53,7 +53,7 @@ def login(token, tenant="production", domain=None):
         console.print("Authenticate with the Nomic API", style=style, justify="center")
         console.print(auth0_auth_endpoint, style=style, justify="center")
         console.print(
-            "Click the above link to retrieve your access token and then run `nomic login [token]`",
+            "Click the above link to retrieve your access token and then run `nomic login \\[token]`",
             style=style,
             justify="center",
         )

nomic-3.6.0/nomic/client.py

@@ -0,0 +1,388 @@
+import json
+import os
+import time
+from collections.abc import Sequence
+from dataclasses import dataclass
+from enum import Enum, auto
+from pathlib import Path
+from typing import Any, Generic, Literal, TypeVar, overload
+from urllib.parse import urlparse
+
+import jsonschema
+import requests
+
+from nomic.dataset import AtlasClass
+
+__all__ = [
+    "NomicClient",
+    "PlatformTask",
+    "TaskFailed",
+    "TaskPending",
+    "UploadedFile",
+]
+
+MAX_FAILRESP_LENGTH = 1_000  # chars
+
+T = TypeVar("T")
+
+client: "AtlasClass | None" = None
+
+
+def get_client():
+    global client
+    if client is None:
+        client = AtlasClass()
+    return client
+
+
+@dataclass(frozen=True)
+class UploadedFile:
+    url: str
+
+
+class Sentinel(Enum):
+    Nothing = auto()
+
+
+class TaskPending(Exception):
+    pass
+
+
+class TaskFailed(Exception):
+    pass
+
+
+class PlatformTask(Generic[T]):
+    """
+    An object representing a task on the Nomic Platform.
+
+    Attributes:
+        id: The ID of the task.
+    """
+
+    _id: str
+    _result: "T | Sentinel"
+
+    def __init__(self, id: str):
+        self._id = id
+        self._result = Sentinel.Nothing
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    def get(self, timeout: "float | None" = None, *, block: bool = True) -> T:
+        """
+        Waits for the task to complete and returns the result.
+
+        Args:
+            timeout: The maximum time to wait for the task to complete.
+            block: Whether to block until the task is complete.
+
+        Returns:
+            The result of the task.
+
+        Raises:
+            TaskPending: If the task is not complete and block is True.
+            TaskFailed: If the task fails.
+        """
+        if self._result is not Sentinel.Nothing:
+            return self._result
+        client = get_client()
+        start_time = time.time()
+        while True:
+            response = client._get(f"/v1/status/{self._id}")
+            raise_for_status_with_body(response)
+            status_resp = response.json()
+            if status_resp["status"] == "COMPLETED":
+                break
+            if status_resp["status"] == "FAILED":
+                raise TaskFailed(status_resp["error"])
+            if not block:
+                raise TaskPending
+            sleeptime = 1  # poll interval
+            if timeout is not None:
+                end_time = start_time + timeout
+                if end_time < (now := time.time()):
+                    raise TaskPending
+                sleeptime = min(sleeptime, end_time - now)
+            time.sleep(sleeptime)
+
+        completed_response = requests.get(status_resp["result_url"])
+        raise_for_status_with_body(completed_response)
+
+        result = status_resp.pop("result", {})
+        result.pop("result_url", None)
+        result.pop("result", None)
+        result["result"] = completed_response.json()
+        result["result"].pop("status", None)
+        result["result"].pop("error", None)
+        self._result = result
+        return result
+
+
+class NomicClient:
+    """Client for the Nomic Platform API."""
+
+    def upload_file(self, path: "str | os.PathLike[str]") -> UploadedFile:
+        """
+        Uploads a file to the Nomic Platform.
+
+        Args:
+            path: The path to the PDF file to upload.
+
+        Returns:
+            An UploadedFile object representing the uploaded file.
+        """
+        client = get_client()
+
+        path = Path(path)
+
+        with path.open("rb") as pdf_file:
+            file_type = path.suffix.lower()
+            if file_type == ".pdf":
+                content_type = "application/pdf"
+            # elif file_type == ".docx":
+            #   content_type = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+            # elif file_type == ".doc":
+            #   content_type = "application/msword"
+            # elif file_type == ".txt":
+            #   content_type = "text/plain"
+            # elif file_type == ".pptx":
+            #   content_type = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+            # elif file_type == ".ppt":
+            #   content_type = "application/vnd.ms-powerpoint"
+            # elif file_type == ".csv":
+            #   content_type = "text/csv"
+            # elif file_type == ".xlsx":
+            #   content_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+            # elif file_type == ".xls":
+            #   content_type = "application/vnd.ms-excel"
+            else:
+                raise ValueError(f"Unsupported file type: {file_type}")
+
+            response = client._post(
+                "/v1/upload",
+                json=dict(files=[{"id": path.name, "size": path.stat().st_size, "content_type": content_type}]),
+            )
+            raise_for_status_with_body(response)
+
+            values = response.json()
+
+            # Extract from the files array
+            file_info = values["files"][0]
+            upload_url = file_info["upload_url"]
+            nomic_url = file_info["nomic_url"]
+
+            # upload the file to the designated pre-signed url
+            resp = requests.put(upload_url, data=pdf_file, headers={"x-amz-server-side-encryption": "AES256"})
+
+        raise_for_status_with_body(resp)
+        return UploadedFile(url=nomic_url)
+
+    @overload
+    def parse(self, file: "str | UploadedFile", *, block: Literal[True] = ...) -> "dict[str, Any]": ...
+    @overload
+    def parse(self, file: "str | UploadedFile", *, block: Literal[False]) -> PlatformTask["dict[str, Any]"]: ...
+    @overload
+    def parse(self, file: "str | UploadedFile", *, block: bool) -> Any: ...
+
+    def parse(self, file: "str | UploadedFile", *, block: bool = True) -> Any:
+        """
+        Parses a document into a structured JSON representation.
+
+        Args:
+            file: The file to parse. Can be a string URL or an UploadedFile object.
+            block: Whether to block until the task is complete.
+
+        Returns:
+            By default, returns the parsed document. If block is False, returns a PlatformTask that can be used to get
+            the result.
+
+        Raises:
+            ValueError: If an invalid URL is passed.
+            TaskFailed: If block is True and the task fails.
+
+        Example:
+            Complete end-to-end workflow with upload and parsing:
+
+            ```python
+            from nomic.documents import upload_file, parse
+
+            # Upload a PDF file
+            file = upload_file("my_document.pdf")
+
+            # Parse the document
+            result = parse(file)
+            print(result)
+            ```
+        """
+        client = get_client()
+
+        response = client._post("/v1/parse", json={"file_url": self._file_to_url(file)})
+        raise_for_status_with_body(response)
+        task = PlatformTask(response.json()["task_id"])
+        if block:
+            return task.get()
+        return task
+
+    @overload
+    def extract(
+        self,
+        files: "str | UploadedFile | Sequence[str | UploadedFile]",
+        schema: "dict[str, Any]",
+        *,
+        block: Literal[True] = ...,
+    ) -> Any: ...
+    @overload
+    def extract(
+        self,
+        files: "str | UploadedFile | Sequence[str | UploadedFile]",
+        schema: "dict[str, Any]",
+        *,
+        block: Literal[False],
+    ) -> PlatformTask[Any]: ...
+    @overload
+    def extract(
+        self,
+        files: "str | UploadedFile | Sequence[str | UploadedFile]",
+        schema: "dict[str, Any]",
+        *,
+        block: bool,
+    ) -> Any: ...
+
+    def extract(
+        self,
+        files: "str | UploadedFile | Sequence[str | UploadedFile]",
+        schema: "dict[str, Any]",
+        *,
+        block: bool = True,
+    ) -> Any:
+        """
+        Extracts structured data from documents.
+
+        Args:
+            files: List of uploaded files to extract from.
+            schema: A JSON schema defining the structure of data to extract.
+            block: Whether to block until the task is complete.
+
+        Returns:
+            By default, returns the extracted data matching the provided schema. If block is False, returns a PlatformTask
+            that can be used to get the result.
+
+        Raises:
+            ValueError: If an invalid URL is passed.
+            TaskFailed: If block is True and the task fails.
+
+        Example:
+            Complete end-to-end workflow with upload and extraction:
+
+            ```python
+            from nomic.documents import upload_file, extract
+
+            # Upload a PDF file
+            file = upload_file("my_document.pdf")
+
+            # Define extraction schema
+            schema = {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "speaker": {"type": "string"},
+                        "content": {"type": "string"},
+                    }
+                },
+            }
+
+            # Extract structured data
+            result = extract(file, schema)
+            print(result)
+            ```
+        """
+        jsonschema.Draft7Validator.check_schema(schema)
+
+        if isinstance(files, (str, UploadedFile)):
+            files = [files]
+
+        client = get_client()
+
+        response = client._post(
+            "/v1/extract", json={"file_urls": list(map(self._file_to_url, files)), "extraction_schema": schema}
+        )
+        raise_for_status_with_body(response)
+        task = PlatformTask(response.json()["task_id"])
+        if block:
+            return task.get()
+        return task
+
+    @staticmethod
+    def _file_to_url(file: "str | UploadedFile") -> str:
+        if isinstance(file, UploadedFile):
+            return file.url
+        parsed = urlparse(file)
+        if parsed.scheme in ("nomic", "http", "https"):
+            return file
+        if parsed.scheme == "file" or (not parsed.scheme and Path(file).exists()):
+            raise ValueError(
+                f"Cannot directly pass local file to platform: {file!r}\nPlease use upload_file() to upload it first."
+            )
+        if not parsed.scheme:
+            raise ValueError(f"Invalid URL: {file!r}")
+        raise ValueError(f"Unsupported scheme {parsed.scheme!r} for URL {file!r}")
+
+
+def raise_for_status_with_body(resp: requests.Response) -> None:
+    """
+    Raises HTTPError if the response is not successful.
+
+    Like Response.raise_for_status, but includes the (truncated) response body in the
+    exception message for improved diagnostics.
+
+    Args:
+        resp: The response to check
+
+    Raises:
+        requests.HTTPError: If the response is not successful
+    """
+    http_error_msg = ""
+    if isinstance(resp.reason, bytes):
+        # We attempt to decode utf-8 first because some servers
+        # choose to localize their reason strings. If the string
+        # isn't utf-8, we fall back to iso-8859-1 for all other
+        # encodings. (See PR #3538)
+        try:
+            reason = resp.reason.decode("utf-8")
+        except UnicodeDecodeError:
+            reason = resp.reason.decode("iso-8859-1")
+    else:
+        reason = resp.reason
+
+    if 400 <= resp.status_code < 500:
+        http_error_msg = f"{resp.status_code} Client Error: {reason} for url: {resp.url}"
+
+    elif 500 <= resp.status_code < 600:
+        http_error_msg = f"{resp.status_code} Server Error: {reason} for url: {resp.url}"
+
+    if http_error_msg:
+        if (ctype := resp.headers.get("content-type")) is not None:
+            http_error_msg += f"\nContent-Type: {ctype}"
+        http_error_msg += f"\nBody: {format_body(resp)}"
+        raise requests.HTTPError(http_error_msg, response=resp)
+
+
+def format_body(resp: requests.Response) -> str:
+    text = None
+    if (ctype := resp.headers.get("content-type")) and "application/json" in ctype.lower():
+        try:
+            data = resp.json()
+            text = json.dumps(data, indent=2, ensure_ascii=False)
+        except Exception:
+            pass
+    if text is None:
+        text = resp.text
+
+    limit = MAX_FAILRESP_LENGTH
+    if len(text) > limit:
+        return text[:limit] + f"\n… [truncated {len(text) - limit} chars]"
+    return text

{nomic-3.5.2 → nomic-3.6.0}/nomic/dataset.py

@@ -8,11 +8,12 @@ import os
 import re
 import time
 import unicodedata
+from collections.abc import Iterable, Mapping
 from contextlib import contextmanager
 from datetime import datetime
 from io import BytesIO
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol, Tuple, TypeVar, Union
 
 import numpy as np
 import pandas as pd
@@ -37,6 +38,20 @@ from .data_operations import AtlasMapData, AtlasMapDuplicates, AtlasMapEmbedding
 from .settings import *
 from .utils import assert_valid_project_id, download_feather
 
+if TYPE_CHECKING:
+    from typing_extensions import TypeAlias
+
+T_co = TypeVar("T_co", covariant=True)
+
+
+class SupportsRead(Protocol[T_co]):
+    def read(self, length: int = ..., /) -> T_co: ...
+
+
+_Data: "TypeAlias" = (
+    "Iterable[bytes] | str | bytes | SupportsRead[str | bytes] | list[tuple[Any, Any]] | tuple[tuple[Any, Any], ...] | Mapping[Any, Any]"
+)
+
 
 class AtlasUser:
     def __init__(self):
@@ -410,6 +425,30 @@ class AtlasClass(object):
 
         return organization_slug, organization_id
 
+    def _post(self, endpoint: str, *, json: "Any | None" = None) -> requests.Response:
+        response = requests.post(
+            self.atlas_api_path + endpoint,
+            headers=self.header,
+            json=json,
+        )
+        return response
+
+    def _get(self, endpoint: str) -> requests.Response:
+        response = requests.get(
+            self.atlas_api_path + endpoint,
+            headers=self.header,
+        )
+        return response
+
+    def _put(self, endpoint: str, *, data: "_Data | None" = None, json: "Any | None" = None) -> requests.Response:
+        response = requests.put(
+            self.atlas_api_path + endpoint,
+            headers=self.header,
+            data=data,
+            json=json,
+        )
+        return response
+
 
 class AtlasIndex:
     """

{nomic-3.5.2 → nomic-3.6.0}/nomic/embed.py

@@ -26,7 +26,7 @@ except ImportError:
 
 atlas_class: Optional[AtlasClass] = None
 
-MAX_TEXT_REQUEST_SIZE = 50
+MAX_TEXT_REQUEST_SIZE = 10
 MAX_IMAGE_REQUEST_SIZE = 512
 MIN_EMBEDDING_DIMENSIONALITY = 64
 

{nomic-3.5.2 → nomic-3.6.0}/nomic.egg-info/PKG-INFO

@@ -1,19 +1,51 @@
 Metadata-Version: 2.1
 Name: nomic
-Version: 3.5.2
+Version: 3.6.0
 Summary: The official Nomic python client.
 Home-page: https://github.com/nomic-ai/nomic
 Author: nomic.ai
 Author-email: support@nomic.ai
-License: UNKNOWN
-Platform: UNKNOWN
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Description-Content-Type: text/markdown
+Requires-Dist: click
+Requires-Dist: jsonlines
+Requires-Dist: jsonschema<5,>=4.23.0
+Requires-Dist: loguru
+Requires-Dist: rich
+Requires-Dist: requests
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: pydantic<3,>=2
+Requires-Dist: tqdm
+Requires-Dist: pyarrow
+Requires-Dist: pillow
+Requires-Dist: pyjwt
 Provides-Extra: local
+Requires-Dist: gpt4all<3,>=2.5.0; extra == "local"
 Provides-Extra: aws
+Requires-Dist: boto3; extra == "aws"
+Requires-Dist: sagemaker; extra == "aws"
 Provides-Extra: all
+Requires-Dist: nomic[aws,local]; extra == "all"
 Provides-Extra: dev
+Requires-Dist: nomic[all]; extra == "dev"
+Requires-Dist: black==24.3.0; extra == "dev"
+Requires-Dist: coverage; extra == "dev"
+Requires-Dist: pylint; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: pyright<=1.1.377; extra == "dev"
+Requires-Dist: myst-parser; extra == "dev"
+Requires-Dist: mkdocs-material; extra == "dev"
+Requires-Dist: mkautodoc; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: mkdocstrings[python]; extra == "dev"
+Requires-Dist: mkdocs-jupyter; extra == "dev"
+Requires-Dist: pillow; extra == "dev"
+Requires-Dist: cairosvg; extra == "dev"
+Requires-Dist: pytorch-lightning; extra == "dev"
+Requires-Dist: pandas; extra == "dev"
 
 <h1 align="center">Nomic Atlas Python Client</h1>
 <h3 align="center">Explore, label, search and share massive datasets in your web browser.</h3>
@@ -243,5 +275,3 @@ Join the discussion on our [:hut: Discord](https://discord.gg/myY5YDR8z8) to ask
 <br>
 
 [Go to top](#)
-
-

{nomic-3.5.2 → nomic-3.6.0}/nomic.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ setup.py
 nomic/__init__.py
 nomic/atlas.py
 nomic/cli.py
+nomic/client.py
 nomic/data_inference.py
 nomic/data_operations.py
 nomic/dataset.py
@@ -19,4 +20,8 @@ nomic.egg-info/top_level.txt
 nomic/aws/__init__.py
 nomic/aws/sagemaker.py
 nomic/pl_callbacks/__init__.py
-nomic/pl_callbacks/pl_callback.py
+nomic/pl_callbacks/pl_callback.py
+tests/__init__.py
+tests/test_atlas_client.py
+tests/test_documents.py
+tests/test_embed.py

{nomic-3.5.2 → nomic-3.6.0}/nomic.egg-info/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 nomic = nomic.cli:cli
-

{nomic-3.5.2 → nomic-3.6.0}/nomic.egg-info/requires.txt

@@ -1,11 +1,12 @@
 click
 jsonlines
+jsonschema<5,>=4.23.0
 loguru
 rich
 requests
 numpy
 pandas
-pydantic
+pydantic<3,>=2
 tqdm
 pyarrow
 pillow

{nomic-3.5.2 → nomic-3.6.0}/setup.py

@@ -23,7 +23,7 @@ with open("README.md") as f:
 
 setup(
     name="nomic",
-    version="3.5.2",
+    version="3.6.0",
     url="https://github.com/nomic-ai/nomic",
     description=description,
     long_description=long_description,
@@ -35,15 +35,17 @@ setup(
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python :: 3",
     ],
+    requires_python='>=3.8',
     install_requires=[
         "click",
         "jsonlines",
+        "jsonschema>=4.23.0,<5",
         "loguru",
         "rich",
         "requests",
         "numpy",
         "pandas",
-        "pydantic",
+        "pydantic>=2,<3",
         "tqdm",
         "pyarrow",
         "pillow",