nomic 3.3.2__tar.gz → 3.3.4__tar.gz

{nomic-3.3.2 → nomic-3.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nomic
-Version: 3.3.2
+Version: 3.3.4
 Summary: The official Nomic python client.
 Home-page: https://github.com/nomic-ai/nomic
 Author: nomic.ai

{nomic-3.3.2 → nomic-3.3.4}/nomic/dataset.py

@@ -1,10 +1,13 @@
 import base64
 import concurrent
 import concurrent.futures
+import importlib.metadata
 import io
 import json
 import os
+import re
 import time
+import unicodedata
 from contextlib import contextmanager
 from datetime import datetime
 from io import BytesIO
@@ -12,6 +15,7 @@ from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
 import numpy as np
+import pandas as pd
 import pyarrow as pa
 import requests
 from loguru import logger
@@ -71,13 +75,20 @@ class AtlasClass(object):
         token = self.credentials["token"]
         self.token = token
 
-        self.header = {"Authorization": f"Bearer {token}"}
+        try:
+            version = importlib.metadata.version("nomic")
+        except Exception:
+            version = "unknown"
+
+        self.header = {"Authorization": f"Bearer {token}", "User-Agent": f"py-nomic/{version}"}
 
         if self.token:
             response = requests.get(
                 self.atlas_api_path + "/v1/user",
                 headers=self.header,
             )
+            if "X-AtlasWarning" in response.headers:
+                logger.warning(response.headers["X-AtlasWarning"])
             response = validate_api_http_response(response)
             if not response.status_code == 200:
                 logger.warning(str(response))
@@ -663,14 +674,22 @@ class AtlasProjection:
         sidecar_suffix = "feather"
         if sidecar_name != "":
             sidecar_suffix = f"{sidecar_name}.feather"
-        for key in tqdm(self._manifest["key"].to_pylist()):
-            sidecar_path = self.tile_destination / f"{key}.{sidecar_suffix}"
-            sidecar_url = (
-                self.dataset.atlas_api_path
-                + f"/v1/project/{self.dataset.id}/index/projection/{self.id}/quadtree/{key}.{sidecar_suffix}"
-            )
-            download_feather(sidecar_url, sidecar_path, headers=self.dataset.header, overwrite=overwrite)
-            downloaded_files.append(sidecar_path)
+        with concurrent.futures.ThreadPoolExecutor(4) as ex:
+            futures = []
+            for key in tqdm(self._manifest["key"].to_pylist()):
+                sidecar_path = self.tile_destination / f"{key}.{sidecar_suffix}"
+                sidecar_url = (
+                    self.dataset.atlas_api_path
+                    + f"/v1/project/{self.dataset.id}/index/projection/{self.id}/quadtree/{key}.{sidecar_suffix}"
+                )
+                futures.append(
+                    ex.submit(
+                        download_feather, sidecar_url, sidecar_path, headers=self.dataset.header, overwrite=overwrite
+                    )
+                )
+                downloaded_files.append(sidecar_path)
+            for f in futures:
+                f.result()
         return downloaded_files
 
     @property
@@ -752,6 +771,15 @@ class AtlasDataset(AtlasClass):
         * **dataset_id** - An alternative way to load a dataset is by passing the dataset_id directly. This only works if a dataset exists.
         """
         assert identifier is not None or dataset_id is not None, "You must pass a dataset identifier"
+        # Normalize identifier.
+        if identifier is not None:
+            s = identifier.split("/", 1)
+            identifier = unicodedata.normalize("NFD", s[-1])  # normalize accents
+            identifier = identifier.lower().replace(" ", "-").replace("_", "-")
+            identifier = re.sub(r"[^a-z0-9-]", "", identifier)
+            identifier = re.sub(r"-+", "-", identifier)
+            if len(s) == 2:
+                identifier = f"{s[0]}/{identifier}"
 
         super().__init__()
 
@@ -760,6 +788,8 @@ class AtlasDataset(AtlasClass):
                 f"Passing organization_name has been removed in Nomic Python client 3.0. Instead identify your dataset with `organization_name/project_name` (e.g. sterling-cooper/november-ads)."
             )
 
+        # Set this before possible early return.
+        self._schema = None
         if dataset_id is not None:
             self.meta = self._get_project_by_id(dataset_id)
             return
@@ -792,7 +822,6 @@ class AtlasDataset(AtlasClass):
             )
 
         self.meta = self._get_project_by_id(project_id=dataset_id)
-        self._schema = None
 
     def delete(self):
         """
@@ -1073,6 +1102,7 @@ class AtlasDataset(AtlasClass):
         else:
             projection = NomicProjectOptions()
 
+        topic_model_was_false = topic_model is False
         if isinstance(topic_model, Dict):
             topic_model = NomicTopicOptions(**topic_model)
         elif isinstance(topic_model, NomicTopicOptions):
@@ -1116,7 +1146,7 @@ class AtlasDataset(AtlasClass):
 
         build_template = {}
         if modality == "embedding":
-            if topic_model.topic_label_field is None:
+            if (not topic_model_was_false) and topic_model.topic_label_field is None:
                 logger.warning(
                     "You did not specify the `topic_label_field` option in your topic_model, your dataset will not contain auto-labeled topics."
                 )
@@ -1373,6 +1403,18 @@ class AtlasDataset(AtlasClass):
             blobs: A list of image paths, bytes, or PIL Images. Use if you want to create an AtlasDataset using image embeddings over your images. Note: Blobs are stored locally only.
             pbar: (Optional). A tqdm progress bar to update.
         """
+        if isinstance(data, DataFrame):
+            cols_before = set(data.columns)
+            for col in cols_before:
+                if col.startswith("_"):
+                    raise ValueError(
+                        f"You are attempting to upload a pandas dataframe with the column name {col}, but columns beginning with '_' are reserved for Atlas internal use. Please rename your column and try again."
+                    )
+            data = pa.Table.from_pandas(data)
+            for newcol in set(data.column_names).difference(cols_before):
+                logger.warning(f"Dropping column {newcol} added in pandas conversion to pyarrow")
+                data = data.drop([newcol])
+
         if embeddings is not None:
             self._add_embeddings(data=data, embeddings=embeddings, pbar=pbar)
         elif isinstance(data, pa.Table) and "_embeddings" in data.column_names:  # type: ignore
@@ -1607,6 +1649,7 @@ class AtlasDataset(AtlasClass):
             close_pbar = True
             pbar = tqdm(total=int(len(data)) // shard_size)
         failed = 0
+        failed_reqs = 0
         succeeded = 0
         errors_504 = 0
         with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
@@ -1655,6 +1698,11 @@ class AtlasDataset(AtlasClass):
                                 failed += shard_size
                                 pbar.update(1)
                                 response.close()
+                            failed_reqs += 1
+                            if failed_reqs > 10:
+                                raise RuntimeError(
+                                    f"{self.identifier}: Too many upload requests have failed at this time. Please try again later."
+                                )
                     else:
                         # A successful upload.
                         succeeded += shard_size

{nomic-3.3.2 → nomic-3.3.4}/nomic.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nomic
-Version: 3.3.2
+Version: 3.3.4
 Summary: The official Nomic python client.
 Home-page: https://github.com/nomic-ai/nomic
 Author: nomic.ai

{nomic-3.3.2 → nomic-3.3.4}/setup.py

@@ -23,7 +23,7 @@ with open("README.md") as f:
 
 setup(
     name="nomic",
-    version="3.3.2",
+    version="3.3.4",
     url="https://github.com/nomic-ai/nomic",
     description=description,
     long_description=long_description,