nomic 3.3.4__tar.gz → 3.4.0__tar.gz

{nomic-3.3.4 → nomic-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nomic
-Version: 3.3.4
+Version: 3.4.0
 Summary: The official Nomic python client.
 Home-page: https://github.com/nomic-ai/nomic
 Author: nomic.ai

{nomic-3.3.4 → nomic-3.4.0}/nomic/atlas.py

@@ -44,6 +44,7 @@ def map_data(
         description: The description of your dataset
         id_field: Specify your data unique id field. This field can be up 36 characters in length. If not specified, one will be created for you named `id_`.
         is_public: Should the dataset be accessible outside your Nomic Atlas organization.
+        indexed_field: The text field from the dataset that will be used to create embeddings, which determines the layout of the data map in Atlas. Required for text data but won't have an impact if uploading embeddings or image blobs.
         projection: Options to adjust Nomic Project - the dimensionality algorithm organizing your dataset.
         topic_model: Options to adjust Nomic Topic - the topic model organizing your dataset.
         duplicate_detection: Options to adjust Nomic Duplicates - the duplicate detection algorithm.
@@ -62,6 +63,10 @@ def map_data(
         modality = "text"
 
     if blobs is not None:
+        if embeddings is not None:
+            raise ValueError(
+                "You cannot pass both `blobs` and `embeddings` to map_data(). To create a map of images, include `blobs` and not `embeddings`. To create a map of embeddings with images as metadata, include your images as a field in your `data` parameter."
+            )
         # change this when we support other modalities
         modality = "image"
         indexed_field = "_blob_hash"

{nomic-3.3.4 → nomic-3.4.0}/nomic/embed.py

@@ -12,6 +12,7 @@ from urllib.parse import urlparse
 import PIL
 import PIL.Image
 import requests
+from tqdm import tqdm
 
 from .dataset import AtlasClass
 from .settings import *
@@ -286,18 +287,28 @@ def _text_embed4all(
         limits = {"cpu": 16, "kompute": 32, "metal": 1024}
         return n_tokens > limits[backend]
 
-    output = _embed4all.embed(
-        texts,
-        prefix=task_type,
-        dimensionality=dimensionality,
-        long_text_mode=long_text_mode,
-        return_dict=True,
-        atlas=True,
-        cancel_cb=cancel_cb if dynamic_mode else None,
-    )
-    ntok = output["n_prompt_tokens"]
+    pb = tqdm(total=len(texts), desc="Embedding texts", unit="inputs")
+    output_embeddings = []
+    ntok = 0
+    batch_size = 64
+    for start in range(0, len(texts), batch_size):
+        end = min(len(texts), start + batch_size)
+        b = end - start
+        out = _embed4all.embed(
+            texts[start:end],
+            prefix=task_type,
+            dimensionality=dimensionality,
+            long_text_mode=long_text_mode,
+            return_dict=True,
+            atlas=True,
+            cancel_cb=cancel_cb if dynamic_mode else None,
+        )
+        ntok += out["n_prompt_tokens"]
+        output_embeddings.extend(out["embeddings"])
+        pb.update(b)
+    pb.close()
     usage = {"prompt_tokens": ntok, "total_tokens": ntok}
-    return {"embeddings": output["embeddings"], "usage": usage, "model": model, "inference_mode": "local"}
+    return {"embeddings": output_embeddings, "usage": usage, "model": model, "inference_mode": "local"}
 
 
 def free_embedding_model() -> None:

{nomic-3.3.4 → nomic-3.4.0}/nomic.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nomic
-Version: 3.3.4
+Version: 3.4.0
 Summary: The official Nomic python client.
 Home-page: https://github.com/nomic-ai/nomic
 Author: nomic.ai

{nomic-3.3.4 → nomic-3.4.0}/setup.py

@@ -23,7 +23,7 @@ with open("README.md") as f:
 
 setup(
     name="nomic",
-    version="3.3.4",
+    version="3.4.0",
     url="https://github.com/nomic-ai/nomic",
     description=description,
     long_description=long_description,