corvic-engine 0.3.0rc81__tar.gz → 0.3.0rc82__tar.gz

{corvic_engine-0.3.0rc81 → corvic_engine-0.3.0rc82}/Cargo.lock

@@ -16,7 +16,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "corvic-engine"
-version = "0.3.0-rc.81"
+version = "0.3.0-rc.82"
 dependencies = [
  "numpy",
  "pyo3",
@@ -57,7 +57,7 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
 [[package]]
 name = "engine-walk"
-version = "0.3.0-rc.81"
+version = "0.3.0-rc.82"
 dependencies = [
  "version_check",
 ]

{corvic_engine-0.3.0rc81 → corvic_engine-0.3.0rc82}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["python"]
 
 [workspace.package]
-version = "0.3.0-rc.81"
+version = "0.3.0-rc.82"
 edition = "2021"
 publish = false
 

{corvic_engine-0.3.0rc81 → corvic_engine-0.3.0rc82}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corvic-engine
-Version: 0.3.0rc81
+Version: 0.3.0rc82
 Classifier: Environment :: Console
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: Implementation :: CPython
@@ -27,6 +27,7 @@ Requires-Dist: umap-learn>=0.5.5 ; extra == 'ml'
 Requires-Dist: pillow>=10.0.0 ; extra == 'ml'
 Requires-Dist: scikit-learn>=1.4.0 ; extra == 'ml'
 Requires-Dist: transformers[torch]>=4.45.0 ; extra == 'ml'
+Requires-Dist: sentencepiece>=0.2.0 ; extra == 'ml'
 Requires-Dist: opentelemetry-api>=1.20.0 ; extra == 'telemetry'
 Requires-Dist: opentelemetry-sdk>=1.20.0 ; extra == 'telemetry'
 Provides-Extra: ml

{corvic_engine-0.3.0rc81 → corvic_engine-0.3.0rc82}/pyproject.toml

@@ -18,7 +18,7 @@ authors = [
 ]
 readme = "README.md"
 description = "Seamless embedding generation and retrieval."
-version = "0.3.0-rc.81" # sync this with tool.poetry.version below
+version = "0.3.0-rc.82" # sync this with tool.poetry.version below
 
 # TODO(ddn): Pull dependencies from tool.poetry.dependencies. We use maturin as
 # the build system but poetry for the development environment.
@@ -52,6 +52,7 @@ ml = [
   "pillow >= 10.0.0",
   "scikit-learn >= 1.4.0",
   "transformers[torch] >= 4.45.0", # Needed for pre-trained torch models
+  "sentencepiece >= 0.2.0", # Needed for SigLIP2
 ]
 
 telemetry = [
@@ -72,18 +73,13 @@ packages = [
 ]
 description = ""
 authors = []
-version = "0.3.0-rc.81"
+version = "0.3.0-rc.82"
 
 [tool.poetry.scripts]
 check = "corvic_check.cli:main"
 
 [tool.poetry.plugins]
 
-[[tool.poetry.source]]
-name = "torch-cpu"
-url = "https://download.pytorch.org/whl/cpu"
-priority = "explicit"
-
 [[tool.poetry.source]]
 name = "torch-cpu-linux"
 url = "https://download.pytorch.org/whl/cpu"
@@ -168,7 +164,10 @@ torch = [
   { platform = "linux", version = "^2.7.0", source = "torch-cpu-linux" },
   { platform = "darwin", version = "^2.7.0", source = "pypi" },
 ]
-torchvision = { version = "^0.22", source = "torch-cpu" }
+torchvision = [
+  { platform = "linux", version = "^0.22.1", source = "torch-cpu-linux" },
+  { platform = "darwin", version = "^0.22.1", source = "pypi" },
+]
 transformers = "^4.51.3"
 umap-learn = "^0.5.5"
 pillow = "10.4.0"

{corvic_engine-0.3.0rc81 → corvic_engine-0.3.0rc82}/python/corvic/op_graph/ops.py

@@ -1260,7 +1260,9 @@ class _Base(OneofProtoWrapper[table_pb2.TableComputeOp], ABC):
                 column_name=column_name,
             )
 
-        field = column.to_frame().to_arrow().schema.field(column_name)
+        # TODO(aneesh): See https://github.com/pola-rs/polars/issues/23111 for
+        # and remove the rechunk eventually.
+        field = column.to_frame().rechunk().to_arrow().schema.field(column_name)
         dtype = field.type
 
         if ftype is None:
@@ -1268,8 +1270,10 @@ class _Base(OneofProtoWrapper[table_pb2.TableComputeOp], ABC):
 
         # Convert array to record batch with dummy column to use pa_scalar and then
         # remove the dummy column.
+        # TODO(aneesh): See https://github.com/pola-rs/polars/issues/23111 for
+        # and remove the rechunk eventually.
         value_batch = pa.record_batch(
-            [column.to_arrow()], schema=pa.schema([field]), metadata=None
+            [column.rechunk().to_arrow()], schema=pa.schema([field]), metadata=None
         )
         structs = pa_scalar.batch_to_structs(value_batch)
         literal_values = [

{corvic_engine-0.3.0rc81 → corvic_engine-0.3.0rc82}/python/corvic/system/__init__.py

@@ -14,6 +14,7 @@ from corvic.system._embedder import (
     EmbedTextContext,
     EmbedTextResult,
     ImageEmbedder,
+    SigLIP2Text,
     TextEmbedder,
 )
 from corvic.system._image_embedder import (
@@ -68,23 +69,26 @@ __all__ = [
     "Client",
     "Clip",
     "ClipText",
+    "CombinedImageEmbedder",
     "DEFAULT_VECTOR_COLUMN_NAMES_TO_SIZES",
     "DataMisplacedError",
     "DimensionReducer",
+    "EmbedImageContext",
+    "EmbedImageResult",
     "EmbedTextContext",
     "EmbedTextResult",
     "ExecutionContext",
     "ExecutionResult",
-    "EmbedImageContext",
-    "EmbedImageResult",
+    "IdentityImageEmbedder",
+    "IdentityTextEmbedder",
     "ImageEmbedder",
     "InMemoryExecutionResult",
     "InMemoryExecutor",
     "OpGraphExecutor",
     "OpGraphPlanner",
     "RandomImageEmbedder",
-    "CombinedImageEmbedder",
     "RandomTextEmbedder",
+    "SigLIP2Text",
     "StagingDB",
     "StorageManager",
     "TableComputeContext",
@@ -97,9 +101,7 @@ __all__ = [
     "VectorSimilarityMetric",
     "get_polars_embedding",
     "get_polars_embedding_length",
+    "image_from_bytes",
     "make_dict_bytes_human_readable",
     "make_list_bytes_human_readable",
-    "image_from_bytes",
-    "IdentityTextEmbedder",
-    "IdentityImageEmbedder",
 ]

{corvic_engine-0.3.0rc81 → corvic_engine-0.3.0rc82}/python/corvic/system/in_memory_executor.py

@@ -215,7 +215,12 @@ class _SchemaAndBatches:
             and not len(dataframe)
         ):
             return cls(expected_schema, [], metrics)
-        table = dataframe.to_arrow()
+        # TODO(aneesh): without this rechunk, conversion to arrow will
+        # occasionally fail and complain about mismatched child array lengths.
+        # This should probably be fixed internally in polars (note that this
+        # still currently happens on polars 1.30.0 - the latest release).
+        # See https://github.com/pola-rs/polars/issues/23111.
+        table = dataframe.rechunk().to_arrow()
         schema = table.schema
         return cls(schema, table.to_batches(), metrics)