corvic-engine 0.3.0rc86__tar.gz → 0.3.0rc87__tar.gz

{corvic_engine-0.3.0rc86 → corvic_engine-0.3.0rc87}/Cargo.lock

@@ -10,13 +10,13 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
 
 [[package]]
 name = "cfg-if"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
 
 [[package]]
 name = "corvic-engine"
-version = "0.3.0-rc.86"
+version = "0.3.0-rc.87"
 dependencies = [
  "numpy",
  "pyo3",
@@ -57,7 +57,7 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
 [[package]]
 name = "engine-walk"
-version = "0.3.0-rc.86"
+version = "0.3.0-rc.87"
 dependencies = [
  "version_check",
 ]
@@ -293,9 +293,9 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
 
 [[package]]
 name = "syn"
-version = "2.0.101"
+version = "2.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
+checksum = "f6397daf94fa90f058bd0fd88429dd9e5738999cca8d701813c80723add80462"
 dependencies = [
  "proc-macro2",
  "quote",

{corvic_engine-0.3.0rc86 → corvic_engine-0.3.0rc87}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["python"]
 
 [workspace.package]
-version = "0.3.0-rc.86"
+version = "0.3.0-rc.87"
 edition = "2021"
 publish = false
 

{corvic_engine-0.3.0rc86 → corvic_engine-0.3.0rc87}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corvic-engine
-Version: 0.3.0rc86
+Version: 0.3.0rc87
 Classifier: Environment :: Console
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: Implementation :: CPython

{corvic_engine-0.3.0rc86 → corvic_engine-0.3.0rc87}/pyproject.toml

@@ -18,7 +18,7 @@ authors = [
 ]
 readme = "README.md"
 description = "Seamless embedding generation and retrieval."
-version = "0.3.0-rc.86" # sync this with tool.poetry.version below
+version = "0.3.0-rc.87" # sync this with tool.poetry.version below
 
 # TODO(ddn): Pull dependencies from tool.poetry.dependencies. We use maturin as
 # the build system but poetry for the development environment.
@@ -73,7 +73,7 @@ packages = [
 ]
 description = ""
 authors = []
-version = "0.3.0-rc.86"
+version = "0.3.0-rc.87"
 
 [tool.poetry.scripts]
 check = "corvic_check.cli:main"

{corvic_engine-0.3.0rc86 → corvic_engine-0.3.0rc87}/python/corvic/emodel/_space.py

@@ -37,13 +37,15 @@ _DEFAULT_CONCAT_SEPARATOR = " "
 
 
 embedding_model_proto_to_name: Final[dict[embedding_models_pb2.Model, str]] = {
-    embedding_models_pb2.MODEL_CUSTOM: "random",
+    embedding_models_pb2.MODEL_CUSTOM: system.RandomTextEmbedder.model_name(),
     embedding_models_pb2.MODEL_SENTENCE_TRANSFORMER: "text-embedding-004",
     embedding_models_pb2.MODEL_GCP_TEXT_EMBEDDING_004: "text-embedding-004",
     embedding_models_pb2.MODEL_OPENAI_TEXT_EMBEDDING_3_SMALL: "text-embedding-3-small",
     embedding_models_pb2.MODEL_OPENAI_TEXT_EMBEDDING_3_LARGE: "text-embedding-3-large",
-    embedding_models_pb2.MODEL_IDENTITY: "identity",
+    embedding_models_pb2.MODEL_IDENTITY: system.IdentityTextEmbedder.model_name(),
     embedding_models_pb2.MODEL_UNSPECIFIED: "",
+    embedding_models_pb2.MODEL_CLIP: system.ClipText.model_name(),
+    embedding_models_pb2.MODEL_SIGLIP2: system.SigLIP2Text.model_name(),
 }
 name_to_proto_embedding_model = {
     name: model for model, name in embedding_model_proto_to_name.items()

{corvic_engine-0.3.0rc86 → corvic_engine-0.3.0rc87}/python/corvic/system/_embedder.py

@@ -39,6 +39,9 @@ class EmbedTextResult:
 class TextEmbedder(Protocol):
     """Use a model to embed text."""
 
+    @classmethod
+    def model_name(cls) -> str: ...
+
     async def aembed(
         self,
         context: EmbedTextContext,
@@ -86,8 +89,12 @@ class LoadedModels:
 class HFModelText(TextEmbedder):
     """Generic text/image embedder from hugging face models."""
 
+    @classmethod
     @abc.abstractmethod
-    def _load_models(self) -> LoadedModels: ...
+    def model_revision(cls) -> str: ...
+
+    @abc.abstractmethod
+    def load_models(self) -> LoadedModels: ...
 
     def embed(
         self, context: EmbedTextContext
@@ -98,7 +105,7 @@ class HFModelText(TextEmbedder):
             case 32:
                 coord_dtype = pl.Float32()
 
-        models = self._load_models()
+        models = self.load_models()
         model = models.model
         processor = models.processor
         model.eval()  # type: ignore[reportAttributeAccess]
@@ -152,7 +159,15 @@ class ClipText(HFModelText):
     overcoming several major challenges in computer vision.
     """
 
-    def _load_models(self):
+    @classmethod
+    def model_name(cls) -> str:
+        return "openai/clip-vit-base-patch32"
+
+    @classmethod
+    def model_revision(cls) -> str:
+        return "5812e510083bb2d23fa43778a39ac065d205ed4d"
+
+    def load_models(self):
         from transformers.models.clip import (
             CLIPModel,
             CLIPProcessor,
@@ -161,15 +176,15 @@ class ClipText(HFModelText):
         model = cast(
             AutoModel,
             CLIPModel.from_pretrained(  # pyright: ignore[reportUnknownMemberType]
-                pretrained_model_name_or_path="openai/clip-vit-base-patch32",
-                revision="5812e510083bb2d23fa43778a39ac065d205ed4d",
+                pretrained_model_name_or_path=self.model_name(),
+                revision=self.model_revision(),
             ),
         )
         processor = cast(
             AutoProcessor,
             CLIPProcessor.from_pretrained(  # pyright: ignore[reportUnknownMemberType]
-                pretrained_model_name_or_path="openai/clip-vit-base-patch32",
-                revision="5812e510083bb2d23fa43778a39ac065d205ed4d",
+                pretrained_model_name_or_path=self.model_name(),
+                revision=self.model_revision(),
                 use_fast=False,
             ),
         )
@@ -179,23 +194,31 @@ class ClipText(HFModelText):
 class SigLIP2Text(HFModelText):
     """SigLIP2 text/image embedder."""
 
-    def _load_models(self):
+    @classmethod
+    def model_name(cls) -> str:
+        return "google/siglip2-base-patch16-512"
+
+    @classmethod
+    def model_revision(cls) -> str:
+        return "a89f5c5093f902bf39d3cd4d81d2c09867f0724b"
+
+    def load_models(self):
         from transformers.models.auto.modeling_auto import AutoModel
         from transformers.models.auto.processing_auto import AutoProcessor
 
         model = cast(
             AutoModel,
             AutoModel.from_pretrained(  # pyright: ignore[reportUnknownMemberType]
-                pretrained_model_name_or_path="google/siglip2-base-patch16-512",
-                revision="a89f5c5093f902bf39d3cd4d81d2c09867f0724b",
+                pretrained_model_name_or_path=self.model_name(),
+                revision=self.model_revision(),
                 device_map="auto",
             ),
         )
         processor = cast(
             AutoProcessor,
             AutoProcessor.from_pretrained(  # pyright: ignore[reportUnknownMemberType]
-                pretrained_model_name_or_path="google/siglip2-base-patch16-512",
-                revision="a89f5c5093f902bf39d3cd4d81d2c09867f0724b",
+                pretrained_model_name_or_path=self.model_name(),
+                revision=self.model_revision(),
                 use_fast=True,
             ),
         )

{corvic_engine-0.3.0rc86 → corvic_engine-0.3.0rc87}/python/corvic/system/_text_embedder.py

@@ -18,6 +18,10 @@ class RandomTextEmbedder(TextEmbedder):
     Useful for testing.
     """
 
+    @classmethod
+    def model_name(cls) -> str:
+        return "random"
+
     def embed(
         self, context: EmbedTextContext
     ) -> Ok[EmbedTextResult] | InvalidArgumentError | InternalError:
@@ -62,6 +66,10 @@ class IdentityTextEmbedder(TextEmbedder):
     - The resulting list is truncated or padded to match the expected vector length.
     """
 
+    @classmethod
+    def model_name(cls) -> str:
+        return "identity"
+
     def _text_to_embedding(
         self, text: str, vector_length: int, *, normalization: bool = False
     ) -> list[float]:

{corvic_engine-0.3.0rc86 → corvic_engine-0.3.0rc87}/python/corvic_generated/embedding/v1/models_pb2.py

@@ -14,7 +14,7 @@ _sym_db = _symbol_database.Default()
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n corvic/embedding/v1/models.proto\x12\x13\x63orvic.embedding.v1\"R\n\nParameters\x12\x30\n\x05model\x18\x01 \x01(\x0e\x32\x1a.corvic.embedding.v1.ModelR\x05model\x12\x12\n\x04ndim\x18\x02 \x01(\x05R\x04ndim\"\xf4\x01\n\x19\x43olumnEmbeddingParameters\x12q\n\x11\x63olumn_parameters\x18\x01 \x03(\x0b\x32\x44.corvic.embedding.v1.ColumnEmbeddingParameters.ColumnParametersEntryR\x10\x63olumnParameters\x1a\x64\n\x15\x43olumnParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x35\n\x05value\x18\x02 \x01(\x0b\x32\x1f.corvic.embedding.v1.ParametersR\x05value:\x02\x38\x01\"\x8f\x01\n\x1e\x43oncatStringAndEmbedParameters\x12!\n\x0c\x63olumn_names\x18\x01 \x03(\tR\x0b\x63olumnNames\x12J\n\x10model_parameters\x18\x02 \x01(\x0b\x32\x1f.corvic.embedding.v1.ParametersR\x0fmodelParameters\"\x89\x01\n\x18\x43oncatAndEmbedParameters\x12!\n\x0c\x63olumn_names\x18\x01 \x03(\tR\x0b\x63olumnNames\x12J\n\x10model_parameters\x18\x02 \x01(\x0b\x32\x1f.corvic.embedding.v1.ParametersR\x0fmodelParameters\".\n\x18\x45mbedAndConcatParameters\x12\x12\n\x04ndim\x18\x01 \x01(\x05R\x04ndim\"a\n\x14ImageModelParameters\x12\x35\n\x05model\x18\x01 \x01(\x0e\x32\x1f.corvic.embedding.v1.ImageModelR\x05model\x12\x12\n\x04ndim\x18\x02 \x01(\x05R\x04ndim\"\x8d\x01\n\x14\x45mbedImageParameters\x12\x1f\n\x0b\x63olumn_name\x18\x01 \x01(\tR\ncolumnName\x12T\n\x10model_parameters\x18\x02 \x01(\x0b\x32).corvic.embedding.v1.ImageModelParametersR\x0fmodelParameters*\xdc\x01\n\x05Model\x12\x15\n\x11MODEL_UNSPECIFIED\x10\x00\x12\"\n\x1aMODEL_SENTENCE_TRANSFORMER\x10\x01\x1a\x02\x08\x01\x12\'\n#MODEL_OPENAI_TEXT_EMBEDDING_3_SMALL\x10\x03\x12\'\n#MODEL_OPENAI_TEXT_EMBEDDING_3_LARGE\x10\x04\x12 \n\x1cMODEL_GCP_TEXT_EMBEDDING_004\x10\x05\x12\x10\n\x0cMODEL_CUSTOM\x10\x02\x12\x12\n\x0eMODEL_IDENTITY\x10\x06*\x8a\x01\n\nImageModel\x12\x1b\n\x17IMAGE_MODEL_UNSPECIFIED\x10\x00\x12\x14\n\x10IMAGE_MODEL_CLIP\x10\x01\x12\x16\n\x12IMAGE_MODEL_CUSTOM\x10\x02\x12\x18\n\x14IMAGE_MODEL_IDENTITY\x10\x03\x12\x17\n\x13IMAGE_MODEL_SIGLIP2\x10\x04\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n corvic/embedding/v1/models.proto\x12\x13\x63orvic.embedding.v1\"R\n\nParameters\x12\x30\n\x05model\x18\x01 \x01(\x0e\x32\x1a.corvic.embedding.v1.ModelR\x05model\x12\x12\n\x04ndim\x18\x02 \x01(\x05R\x04ndim\"\xf4\x01\n\x19\x43olumnEmbeddingParameters\x12q\n\x11\x63olumn_parameters\x18\x01 \x03(\x0b\x32\x44.corvic.embedding.v1.ColumnEmbeddingParameters.ColumnParametersEntryR\x10\x63olumnParameters\x1a\x64\n\x15\x43olumnParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x35\n\x05value\x18\x02 \x01(\x0b\x32\x1f.corvic.embedding.v1.ParametersR\x05value:\x02\x38\x01\"\x8f\x01\n\x1e\x43oncatStringAndEmbedParameters\x12!\n\x0c\x63olumn_names\x18\x01 \x03(\tR\x0b\x63olumnNames\x12J\n\x10model_parameters\x18\x02 \x01(\x0b\x32\x1f.corvic.embedding.v1.ParametersR\x0fmodelParameters\"\x89\x01\n\x18\x43oncatAndEmbedParameters\x12!\n\x0c\x63olumn_names\x18\x01 \x03(\tR\x0b\x63olumnNames\x12J\n\x10model_parameters\x18\x02 \x01(\x0b\x32\x1f.corvic.embedding.v1.ParametersR\x0fmodelParameters\".\n\x18\x45mbedAndConcatParameters\x12\x12\n\x04ndim\x18\x01 \x01(\x05R\x04ndim\"a\n\x14ImageModelParameters\x12\x35\n\x05model\x18\x01 \x01(\x0e\x32\x1f.corvic.embedding.v1.ImageModelR\x05model\x12\x12\n\x04ndim\x18\x02 \x01(\x05R\x04ndim\"\x8d\x01\n\x14\x45mbedImageParameters\x12\x1f\n\x0b\x63olumn_name\x18\x01 \x01(\tR\ncolumnName\x12T\n\x10model_parameters\x18\x02 \x01(\x0b\x32).corvic.embedding.v1.ImageModelParametersR\x0fmodelParameters*\xff\x01\n\x05Model\x12\x15\n\x11MODEL_UNSPECIFIED\x10\x00\x12\"\n\x1aMODEL_SENTENCE_TRANSFORMER\x10\x01\x1a\x02\x08\x01\x12\'\n#MODEL_OPENAI_TEXT_EMBEDDING_3_SMALL\x10\x03\x12\'\n#MODEL_OPENAI_TEXT_EMBEDDING_3_LARGE\x10\x04\x12 \n\x1cMODEL_GCP_TEXT_EMBEDDING_004\x10\x05\x12\x10\n\x0cMODEL_CUSTOM\x10\x02\x12\x12\n\x0eMODEL_IDENTITY\x10\x06\x12\x0e\n\nMODEL_CLIP\x10\x07\x12\x11\n\rMODEL_SIGLIP2\x10\x08*\x8a\x01\n\nImageModel\x12\x1b\n\x17IMAGE_MODEL_UNSPECIFIED\x10\x00\x12\x14\n\x10IMAGE_MODEL_CLIP\x10\x01\x12\x16\n\x12IMAGE_MODEL_CUSTOM\x10\x02\x12\x18\n\x14IMAGE_MODEL_IDENTITY\x10\x03\x12\x17\n\x13IMAGE_MODEL_SIGLIP2\x10\x04\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -26,9 +26,9 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_COLUMNEMBEDDINGPARAMETERS_COLUMNPARAMETERSENTRY']._options = None
   _globals['_COLUMNEMBEDDINGPARAMETERS_COLUMNPARAMETERSENTRY']._serialized_options = b'8\001'
   _globals['_MODEL']._serialized_start=966
-  _globals['_MODEL']._serialized_end=1186
-  _globals['_IMAGEMODEL']._serialized_start=1189
-  _globals['_IMAGEMODEL']._serialized_end=1327
+  _globals['_MODEL']._serialized_end=1221
+  _globals['_IMAGEMODEL']._serialized_start=1224
+  _globals['_IMAGEMODEL']._serialized_end=1362
   _globals['_PARAMETERS']._serialized_start=57
   _globals['_PARAMETERS']._serialized_end=139
   _globals['_COLUMNEMBEDDINGPARAMETERS']._serialized_start=142

{corvic_engine-0.3.0rc86 → corvic_engine-0.3.0rc87}/python/corvic_generated/embedding/v1/models_pb2.pyi

@@ -15,6 +15,8 @@ class Model(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     MODEL_GCP_TEXT_EMBEDDING_004: _ClassVar[Model]
     MODEL_CUSTOM: _ClassVar[Model]
     MODEL_IDENTITY: _ClassVar[Model]
+    MODEL_CLIP: _ClassVar[Model]
+    MODEL_SIGLIP2: _ClassVar[Model]
 
 class ImageModel(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
@@ -30,6 +32,8 @@ MODEL_OPENAI_TEXT_EMBEDDING_3_LARGE: Model
 MODEL_GCP_TEXT_EMBEDDING_004: Model
 MODEL_CUSTOM: Model
 MODEL_IDENTITY: Model
+MODEL_CLIP: Model
+MODEL_SIGLIP2: Model
 IMAGE_MODEL_UNSPECIFIED: ImageModel
 IMAGE_MODEL_CLIP: ImageModel
 IMAGE_MODEL_CUSTOM: ImageModel