corvic-engine 0.3.0rc42__tar.gz → 0.3.0rc44__tar.gz

{corvic_engine-0.3.0rc42 → corvic_engine-0.3.0rc44}/Cargo.lock

@@ -22,7 +22,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "corvic-engine"
-version = "0.3.0-rc.42"
+version = "0.3.0-rc.44"
 dependencies = [
  "numpy",
  "pyo3",
@@ -63,7 +63,7 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
 [[package]]
 name = "engine-walk"
-version = "0.3.0-rc.42"
+version = "0.3.0-rc.44"
 dependencies = [
  "version_check",
 ]

{corvic_engine-0.3.0rc42 → corvic_engine-0.3.0rc44}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["python"]
 
 [workspace.package]
-version = "0.3.0-rc.42"
+version = "0.3.0-rc.44"
 edition = "2021"
 publish = false
 

{corvic_engine-0.3.0rc42 → corvic_engine-0.3.0rc44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corvic-engine
-Version: 0.3.0rc42
+Version: 0.3.0rc44
 Classifier: Environment :: Console
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: Implementation :: CPython
@@ -11,25 +11,25 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Rust
 Classifier: Topic :: Scientific/Engineering
-Requires-Dist: cachetools >=5
-Requires-Dist: duckdb >=1.0.0
-Requires-Dist: more-itertools >=10
-Requires-Dist: numpy >=1.26
-Requires-Dist: polars >=1.7.1
-Requires-Dist: protobuf >=4.25
-Requires-Dist: protovalidate >=0.3
-Requires-Dist: pyarrow >=17
-Requires-Dist: sqlalchemy >=2
-Requires-Dist: sqlglot >=25.6.0, <26
-Requires-Dist: structlog >=24
+Requires-Dist: cachetools>=5
+Requires-Dist: duckdb>=1.0.0
+Requires-Dist: more-itertools>=10
+Requires-Dist: numpy>=1.26
+Requires-Dist: polars>=1.7.1
+Requires-Dist: protobuf>=4.25
+Requires-Dist: protovalidate>=0.3
+Requires-Dist: pyarrow>=17
+Requires-Dist: sqlalchemy>=2
+Requires-Dist: sqlglot>=25.6.0,<26
+Requires-Dist: structlog>=24
 Requires-Dist: tqdm
-Requires-Dist: typing-extensions >=4.9
-Requires-Dist: umap-learn >=0.5.5 ; extra == 'ml'
-Requires-Dist: pillow >=10.0.0 ; extra == 'ml'
-Requires-Dist: scikit-learn >=1.4.0 ; extra == 'ml'
-Requires-Dist: transformers[torch] >=4.45.0 ; extra == 'ml'
-Requires-Dist: opentelemetry-api >=1.20.0 ; extra == 'telemetry'
-Requires-Dist: opentelemetry-sdk >=1.20.0 ; extra == 'telemetry'
+Requires-Dist: typing-extensions>=4.9
+Requires-Dist: umap-learn>=0.5.5 ; extra == 'ml'
+Requires-Dist: pillow>=10.0.0 ; extra == 'ml'
+Requires-Dist: scikit-learn>=1.4.0 ; extra == 'ml'
+Requires-Dist: transformers[torch]>=4.45.0 ; extra == 'ml'
+Requires-Dist: opentelemetry-api>=1.20.0 ; extra == 'telemetry'
+Requires-Dist: opentelemetry-sdk>=1.20.0 ; extra == 'telemetry'
 Provides-Extra: ml
 Provides-Extra: telemetry
 License-File: LICENSE

{corvic_engine-0.3.0rc42 → corvic_engine-0.3.0rc44}/pyproject.toml

@@ -20,7 +20,7 @@ authors = [
 ]
 readme = "README.md"
 description = "Seamless embedding generation and retrieval."
-version = "0.3.0-rc.42" # sync this with tool.poetry.version below
+version = "0.3.0-rc.44" # sync this with tool.poetry.version below
 
 # TODO(ddn): Pull dependencies from tool.poetry.dependencies. We use maturin as
 # the build system but poetry for the development environment.
@@ -74,7 +74,7 @@ packages = [
 ]
 description = ""
 authors = []
-version = "0.3.0-rc.42"
+version = "0.3.0-rc.44"
 
 [tool.poetry.scripts]
 check = "corvic_check.cli:main"

{corvic_engine-0.3.0rc42 → corvic_engine-0.3.0rc44}/python/corvic/embedding_metric/embeddings.py

@@ -84,8 +84,21 @@ def stable_rank(
         case Ok():
             pass
 
-    numerator = norm(embeddings, order="fro")
-    denominator = norm(embeddings, order=2) ** 2
+    try:
+        numerator = norm(embeddings, order="fro")
+        denominator = norm(embeddings, order=2) ** 2
+    except np.linalg.LinAlgError as err:
+        return InvalidArgumentError.from_(err)
+
+    if not np.isfinite(numerator):
+        return InvalidArgumentError("embeddings norm_fro is not finite")
+
+    if not np.isfinite(denominator):
+        return InvalidArgumentError("embeddings norm_2 is not finite")
+
+    if not denominator:
+        return InvalidArgumentError("embeddings norm_2 is zero")
+
     metric = numerator / denominator
     if normalize:
         metric = 1 - 1 / (1 + metric)
@@ -162,7 +175,10 @@ def ne_sum(
             pass
 
     covariance = np.cov(embeddings.T)
-    eigenvalues = linalg.eigvals(covariance)
+    try:
+        eigenvalues = linalg.eigvals(covariance)
+    except np.linalg.LinAlgError as err:
+        return InvalidArgumentError.from_(err)
 
     # Discard imaginary part
     eigenvalues = eigenvalues.real
@@ -172,7 +188,7 @@ def ne_sum(
     eigenvalues = eigenvalues[sorted_indices]
 
     if eigenvalues[0] == 0:
-        return Ok(0)
+        return Ok(0.0)
 
     ne_sum_value = float(np.sum(eigenvalues) / eigenvalues[0])
     if normalize:
@@ -227,7 +243,11 @@ def condition_number(
         case Ok():
             pass
 
-    metric = float(np.linalg.cond(embeddings, p=p))
+    try:
+        metric = float(np.linalg.cond(embeddings, p=p))
+    except np.linalg.LinAlgError as err:
+        return InvalidArgumentError.from_(err)
+
     if normalize:
         metric = 1 - 1 / (1 + metric)
     return Ok(metric)
@@ -246,7 +266,11 @@ def rcondition_number(
         case Ok():
             pass
 
-    metric = float(1 / np.linalg.cond(embeddings, p=p))
+    try:
+        metric = float(1 / np.linalg.cond(embeddings, p=p))
+    except np.linalg.LinAlgError as err:
+        return InvalidArgumentError.from_(err)
+
     if normalize:
         metric = 1 - 1 / (1 + metric)
     return Ok(metric)

{corvic_engine-0.3.0rc42 → corvic_engine-0.3.0rc44}/python/corvic/model/_completion_model.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import copy
 import datetime
 from collections.abc import Iterable, Sequence
-from typing import TypeAlias
+from typing import Literal, TypeAlias
 
 from sqlalchemy import orm as sa_orm
 
@@ -19,6 +19,7 @@ from corvic.model._proto_orm_convert import (
 )
 from corvic.result import InvalidArgumentError, NotFoundError, Ok
 from corvic_generated.model.v1alpha import models_pb2
+from corvic_generated.orm.v1 import completion_model_pb2
 
 CompletionModelID: TypeAlias = orm.CompletionModelID
 OrgID: TypeAlias = orm.OrgID
@@ -62,12 +63,46 @@ class CompletionModel(
         return OrgID(self.proto_self.org_id)
 
     @property
-    def model_name(self) -> str:
-        return self.proto_self.model_name
+    def provider(self) -> Literal["openai-generic", "azure-openai"] | None:
+        match self.proto_self.parameters.WhichOneof("params"):
+            case "azure_openai_parameters":
+                return "azure-openai"
+            case "generic_openai_parameters":
+                return "openai-generic"
+            case _:
+                return None
 
     @property
-    def endpoint(self) -> str:
-        return self.proto_self.endpoint
+    def parameters(
+        self,
+    ) -> (
+        completion_model_pb2.AzureOpenAIParameters
+        | completion_model_pb2.GenericOpenAIParameters
+        | None
+    ):
+        match self.provider:
+            case "azure-openai":
+                return self.azure_openai_parameters
+            case "openai-generic":
+                return self.generic_openai_parameters
+            case None:
+                return None
+
+    @property
+    def azure_openai_parameters(
+        self,
+    ) -> completion_model_pb2.AzureOpenAIParameters | None:
+        if self.proto_self.parameters.HasField("azure_openai_parameters"):
+            return self.proto_self.parameters.azure_openai_parameters
+        return None
+
+    @property
+    def generic_openai_parameters(
+        self,
+    ) -> completion_model_pb2.GenericOpenAIParameters | None:
+        if self.proto_self.parameters.HasField("generic_openai_parameters"):
+            return self.proto_self.parameters.generic_openai_parameters
+        return None
 
     @property
     def secret_api_key(self) -> str:
@@ -83,8 +118,7 @@ class CompletionModel(
         *,
         name: str,
         description: str,
-        model_name: str,
-        endpoint: str,
+        parameters: completion_model_pb2.CompletionModelParameters,
         secret_api_key: str,
         client: system.Client | None = None,
     ):
@@ -94,8 +128,7 @@ class CompletionModel(
             models_pb2.CompletionModel(
                 name=name,
                 description=description,
-                model_name=model_name,
-                endpoint=endpoint,
+                parameters=parameters,
                 secret_api_key=secret_api_key,
             ),
         )
@@ -153,14 +186,11 @@ class CompletionModel(
         proto_self.description = description
         return CompletionModel(self.client, proto_self)
 
-    def with_model_name(self, model_name: str) -> CompletionModel:
-        proto_self = copy.deepcopy(self.proto_self)
-        proto_self.model_name = model_name
-        return CompletionModel(self.client, proto_self)
-
-    def with_endpoint(self, endpoint: str) -> CompletionModel:
+    def with_parameters(
+        self, parameters: completion_model_pb2.CompletionModelParameters
+    ) -> CompletionModel:
         proto_self = copy.deepcopy(self.proto_self)
-        proto_self.endpoint = endpoint
+        proto_self.parameters.CopyFrom(parameters)
         return CompletionModel(self.client, proto_self)
 
     def with_secret_api_key(self, secret_api_key: str) -> CompletionModel:

{corvic_engine-0.3.0rc42 → corvic_engine-0.3.0rc44}/python/corvic/model/_feature_view.py

@@ -1075,6 +1075,7 @@ class FeatureView(BaseModel[FeatureViewID, models_pb2.FeatureView, orm.FeatureVi
             )
 
         proto_feature_view_source = models_pb2.FeatureViewSource(
+            room_id=str(source.room_id),
             table_op_graph=new_table.op_graph.to_proto(),
             drop_disconnected=drop_disconnected,
             source=source.proto_self,

{corvic_engine-0.3.0rc42 → corvic_engine-0.3.0rc44}/python/corvic/model/_proto_orm_convert.py

@@ -66,6 +66,7 @@ def _translate_orm_ids(
             | models_pb2.Space()
             | models_pb2.Agent()
             | models_pb2.Pipeline()
+            | models_pb2.FeatureViewSource()
         ):
             room_id = orm.RoomID(proto_obj.room_id)
             match room_id.to_db():
@@ -73,7 +74,7 @@ def _translate_orm_ids(
                     return err
                 case Ok():
                     pass
-        case models_pb2.FeatureViewSource() | models_pb2.CompletionModel():
+        case models_pb2.CompletionModel():
             room_id = None
         case models_pb2.Room():
             room_id = cast(orm.RoomID, obj_id)
@@ -146,6 +147,7 @@ def feature_view_source_orm_to_proto(
 ) -> models_pb2.FeatureViewSource:
     return models_pb2.FeatureViewSource(
         id=str(feature_view_source_orm.id),
+        room_id=str(feature_view_source_orm.room_id),
         source=source_orm_to_proto(feature_view_source_orm.source),
         table_op_graph=feature_view_source_orm.table_op_graph,
         drop_disconnected=feature_view_source_orm.drop_disconnected,
@@ -225,8 +227,7 @@ def completion_model_orm_to_proto(
         name=completion_model_orm.name,
         description=completion_model_orm.description,
         org_id=str(completion_model_orm.org_id),
-        model_name=completion_model_orm.model_name,
-        endpoint=completion_model_orm.endpoint,
+        parameters=completion_model_orm.parameters,
         secret_api_key=completion_model_orm.secret_api_key,
         created_at=timestamp_orm_to_proto(completion_model_orm.created_at),
     )
@@ -263,7 +264,9 @@ def resource_proto_to_orm(
         latest_event=proto_obj.recent_events[-1] if proto_obj.recent_events else None,
         room_id=ids.room_id,
         source_associations=[
-            orm.SourceResourceAssociation(source_id=src_id, resource_id=ids.obj_id)
+            orm.SourceResourceAssociation(
+                room_id=ids.room_id, source_id=src_id, resource_id=ids.obj_id
+            )
             for src_id in source_ids
         ],
     )
@@ -317,7 +320,10 @@ def pipeline_proto_to_orm(  # noqa: C901
                     session.add(resource_orm)
                 session.merge(
                     orm.PipelineInput(
-                        pipeline=orm_obj, resource=resource_orm, name=name
+                        room_id=resource_orm.room_id,
+                        pipeline=orm_obj,
+                        resource=resource_orm,
+                        name=name,
                     )
                 )
 
@@ -333,7 +339,12 @@ def pipeline_proto_to_orm(  # noqa: C901
                 else:
                     session.add(source_orm)
                 session.merge(
-                    orm.PipelineOutput(pipeline=orm_obj, source=source_orm, name=name)
+                    orm.PipelineOutput(
+                        room_id=source_orm.room_id,
+                        pipeline=orm_obj,
+                        source=source_orm,
+                        name=name,
+                    )
                 )
     if proto_obj.org_id:
         org_id = orm.OrgID(proto_obj.org_id)
@@ -356,7 +367,9 @@ def source_proto_to_orm(
     resource_id = orm.ResourceID(proto_obj.resource_id)
     if resource_id:
         associations = [
-            orm.SourceResourceAssociation(source_id=ids.obj_id, resource_id=resource_id)
+            orm.SourceResourceAssociation(
+                room_id=ids.room_id, source_id=ids.obj_id, resource_id=resource_id
+            )
         ]
     else:
         associations = list[orm.SourceResourceAssociation]()
@@ -499,6 +512,7 @@ def feature_view_source_proto_to_orm(
     else:
         session.add(source)
     orm_obj = orm.FeatureViewSource(
+        room_id=source.room_id,
         table_op_graph=proto_obj.table_op_graph,
         drop_disconnected=proto_obj.drop_disconnected,
         source=source,
@@ -543,8 +557,7 @@ def completion_model_proto_to_orm(
         id=ids.obj_id,
         name=proto_obj.name,
         description=proto_obj.description,
-        model_name=proto_obj.model_name,
-        endpoint=proto_obj.endpoint,
+        parameters=proto_obj.parameters,
         secret_api_key=proto_obj.secret_api_key,
     )
 

{corvic_engine-0.3.0rc42 → corvic_engine-0.3.0rc44}/python/corvic/model/_space.py

@@ -782,50 +782,67 @@ class TabularSpace(Space):
 
             embedding_column_tmp_name = f"__embed-{uuid.uuid4()}"
 
-            op = (
-                op.concat_list(
-                    column_names=embedding_column_tmp_names,
-                    concat_list_column_name=embedding_column_tmp_name,
-                )
-                .and_then(
-                    lambda t,
-                    embedding_column_name=embedding_column_tmp_name: op_graph.op.coordinates_from_embedding(  # noqa: E501
-                        table=t,
-                        embedding_column_name=embedding_column_name,
-                        output_dims=parameters.ndim,
-                    )
+            # Avoid 0 padding for spaces with small numbers of columns
+            target_list_length = min(parameters.ndim, len(embedding_column_tmp_names))
+
+            def reduce_dimension(
+                op: op_graph.Op,
+                embedding_column_tmp_name=embedding_column_tmp_name,
+                target_list_length=target_list_length,
+            ):
+                return op.truncate_list(
+                    list_column_name=embedding_column_tmp_name,
+                    target_list_length=target_list_length,
+                    padding_value=0,
                 )
-                .and_then(
-                    lambda t,
-                    pk_field=pk_field,
-                    embedding_column_tmp_name=embedding_column_tmp_name,
-                    output_source=output_source: t.select_columns(
-                        [pk_field.name, embedding_column_tmp_name]
-                    )
+
+            def select_columns(
+                op: op_graph.Op,
+                pk_field=pk_field,
+                embedding_column_tmp_name=embedding_column_tmp_name,
+            ):
+                return op.select_columns([pk_field.name, embedding_column_tmp_name])
+
+            def update_feature_types(
+                op: op_graph.Op,
+                embedding_column_tmp_name=embedding_column_tmp_name,
+            ):
+                return op.update_feature_types(
+                    {embedding_column_tmp_name: op_graph.feature_type.embedding()}
                 )
-                .and_then(
-                    lambda t,
-                    embedding_column_tmp_name=embedding_column_tmp_name: t.update_feature_types(  # noqa: E501
-                        {embedding_column_tmp_name: op_graph.feature_type.embedding()}
-                    )
+
+            def rename_columns(
+                op: op_graph.Op,
+                pk_field=pk_field,
+                embedding_column_tmp_name=embedding_column_tmp_name,
+            ):
+                return op.rename_columns(
+                    {
+                        pk_field.name: "entity_id",
+                        embedding_column_tmp_name: "embedding",
+                    }
                 )
-                .and_then(
-                    lambda t,
-                    pk_field=pk_field,
-                    embedding_column_tmp_name=embedding_column_tmp_name: t.rename_columns(  # noqa: E501
-                        {
-                            pk_field.name: "entity_id",
-                            embedding_column_tmp_name: "embedding",
-                        }
-                    )
+
+            def add_literal_column(
+                op: op_graph.Op,
+                output_source=output_source,
+            ):
+                return op.add_literal_column(
+                    "source_id",
+                    str(output_source.id),
+                    pa.string(),
                 )
-                .and_then(
-                    lambda t, output_source=output_source: t.add_literal_column(
-                        "source_id",
-                        str(output_source.id),
-                        pa.string(),
-                    )
+
+            op = (
+                op.concat_list(
+                    column_names=embedding_column_tmp_names,
+                    concat_list_column_name=embedding_column_tmp_name,
                 )
+                .and_then(reduce_dimension)
+                .and_then(select_columns)
+                .and_then(update_feature_types)
+                .and_then(rename_columns)
+                .and_then(add_literal_column)
             )
 
             match op: