corvic-engine 0.3.0rc72__tar.gz → 0.3.0rc74__tar.gz

{corvic_engine-0.3.0rc72 → corvic_engine-0.3.0rc74}/Cargo.lock

@@ -16,7 +16,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "corvic-engine"
-version = "0.3.0-rc.72"
+version = "0.3.0-rc.74"
 dependencies = [
  "numpy",
  "pyo3",
@@ -57,7 +57,7 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
 [[package]]
 name = "engine-walk"
-version = "0.3.0-rc.72"
+version = "0.3.0-rc.74"
 dependencies = [
  "version_check",
 ]

{corvic_engine-0.3.0rc72 → corvic_engine-0.3.0rc74}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["python"]
 
 [workspace.package]
-version = "0.3.0-rc.72"
+version = "0.3.0-rc.74"
 edition = "2021"
 publish = false
 

{corvic_engine-0.3.0rc72 → corvic_engine-0.3.0rc74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corvic-engine
-Version: 0.3.0rc72
+Version: 0.3.0rc74
 Classifier: Environment :: Console
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: Implementation :: CPython

{corvic_engine-0.3.0rc72 → corvic_engine-0.3.0rc74}/pyproject.toml

@@ -18,7 +18,7 @@ authors = [
 ]
 readme = "README.md"
 description = "Seamless embedding generation and retrieval."
-version = "0.3.0-rc.72" # sync this with tool.poetry.version below
+version = "0.3.0-rc.74" # sync this with tool.poetry.version below
 
 # TODO(ddn): Pull dependencies from tool.poetry.dependencies. We use maturin as
 # the build system but poetry for the development environment.
@@ -72,7 +72,7 @@ packages = [
 ]
 description = ""
 authors = []
-version = "0.3.0-rc.72"
+version = "0.3.0-rc.74"
 
 [tool.poetry.scripts]
 check = "corvic_check.cli:main"

{corvic_engine-0.3.0rc72 → corvic_engine-0.3.0rc74}/python/corvic/model/_proto_orm_convert.py

@@ -5,7 +5,6 @@ from typing import Any, Protocol, TypeVar
 import sqlalchemy as sa
 import sqlalchemy.orm as sa_orm
 from google.protobuf import timestamp_pb2
-from more_itertools import flatten
 
 from corvic import orm
 from corvic.result import InternalError, InvalidArgumentError, Ok
@@ -59,6 +58,12 @@ OrmBelongsToOrgObj = TypeVar("OrmBelongsToOrgObj", bound=_OrmBelongsToOrgModel[A
 OrmBelongsToRoomObj = TypeVar("OrmBelongsToRoomObj", bound=_OrmBelongsToRoomModel[Any])
 
 
+def _orm_id_to_str(id: orm.ID | None):
+    if id:
+        return str(id)
+    return ""
+
+
 def _translate_orm_id(
     obj_id: str, id_class: type[ID]
 ) -> Ok[ID | None] | orm.InvalidORMIdentifierError:
@@ -88,9 +93,9 @@ def resource_orm_to_proto(resource_orm: orm.Resource) -> models_pb2.Resource:
     pipeline_id = ""
     if resource_orm.pipeline_ref:
         pipeline_input_name = resource_orm.pipeline_ref.name
-        pipeline_id = str(resource_orm.pipeline_ref.pipeline_id)
+        pipeline_id = _orm_id_to_str(resource_orm.pipeline_ref.pipeline_id)
     return models_pb2.Resource(
-        id=str(resource_orm.id),
+        id=_orm_id_to_str(resource_orm.id),
         name=resource_orm.name,
         description=resource_orm.description,
         mime_type=resource_orm.mime_type,
@@ -98,8 +103,8 @@ def resource_orm_to_proto(resource_orm: orm.Resource) -> models_pb2.Resource:
         md5=resource_orm.md5,
         size=resource_orm.size,
         original_path=resource_orm.original_path,
-        room_id=str(resource_orm.room_id),
-        org_id=str(resource_orm.org_id),
+        room_id=_orm_id_to_str(resource_orm.room_id),
+        org_id=_orm_id_to_str(resource_orm.org_id),
         recent_events=[resource_orm.latest_event] if resource_orm.latest_event else [],
         is_terminal=bool(resource_orm.is_terminal),
         pipeline_id=pipeline_id,
@@ -110,12 +115,12 @@ def resource_orm_to_proto(resource_orm: orm.Resource) -> models_pb2.Resource:
 
 def source_orm_to_proto(source_orm: orm.Source) -> models_pb2.Source:
     return models_pb2.Source(
-        id=str(source_orm.id),
+        id=_orm_id_to_str(source_orm.id),
         name=source_orm.name,
         table_op_graph=source_orm.table_op_graph,
-        room_id=str(source_orm.room_id),
-        org_id=str(source_orm.org_id),
-        pipeline_id=str(source_orm.pipeline_ref.pipeline_id)
+        room_id=_orm_id_to_str(source_orm.room_id),
+        org_id=_orm_id_to_str(source_orm.org_id),
+        pipeline_id=_orm_id_to_str(source_orm.pipeline_ref.pipeline_id)
         if source_orm.pipeline_ref
         else "",
         created_at=timestamp_orm_to_proto(source_orm.created_at),
@@ -132,12 +137,12 @@ def feature_view_source_orm_to_proto(
         # with the source's opgraph
         op = feature_view_source_orm.source.table_op_graph
     return models_pb2.FeatureViewSource(
-        id=str(feature_view_source_orm.id),
-        room_id=str(feature_view_source_orm.room_id),
+        id=_orm_id_to_str(feature_view_source_orm.id),
+        room_id=_orm_id_to_str(feature_view_source_orm.room_id),
         source=source_orm_to_proto(feature_view_source_orm.source),
         table_op_graph=op,
         drop_disconnected=feature_view_source_orm.drop_disconnected,
-        org_id=str(feature_view_source_orm.org_id),
+        org_id=_orm_id_to_str(feature_view_source_orm.org_id),
         created_at=timestamp_orm_to_proto(feature_view_source_orm.created_at),
     )
 
@@ -146,16 +151,16 @@ def feature_view_orm_to_proto(
     feature_view_orm: orm.FeatureView,
 ) -> models_pb2.FeatureView:
     return models_pb2.FeatureView(
-        id=str(feature_view_orm.id),
+        id=_orm_id_to_str(feature_view_orm.id),
         name=feature_view_orm.name,
         description=feature_view_orm.description,
-        room_id=str(feature_view_orm.room_id),
+        room_id=_orm_id_to_str(feature_view_orm.room_id),
         feature_view_output=feature_view_orm.feature_view_output,
         feature_view_sources=[
             feature_view_source_orm_to_proto(fvs)
             for fvs in feature_view_orm.feature_view_sources
         ],
-        org_id=str(feature_view_orm.org_id),
+        org_id=_orm_id_to_str(feature_view_orm.org_id),
         created_at=timestamp_orm_to_proto(feature_view_orm.created_at),
     )
 
@@ -164,15 +169,15 @@ def pipeline_orm_to_proto(
     pipeline_orm: orm.Pipeline,
 ) -> models_pb2.Pipeline:
     return models_pb2.Pipeline(
-        id=str(pipeline_orm.id),
+        id=_orm_id_to_str(pipeline_orm.id),
         name=pipeline_orm.name,
-        room_id=str(pipeline_orm.room_id),
+        room_id=_orm_id_to_str(pipeline_orm.room_id),
         source_outputs={
             output_obj.name: source_orm_to_proto(output_obj.source)
             for output_obj in pipeline_orm.outputs
         },
         pipeline_transformation=pipeline_orm.transformation,
-        org_id=str(pipeline_orm.org_id),
+        org_id=_orm_id_to_str(pipeline_orm.org_id),
         description=pipeline_orm.description,
         created_at=timestamp_orm_to_proto(pipeline_orm.created_at),
     )
@@ -180,23 +185,23 @@ def pipeline_orm_to_proto(
 
 def space_orm_to_proto(space_orm: orm.Space) -> models_pb2.Space:
     return models_pb2.Space(
-        id=str(space_orm.id),
+        id=_orm_id_to_str(space_orm.id),
         name=space_orm.name,
         description=space_orm.description,
-        room_id=str(space_orm.room_id),
+        room_id=_orm_id_to_str(space_orm.room_id),
         space_parameters=space_orm.parameters,
         feature_view=feature_view_orm_to_proto(space_orm.feature_view),
         auto_sync=space_orm.auto_sync if space_orm.auto_sync is not None else False,
-        org_id=str(space_orm.org_id),
+        org_id=_orm_id_to_str(space_orm.org_id),
         created_at=timestamp_orm_to_proto(space_orm.created_at),
     )
 
 
 def room_orm_to_proto(room_orm: orm.Room) -> models_pb2.Room:
     return models_pb2.Room(
-        id=str(room_orm.id),
+        id=_orm_id_to_str(room_orm.id),
         name=room_orm.name,
-        org_id=str(room_orm.org_id),
+        org_id=_orm_id_to_str(room_orm.org_id),
         created_at=timestamp_orm_to_proto(room_orm.created_at),
     )
 
@@ -205,10 +210,10 @@ def completion_model_orm_to_proto(
     completion_model_orm: orm.CompletionModel,
 ) -> models_pb2.CompletionModel:
     return models_pb2.CompletionModel(
-        id=str(completion_model_orm.id),
+        id=_orm_id_to_str(completion_model_orm.id),
         name=completion_model_orm.name,
         description=completion_model_orm.description,
-        org_id=str(completion_model_orm.org_id),
+        org_id=_orm_id_to_str(completion_model_orm.org_id),
         parameters=completion_model_orm.parameters,
         secret_api_key=completion_model_orm.secret_api_key,
         created_at=timestamp_orm_to_proto(completion_model_orm.created_at),
@@ -339,7 +344,6 @@ def _ensure_id(
 def pipeline_proto_to_orm(  # noqa: C901
     proto_obj: models_pb2.Pipeline, session: sa_orm.Session
 ) -> Ok[orm.Pipeline] | orm.InvalidORMIdentifierError | InvalidArgumentError:
-    inputs = list[orm.PipelineInput]()
     orm_obj = orm.Pipeline(
         name=proto_obj.name,
         transformation=proto_obj.pipeline_transformation,
@@ -375,9 +379,9 @@ def pipeline_proto_to_orm(  # noqa: C901
 
     if proto_obj.org_id:
         org_id = orm.OrgID(proto_obj.org_id)
-        for obj in flatten((inputs, outputs)):
+        for obj in outputs:
             obj.org_id = org_id
-    for obj in flatten((inputs, outputs)):
+    for obj in outputs:
         session.merge(obj)
     return Ok(orm_obj)
 
@@ -493,8 +497,8 @@ def _feature_view_source_proto_to_orm(
         case Ok(source_id):
             pass
 
+    proto_obj.room_id = proto_obj.room_id or str(room_id)
     orm_obj = orm.FeatureViewSource(
-        room_id=room_id,
         table_op_graph=proto_obj.table_op_graph,
         drop_disconnected=proto_obj.drop_disconnected,
         source_id=source_id,

{corvic_engine-0.3.0rc72 → corvic_engine-0.3.0rc74}/python/corvic/orm/base.py

@@ -134,7 +134,11 @@ class Base(sa_orm.MappedAsDataclass, sa_orm.DeclarativeBase):
     }
 
     _created_at: sa_orm.Mapped[datetime] = sa_orm.mapped_column(
-        "created_at", sa.DateTime(timezone=True), server_default=utc_now(), init=False
+        "created_at",
+        sa.DateTime(timezone=True),
+        server_default=utc_now(),
+        init=False,
+        index=True,
     )
 
     _updated_at: sa_orm.Mapped[datetime] = sa_orm.mapped_column(

{corvic_engine-0.3.0rc72 → corvic_engine-0.3.0rc74}/python/corvic/system/_dimension_reduction.py

@@ -1,3 +1,4 @@
+import sys
 from typing import Any, Protocol
 
 import numpy as np
@@ -71,6 +72,7 @@ class UmapDimensionReducer(DimensionReducer):
             init=init,
             low_memory=False,
             verbose=False,
+            tqdm_kwds={"file": sys.stdout},
         )
         return Ok(projector.fit_transform(vectors))
 

{corvic_engine-0.3.0rc72 → corvic_engine-0.3.0rc74}/python/corvic/system/in_memory_executor.py

@@ -775,8 +775,9 @@ class InMemoryExecutor(OpGraphExecutor):
         right_lf = right_lf.select(
             [
                 col
-                for col in right_lf.columns
-                if col in op.right_join_columns or col not in left_lf.columns
+                for col in right_lf.collect_schema().names()
+                if col in op.right_join_columns
+                or col not in left_lf.collect_schema().names()
             ]
         )
         metrics = right_lfm.metrics.copy()