corvic-engine 0.3.0rc82__tar.gz → 0.3.0rc84__tar.gz

{corvic_engine-0.3.0rc82 → corvic_engine-0.3.0rc84}/Cargo.lock

@@ -16,7 +16,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "corvic-engine"
-version = "0.3.0-rc.82"
+version = "0.3.0-rc.84"
 dependencies = [
  "numpy",
  "pyo3",
@@ -57,7 +57,7 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
 [[package]]
 name = "engine-walk"
-version = "0.3.0-rc.82"
+version = "0.3.0-rc.84"
 dependencies = [
  "version_check",
 ]
@@ -165,9 +165,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 
 [[package]]
 name = "portable-atomic"
-version = "1.11.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
+checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
 
 [[package]]
 name = "portable-atomic-util"

{corvic_engine-0.3.0rc82 → corvic_engine-0.3.0rc84}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["python"]
 
 [workspace.package]
-version = "0.3.0-rc.82"
+version = "0.3.0-rc.84"
 edition = "2021"
 publish = false
 

{corvic_engine-0.3.0rc82 → corvic_engine-0.3.0rc84}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corvic-engine
-Version: 0.3.0rc82
+Version: 0.3.0rc84
 Classifier: Environment :: Console
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: Implementation :: CPython
@@ -22,12 +22,12 @@ Requires-Dist: sqlalchemy>=2
 Requires-Dist: sqlglot>=25.6.0,<26
 Requires-Dist: structlog>=24
 Requires-Dist: tqdm
-Requires-Dist: typing-extensions>=4.9
 Requires-Dist: umap-learn>=0.5.5 ; extra == 'ml'
 Requires-Dist: pillow>=10.0.0 ; extra == 'ml'
 Requires-Dist: scikit-learn>=1.4.0 ; extra == 'ml'
 Requires-Dist: transformers[torch]>=4.45.0 ; extra == 'ml'
 Requires-Dist: sentencepiece>=0.2.0 ; extra == 'ml'
+Requires-Dist: torchvision==0.22.1 ; extra == 'ml'
 Requires-Dist: opentelemetry-api>=1.20.0 ; extra == 'telemetry'
 Requires-Dist: opentelemetry-sdk>=1.20.0 ; extra == 'telemetry'
 Provides-Extra: ml

{corvic_engine-0.3.0rc82 → corvic_engine-0.3.0rc84}/pyproject.toml

@@ -18,7 +18,7 @@ authors = [
 ]
 readme = "README.md"
 description = "Seamless embedding generation and retrieval."
-version = "0.3.0-rc.82" # sync this with tool.poetry.version below
+version = "0.3.0-rc.84" # sync this with tool.poetry.version below
 
 # TODO(ddn): Pull dependencies from tool.poetry.dependencies. We use maturin as
 # the build system but poetry for the development environment.
@@ -41,7 +41,6 @@ dependencies = [
   "sqlglot >= 25.6.0, < 26",
   "structlog >= 24",
   "tqdm",
-  "typing-extensions >= 4.9",
 ]
 
 [project.optional-dependencies]
@@ -53,6 +52,7 @@ ml = [
   "scikit-learn >= 1.4.0",
   "transformers[torch] >= 4.45.0", # Needed for pre-trained torch models
   "sentencepiece >= 0.2.0", # Needed for SigLIP2
+  "torchvision == 0.22.1",
 ]
 
 telemetry = [
@@ -73,7 +73,7 @@ packages = [
 ]
 description = ""
 authors = []
-version = "0.3.0-rc.82"
+version = "0.3.0-rc.84"
 
 [tool.poetry.scripts]
 check = "corvic_check.cli:main"
@@ -107,8 +107,6 @@ sqlalchemy = "2.0.36"
 sqlglot = { extras = ["rs"], version = "^25.8.1" }
 structlog = "^24.1.0"
 tqdm = "^4.66.3"
-types-cachetools = "^5.3.0"
-typing-extensions = "^4.9.0"
 
 [tool.poetry.group.dev.dependencies]
 anyio = "^4"
@@ -160,9 +158,10 @@ scikit-learn = "^1.4.0"
 optional = true
 
 [tool.poetry.group.ml.dependencies]
+sentencepiece = "^0.2.0"
 torch = [
-  { platform = "linux", version = "^2.7.0", source = "torch-cpu-linux" },
-  { platform = "darwin", version = "^2.7.0", source = "pypi" },
+  { platform = "linux", version = "^2.7.1", source = "torch-cpu-linux" },
+  { platform = "darwin", version = "^2.7.1", source = "pypi" },
 ]
 torchvision = [
   { platform = "linux", version = "^0.22.1", source = "torch-cpu-linux" },
@@ -409,9 +408,9 @@ root_packages = [
   "corvic.context",
   "corvic.embed",
   "corvic.embedding_metric",
+  "corvic.emodel",
   "corvic.engine",
   "corvic.eorm",
-  "corvic.model",
   "corvic.op_graph",
   "corvic.orm",
   "corvic.pa_scalar",
@@ -421,6 +420,7 @@ root_packages = [
   "corvic.system",
   "corvic.system_sqlite",
   "corvic.table",
+  "corvic.transfer",
   "corvic.version",
   "corvic.well_known_types",
   "corvic_check",

{corvic_engine-0.3.0rc82/python/corvic/model → corvic_engine-0.3.0rc84/python/corvic/emodel}/__init__.py

@@ -1,26 +1,34 @@
 """Data modeling objects for creating corvic pipelines."""
 
-import corvic.model._feature_type as feature_type
-from corvic.model._base_model import (
-    BaseModel,
-    BelongsToOrgModel,
-    BelongsToRoomModel,
-    HasProtoSelf,
-    UsesOrmID,
-    non_empty_timestamp_to_datetime,
+import corvic.emodel._feature_type as feature_type
+from corvic import transfer
+from corvic.emodel._base_model import (
+    BelongsToOrgModelMixin,
+    BelongsToRoomModelMixin,
+    HasIdOrmBackedModel,
+    NoIdModel,
+    OrgWideStandardModel,
+    OrmBelongsToRoomT,
+    OrmHasIdBelongsToOrgT,
+    OrmHasIdBelongsToRoomT,
+    ProtoBelongsToOrgT,
+    ProtoBelongsToRoomT,
+    ProtoHasIdBelongsToOrgT,
+    ProtoHasIdBelongsToRoomT,
+    StandardModel,
 )
-from corvic.model._completion_model import (
+from corvic.emodel._completion_model import (
     CompletionModel,
     CompletionModelID,
 )
-from corvic.model._feature_view import (
+from corvic.emodel._feature_view import (
     Column,
     DeepGnnCsvUrlMetadata,
     FeatureView,
     FeatureViewEdgeTableMetadata,
     FeatureViewRelationshipsMetadata,
 )
-from corvic.model._pipeline import (
+from corvic.emodel._pipeline import (
     ChunkPdfsPipeline,
     OcrPdfsPipeline,
     Pipeline,
@@ -29,29 +37,22 @@ from corvic.model._pipeline import (
     SpecificPipeline,
     UnknownTransformationPipeline,
 )
-from corvic.model._proto_orm_convert import (
-    UNCOMMITTED_ID_PREFIX,
-    OrmBelongsToOrgObj,
-    OrmBelongsToRoomObj,
-    OrmObj,
-    ProtoBelongsToOrgObj,
-    ProtoBelongsToRoomObj,
-    ProtoObj,
+from corvic.emodel._proto_orm_convert import (
     add_orm_org_mixin_to_session,
     add_orm_room_mixin_to_session,
     space_orm_to_proto,
     timestamp_orm_to_proto,
 )
-from corvic.model._resource import (
+from corvic.emodel._resource import (
     Resource,
     ResourceID,
 )
-from corvic.model._room import (
+from corvic.emodel._room import (
     Room,
     RoomID,
 )
-from corvic.model._source import Source, SourceID
-from corvic.model._space import (
+from corvic.emodel._source import Source, SourceID
+from corvic.emodel._space import (
     ConcatAndEmbedParameters,
     EmbedAndConcatParameters,
     EmbedImageParameters,
@@ -71,11 +72,11 @@ from corvic.model._space import (
 )
 
 FeatureType = feature_type.FeatureType
+OrmIdT = transfer.OrmIdT
 
 __all__ = [
-    "BaseModel",
-    "BelongsToOrgModel",
-    "BelongsToRoomModel",
+    "BelongsToOrgModelMixin",
+    "BelongsToRoomModelMixin",
     "ChunkPdfsPipeline",
     "Column",
     "CompletionModel",
@@ -88,18 +89,22 @@ __all__ = [
     "FeatureView",
     "FeatureViewEdgeTableMetadata",
     "FeatureViewRelationshipsMetadata",
-    "HasProtoSelf",
+    "HasIdOrmBackedModel",
     "ImageSpace",
+    "NoIdModel",
     "Node2VecParameters",
     "OcrPdfsPipeline",
-    "OrmObj",
-    "OrmBelongsToOrgObj",
-    "OrmBelongsToRoomObj",
+    "OrgWideStandardModel",
+    "OrmBelongsToRoomT",
+    "OrmHasIdBelongsToOrgT",
+    "OrmHasIdBelongsToRoomT",
+    "OrmIdT",
     "Pipeline",
     "PipelineID",
-    "ProtoObj",
-    "ProtoBelongsToOrgObj",
-    "ProtoBelongsToRoomObj",
+    "ProtoBelongsToOrgT",
+    "ProtoBelongsToRoomT",
+    "ProtoHasIdBelongsToOrgT",
+    "ProtoHasIdBelongsToRoomT",
     "RelationalSpace",
     "Resource",
     "ResourceID",
@@ -113,11 +118,10 @@ __all__ = [
     "SpecificPipeline",
     "SpecificSpace",
     "SpecificSpaceParameters",
+    "StandardModel",
     "TabularSpace",
-    "UNCOMMITTED_ID_PREFIX",
-    "UnknownTransformationPipeline",
     "UnknownSpace",
-    "UsesOrmID",
+    "UnknownTransformationPipeline",
     "add_orm_org_mixin_to_session",
     "add_orm_room_mixin_to_session",
     "embedding_model_proto_to_name",
@@ -125,7 +129,6 @@ __all__ = [
     "image_model_can_embed_images",
     "image_model_can_embed_text",
     "image_model_proto_to_name",
-    "non_empty_timestamp_to_datetime",
     "space_orm_to_proto",
     "timestamp_orm_to_proto",
 ]

corvic_engine-0.3.0rc84/python/corvic/emodel/_base_model.py

@@ -0,0 +1,161 @@
+import datetime
+from collections.abc import Callable, Iterable
+from typing import Generic
+
+import sqlalchemy as sa
+import sqlalchemy.orm as sa_orm
+
+from corvic import eorm, system, transfer
+from corvic.emodel._proto_orm_convert import (
+    OrmBelongsToRoomT,
+    OrmHasIdBelongsToOrgT,
+    OrmHasIdBelongsToRoomT,
+    ProtoBelongsToOrgT,
+    ProtoBelongsToRoomT,
+    ProtoHasIdBelongsToOrgT,
+    ProtoHasIdBelongsToRoomT,
+)
+from corvic.result import (
+    InvalidArgumentError,
+    NotFoundError,
+    Ok,
+)
+
+
+def _make_room_filter_query_transfrom(
+    room_id: eorm.RoomID | None,
+    existing_transform: Callable[
+        [sa.Select[tuple[transfer.OrmT]]], sa.Select[tuple[transfer.OrmT]]
+    ]
+    | None,
+):
+    def query_transform(
+        query: sa.Select[tuple[transfer.OrmT]],
+    ) -> sa.Select[tuple[transfer.OrmT]]:
+        if room_id:
+            query = query.filter_by(room_id=room_id)
+        if existing_transform:
+            query = existing_transform(query)
+        return query
+
+    if room_id:
+        return query_transform
+    return existing_transform
+
+
+class OrmBackedModel(
+    Generic[transfer.ProtoT, transfer.OrmT],
+    transfer.OrmBackedProto[transfer.ProtoT, transfer.OrmT],
+):
+    @classmethod
+    def list_as_proto(
+        cls,
+        client: system.Client,
+        *,
+        limit: int | None = None,
+        room_id: eorm.RoomID | None = None,
+        created_before: datetime.datetime | None = None,
+        additional_query_transform: Callable[
+            [sa.Select[tuple[transfer.OrmT]]], sa.Select[tuple[transfer.OrmT]]
+        ]
+        | None = None,
+        existing_session: sa_orm.Session | None = None,
+    ) -> Ok[list[transfer.ProtoT]] | NotFoundError | InvalidArgumentError:
+        return super().list_as_proto(
+            client,
+            limit=limit,
+            created_before=created_before,
+            additional_query_transform=_make_room_filter_query_transfrom(
+                room_id, additional_query_transform
+            ),
+            existing_session=existing_session,
+        )
+
+
+class HasIdOrmBackedModel(
+    Generic[transfer.OrmIdT, transfer.ProtoHasIdT, transfer.OrmHasIdT],
+    transfer.HasIdOrmBackedProto[
+        transfer.OrmIdT, transfer.ProtoHasIdT, transfer.OrmHasIdT
+    ],
+):
+    @classmethod
+    def list_as_proto(
+        cls,
+        client: system.Client,
+        *,
+        limit: int | None = None,
+        room_id: eorm.RoomID | None = None,
+        created_before: datetime.datetime | None = None,
+        ids: Iterable[transfer.OrmIdT] | None = None,
+        additional_query_transform: Callable[
+            [sa.Select[tuple[transfer.OrmHasIdT]]], sa.Select[tuple[transfer.OrmHasIdT]]
+        ]
+        | None = None,
+        existing_session: sa_orm.Session | None = None,
+    ) -> Ok[list[transfer.ProtoHasIdT]] | NotFoundError | InvalidArgumentError:
+        return super().list_as_proto(
+            client,
+            limit=limit,
+            created_before=created_before,
+            ids=ids,
+            additional_query_transform=_make_room_filter_query_transfrom(
+                room_id, additional_query_transform
+            ),
+            existing_session=existing_session,
+        )
+
+
+class BelongsToOrgModelMixin(
+    Generic[ProtoBelongsToOrgT], transfer.HasProtoSelf[ProtoBelongsToOrgT]
+):
+    """Base for orm wrappers with org mixin providing a unified update mechanism."""
+
+    @property
+    def org_id(self) -> eorm.OrgID:
+        return eorm.OrgID().from_str(self.proto_self.org_id)
+
+
+class BelongsToRoomModelMixin(
+    Generic[ProtoBelongsToRoomT], transfer.HasProtoSelf[ProtoBelongsToRoomT]
+):
+    """Base for orm wrappers with room mixin providing a unified update mechanism."""
+
+    @property
+    def room_id(self) -> eorm.RoomID:
+        return eorm.RoomID().from_str(self.proto_self.room_id)
+
+
+class OrgWideStandardModel(
+    Generic[transfer.OrmIdT, ProtoHasIdBelongsToOrgT, OrmHasIdBelongsToOrgT],
+    BelongsToOrgModelMixin[ProtoHasIdBelongsToOrgT],
+    HasIdOrmBackedModel[
+        transfer.OrmIdT, ProtoHasIdBelongsToOrgT, OrmHasIdBelongsToOrgT
+    ],
+):
+    """Base for most models, though the lion's share are StandardModels.
+
+    Most models are typically referenced by an ID, and belong to an org.
+    """
+
+
+class StandardModel(
+    Generic[transfer.OrmIdT, ProtoHasIdBelongsToRoomT, OrmHasIdBelongsToRoomT],
+    BelongsToRoomModelMixin[ProtoHasIdBelongsToRoomT],
+    OrgWideStandardModel[
+        transfer.OrmIdT, ProtoHasIdBelongsToRoomT, OrmHasIdBelongsToRoomT
+    ],
+):
+    """Base for most models.
+
+    Like OrgWideStandardModel, but more specific. The majority of models belong to a
+    room.
+    """
+
+
+class NoIdModel(
+    Generic[ProtoBelongsToRoomT, OrmBelongsToRoomT],
+    BelongsToRoomModelMixin[ProtoBelongsToRoomT],
+    BelongsToOrgModelMixin[ProtoBelongsToRoomT],
+    transfer.OrmBackedProto[ProtoBelongsToRoomT, OrmBelongsToRoomT],
+):
+    """Like StandardModel, but for objects that are not usually referenced by ID."""

{corvic_engine-0.3.0rc82/python/corvic/model → corvic_engine-0.3.0rc84/python/corvic/emodel}/_completion_model.py

@@ -9,10 +9,10 @@ from typing import Literal, TypeAlias
 
 from sqlalchemy import orm as sa_orm
 
-from corvic import eorm, system
-from corvic.model._base_model import BelongsToOrgModel, non_empty_timestamp_to_datetime
-from corvic.model._defaults import Defaults
-from corvic.model._proto_orm_convert import (
+from corvic import eorm, system, transfer
+from corvic.emodel._base_model import OrgWideStandardModel
+from corvic.emodel._defaults import Defaults
+from corvic.emodel._proto_orm_convert import (
     completion_model_delete_orms,
     completion_model_orm_to_proto,
     completion_model_proto_to_orm,
@@ -26,9 +26,9 @@ OrgID: TypeAlias = eorm.OrgID
 
 
 class CompletionModel(
-    BelongsToOrgModel[
+    OrgWideStandardModel[
         CompletionModelID, models_pb2.CompletionModel, eorm.CompletionModel
-    ]
+    ],
 ):
     """Completion Models."""
 
@@ -112,11 +112,13 @@ class CompletionModel(
 
     @property
     def last_validation_time(self) -> datetime.datetime | None:
-        return non_empty_timestamp_to_datetime(self.proto_self.last_validation_time)
+        return transfer.non_empty_timestamp_to_datetime(
+            self.proto_self.last_validation_time
+        )
 
     @property
     def last_successful_validation(self) -> datetime.datetime | None:
-        return non_empty_timestamp_to_datetime(
+        return transfer.non_empty_timestamp_to_datetime(
             self.proto_self.last_successful_validation
         )
 

{corvic_engine-0.3.0rc82/python/corvic/model → corvic_engine-0.3.0rc84/python/corvic/emodel}/_feature_type.py

@@ -1,4 +1,4 @@
-import corvic.model._source as source
+import corvic.emodel._source as source
 from corvic.op_graph import feature_type
 
 # types

{corvic_engine-0.3.0rc82/python/corvic/model → corvic_engine-0.3.0rc84/python/corvic/emodel}/_feature_view.py

@@ -16,15 +16,15 @@ from more_itertools import flatten
 from sqlalchemy import orm as sa_orm
 from sqlalchemy.orm.interfaces import LoaderOption
 
-from corvic import eorm, op_graph, system
-from corvic.model._base_model import BelongsToRoomModel, UsesOrmID
-from corvic.model._defaults import Defaults
-from corvic.model._proto_orm_convert import (
+from corvic import eorm, op_graph, system, transfer
+from corvic.emodel._base_model import StandardModel
+from corvic.emodel._defaults import Defaults
+from corvic.emodel._proto_orm_convert import (
     feature_view_delete_orms,
     feature_view_orm_to_proto,
     feature_view_proto_to_orm,
 )
-from corvic.model._source import Source, SourceID
+from corvic.emodel._source import Source, SourceID
 from corvic.result import InvalidArgumentError, NotFoundError, Ok
 from corvic.table import (
     DataclassAsTypedMetadataMixin,
@@ -539,7 +539,9 @@ class RelationshipPath:
         return f"end-{uuid.uuid4()}"
 
 
-class FeatureViewSource(UsesOrmID[FeatureViewSourceID, models_pb2.FeatureViewSource]):
+class FeatureViewSource(
+    transfer.UsesOrmID[FeatureViewSourceID, models_pb2.FeatureViewSource]
+):
     """A table from a source with some extra operations defined by a feature view."""
 
     @classmethod
@@ -642,7 +644,7 @@ class DeepGnnCsvUrlMetadata(DataclassAsTypedMetadataMixin):
 
 
 class FeatureView(
-    BelongsToRoomModel[FeatureViewID, models_pb2.FeatureView, eorm.FeatureView]
+    StandardModel[FeatureViewID, models_pb2.FeatureView, eorm.FeatureView]
 ):
     """FeatureViews describe how Sources should be modeled to create a feature space.
 

{corvic_engine-0.3.0rc82/python/corvic/model → corvic_engine-0.3.0rc84/python/corvic/emodel}/_pipeline.py

@@ -14,14 +14,14 @@ from sqlalchemy.orm.interfaces import LoaderOption
 
 import corvic.table
 from corvic import eorm, op_graph, system
-from corvic.model._base_model import BelongsToRoomModel
-from corvic.model._defaults import Defaults
-from corvic.model._proto_orm_convert import (
+from corvic.emodel._base_model import StandardModel
+from corvic.emodel._defaults import Defaults
+from corvic.emodel._proto_orm_convert import (
     pipeline_delete_orms,
     pipeline_orm_to_proto,
     pipeline_proto_to_orm,
 )
-from corvic.model._source import Source
+from corvic.emodel._source import Source
 from corvic.result import InvalidArgumentError, NotFoundError, Ok, UnavailableError
 from corvic_generated.model.v1alpha import models_pb2
 from corvic_generated.orm.v1 import pipeline_pb2
@@ -30,7 +30,7 @@ PipelineID: TypeAlias = eorm.PipelineID
 RoomID: TypeAlias = eorm.RoomID
 
 
-class Pipeline(BelongsToRoomModel[PipelineID, models_pb2.Pipeline, eorm.Pipeline]):
+class Pipeline(StandardModel[PipelineID, models_pb2.Pipeline, eorm.Pipeline]):
     """Pipelines map resources to sources."""
 
     @classmethod