corvic-engine 0.3.0rc95__tar.gz → 0.3.0rc97__tar.gz

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/Cargo.lock

@@ -16,7 +16,7 @@ checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
 
 [[package]]
 name = "corvic-engine"
-version = "0.3.0-rc.95"
+version = "0.3.0-rc.97"
 dependencies = [
  "numpy",
  "pyo3",
@@ -57,7 +57,7 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
 [[package]]
 name = "engine-walk"
-version = "0.3.0-rc.95"
+version = "0.3.0-rc.97"
 dependencies = [
  "version_check",
 ]

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["python"]
 
 [workspace.package]
-version = "0.3.0-rc.95"
+version = "0.3.0-rc.97"
 edition = "2021"
 publish = false
 

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corvic-engine
-Version: 0.3.0rc95
+Version: 0.3.0rc97
 Classifier: Environment :: Console
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: Implementation :: CPython
@@ -10,16 +10,16 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Rust
 Classifier: Topic :: Scientific/Engineering
-Requires-Dist: cachetools>=5
+Requires-Dist: cachetools>=6
 Requires-Dist: duckdb>=1.0.0
 Requires-Dist: more-itertools>=10
 Requires-Dist: numpy>=1.26
 Requires-Dist: polars>=1.7.1
 Requires-Dist: protobuf>=4.25
 Requires-Dist: protovalidate>=0.3
-Requires-Dist: pyarrow>=17
+Requires-Dist: pyarrow>=20
 Requires-Dist: sqlalchemy>=2
-Requires-Dist: sqlglot>=25.6.0,<26
+Requires-Dist: sqlglot>=26,<27
 Requires-Dist: structlog>=24
 Requires-Dist: tqdm
 Requires-Dist: umap-learn>=0.5.5 ; extra == 'ml'

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/pyproject.toml

@@ -18,7 +18,7 @@ authors = [
 ]
 readme = "README.md"
 description = "Seamless embedding generation and retrieval."
-version = "0.3.0-rc.95" # sync this with tool.poetry.version below
+version = "0.3.0-rc.97" # sync this with tool.poetry.version below
 
 # TODO(ddn): Pull dependencies from tool.poetry.dependencies. We use maturin as
 # the build system but poetry for the development environment.
@@ -29,16 +29,16 @@ version = "0.3.0-rc.95" # sync this with tool.poetry.version below
 # don't overimpose on users: either ">= x" or ">= 0.x" for semvar packages or
 # minor versions for notable semvar exceptions: protobuf, numpy
 dependencies = [
-  "cachetools >= 5",
+  "cachetools >= 6",
   "duckdb >= 1.0.0",
   "more-itertools >= 10",
   "numpy >= 1.26",
   "polars >= 1.7.1",
   "protobuf >= 4.25",
   "protovalidate >= 0.3",
-  "pyarrow >= 17",
+  "pyarrow >= 20",
   "sqlalchemy >= 2",
-  "sqlglot >= 25.6.0, < 26",
+  "sqlglot >= 26, < 27",
   "structlog >= 24",
   "tqdm",
 ]
@@ -73,7 +73,7 @@ packages = [
 ]
 description = ""
 authors = []
-version = "0.3.0-rc.95"
+version = "0.3.0-rc.97"
 
 [tool.poetry.scripts]
 check = "corvic_check.cli:main"
@@ -90,22 +90,24 @@ name = "pypi"
 priority = "primary"
 
 [tool.poetry.dependencies]
-cachetools = "^5.3.0"
+cachetools = "^6.1.0"
 duckdb = "1.1.3"
 more-itertools = "^10.2.0"
+# Numpy 2.0 is blocked by gensim
+# https://github.com/piskvorky/gensim/issues/3560
 numpy = "^1.26.3"
 polars = "^1.25.2"
 # buf (which we use to generate protobuf code) doesn't yet have a plugin
 # that supports patch versions greater than 8.
 protobuf = "=4.25.8"
 protovalidate = "^0.7.1"
-pyarrow = "^17"
+pyarrow = "^20"
 # Should match build matrix for any native code dependencies (namely pyarrow)
 python = ">=3.11,<3.13"
-sqlalchemy = "2.0.36"
+sqlalchemy = "2.0.41"
 # Rust tokenizer for better performances
-sqlglot = { extras = ["rs"], version = "^25.8.1" }
-structlog = "^24.1.0"
+sqlglot = { extras = ["rs"], version = "^26.33.0" }
+structlog = "^25.4.0"
 tqdm = "^4.66.3"
 
 [tool.poetry.group.dev.dependencies]
@@ -118,12 +120,12 @@ coverage = "^7.3.1"
 grpcio = "1.62.1"
 import-linter = "^2.0"
 maturin = "^1.4.0"
-pdoc = "^14.5.1"
+pdoc = "^15"
 pre-commit = "^4.0.0"
 psycopg = { extras = ["binary"], version = "^3.2" }
-pyright = "1.1.400"
+pyright = "1.1.403"
 pytest = "^8.1.1"
-pytest-benchmark = "^4.0.0"
+pytest-benchmark = "^5"
 pytest-fail-slow = "^0.6.0"
 pytest-socket = "^0.7.0"
 pytest-timeout = "^2.2.0"
@@ -132,21 +134,19 @@ hypothesis = "^6.135"
 yq = "^3.4.3" # includes xq for parsing xml test results
 
 # Required to generate stubs for pyarrow, but not used by corvic itself
-pyarrow-stubs = "^17"
-fsspec = "^2024"
-pandas = "^2"
+pyarrow-stubs = "^20.0.0.20250627"
 
 [tool.poetry.group.docs.dependencies]
-sphinx-autodoc-typehints = "^2.4"
+sphinx-autodoc-typehints = "^3.2"
 sphinx-rtd-theme = "^3.0"
 sphinx = "^8.0.2"
-pydata-sphinx-theme = "^0.15.2"
+pydata-sphinx-theme = "^0.16.1"
 
 [tool.poetry.group.check-license]
 optional = true
 
 [tool.poetry.group.check-license.dependencies]
-pip-licenses = "^4.3.3"
+pip-licenses-cli = "^1.2.0"
 
 [tool.poetry.group.test.dependencies]
 fastnode2vec = "^0.0.7"

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/python/corvic/emodel/_space.py

@@ -10,8 +10,8 @@ from collections.abc import Iterable, Mapping, Sequence
 from typing import Final, Literal, Self, TypeAlias, cast
 
 import pyarrow as pa
-import sqlalchemy as sa
 from sqlalchemy import orm as sa_orm
+from sqlalchemy.orm.interfaces import LoaderOption
 
 from corvic import eorm, op_graph, system
 from corvic.emodel._base_model import StandardModel
@@ -149,7 +149,7 @@ class Space(StandardModel[SpaceID, models_pb2.Space, eorm.Space]):
         return space_delete_orms(ids, session)
 
     @classmethod
-    def orm_load_options(cls) -> list[sa.LoaderOption]:
+    def orm_load_options(cls) -> list[LoaderOption]:
         return [
             sa_orm.selectinload(eorm.Space.feature_view)
             .selectinload(eorm.FeatureView.feature_view_sources)

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/python/corvic/op_graph/_schema.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+from typing import cast
+
 import polars as pl
 import pyarrow as pa
 
@@ -70,6 +72,11 @@ def _upgrade_to_polars_dtype(dtype: pa.DataType) -> pa.DataType:
         # polars does not support float16 unless explicitly enabled
         return pa.float32()
     if isinstance(dtype, pa.ListType | pa.LargeListType | pa.FixedSizeListType):
+        dtype = cast(
+            "pa.ListType[pa.DataType] | pa.LargeListType[pa.DataType] \
+                | pa.FixedSizeListType[pa.DataType]",
+            dtype,
+        )
         # modify large lists as well since its value datatype
         # may need to be updated
         return pa.large_list(

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/python/corvic/op_graph/ops.py

@@ -695,6 +695,11 @@ def _validate_embedding_column(
         )
     dtype = embedding_field.dtype
     if isinstance(dtype, pa.ListType | pa.LargeListType | pa.FixedSizeListType):
+        dtype = cast(
+            "pa.ListType[pa.DataType] | pa.LargeListType[pa.DataType] \
+                | pa.FixedSizeListType[pa.DataType]",
+            dtype,
+        )
         inner_dtype = dtype.value_field.type
         if not (
             pa.types.is_floating(inner_dtype)
@@ -1615,14 +1620,21 @@ class _Base(OneofProtoWrapper[table_pb2.TableComputeOp], ABC):
     ) -> Ok[OutputCsv] | InvalidArgumentError:
         # Some execution engines don't support nested dtypes
         for field in self.schema:
+            dtype = field.dtype
             if isinstance(
-                field.dtype,
+                dtype,
                 pa.StructType | pa.ListType | pa.LargeListType | pa.FixedSizeListType,
             ):
+                dtype = cast(
+                    "pa.StructType | pa.ListType[pa.DataType] \
+                        | pa.LargeListType[pa.DataType] \
+                        | pa.FixedSizeListType[pa.DataType]",
+                    dtype,
+                )
                 return InvalidArgumentError(
                     "nested fields are not supported",
                     name=field.name,
-                    dtype=str(field.dtype),
+                    dtype=str(dtype),
                 )
         return Ok(
             from_proto(
@@ -1653,6 +1665,11 @@ class _Base(OneofProtoWrapper[table_pb2.TableComputeOp], ABC):
             return InvalidArgumentError(
                 "given column must be a list", column_name=list_column_name
             )
+        column_type = cast(
+            "pa.ListType[pa.DataType] | pa.LargeListType[pa.DataType] \
+                | pa.FixedSizeListType[pa.DataType]",
+            column_type,
+        )
         elem_type = _list_elem_type(column_type)
         if target_list_length < 1:
             return InvalidArgumentError(
@@ -3196,13 +3213,19 @@ def _make_schema_for_unnest_list(op: UnnestList):
     def gen_fields() -> Iterable[Field]:
         for field in schema:
             if field.name == op.list_column_name:
-                if not isinstance(field.dtype, pa.LargeListType):
+                dtype = field.dtype
+                if not isinstance(dtype, pa.LargeListType):
                     raise InvalidArgumentError(
                         "unnest cannot be done on a non-list column"
                     )
 
+                dtype = cast(
+                    "pa.LargeListType[pa.DataType]",
+                    dtype,
+                )
+
                 yield from (
-                    Field(column_name, dtype=field.dtype.value_type, ftype=field.ftype)
+                    Field(column_name, dtype=dtype.value_type, ftype=field.ftype)
                     for column_name in op.column_names
                 )
             else:

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/python/corvic/orm/_soft_delete.py

@@ -7,6 +7,9 @@ from typing import Any, LiteralString
 import sqlalchemy as sa
 from sqlalchemy import event, exc
 from sqlalchemy import orm as sa_orm
+from sqlalchemy.exc import (
+    DBAPIError,
+)
 from sqlalchemy.ext import hybrid
 from sqlalchemy.ext.hybrid import hybrid_property
 
@@ -161,7 +164,7 @@ class Session(sa_orm.Session):
         traceback: TracebackType | None,
     ):
         super().__exit__(type_, value, traceback)
-        if isinstance(value, sa.exc.DBAPIError):
+        if isinstance(value, DBAPIError):
             raise dbapi_error_to_result(value) from value
 
     def _track_soft_deleted(self, instance: object):

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/python/corvic/orm/errors.py

@@ -1,6 +1,14 @@
 """Errors specific to communicating with the database."""
 
-import sqlalchemy as sa
+from sqlalchemy.exc import (
+    DBAPIError,
+    IntegrityError,
+    InterfaceError,
+    InternalError,
+    NotSupportedError,
+    OperationalError,
+    ProgrammingError,
+)
 
 from corvic import result
 
@@ -20,24 +28,20 @@ class DeletedObjectError(result.Error):
     """
 
 
-def dbapi_error_to_result(err: sa.exc.DBAPIError):
+def dbapi_error_to_result(err: DBAPIError):
     # based on https://docs.sqlalchemy.org/en/20/errors.html
     match err:
-        case sa.exc.NotSupportedError():
+        case NotSupportedError():
             # raised in the unexpected case that we're doing something that the
             # database just doesn't support
             raise result.InternalError.from_(err) from err
-        case (
-            sa.exc.OperationalError()
-            | sa.exc.InterfaceError()
-            | sa.exc.InternalError()
-        ):
+        case OperationalError() | InterfaceError() | InternalError():
             # These are commonly things that are outside of our control that might
             # succeed on retry, e.g., connections being dropped
             return result.UnavailableError.from_(err)
-        case sa.exc.IntegrityError():
+        case IntegrityError():
             return result.InvalidArgumentError.from_(err)
-        case sa.exc.ProgrammingError():
+        case ProgrammingError():
             if "could not serialize" in str(err):
                 return result.UnavailableError.from_(err)
         case _:

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/python/corvic/pa_scalar/_from_value.py

@@ -3,7 +3,7 @@ import decimal
 from abc import abstractmethod
 from collections.abc import Callable
 from datetime import date
-from typing import Final, Literal, TypeAlias, cast, overload
+from typing import Any, Final, Literal, TypeAlias, cast, overload
 
 import pyarrow as pa
 from google.protobuf import struct_pb2
@@ -584,7 +584,7 @@ def _visit_large_list(
 def _visit_dictionary(
     value: struct_pb2.Value, dtype: pa.DataType, error_handler: _ErrorHandler
 ) -> object:
-    dtype = cast(pa.DictionaryType, dtype)
+    dtype = cast("pa.DictionaryType[Any, Any, Any]", dtype)
     return _visit(value, dtype.value_type, error_handler)
 
 
@@ -782,4 +782,4 @@ def from_value(
     except (pa.ArrowTypeError, pa.ArrowInvalid, OverflowError) as exc:
         if errors == "strict":
             return InvalidArgumentError(_extract_error_message(exc))
-        return Ok(pa.NA)
+        return Ok(pa.NullScalar())

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/python/corvic/pa_scalar/_to_value.py

@@ -322,7 +322,7 @@ def _visit_dictionary(
     value: _PyValue,
     dtype: pa.DataType,
 ) -> struct_pb2.Value:
-    dtype = cast(pa.DictionaryType, dtype)
+    dtype = cast("pa.DictionaryType[Any, Any, Any]", dtype)
     return _visit(value, dtype.value_type)
 
 

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/python/corvic/system_sqlite/client.py

@@ -11,6 +11,7 @@ from typing import Protocol
 
 import duckdb
 import sqlalchemy as sa
+from sqlalchemy import event
 
 import corvic.context
 import corvic.eorm
@@ -40,7 +41,7 @@ def _context_requester_org_is_superuser() -> Iterator[None]:
         yield
 
 
-@sa.event.listens_for(sa.Engine, "connect")
+@event.listens_for(sa.Engine, "connect")
 def set_sqlite_pragma(dbapi_connection: sqlite3.Connection | None, _) -> None:
     """Tell sqlite to respect foreign key constraints.
 

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/python/corvic/system_sqlite/staging.py

@@ -3,7 +3,7 @@
 import functools
 import uuid
 from collections.abc import Callable, Iterable, Mapping
-from typing import Final
+from typing import Final, cast
 
 import duckdb
 import pyarrow as pa
@@ -47,7 +47,9 @@ def _patch_schema_for_storage(
             patched_schema.append(_patch_list_field(field, old_field))
         else:
             patched_schema.append(field)
-    return pa.schema(patched_schema, new_schema.metadata)
+    schema_metadata = new_schema.metadata
+    # dict[bytes, bytes] should be valid for a dict[bytes | str, bytes | str]
+    return pa.schema(patched_schema, schema_metadata)  # pyright: ignore[reportArgumentType]
 
 
 def _wrap_list_type(
@@ -79,6 +81,17 @@ def _patch_list_field(
         )
     ):
         return new_field
+    new_field_type = cast(
+        "pa.ListType[pa.DataType] | pa.LargeListType[pa.DataType] \
+            | pa.FixedSizeListType[pa.DataType]",
+        new_field_type,
+    )
+    old_field_type = cast(
+        "pa.ListType[pa.DataType] | pa.LargeListType[pa.DataType] \
+            | pa.FixedSizeListType[pa.DataType]",
+        old_field_type,
+    )
+
     new_list_field = new_field_type.value_field
     old_list_type = old_field_type.value_field.type
 
@@ -449,7 +462,7 @@ class DuckDBStaging(StagingDB):
                 for field in storage_schema:
                     if pa.types.is_null(field.type):
                         batch = batch.drop_columns(field.name)
-                        column = pa.nulls(size=len(batch))  # pyright: ignore[reportUnknownMemberType]
+                        column = pa.nulls(size=len(batch))
                         batch = batch.append_column(
                             field.with_type(pa.null()),
                             column,

{corvic_engine-0.3.0rc95 → corvic_engine-0.3.0rc97}/python/corvic/transfer/_orm_backed_proto.py

@@ -10,6 +10,11 @@ import sqlalchemy as sa
 import sqlalchemy.orm as sa_orm
 import structlog
 from google.protobuf import timestamp_pb2
+from sqlalchemy.exc import (
+    DBAPIError,
+    IntegrityError,
+)
+from sqlalchemy.ext.hybrid import hybrid_property
 
 from corvic import orm, system
 from corvic.result import InvalidArgumentError, NotFoundError, Ok, UnavailableError
@@ -23,12 +28,14 @@ _logger = structlog.get_logger()
 
 
 class OrmModel(Protocol):
-    @sa.ext.hybrid.hybrid_property
+    @hybrid_property
     def created_at(self) -> datetime.datetime | None: ...
 
     @created_at.inplace.expression
     @classmethod
-    def _created_at_expression(cls): ...
+    def _created_at_expression(
+        cls,
+    ) -> ...: ...
 
 
 class OrmHasIdModel(OrmModel, Protocol[OrmIdT]):
@@ -174,7 +181,7 @@ class OrmBackedProto(Generic[ProtoT, OrmT], HasProtoSelf[ProtoT]):
                     self.proto_self, session
                 ).unwrap_or_raise()
                 session.commit()
-            except sa.exc.DBAPIError as err:
+            except DBAPIError as err:
                 return orm.dbapi_error_to_result(err)
             return Ok(
                 self.__class__(
@@ -196,7 +203,7 @@ class OrmBackedProto(Generic[ProtoT, OrmT], HasProtoSelf[ProtoT]):
             _ = self.proto_to_orm(self.proto_self, session).unwrap_or_raise()
             session.flush()
         # TODO(thunt): Possibly separate out DatabaseError into a precondition error
-        except sa.exc.DBAPIError as err:
+        except DBAPIError as err:
             return orm.dbapi_error_to_result(err)
         return Ok(None)
 
@@ -246,7 +253,7 @@ class HasIdOrmBackedProto(
                     case Ok(None):
                         pass
                 session.commit()
-            except sa.exc.IntegrityError as exc:
+            except IntegrityError as exc:
                 return InvalidArgumentError.from_(exc)
 
             new_proto_self = copy.copy(self.proto_self)