corvic-engine 0.3.0rc56__tar.gz → 0.3.0rc57__tar.gz

{corvic_engine-0.3.0rc56 → corvic_engine-0.3.0rc57}/Cargo.lock

@@ -22,7 +22,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "corvic-engine"
-version = "0.3.0-rc.56"
+version = "0.3.0-rc.57"
 dependencies = [
  "numpy",
  "pyo3",
@@ -63,7 +63,7 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
 [[package]]
 name = "engine-walk"
-version = "0.3.0-rc.56"
+version = "0.3.0-rc.57"
 dependencies = [
  "version_check",
 ]

{corvic_engine-0.3.0rc56 → corvic_engine-0.3.0rc57}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["python"]
 
 [workspace.package]
-version = "0.3.0-rc.56"
+version = "0.3.0-rc.57"
 edition = "2021"
 publish = false
 

{corvic_engine-0.3.0rc56 → corvic_engine-0.3.0rc57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corvic-engine
-Version: 0.3.0rc56
+Version: 0.3.0rc57
 Classifier: Environment :: Console
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: Implementation :: CPython

{corvic_engine-0.3.0rc56 → corvic_engine-0.3.0rc57}/pyproject.toml

@@ -20,7 +20,7 @@ authors = [
 ]
 readme = "README.md"
 description = "Seamless embedding generation and retrieval."
-version = "0.3.0-rc.56" # sync this with tool.poetry.version below
+version = "0.3.0-rc.57" # sync this with tool.poetry.version below
 
 # TODO(ddn): Pull dependencies from tool.poetry.dependencies. We use maturin as
 # the build system but poetry for the development environment.
@@ -74,7 +74,7 @@ packages = [
 ]
 description = ""
 authors = []
-version = "0.3.0-rc.56"
+version = "0.3.0-rc.57"
 
 [tool.poetry.scripts]
 check = "corvic_check.cli:main"

{corvic_engine-0.3.0rc56 → corvic_engine-0.3.0rc57}/python/corvic/model/_space.py

@@ -836,13 +836,20 @@ class TabularSpace(Space):
                     pa.string(),
                 )
 
-            op = (
-                op.concat_list(
+            if len(embedding_column_tmp_names):
+                op = op.concat_list(
                     column_names=embedding_column_tmp_names,
                     concat_list_column_name=embedding_column_tmp_name,
+                ).and_then(reduce_dimension)
+            else:
+                op = op.add_literal_column(
+                    column_name=embedding_column_tmp_name,
+                    literal=[],
+                    dtype=pa.large_list(pa.float32()),
+                    ftype=op_graph.feature_type.embedding(),
                 )
-                .and_then(reduce_dimension)
-                .and_then(select_columns)
+            op = (
+                op.and_then(select_columns)
                 .and_then(update_feature_types)
                 .and_then(rename_columns)
                 .and_then(add_literal_column)

{corvic_engine-0.3.0rc56 → corvic_engine-0.3.0rc57}/python/corvic/op_graph/_schema.py

@@ -2,8 +2,6 @@
 
 from __future__ import annotations
 
-from collections.abc import Sequence
-
 import polars as pl
 import pyarrow as pa
 
@@ -14,7 +12,7 @@ from corvic.op_graph.feature_types import FeatureType
 
 def _infer_possible_feature_types(
     data_type: pa.DataType,
-) -> Sequence[FeatureType]:
+) -> list[FeatureType]:
     if pa.types.is_integer(data_type):
         # TODO(thunt): Add multi-categorical in future versions
         return [
@@ -168,6 +166,6 @@ class Field:
     def rename(self, new_name: str) -> Field:
         return Field(new_name, self.dtype, self.ftype)
 
-    def possible_feature_types(self) -> Sequence[FeatureType]:
+    def possible_feature_types(self) -> list[FeatureType]:
         """Infer possible feature types given the data type."""
         return _infer_possible_feature_types(self.dtype)

{corvic_engine-0.3.0rc56 → corvic_engine-0.3.0rc57}/python/corvic/op_graph/ops.py

@@ -59,7 +59,7 @@ from corvic.op_graph.row_filters import RowFilter
 from corvic.op_graph.row_filters import from_proto as row_filters_from_proto
 from corvic.op_graph.sample_strategy import SampleStrategy
 from corvic.op_graph.sample_strategy import from_proto as sample_strategy_from_proto
-from corvic.pa_scalar import PyValue, from_value, to_value
+from corvic.pa_scalar import PyValue, Scalar, from_value, to_value
 from corvic.proto_wrapper import OneofProtoWrapper
 from corvic.result import InternalError, InvalidArgumentError, Ok
 from corvic_generated.algorithm.graph.v1 import graph_pb2
@@ -1275,7 +1275,7 @@ class _Base(OneofProtoWrapper[table_pb2.TableComputeOp], ABC):
     def add_literal_column(
         self,
         column_name: str,
-        literal: struct_pb2.Value | float | str | bool,
+        literal: struct_pb2.Value | Scalar | PyValue,
         dtype: pa.DataType,
         ftype: FeatureType | None = None,
     ) -> Ok[AddLiteralColumn] | InvalidArgumentError:
@@ -1313,8 +1313,13 @@ class _Base(OneofProtoWrapper[table_pb2.TableComputeOp], ABC):
         separator: str,
     ) -> Ok[CombineColumns] | InvalidArgumentError:
         for col in column_names:
-            if not self.schema.has_column(col):
+            field = self.schema.get(col)
+            if not field:
                 return InvalidArgumentError("no column with given name", given_name=col)
+            if pa.types.is_binary(field.dtype) | pa.types.is_large_binary(field.dtype):
+                return InvalidArgumentError(
+                    "cannot concat binary columns", given_name=col
+                )
 
         if self.schema.has_column(combined_column_name):
             return InvalidArgumentError("name given for combined column already exists")
@@ -1698,6 +1703,9 @@ class _Base(OneofProtoWrapper[table_pb2.TableComputeOp], ABC):
         Returns:
             An AddDecisionTreeSummaryOp.
         """
+        if max_depth < 0:
+            return InvalidArgumentError("max_depth must be strictly positive")
+
         return Ok(
             from_proto(
                 table_pb2.AddDecisionTreeSummaryOp(
@@ -2938,16 +2946,26 @@ def _make_schema_for_combine_columns(op: CombineColumns):
                 if column.ftype != ftype:
                     ftype = feature_type.unknown()
 
-            dtype = (
-                pl.DataFrame(schema=schema.to_polars())
-                .with_columns(
-                    pl.concat_list(*op.column_names).alias(op.combined_column_name)
+            if op.column_names:
+                dtype = (
+                    pl.DataFrame(schema=schema.to_polars())
+                    .with_columns(
+                        pl.concat_list(*op.column_names).alias(op.combined_column_name)
+                    )
+                    .to_arrow()
+                    .schema.field(op.combined_column_name)
+                    .type
+                )
+            else:
+                dtype = (
+                    pl.DataFrame(schema=schema.to_polars())
+                    .with_columns(
+                        pl.Series(op.combined_column_name, [], pl.List(pl.Float32))
+                    )
+                    .to_arrow()
+                    .schema.field(op.combined_column_name)
+                    .type
                 )
-                .to_arrow()
-                .schema.field(op.combined_column_name)
-                .type
-            )
-
     return Schema(
         [
             *op.source.schema,

{corvic_engine-0.3.0rc56 → corvic_engine-0.3.0rc57}/python/corvic/system/in_memory_executor.py

@@ -12,6 +12,7 @@ from typing import Any, Final, cast
 
 import numpy as np
 import polars as pl
+import polars.selectors as cs
 import pyarrow as pa
 import pyarrow.parquet as pq
 import structlog
@@ -778,9 +779,14 @@ class InMemoryExecutor(OpGraphExecutor):
                 )
 
             case op_graph.ConcatList():
-                result_df = source_df.with_columns(
-                    pl.concat_list(*op.column_names).alias(op.combined_column_name)
-                )
+                if op.column_names:
+                    result_df = source_df.with_columns(
+                        pl.concat_list(*op.column_names).alias(op.combined_column_name)
+                    )
+                else:
+                    result_df = source_df.with_columns(
+                        pl.Series(op.combined_column_name, [])
+                    )
 
         return Ok(_SchemaAndBatches.from_dataframe(result_df, source_batches.metrics))
 
@@ -1394,17 +1400,41 @@ class InMemoryExecutor(OpGraphExecutor):
                 pass
             case err:
                 return err
+
         df_input = _as_df(source_batches)
+        dataframe = df_input[list({*op.feature_column_names, op.label_column_name})]
+        boolean_columns = [
+            name
+            for name, dtype in dataframe.schema.items()
+            if dtype == pl.Boolean() and name in op.feature_column_names
+        ]
 
-        features = df_input[op.feature_column_names]
-        classes = df_input[op.label_column_name]
+        # Drop Nan and Null and infinite rows as not supported by decision tree
+        dataframe = dataframe.with_columns(
+            *[pl.col(col).cast(pl.Float32) for col in op.feature_column_names]
+        )
+        dataframe = dataframe.drop_nans().drop_nulls()
+        try:
+            # is_infinite is not implemented for all datatypes
+            dataframe = dataframe.filter(~pl.any_horizontal(cs.numeric().is_infinite()))
+        except pl.exceptions.InvalidOperationError as err:
+            return InvalidArgumentError.from_(err)
+
+        if not len(dataframe):
+            return InvalidArgumentError(
+                "a minimum of 1 sample is required by DecisionTreeClassifier"
+            )
+        features = dataframe[op.feature_column_names]
+        classes = dataframe[op.label_column_name]
         max_depth = op.max_depth
 
-        binary_columns = [
-            name for name, dtype in features.schema.items() if dtype == pl.Boolean()
-        ]
-
         from sklearn.tree import DecisionTreeClassifier, export_graphviz, export_text
+        from sklearn.utils.multiclass import check_classification_targets
+
+        try:
+            check_classification_targets(classes)
+        except ValueError as err:
+            return InvalidArgumentError.from_(err)
 
         decision_tree = DecisionTreeClassifier(random_state=0, max_depth=max_depth)
         try:
@@ -1426,11 +1456,11 @@ class InMemoryExecutor(OpGraphExecutor):
             max_depth=max_depth,
         )
 
-        for binary_column in binary_columns:
+        for boolean_column in boolean_columns:
             tree_str = tree_str.replace(
-                f"{binary_column} <= 0.50", f"NOT {binary_column}"
+                f"{boolean_column} <= 0.50", f"NOT {boolean_column}"
             )
-            tree_str = tree_str.replace(f"{binary_column} >  0.50", binary_column)
+            tree_str = tree_str.replace(f"{boolean_column} >  0.50", boolean_column)
 
         metrics = source_batches.metrics.copy()
         metrics[op.output_metric_key] = table_pb2.DecisionTreeSummary(

{corvic_engine-0.3.0rc56 → corvic_engine-0.3.0rc57}/python/corvic/table/table.py

@@ -412,12 +412,18 @@ class Table:
             case _:
                 return more_itertools.flatten(map(cls._get_staging_ops, op.sources()))
 
-    def head(self) -> InvalidArgumentError | Ok[Table]:
-        """Get up to the first 10 rows of the table."""
-        return self.op_graph.limit_rows(num_rows=10).map(
+    def head(self, num_rows: int = 10) -> InvalidArgumentError | Ok[Table]:
+        """Get the first `num_rows` rows of the table."""
+        return self.op_graph.limit_rows(num_rows=num_rows).map(
             lambda op: Table(self.client, op)
         )
 
+    def sample_rows(self, num_rows: int = 10) -> InvalidArgumentError | Ok[Table]:
+        """Get a sample of `num_rows` rows of the table."""
+        return self.op_graph.sample_rows(
+            sample_strategy=op_graph.sample_strategy.uniform_random(), num_rows=num_rows
+        ).map(lambda op: Table(self.client, op))
+
     def distinct_rows(self) -> Table:
         return Table(
             self.client,