corvic-engine 0.3.0rc56__cp38-abi3-win_amd64.whl → 0.3.0rc57__cp38-abi3-win_amd64.whl

corvic/model/_space.py

@@ -836,13 +836,20 @@ class TabularSpace(Space):
                     pa.string(),
                 )
 
-            op = (
-                op.concat_list(
+            if len(embedding_column_tmp_names):
+                op = op.concat_list(
                     column_names=embedding_column_tmp_names,
                     concat_list_column_name=embedding_column_tmp_name,
+                ).and_then(reduce_dimension)
+            else:
+                op = op.add_literal_column(
+                    column_name=embedding_column_tmp_name,
+                    literal=[],
+                    dtype=pa.large_list(pa.float32()),
+                    ftype=op_graph.feature_type.embedding(),
                 )
-                .and_then(reduce_dimension)
-                .and_then(select_columns)
+            op = (
+                op.and_then(select_columns)
                 .and_then(update_feature_types)
                 .and_then(rename_columns)
                 .and_then(add_literal_column)

corvic/op_graph/_schema.py

@@ -2,8 +2,6 @@
 
 from __future__ import annotations
 
-from collections.abc import Sequence
-
 import polars as pl
 import pyarrow as pa
 
@@ -14,7 +12,7 @@ from corvic.op_graph.feature_types import FeatureType
 
 def _infer_possible_feature_types(
     data_type: pa.DataType,
-) -> Sequence[FeatureType]:
+) -> list[FeatureType]:
     if pa.types.is_integer(data_type):
         # TODO(thunt): Add multi-categorical in future versions
         return [
@@ -168,6 +166,6 @@ class Field:
     def rename(self, new_name: str) -> Field:
         return Field(new_name, self.dtype, self.ftype)
 
-    def possible_feature_types(self) -> Sequence[FeatureType]:
+    def possible_feature_types(self) -> list[FeatureType]:
         """Infer possible feature types given the data type."""
         return _infer_possible_feature_types(self.dtype)

corvic/op_graph/ops.py

@@ -59,7 +59,7 @@ from corvic.op_graph.row_filters import RowFilter
 from corvic.op_graph.row_filters import from_proto as row_filters_from_proto
 from corvic.op_graph.sample_strategy import SampleStrategy
 from corvic.op_graph.sample_strategy import from_proto as sample_strategy_from_proto
-from corvic.pa_scalar import PyValue, from_value, to_value
+from corvic.pa_scalar import PyValue, Scalar, from_value, to_value
 from corvic.proto_wrapper import OneofProtoWrapper
 from corvic.result import InternalError, InvalidArgumentError, Ok
 from corvic_generated.algorithm.graph.v1 import graph_pb2
@@ -1275,7 +1275,7 @@ class _Base(OneofProtoWrapper[table_pb2.TableComputeOp], ABC):
     def add_literal_column(
         self,
         column_name: str,
-        literal: struct_pb2.Value | float | str | bool,
+        literal: struct_pb2.Value | Scalar | PyValue,
         dtype: pa.DataType,
         ftype: FeatureType | None = None,
     ) -> Ok[AddLiteralColumn] | InvalidArgumentError:
@@ -1313,8 +1313,13 @@ class _Base(OneofProtoWrapper[table_pb2.TableComputeOp], ABC):
         separator: str,
     ) -> Ok[CombineColumns] | InvalidArgumentError:
         for col in column_names:
-            if not self.schema.has_column(col):
+            field = self.schema.get(col)
+            if not field:
                 return InvalidArgumentError("no column with given name", given_name=col)
+            if pa.types.is_binary(field.dtype) | pa.types.is_large_binary(field.dtype):
+                return InvalidArgumentError(
+                    "cannot concat binary columns", given_name=col
+                )
 
         if self.schema.has_column(combined_column_name):
             return InvalidArgumentError("name given for combined column already exists")
@@ -1698,6 +1703,9 @@ class _Base(OneofProtoWrapper[table_pb2.TableComputeOp], ABC):
         Returns:
             An AddDecisionTreeSummaryOp.
         """
+        if max_depth < 0:
+            return InvalidArgumentError("max_depth must be strictly positive")
+
         return Ok(
             from_proto(
                 table_pb2.AddDecisionTreeSummaryOp(
@@ -2938,16 +2946,26 @@ def _make_schema_for_combine_columns(op: CombineColumns):
                 if column.ftype != ftype:
                     ftype = feature_type.unknown()
 
-            dtype = (
-                pl.DataFrame(schema=schema.to_polars())
-                .with_columns(
-                    pl.concat_list(*op.column_names).alias(op.combined_column_name)
+            if op.column_names:
+                dtype = (
+                    pl.DataFrame(schema=schema.to_polars())
+                    .with_columns(
+                        pl.concat_list(*op.column_names).alias(op.combined_column_name)
+                    )
+                    .to_arrow()
+                    .schema.field(op.combined_column_name)
+                    .type
+                )
+            else:
+                dtype = (
+                    pl.DataFrame(schema=schema.to_polars())
+                    .with_columns(
+                        pl.Series(op.combined_column_name, [], pl.List(pl.Float32))
+                    )
+                    .to_arrow()
+                    .schema.field(op.combined_column_name)
+                    .type
                 )
-                .to_arrow()
-                .schema.field(op.combined_column_name)
-                .type
-            )
-
     return Schema(
         [
             *op.source.schema,

corvic/system/in_memory_executor.py

@@ -12,6 +12,7 @@ from typing import Any, Final, cast
 
 import numpy as np
 import polars as pl
+import polars.selectors as cs
 import pyarrow as pa
 import pyarrow.parquet as pq
 import structlog
@@ -778,9 +779,14 @@ class InMemoryExecutor(OpGraphExecutor):
                 )
 
             case op_graph.ConcatList():
-                result_df = source_df.with_columns(
-                    pl.concat_list(*op.column_names).alias(op.combined_column_name)
-                )
+                if op.column_names:
+                    result_df = source_df.with_columns(
+                        pl.concat_list(*op.column_names).alias(op.combined_column_name)
+                    )
+                else:
+                    result_df = source_df.with_columns(
+                        pl.Series(op.combined_column_name, [])
+                    )
 
         return Ok(_SchemaAndBatches.from_dataframe(result_df, source_batches.metrics))
 
@@ -1394,17 +1400,41 @@ class InMemoryExecutor(OpGraphExecutor):
                 pass
             case err:
                 return err
+
         df_input = _as_df(source_batches)
+        dataframe = df_input[list({*op.feature_column_names, op.label_column_name})]
+        boolean_columns = [
+            name
+            for name, dtype in dataframe.schema.items()
+            if dtype == pl.Boolean() and name in op.feature_column_names
+        ]
 
-        features = df_input[op.feature_column_names]
-        classes = df_input[op.label_column_name]
+        # Drop Nan and Null and infinite rows as not supported by decision tree
+        dataframe = dataframe.with_columns(
+            *[pl.col(col).cast(pl.Float32) for col in op.feature_column_names]
+        )
+        dataframe = dataframe.drop_nans().drop_nulls()
+        try:
+            # is_infinite is not implemented for all datatypes
+            dataframe = dataframe.filter(~pl.any_horizontal(cs.numeric().is_infinite()))
+        except pl.exceptions.InvalidOperationError as err:
+            return InvalidArgumentError.from_(err)
+
+        if not len(dataframe):
+            return InvalidArgumentError(
+                "a minimum of 1 sample is required by DecisionTreeClassifier"
+            )
+        features = dataframe[op.feature_column_names]
+        classes = dataframe[op.label_column_name]
         max_depth = op.max_depth
 
-        binary_columns = [
-            name for name, dtype in features.schema.items() if dtype == pl.Boolean()
-        ]
-
         from sklearn.tree import DecisionTreeClassifier, export_graphviz, export_text
+        from sklearn.utils.multiclass import check_classification_targets
+
+        try:
+            check_classification_targets(classes)
+        except ValueError as err:
+            return InvalidArgumentError.from_(err)
 
         decision_tree = DecisionTreeClassifier(random_state=0, max_depth=max_depth)
         try:
@@ -1426,11 +1456,11 @@ class InMemoryExecutor(OpGraphExecutor):
             max_depth=max_depth,
         )
 
-        for binary_column in binary_columns:
+        for boolean_column in boolean_columns:
             tree_str = tree_str.replace(
-                f"{binary_column} <= 0.50", f"NOT {binary_column}"
+                f"{boolean_column} <= 0.50", f"NOT {boolean_column}"
             )
-            tree_str = tree_str.replace(f"{binary_column} >  0.50", binary_column)
+            tree_str = tree_str.replace(f"{boolean_column} >  0.50", boolean_column)
 
         metrics = source_batches.metrics.copy()
         metrics[op.output_metric_key] = table_pb2.DecisionTreeSummary(

corvic/table/table.py

@@ -412,12 +412,18 @@ class Table:
             case _:
                 return more_itertools.flatten(map(cls._get_staging_ops, op.sources()))
 
-    def head(self) -> InvalidArgumentError | Ok[Table]:
-        """Get up to the first 10 rows of the table."""
-        return self.op_graph.limit_rows(num_rows=10).map(
+    def head(self, num_rows: int = 10) -> InvalidArgumentError | Ok[Table]:
+        """Get the first `num_rows` rows of the table."""
+        return self.op_graph.limit_rows(num_rows=num_rows).map(
             lambda op: Table(self.client, op)
         )
 
+    def sample_rows(self, num_rows: int = 10) -> InvalidArgumentError | Ok[Table]:
+        """Get a sample of `num_rows` rows of the table."""
+        return self.op_graph.sample_rows(
+            sample_strategy=op_graph.sample_strategy.uniform_random(), num_rows=num_rows
+        ).map(lambda op: Table(self.client, op))
+
     def distinct_rows(self) -> Table:
         return Table(
             self.client,

{corvic_engine-0.3.0rc56.dist-info → corvic_engine-0.3.0rc57.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corvic-engine
-Version: 0.3.0rc56
+Version: 0.3.0rc57
 Classifier: Environment :: Console
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: Implementation :: CPython

{corvic_engine-0.3.0rc56.dist-info → corvic_engine-0.3.0rc57.dist-info}/RECORD

@@ -1,6 +1,6 @@
-corvic_engine-0.3.0rc56.dist-info/METADATA,sha256=kGcTUywSpPxHTYF5tu-LPySqBXMZgKxvNVgkOssAn0A,1876
-corvic_engine-0.3.0rc56.dist-info/WHEEL,sha256=hKPP3BCTWtTwj6SFaSI--T5aOGqh_llYfbZ_BsqivwA,94
-corvic_engine-0.3.0rc56.dist-info/licenses/LICENSE,sha256=DSS1OD0oIgssKOmAzkMRBv5jvvVuZQbrIv8lpl9DXY8,1035
+corvic_engine-0.3.0rc57.dist-info/METADATA,sha256=T40bDZQGa5aI1lYDYYWrtEwMrowLcANHpje9gUdPpsM,1876
+corvic_engine-0.3.0rc57.dist-info/WHEEL,sha256=hKPP3BCTWtTwj6SFaSI--T5aOGqh_llYfbZ_BsqivwA,94
+corvic_engine-0.3.0rc57.dist-info/licenses/LICENSE,sha256=DSS1OD0oIgssKOmAzkMRBv5jvvVuZQbrIv8lpl9DXY8,1035
 corvic/context/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 corvic/context/__init__.py,sha256=zBnPiP-tStGSVMG_0-G_0ay6-yIX2aerW_oYRzAex74,1702
 corvic/embed/node2vec.py,sha256=XIJjFDdT-JnmZ43lgP-K-dLgnR17L_uaJqBPAYlsPsk,11148
@@ -25,19 +25,19 @@ corvic/model/_proto_orm_convert.py,sha256=jmzmaaUkSxeHB5OMef92AyGw7sorJ6pP4ylbeK
 corvic/model/_resource.py,sha256=w5m6mmD8KrHJ8efPTfRV0JKaCmkDRaxlGeuRMmVbw10,7773
 corvic/model/_room.py,sha256=36mXngZ38L4mr6_LgUm-QgsUUaoGMiYQRfvXLV_jd-4,2914
 corvic/model/_source.py,sha256=A1Jk4r5mB0f-Y3L8esaQFCUAu7CCTlwAm7f4qSnvjsM,9603
-corvic/model/_space.py,sha256=13ggLTCQMNTYYpP5PldMqtJiKp3sWOVRhQcktmoHefA,35590
+corvic/model/_space.py,sha256=ZljalsBDrcnsx2sUOpJd6qQO2nFYDFttNoJMiLdGTBM,35922
 corvic/model/__init__.py,sha256=Lb-yC04t17Hr2TlnGfn5Ewzd2h1nH4hb9tKdMNAak9s,3075
 corvic/op_graph/aggregation.py,sha256=8X6vqXD7dLHrhYJU0BqmhUsWGbzD1zSP5Db5VHdIru4,6187
 corvic/op_graph/encoders.py,sha256=93wYoBCn_us5lRCkqvjaP0LTg3LBB3yEfhzICv06bB0,10460
 corvic/op_graph/errors.py,sha256=I4NE5053d0deGm5xx5EmyP4f98qx42xnIsW1IA-2hy4,163
 corvic/op_graph/feature_types.py,sha256=ZE6onUGW4Xa7tPL4XgRVQ1Tvj5FVJJ66di3ShDTR0Ak,9623
-corvic/op_graph/ops.py,sha256=G2bDIK_hlKJxqOX5Xu9hEoLBkdqi-TsZSn6tTagqjgg,109823
+corvic/op_graph/ops.py,sha256=1YOFnnN6WgzBajkXRM9UgdMLd-NEfa4tRmIQVj5cyeo,110637
 corvic/op_graph/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 corvic/op_graph/row_filters/_jsonlogic.py,sha256=tBd-wOwE6AIx9XEkuSVdBx9iB08nsdHJdvNzEmWzrB0,6432
 corvic/op_graph/row_filters/_row_filters.py,sha256=d7oUbB-vThi-Kn5GupGnEwr5UNlsGFgCgR3Q7NR_tkI,9554
 corvic/op_graph/row_filters/__init__.py,sha256=1sibH_kLw7t_9bpRccnEGWqdCiN0VaUh9LMMIMCRyL8,575
 corvic/op_graph/sample_strategy.py,sha256=DrbtJ3ORkIRfyIE_FdlOh_UMnCW_K9jL1LeonVYb3bU,3007
-corvic/op_graph/_schema.py,sha256=STbxY5PIqIA6xkSDeK8k72Nutsxq5jGe7e_aT35aznI,5733
+corvic/op_graph/_schema.py,sha256=7Uuun9e6PRrtOeJLsFD8VzkwWeUpbnBcD37NpMKOcmQ,5685
 corvic/op_graph/_transformations.py,sha256=L9Au_GcciPynww4ZXojMtNdPJ36Qboc9gn0bVzXLifU,9445
 corvic/op_graph/__init__.py,sha256=1DMrQfuuS3FkLa9DXYDjSDLurdxxpG5H1jB2ctaa9xo,1444
 corvic/orm/base.py,sha256=95nkqycCZ1FaWAhTsa7zbZ0YuwNFkMUW7Wk8yhtYau8,8824
@@ -68,7 +68,7 @@ corvic/sql/parse_ops.py,sha256=1ZXVlDzIzqwW_KP0mwMxaY91tLSXqpeaUHyrGJkh56o,29444
 corvic/sql/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 corvic/sql/__init__.py,sha256=kZ1a39KVZ08P8Bg6XuXDLD_dTQX0k620u4nwxZF4SnY,303
 corvic/system/client.py,sha256=hGhZX8RtHrFEOlOmJNlUHktOZrutOwNYUY_a1htQSrg,821
-corvic/system/in_memory_executor.py,sha256=t5zYx2SC7SXlG_iGa5gKnaTgOKUoTS6FQUL9FYVFex0,67586
+corvic/system/in_memory_executor.py,sha256=tRYzoVCNHemlpPfYRaVM_Nc3uFsLYaOFof1nVR-6hGc,68943
 corvic/system/op_graph_executor.py,sha256=gXFnVkemS5EwNegJdU-xVAfMLPULqMFPF7d3EG3AD_U,3482
 corvic/system/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 corvic/system/staging.py,sha256=K5P5moiuAMfPx7lxK4mArxeURBwKoyB6x9HGu9JJ16E,1846
@@ -86,7 +86,7 @@ corvic/system_sqlite/rdbms_blob_store.py,sha256=gTP_tQfTVb3wzZkzo8ys1zaz0rSrERzb
 corvic/system_sqlite/staging.py,sha256=P6XdWhjpgcpOZkYxKEjpsTxaAdBKOeSVfARjqt4_xJA,16948
 corvic/system_sqlite/__init__.py,sha256=F4UN9vFsXiDY2AKk1jYZPuWWJpSugKHS7ghXeZYlbZs,390
 corvic/table/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-corvic/table/table.py,sha256=wgA4d1QslrGu7rvzyiob21hLd7esyjgO_QntaSYl-4Q,25038
+corvic/table/table.py,sha256=v3MTV_nHaSAXFjPurn0Gp9Pe4UVL8RhYUHhxR6MVfmE,25396
 corvic/table/__init__.py,sha256=Gj0IR8BQF5PZK92Us7PP0ZigMsVyrfWJupzH8TgzRQk,588
 corvic/version/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 corvic/version/__init__.py,sha256=JlkRLvKXsu3zIxhdynO_0Ub5NfQOvGjfwCRkNnaOu9U,1125
@@ -244,5 +244,5 @@ corvic_generated/status/v1/event_pb2.pyi,sha256=eU-ibrYpvEAJSIDlSa62-bC96AQU1ykF
 corvic_generated/status/v1/event_pb2_grpc.pyi,sha256=H9-ADaiKR9iyVZvmnXutZqWwRRCDxjUIktkfJrJFIHg,417
 corvic_generated/status/v1/service_pb2.pyi,sha256=iXLR2FOKQJpBgvBzpD2kVwcYOCksP2aRwK4JYaI9CBw,558
 corvic_generated/status/v1/service_pb2_grpc.pyi,sha256=OoAnaZ64FD0UTzPoRhYvQU8ecoilhHj3ySjSfHbVDaU,1501
-corvic/engine/_native.pyd,sha256=XkU3bVVXAk3up15IfaE0ih1d0_Lo8jRl_mJp1ZwbBls,438272
-corvic_engine-0.3.0rc56.dist-info/RECORD,,
+corvic/engine/_native.pyd,sha256=-QxbxEBeQo7SFPLJlh84nSQEIcAM-zOfcmElul0VQ1U,438272
+corvic_engine-0.3.0rc57.dist-info/RECORD,,