snowflake-ml-python 1.1.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

snowflake/ml/modeling/ensemble/voting_regressor.py

@@ -22,17 +22,19 @@ from sklearn.utils.metaestimators import available_if
 from snowflake.ml.modeling.framework.base import BaseTransformer, _process_cols
 from snowflake.ml._internal import telemetry
 from snowflake.ml._internal.exceptions import error_codes, exceptions, modeling_error_messages
+from snowflake.ml._internal.env_utils import SNOWML_SPROC_ENV
 from snowflake.ml._internal.utils import pkg_version_utils, identifier
-from snowflake.snowpark import DataFrame
+from snowflake.snowpark import DataFrame, Session
 from snowflake.snowpark._internal.type_utils import convert_sp_to_sf_type
 from snowflake.ml.modeling._internal.snowpark_handlers import SnowparkHandlers as HandlersImpl
+from snowflake.ml.modeling._internal.model_trainer_builder import ModelTrainerBuilder
+from snowflake.ml.modeling._internal.model_trainer import ModelTrainer
 from snowflake.ml.modeling._internal.estimator_utils import (
     gather_dependencies,
     original_estimator_has_callable,
     transform_snowml_obj_to_sklearn_obj,
     validate_sklearn_args,
 )
-from snowflake.ml.modeling._internal.snowpark_handlers import SklearnWrapperProvider
 from snowflake.ml.modeling._internal.estimator_protocols import FitPredictHandlers
 
 from snowflake.ml.model.model_signature import (
@@ -52,7 +54,6 @@ _PROJECT = "ModelDevelopment"
 _SUBPROJECT = "".join([s.capitalize() for s in "sklearn.ensemble".replace("sklearn.", "").split("_")])
 
 
-
 class VotingRegressor(BaseTransformer):
     r"""Prediction voting regressor for unfitted estimators
     For more details on this class, see [sklearn.ensemble.VotingRegressor]
@@ -60,61 +61,70 @@ class VotingRegressor(BaseTransformer):
 
     Parameters
     ----------
-    estimators: list of (str, estimator) tuples
-        Invoking the ``fit`` method on the ``VotingRegressor`` will fit clones
-        of those original estimators that will be stored in the class attribute
-        ``self.estimators_``. An estimator can be set to ``'drop'`` using
-        :meth:`set_params`.
-
-    weights: array-like of shape (n_regressors,), default=None
-        Sequence of weights (`float` or `int`) to weight the occurrences of
-        predicted values before averaging. Uses uniform weights if `None`.
-
-    n_jobs: int, default=None
-        The number of jobs to run in parallel for ``fit``.
-        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
-        ``-1`` means using all processors. See :term:`Glossary <n_jobs>`
-        for more details.
-
-    verbose: bool, default=False
-        If True, the time elapsed while fitting will be printed as it
-        is completed.
 
     input_cols: Optional[Union[str, List[str]]]
         A string or list of strings representing column names that contain features.
         If this parameter is not specified, all columns in the input DataFrame except
         the columns specified by label_cols, sample_weight_col, and passthrough_cols
-        parameters are considered input columns.
-
+        parameters are considered input columns. Input columns can also be set after
+        initialization with the `set_input_cols` method.
+    
     label_cols: Optional[Union[str, List[str]]]
         A string or list of strings representing column names that contain labels.
-        This is a required param for estimators, as there is no way to infer these
-        columns. If this parameter is not specified, then object is fitted without
-        labels (like a transformer).
+        Label columns must be specified with this parameter during initialization
+        or with the `set_label_cols` method before fitting.
 
     output_cols: Optional[Union[str, List[str]]]
         A string or list of strings representing column names that will store the
         output of predict and transform operations. The length of output_cols must
-        match the expected number of output columns from the specific estimator or
+        match the expected number of output columns from the specific predictor or
         transformer class used.
-        If this parameter is not specified, output column names are derived by
-        adding an OUTPUT_ prefix to the label column names. These inferred output
-        column names work for estimator's predict() method, but output_cols must
-        be set explicitly for transformers.
+        If you omit this parameter, output column names are derived by adding an
+        OUTPUT_ prefix to the label column names for supervised estimators, or
+        OUTPUT_<IDX>for unsupervised estimators. These inferred output column names
+        work for predictors, but output_cols must be set explicitly for transformers.
+        In general, explicitly specifying output column names is clearer, especially
+        if you don’t specify the input column names.
+        To transform in place, pass the same names for input_cols and output_cols.
+        be set explicitly for transformers. Output columns can also be set after
+        initialization with the `set_output_cols` method.
 
     sample_weight_col: Optional[str]
         A string representing the column name containing the sample weights.
-        This argument is only required when working with weighted datasets.
+        This argument is only required when working with weighted datasets. Sample
+        weight column can also be set after initialization with the
+        `set_sample_weight_col` method.
 
     passthrough_cols: Optional[Union[str, List[str]]]
         A string or a list of strings indicating column names to be excluded from any
         operations (such as train, transform, or inference). These specified column(s)
         will remain untouched throughout the process. This option is helpful in scenarios
         requiring automatic input_cols inference, but need to avoid using specific
-        columns, like index columns, during training or inference.
+        columns, like index columns, during training or inference. Passthrough columns
+        can also be set after initialization with the `set_passthrough_cols` method.
 
     drop_input_cols: Optional[bool], default=False
         If set, the response of predict(), transform() methods will not contain input columns.
+
+    estimators: list of (str, estimator) tuples
+        Invoking the ``fit`` method on the ``VotingRegressor`` will fit clones
+        of those original estimators that will be stored in the class attribute
+        ``self.estimators_``. An estimator can be set to ``'drop'`` using
+        :meth:`set_params`.
+
+    weights: array-like of shape (n_regressors,), default=None
+        Sequence of weights (`float` or `int`) to weight the occurrences of
+        predicted values before averaging. Uses uniform weights if `None`.
+
+    n_jobs: int, default=None
+        The number of jobs to run in parallel for ``fit``.
+        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+        ``-1`` means using all processors. See :term:`Glossary <n_jobs>`
+        for more details.
+
+    verbose: bool, default=False
+        If True, the time elapsed while fitting will be printed as it
+        is completed.
     """
 
     def __init__(  # type: ignore[no-untyped-def]
@@ -139,7 +149,7 @@ class VotingRegressor(BaseTransformer):
         self.set_passthrough_cols(passthrough_cols)
         self.set_drop_input_cols(drop_input_cols)
         self.set_sample_weight_col(sample_weight_col)        
-        deps = set(SklearnWrapperProvider().dependencies)
+        deps: Set[str] = set([f'numpy=={np.__version__}', f'scikit-learn=={sklearn.__version__}', f'cloudpickle=={cp.__version__}'])
         deps = deps | gather_dependencies(estimators)
         self._deps = list(deps)
         estimators = transform_snowml_obj_to_sklearn_obj(estimators)
@@ -151,13 +161,14 @@ class VotingRegressor(BaseTransformer):
             args=init_args,
             klass=sklearn.ensemble.VotingRegressor
         )
-        self._sklearn_object = sklearn.ensemble.VotingRegressor(
+        self._sklearn_object: Any = sklearn.ensemble.VotingRegressor(
             **cleaned_up_init_args,
         )
         self._model_signature_dict: Optional[Dict[str, ModelSignature]] = None
         # If user used snowpark dataframe during fit, here it stores the snowpark input_cols, otherwise the processed input_cols
         self._snowpark_cols: Optional[List[str]] = self.input_cols
-        self._handlers: FitPredictHandlers = HandlersImpl(class_name=VotingRegressor.__class__.__name__, subproject=_SUBPROJECT, autogenerated=True, wrapper_provider=SklearnWrapperProvider())
+        self._handlers: FitPredictHandlers = HandlersImpl(class_name=VotingRegressor.__class__.__name__, subproject=_SUBPROJECT, autogenerated=True)
+        self._autogenerated = True
 
     def _get_rand_id(self) -> str:
         """
@@ -213,54 +224,48 @@ class VotingRegressor(BaseTransformer):
             self
         """
         self._infer_input_output_cols(dataset)
-        if isinstance(dataset, pd.DataFrame):
-            assert self._sklearn_object is not None  # keep mypy happy
-            self._sklearn_object = self._handlers.fit_pandas(
-                dataset,
-                self._sklearn_object,
-                self.input_cols,
-                self.label_cols,
-                self.sample_weight_col
-            )
-        elif isinstance(dataset, DataFrame):
-            self._fit_snowpark(dataset)
-        else:
-            raise TypeError(
-                f"Unexpected dataset type: {type(dataset)}."
-                "Supported dataset types: snowpark.DataFrame, pandas.DataFrame."
-            )
+        if isinstance(dataset, DataFrame):
+            session = dataset._session
+            assert session is not None  # keep mypy happy
+            # Validate that key package version in user workspace are supported in snowflake conda channel
+            # If customer doesn't have package in conda channel, replace the ones have the closest versions
+            self._deps = pkg_version_utils.get_valid_pkg_versions_supported_in_snowflake_conda_channel(
+                pkg_versions=self._get_dependencies(), session=session, subproject=_SUBPROJECT)
+            
+            # Specify input columns so column pruning will be enforced
+            selected_cols = self._get_active_columns()
+            if len(selected_cols) > 0:
+                dataset = dataset.select(selected_cols)
+
+            self._snowpark_cols = dataset.select(self.input_cols).columns
+
+             # If we are already in a stored procedure, no need to kick off another one.
+            if SNOWML_SPROC_ENV in os.environ:
+                statement_params = telemetry.get_function_usage_statement_params(
+                    project=_PROJECT,
+                    subproject=_SUBPROJECT,
+                    function_name=telemetry.get_statement_params_full_func_name(inspect.currentframe(), VotingRegressor.__class__.__name__),
+                    api_calls=[Session.call],
+                    custom_tags=dict([("autogen", True)]) if self._autogenerated else None,
+                )
+                pd_df: pd.DataFrame = dataset.to_pandas(statement_params=statement_params)
+                pd_df.columns = dataset.columns
+                dataset = pd_df
+
+        model_trainer = ModelTrainerBuilder.build(
+            estimator=self._sklearn_object,
+            dataset=dataset,
+            input_cols=self.input_cols,
+            label_cols=self.label_cols,
+            sample_weight_col=self.sample_weight_col,
+            autogenerated=self._autogenerated,
+            subproject=_SUBPROJECT
+        )
+        self._sklearn_object = model_trainer.train()
         self._is_fitted = True
         self._get_model_signatures(dataset)
         return self
 
-    def _fit_snowpark(self, dataset: DataFrame) -> None:
-        session = dataset._session
-        assert session is not None  # keep mypy happy
-        # Validate that key package version in user workspace are supported in snowflake conda channel
-        # If customer doesn't have package in conda channel, replace the ones have the closest versions
-        self._deps = pkg_version_utils.get_valid_pkg_versions_supported_in_snowflake_conda_channel(
-            pkg_versions=self._get_dependencies(), session=session, subproject=_SUBPROJECT)
-        
-        # Specify input columns so column pruning will be enforced
-        selected_cols = self._get_active_columns()
-        if len(selected_cols) > 0:
-            dataset = dataset.select(selected_cols)
-
-        estimator = self._sklearn_object
-        assert estimator is not None  # Keep mypy happy
-
-        self._snowpark_cols = dataset.select(self.input_cols).columns
-
-        self._sklearn_object = self._handlers.fit_snowpark(
-            dataset,
-            session,
-            estimator,
-            ["snowflake-snowpark-python"] + self._get_dependencies(),
-            self.input_cols,
-            self.label_cols,
-            self.sample_weight_col,
-        )
-
     def _get_pass_through_columns(self, dataset: DataFrame) -> List[str]:
         if self._drop_input_cols:
             return []
@@ -448,11 +453,6 @@ class VotingRegressor(BaseTransformer):
         subproject=_SUBPROJECT,
         custom_tags=dict([("autogen", True)]),
     )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def predict(self, dataset: Union[DataFrame, pd.DataFrame]) -> Union[DataFrame, pd.DataFrame]:
         """Predict regression target for X
         For more details on this function, see [sklearn.ensemble.VotingRegressor.predict]
@@ -506,11 +506,6 @@ class VotingRegressor(BaseTransformer):
         subproject=_SUBPROJECT,
         custom_tags=dict([("autogen", True)]),
     )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def transform(self, dataset: Union[DataFrame, pd.DataFrame]) -> Union[DataFrame, pd.DataFrame]:
         """Return predictions for X for each estimator
         For more details on this function, see [sklearn.ensemble.VotingRegressor.transform]
@@ -569,7 +564,8 @@ class VotingRegressor(BaseTransformer):
         if False:
             self.fit(dataset)
             assert self._sklearn_object is not None
-            return self._sklearn_object.labels_
+            labels : npt.NDArray[Any] = self._sklearn_object.labels_
+            return labels
         else:
             raise NotImplementedError
 
@@ -605,6 +601,7 @@ class VotingRegressor(BaseTransformer):
             output_cols = []
 
         # Make sure column names are valid snowflake identifiers.
+        assert output_cols is not None  # Make MyPy happy
         rv = [identifier.rename_to_valid_snowflake_identifier(c) for c in output_cols]
 
         return rv
@@ -615,11 +612,6 @@ class VotingRegressor(BaseTransformer):
         subproject=_SUBPROJECT,
         custom_tags=dict([("autogen", True)]),
     )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def predict_proba(
         self, dataset: Union[DataFrame, pd.DataFrame], output_cols_prefix: str = "predict_proba_"
     ) -> Union[DataFrame, pd.DataFrame]:
@@ -660,11 +652,6 @@ class VotingRegressor(BaseTransformer):
         subproject=_SUBPROJECT,
         custom_tags=dict([("autogen", True)]),
     )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def predict_log_proba(
         self, dataset: Union[DataFrame, pd.DataFrame], output_cols_prefix: str = "predict_log_proba_"
     ) -> Union[DataFrame, pd.DataFrame]:
@@ -701,16 +688,6 @@ class VotingRegressor(BaseTransformer):
         return output_df
 
     @available_if(original_estimator_has_callable("decision_function"))  # type: ignore[misc]
-    @telemetry.send_api_usage_telemetry(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def decision_function(
         self, dataset: Union[DataFrame, pd.DataFrame], output_cols_prefix: str = "decision_function_"
     ) -> Union[DataFrame, pd.DataFrame]:
@@ -811,11 +788,6 @@ class VotingRegressor(BaseTransformer):
         subproject=_SUBPROJECT,
         custom_tags=dict([("autogen", True)]),
     )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def kneighbors(
         self,
         dataset: Union[DataFrame, pd.DataFrame],
@@ -875,9 +847,9 @@ class VotingRegressor(BaseTransformer):
             # For classifier, the type of predict is the same as the type of label
             if self._sklearn_object._estimator_type == 'classifier':
                  # label columns is the desired type for output
-                outputs = _infer_signature(dataset[self.label_cols], "output", use_snowflake_identifiers=True)
+                outputs = list(_infer_signature(dataset[self.label_cols], "output", use_snowflake_identifiers=True))
                 # rename the output columns
-                outputs = model_signature_utils.rename_features(outputs, self.output_cols)
+                outputs = list(model_signature_utils.rename_features(outputs, self.output_cols))
                 self._model_signature_dict["predict"] = ModelSignature(inputs,
                                                                        ([] if self._drop_input_cols else inputs)
                                                                        + outputs)