snowflake-ml-python 1.1.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

snowflake/ml/modeling/multiclass/output_code_classifier.py

@@ -22,17 +22,19 @@ from sklearn.utils.metaestimators import available_if
 from snowflake.ml.modeling.framework.base import BaseTransformer, _process_cols
 from snowflake.ml._internal import telemetry
 from snowflake.ml._internal.exceptions import error_codes, exceptions, modeling_error_messages
+from snowflake.ml._internal.env_utils import SNOWML_SPROC_ENV
 from snowflake.ml._internal.utils import pkg_version_utils, identifier
-from snowflake.snowpark import DataFrame
+from snowflake.snowpark import DataFrame, Session
 from snowflake.snowpark._internal.type_utils import convert_sp_to_sf_type
 from snowflake.ml.modeling._internal.snowpark_handlers import SnowparkHandlers as HandlersImpl
+from snowflake.ml.modeling._internal.model_trainer_builder import ModelTrainerBuilder
+from snowflake.ml.modeling._internal.model_trainer import ModelTrainer
 from snowflake.ml.modeling._internal.estimator_utils import (
     gather_dependencies,
     original_estimator_has_callable,
     transform_snowml_obj_to_sklearn_obj,
     validate_sklearn_args,
 )
-from snowflake.ml.modeling._internal.snowpark_handlers import SklearnWrapperProvider
 from snowflake.ml.modeling._internal.estimator_protocols import FitPredictHandlers
 
 from snowflake.ml.model.model_signature import (
@@ -52,7 +54,6 @@ _PROJECT = "ModelDevelopment"
 _SUBPROJECT = "".join([s.capitalize() for s in "sklearn.multiclass".replace("sklearn.", "").split("_")])
 
 
-
 class OutputCodeClassifier(BaseTransformer):
     r"""(Error-Correcting) Output-Code multiclass strategy
     For more details on this class, see [sklearn.multiclass.OutputCodeClassifier]
@@ -60,64 +61,73 @@ class OutputCodeClassifier(BaseTransformer):
 
     Parameters
     ----------
-    estimator: estimator object
-        An estimator object implementing :term:`fit` and one of
-        :term:`decision_function` or :term:`predict_proba`.
-
-    code_size: float, default=1.5
-        Percentage of the number of classes to be used to create the code book.
-        A number between 0 and 1 will require fewer classifiers than
-        one-vs-the-rest. A number greater than 1 will require more classifiers
-        than one-vs-the-rest.
-
-    random_state: int, RandomState instance, default=None
-        The generator used to initialize the codebook.
-        Pass an int for reproducible output across multiple function calls.
-        See :term:`Glossary <random_state>`.
-
-    n_jobs: int, default=None
-        The number of jobs to use for the computation: the multiclass problems
-        are computed in parallel.
-
-        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
-        ``-1`` means using all processors. See :term:`Glossary <n_jobs>`
-        for more details.
 
     input_cols: Optional[Union[str, List[str]]]
         A string or list of strings representing column names that contain features.
         If this parameter is not specified, all columns in the input DataFrame except
         the columns specified by label_cols, sample_weight_col, and passthrough_cols
-        parameters are considered input columns.
-
+        parameters are considered input columns. Input columns can also be set after
+        initialization with the `set_input_cols` method.
+    
     label_cols: Optional[Union[str, List[str]]]
         A string or list of strings representing column names that contain labels.
-        This is a required param for estimators, as there is no way to infer these
-        columns. If this parameter is not specified, then object is fitted without
-        labels (like a transformer).
+        Label columns must be specified with this parameter during initialization
+        or with the `set_label_cols` method before fitting.
 
     output_cols: Optional[Union[str, List[str]]]
         A string or list of strings representing column names that will store the
         output of predict and transform operations. The length of output_cols must
-        match the expected number of output columns from the specific estimator or
+        match the expected number of output columns from the specific predictor or
         transformer class used.
-        If this parameter is not specified, output column names are derived by
-        adding an OUTPUT_ prefix to the label column names. These inferred output
-        column names work for estimator's predict() method, but output_cols must
-        be set explicitly for transformers.
+        If you omit this parameter, output column names are derived by adding an
+        OUTPUT_ prefix to the label column names for supervised estimators, or
+        OUTPUT_<IDX>for unsupervised estimators. These inferred output column names
+        work for predictors, but output_cols must be set explicitly for transformers.
+        In general, explicitly specifying output column names is clearer, especially
+        if you don’t specify the input column names.
+        To transform in place, pass the same names for input_cols and output_cols.
+        be set explicitly for transformers. Output columns can also be set after
+        initialization with the `set_output_cols` method.
 
     sample_weight_col: Optional[str]
         A string representing the column name containing the sample weights.
-        This argument is only required when working with weighted datasets.
+        This argument is only required when working with weighted datasets. Sample
+        weight column can also be set after initialization with the
+        `set_sample_weight_col` method.
 
     passthrough_cols: Optional[Union[str, List[str]]]
         A string or a list of strings indicating column names to be excluded from any
         operations (such as train, transform, or inference). These specified column(s)
         will remain untouched throughout the process. This option is helpful in scenarios
         requiring automatic input_cols inference, but need to avoid using specific
-        columns, like index columns, during training or inference.
+        columns, like index columns, during training or inference. Passthrough columns
+        can also be set after initialization with the `set_passthrough_cols` method.
 
     drop_input_cols: Optional[bool], default=False
         If set, the response of predict(), transform() methods will not contain input columns.
+
+    estimator: estimator object
+        An estimator object implementing :term:`fit` and one of
+        :term:`decision_function` or :term:`predict_proba`.
+
+    code_size: float, default=1.5
+        Percentage of the number of classes to be used to create the code book.
+        A number between 0 and 1 will require fewer classifiers than
+        one-vs-the-rest. A number greater than 1 will require more classifiers
+        than one-vs-the-rest.
+
+    random_state: int, RandomState instance, default=None
+        The generator used to initialize the codebook.
+        Pass an int for reproducible output across multiple function calls.
+        See :term:`Glossary <random_state>`.
+
+    n_jobs: int, default=None
+        The number of jobs to use for the computation: the multiclass problems
+        are computed in parallel.
+
+        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+        ``-1`` means using all processors. See :term:`Glossary <n_jobs>`
+        for more details.
     """
 
     def __init__(  # type: ignore[no-untyped-def]
@@ -142,7 +152,7 @@ class OutputCodeClassifier(BaseTransformer):
         self.set_passthrough_cols(passthrough_cols)
         self.set_drop_input_cols(drop_input_cols)
         self.set_sample_weight_col(sample_weight_col)        
-        deps = set(SklearnWrapperProvider().dependencies)
+        deps: Set[str] = set([f'numpy=={np.__version__}', f'scikit-learn=={sklearn.__version__}', f'cloudpickle=={cp.__version__}'])
         deps = deps | gather_dependencies(estimator)
         self._deps = list(deps)
         estimator = transform_snowml_obj_to_sklearn_obj(estimator)
@@ -154,13 +164,14 @@ class OutputCodeClassifier(BaseTransformer):
             args=init_args,
             klass=sklearn.multiclass.OutputCodeClassifier
         )
-        self._sklearn_object = sklearn.multiclass.OutputCodeClassifier(
+        self._sklearn_object: Any = sklearn.multiclass.OutputCodeClassifier(
             **cleaned_up_init_args,
         )
         self._model_signature_dict: Optional[Dict[str, ModelSignature]] = None
         # If user used snowpark dataframe during fit, here it stores the snowpark input_cols, otherwise the processed input_cols
         self._snowpark_cols: Optional[List[str]] = self.input_cols
-        self._handlers: FitPredictHandlers = HandlersImpl(class_name=OutputCodeClassifier.__class__.__name__, subproject=_SUBPROJECT, autogenerated=True, wrapper_provider=SklearnWrapperProvider())
+        self._handlers: FitPredictHandlers = HandlersImpl(class_name=OutputCodeClassifier.__class__.__name__, subproject=_SUBPROJECT, autogenerated=True)
+        self._autogenerated = True
 
     def _get_rand_id(self) -> str:
         """
@@ -216,54 +227,48 @@ class OutputCodeClassifier(BaseTransformer):
             self
         """
         self._infer_input_output_cols(dataset)
-        if isinstance(dataset, pd.DataFrame):
-            assert self._sklearn_object is not None  # keep mypy happy
-            self._sklearn_object = self._handlers.fit_pandas(
-                dataset,
-                self._sklearn_object,
-                self.input_cols,
-                self.label_cols,
-                self.sample_weight_col
-            )
-        elif isinstance(dataset, DataFrame):
-            self._fit_snowpark(dataset)
-        else:
-            raise TypeError(
-                f"Unexpected dataset type: {type(dataset)}."
-                "Supported dataset types: snowpark.DataFrame, pandas.DataFrame."
-            )
+        if isinstance(dataset, DataFrame):
+            session = dataset._session
+            assert session is not None  # keep mypy happy
+            # Validate that key package version in user workspace are supported in snowflake conda channel
+            # If customer doesn't have package in conda channel, replace the ones have the closest versions
+            self._deps = pkg_version_utils.get_valid_pkg_versions_supported_in_snowflake_conda_channel(
+                pkg_versions=self._get_dependencies(), session=session, subproject=_SUBPROJECT)
+            
+            # Specify input columns so column pruning will be enforced
+            selected_cols = self._get_active_columns()
+            if len(selected_cols) > 0:
+                dataset = dataset.select(selected_cols)
+
+            self._snowpark_cols = dataset.select(self.input_cols).columns
+
+             # If we are already in a stored procedure, no need to kick off another one.
+            if SNOWML_SPROC_ENV in os.environ:
+                statement_params = telemetry.get_function_usage_statement_params(
+                    project=_PROJECT,
+                    subproject=_SUBPROJECT,
+                    function_name=telemetry.get_statement_params_full_func_name(inspect.currentframe(), OutputCodeClassifier.__class__.__name__),
+                    api_calls=[Session.call],
+                    custom_tags=dict([("autogen", True)]) if self._autogenerated else None,
+                )
+                pd_df: pd.DataFrame = dataset.to_pandas(statement_params=statement_params)
+                pd_df.columns = dataset.columns
+                dataset = pd_df
+
+        model_trainer = ModelTrainerBuilder.build(
+            estimator=self._sklearn_object,
+            dataset=dataset,
+            input_cols=self.input_cols,
+            label_cols=self.label_cols,
+            sample_weight_col=self.sample_weight_col,
+            autogenerated=self._autogenerated,
+            subproject=_SUBPROJECT
+        )
+        self._sklearn_object = model_trainer.train()
         self._is_fitted = True
         self._get_model_signatures(dataset)
         return self
 
-    def _fit_snowpark(self, dataset: DataFrame) -> None:
-        session = dataset._session
-        assert session is not None  # keep mypy happy
-        # Validate that key package version in user workspace are supported in snowflake conda channel
-        # If customer doesn't have package in conda channel, replace the ones have the closest versions
-        self._deps = pkg_version_utils.get_valid_pkg_versions_supported_in_snowflake_conda_channel(
-            pkg_versions=self._get_dependencies(), session=session, subproject=_SUBPROJECT)
-        
-        # Specify input columns so column pruning will be enforced
-        selected_cols = self._get_active_columns()
-        if len(selected_cols) > 0:
-            dataset = dataset.select(selected_cols)
-
-        estimator = self._sklearn_object
-        assert estimator is not None  # Keep mypy happy
-
-        self._snowpark_cols = dataset.select(self.input_cols).columns
-
-        self._sklearn_object = self._handlers.fit_snowpark(
-            dataset,
-            session,
-            estimator,
-            ["snowflake-snowpark-python"] + self._get_dependencies(),
-            self.input_cols,
-            self.label_cols,
-            self.sample_weight_col,
-        )
-
     def _get_pass_through_columns(self, dataset: DataFrame) -> List[str]:
         if self._drop_input_cols:
             return []
@@ -451,11 +456,6 @@ class OutputCodeClassifier(BaseTransformer):
         subproject=_SUBPROJECT,
         custom_tags=dict([("autogen", True)]),
     )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def predict(self, dataset: Union[DataFrame, pd.DataFrame]) -> Union[DataFrame, pd.DataFrame]:
         """Predict multi-class targets using underlying estimators
         For more details on this function, see [sklearn.multiclass.OutputCodeClassifier.predict]
@@ -509,11 +509,6 @@ class OutputCodeClassifier(BaseTransformer):
         subproject=_SUBPROJECT,
         custom_tags=dict([("autogen", True)]),
     )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def transform(self, dataset: Union[DataFrame, pd.DataFrame]) -> Union[DataFrame, pd.DataFrame]:
         """Method not supported for this class.
 
@@ -570,7 +565,8 @@ class OutputCodeClassifier(BaseTransformer):
         if False:
             self.fit(dataset)
             assert self._sklearn_object is not None
-            return self._sklearn_object.labels_
+            labels : npt.NDArray[Any] = self._sklearn_object.labels_
+            return labels
         else:
             raise NotImplementedError
 
@@ -606,6 +602,7 @@ class OutputCodeClassifier(BaseTransformer):
             output_cols = []
 
         # Make sure column names are valid snowflake identifiers.
+        assert output_cols is not None  # Make MyPy happy
         rv = [identifier.rename_to_valid_snowflake_identifier(c) for c in output_cols]
 
         return rv
@@ -616,11 +613,6 @@ class OutputCodeClassifier(BaseTransformer):
         subproject=_SUBPROJECT,
         custom_tags=dict([("autogen", True)]),
     )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def predict_proba(
         self, dataset: Union[DataFrame, pd.DataFrame], output_cols_prefix: str = "predict_proba_"
     ) -> Union[DataFrame, pd.DataFrame]:
@@ -661,11 +653,6 @@ class OutputCodeClassifier(BaseTransformer):
         subproject=_SUBPROJECT,
         custom_tags=dict([("autogen", True)]),
     )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def predict_log_proba(
         self, dataset: Union[DataFrame, pd.DataFrame], output_cols_prefix: str = "predict_log_proba_"
     ) -> Union[DataFrame, pd.DataFrame]:
@@ -702,16 +689,6 @@ class OutputCodeClassifier(BaseTransformer):
         return output_df
 
     @available_if(original_estimator_has_callable("decision_function"))  # type: ignore[misc]
-    @telemetry.send_api_usage_telemetry(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def decision_function(
         self, dataset: Union[DataFrame, pd.DataFrame], output_cols_prefix: str = "decision_function_"
     ) -> Union[DataFrame, pd.DataFrame]:
@@ -812,11 +789,6 @@ class OutputCodeClassifier(BaseTransformer):
         subproject=_SUBPROJECT,
         custom_tags=dict([("autogen", True)]),
     )
-    @telemetry.add_stmt_params_to_df(
-        project=_PROJECT,
-        subproject=_SUBPROJECT,
-        custom_tags=dict([("autogen", True)]),
-    )
     def kneighbors(
         self,
         dataset: Union[DataFrame, pd.DataFrame],
@@ -876,9 +848,9 @@ class OutputCodeClassifier(BaseTransformer):
             # For classifier, the type of predict is the same as the type of label
             if self._sklearn_object._estimator_type == 'classifier':
                  # label columns is the desired type for output
-                outputs = _infer_signature(dataset[self.label_cols], "output", use_snowflake_identifiers=True)
+                outputs = list(_infer_signature(dataset[self.label_cols], "output", use_snowflake_identifiers=True))
                 # rename the output columns
-                outputs = model_signature_utils.rename_features(outputs, self.output_cols)
+                outputs = list(model_signature_utils.rename_features(outputs, self.output_cols))
                 self._model_signature_dict["predict"] = ModelSignature(inputs,
                                                                        ([] if self._drop_input_cols else inputs)
                                                                        + outputs)