snowflake-ml-python 1.4.0__py3-none-any.whl → 1.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/ml/_internal/env_utils.py +11 -1
- snowflake/ml/_internal/utils/identifier.py +3 -1
- snowflake/ml/_internal/utils/sql_identifier.py +2 -6
- snowflake/ml/feature_store/feature_store.py +151 -78
- snowflake/ml/feature_store/feature_view.py +12 -24
- snowflake/ml/fileset/sfcfs.py +56 -50
- snowflake/ml/fileset/stage_fs.py +48 -13
- snowflake/ml/model/_client/model/model_version_impl.py +2 -50
- snowflake/ml/model/_client/ops/model_ops.py +78 -29
- snowflake/ml/model/_client/sql/model.py +23 -2
- snowflake/ml/model/_client/sql/model_version.py +22 -1
- snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +19 -54
- snowflake/ml/model/_model_composer/model_manifest/model_manifest_schema.py +8 -1
- snowflake/ml/model/_model_composer/model_method/model_method.py +6 -10
- snowflake/ml/model/_packager/model_handlers/catboost.py +206 -0
- snowflake/ml/model/_packager/model_handlers/lightgbm.py +218 -0
- snowflake/ml/model/_packager/model_handlers/sklearn.py +3 -0
- snowflake/ml/model/_packager/model_meta/_core_requirements.py +1 -1
- snowflake/ml/model/_packager/model_meta/model_meta.py +36 -6
- snowflake/ml/model/_packager/model_meta/model_meta_schema.py +20 -1
- snowflake/ml/model/_packager/model_meta_migrator/migrator_plans.py +3 -1
- snowflake/ml/model/_packager/model_packager.py +2 -2
- snowflake/ml/model/{_model_composer/model_runtime/_runtime_requirements.py → _packager/model_runtime/_snowml_inference_alternative_requirements.py} +1 -1
- snowflake/ml/model/_packager/model_runtime/model_runtime.py +137 -0
- snowflake/ml/model/type_hints.py +21 -2
- snowflake/ml/modeling/_internal/estimator_utils.py +16 -11
- snowflake/ml/modeling/_internal/local_implementations/pandas_handlers.py +4 -1
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +13 -14
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_handlers.py +5 -5
- snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +195 -123
- snowflake/ml/modeling/cluster/affinity_propagation.py +195 -123
- snowflake/ml/modeling/cluster/agglomerative_clustering.py +195 -123
- snowflake/ml/modeling/cluster/birch.py +195 -123
- snowflake/ml/modeling/cluster/bisecting_k_means.py +195 -123
- snowflake/ml/modeling/cluster/dbscan.py +195 -123
- snowflake/ml/modeling/cluster/feature_agglomeration.py +195 -123
- snowflake/ml/modeling/cluster/k_means.py +195 -123
- snowflake/ml/modeling/cluster/mean_shift.py +195 -123
- snowflake/ml/modeling/cluster/mini_batch_k_means.py +195 -123
- snowflake/ml/modeling/cluster/optics.py +195 -123
- snowflake/ml/modeling/cluster/spectral_biclustering.py +195 -123
- snowflake/ml/modeling/cluster/spectral_clustering.py +195 -123
- snowflake/ml/modeling/cluster/spectral_coclustering.py +195 -123
- snowflake/ml/modeling/compose/column_transformer.py +195 -123
- snowflake/ml/modeling/compose/transformed_target_regressor.py +195 -123
- snowflake/ml/modeling/covariance/elliptic_envelope.py +195 -123
- snowflake/ml/modeling/covariance/empirical_covariance.py +195 -123
- snowflake/ml/modeling/covariance/graphical_lasso.py +195 -123
- snowflake/ml/modeling/covariance/graphical_lasso_cv.py +195 -123
- snowflake/ml/modeling/covariance/ledoit_wolf.py +195 -123
- snowflake/ml/modeling/covariance/min_cov_det.py +195 -123
- snowflake/ml/modeling/covariance/oas.py +195 -123
- snowflake/ml/modeling/covariance/shrunk_covariance.py +195 -123
- snowflake/ml/modeling/decomposition/dictionary_learning.py +195 -123
- snowflake/ml/modeling/decomposition/factor_analysis.py +195 -123
- snowflake/ml/modeling/decomposition/fast_ica.py +195 -123
- snowflake/ml/modeling/decomposition/incremental_pca.py +195 -123
- snowflake/ml/modeling/decomposition/kernel_pca.py +195 -123
- snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +195 -123
- snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +195 -123
- snowflake/ml/modeling/decomposition/pca.py +195 -123
- snowflake/ml/modeling/decomposition/sparse_pca.py +195 -123
- snowflake/ml/modeling/decomposition/truncated_svd.py +195 -123
- snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +195 -123
- snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +195 -123
- snowflake/ml/modeling/ensemble/ada_boost_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/ada_boost_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/bagging_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/bagging_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/extra_trees_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/extra_trees_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/isolation_forest.py +195 -123
- snowflake/ml/modeling/ensemble/random_forest_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/random_forest_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/stacking_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/voting_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/voting_regressor.py +195 -123
- snowflake/ml/modeling/feature_selection/generic_univariate_select.py +195 -123
- snowflake/ml/modeling/feature_selection/select_fdr.py +195 -123
- snowflake/ml/modeling/feature_selection/select_fpr.py +195 -123
- snowflake/ml/modeling/feature_selection/select_fwe.py +195 -123
- snowflake/ml/modeling/feature_selection/select_k_best.py +195 -123
- snowflake/ml/modeling/feature_selection/select_percentile.py +195 -123
- snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +195 -123
- snowflake/ml/modeling/feature_selection/variance_threshold.py +195 -123
- snowflake/ml/modeling/framework/_utils.py +8 -1
- snowflake/ml/modeling/framework/base.py +9 -1
- snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +195 -123
- snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +195 -123
- snowflake/ml/modeling/impute/iterative_imputer.py +195 -123
- snowflake/ml/modeling/impute/knn_imputer.py +195 -123
- snowflake/ml/modeling/impute/missing_indicator.py +195 -123
- snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +195 -123
- snowflake/ml/modeling/kernel_approximation/nystroem.py +195 -123
- snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +195 -123
- snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +195 -123
- snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +195 -123
- snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +195 -123
- snowflake/ml/modeling/lightgbm/lgbm_classifier.py +195 -123
- snowflake/ml/modeling/lightgbm/lgbm_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/ard_regression.py +195 -123
- snowflake/ml/modeling/linear_model/bayesian_ridge.py +195 -123
- snowflake/ml/modeling/linear_model/elastic_net.py +195 -123
- snowflake/ml/modeling/linear_model/elastic_net_cv.py +195 -123
- snowflake/ml/modeling/linear_model/gamma_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/huber_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/lars.py +195 -123
- snowflake/ml/modeling/linear_model/lars_cv.py +195 -123
- snowflake/ml/modeling/linear_model/lasso.py +195 -123
- snowflake/ml/modeling/linear_model/lasso_cv.py +195 -123
- snowflake/ml/modeling/linear_model/lasso_lars.py +195 -123
- snowflake/ml/modeling/linear_model/lasso_lars_cv.py +195 -123
- snowflake/ml/modeling/linear_model/lasso_lars_ic.py +195 -123
- snowflake/ml/modeling/linear_model/linear_regression.py +195 -123
- snowflake/ml/modeling/linear_model/logistic_regression.py +195 -123
- snowflake/ml/modeling/linear_model/logistic_regression_cv.py +195 -123
- snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +195 -123
- snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +195 -123
- snowflake/ml/modeling/linear_model/multi_task_lasso.py +195 -123
- snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +195 -123
- snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +195 -123
- snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +195 -123
- snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/perceptron.py +195 -123
- snowflake/ml/modeling/linear_model/poisson_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/ransac_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/ridge.py +195 -123
- snowflake/ml/modeling/linear_model/ridge_classifier.py +195 -123
- snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +195 -123
- snowflake/ml/modeling/linear_model/ridge_cv.py +195 -123
- snowflake/ml/modeling/linear_model/sgd_classifier.py +195 -123
- snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +195 -123
- snowflake/ml/modeling/linear_model/sgd_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/theil_sen_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/tweedie_regressor.py +195 -123
- snowflake/ml/modeling/manifold/isomap.py +195 -123
- snowflake/ml/modeling/manifold/mds.py +195 -123
- snowflake/ml/modeling/manifold/spectral_embedding.py +195 -123
- snowflake/ml/modeling/manifold/tsne.py +195 -123
- snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +195 -123
- snowflake/ml/modeling/mixture/gaussian_mixture.py +195 -123
- snowflake/ml/modeling/model_selection/grid_search_cv.py +42 -18
- snowflake/ml/modeling/model_selection/randomized_search_cv.py +42 -18
- snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +195 -123
- snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +195 -123
- snowflake/ml/modeling/multiclass/output_code_classifier.py +195 -123
- snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +195 -123
- snowflake/ml/modeling/naive_bayes/categorical_nb.py +195 -123
- snowflake/ml/modeling/naive_bayes/complement_nb.py +195 -123
- snowflake/ml/modeling/naive_bayes/gaussian_nb.py +195 -123
- snowflake/ml/modeling/naive_bayes/multinomial_nb.py +195 -123
- snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +195 -123
- snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +195 -123
- snowflake/ml/modeling/neighbors/kernel_density.py +195 -123
- snowflake/ml/modeling/neighbors/local_outlier_factor.py +195 -123
- snowflake/ml/modeling/neighbors/nearest_centroid.py +195 -123
- snowflake/ml/modeling/neighbors/nearest_neighbors.py +195 -123
- snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +195 -123
- snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +195 -123
- snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +195 -123
- snowflake/ml/modeling/neural_network/bernoulli_rbm.py +195 -123
- snowflake/ml/modeling/neural_network/mlp_classifier.py +195 -123
- snowflake/ml/modeling/neural_network/mlp_regressor.py +195 -123
- snowflake/ml/modeling/pipeline/pipeline.py +4 -4
- snowflake/ml/modeling/preprocessing/binarizer.py +1 -5
- snowflake/ml/modeling/preprocessing/k_bins_discretizer.py +1 -5
- snowflake/ml/modeling/preprocessing/label_encoder.py +1 -5
- snowflake/ml/modeling/preprocessing/max_abs_scaler.py +1 -5
- snowflake/ml/modeling/preprocessing/min_max_scaler.py +10 -12
- snowflake/ml/modeling/preprocessing/normalizer.py +1 -5
- snowflake/ml/modeling/preprocessing/one_hot_encoder.py +1 -5
- snowflake/ml/modeling/preprocessing/ordinal_encoder.py +1 -5
- snowflake/ml/modeling/preprocessing/polynomial_features.py +195 -123
- snowflake/ml/modeling/preprocessing/robust_scaler.py +1 -5
- snowflake/ml/modeling/preprocessing/standard_scaler.py +11 -11
- snowflake/ml/modeling/semi_supervised/label_propagation.py +195 -123
- snowflake/ml/modeling/semi_supervised/label_spreading.py +195 -123
- snowflake/ml/modeling/svm/linear_svc.py +195 -123
- snowflake/ml/modeling/svm/linear_svr.py +195 -123
- snowflake/ml/modeling/svm/nu_svc.py +195 -123
- snowflake/ml/modeling/svm/nu_svr.py +195 -123
- snowflake/ml/modeling/svm/svc.py +195 -123
- snowflake/ml/modeling/svm/svr.py +195 -123
- snowflake/ml/modeling/tree/decision_tree_classifier.py +195 -123
- snowflake/ml/modeling/tree/decision_tree_regressor.py +195 -123
- snowflake/ml/modeling/tree/extra_tree_classifier.py +195 -123
- snowflake/ml/modeling/tree/extra_tree_regressor.py +195 -123
- snowflake/ml/modeling/xgboost/xgb_classifier.py +195 -123
- snowflake/ml/modeling/xgboost/xgb_regressor.py +195 -123
- snowflake/ml/modeling/xgboost/xgbrf_classifier.py +195 -123
- snowflake/ml/modeling/xgboost/xgbrf_regressor.py +195 -123
- snowflake/ml/registry/registry.py +1 -1
- snowflake/ml/version.py +1 -1
- {snowflake_ml_python-1.4.0.dist-info → snowflake_ml_python-1.4.1.dist-info}/METADATA +68 -57
- {snowflake_ml_python-1.4.0.dist-info → snowflake_ml_python-1.4.1.dist-info}/RECORD +202 -200
- snowflake/ml/model/_model_composer/model_runtime/model_runtime.py +0 -97
- {snowflake_ml_python-1.4.0.dist-info → snowflake_ml_python-1.4.1.dist-info}/LICENSE.txt +0 -0
- {snowflake_ml_python-1.4.0.dist-info → snowflake_ml_python-1.4.1.dist-info}/WHEEL +0 -0
- {snowflake_ml_python-1.4.0.dist-info → snowflake_ml_python-1.4.1.dist-info}/top_level.txt +0 -0
@@ -33,6 +33,15 @@ from snowflake.ml.modeling._internal.transformer_protocols import (
|
|
33
33
|
BatchInferenceKwargsTypedDict,
|
34
34
|
ScoreKwargsTypedDict
|
35
35
|
)
|
36
|
+
from snowflake.ml.model._signatures import utils as model_signature_utils
|
37
|
+
from snowflake.ml.model.model_signature import (
|
38
|
+
BaseFeatureSpec,
|
39
|
+
DataType,
|
40
|
+
FeatureSpec,
|
41
|
+
ModelSignature,
|
42
|
+
_infer_signature,
|
43
|
+
_rename_signature_with_snowflake_identifiers,
|
44
|
+
)
|
36
45
|
|
37
46
|
from snowflake.ml.modeling._internal.model_transformer_builder import ModelTransformerBuilder
|
38
47
|
|
@@ -43,16 +52,6 @@ from snowflake.ml.modeling._internal.estimator_utils import (
|
|
43
52
|
validate_sklearn_args,
|
44
53
|
)
|
45
54
|
|
46
|
-
from snowflake.ml.model.model_signature import (
|
47
|
-
DataType,
|
48
|
-
FeatureSpec,
|
49
|
-
ModelSignature,
|
50
|
-
_infer_signature,
|
51
|
-
_rename_signature_with_snowflake_identifiers,
|
52
|
-
BaseFeatureSpec,
|
53
|
-
)
|
54
|
-
from snowflake.ml.model._signatures import utils as model_signature_utils
|
55
|
-
|
56
55
|
_PROJECT = "ModelDevelopment"
|
57
56
|
# Derive subproject from module name by removing "sklearn"
|
58
57
|
# and converting module name from underscore to CamelCase
|
@@ -253,12 +252,7 @@ class MeanShift(BaseTransformer):
|
|
253
252
|
)
|
254
253
|
return selected_cols
|
255
254
|
|
256
|
-
|
257
|
-
project=_PROJECT,
|
258
|
-
subproject=_SUBPROJECT,
|
259
|
-
custom_tags=dict([("autogen", True)]),
|
260
|
-
)
|
261
|
-
def fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "MeanShift":
|
255
|
+
def _fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "MeanShift":
|
262
256
|
"""Perform clustering
|
263
257
|
For more details on this function, see [sklearn.cluster.MeanShift.fit]
|
264
258
|
(https://scikit-learn.org/stable/modules/generated/sklearn.cluster.MeanShift.html#sklearn.cluster.MeanShift.fit)
|
@@ -285,12 +279,14 @@ class MeanShift(BaseTransformer):
|
|
285
279
|
|
286
280
|
self._snowpark_cols = dataset.select(self.input_cols).columns
|
287
281
|
|
288
|
-
|
282
|
+
# If we are already in a stored procedure, no need to kick off another one.
|
289
283
|
if SNOWML_SPROC_ENV in os.environ:
|
290
284
|
statement_params = telemetry.get_function_usage_statement_params(
|
291
285
|
project=_PROJECT,
|
292
286
|
subproject=_SUBPROJECT,
|
293
|
-
function_name=telemetry.get_statement_params_full_func_name(
|
287
|
+
function_name=telemetry.get_statement_params_full_func_name(
|
288
|
+
inspect.currentframe(), MeanShift.__class__.__name__
|
289
|
+
),
|
294
290
|
api_calls=[Session.call],
|
295
291
|
custom_tags=dict([("autogen", True)]) if self._autogenerated else None,
|
296
292
|
)
|
@@ -311,7 +307,7 @@ class MeanShift(BaseTransformer):
|
|
311
307
|
)
|
312
308
|
self._sklearn_object = model_trainer.train()
|
313
309
|
self._is_fitted = True
|
314
|
-
self.
|
310
|
+
self._generate_model_signatures(dataset)
|
315
311
|
return self
|
316
312
|
|
317
313
|
def _batch_inference_validate_snowpark(
|
@@ -387,7 +383,9 @@ class MeanShift(BaseTransformer):
|
|
387
383
|
# when it is classifier, infer the datatype from label columns
|
388
384
|
if expected_type_inferred == "" and 'predict' in self.model_signatures:
|
389
385
|
# Batch inference takes a single expected output column type. Use the first columns type for now.
|
390
|
-
label_cols_signatures = [
|
386
|
+
label_cols_signatures = [
|
387
|
+
row for row in self.model_signatures['predict'].outputs if row.name in self.output_cols
|
388
|
+
]
|
391
389
|
if len(label_cols_signatures) == 0:
|
392
390
|
error_str = f"Output columns {self.output_cols} do not match model signatures {self.model_signatures['predict'].outputs}."
|
393
391
|
raise exceptions.SnowflakeMLException(
|
@@ -395,25 +393,22 @@ class MeanShift(BaseTransformer):
|
|
395
393
|
original_exception=ValueError(error_str),
|
396
394
|
)
|
397
395
|
|
398
|
-
expected_type_inferred = convert_sp_to_sf_type(
|
399
|
-
label_cols_signatures[0].as_snowpark_type()
|
400
|
-
)
|
396
|
+
expected_type_inferred = convert_sp_to_sf_type(label_cols_signatures[0].as_snowpark_type())
|
401
397
|
|
402
398
|
self._deps = self._batch_inference_validate_snowpark(dataset=dataset, inference_method=inference_method)
|
403
|
-
assert isinstance(
|
399
|
+
assert isinstance(
|
400
|
+
dataset._session, Session
|
401
|
+
) # mypy does not recognize the check in _batch_inference_validate_snowpark()
|
404
402
|
|
405
403
|
transform_kwargs = dict(
|
406
|
-
session
|
407
|
-
dependencies
|
408
|
-
drop_input_cols
|
409
|
-
expected_output_cols_type
|
404
|
+
session=dataset._session,
|
405
|
+
dependencies=self._deps,
|
406
|
+
drop_input_cols=self._drop_input_cols,
|
407
|
+
expected_output_cols_type=expected_type_inferred,
|
410
408
|
)
|
411
409
|
|
412
410
|
elif isinstance(dataset, pd.DataFrame):
|
413
|
-
transform_kwargs = dict(
|
414
|
-
snowpark_input_cols = self._snowpark_cols,
|
415
|
-
drop_input_cols = self._drop_input_cols
|
416
|
-
)
|
411
|
+
transform_kwargs = dict(snowpark_input_cols=self._snowpark_cols, drop_input_cols=self._drop_input_cols)
|
417
412
|
|
418
413
|
transform_handlers = ModelTransformerBuilder.build(
|
419
414
|
dataset=dataset,
|
@@ -453,7 +448,7 @@ class MeanShift(BaseTransformer):
|
|
453
448
|
Transformed dataset.
|
454
449
|
"""
|
455
450
|
super()._check_dataset_type(dataset)
|
456
|
-
inference_method="transform"
|
451
|
+
inference_method = "transform"
|
457
452
|
|
458
453
|
# This dictionary contains optional kwargs for batch inference. These kwargs
|
459
454
|
# are specific to the type of dataset used.
|
@@ -490,17 +485,14 @@ class MeanShift(BaseTransformer):
|
|
490
485
|
assert isinstance(dataset._session, Session) # mypy does not recognize the check in _batch_inference_validate_snowpark()
|
491
486
|
|
492
487
|
transform_kwargs = dict(
|
493
|
-
session
|
494
|
-
dependencies
|
495
|
-
drop_input_cols
|
496
|
-
expected_output_cols_type
|
488
|
+
session=dataset._session,
|
489
|
+
dependencies=self._deps,
|
490
|
+
drop_input_cols=self._drop_input_cols,
|
491
|
+
expected_output_cols_type=expected_dtype,
|
497
492
|
)
|
498
493
|
|
499
494
|
elif isinstance(dataset, pd.DataFrame):
|
500
|
-
transform_kwargs = dict(
|
501
|
-
snowpark_input_cols = self._snowpark_cols,
|
502
|
-
drop_input_cols = self._drop_input_cols
|
503
|
-
)
|
495
|
+
transform_kwargs = dict(snowpark_input_cols=self._snowpark_cols, drop_input_cols=self._drop_input_cols)
|
504
496
|
|
505
497
|
transform_handlers = ModelTransformerBuilder.build(
|
506
498
|
dataset=dataset,
|
@@ -519,7 +511,11 @@ class MeanShift(BaseTransformer):
|
|
519
511
|
return output_df
|
520
512
|
|
521
513
|
@available_if(original_estimator_has_callable("fit_predict")) # type: ignore[misc]
|
522
|
-
def fit_predict(
|
514
|
+
def fit_predict(
|
515
|
+
self,
|
516
|
+
dataset: Union[DataFrame, pd.DataFrame],
|
517
|
+
output_cols_prefix: str = "fit_predict_",
|
518
|
+
) -> Union[DataFrame, pd.DataFrame]:
|
523
519
|
""" Perform clustering on `X` and returns cluster labels
|
524
520
|
For more details on this function, see [sklearn.cluster.MeanShift.fit_predict]
|
525
521
|
(https://scikit-learn.org/stable/modules/generated/sklearn.cluster.MeanShift.html#sklearn.cluster.MeanShift.fit_predict)
|
@@ -546,7 +542,9 @@ class MeanShift(BaseTransformer):
|
|
546
542
|
)
|
547
543
|
output_result, fitted_estimator = model_trainer.train_fit_predict(
|
548
544
|
drop_input_cols=self._drop_input_cols,
|
549
|
-
expected_output_cols_list=
|
545
|
+
expected_output_cols_list=(
|
546
|
+
self.output_cols if self.output_cols else self._get_output_column_names(output_cols_prefix)
|
547
|
+
),
|
550
548
|
)
|
551
549
|
self._sklearn_object = fitted_estimator
|
552
550
|
self._is_fitted = True
|
@@ -563,6 +561,62 @@ class MeanShift(BaseTransformer):
|
|
563
561
|
assert self._sklearn_object is not None
|
564
562
|
return self._sklearn_object.embedding_
|
565
563
|
|
564
|
+
|
565
|
+
def _get_output_column_names(self, output_cols_prefix: str, output_cols: Optional[List[str]] = None) -> List[str]:
|
566
|
+
""" Returns the list of output columns for predict_proba(), decision_function(), etc.. functions.
|
567
|
+
Returns a list with output_cols_prefix as the only element if the estimator is not a classifier.
|
568
|
+
"""
|
569
|
+
output_cols_prefix = identifier.resolve_identifier(output_cols_prefix)
|
570
|
+
# The following condition is introduced for kneighbors methods, and not used in other methods
|
571
|
+
if output_cols:
|
572
|
+
output_cols = [
|
573
|
+
identifier.concat_names([output_cols_prefix, identifier.resolve_identifier(c)])
|
574
|
+
for c in output_cols
|
575
|
+
]
|
576
|
+
elif getattr(self._sklearn_object, "classes_", None) is None:
|
577
|
+
output_cols = [output_cols_prefix]
|
578
|
+
elif self._sklearn_object is not None:
|
579
|
+
classes = self._sklearn_object.classes_
|
580
|
+
if isinstance(classes, numpy.ndarray):
|
581
|
+
output_cols = [f'{output_cols_prefix}{str(c)}' for c in classes.tolist()]
|
582
|
+
elif isinstance(classes, list) and len(classes) > 0 and isinstance(classes[0], numpy.ndarray):
|
583
|
+
# If the estimator is a multioutput estimator, classes_ will be a list of ndarrays.
|
584
|
+
output_cols = []
|
585
|
+
for i, cl in enumerate(classes):
|
586
|
+
# For binary classification, there is only one output column for each class
|
587
|
+
# ndarray as the two classes are complementary.
|
588
|
+
if len(cl) == 2:
|
589
|
+
output_cols.append(f'{output_cols_prefix}{i}_{cl[0]}')
|
590
|
+
else:
|
591
|
+
output_cols.extend([
|
592
|
+
f'{output_cols_prefix}{i}_{c}' for c in cl.tolist()
|
593
|
+
])
|
594
|
+
else:
|
595
|
+
output_cols = []
|
596
|
+
|
597
|
+
# Make sure column names are valid snowflake identifiers.
|
598
|
+
assert output_cols is not None # Make MyPy happy
|
599
|
+
rv = [identifier.rename_to_valid_snowflake_identifier(c) for c in output_cols]
|
600
|
+
|
601
|
+
return rv
|
602
|
+
|
603
|
+
def _align_expected_output_names(
|
604
|
+
self, method: str, dataset: DataFrame, expected_output_cols_list: List[str], output_cols_prefix: str
|
605
|
+
) -> List[str]:
|
606
|
+
# in case the inferred output column names dimension is different
|
607
|
+
# we use one line of snowpark dataframe and put it into sklearn estimator using pandas
|
608
|
+
output_df_pd = getattr(self, method)(dataset.limit(1).to_pandas(), output_cols_prefix)
|
609
|
+
output_df_columns = list(output_df_pd.columns)
|
610
|
+
output_df_columns_set: Set[str] = set(output_df_columns) - set(dataset.columns)
|
611
|
+
if self.sample_weight_col:
|
612
|
+
output_df_columns_set -= set(self.sample_weight_col)
|
613
|
+
# if the dimension of inferred output column names is correct; use it
|
614
|
+
if len(expected_output_cols_list) == len(output_df_columns_set):
|
615
|
+
return expected_output_cols_list
|
616
|
+
# otherwise, use the sklearn estimator's output
|
617
|
+
else:
|
618
|
+
return sorted(list(output_df_columns_set), key=lambda x: output_df_columns.index(x))
|
619
|
+
|
566
620
|
@available_if(original_estimator_has_callable("predict_proba")) # type: ignore[misc]
|
567
621
|
@telemetry.send_api_usage_telemetry(
|
568
622
|
project=_PROJECT,
|
@@ -593,24 +647,28 @@ class MeanShift(BaseTransformer):
|
|
593
647
|
# are specific to the type of dataset used.
|
594
648
|
transform_kwargs: BatchInferenceKwargsTypedDict = dict()
|
595
649
|
|
650
|
+
expected_output_cols = self._get_output_column_names(output_cols_prefix)
|
651
|
+
|
596
652
|
if isinstance(dataset, DataFrame):
|
597
653
|
self._deps = self._batch_inference_validate_snowpark(
|
598
654
|
dataset=dataset,
|
599
655
|
inference_method=inference_method,
|
600
656
|
)
|
601
|
-
assert isinstance(
|
657
|
+
assert isinstance(
|
658
|
+
dataset._session, Session
|
659
|
+
) # mypy does not recognize the check in _batch_inference_validate_snowpark()
|
602
660
|
transform_kwargs = dict(
|
603
661
|
session=dataset._session,
|
604
662
|
dependencies=self._deps,
|
605
|
-
drop_input_cols
|
663
|
+
drop_input_cols=self._drop_input_cols,
|
606
664
|
expected_output_cols_type="float",
|
607
665
|
)
|
666
|
+
expected_output_cols = self._align_expected_output_names(
|
667
|
+
inference_method, dataset, expected_output_cols, output_cols_prefix
|
668
|
+
)
|
608
669
|
|
609
670
|
elif isinstance(dataset, pd.DataFrame):
|
610
|
-
transform_kwargs = dict(
|
611
|
-
snowpark_input_cols = self._snowpark_cols,
|
612
|
-
drop_input_cols = self._drop_input_cols
|
613
|
-
)
|
671
|
+
transform_kwargs = dict(snowpark_input_cols=self._snowpark_cols, drop_input_cols=self._drop_input_cols)
|
614
672
|
|
615
673
|
transform_handlers = ModelTransformerBuilder.build(
|
616
674
|
dataset=dataset,
|
@@ -622,7 +680,7 @@ class MeanShift(BaseTransformer):
|
|
622
680
|
output_df: DATAFRAME_TYPE = transform_handlers.batch_inference(
|
623
681
|
inference_method=inference_method,
|
624
682
|
input_cols=self.input_cols,
|
625
|
-
expected_output_cols=
|
683
|
+
expected_output_cols=expected_output_cols,
|
626
684
|
**transform_kwargs
|
627
685
|
)
|
628
686
|
return output_df
|
@@ -652,7 +710,8 @@ class MeanShift(BaseTransformer):
|
|
652
710
|
Output dataset with log probability of the sample for each class in the model.
|
653
711
|
"""
|
654
712
|
super()._check_dataset_type(dataset)
|
655
|
-
inference_method="predict_log_proba"
|
713
|
+
inference_method = "predict_log_proba"
|
714
|
+
expected_output_cols = self._get_output_column_names(output_cols_prefix)
|
656
715
|
|
657
716
|
# This dictionary contains optional kwargs for batch inference. These kwargs
|
658
717
|
# are specific to the type of dataset used.
|
@@ -663,18 +722,20 @@ class MeanShift(BaseTransformer):
|
|
663
722
|
dataset=dataset,
|
664
723
|
inference_method=inference_method,
|
665
724
|
)
|
666
|
-
assert isinstance(
|
725
|
+
assert isinstance(
|
726
|
+
dataset._session, Session
|
727
|
+
) # mypy does not recognize the check in _batch_inference_validate_snowpark()
|
667
728
|
transform_kwargs = dict(
|
668
729
|
session=dataset._session,
|
669
730
|
dependencies=self._deps,
|
670
|
-
drop_input_cols
|
731
|
+
drop_input_cols=self._drop_input_cols,
|
671
732
|
expected_output_cols_type="float",
|
672
733
|
)
|
734
|
+
expected_output_cols = self._align_expected_output_names(
|
735
|
+
inference_method, dataset, expected_output_cols, output_cols_prefix
|
736
|
+
)
|
673
737
|
elif isinstance(dataset, pd.DataFrame):
|
674
|
-
transform_kwargs = dict(
|
675
|
-
snowpark_input_cols = self._snowpark_cols,
|
676
|
-
drop_input_cols = self._drop_input_cols
|
677
|
-
)
|
738
|
+
transform_kwargs = dict(snowpark_input_cols=self._snowpark_cols, drop_input_cols=self._drop_input_cols)
|
678
739
|
|
679
740
|
transform_handlers = ModelTransformerBuilder.build(
|
680
741
|
dataset=dataset,
|
@@ -687,7 +748,7 @@ class MeanShift(BaseTransformer):
|
|
687
748
|
output_df: DATAFRAME_TYPE = transform_handlers.batch_inference(
|
688
749
|
inference_method=inference_method,
|
689
750
|
input_cols=self.input_cols,
|
690
|
-
expected_output_cols=
|
751
|
+
expected_output_cols=expected_output_cols,
|
691
752
|
**transform_kwargs
|
692
753
|
)
|
693
754
|
return output_df
|
@@ -713,30 +774,34 @@ class MeanShift(BaseTransformer):
|
|
713
774
|
Output dataset with results of the decision function for the samples in input dataset.
|
714
775
|
"""
|
715
776
|
super()._check_dataset_type(dataset)
|
716
|
-
inference_method="decision_function"
|
777
|
+
inference_method = "decision_function"
|
717
778
|
|
718
779
|
# This dictionary contains optional kwargs for batch inference. These kwargs
|
719
780
|
# are specific to the type of dataset used.
|
720
781
|
transform_kwargs: BatchInferenceKwargsTypedDict = dict()
|
721
782
|
|
783
|
+
expected_output_cols = self._get_output_column_names(output_cols_prefix)
|
784
|
+
|
722
785
|
if isinstance(dataset, DataFrame):
|
723
786
|
self._deps = self._batch_inference_validate_snowpark(
|
724
787
|
dataset=dataset,
|
725
788
|
inference_method=inference_method,
|
726
789
|
)
|
727
|
-
assert isinstance(
|
790
|
+
assert isinstance(
|
791
|
+
dataset._session, Session
|
792
|
+
) # mypy does not recognize the check in _batch_inference_validate_snowpark()
|
728
793
|
transform_kwargs = dict(
|
729
794
|
session=dataset._session,
|
730
795
|
dependencies=self._deps,
|
731
|
-
drop_input_cols
|
796
|
+
drop_input_cols=self._drop_input_cols,
|
732
797
|
expected_output_cols_type="float",
|
733
798
|
)
|
799
|
+
expected_output_cols = self._align_expected_output_names(
|
800
|
+
inference_method, dataset, expected_output_cols, output_cols_prefix
|
801
|
+
)
|
734
802
|
|
735
803
|
elif isinstance(dataset, pd.DataFrame):
|
736
|
-
transform_kwargs = dict(
|
737
|
-
snowpark_input_cols = self._snowpark_cols,
|
738
|
-
drop_input_cols = self._drop_input_cols
|
739
|
-
)
|
804
|
+
transform_kwargs = dict(snowpark_input_cols=self._snowpark_cols, drop_input_cols=self._drop_input_cols)
|
740
805
|
|
741
806
|
transform_handlers = ModelTransformerBuilder.build(
|
742
807
|
dataset=dataset,
|
@@ -749,7 +814,7 @@ class MeanShift(BaseTransformer):
|
|
749
814
|
output_df: DATAFRAME_TYPE = transform_handlers.batch_inference(
|
750
815
|
inference_method=inference_method,
|
751
816
|
input_cols=self.input_cols,
|
752
|
-
expected_output_cols=
|
817
|
+
expected_output_cols=expected_output_cols,
|
753
818
|
**transform_kwargs
|
754
819
|
)
|
755
820
|
return output_df
|
@@ -778,12 +843,14 @@ class MeanShift(BaseTransformer):
|
|
778
843
|
Output dataset with probability of the sample for each class in the model.
|
779
844
|
"""
|
780
845
|
super()._check_dataset_type(dataset)
|
781
|
-
inference_method="score_samples"
|
846
|
+
inference_method = "score_samples"
|
782
847
|
|
783
848
|
# This dictionary contains optional kwargs for batch inference. These kwargs
|
784
849
|
# are specific to the type of dataset used.
|
785
850
|
transform_kwargs: BatchInferenceKwargsTypedDict = dict()
|
786
851
|
|
852
|
+
expected_output_cols = self._get_output_column_names(output_cols_prefix)
|
853
|
+
|
787
854
|
if isinstance(dataset, DataFrame):
|
788
855
|
self._deps = self._batch_inference_validate_snowpark(
|
789
856
|
dataset=dataset,
|
@@ -796,6 +863,9 @@ class MeanShift(BaseTransformer):
|
|
796
863
|
drop_input_cols = self._drop_input_cols,
|
797
864
|
expected_output_cols_type="float",
|
798
865
|
)
|
866
|
+
expected_output_cols = self._align_expected_output_names(
|
867
|
+
inference_method, dataset, expected_output_cols, output_cols_prefix
|
868
|
+
)
|
799
869
|
|
800
870
|
elif isinstance(dataset, pd.DataFrame):
|
801
871
|
transform_kwargs = dict(
|
@@ -814,7 +884,7 @@ class MeanShift(BaseTransformer):
|
|
814
884
|
output_df: DATAFRAME_TYPE = transform_handlers.batch_inference(
|
815
885
|
inference_method=inference_method,
|
816
886
|
input_cols=self.input_cols,
|
817
|
-
expected_output_cols=
|
887
|
+
expected_output_cols=expected_output_cols,
|
818
888
|
**transform_kwargs
|
819
889
|
)
|
820
890
|
return output_df
|
@@ -959,50 +1029,84 @@ class MeanShift(BaseTransformer):
|
|
959
1029
|
)
|
960
1030
|
return output_df
|
961
1031
|
|
1032
|
+
|
1033
|
+
|
1034
|
+
def to_sklearn(self) -> Any:
|
1035
|
+
"""Get sklearn.cluster.MeanShift object.
|
1036
|
+
"""
|
1037
|
+
if self._sklearn_object is None:
|
1038
|
+
self._sklearn_object = self._create_sklearn_object()
|
1039
|
+
return self._sklearn_object
|
1040
|
+
|
1041
|
+
def to_xgboost(self) -> Any:
|
1042
|
+
raise exceptions.SnowflakeMLException(
|
1043
|
+
error_code=error_codes.METHOD_NOT_ALLOWED,
|
1044
|
+
original_exception=AttributeError(
|
1045
|
+
modeling_error_messages.UNSUPPORTED_MODEL_CONVERSION.format(
|
1046
|
+
"to_xgboost()",
|
1047
|
+
"to_sklearn()"
|
1048
|
+
)
|
1049
|
+
),
|
1050
|
+
)
|
1051
|
+
|
1052
|
+
def to_lightgbm(self) -> Any:
|
1053
|
+
raise exceptions.SnowflakeMLException(
|
1054
|
+
error_code=error_codes.METHOD_NOT_ALLOWED,
|
1055
|
+
original_exception=AttributeError(
|
1056
|
+
modeling_error_messages.UNSUPPORTED_MODEL_CONVERSION.format(
|
1057
|
+
"to_lightgbm()",
|
1058
|
+
"to_sklearn()"
|
1059
|
+
)
|
1060
|
+
),
|
1061
|
+
)
|
962
1062
|
|
963
|
-
def
|
1063
|
+
def _get_dependencies(self) -> List[str]:
|
1064
|
+
return self._deps
|
1065
|
+
|
1066
|
+
|
1067
|
+
def _generate_model_signatures(self, dataset: Union[DataFrame, pd.DataFrame]) -> None:
|
964
1068
|
self._model_signature_dict = dict()
|
965
1069
|
|
966
1070
|
PROB_FUNCTIONS = ["predict_log_proba", "predict_proba", "decision_function"]
|
967
1071
|
|
968
|
-
inputs = list(_infer_signature(dataset[self.input_cols], "input"))
|
1072
|
+
inputs = list(_infer_signature(dataset[self.input_cols], "input", use_snowflake_identifiers=True))
|
969
1073
|
outputs: List[BaseFeatureSpec] = []
|
970
1074
|
if hasattr(self, "predict"):
|
971
1075
|
# keep mypy happy
|
972
|
-
assert self._sklearn_object is not None and hasattr(self._sklearn_object, "_estimator_type")
|
1076
|
+
assert self._sklearn_object is not None and hasattr(self._sklearn_object, "_estimator_type")
|
973
1077
|
# For classifier, the type of predict is the same as the type of label
|
974
|
-
if self._sklearn_object._estimator_type ==
|
975
|
-
|
1078
|
+
if self._sklearn_object._estimator_type == "classifier":
|
1079
|
+
# label columns is the desired type for output
|
976
1080
|
outputs = list(_infer_signature(dataset[self.label_cols], "output", use_snowflake_identifiers=True))
|
977
1081
|
# rename the output columns
|
978
1082
|
outputs = list(model_signature_utils.rename_features(outputs, self.output_cols))
|
979
|
-
self._model_signature_dict["predict"] = ModelSignature(
|
980
|
-
|
981
|
-
|
1083
|
+
self._model_signature_dict["predict"] = ModelSignature(
|
1084
|
+
inputs, ([] if self._drop_input_cols else inputs) + outputs
|
1085
|
+
)
|
982
1086
|
# For mixture models that use the density mixin, `predict` returns the argmax of the log prob.
|
983
1087
|
# For outlier models, returns -1 for outliers and 1 for inliers.
|
984
|
-
# Clusterer returns int64 cluster labels.
|
1088
|
+
# Clusterer returns int64 cluster labels.
|
985
1089
|
elif self._sklearn_object._estimator_type in ["DensityEstimator", "clusterer", "outlier_detector"]:
|
986
1090
|
outputs = [FeatureSpec(dtype=DataType.INT64, name=c) for c in self.output_cols]
|
987
|
-
self._model_signature_dict["predict"] = ModelSignature(
|
988
|
-
|
989
|
-
|
990
|
-
|
1091
|
+
self._model_signature_dict["predict"] = ModelSignature(
|
1092
|
+
inputs, ([] if self._drop_input_cols else inputs) + outputs
|
1093
|
+
)
|
1094
|
+
|
991
1095
|
# For regressor, the type of predict is float64
|
992
|
-
elif self._sklearn_object._estimator_type ==
|
1096
|
+
elif self._sklearn_object._estimator_type == "regressor":
|
993
1097
|
outputs = [FeatureSpec(dtype=DataType.DOUBLE, name=c) for c in self.output_cols]
|
994
|
-
self._model_signature_dict["predict"] = ModelSignature(
|
995
|
-
|
996
|
-
|
997
|
-
|
1098
|
+
self._model_signature_dict["predict"] = ModelSignature(
|
1099
|
+
inputs, ([] if self._drop_input_cols else inputs) + outputs
|
1100
|
+
)
|
1101
|
+
|
998
1102
|
for prob_func in PROB_FUNCTIONS:
|
999
1103
|
if hasattr(self, prob_func):
|
1000
1104
|
output_cols_prefix: str = f"{prob_func}_"
|
1001
1105
|
output_column_names = self._get_output_column_names(output_cols_prefix)
|
1002
1106
|
outputs = [FeatureSpec(dtype=DataType.DOUBLE, name=c) for c in output_column_names]
|
1003
|
-
self._model_signature_dict[prob_func] = ModelSignature(
|
1004
|
-
|
1005
|
-
|
1107
|
+
self._model_signature_dict[prob_func] = ModelSignature(
|
1108
|
+
inputs, ([] if self._drop_input_cols else inputs) + outputs
|
1109
|
+
)
|
1006
1110
|
|
1007
1111
|
# Output signature names may still need to be renamed, since they were not created with `_infer_signature`.
|
1008
1112
|
items = list(self._model_signature_dict.items())
|
@@ -1015,10 +1119,10 @@ class MeanShift(BaseTransformer):
|
|
1015
1119
|
"""Returns model signature of current class.
|
1016
1120
|
|
1017
1121
|
Raises:
|
1018
|
-
|
1122
|
+
SnowflakeMLException: If estimator is not fitted, then model signature cannot be inferred
|
1019
1123
|
|
1020
1124
|
Returns:
|
1021
|
-
Dict
|
1125
|
+
Dict with each method and its input output signature
|
1022
1126
|
"""
|
1023
1127
|
if self._model_signature_dict is None:
|
1024
1128
|
raise exceptions.SnowflakeMLException(
|
@@ -1026,35 +1130,3 @@ class MeanShift(BaseTransformer):
|
|
1026
1130
|
original_exception=RuntimeError("Estimator not fitted before accessing property model_signatures!"),
|
1027
1131
|
)
|
1028
1132
|
return self._model_signature_dict
|
1029
|
-
|
1030
|
-
def to_sklearn(self) -> Any:
|
1031
|
-
"""Get sklearn.cluster.MeanShift object.
|
1032
|
-
"""
|
1033
|
-
if self._sklearn_object is None:
|
1034
|
-
self._sklearn_object = self._create_sklearn_object()
|
1035
|
-
return self._sklearn_object
|
1036
|
-
|
1037
|
-
def to_xgboost(self) -> Any:
|
1038
|
-
raise exceptions.SnowflakeMLException(
|
1039
|
-
error_code=error_codes.METHOD_NOT_ALLOWED,
|
1040
|
-
original_exception=AttributeError(
|
1041
|
-
modeling_error_messages.UNSUPPORTED_MODEL_CONVERSION.format(
|
1042
|
-
"to_xgboost()",
|
1043
|
-
"to_sklearn()"
|
1044
|
-
)
|
1045
|
-
),
|
1046
|
-
)
|
1047
|
-
|
1048
|
-
def to_lightgbm(self) -> Any:
|
1049
|
-
raise exceptions.SnowflakeMLException(
|
1050
|
-
error_code=error_codes.METHOD_NOT_ALLOWED,
|
1051
|
-
original_exception=AttributeError(
|
1052
|
-
modeling_error_messages.UNSUPPORTED_MODEL_CONVERSION.format(
|
1053
|
-
"to_lightgbm()",
|
1054
|
-
"to_sklearn()"
|
1055
|
-
)
|
1056
|
-
),
|
1057
|
-
)
|
1058
|
-
|
1059
|
-
def _get_dependencies(self) -> List[str]:
|
1060
|
-
return self._deps
|