snowflake-ml-python 1.4.0__py3-none-any.whl → 1.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/ml/_internal/env_utils.py +11 -1
- snowflake/ml/_internal/utils/identifier.py +3 -1
- snowflake/ml/_internal/utils/sql_identifier.py +2 -6
- snowflake/ml/feature_store/feature_store.py +151 -78
- snowflake/ml/feature_store/feature_view.py +12 -24
- snowflake/ml/fileset/sfcfs.py +56 -50
- snowflake/ml/fileset/stage_fs.py +48 -13
- snowflake/ml/model/_client/model/model_version_impl.py +2 -50
- snowflake/ml/model/_client/ops/model_ops.py +78 -29
- snowflake/ml/model/_client/sql/model.py +23 -2
- snowflake/ml/model/_client/sql/model_version.py +22 -1
- snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +19 -54
- snowflake/ml/model/_model_composer/model_manifest/model_manifest_schema.py +8 -1
- snowflake/ml/model/_model_composer/model_method/model_method.py +6 -10
- snowflake/ml/model/_packager/model_handlers/catboost.py +206 -0
- snowflake/ml/model/_packager/model_handlers/lightgbm.py +218 -0
- snowflake/ml/model/_packager/model_handlers/sklearn.py +3 -0
- snowflake/ml/model/_packager/model_meta/_core_requirements.py +1 -1
- snowflake/ml/model/_packager/model_meta/model_meta.py +36 -6
- snowflake/ml/model/_packager/model_meta/model_meta_schema.py +20 -1
- snowflake/ml/model/_packager/model_meta_migrator/migrator_plans.py +3 -1
- snowflake/ml/model/_packager/model_packager.py +2 -2
- snowflake/ml/model/{_model_composer/model_runtime/_runtime_requirements.py → _packager/model_runtime/_snowml_inference_alternative_requirements.py} +1 -1
- snowflake/ml/model/_packager/model_runtime/model_runtime.py +137 -0
- snowflake/ml/model/type_hints.py +21 -2
- snowflake/ml/modeling/_internal/estimator_utils.py +16 -11
- snowflake/ml/modeling/_internal/local_implementations/pandas_handlers.py +4 -1
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +13 -14
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_handlers.py +5 -5
- snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +195 -123
- snowflake/ml/modeling/cluster/affinity_propagation.py +195 -123
- snowflake/ml/modeling/cluster/agglomerative_clustering.py +195 -123
- snowflake/ml/modeling/cluster/birch.py +195 -123
- snowflake/ml/modeling/cluster/bisecting_k_means.py +195 -123
- snowflake/ml/modeling/cluster/dbscan.py +195 -123
- snowflake/ml/modeling/cluster/feature_agglomeration.py +195 -123
- snowflake/ml/modeling/cluster/k_means.py +195 -123
- snowflake/ml/modeling/cluster/mean_shift.py +195 -123
- snowflake/ml/modeling/cluster/mini_batch_k_means.py +195 -123
- snowflake/ml/modeling/cluster/optics.py +195 -123
- snowflake/ml/modeling/cluster/spectral_biclustering.py +195 -123
- snowflake/ml/modeling/cluster/spectral_clustering.py +195 -123
- snowflake/ml/modeling/cluster/spectral_coclustering.py +195 -123
- snowflake/ml/modeling/compose/column_transformer.py +195 -123
- snowflake/ml/modeling/compose/transformed_target_regressor.py +195 -123
- snowflake/ml/modeling/covariance/elliptic_envelope.py +195 -123
- snowflake/ml/modeling/covariance/empirical_covariance.py +195 -123
- snowflake/ml/modeling/covariance/graphical_lasso.py +195 -123
- snowflake/ml/modeling/covariance/graphical_lasso_cv.py +195 -123
- snowflake/ml/modeling/covariance/ledoit_wolf.py +195 -123
- snowflake/ml/modeling/covariance/min_cov_det.py +195 -123
- snowflake/ml/modeling/covariance/oas.py +195 -123
- snowflake/ml/modeling/covariance/shrunk_covariance.py +195 -123
- snowflake/ml/modeling/decomposition/dictionary_learning.py +195 -123
- snowflake/ml/modeling/decomposition/factor_analysis.py +195 -123
- snowflake/ml/modeling/decomposition/fast_ica.py +195 -123
- snowflake/ml/modeling/decomposition/incremental_pca.py +195 -123
- snowflake/ml/modeling/decomposition/kernel_pca.py +195 -123
- snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +195 -123
- snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +195 -123
- snowflake/ml/modeling/decomposition/pca.py +195 -123
- snowflake/ml/modeling/decomposition/sparse_pca.py +195 -123
- snowflake/ml/modeling/decomposition/truncated_svd.py +195 -123
- snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +195 -123
- snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +195 -123
- snowflake/ml/modeling/ensemble/ada_boost_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/ada_boost_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/bagging_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/bagging_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/extra_trees_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/extra_trees_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/isolation_forest.py +195 -123
- snowflake/ml/modeling/ensemble/random_forest_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/random_forest_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/stacking_regressor.py +195 -123
- snowflake/ml/modeling/ensemble/voting_classifier.py +195 -123
- snowflake/ml/modeling/ensemble/voting_regressor.py +195 -123
- snowflake/ml/modeling/feature_selection/generic_univariate_select.py +195 -123
- snowflake/ml/modeling/feature_selection/select_fdr.py +195 -123
- snowflake/ml/modeling/feature_selection/select_fpr.py +195 -123
- snowflake/ml/modeling/feature_selection/select_fwe.py +195 -123
- snowflake/ml/modeling/feature_selection/select_k_best.py +195 -123
- snowflake/ml/modeling/feature_selection/select_percentile.py +195 -123
- snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +195 -123
- snowflake/ml/modeling/feature_selection/variance_threshold.py +195 -123
- snowflake/ml/modeling/framework/_utils.py +8 -1
- snowflake/ml/modeling/framework/base.py +9 -1
- snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +195 -123
- snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +195 -123
- snowflake/ml/modeling/impute/iterative_imputer.py +195 -123
- snowflake/ml/modeling/impute/knn_imputer.py +195 -123
- snowflake/ml/modeling/impute/missing_indicator.py +195 -123
- snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +195 -123
- snowflake/ml/modeling/kernel_approximation/nystroem.py +195 -123
- snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +195 -123
- snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +195 -123
- snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +195 -123
- snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +195 -123
- snowflake/ml/modeling/lightgbm/lgbm_classifier.py +195 -123
- snowflake/ml/modeling/lightgbm/lgbm_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/ard_regression.py +195 -123
- snowflake/ml/modeling/linear_model/bayesian_ridge.py +195 -123
- snowflake/ml/modeling/linear_model/elastic_net.py +195 -123
- snowflake/ml/modeling/linear_model/elastic_net_cv.py +195 -123
- snowflake/ml/modeling/linear_model/gamma_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/huber_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/lars.py +195 -123
- snowflake/ml/modeling/linear_model/lars_cv.py +195 -123
- snowflake/ml/modeling/linear_model/lasso.py +195 -123
- snowflake/ml/modeling/linear_model/lasso_cv.py +195 -123
- snowflake/ml/modeling/linear_model/lasso_lars.py +195 -123
- snowflake/ml/modeling/linear_model/lasso_lars_cv.py +195 -123
- snowflake/ml/modeling/linear_model/lasso_lars_ic.py +195 -123
- snowflake/ml/modeling/linear_model/linear_regression.py +195 -123
- snowflake/ml/modeling/linear_model/logistic_regression.py +195 -123
- snowflake/ml/modeling/linear_model/logistic_regression_cv.py +195 -123
- snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +195 -123
- snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +195 -123
- snowflake/ml/modeling/linear_model/multi_task_lasso.py +195 -123
- snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +195 -123
- snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +195 -123
- snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +195 -123
- snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/perceptron.py +195 -123
- snowflake/ml/modeling/linear_model/poisson_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/ransac_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/ridge.py +195 -123
- snowflake/ml/modeling/linear_model/ridge_classifier.py +195 -123
- snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +195 -123
- snowflake/ml/modeling/linear_model/ridge_cv.py +195 -123
- snowflake/ml/modeling/linear_model/sgd_classifier.py +195 -123
- snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +195 -123
- snowflake/ml/modeling/linear_model/sgd_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/theil_sen_regressor.py +195 -123
- snowflake/ml/modeling/linear_model/tweedie_regressor.py +195 -123
- snowflake/ml/modeling/manifold/isomap.py +195 -123
- snowflake/ml/modeling/manifold/mds.py +195 -123
- snowflake/ml/modeling/manifold/spectral_embedding.py +195 -123
- snowflake/ml/modeling/manifold/tsne.py +195 -123
- snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +195 -123
- snowflake/ml/modeling/mixture/gaussian_mixture.py +195 -123
- snowflake/ml/modeling/model_selection/grid_search_cv.py +42 -18
- snowflake/ml/modeling/model_selection/randomized_search_cv.py +42 -18
- snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +195 -123
- snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +195 -123
- snowflake/ml/modeling/multiclass/output_code_classifier.py +195 -123
- snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +195 -123
- snowflake/ml/modeling/naive_bayes/categorical_nb.py +195 -123
- snowflake/ml/modeling/naive_bayes/complement_nb.py +195 -123
- snowflake/ml/modeling/naive_bayes/gaussian_nb.py +195 -123
- snowflake/ml/modeling/naive_bayes/multinomial_nb.py +195 -123
- snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +195 -123
- snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +195 -123
- snowflake/ml/modeling/neighbors/kernel_density.py +195 -123
- snowflake/ml/modeling/neighbors/local_outlier_factor.py +195 -123
- snowflake/ml/modeling/neighbors/nearest_centroid.py +195 -123
- snowflake/ml/modeling/neighbors/nearest_neighbors.py +195 -123
- snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +195 -123
- snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +195 -123
- snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +195 -123
- snowflake/ml/modeling/neural_network/bernoulli_rbm.py +195 -123
- snowflake/ml/modeling/neural_network/mlp_classifier.py +195 -123
- snowflake/ml/modeling/neural_network/mlp_regressor.py +195 -123
- snowflake/ml/modeling/pipeline/pipeline.py +4 -4
- snowflake/ml/modeling/preprocessing/binarizer.py +1 -5
- snowflake/ml/modeling/preprocessing/k_bins_discretizer.py +1 -5
- snowflake/ml/modeling/preprocessing/label_encoder.py +1 -5
- snowflake/ml/modeling/preprocessing/max_abs_scaler.py +1 -5
- snowflake/ml/modeling/preprocessing/min_max_scaler.py +10 -12
- snowflake/ml/modeling/preprocessing/normalizer.py +1 -5
- snowflake/ml/modeling/preprocessing/one_hot_encoder.py +1 -5
- snowflake/ml/modeling/preprocessing/ordinal_encoder.py +1 -5
- snowflake/ml/modeling/preprocessing/polynomial_features.py +195 -123
- snowflake/ml/modeling/preprocessing/robust_scaler.py +1 -5
- snowflake/ml/modeling/preprocessing/standard_scaler.py +11 -11
- snowflake/ml/modeling/semi_supervised/label_propagation.py +195 -123
- snowflake/ml/modeling/semi_supervised/label_spreading.py +195 -123
- snowflake/ml/modeling/svm/linear_svc.py +195 -123
- snowflake/ml/modeling/svm/linear_svr.py +195 -123
- snowflake/ml/modeling/svm/nu_svc.py +195 -123
- snowflake/ml/modeling/svm/nu_svr.py +195 -123
- snowflake/ml/modeling/svm/svc.py +195 -123
- snowflake/ml/modeling/svm/svr.py +195 -123
- snowflake/ml/modeling/tree/decision_tree_classifier.py +195 -123
- snowflake/ml/modeling/tree/decision_tree_regressor.py +195 -123
- snowflake/ml/modeling/tree/extra_tree_classifier.py +195 -123
- snowflake/ml/modeling/tree/extra_tree_regressor.py +195 -123
- snowflake/ml/modeling/xgboost/xgb_classifier.py +195 -123
- snowflake/ml/modeling/xgboost/xgb_regressor.py +195 -123
- snowflake/ml/modeling/xgboost/xgbrf_classifier.py +195 -123
- snowflake/ml/modeling/xgboost/xgbrf_regressor.py +195 -123
- snowflake/ml/registry/registry.py +1 -1
- snowflake/ml/version.py +1 -1
- {snowflake_ml_python-1.4.0.dist-info → snowflake_ml_python-1.4.1.dist-info}/METADATA +68 -57
- {snowflake_ml_python-1.4.0.dist-info → snowflake_ml_python-1.4.1.dist-info}/RECORD +202 -200
- snowflake/ml/model/_model_composer/model_runtime/model_runtime.py +0 -97
- {snowflake_ml_python-1.4.0.dist-info → snowflake_ml_python-1.4.1.dist-info}/LICENSE.txt +0 -0
- {snowflake_ml_python-1.4.0.dist-info → snowflake_ml_python-1.4.1.dist-info}/WHEEL +0 -0
- {snowflake_ml_python-1.4.0.dist-info → snowflake_ml_python-1.4.1.dist-info}/top_level.txt +0 -0
@@ -33,6 +33,15 @@ from snowflake.ml.modeling._internal.transformer_protocols import (
|
|
33
33
|
BatchInferenceKwargsTypedDict,
|
34
34
|
ScoreKwargsTypedDict
|
35
35
|
)
|
36
|
+
from snowflake.ml.model._signatures import utils as model_signature_utils
|
37
|
+
from snowflake.ml.model.model_signature import (
|
38
|
+
BaseFeatureSpec,
|
39
|
+
DataType,
|
40
|
+
FeatureSpec,
|
41
|
+
ModelSignature,
|
42
|
+
_infer_signature,
|
43
|
+
_rename_signature_with_snowflake_identifiers,
|
44
|
+
)
|
36
45
|
|
37
46
|
from snowflake.ml.modeling._internal.model_transformer_builder import ModelTransformerBuilder
|
38
47
|
|
@@ -43,16 +52,6 @@ from snowflake.ml.modeling._internal.estimator_utils import (
|
|
43
52
|
validate_sklearn_args,
|
44
53
|
)
|
45
54
|
|
46
|
-
from snowflake.ml.model.model_signature import (
|
47
|
-
DataType,
|
48
|
-
FeatureSpec,
|
49
|
-
ModelSignature,
|
50
|
-
_infer_signature,
|
51
|
-
_rename_signature_with_snowflake_identifiers,
|
52
|
-
BaseFeatureSpec,
|
53
|
-
)
|
54
|
-
from snowflake.ml.model._signatures import utils as model_signature_utils
|
55
|
-
|
56
55
|
_PROJECT = "ModelDevelopment"
|
57
56
|
# Derive subproject from module name by removing "sklearn"
|
58
57
|
# and converting module name from underscore to CamelCase
|
@@ -224,12 +223,7 @@ class MissingIndicator(BaseTransformer):
|
|
224
223
|
)
|
225
224
|
return selected_cols
|
226
225
|
|
227
|
-
|
228
|
-
project=_PROJECT,
|
229
|
-
subproject=_SUBPROJECT,
|
230
|
-
custom_tags=dict([("autogen", True)]),
|
231
|
-
)
|
232
|
-
def fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "MissingIndicator":
|
226
|
+
def _fit(self, dataset: Union[DataFrame, pd.DataFrame]) -> "MissingIndicator":
|
233
227
|
"""Fit the transformer on `X`
|
234
228
|
For more details on this function, see [sklearn.impute.MissingIndicator.fit]
|
235
229
|
(https://scikit-learn.org/stable/modules/generated/sklearn.impute.MissingIndicator.html#sklearn.impute.MissingIndicator.fit)
|
@@ -256,12 +250,14 @@ class MissingIndicator(BaseTransformer):
|
|
256
250
|
|
257
251
|
self._snowpark_cols = dataset.select(self.input_cols).columns
|
258
252
|
|
259
|
-
|
253
|
+
# If we are already in a stored procedure, no need to kick off another one.
|
260
254
|
if SNOWML_SPROC_ENV in os.environ:
|
261
255
|
statement_params = telemetry.get_function_usage_statement_params(
|
262
256
|
project=_PROJECT,
|
263
257
|
subproject=_SUBPROJECT,
|
264
|
-
function_name=telemetry.get_statement_params_full_func_name(
|
258
|
+
function_name=telemetry.get_statement_params_full_func_name(
|
259
|
+
inspect.currentframe(), MissingIndicator.__class__.__name__
|
260
|
+
),
|
265
261
|
api_calls=[Session.call],
|
266
262
|
custom_tags=dict([("autogen", True)]) if self._autogenerated else None,
|
267
263
|
)
|
@@ -282,7 +278,7 @@ class MissingIndicator(BaseTransformer):
|
|
282
278
|
)
|
283
279
|
self._sklearn_object = model_trainer.train()
|
284
280
|
self._is_fitted = True
|
285
|
-
self.
|
281
|
+
self._generate_model_signatures(dataset)
|
286
282
|
return self
|
287
283
|
|
288
284
|
def _batch_inference_validate_snowpark(
|
@@ -356,7 +352,9 @@ class MissingIndicator(BaseTransformer):
|
|
356
352
|
# when it is classifier, infer the datatype from label columns
|
357
353
|
if expected_type_inferred == "" and 'predict' in self.model_signatures:
|
358
354
|
# Batch inference takes a single expected output column type. Use the first columns type for now.
|
359
|
-
label_cols_signatures = [
|
355
|
+
label_cols_signatures = [
|
356
|
+
row for row in self.model_signatures['predict'].outputs if row.name in self.output_cols
|
357
|
+
]
|
360
358
|
if len(label_cols_signatures) == 0:
|
361
359
|
error_str = f"Output columns {self.output_cols} do not match model signatures {self.model_signatures['predict'].outputs}."
|
362
360
|
raise exceptions.SnowflakeMLException(
|
@@ -364,25 +362,22 @@ class MissingIndicator(BaseTransformer):
|
|
364
362
|
original_exception=ValueError(error_str),
|
365
363
|
)
|
366
364
|
|
367
|
-
expected_type_inferred = convert_sp_to_sf_type(
|
368
|
-
label_cols_signatures[0].as_snowpark_type()
|
369
|
-
)
|
365
|
+
expected_type_inferred = convert_sp_to_sf_type(label_cols_signatures[0].as_snowpark_type())
|
370
366
|
|
371
367
|
self._deps = self._batch_inference_validate_snowpark(dataset=dataset, inference_method=inference_method)
|
372
|
-
assert isinstance(
|
368
|
+
assert isinstance(
|
369
|
+
dataset._session, Session
|
370
|
+
) # mypy does not recognize the check in _batch_inference_validate_snowpark()
|
373
371
|
|
374
372
|
transform_kwargs = dict(
|
375
|
-
session
|
376
|
-
dependencies
|
377
|
-
drop_input_cols
|
378
|
-
expected_output_cols_type
|
373
|
+
session=dataset._session,
|
374
|
+
dependencies=self._deps,
|
375
|
+
drop_input_cols=self._drop_input_cols,
|
376
|
+
expected_output_cols_type=expected_type_inferred,
|
379
377
|
)
|
380
378
|
|
381
379
|
elif isinstance(dataset, pd.DataFrame):
|
382
|
-
transform_kwargs = dict(
|
383
|
-
snowpark_input_cols = self._snowpark_cols,
|
384
|
-
drop_input_cols = self._drop_input_cols
|
385
|
-
)
|
380
|
+
transform_kwargs = dict(snowpark_input_cols=self._snowpark_cols, drop_input_cols=self._drop_input_cols)
|
386
381
|
|
387
382
|
transform_handlers = ModelTransformerBuilder.build(
|
388
383
|
dataset=dataset,
|
@@ -424,7 +419,7 @@ class MissingIndicator(BaseTransformer):
|
|
424
419
|
Transformed dataset.
|
425
420
|
"""
|
426
421
|
super()._check_dataset_type(dataset)
|
427
|
-
inference_method="transform"
|
422
|
+
inference_method = "transform"
|
428
423
|
|
429
424
|
# This dictionary contains optional kwargs for batch inference. These kwargs
|
430
425
|
# are specific to the type of dataset used.
|
@@ -461,17 +456,14 @@ class MissingIndicator(BaseTransformer):
|
|
461
456
|
assert isinstance(dataset._session, Session) # mypy does not recognize the check in _batch_inference_validate_snowpark()
|
462
457
|
|
463
458
|
transform_kwargs = dict(
|
464
|
-
session
|
465
|
-
dependencies
|
466
|
-
drop_input_cols
|
467
|
-
expected_output_cols_type
|
459
|
+
session=dataset._session,
|
460
|
+
dependencies=self._deps,
|
461
|
+
drop_input_cols=self._drop_input_cols,
|
462
|
+
expected_output_cols_type=expected_dtype,
|
468
463
|
)
|
469
464
|
|
470
465
|
elif isinstance(dataset, pd.DataFrame):
|
471
|
-
transform_kwargs = dict(
|
472
|
-
snowpark_input_cols = self._snowpark_cols,
|
473
|
-
drop_input_cols = self._drop_input_cols
|
474
|
-
)
|
466
|
+
transform_kwargs = dict(snowpark_input_cols=self._snowpark_cols, drop_input_cols=self._drop_input_cols)
|
475
467
|
|
476
468
|
transform_handlers = ModelTransformerBuilder.build(
|
477
469
|
dataset=dataset,
|
@@ -490,7 +482,11 @@ class MissingIndicator(BaseTransformer):
|
|
490
482
|
return output_df
|
491
483
|
|
492
484
|
@available_if(original_estimator_has_callable("fit_predict")) # type: ignore[misc]
|
493
|
-
def fit_predict(
|
485
|
+
def fit_predict(
|
486
|
+
self,
|
487
|
+
dataset: Union[DataFrame, pd.DataFrame],
|
488
|
+
output_cols_prefix: str = "fit_predict_",
|
489
|
+
) -> Union[DataFrame, pd.DataFrame]:
|
494
490
|
""" Method not supported for this class.
|
495
491
|
|
496
492
|
|
@@ -515,7 +511,9 @@ class MissingIndicator(BaseTransformer):
|
|
515
511
|
)
|
516
512
|
output_result, fitted_estimator = model_trainer.train_fit_predict(
|
517
513
|
drop_input_cols=self._drop_input_cols,
|
518
|
-
expected_output_cols_list=
|
514
|
+
expected_output_cols_list=(
|
515
|
+
self.output_cols if self.output_cols else self._get_output_column_names(output_cols_prefix)
|
516
|
+
),
|
519
517
|
)
|
520
518
|
self._sklearn_object = fitted_estimator
|
521
519
|
self._is_fitted = True
|
@@ -532,6 +530,62 @@ class MissingIndicator(BaseTransformer):
|
|
532
530
|
assert self._sklearn_object is not None
|
533
531
|
return self._sklearn_object.embedding_
|
534
532
|
|
533
|
+
|
534
|
+
def _get_output_column_names(self, output_cols_prefix: str, output_cols: Optional[List[str]] = None) -> List[str]:
|
535
|
+
""" Returns the list of output columns for predict_proba(), decision_function(), etc.. functions.
|
536
|
+
Returns a list with output_cols_prefix as the only element if the estimator is not a classifier.
|
537
|
+
"""
|
538
|
+
output_cols_prefix = identifier.resolve_identifier(output_cols_prefix)
|
539
|
+
# The following condition is introduced for kneighbors methods, and not used in other methods
|
540
|
+
if output_cols:
|
541
|
+
output_cols = [
|
542
|
+
identifier.concat_names([output_cols_prefix, identifier.resolve_identifier(c)])
|
543
|
+
for c in output_cols
|
544
|
+
]
|
545
|
+
elif getattr(self._sklearn_object, "classes_", None) is None:
|
546
|
+
output_cols = [output_cols_prefix]
|
547
|
+
elif self._sklearn_object is not None:
|
548
|
+
classes = self._sklearn_object.classes_
|
549
|
+
if isinstance(classes, numpy.ndarray):
|
550
|
+
output_cols = [f'{output_cols_prefix}{str(c)}' for c in classes.tolist()]
|
551
|
+
elif isinstance(classes, list) and len(classes) > 0 and isinstance(classes[0], numpy.ndarray):
|
552
|
+
# If the estimator is a multioutput estimator, classes_ will be a list of ndarrays.
|
553
|
+
output_cols = []
|
554
|
+
for i, cl in enumerate(classes):
|
555
|
+
# For binary classification, there is only one output column for each class
|
556
|
+
# ndarray as the two classes are complementary.
|
557
|
+
if len(cl) == 2:
|
558
|
+
output_cols.append(f'{output_cols_prefix}{i}_{cl[0]}')
|
559
|
+
else:
|
560
|
+
output_cols.extend([
|
561
|
+
f'{output_cols_prefix}{i}_{c}' for c in cl.tolist()
|
562
|
+
])
|
563
|
+
else:
|
564
|
+
output_cols = []
|
565
|
+
|
566
|
+
# Make sure column names are valid snowflake identifiers.
|
567
|
+
assert output_cols is not None # Make MyPy happy
|
568
|
+
rv = [identifier.rename_to_valid_snowflake_identifier(c) for c in output_cols]
|
569
|
+
|
570
|
+
return rv
|
571
|
+
|
572
|
+
def _align_expected_output_names(
|
573
|
+
self, method: str, dataset: DataFrame, expected_output_cols_list: List[str], output_cols_prefix: str
|
574
|
+
) -> List[str]:
|
575
|
+
# in case the inferred output column names dimension is different
|
576
|
+
# we use one line of snowpark dataframe and put it into sklearn estimator using pandas
|
577
|
+
output_df_pd = getattr(self, method)(dataset.limit(1).to_pandas(), output_cols_prefix)
|
578
|
+
output_df_columns = list(output_df_pd.columns)
|
579
|
+
output_df_columns_set: Set[str] = set(output_df_columns) - set(dataset.columns)
|
580
|
+
if self.sample_weight_col:
|
581
|
+
output_df_columns_set -= set(self.sample_weight_col)
|
582
|
+
# if the dimension of inferred output column names is correct; use it
|
583
|
+
if len(expected_output_cols_list) == len(output_df_columns_set):
|
584
|
+
return expected_output_cols_list
|
585
|
+
# otherwise, use the sklearn estimator's output
|
586
|
+
else:
|
587
|
+
return sorted(list(output_df_columns_set), key=lambda x: output_df_columns.index(x))
|
588
|
+
|
535
589
|
@available_if(original_estimator_has_callable("predict_proba")) # type: ignore[misc]
|
536
590
|
@telemetry.send_api_usage_telemetry(
|
537
591
|
project=_PROJECT,
|
@@ -562,24 +616,28 @@ class MissingIndicator(BaseTransformer):
|
|
562
616
|
# are specific to the type of dataset used.
|
563
617
|
transform_kwargs: BatchInferenceKwargsTypedDict = dict()
|
564
618
|
|
619
|
+
expected_output_cols = self._get_output_column_names(output_cols_prefix)
|
620
|
+
|
565
621
|
if isinstance(dataset, DataFrame):
|
566
622
|
self._deps = self._batch_inference_validate_snowpark(
|
567
623
|
dataset=dataset,
|
568
624
|
inference_method=inference_method,
|
569
625
|
)
|
570
|
-
assert isinstance(
|
626
|
+
assert isinstance(
|
627
|
+
dataset._session, Session
|
628
|
+
) # mypy does not recognize the check in _batch_inference_validate_snowpark()
|
571
629
|
transform_kwargs = dict(
|
572
630
|
session=dataset._session,
|
573
631
|
dependencies=self._deps,
|
574
|
-
drop_input_cols
|
632
|
+
drop_input_cols=self._drop_input_cols,
|
575
633
|
expected_output_cols_type="float",
|
576
634
|
)
|
635
|
+
expected_output_cols = self._align_expected_output_names(
|
636
|
+
inference_method, dataset, expected_output_cols, output_cols_prefix
|
637
|
+
)
|
577
638
|
|
578
639
|
elif isinstance(dataset, pd.DataFrame):
|
579
|
-
transform_kwargs = dict(
|
580
|
-
snowpark_input_cols = self._snowpark_cols,
|
581
|
-
drop_input_cols = self._drop_input_cols
|
582
|
-
)
|
640
|
+
transform_kwargs = dict(snowpark_input_cols=self._snowpark_cols, drop_input_cols=self._drop_input_cols)
|
583
641
|
|
584
642
|
transform_handlers = ModelTransformerBuilder.build(
|
585
643
|
dataset=dataset,
|
@@ -591,7 +649,7 @@ class MissingIndicator(BaseTransformer):
|
|
591
649
|
output_df: DATAFRAME_TYPE = transform_handlers.batch_inference(
|
592
650
|
inference_method=inference_method,
|
593
651
|
input_cols=self.input_cols,
|
594
|
-
expected_output_cols=
|
652
|
+
expected_output_cols=expected_output_cols,
|
595
653
|
**transform_kwargs
|
596
654
|
)
|
597
655
|
return output_df
|
@@ -621,7 +679,8 @@ class MissingIndicator(BaseTransformer):
|
|
621
679
|
Output dataset with log probability of the sample for each class in the model.
|
622
680
|
"""
|
623
681
|
super()._check_dataset_type(dataset)
|
624
|
-
inference_method="predict_log_proba"
|
682
|
+
inference_method = "predict_log_proba"
|
683
|
+
expected_output_cols = self._get_output_column_names(output_cols_prefix)
|
625
684
|
|
626
685
|
# This dictionary contains optional kwargs for batch inference. These kwargs
|
627
686
|
# are specific to the type of dataset used.
|
@@ -632,18 +691,20 @@ class MissingIndicator(BaseTransformer):
|
|
632
691
|
dataset=dataset,
|
633
692
|
inference_method=inference_method,
|
634
693
|
)
|
635
|
-
assert isinstance(
|
694
|
+
assert isinstance(
|
695
|
+
dataset._session, Session
|
696
|
+
) # mypy does not recognize the check in _batch_inference_validate_snowpark()
|
636
697
|
transform_kwargs = dict(
|
637
698
|
session=dataset._session,
|
638
699
|
dependencies=self._deps,
|
639
|
-
drop_input_cols
|
700
|
+
drop_input_cols=self._drop_input_cols,
|
640
701
|
expected_output_cols_type="float",
|
641
702
|
)
|
703
|
+
expected_output_cols = self._align_expected_output_names(
|
704
|
+
inference_method, dataset, expected_output_cols, output_cols_prefix
|
705
|
+
)
|
642
706
|
elif isinstance(dataset, pd.DataFrame):
|
643
|
-
transform_kwargs = dict(
|
644
|
-
snowpark_input_cols = self._snowpark_cols,
|
645
|
-
drop_input_cols = self._drop_input_cols
|
646
|
-
)
|
707
|
+
transform_kwargs = dict(snowpark_input_cols=self._snowpark_cols, drop_input_cols=self._drop_input_cols)
|
647
708
|
|
648
709
|
transform_handlers = ModelTransformerBuilder.build(
|
649
710
|
dataset=dataset,
|
@@ -656,7 +717,7 @@ class MissingIndicator(BaseTransformer):
|
|
656
717
|
output_df: DATAFRAME_TYPE = transform_handlers.batch_inference(
|
657
718
|
inference_method=inference_method,
|
658
719
|
input_cols=self.input_cols,
|
659
|
-
expected_output_cols=
|
720
|
+
expected_output_cols=expected_output_cols,
|
660
721
|
**transform_kwargs
|
661
722
|
)
|
662
723
|
return output_df
|
@@ -682,30 +743,34 @@ class MissingIndicator(BaseTransformer):
|
|
682
743
|
Output dataset with results of the decision function for the samples in input dataset.
|
683
744
|
"""
|
684
745
|
super()._check_dataset_type(dataset)
|
685
|
-
inference_method="decision_function"
|
746
|
+
inference_method = "decision_function"
|
686
747
|
|
687
748
|
# This dictionary contains optional kwargs for batch inference. These kwargs
|
688
749
|
# are specific to the type of dataset used.
|
689
750
|
transform_kwargs: BatchInferenceKwargsTypedDict = dict()
|
690
751
|
|
752
|
+
expected_output_cols = self._get_output_column_names(output_cols_prefix)
|
753
|
+
|
691
754
|
if isinstance(dataset, DataFrame):
|
692
755
|
self._deps = self._batch_inference_validate_snowpark(
|
693
756
|
dataset=dataset,
|
694
757
|
inference_method=inference_method,
|
695
758
|
)
|
696
|
-
assert isinstance(
|
759
|
+
assert isinstance(
|
760
|
+
dataset._session, Session
|
761
|
+
) # mypy does not recognize the check in _batch_inference_validate_snowpark()
|
697
762
|
transform_kwargs = dict(
|
698
763
|
session=dataset._session,
|
699
764
|
dependencies=self._deps,
|
700
|
-
drop_input_cols
|
765
|
+
drop_input_cols=self._drop_input_cols,
|
701
766
|
expected_output_cols_type="float",
|
702
767
|
)
|
768
|
+
expected_output_cols = self._align_expected_output_names(
|
769
|
+
inference_method, dataset, expected_output_cols, output_cols_prefix
|
770
|
+
)
|
703
771
|
|
704
772
|
elif isinstance(dataset, pd.DataFrame):
|
705
|
-
transform_kwargs = dict(
|
706
|
-
snowpark_input_cols = self._snowpark_cols,
|
707
|
-
drop_input_cols = self._drop_input_cols
|
708
|
-
)
|
773
|
+
transform_kwargs = dict(snowpark_input_cols=self._snowpark_cols, drop_input_cols=self._drop_input_cols)
|
709
774
|
|
710
775
|
transform_handlers = ModelTransformerBuilder.build(
|
711
776
|
dataset=dataset,
|
@@ -718,7 +783,7 @@ class MissingIndicator(BaseTransformer):
|
|
718
783
|
output_df: DATAFRAME_TYPE = transform_handlers.batch_inference(
|
719
784
|
inference_method=inference_method,
|
720
785
|
input_cols=self.input_cols,
|
721
|
-
expected_output_cols=
|
786
|
+
expected_output_cols=expected_output_cols,
|
722
787
|
**transform_kwargs
|
723
788
|
)
|
724
789
|
return output_df
|
@@ -747,12 +812,14 @@ class MissingIndicator(BaseTransformer):
|
|
747
812
|
Output dataset with probability of the sample for each class in the model.
|
748
813
|
"""
|
749
814
|
super()._check_dataset_type(dataset)
|
750
|
-
inference_method="score_samples"
|
815
|
+
inference_method = "score_samples"
|
751
816
|
|
752
817
|
# This dictionary contains optional kwargs for batch inference. These kwargs
|
753
818
|
# are specific to the type of dataset used.
|
754
819
|
transform_kwargs: BatchInferenceKwargsTypedDict = dict()
|
755
820
|
|
821
|
+
expected_output_cols = self._get_output_column_names(output_cols_prefix)
|
822
|
+
|
756
823
|
if isinstance(dataset, DataFrame):
|
757
824
|
self._deps = self._batch_inference_validate_snowpark(
|
758
825
|
dataset=dataset,
|
@@ -765,6 +832,9 @@ class MissingIndicator(BaseTransformer):
|
|
765
832
|
drop_input_cols = self._drop_input_cols,
|
766
833
|
expected_output_cols_type="float",
|
767
834
|
)
|
835
|
+
expected_output_cols = self._align_expected_output_names(
|
836
|
+
inference_method, dataset, expected_output_cols, output_cols_prefix
|
837
|
+
)
|
768
838
|
|
769
839
|
elif isinstance(dataset, pd.DataFrame):
|
770
840
|
transform_kwargs = dict(
|
@@ -783,7 +853,7 @@ class MissingIndicator(BaseTransformer):
|
|
783
853
|
output_df: DATAFRAME_TYPE = transform_handlers.batch_inference(
|
784
854
|
inference_method=inference_method,
|
785
855
|
input_cols=self.input_cols,
|
786
|
-
expected_output_cols=
|
856
|
+
expected_output_cols=expected_output_cols,
|
787
857
|
**transform_kwargs
|
788
858
|
)
|
789
859
|
return output_df
|
@@ -928,50 +998,84 @@ class MissingIndicator(BaseTransformer):
|
|
928
998
|
)
|
929
999
|
return output_df
|
930
1000
|
|
1001
|
+
|
1002
|
+
|
1003
|
+
def to_sklearn(self) -> Any:
|
1004
|
+
"""Get sklearn.impute.MissingIndicator object.
|
1005
|
+
"""
|
1006
|
+
if self._sklearn_object is None:
|
1007
|
+
self._sklearn_object = self._create_sklearn_object()
|
1008
|
+
return self._sklearn_object
|
1009
|
+
|
1010
|
+
def to_xgboost(self) -> Any:
|
1011
|
+
raise exceptions.SnowflakeMLException(
|
1012
|
+
error_code=error_codes.METHOD_NOT_ALLOWED,
|
1013
|
+
original_exception=AttributeError(
|
1014
|
+
modeling_error_messages.UNSUPPORTED_MODEL_CONVERSION.format(
|
1015
|
+
"to_xgboost()",
|
1016
|
+
"to_sklearn()"
|
1017
|
+
)
|
1018
|
+
),
|
1019
|
+
)
|
1020
|
+
|
1021
|
+
def to_lightgbm(self) -> Any:
|
1022
|
+
raise exceptions.SnowflakeMLException(
|
1023
|
+
error_code=error_codes.METHOD_NOT_ALLOWED,
|
1024
|
+
original_exception=AttributeError(
|
1025
|
+
modeling_error_messages.UNSUPPORTED_MODEL_CONVERSION.format(
|
1026
|
+
"to_lightgbm()",
|
1027
|
+
"to_sklearn()"
|
1028
|
+
)
|
1029
|
+
),
|
1030
|
+
)
|
931
1031
|
|
932
|
-
def
|
1032
|
+
def _get_dependencies(self) -> List[str]:
|
1033
|
+
return self._deps
|
1034
|
+
|
1035
|
+
|
1036
|
+
def _generate_model_signatures(self, dataset: Union[DataFrame, pd.DataFrame]) -> None:
|
933
1037
|
self._model_signature_dict = dict()
|
934
1038
|
|
935
1039
|
PROB_FUNCTIONS = ["predict_log_proba", "predict_proba", "decision_function"]
|
936
1040
|
|
937
|
-
inputs = list(_infer_signature(dataset[self.input_cols], "input"))
|
1041
|
+
inputs = list(_infer_signature(dataset[self.input_cols], "input", use_snowflake_identifiers=True))
|
938
1042
|
outputs: List[BaseFeatureSpec] = []
|
939
1043
|
if hasattr(self, "predict"):
|
940
1044
|
# keep mypy happy
|
941
|
-
assert self._sklearn_object is not None and hasattr(self._sklearn_object, "_estimator_type")
|
1045
|
+
assert self._sklearn_object is not None and hasattr(self._sklearn_object, "_estimator_type")
|
942
1046
|
# For classifier, the type of predict is the same as the type of label
|
943
|
-
if self._sklearn_object._estimator_type ==
|
944
|
-
|
1047
|
+
if self._sklearn_object._estimator_type == "classifier":
|
1048
|
+
# label columns is the desired type for output
|
945
1049
|
outputs = list(_infer_signature(dataset[self.label_cols], "output", use_snowflake_identifiers=True))
|
946
1050
|
# rename the output columns
|
947
1051
|
outputs = list(model_signature_utils.rename_features(outputs, self.output_cols))
|
948
|
-
self._model_signature_dict["predict"] = ModelSignature(
|
949
|
-
|
950
|
-
|
1052
|
+
self._model_signature_dict["predict"] = ModelSignature(
|
1053
|
+
inputs, ([] if self._drop_input_cols else inputs) + outputs
|
1054
|
+
)
|
951
1055
|
# For mixture models that use the density mixin, `predict` returns the argmax of the log prob.
|
952
1056
|
# For outlier models, returns -1 for outliers and 1 for inliers.
|
953
|
-
# Clusterer returns int64 cluster labels.
|
1057
|
+
# Clusterer returns int64 cluster labels.
|
954
1058
|
elif self._sklearn_object._estimator_type in ["DensityEstimator", "clusterer", "outlier_detector"]:
|
955
1059
|
outputs = [FeatureSpec(dtype=DataType.INT64, name=c) for c in self.output_cols]
|
956
|
-
self._model_signature_dict["predict"] = ModelSignature(
|
957
|
-
|
958
|
-
|
959
|
-
|
1060
|
+
self._model_signature_dict["predict"] = ModelSignature(
|
1061
|
+
inputs, ([] if self._drop_input_cols else inputs) + outputs
|
1062
|
+
)
|
1063
|
+
|
960
1064
|
# For regressor, the type of predict is float64
|
961
|
-
elif self._sklearn_object._estimator_type ==
|
1065
|
+
elif self._sklearn_object._estimator_type == "regressor":
|
962
1066
|
outputs = [FeatureSpec(dtype=DataType.DOUBLE, name=c) for c in self.output_cols]
|
963
|
-
self._model_signature_dict["predict"] = ModelSignature(
|
964
|
-
|
965
|
-
|
966
|
-
|
1067
|
+
self._model_signature_dict["predict"] = ModelSignature(
|
1068
|
+
inputs, ([] if self._drop_input_cols else inputs) + outputs
|
1069
|
+
)
|
1070
|
+
|
967
1071
|
for prob_func in PROB_FUNCTIONS:
|
968
1072
|
if hasattr(self, prob_func):
|
969
1073
|
output_cols_prefix: str = f"{prob_func}_"
|
970
1074
|
output_column_names = self._get_output_column_names(output_cols_prefix)
|
971
1075
|
outputs = [FeatureSpec(dtype=DataType.DOUBLE, name=c) for c in output_column_names]
|
972
|
-
self._model_signature_dict[prob_func] = ModelSignature(
|
973
|
-
|
974
|
-
|
1076
|
+
self._model_signature_dict[prob_func] = ModelSignature(
|
1077
|
+
inputs, ([] if self._drop_input_cols else inputs) + outputs
|
1078
|
+
)
|
975
1079
|
|
976
1080
|
# Output signature names may still need to be renamed, since they were not created with `_infer_signature`.
|
977
1081
|
items = list(self._model_signature_dict.items())
|
@@ -984,10 +1088,10 @@ class MissingIndicator(BaseTransformer):
|
|
984
1088
|
"""Returns model signature of current class.
|
985
1089
|
|
986
1090
|
Raises:
|
987
|
-
|
1091
|
+
SnowflakeMLException: If estimator is not fitted, then model signature cannot be inferred
|
988
1092
|
|
989
1093
|
Returns:
|
990
|
-
Dict
|
1094
|
+
Dict with each method and its input output signature
|
991
1095
|
"""
|
992
1096
|
if self._model_signature_dict is None:
|
993
1097
|
raise exceptions.SnowflakeMLException(
|
@@ -995,35 +1099,3 @@ class MissingIndicator(BaseTransformer):
|
|
995
1099
|
original_exception=RuntimeError("Estimator not fitted before accessing property model_signatures!"),
|
996
1100
|
)
|
997
1101
|
return self._model_signature_dict
|
998
|
-
|
999
|
-
def to_sklearn(self) -> Any:
|
1000
|
-
"""Get sklearn.impute.MissingIndicator object.
|
1001
|
-
"""
|
1002
|
-
if self._sklearn_object is None:
|
1003
|
-
self._sklearn_object = self._create_sklearn_object()
|
1004
|
-
return self._sklearn_object
|
1005
|
-
|
1006
|
-
def to_xgboost(self) -> Any:
|
1007
|
-
raise exceptions.SnowflakeMLException(
|
1008
|
-
error_code=error_codes.METHOD_NOT_ALLOWED,
|
1009
|
-
original_exception=AttributeError(
|
1010
|
-
modeling_error_messages.UNSUPPORTED_MODEL_CONVERSION.format(
|
1011
|
-
"to_xgboost()",
|
1012
|
-
"to_sklearn()"
|
1013
|
-
)
|
1014
|
-
),
|
1015
|
-
)
|
1016
|
-
|
1017
|
-
def to_lightgbm(self) -> Any:
|
1018
|
-
raise exceptions.SnowflakeMLException(
|
1019
|
-
error_code=error_codes.METHOD_NOT_ALLOWED,
|
1020
|
-
original_exception=AttributeError(
|
1021
|
-
modeling_error_messages.UNSUPPORTED_MODEL_CONVERSION.format(
|
1022
|
-
"to_lightgbm()",
|
1023
|
-
"to_sklearn()"
|
1024
|
-
)
|
1025
|
-
),
|
1026
|
-
)
|
1027
|
-
|
1028
|
-
def _get_dependencies(self) -> List[str]:
|
1029
|
-
return self._deps
|