snowflake-ml-python 1.5.0__py3-none-any.whl → 1.5.1__py3-none-any.whl

snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py

@@ -4,7 +4,7 @@ import io
 import os
 import posixpath
 import sys
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 import cloudpickle as cp
 import numpy as np
@@ -154,7 +154,7 @@ def construct_cv_results(
     return multimetric, estimator._format_results(param_grid, n_split, out)
 
 
-def construct_cv_results_new_implementation(
+def construct_cv_results_memory_efficient_version(
     estimator: Union[GridSearchCV, RandomizedSearchCV],
     n_split: int,
     param_grid: List[Dict[str, Any]],
@@ -205,12 +205,35 @@ def construct_cv_results_new_implementation(
         with io.BytesIO(hex_str) as f_reload:
             out = cp.load(f_reload)
             all_out.extend(out)
+
+    # because original SearchCV is ranked by parameter first and cv second,
+    # to make the memory efficient, we implemented by fitting on cv first and parameter second
+    # when retrieving the results back, the ordering should revert back to remain the same result as original SearchCV
+    def generate_the_order_by_parameter_index(all_combination_length: int) -> List[int]:
+        pattern = []
+        for i in range(all_combination_length):
+            if i % parameter_grid_length == 0:
+                pattern.append(i)
+        for i in range(1, parameter_grid_length):
+            for j in range(all_combination_length):
+                if j % parameter_grid_length == i:
+                    pattern.append(j)
+        return pattern
+
+    def rerank_array(original_array: List[Any], pattern: List[int]) -> List[Any]:
+        reranked_array = []
+        for index in pattern:
+            reranked_array.append(original_array[index])
+        return reranked_array
+
+    pattern = generate_the_order_by_parameter_index(len(all_out))
+    reranked_all_out = rerank_array(all_out, pattern)
     first_test_score = all_out[0]["test_scores"]
-    return first_test_score, estimator._format_results(param_grid, n_split, all_out)
+    return first_test_score, estimator._format_results(param_grid, n_split, reranked_all_out)
 
 
 cp.register_pickle_by_value(inspect.getmodule(construct_cv_results))
-cp.register_pickle_by_value(inspect.getmodule(construct_cv_results_new_implementation))
+cp.register_pickle_by_value(inspect.getmodule(construct_cv_results_memory_efficient_version))
 
 
 class DistributedHPOTrainer(SnowparkModelTrainer):
@@ -661,7 +684,7 @@ class DistributedHPOTrainer(SnowparkModelTrainer):
 
         return fit_estimator
 
-    def fit_search_snowpark_new_implementation(
+    def fit_search_snowpark_enable_efficient_memory_usage(
         self,
         param_grid: Union[model_selection.ParameterGrid, model_selection.ParameterSampler],
         dataset: DataFrame,
@@ -718,7 +741,7 @@ class DistributedHPOTrainer(SnowparkModelTrainer):
                 inspect.currentframe(), self.__class__.__name__
             ),
             api_calls=[udtf],
-            custom_tags=dict([("hpo_udtf", True)]),
+            custom_tags=dict([("hpo_memory_efficient", True)]),
         )
 
         # Put locally serialized estimator on stage.
@@ -960,22 +983,26 @@ class DistributedHPOTrainer(SnowparkModelTrainer):
                     self.base_estimator = base_estimator
                     self.fit_and_score_kwargs = fit_and_score_kwargs
                     self.fit_score_params: List[Any] = []
-                    self.cached_train_test_indices = []
-                    # Calculate the full index here to avoid duplicate calculation (which consumes a lot of memory)
-                    full_index = np.arange(DATA_LENGTH)
-                    for i in range(n_splits):
-                        self.cached_train_test_indices.extend(
-                            [[np.setdiff1d(full_index, self.test_indices[i]), self.test_indices[i]]]
-                        )
+                    self.cv_indices_set: Set[int] = set()
 
                 def process(self, idx: int, params_idx: int, cv_idx: int) -> None:
                     self.fit_score_params.extend([[idx, params_idx, cv_idx]])
+                    self.cv_indices_set.add(cv_idx)
 
                 def end_partition(self) -> Iterator[Tuple[int, str]]:
                     from sklearn.base import clone
                     from sklearn.model_selection._validation import _fit_and_score
                     from sklearn.utils.parallel import Parallel, delayed
 
+                    cached_train_test_indices = {}
+                    # Calculate the full index here to avoid duplicate calculation (which consumes a lot of memory)
+                    full_index = np.arange(DATA_LENGTH)
+                    for i in self.cv_indices_set:
+                        cached_train_test_indices[i] = [
+                            np.setdiff1d(full_index, self.test_indices[i]),
+                            self.test_indices[i],
+                        ]
+
                     parallel = Parallel(n_jobs=_N_JOBS, pre_dispatch=_PRE_DISPATCH)
 
                     out = parallel(
@@ -983,8 +1010,8 @@ class DistributedHPOTrainer(SnowparkModelTrainer):
                             clone(self.base_estimator),
                             self.X,
                             self.y,
-                            train=self.cached_train_test_indices[split_idx][0],
-                            test=self.cached_train_test_indices[split_idx][1],
+                            train=cached_train_test_indices[split_idx][0],
+                            test=cached_train_test_indices[split_idx][1],
                             parameters=self.params_to_evaluate[cand_idx],
                             split_progress=(split_idx, n_splits),
                             candidate_progress=(cand_idx, n_candidates),
@@ -1005,7 +1032,9 @@ class DistributedHPOTrainer(SnowparkModelTrainer):
 
             session.udtf.register(
                 SearchCV,
-                output_schema=StructType([StructField("IDX", IntegerType()), StructField("CV_RESULTS", StringType())]),
+                output_schema=StructType(
+                    [StructField("FIRST_IDX", IntegerType()), StructField("EACH_CV_RESULTS", StringType())]
+                ),
                 input_types=[IntegerType(), IntegerType(), IntegerType()],
                 name=random_udtf_name,
                 packages=required_deps,  # type: ignore[arg-type]
@@ -1020,8 +1049,8 @@ class DistributedHPOTrainer(SnowparkModelTrainer):
             # param_indices is for the index for each parameter grid;
             # cv_indices is for the index for each cross_validator's fold;
             # param_cv_indices is for the index for the product of (len(param_indices) * len(cv_indices))
-            param_indices, cv_indices = zip(
-                *product(range(parameter_grid_length), range(cross_validator_indices_length))
+            cv_indices, param_indices = zip(
+                *product(range(cross_validator_indices_length), range(parameter_grid_length))
             )
 
             indices_info_pandas = pd.DataFrame(
@@ -1042,11 +1071,11 @@ class DistributedHPOTrainer(SnowparkModelTrainer):
                 ),
             )
 
-            first_test_score, cv_results_ = construct_cv_results_new_implementation(
+            first_test_score, cv_results_ = construct_cv_results_memory_efficient_version(
                 estimator,
                 n_splits,
                 list(param_grid),
-                HP_raw_results.select("CV_RESULTS").sort(F.col("IDX")).collect(),
+                HP_raw_results.select("EACH_CV_RESULTS").sort(F.col("FIRST_IDX")).collect(),
                 cross_validator_indices_length,
                 parameter_grid_length,
             )
@@ -1163,7 +1192,7 @@ class DistributedHPOTrainer(SnowparkModelTrainer):
             pkg_versions=model_spec.pkgDependencies, session=self.session
         )
         if ENABLE_EFFICIENT_MEMORY_USAGE:
-            return self.fit_search_snowpark_new_implementation(
+            return self.fit_search_snowpark_enable_efficient_memory_usage(
                 param_grid=param_grid,
                 dataset=self.dataset,
                 session=self.session,

snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_trainer.py

@@ -45,6 +45,7 @@ cp.register_pickle_by_value(inspect.getmodule(identifier.get_inferred_name))
 cp.register_pickle_by_value(inspect.getmodule(handle_inference_result))
 
 _PROJECT = "ModelDevelopment"
+_ENABLE_ANONYMOUS_SPROC = False
 
 
 class SnowparkModelTrainer:
@@ -251,6 +252,27 @@ class SnowparkModelTrainer:
 
         return fit_wrapper_function
 
+    def _get_fit_wrapper_sproc_anonymous(self, statement_params: Dict[str, str]) -> StoredProcedure:
+        model_spec = ModelSpecificationsBuilder.build(model=self.estimator)
+        fit_sproc_name = random_name_for_temp_object(TempObjectType.PROCEDURE)
+
+        relaxed_dependencies = pkg_version_utils.get_valid_pkg_versions_supported_in_snowflake_conda_channel(
+            pkg_versions=model_spec.pkgDependencies, session=self.session
+        )
+
+        fit_wrapper_sproc = self.session.sproc.register(
+            func=self._build_fit_wrapper_sproc(model_spec=model_spec),
+            is_permanent=False,
+            name=fit_sproc_name,
+            packages=["snowflake-snowpark-python"] + relaxed_dependencies,  # type: ignore[arg-type]
+            replace=True,
+            session=self.session,
+            statement_params=statement_params,
+            anonymous=True,
+        )
+
+        return fit_wrapper_sproc
+
     def _get_fit_wrapper_sproc(self, statement_params: Dict[str, str]) -> StoredProcedure:
         # If the sproc already exists, don't register.
         if not hasattr(self.session, "_FIT_WRAPPER_SPROCS"):
@@ -510,6 +532,28 @@ class SnowparkModelTrainer:
 
         return fit_transform_wrapper_function
 
+    def _get_fit_predict_wrapper_sproc_anonymous(self, statement_params: Dict[str, str]) -> StoredProcedure:
+        model_spec = ModelSpecificationsBuilder.build(model=self.estimator)
+
+        fit_predict_sproc_name = random_name_for_temp_object(TempObjectType.PROCEDURE)
+
+        relaxed_dependencies = pkg_version_utils.get_valid_pkg_versions_supported_in_snowflake_conda_channel(
+            pkg_versions=model_spec.pkgDependencies, session=self.session
+        )
+
+        fit_predict_wrapper_sproc = self.session.sproc.register(
+            func=self._build_fit_predict_wrapper_sproc(model_spec=model_spec),
+            is_permanent=False,
+            name=fit_predict_sproc_name,
+            packages=["snowflake-snowpark-python"] + relaxed_dependencies,  # type: ignore[arg-type]
+            replace=True,
+            session=self.session,
+            statement_params=statement_params,
+            anonymous=True,
+        )
+
+        return fit_predict_wrapper_sproc
+
     def _get_fit_predict_wrapper_sproc(self, statement_params: Dict[str, str]) -> StoredProcedure:
         # If the sproc already exists, don't register.
         if not hasattr(self.session, "_FIT_WRAPPER_SPROCS"):
@@ -545,6 +589,27 @@ class SnowparkModelTrainer:
 
         return fit_predict_wrapper_sproc
 
+    def _get_fit_transform_wrapper_sproc_anonymous(self, statement_params: Dict[str, str]) -> StoredProcedure:
+        model_spec = ModelSpecificationsBuilder.build(model=self.estimator)
+
+        fit_transform_sproc_name = random_name_for_temp_object(TempObjectType.PROCEDURE)
+
+        relaxed_dependencies = pkg_version_utils.get_valid_pkg_versions_supported_in_snowflake_conda_channel(
+            pkg_versions=model_spec.pkgDependencies, session=self.session
+        )
+
+        fit_transform_wrapper_sproc = self.session.sproc.register(
+            func=self._build_fit_transform_wrapper_sproc(model_spec=model_spec),
+            is_permanent=False,
+            name=fit_transform_sproc_name,
+            packages=["snowflake-snowpark-python"] + relaxed_dependencies,  # type: ignore[arg-type]
+            replace=True,
+            session=self.session,
+            statement_params=statement_params,
+            anonymous=True,
+        )
+        return fit_transform_wrapper_sproc
+
     def _get_fit_transform_wrapper_sproc(self, statement_params: Dict[str, str]) -> StoredProcedure:
         # If the sproc already exists, don't register.
         if not hasattr(self.session, "_FIT_WRAPPER_SPROCS"):
@@ -612,7 +677,10 @@ class SnowparkModelTrainer:
             custom_tags=dict([("autogen", True)]) if self._autogenerated else None,
         )
 
-        fit_wrapper_sproc = self._get_fit_wrapper_sproc(statement_params=statement_params)
+        if _ENABLE_ANONYMOUS_SPROC:
+            fit_wrapper_sproc = self._get_fit_wrapper_sproc_anonymous(statement_params=statement_params)
+        else:
+            fit_wrapper_sproc = self._get_fit_wrapper_sproc(statement_params=statement_params)
 
         try:
             sproc_export_file_name: str = fit_wrapper_sproc(
@@ -680,7 +748,11 @@ class SnowparkModelTrainer:
             custom_tags=dict([("autogen", True)]) if self._autogenerated else None,
         )
 
-        fit_predict_wrapper_sproc = self._get_fit_predict_wrapper_sproc(statement_params=statement_params)
+        if _ENABLE_ANONYMOUS_SPROC:
+            fit_predict_wrapper_sproc = self._get_fit_predict_wrapper_sproc_anonymous(statement_params=statement_params)
+        else:
+            fit_predict_wrapper_sproc = self._get_fit_predict_wrapper_sproc(statement_params=statement_params)
+
         fit_predict_result_name = random_name_for_temp_object(TempObjectType.TABLE)
 
         sproc_export_file_name: str = fit_predict_wrapper_sproc(
@@ -741,7 +813,13 @@ class SnowparkModelTrainer:
             custom_tags=dict([("autogen", True)]) if self._autogenerated else None,
         )
 
-        fit_transform_wrapper_sproc = self._get_fit_transform_wrapper_sproc(statement_params=statement_params)
+        if _ENABLE_ANONYMOUS_SPROC:
+            fit_transform_wrapper_sproc = self._get_fit_transform_wrapper_sproc_anonymous(
+                statement_params=statement_params
+            )
+        else:
+            fit_transform_wrapper_sproc = self._get_fit_transform_wrapper_sproc(statement_params=statement_params)
+
         fit_transform_result_name = random_name_for_temp_object(TempObjectType.TABLE)
 
         sproc_export_file_name: str = fit_transform_wrapper_sproc(

snowflake/ml/modeling/framework/base.py

@@ -16,7 +16,7 @@ from snowflake.ml._internal.exceptions import (
     exceptions,
     modeling_error_messages,
 )
-from snowflake.ml._internal.lineage import data_source, dataset_dataframe
+from snowflake.ml._internal.lineage import data_source, lineage_utils
 from snowflake.ml._internal.utils import identifier, parallelize
 from snowflake.ml.modeling.framework import _utils
 from snowflake.snowpark import functions as F
@@ -430,8 +430,9 @@ class BaseEstimator(Base):
     )
     def fit(self, dataset: Union[snowpark.DataFrame, pd.DataFrame]) -> "BaseEstimator":
         """Runs universal logics for all fit implementations."""
-        if isinstance(dataset, dataset_dataframe.DatasetDataFrame):
-            self._data_sources = dataset._get_sources()
+        self._data_sources = getattr(dataset, lineage_utils.DATA_SOURCES_ATTR, None)
+        if self._data_sources:
+            assert all(isinstance(ds, data_source.DataSource) for ds in self._data_sources)
         return self._fit(dataset)
 
     @abstractmethod

snowflake/ml/modeling/pipeline/pipeline.py

@@ -115,7 +115,7 @@ class Pipeline(base.BaseTransformer):
         self._feature_names_in: List[np.ndarray[Any, np.dtype[Any]]] = []
         self._n_features_in: List[int] = []
         self._transformers_to_input_indices: Dict[str, List[int]] = {}
-        self._is_convertible_to_sklearn = True
+        self._modifies_label_or_sample_weight = True
 
         self._model_signature_dict: Optional[Dict[str, ModelSignature]] = None
 
@@ -126,6 +126,9 @@ class Pipeline(base.BaseTransformer):
         self._deps = list(deps)
         self._sklearn_object = None
         self.label_cols = self._get_label_cols()
+        self._is_convertible_to_sklearn = self._is_convertible_to_sklearn_object()
+
+        self._send_pipeline_configuration_telemetry()
 
     @staticmethod
     def _is_estimator(obj: object) -> bool:
@@ -228,7 +231,7 @@ class Pipeline(base.BaseTransformer):
         return [c for c in columns if c not in target_cols]
 
     def _append_step_feature_consumption_info(self, step_name: str, all_cols: List[str], input_cols: List[str]) -> None:
-        if self._is_convertible_to_sklearn:
+        if self._modifies_label_or_sample_weight:
             all_cols = self._get_sanitized_list_of_columns(all_cols)
             self._feature_names_in.append(np.asarray(all_cols, dtype=object))
             self._n_features_in.append(len(all_cols))
@@ -248,7 +251,7 @@ class Pipeline(base.BaseTransformer):
         self, dataset: Union[snowpark.DataFrame, pd.DataFrame]
     ) -> Union[snowpark.DataFrame, pd.DataFrame]:
         self._reset()
-        self._is_convertible_to_sklearn = not self._is_pipeline_modifying_label_or_sample_weight()
+        self._modifies_label_or_sample_weight = not self._is_pipeline_modifying_label_or_sample_weight()
         transformed_dataset = dataset
         for name, trans in self._get_transformers():
             self._append_step_feature_consumption_info(
@@ -425,7 +428,7 @@ class Pipeline(base.BaseTransformer):
         )
 
         if self._can_be_trained_in_ml_runtime(dataset):
-            if not self._is_convertible_to_sklearn_object():
+            if not self._is_convertible_to_sklearn:
                 raise ValueError("This pipeline cannot be converted to an sklearn pipeline.")
             self._fit_ml_runtime(dataset)
 
@@ -947,7 +950,7 @@ class Pipeline(base.BaseTransformer):
         if not os.environ.get(IN_ML_RUNTIME_ENV_VAR):
             return False
 
-        return self._is_convertible_to_sklearn_object()
+        return self._is_convertible_to_sklearn
 
     @staticmethod
     def _wrap_transformer_in_column_transformer(
@@ -1003,7 +1006,7 @@ class Pipeline(base.BaseTransformer):
         if not self._is_fitted:
             return self._create_unfitted_sklearn_object()
 
-        if not self._is_convertible_to_sklearn:
+        if not self._modifies_label_or_sample_weight:
             raise exceptions.SnowflakeMLException(
                 error_code=error_codes.METHOD_NOT_ALLOWED,
                 original_exception=ValueError(
@@ -1109,7 +1112,24 @@ class Pipeline(base.BaseTransformer):
             else:
                 return self._create_sklearn_object()
         else:
-            if self._is_convertible_to_sklearn_object():
+            if self._is_convertible_to_sklearn:
                 return self._create_unfitted_sklearn_object()
             else:
                 raise ValueError("This pipeline can not be converted to an sklearn pipeline.")
+
+    def _send_pipeline_configuration_telemetry(self) -> None:
+        """Track information about the pipeline setup. Currently, we want to track:
+        - Whether the pipeline is converible to an sklearn pipeline
+        - Whether the pipeline is being used in the SPCS ml runtime.
+        """
+
+        telemetry_data = {
+            "pipeline_is_convertible_to_sklearn": self._is_convertible_to_sklearn,
+            "in_spcs_ml_runtime": bool(os.environ.get(IN_ML_RUNTIME_ENV_VAR)),
+        }
+        telemetry.send_custom_usage(
+            project=_PROJECT,
+            subproject=_SUBPROJECT,
+            telemetry_type=telemetry.TelemetryField.TYPE_SNOWML_PIPELINE_USAGE.value,
+            data=telemetry_data,
+        )

snowflake/ml/registry/_manager/model_manager.py

@@ -48,20 +48,29 @@ class ModelManager:
         options: Optional[model_types.ModelSaveOption] = None,
         statement_params: Optional[Dict[str, Any]] = None,
     ) -> model_version_impl.ModelVersion:
-        model_name_id = sql_identifier.SqlIdentifier(model_name)
+        database_name_id, schema_name_id, model_name_id = sql_identifier.parse_fully_qualified_name(model_name)
 
         if not version_name:
             version_name = self._hrid_generator.generate()[1]
         version_name_id = sql_identifier.SqlIdentifier(version_name)
 
         if self._model_ops.validate_existence(
-            model_name=model_name_id, statement_params=statement_params
+            database_name=database_name_id,
+            schema_name=schema_name_id,
+            model_name=model_name_id,
+            statement_params=statement_params,
         ) and self._model_ops.validate_existence(
-            model_name=model_name_id, version_name=version_name_id, statement_params=statement_params
+            database_name=database_name_id,
+            schema_name=schema_name_id,
+            model_name=model_name_id,
+            version_name=version_name_id,
+            statement_params=statement_params,
         ):
             raise ValueError(f"Model {model_name} version {version_name} already existed.")
 
         stage_path = self._model_ops.prepare_model_stage_path(
+            database_name=database_name_id,
+            schema_name=schema_name_id,
             statement_params=statement_params,
         )
 
@@ -85,13 +94,19 @@ class ModelManager:
 
         self._model_ops.create_from_stage(
             composed_model=mc,
+            database_name=database_name_id,
+            schema_name=schema_name_id,
             model_name=model_name_id,
             version_name=version_name_id,
             statement_params=statement_params,
         )
 
         mv = model_version_impl.ModelVersion._ref(
-            self._model_ops,
+            model_ops.ModelOperator(
+                self._model_ops._session,
+                database_name=database_name_id or self._database_name,
+                schema_name=schema_name_id or self._schema_name,
+            ),
             model_name=model_name_id,
             version_name=version_name_id,
         )
@@ -102,6 +117,8 @@ class ModelManager:
         if metrics:
             self._model_ops._metadata_ops.save(
                 metadata_ops.ModelVersionMetadataSchema(metrics=metrics),
+                database_name=database_name_id,
+                schema_name=schema_name_id,
                 model_name=model_name_id,
                 version_name=version_name_id,
                 statement_params=statement_params,
@@ -115,13 +132,19 @@ class ModelManager:
         *,
         statement_params: Optional[Dict[str, Any]] = None,
     ) -> model_impl.Model:
-        model_name_id = sql_identifier.SqlIdentifier(model_name)
+        database_name_id, schema_name_id, model_name_id = sql_identifier.parse_fully_qualified_name(model_name)
         if self._model_ops.validate_existence(
+            database_name=database_name_id,
+            schema_name=schema_name_id,
             model_name=model_name_id,
             statement_params=statement_params,
         ):
             return model_impl.Model._ref(
-                self._model_ops,
+                model_ops.ModelOperator(
+                    self._model_ops._session,
+                    database_name=database_name_id or self._database_name,
+                    schema_name=schema_name_id or self._schema_name,
+                ),
                 model_name=model_name_id,
             )
         else:
@@ -133,6 +156,8 @@ class ModelManager:
         statement_params: Optional[Dict[str, Any]] = None,
     ) -> List[model_impl.Model]:
         model_names = self._model_ops.list_models_or_versions(
+            database_name=None,
+            schema_name=None,
             statement_params=statement_params,
         )
         return [
@@ -149,6 +174,8 @@ class ModelManager:
         statement_params: Optional[Dict[str, Any]] = None,
     ) -> pd.DataFrame:
         rows = self._model_ops.show_models_or_versions(
+            database_name=None,
+            schema_name=None,
             statement_params=statement_params,
         )
         return pd.DataFrame([row.as_dict() for row in rows])
@@ -159,9 +186,11 @@ class ModelManager:
         *,
         statement_params: Optional[Dict[str, Any]] = None,
     ) -> None:
-        model_name_id = sql_identifier.SqlIdentifier(model_name)
+        database_name_id, schema_name_id, model_name_id = sql_identifier.parse_fully_qualified_name(model_name)
 
         self._model_ops.delete_model_or_version(
+            database_name=database_name_id,
+            schema_name=schema_name_id,
             model_name=model_name_id,
             statement_params=statement_params,
         )

snowflake/ml/version.py

@@ -1 +1 @@
-VERSION="1.5.0"
+VERSION="1.5.1"

{snowflake_ml_python-1.5.0.dist-info → snowflake_ml_python-1.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: snowflake-ml-python
-Version: 1.5.0
+Version: 1.5.1
 Summary: The machine learning client library that is used for interacting with Snowflake to build machine learning solutions.
 Author-email: "Snowflake, Inc" <support@snowflake.com>
 License: 
@@ -236,7 +236,6 @@ License-File: LICENSE.txt
 Requires-Dist: absl-py <2,>=0.15
 Requires-Dist: anyio <4,>=3.5.0
 Requires-Dist: cachetools <6,>=3.1.1
-Requires-Dist: catboost <1.3,>=1.2.0
 Requires-Dist: cloudpickle >=2.0.0
 Requires-Dist: fsspec[http] <2024,>=2022.11
 Requires-Dist: importlib-resources <7,>=6.1.1
@@ -256,19 +255,22 @@ Requires-Dist: sqlparse <1,>=0.4
 Requires-Dist: typing-extensions <5,>=4.1.0
 Requires-Dist: xgboost <2,>=1.7.3
 Provides-Extra: all
-Requires-Dist: lightgbm <4.2,>=3.3.5 ; extra == 'all'
+Requires-Dist: catboost <2,>=1.2.0 ; extra == 'all'
+Requires-Dist: lightgbm <5,>=3.3.5 ; extra == 'all'
 Requires-Dist: mlflow <2.4,>=2.1.0 ; extra == 'all'
 Requires-Dist: peft <1,>=0.5.0 ; extra == 'all'
 Requires-Dist: sentence-transformers <3,>=2.2.2 ; extra == 'all'
-Requires-Dist: sentencepiece <0.2,>=0.1.95 ; extra == 'all'
+Requires-Dist: sentencepiece <1,>=0.1.95 ; extra == 'all'
 Requires-Dist: shap ==0.42.1 ; extra == 'all'
 Requires-Dist: tensorflow <3,>=2.10 ; extra == 'all'
 Requires-Dist: tokenizers <1,>=0.10 ; extra == 'all'
 Requires-Dist: torch <3,>=2.0.1 ; extra == 'all'
 Requires-Dist: torchdata <1,>=0.4 ; extra == 'all'
 Requires-Dist: transformers <5,>=4.32.1 ; extra == 'all'
+Provides-Extra: catboost
+Requires-Dist: catboost <2,>=1.2.0 ; extra == 'catboost'
 Provides-Extra: lightgbm
-Requires-Dist: lightgbm <4.2,>=3.3.5 ; extra == 'lightgbm'
+Requires-Dist: lightgbm <5,>=3.3.5 ; extra == 'lightgbm'
 Provides-Extra: llm
 Requires-Dist: peft <1,>=0.5.0 ; extra == 'llm'
 Provides-Extra: mlflow
@@ -282,7 +284,7 @@ Requires-Dist: torch <3,>=2.0.1 ; extra == 'torch'
 Requires-Dist: torchdata <1,>=0.4 ; extra == 'torch'
 Provides-Extra: transformers
 Requires-Dist: sentence-transformers <3,>=2.2.2 ; extra == 'transformers'
-Requires-Dist: sentencepiece <0.2,>=0.1.95 ; extra == 'transformers'
+Requires-Dist: sentencepiece <1,>=0.1.95 ; extra == 'transformers'
 Requires-Dist: tokenizers <1,>=0.10 ; extra == 'transformers'
 Requires-Dist: transformers <5,>=4.32.1 ; extra == 'transformers'
 
@@ -371,6 +373,31 @@ be compatibility issues. Server-side functionality that `snowflake-ml-python` de
 
 # Release History
 
+## 1.5.1
+
+### Bug Fixes
+
+- Dataset: Fix `snowflake.connector.errors.DataError: Query Result did not match expected number of rows` when accessing
+  DatasetVersion properties when case insensitive `SHOW VERSIONS IN DATASET` check matches multiple version names.
+- Dataset: Fix bug in SnowFS bulk file read when used with DuckDB
+- Registry: Fixed a bug when loading old models.
+- Lineage: Fix Dataset source lineage propagation through `snowpark.DataFrame` transformations
+
+### Behavior Changes
+
+- Feature Store: convert clear() into a private function. Also make it deletes feature views and entities only.
+- Feature Store: Use NULL as default value for timestamp tag value.
+
+### New Features
+
+- Feature Store: Added new `snowflake.ml.feature_store.setup_feature_store()` API to assist Feature Store RBAC setup.
+- Feature Store: Add `output_type` argument to `FeatureStore.generate_dataset()` to allow generating data snapshots
+  as Datasets or Tables.
+- Registry: `log_model`, `get_model`, `delete_model` now supports fully qualified name.
+- Modeling: Supports anonymous stored procedure during fit calls so that modeling would not require sufficient
+  permissions to operate on schema. Please call
+  `import snowflake.ml.modeling.parameters.enable_anonymous_sproc  # noqa: F401`
+
 ## 1.5.0
 
 ### Bug Fixes
@@ -411,12 +438,19 @@ be compatibility issues. Server-side functionality that `snowflake-ml-python` de
 
 #### Feature Store (PrPr)
 
-`FeatureStore.generate_dataset` argument list has been changed to match the new
+- `FeatureStore.generate_dataset` argument list has been changed to match the new
 `snowflake.ml.dataset.Dataset` definition
 
-- `materialized_table` has been removed and replaced with `name` and `version`.
-- `name` moved to first positional argument
-- `save_mode` has been removed as `merge` behavior is no longer supported. The new behavior is always `errorifexists`.
+  - `materialized_table` has been removed and replaced with `name` and `version`.
+  - `name` moved to first positional argument
+  - `save_mode` has been removed as `merge` behavior is no longer supported. The new behavior is always `errorifexists`.
+
+- Change feature view version type from str to `FeatureViewVersion`. It is a restricted string literal.
+
+- Remove as_dataframe arg from FeatureStore.list_feature_views(), now always returns result as DataFrame.
+
+- Combines few metadata tags into a new tag: SNOWML_FEATURE_VIEW_METADATA. This will make previously created feature views
+not readable by new SDK.
 
 ### New Features
 
@@ -432,6 +466,10 @@ be compatibility issues. Server-side functionality that `snowflake-ml-python` de
     and `Dataset.read.to_tf_dataset()` respectively.
 - Added `fsspec` style file integration using `Dataset.read.files()` and `Dataset.read.filesystem()`
 
+#### Feature Store
+
+- use new tag_reference_internal to speed up metadata lookup.
+
 ## 1.4.1 (2024-04-18)
 
 ### New Features
@@ -443,6 +481,10 @@ be compatibility issues. Server-side functionality that `snowflake-ml-python` de
 
 - Registry: Fix a bug that leads to relax_version option is not working.
 
+### Behavior changes
+
+- Feature Store: update_feature_view takes refresh_freq and warehouse as argument.
+
 ## 1.4.0 (2024-04-08)
 
 ### Bug Fixes
@@ -464,6 +506,8 @@ be compatibility issues. Server-side functionality that `snowflake-ml-python` de
 
 - Registry: `apply` method is no longer by default logged when logging a xgboost model. If that is required, it could
   be specified manually when logging the model by `log_model(..., options={"target_methods": ["apply", ...]})`.
+- Feature Store: register_entity returns an entity object.
+- Feature Store: register_feature_view `block=true` becomes default.
 
 ### New Features