snowflake-ml-python 1.8.6__py3-none-any.whl → 1.9.1__py3-none-any.whl

snowflake/ml/model/_client/service/model_deployment_spec_schema.py

@@ -2,6 +2,8 @@ from typing import Optional
 
 from pydantic import BaseModel
 
+BaseModel.model_config["protected_namespaces"] = ()
+
 
 class Model(BaseModel):
     name: str
@@ -53,7 +55,7 @@ class HuggingFaceModel(BaseModel):
     hf_model_name: str
     task: Optional[str] = None
     tokenizer: Optional[str] = None
-    hf_token: Optional[str] = None
+    token: Optional[str] = None
     trust_remote_code: Optional[bool] = False
     revision: Optional[str] = None
     hf_model_kwargs: Optional[str] = "{}"

snowflake/ml/model/_model_composer/model_composer.py

@@ -3,7 +3,7 @@ import tempfile
 import uuid
 import warnings
 from types import ModuleType
-from typing import Any, Optional, Union
+from typing import TYPE_CHECKING, Any, Optional, Union
 from urllib import parse
 
 from absl import logging
@@ -21,6 +21,9 @@ from snowflake.ml.model._packager.model_meta import model_meta
 from snowflake.snowpark import Session
 from snowflake.snowpark._internal import utils as snowpark_utils
 
+if TYPE_CHECKING:
+    from snowflake.ml.experiment._experiment_info import ExperimentInfo
+
 
 class ModelComposer:
     """Top-level class to construct contents in a MODEL object in SQL.
@@ -136,6 +139,7 @@ class ModelComposer:
         ext_modules: Optional[list[ModuleType]] = None,
         code_paths: Optional[list[str]] = None,
         task: model_types.Task = model_types.Task.UNKNOWN,
+        experiment_info: Optional["ExperimentInfo"] = None,
         options: Optional[model_types.ModelSaveOption] = None,
     ) -> model_meta.ModelMetadata:
         # set enable_explainability=False if the model is not runnable in WH or the target platforms include SPCS
@@ -230,6 +234,7 @@ class ModelComposer:
             options=options,
             user_files=user_files,
             data_sources=self._get_data_sources(model, sample_input_data),
+            experiment_info=experiment_info,
             target_platforms=target_platforms,
         )
 

snowflake/ml/model/_model_composer/model_manifest/model_manifest.py

@@ -2,11 +2,12 @@ import collections
 import logging
 import pathlib
 import warnings
-from typing import Optional, cast
+from typing import TYPE_CHECKING, Optional, cast
 
 import yaml
 
 from snowflake.ml._internal import env_utils
+from snowflake.ml._internal.exceptions import error_codes, exceptions
 from snowflake.ml.data import data_source
 from snowflake.ml.model import type_hints
 from snowflake.ml.model._model_composer.model_manifest import model_manifest_schema
@@ -22,6 +23,9 @@ from snowflake.ml.model._packager.model_meta import (
 )
 from snowflake.ml.model._packager.model_runtime import model_runtime
 
+if TYPE_CHECKING:
+    from snowflake.ml.experiment._experiment_info import ExperimentInfo
+
 logger = logging.getLogger(__name__)
 
 
@@ -48,22 +52,50 @@ class ModelManifest:
         user_files: Optional[dict[str, list[str]]] = None,
         options: Optional[type_hints.ModelSaveOption] = None,
         data_sources: Optional[list[data_source.DataSource]] = None,
+        experiment_info: Optional["ExperimentInfo"] = None,
         target_platforms: Optional[list[type_hints.TargetPlatform]] = None,
     ) -> None:
         if options is None:
             options = {}
 
+        has_pip_requirements = len(model_meta.env.pip_requirements) > 0
+        only_spcs = (
+            target_platforms
+            and len(target_platforms) == 1
+            and target_platforms[0] == type_hints.TargetPlatform.SNOWPARK_CONTAINER_SERVICES
+        )
+
         if "relax_version" not in options:
-            warnings.warn(
-                (
-                    "`relax_version` is not set and therefore defaulted to True. Dependency version constraints relaxed"
-                    " from ==x.y.z to >=x.y, <(x+1). To use specific dependency versions for compatibility, "
-                    "reproducibility, etc., set `options={'relax_version': False}` when logging the model."
-                ),
-                category=UserWarning,
-                stacklevel=2,
-            )
-        relax_version = options.get("relax_version", True)
+            if has_pip_requirements or only_spcs:
+                logger.info(
+                    "Setting `relax_version=False` as this model will run in Snowpark Container Services "
+                    "or in Warehouse with a specified artifact_repository_map where exact version "
+                    " specifications will be honored."
+                )
+                relax_version = False
+            else:
+                warnings.warn(
+                    (
+                        "`relax_version` is not set and therefore defaulted to True. Dependency version constraints"
+                        " relaxed from ==x.y.z to >=x.y, <(x+1). To use specific dependency versions for compatibility,"
+                        " reproducibility, etc., set `options={'relax_version': False}` when logging the model."
+                    ),
+                    category=UserWarning,
+                    stacklevel=2,
+                )
+                relax_version = True
+            options["relax_version"] = relax_version
+        else:
+            relax_version = options.get("relax_version", True)
+            if relax_version and (has_pip_requirements or only_spcs):
+                raise exceptions.SnowflakeMLException(
+                    error_code=error_codes.INVALID_ARGUMENT,
+                    original_exception=ValueError(
+                        "Setting `relax_version=True` is only allowed for models to be run in Warehouse with "
+                        "Snowflake Conda Channel dependencies. It cannot be used with pip requirements or when "
+                        "targeting only Snowpark Container Services."
+                    ),
+                )
 
         runtime_to_use = model_runtime.ModelRuntime(
             name=self._DEFAULT_RUNTIME_NAME,
@@ -155,7 +187,7 @@ class ModelManifest:
         if self.user_files:
             manifest_dict["user_files"] = [user_file.save(self.workspace_path) for user_file in self.user_files]
 
-        lineage_sources = self._extract_lineage_info(data_sources)
+        lineage_sources = self._extract_lineage_info(data_sources, experiment_info)
         if lineage_sources:
             manifest_dict["lineage_sources"] = lineage_sources
 
@@ -182,7 +214,9 @@ class ModelManifest:
         return res
 
     def _extract_lineage_info(
-        self, data_sources: Optional[list[data_source.DataSource]]
+        self,
+        data_sources: Optional[list[data_source.DataSource]],
+        experiment_info: Optional["ExperimentInfo"],
     ) -> list[model_manifest_schema.LineageSourceDict]:
         result = []
         if data_sources:
@@ -201,4 +235,12 @@ class ModelManifest:
                             type=model_manifest_schema.LineageSourceTypes.QUERY.value, entity=source.sql
                         )
                     )
+        if experiment_info:
+            result.append(
+                model_manifest_schema.LineageSourceDict(
+                    type=model_manifest_schema.LineageSourceTypes.EXPERIMENT.value,
+                    entity=experiment_info.fully_qualified_name,
+                    version=experiment_info.run_name,
+                )
+            )
         return result

snowflake/ml/model/_model_composer/model_manifest/model_manifest_schema.py

@@ -83,6 +83,7 @@ class SnowparkMLDataDict(TypedDict):
 class LineageSourceTypes(enum.Enum):
     DATASET = "DATASET"
     QUERY = "QUERY"
+    EXPERIMENT = "EXPERIMENT"
 
 
 class LineageSourceDict(TypedDict):

snowflake/ml/model/_packager/model_env/model_env.py

@@ -9,6 +9,7 @@ from packaging import requirements, version
 
 from snowflake.ml import version as snowml_version
 from snowflake.ml._internal import env as snowml_env, env_utils
+from snowflake.ml.model import type_hints as model_types
 from snowflake.ml.model._packager.model_meta import model_meta_schema
 
 # requirement: Full version requirement where name is conda package name.
@@ -30,6 +31,7 @@ class ModelEnv:
         conda_env_rel_path: Optional[str] = None,
         pip_requirements_rel_path: Optional[str] = None,
         prefer_pip: bool = False,
+        target_platforms: Optional[list[model_types.TargetPlatform]] = None,
     ) -> None:
         if conda_env_rel_path is None:
             conda_env_rel_path = os.path.join(_DEFAULT_ENV_DIR, _DEFAULT_CONDA_ENV_FILENAME)
@@ -45,6 +47,8 @@ class ModelEnv:
         self._python_version: version.Version = version.parse(snowml_env.PYTHON_VERSION)
         self._cuda_version: Optional[version.Version] = None
         self._snowpark_ml_version: version.Version = version.parse(snowml_version.VERSION)
+        self._target_platforms = target_platforms
+        self._warnings_shown: set[str] = set()
 
     @property
     def conda_dependencies(self) -> list[str]:
@@ -116,6 +120,17 @@ class ModelEnv:
         if snowpark_ml_version:
             self._snowpark_ml_version = version.parse(snowpark_ml_version)
 
+    @property
+    def targets_warehouse(self) -> bool:
+        """Returns True if warehouse is a target platform."""
+        return self._target_platforms is None or model_types.TargetPlatform.WAREHOUSE in self._target_platforms
+
+    def _warn_once(self, message: str, stacklevel: int = 2) -> None:
+        """Show warning only once per ModelEnv instance."""
+        if message not in self._warnings_shown:
+            warnings.warn(message, category=UserWarning, stacklevel=stacklevel)
+            self._warnings_shown.add(message)
+
     def include_if_absent(
         self,
         pkgs: list[ModelDependency],
@@ -130,14 +145,14 @@ class ModelEnv:
         """
         if (self.pip_requirements or self.prefer_pip) and not self.conda_dependencies and pkgs:
             pip_pkg_reqs: list[str] = []
-            warnings.warn(
-                (
-                    "Dependencies specified from pip requirements."
-                    " This may prevent model deploying to Snowflake Warehouse."
-                ),
-                category=UserWarning,
-                stacklevel=2,
-            )
+            if self.targets_warehouse:
+                self._warn_once(
+                    (
+                        "Dependencies specified from pip requirements."
+                        " This may prevent model deploying to Snowflake Warehouse."
+                    ),
+                    stacklevel=2,
+                )
             for conda_req_str, pip_name in pkgs:
                 _, conda_req = env_utils._validate_conda_dependency_string(conda_req_str)
                 pip_req = requirements.Requirement(f"{pip_name}{conda_req.specifier}")
@@ -162,16 +177,15 @@ class ModelEnv:
                 req_to_add.name = conda_req.name
             else:
                 req_to_add = conda_req
-            show_warning_message = conda_req_channel == env_utils.DEFAULT_CHANNEL_NAME
+            show_warning_message = conda_req_channel == env_utils.DEFAULT_CHANNEL_NAME and self.targets_warehouse
 
             if any(added_pip_req.name == pip_name for added_pip_req in self._pip_requirements):
                 if show_warning_message:
-                    warnings.warn(
+                    self._warn_once(
                         (
                             f"Basic dependency {req_to_add.name} specified from pip requirements."
                             " This may prevent model deploying to Snowflake Warehouse."
                         ),
-                        category=UserWarning,
                         stacklevel=2,
                     )
                 continue
@@ -182,12 +196,11 @@ class ModelEnv:
                 pass
             except env_utils.DuplicateDependencyInMultipleChannelsError:
                 if show_warning_message:
-                    warnings.warn(
+                    self._warn_once(
                         (
                             f"Basic dependency {req_to_add.name} specified from non-Snowflake channel."
                             + " This may prevent model deploying to Snowflake Warehouse."
                         ),
-                        category=UserWarning,
                         stacklevel=2,
                     )
 
@@ -272,22 +285,20 @@ class ModelEnv:
         )
 
         for channel, channel_dependencies in conda_dependencies_dict.items():
-            if channel != env_utils.DEFAULT_CHANNEL_NAME:
-                warnings.warn(
+            if channel != env_utils.DEFAULT_CHANNEL_NAME and self.targets_warehouse:
+                self._warn_once(
                     (
                         "Found dependencies specified in the conda file from non-Snowflake channel."
                         " This may prevent model deploying to Snowflake Warehouse."
                     ),
-                    category=UserWarning,
                     stacklevel=2,
                 )
-            if len(channel_dependencies) == 0 and channel not in self._conda_dependencies:
-                warnings.warn(
+            if len(channel_dependencies) == 0 and channel not in self._conda_dependencies and self.targets_warehouse:
+                self._warn_once(
                     (
                         f"Found additional conda channel {channel} specified in the conda file."
                         " This may prevent model deploying to Snowflake Warehouse."
                     ),
-                    category=UserWarning,
                     stacklevel=2,
                 )
                 self._conda_dependencies[channel] = []
@@ -298,22 +309,20 @@ class ModelEnv:
                 except env_utils.DuplicateDependencyError:
                     pass
                 except env_utils.DuplicateDependencyInMultipleChannelsError:
-                    warnings.warn(
+                    self._warn_once(
                         (
                             f"Dependency {channel_dependency.name} appeared in multiple channels as conda dependency."
                             " This may be unintentional."
                         ),
-                        category=UserWarning,
                         stacklevel=2,
                     )
 
-        if pip_requirements_list:
-            warnings.warn(
+        if pip_requirements_list and self.targets_warehouse:
+            self._warn_once(
                 (
                     "Found dependencies specified as pip requirements."
                     " This may prevent model deploying to Snowflake Warehouse."
                 ),
-                category=UserWarning,
                 stacklevel=2,
             )
             for pip_dependency in pip_requirements_list:
@@ -333,13 +342,12 @@ class ModelEnv:
     def load_from_pip_file(self, pip_requirements_path: pathlib.Path) -> None:
         pip_requirements_list = env_utils.load_requirements_file(pip_requirements_path)
 
-        if pip_requirements_list:
-            warnings.warn(
+        if pip_requirements_list and self.targets_warehouse:
+            self._warn_once(
                 (
                     "Found dependencies specified as pip requirements."
                     " This may prevent model deploying to Snowflake Warehouse."
                 ),
-                category=UserWarning,
                 stacklevel=2,
             )
             for pip_dependency in pip_requirements_list:

snowflake/ml/model/_packager/model_handlers/huggingface_pipeline.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import os
 import warnings
 from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast, final
@@ -23,9 +24,13 @@ from snowflake.ml.model._signatures import utils as model_signature_utils
 from snowflake.ml.model.models import huggingface_pipeline
 from snowflake.snowpark._internal import utils as snowpark_utils
 
+logger = logging.getLogger(__name__)
+
 if TYPE_CHECKING:
     import transformers
 
+DEFAULT_CHAT_TEMPLATE = "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"  # noqa: E501
+
 
 def get_requirements_from_task(task: str, spcs_only: bool = False) -> list[model_env.ModelDependency]:
     # Text
@@ -326,6 +331,23 @@ class HuggingFacePipelineHandler(
                 **device_config,
             )
 
+            # If the task is text-generation, and the tokenizer does not have a chat_template,
+            # set the default chat template.
+            if (
+                hasattr(m, "task")
+                and m.task == "text-generation"
+                and hasattr(m.tokenizer, "chat_template")
+                and not m.tokenizer.chat_template
+            ):
+                warnings.warn(
+                    "The tokenizer does not have default chat_template. "
+                    "Setting the chat_template to default ChatML template.",
+                    UserWarning,
+                    stacklevel=1,
+                )
+                logger.info(DEFAULT_CHAT_TEMPLATE)
+                m.tokenizer.chat_template = DEFAULT_CHAT_TEMPLATE
+
             m.__dict__.update(pipeline_params)
 
         else:
@@ -481,8 +503,25 @@ class HuggingFacePipelineHandler(
 
         # To enable batch_size > 1 for LLM
         # Pipe might not have tokenizer, but should always have a model, and model should always have a config.
-        if getattr(pipe, "tokenizer", None) is not None and pipe.tokenizer.pad_token_id is None:
-            pipe.tokenizer.pad_token_id = pipe.model.config.eos_token_id
+        if (
+            getattr(pipe, "tokenizer", None) is not None
+            and pipe.tokenizer.pad_token_id is None
+            and hasattr(pipe.model.config, "eos_token_id")
+        ):
+            if isinstance(pipe.model.config.eos_token_id, int):
+                pipe.tokenizer.pad_token_id = pipe.model.config.eos_token_id
+            elif (
+                isinstance(pipe.model.config.eos_token_id, list)
+                and len(pipe.model.config.eos_token_id) > 0
+                and isinstance(pipe.model.config.eos_token_id[0], int)
+            ):
+                pipe.tokenizer.pad_token_id = pipe.model.config.eos_token_id[0]
+            else:
+                warnings.warn(
+                    f"Unexpected type of eos_token_id: {type(pipe.model.config.eos_token_id)}. "
+                    "Not setting pad_token_id to eos_token_id.",
+                    stacklevel=2,
+                )
 
         _HFPipelineModel = _create_custom_model(pipe, model_meta)
         hg_pipe_model = _HFPipelineModel(custom_model.ModelContext())

snowflake/ml/model/_packager/model_handlers/pytorch.py

@@ -167,7 +167,11 @@ class PyTorchHandler(_base.BaseModelHandler["torch.nn.Module"]):
         model_blob_metadata = model_blobs_metadata[name]
         model_blob_filename = model_blob_metadata.path
         with open(os.path.join(model_blob_path, model_blob_filename), "rb") as f:
-            m = torch.load(f, map_location="cuda" if kwargs.get("use_gpu", False) else "cpu")
+            m = torch.load(
+                f,
+                map_location="cuda" if kwargs.get("use_gpu", False) else "cpu",
+                weights_only=False,
+            )
         assert isinstance(m, torch.nn.Module)
 
         return m

snowflake/ml/model/_packager/model_meta/model_meta.py

@@ -110,6 +110,7 @@ def create_model_metadata(
         python_version=python_version,
         embed_local_ml_library=embed_local_ml_library,
         prefer_pip=prefer_pip,
+        target_platforms=target_platforms,
     )
 
     if embed_local_ml_library:
@@ -162,8 +163,9 @@ def _create_env_for_model_metadata(
     python_version: Optional[str] = None,
     embed_local_ml_library: bool = False,
     prefer_pip: bool = False,
+    target_platforms: Optional[list[model_types.TargetPlatform]] = None,
 ) -> model_env.ModelEnv:
-    env = model_env.ModelEnv(prefer_pip=prefer_pip)
+    env = model_env.ModelEnv(prefer_pip=prefer_pip, target_platforms=target_platforms)
 
     # Mypy doesn't like getter and setter have different types. See python/mypy #3004
     env.conda_dependencies = conda_dependencies  # type: ignore[assignment]

snowflake/ml/model/_packager/model_runtime/_snowml_inference_alternative_requirements.py

@@ -10,7 +10,7 @@ REQUIREMENTS = [
     "cryptography",
     "fsspec>=2024.6.1,<2026",
     "importlib_resources>=6.1.1, <7",
-    "numpy>=1.23,<2",
+    "numpy>=1.23,<3",
     "packaging>=20.9,<25",
     "pandas>=2.1.4,<3",
     "pyarrow",
@@ -28,6 +28,7 @@ REQUIREMENTS = [
     "snowflake-snowpark-python>=1.17.0,<2,!=1.26.0",
     "snowflake.core>=1.0.2,<2",
     "sqlparse>=0.4,<1",
+    "tqdm<5",
     "typing-extensions>=4.1.0,<5",
     "xgboost>=1.7.3,<3",
 ]

snowflake/ml/model/_packager/model_runtime/model_runtime.py

@@ -98,9 +98,9 @@ class ModelRuntime:
             dependencies=model_meta_schema.ModelRuntimeDependenciesDict(
                 conda=env_dict["conda"],
                 pip=env_dict["pip"],
-                artifact_repository_map=env_dict["artifact_repository_map"]
-                if env_dict.get("artifact_repository_map") is not None
-                else {},
+                artifact_repository_map=(
+                    env_dict["artifact_repository_map"] if env_dict.get("artifact_repository_map") is not None else {}
+                ),
             ),
             resource_constraint=env_dict["resource_constraint"],
         )

snowflake/ml/model/_signatures/snowpark_handler.py

@@ -60,12 +60,19 @@ class SnowparkDataFrameHandler(base_handler.BaseDataHandler[snowflake.snowpark.D
         data: snowflake.snowpark.DataFrame,
         ensure_serializable: bool = True,
         features: Optional[Sequence[core.BaseFeatureSpec]] = None,
+        statement_params: Optional[dict[str, Any]] = None,
     ) -> pd.DataFrame:
         # This method do things on top of to_pandas, to make sure the local dataframe got is in correct shape.
         dtype_map = {}
+
         if features:
+            quoted_identifiers_ignore_case = SnowparkDataFrameHandler._is_quoted_identifiers_ignore_case_enabled(
+                data.session, statement_params
+            )
             for feature in features:
-                dtype_map[feature.name] = feature.as_dtype()
+                feature_name = feature.name.upper() if quoted_identifiers_ignore_case else feature.name
+                dtype_map[feature_name] = feature.as_dtype()
+
         df_local = data.to_pandas()
 
         # This is because Array will become string (Even though the correct schema is set)
@@ -93,6 +100,7 @@ class SnowparkDataFrameHandler(base_handler.BaseDataHandler[snowflake.snowpark.D
         df: pd.DataFrame,
         keep_order: bool = False,
         features: Optional[Sequence[core.BaseFeatureSpec]] = None,
+        statement_params: Optional[dict[str, Any]] = None,
     ) -> snowflake.snowpark.DataFrame:
         # This method is necessary to create the Snowpark Dataframe in correct schema.
         # However, in this case, the order could not be preserved. Thus, a _ID column has to be added,
@@ -100,6 +108,12 @@ class SnowparkDataFrameHandler(base_handler.BaseDataHandler[snowflake.snowpark.D
         # Although in this case, the column with array type can get correct ARRAY type, however, the element
         # type is not preserved, and will become string type. This affect the implementation of convert_from_df.
         df = pandas_handler.PandasDataFrameHandler.convert_to_df(df)
+        quoted_identifiers_ignore_case = SnowparkDataFrameHandler._is_quoted_identifiers_ignore_case_enabled(
+            session, statement_params
+        )
+        if quoted_identifiers_ignore_case:
+            df.columns = [str(col).upper() for col in df.columns]
+
         df_cols = df.columns
         if df_cols.dtype != np.object_:
             raise snowml_exceptions.SnowflakeMLException(
@@ -116,9 +130,47 @@ class SnowparkDataFrameHandler(base_handler.BaseDataHandler[snowflake.snowpark.D
         column_names = []
         columns = []
         for feature in features:
-            column_names.append(identifier.get_inferred_name(feature.name))
-            columns.append(F.col(identifier.get_inferred_name(feature.name)).cast(feature.as_snowpark_type()))
+            feature_name = identifier.get_inferred_name(feature.name)
+            if quoted_identifiers_ignore_case:
+                feature_name = feature_name.upper()
+            column_names.append(feature_name)
+            columns.append(F.col(feature_name).cast(feature.as_snowpark_type()))
 
         sp_df = sp_df.with_columns(column_names, columns)
 
         return sp_df
+
+    @staticmethod
+    def _is_quoted_identifiers_ignore_case_enabled(
+        session: snowflake.snowpark.Session, statement_params: Optional[dict[str, Any]] = None
+    ) -> bool:
+        """
+        Check if QUOTED_IDENTIFIERS_IGNORE_CASE parameter is enabled.
+
+        Args:
+            session: Snowpark session to check parameter for
+            statement_params: Optional statement parameters to check first
+
+        Returns:
+            bool: True if QUOTED_IDENTIFIERS_IGNORE_CASE is enabled, False otherwise
+            Returns False if the parameter cannot be retrieved (e.g., in stored procedures)
+        """
+        if statement_params is not None:
+            for key, value in statement_params.items():
+                if key.upper() == "QUOTED_IDENTIFIERS_IGNORE_CASE":
+                    parameter_value = str(value)
+                    return parameter_value.lower() == "true"
+
+        try:
+            result = session.sql(
+                "SHOW PARAMETERS LIKE 'QUOTED_IDENTIFIERS_IGNORE_CASE' IN SESSION",
+                _emit_ast=False,
+            ).collect(_emit_ast=False)
+
+            parameter_value = str(result[0].value)
+            return parameter_value.lower() == "true"
+
+        except Exception:
+            # Parameter query can fail in certain environments (e.g., in stored procedures)
+            # In that case, assume default behavior (case-sensitive)
+            return False