snowflake-ml-python 1.2.0__py3-none-any.whl → 1.2.2__py3-none-any.whl

snowflake/ml/model/_deploy_client/snowservice/deploy_options.py

@@ -1,6 +1,6 @@
 import inspect
 import logging
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from snowflake.ml._internal.exceptions import (
     error_codes,
@@ -16,6 +16,7 @@ class SnowServiceDeployOptions:
         self,
         compute_pool: str,
         *,
+        external_access_integrations: List[str],
         image_repo: Optional[str] = None,
         min_instances: Optional[int] = 1,
         max_instances: Optional[int] = 1,
@@ -34,7 +35,15 @@ class SnowServiceDeployOptions:
 
         Args:
             compute_pool: SnowService compute pool name. Please refer to official doc for how to create a
-                compute pool: https://docs.snowflake.com/LIMITEDACCESS/snowpark-containers/reference/compute-pool
+                compute pool:
+                https://docs.snowflake.com/en/developer-guide/snowpark-container-services/working-with-compute-pool
+            external_access_integrations: External Access Integrations name used to build image and deploy the model.
+                Please refer to the doc for how to create an External Access Integrations: https://docs.snowflake.com/
+                developer-guide/snowpark-container-services/additional-considerations-services-jobs
+                #configuring-network-capabilities .
+                To make sure your image could be built, access to the following endpoint must be allowed.
+                docker.com:80, docker.com:443, anaconda.com:80, anaconda.com:443, anaconda.org:80, anaconda.org:443,
+                pypi.org:80, pypi.org:443
             image_repo: SnowService image repo path. e.g. "<image_registry>/<db>/<schema>/<repo>". Default to auto
                 inferred based on session information.
             min_instances: Minimum number of service replicas. Default to 1.
@@ -70,6 +79,7 @@ class SnowServiceDeployOptions:
         self.model_in_image = model_in_image
         self.debug_mode = debug_mode
         self.enable_ingress = enable_ingress
+        self.external_access_integrations = external_access_integrations
 
         if self.num_workers is None and self.use_gpu:
             logger.info("num_workers has been defaulted to 1 when using GPU.")

snowflake/ml/model/_deploy_client/utils/snowservice_client.py

@@ -2,7 +2,7 @@ import json
 import logging
 import textwrap
 import time
-from typing import Optional
+from typing import List, Optional
 
 from snowflake.ml._internal.exceptions import (
     error_codes,
@@ -36,6 +36,7 @@ class SnowServiceClient:
         service_name: str,
         compute_pool: str,
         spec_stage_location: str,
+        external_access_integrations: List[str],
         *,
         min_instances: Optional[int] = 1,
         max_instances: Optional[int] = 1,
@@ -48,6 +49,7 @@ class SnowServiceClient:
             service_name: Name of the service.
             min_instances: Minimum number of service replicas.
             max_instances: Maximum number of service replicas.
+            external_access_integrations: EAIs for network connection.
             compute_pool: Name of the compute pool.
             spec_stage_location: Stage path for the service spec.
         """
@@ -61,13 +63,14 @@ class SnowServiceClient:
                 SPEC = '{path}'
                 MIN_INSTANCES={min_instances}
                 MAX_INSTANCES={max_instances}
+                EXTERNAL_ACCESS_INTEGRATIONS = ({', '.join(external_access_integrations)})
             """
         )
         logger.info(f"Creating service {service_name}")
         logger.debug(f"Create service with SQL: \n {sql}")
         self.session.sql(sql).collect()
 
-    def create_job(self, compute_pool: str, spec_stage_location: str) -> None:
+    def create_job(self, compute_pool: str, spec_stage_location: str, external_access_integrations: List[str]) -> None:
         """Execute the job creation SQL command. Note that the job creation is synchronous, hence we execute it in a
         async way so that we can query the log in the meantime.
 
@@ -76,7 +79,7 @@ class SnowServiceClient:
         Args:
             compute_pool: name of the compute pool
             spec_stage_location: path to the stage location where the spec is located at.
-
+            external_access_integrations: EAIs for network connection.
         """
         stage, path = uri.get_stage_and_path(spec_stage_location)
         sql = textwrap.dedent(
@@ -85,6 +88,7 @@ class SnowServiceClient:
             IN COMPUTE POOL {compute_pool}
             FROM {stage}
             SPEC = '{path}'
+            EXTERNAL_ACCESS_INTEGRATIONS = ({', '.join(external_access_integrations)})
             """
         )
         logger.debug(f"Create job with SQL: \n {sql}")

snowflake/ml/model/_model_composer/model_manifest/model_manifest.py

@@ -4,7 +4,6 @@ from typing import Any, Dict, List, Optional, cast
 
 import yaml
 
-from snowflake.ml._internal.utils import snowflake_env
 from snowflake.ml.model import type_hints
 from snowflake.ml.model._model_composer.model_manifest import model_manifest_schema
 from snowflake.ml.model._model_composer.model_method import (
@@ -84,11 +83,7 @@ class ModelManifest:
             ],
         )
 
-        if (
-            snowflake_env.get_current_snowflake_version(session)
-            >= model_manifest_schema.MANIFEST_USER_DATA_ENABLE_VERSION
-        ):
-            manifest_dict["user_data"] = self.generate_user_data_with_client_data(model_meta)
+        manifest_dict["user_data"] = self.generate_user_data_with_client_data(model_meta)
 
         with (self.workspace_path / ModelManifest.MANIFEST_FILE_REL_PATH).open("w", encoding="utf-8") as f:
             # Anchors are not supported in the server, avoid that.

snowflake/ml/model/_model_composer/model_manifest/model_manifest_schema.py

@@ -2,14 +2,12 @@
 
 from typing import Any, Dict, List, Literal, TypedDict
 
-from packaging import version
 from typing_extensions import NotRequired, Required
 
 from snowflake.ml.model import model_signature
 
 MODEL_MANIFEST_VERSION = "1.0"
 
-MANIFEST_USER_DATA_ENABLE_VERSION = version.parse("8.2.0")
 MANIFEST_CLIENT_DATA_KEY_NAME = "snowpark_ml_data"
 MANIFEST_CLIENT_DATA_SCHEMA_VERSION = "2024-02-01"
 

snowflake/ml/model/_model_composer/model_runtime/_runtime_requirements.py

@@ -1 +1,10 @@
-REQUIREMENTS = ['absl-py>=0.15,<2', 'anyio>=3.5.0,<4', 'numpy>=1.23,<2', 'packaging>=20.9,<24', 'pandas>=1.0.0,<2', 'pyyaml>=6.0,<7', 'snowflake-snowpark-python>=1.8.0,<2', 'typing-extensions>=4.1.0,<5']
+REQUIREMENTS = [
+    "absl-py>=0.15,<2",
+    "anyio>=3.5.0,<4",
+    "numpy>=1.23,<2",
+    "packaging>=20.9,<24",
+    "pandas>=1.0.0,<2",
+    "pyyaml>=6.0,<7",
+    "snowflake-snowpark-python>=1.8.0,<2",
+    "typing-extensions>=4.1.0,<5"
+]

snowflake/ml/model/_model_composer/model_runtime/model_runtime.py

@@ -62,7 +62,6 @@ class ModelRuntime:
                     model_env.ModelDependency(requirement=dep, pip_name=requirements.Requirement(dep).name)
                     for dep in _UDF_INFERENCE_DEPENDENCIES
                 ],
-                check_local_version=True,
             )
         else:
             self.runtime_env.include_if_absent(
@@ -70,7 +69,6 @@ class ModelRuntime:
                     model_env.ModelDependency(requirement=dep, pip_name=requirements.Requirement(dep).name)
                     for dep in _UDF_INFERENCE_DEPENDENCIES + [snowml_pkg_spec]
                 ],
-                check_local_version=True,
             )
 
     def save(self, workspace_path: pathlib.Path) -> model_manifest_schema.ModelRuntimeDict:

snowflake/ml/model/_packager/model_meta/_core_requirements.py

@@ -1 +1,11 @@
-REQUIREMENTS = ['absl-py>=0.15,<2', 'anyio>=3.5.0,<4', 'cloudpickle>=2.0.0', 'numpy>=1.23,<2', 'packaging>=20.9,<24', 'pandas>=1.0.0,<2', 'pyyaml>=6.0,<7', 'snowflake-snowpark-python>=1.8.0,<2', 'typing-extensions>=4.1.0,<5']
+REQUIREMENTS = [
+    "absl-py>=0.15,<2",
+    "anyio>=3.5.0,<4",
+    "cloudpickle>=2.0.0",
+    "numpy>=1.23,<2",
+    "packaging>=20.9,<24",
+    "pandas>=1.0.0,<2",
+    "pyyaml>=6.0,<7",
+    "snowflake-snowpark-python>=1.8.0,<2",
+    "typing-extensions>=4.1.0,<5"
+]

snowflake/ml/model/_packager/model_meta/_packaging_requirements.py

@@ -0,0 +1,3 @@
+REQUIREMENTS = [
+    "cloudpickle>=2.0.0"
+]

snowflake/ml/model/_packager/model_meta/model_meta.py

@@ -18,6 +18,7 @@ from snowflake.ml.model import model_signature, type_hints as model_types
 from snowflake.ml.model._packager.model_env import model_env
 from snowflake.ml.model._packager.model_meta import (
     _core_requirements,
+    _packaging_requirements,
     model_blob_meta,
     model_meta_schema,
 )
@@ -26,7 +27,8 @@ from snowflake.ml.model._packager.model_meta_migrator import migrator_plans
 MODEL_METADATA_FILE = "model.yaml"
 MODEL_CODE_DIR = "code"
 
-_PACKAGING_CORE_DEPENDENCIES = _core_requirements.REQUIREMENTS
+_PACKAGING_CORE_DEPENDENCIES = _core_requirements.REQUIREMENTS  # Legacy Model only
+_PACKAGING_REQUIREMENTS = _packaging_requirements.REQUIREMENTS  # New Model only
 _SNOWFLAKE_PKG_NAME = "snowflake"
 _SNOWFLAKE_ML_PKG_NAME = f"{_SNOWFLAKE_PKG_NAME}.ml"
 
@@ -73,6 +75,8 @@ def create_model_metadata(
     model_dir_path = os.path.normpath(model_dir_path)
     embed_local_ml_library = kwargs.pop("embed_local_ml_library", False)
     legacy_save = kwargs.pop("_legacy_save", False)
+    relax_version = kwargs.pop("relax_version", False)
+
     if embed_local_ml_library:
         # Use the last one which is loaded first, that is mean, it is loaded from site-packages.
         # We could make sure that user does not overwrite our library with their code follow the same naming.
@@ -94,6 +98,8 @@ def create_model_metadata(
         pip_requirements=pip_requirements,
         python_version=python_version,
         embed_local_ml_library=embed_local_ml_library,
+        legacy_save=legacy_save,
+        relax_version=relax_version,
     )
 
     if embed_local_ml_library:
@@ -146,6 +152,8 @@ def _create_env_for_model_metadata(
     pip_requirements: Optional[List[str]] = None,
     python_version: Optional[str] = None,
     embed_local_ml_library: bool = False,
+    legacy_save: bool = False,
+    relax_version: bool = False,
 ) -> model_env.ModelEnv:
     env = model_env.ModelEnv()
 
@@ -154,11 +162,14 @@ def _create_env_for_model_metadata(
     env.pip_requirements = pip_requirements  # type: ignore[assignment]
     env.python_version = python_version  # type: ignore[assignment]
     env.snowpark_ml_version = snowml_env.VERSION
+
+    requirements_to_add = _PACKAGING_CORE_DEPENDENCIES if legacy_save else _PACKAGING_REQUIREMENTS
+
     if embed_local_ml_library:
         env.include_if_absent(
             [
                 model_env.ModelDependency(requirement=dep, pip_name=requirements.Requirement(dep).name)
-                for dep in _PACKAGING_CORE_DEPENDENCIES
+                for dep in requirements_to_add
             ],
             check_local_version=True,
         )
@@ -166,11 +177,14 @@ def _create_env_for_model_metadata(
         env.include_if_absent(
             [
                 model_env.ModelDependency(requirement=dep, pip_name=requirements.Requirement(dep).name)
-                for dep in _PACKAGING_CORE_DEPENDENCIES + [env_utils.SNOWPARK_ML_PKG_NAME]
+                for dep in requirements_to_add + [env_utils.SNOWPARK_ML_PKG_NAME]
             ],
             check_local_version=True,
         )
 
+    if relax_version:
+        env.relax_version()
+
     return env
 
 

snowflake/ml/model/model_signature.py

@@ -1,6 +1,18 @@
 import enum
+import json
 import warnings
-from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple, Type
+from typing import (
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+    cast,
+)
 
 import numpy as np
 import pandas as pd
@@ -337,6 +349,31 @@ class SnowparkIdentifierRule(enum.Enum):
             assert_never(self)
 
 
+def _get_dataframe_values_range(
+    df: snowflake.snowpark.DataFrame,
+) -> Dict[str, Union[Tuple[int, int], Tuple[float, float]]]:
+    columns = [
+        F.array_construct(F.min(field.name), F.max(field.name)).as_(field.name)
+        for field in df.schema.fields
+        if isinstance(field.datatype, spt._NumericType)
+    ]
+    if not columns:
+        return {}
+    res = df.select(columns).collect()
+    if len(res) != 1:
+        raise snowml_exceptions.SnowflakeMLException(
+            error_code=error_codes.INTERNAL_SNOWML_ERROR,
+            original_exception=ValueError(f"Unable to get the value range of fields {df.columns}"),
+        )
+    return cast(
+        Dict[str, Union[Tuple[int, int], Tuple[float, float]]],
+        {
+            sql_identifier.SqlIdentifier(k, case_sensitive=True).identifier(): (json.loads(v)[0], json.loads(v)[1])
+            for k, v in res[0].as_dict().items()
+        },
+    )
+
+
 def _validate_snowpark_data(
     data: snowflake.snowpark.DataFrame, features: Sequence[core.BaseFeatureSpec]
 ) -> SnowparkIdentifierRule:
@@ -361,6 +398,7 @@ def _validate_snowpark_data(
         SnowparkIdentifierRule.NORMALIZED: [],
     }
     schema = data.schema
+    values_range = _get_dataframe_values_range(data)
     for identifier_rule in errors.keys():
         for feature in features:
             try:
@@ -401,8 +439,11 @@ def _validate_snowpark_data(
                                     + f"Feature is a scalar feature, while {field.name} is not."
                                 ),
                             )
+                            continue
                         try:
-                            _validate_snowpark_type_feature(data, field, ft_type, feature.name)
+                            _validate_snowpark_type_feature(
+                                data, field, ft_type, feature.name, values_range.get(field.name, None)
+                            )
                         except snowml_exceptions.SnowflakeMLException as e:
                             errors[identifier_rule].append(e.original_exception)
                     break
@@ -433,17 +474,12 @@ If using the inferred names from model signatures, there are the following error
 
 
 def _validate_snowpark_type_feature(
-    df: snowflake.snowpark.DataFrame, field: spt.StructField, ft_type: DataType, ft_name: str
+    df: snowflake.snowpark.DataFrame,
+    field: spt.StructField,
+    ft_type: DataType,
+    ft_name: str,
+    value_range: Optional[Union[Tuple[int, int], Tuple[float, float]]],
 ) -> None:
-    def get_value_range(field_name: str) -> Tuple[int, int]:
-        res = df.select(F.min(field_name).as_("MIN"), F.max(field_name).as_("MAX")).collect()
-        if len(res) != 1:
-            raise snowml_exceptions.SnowflakeMLException(
-                error_code=error_codes.INTERNAL_SNOWML_ERROR,
-                original_exception=ValueError(f"Unable to get the value range of field {field_name}"),
-            )
-        return res[0].MIN, res[0].MAX
-
     field_data_type = field.datatype
     col_name = identifier.get_unescaped_names(field.name)
 
@@ -465,16 +501,27 @@ def _validate_snowpark_type_feature(
                 error_code=error_codes.INVALID_DATA,
                 original_exception=ValueError(
                     f"Data Validation Error in feature {ft_name}: "
-                    + f"Feature type {ft_type} is not met by column {col_name}."
+                    f"Feature type {ft_type} is not met by column {col_name} "
+                    f"because of its original type {field_data_type}"
+                ),
+            )
+        if value_range is None:
+            raise snowml_exceptions.SnowflakeMLException(
+                error_code=error_codes.INVALID_DATA,
+                original_exception=ValueError(
+                    f"Data Validation Error in feature {ft_name}: "
+                    f"Feature type {ft_type} is not met by column {col_name} "
+                    f"because of its original type {field_data_type} is non-Numeric."
                 ),
             )
-        min_v, max_v = get_value_range(field.name)
+        min_v, max_v = value_range
         if max_v > np.iinfo(ft_type._numpy_type).max or min_v < np.iinfo(ft_type._numpy_type).min:
             raise snowml_exceptions.SnowflakeMLException(
                 error_code=error_codes.INVALID_DATA,
                 original_exception=ValueError(
                     f"Data Validation Error in feature {ft_name}: "
-                    + f"Feature type {ft_type} is not met by column {col_name}."
+                    f"Feature type {ft_type} is not met by column {col_name} "
+                    f"because it overflows with min"
                 ),
             )
     elif ft_type in [core.DataType.FLOAT, core.DataType.DOUBLE]:
@@ -494,7 +541,16 @@ def _validate_snowpark_type_feature(
                     + f"Feature type {ft_type} is not met by column {col_name}."
                 ),
             )
-        min_v, max_v = get_value_range(field.name)
+        if value_range is None:
+            raise snowml_exceptions.SnowflakeMLException(
+                error_code=error_codes.INVALID_DATA,
+                original_exception=ValueError(
+                    f"Data Validation Error in feature {ft_name}: "
+                    f"Feature type {ft_type} is not met by column {col_name} "
+                    f"because of its original type {field_data_type} is non-Numeric."
+                ),
+            )
+        min_v, max_v = value_range
         if (
             max_v > np.finfo(ft_type._numpy_type).max  # type: ignore[arg-type]
             or min_v < np.finfo(ft_type._numpy_type).min  # type: ignore[arg-type]

snowflake/ml/model/type_hints.py

@@ -3,6 +3,7 @@ from typing import (
     TYPE_CHECKING,
     Any,
     Dict,
+    List,
     Literal,
     Optional,
     Sequence,
@@ -173,6 +174,13 @@ class SnowparkContainerServiceDeployOptions(DeployOptions):
     debug_mode: When set to True, deployment artifacts will be persisted in a local temp directory.
     enable_ingress: When set to True, will expose HTTP endpoint for access to the predict method of the created
         service.
+    external_access_integrations: External Access Integrations name used to build image and deploy the model.
+        Please refer to the doc for how to create an External Access Integrations: https://docs.snowflake.com/
+        developer-guide/snowpark-container-services/additional-considerations-services-jobs
+        #configuring-network-capabilities .
+        To make sure your image could be built, access to the following endpoint must be allowed.
+        docker.com:80, docker.com:443, anaconda.com:80, anaconda.com:443, anaconda.org:80, anaconda.org:443,
+        pypi.org:80, pypi.org:443
     """
 
     compute_pool: str
@@ -187,6 +195,7 @@ class SnowparkContainerServiceDeployOptions(DeployOptions):
     model_in_image: NotRequired[bool]
     debug_mode: NotRequired[bool]
     enable_ingress: NotRequired[bool]
+    external_access_integrations: List[str]
 
 
 class ModelMethodSaveOptions(TypedDict):
@@ -198,9 +207,12 @@ class BaseModelSaveOption(TypedDict):
     """Options for saving the model.
 
     embed_local_ml_library: Embedding local SnowML into the code directory of the folder.
+    relax_version: Whether or not relax the version constraints of the dependencies if unresolvable. It detects any
+        ==x.y.z in specifiers and replaced with >=x.y, <(x+1). Defaults to False.
     """
 
     embed_local_ml_library: NotRequired[bool]
+    relax_version: NotRequired[bool]
     _legacy_save: NotRequired[bool]
     method_options: NotRequired[Dict[str, ModelMethodSaveOptions]]
 

snowflake/ml/modeling/_internal/estimator_protocols.py

@@ -6,47 +6,7 @@ from snowflake.snowpark import DataFrame, Session
 
 
 # TODO: Add more specific entities to type hint estimators instead of using `object`.
-class FitPredictHandlers(Protocol):
-    def batch_inference(
-        self,
-        dataset: DataFrame,
-        session: Session,
-        estimator: object,
-        dependencies: List[str],
-        inference_method: str,
-        input_cols: List[str],
-        pass_through_columns: List[str],
-        expected_output_cols_list: List[str],
-        expected_output_cols_type: str = "",
-    ) -> DataFrame:
-        raise NotImplementedError
-
-    def score_pandas(
-        self,
-        dataset: pd.DataFrame,
-        estimator: object,
-        input_cols: List[str],
-        label_cols: List[str],
-        sample_weight_col: Optional[str],
-    ) -> float:
-        raise NotImplementedError
-
-    def score_snowpark(
-        self,
-        dataset: DataFrame,
-        session: Session,
-        estimator: object,
-        dependencies: List[str],
-        score_sproc_imports: List[str],
-        input_cols: List[str],
-        label_cols: List[str],
-        sample_weight_col: Optional[str],
-    ) -> float:
-        raise NotImplementedError
-
-
-# TODO: Add more specific entities to type hint estimators instead of using `object`.
-class CVHandlers(Protocol):
+class TransformerHandlers(Protocol):
     def batch_inference(
         self,
         dataset: DataFrame,

snowflake/ml/modeling/_internal/model_trainer_builder.py

@@ -4,17 +4,21 @@ import pandas as pd
 from sklearn import model_selection
 
 from snowflake.ml._internal.exceptions import error_codes, exceptions
-from snowflake.ml.modeling._internal.distributed_hpo_trainer import (
-    DistributedHPOTrainer,
-)
 from snowflake.ml.modeling._internal.estimator_utils import (
     get_module_name,
     is_single_node,
 )
+from snowflake.ml.modeling._internal.local_implementations.pandas_trainer import (
+    PandasModelTrainer,
+)
 from snowflake.ml.modeling._internal.model_trainer import ModelTrainer
-from snowflake.ml.modeling._internal.pandas_trainer import PandasModelTrainer
-from snowflake.ml.modeling._internal.snowpark_trainer import SnowparkModelTrainer
-from snowflake.ml.modeling._internal.xgboost_external_memory_trainer import (
+from snowflake.ml.modeling._internal.snowpark_implementations.distributed_hpo_trainer import (
+    DistributedHPOTrainer,
+)
+from snowflake.ml.modeling._internal.snowpark_implementations.snowpark_trainer import (
+    SnowparkModelTrainer,
+)
+from snowflake.ml.modeling._internal.snowpark_implementations.xgboost_external_memory_trainer import (
     XGBoostExternalMemoryTrainer,
 )
 from snowflake.snowpark import DataFrame, Session
@@ -76,9 +80,9 @@ class ModelTrainerBuilder:
         batch_size: int = -1,
     ) -> ModelTrainer:
         """
-        Builder method that creates an approproiate ModelTrainer instance based on the given params.
+        Builder method that creates an appropriate ModelTrainer instance based on the given params.
         """
-        assert input_cols is not None  # Make MyPy happpy
+        assert input_cols is not None  # Make MyPy happy
         if isinstance(dataset, pd.DataFrame):
             return PandasModelTrainer(
                 estimator=estimator,
@@ -100,7 +104,7 @@ class ModelTrainerBuilder:
                 "subproject": subproject,
             }
 
-            assert dataset._session is not None  # Make MyPy happpy
+            assert dataset._session is not None  # Make MyPy happy
             if isinstance(estimator, model_selection.GridSearchCV) or isinstance(
                 estimator, model_selection.RandomizedSearchCV
             ):