snowflake-ml-python 1.9.2__py3-none-any.whl → 1.11.0__py3-none-any.whl

snowflake/ml/registry/_manager/model_manager.py

@@ -4,16 +4,16 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 import pandas as pd
 from absl.logging import logging
 
-from snowflake.ml._internal import env, platform_capabilities, telemetry
+from snowflake.ml._internal import platform_capabilities, telemetry
 from snowflake.ml._internal.exceptions import error_codes, exceptions
 from snowflake.ml._internal.human_readable_id import hrid_generator
 from snowflake.ml._internal.utils import sql_identifier
-from snowflake.ml.model import model_signature, target_platform, task, type_hints
+from snowflake.ml.model import model_signature, task, type_hints
 from snowflake.ml.model._client.model import model_impl, model_version_impl
 from snowflake.ml.model._client.ops import metadata_ops, model_ops, service_ops
 from snowflake.ml.model._model_composer import model_composer
-from snowflake.ml.model._model_composer.model_manifest import model_manifest_schema
 from snowflake.ml.model._packager.model_meta import model_meta
+from snowflake.ml.registry._manager import model_parameter_reconciler
 from snowflake.snowpark import exceptions as snowpark_exceptions, session
 from snowflake.snowpark._internal import utils as snowpark_utils
 
@@ -46,6 +46,7 @@ class ModelManager:
         *,
         model: Union[type_hints.SupportedModelType, model_version_impl.ModelVersion],
         model_name: str,
+        progress_status: type_hints.ProgressStatus,
         version_name: Optional[str] = None,
         comment: Optional[str] = None,
         metrics: Optional[dict[str, Any]] = None,
@@ -64,7 +65,6 @@ class ModelManager:
         experiment_info: Optional["ExperimentInfo"] = None,
         options: Optional[type_hints.ModelSaveOption] = None,
         statement_params: Optional[dict[str, Any]] = None,
-        progress_status: Optional[Any] = None,
     ) -> model_version_impl.ModelVersion:
 
         database_name_id, schema_name_id, model_name_id = self._parse_fully_qualified_name(model_name)
@@ -158,6 +158,7 @@ class ModelManager:
         *,
         model_name: str,
         version_name: str,
+        progress_status: type_hints.ProgressStatus,
         comment: Optional[str] = None,
         metrics: Optional[dict[str, Any]] = None,
         conda_dependencies: Optional[list[str]] = None,
@@ -175,7 +176,6 @@ class ModelManager:
         experiment_info: Optional["ExperimentInfo"] = None,
         options: Optional[type_hints.ModelSaveOption] = None,
         statement_params: Optional[dict[str, Any]] = None,
-        progress_status: Optional[Any] = None,
     ) -> model_version_impl.ModelVersion:
         database_name_id, schema_name_id, model_name_id = sql_identifier.parse_fully_qualified_name(model_name)
         version_name_id = sql_identifier.SqlIdentifier(version_name)
@@ -220,57 +220,30 @@ class ModelManager:
                 statement_params=statement_params,
             )
 
-        platforms = None
-        # User specified target platforms are defaulted to None and will not show up in the generated manifest.
-        if target_platforms:
-            # Convert any string target platforms to TargetPlatform objects
-            platforms = [type_hints.TargetPlatform(platform) for platform in target_platforms]
-        else:
-            # Default the target platform to warehouse if not specified and any table function exists
-            if options and (
-                options.get("function_type") == model_manifest_schema.ModelMethodFunctionTypes.TABLE_FUNCTION.value
-                or (
-                    any(
-                        opt.get("function_type") == "TABLE_FUNCTION"
-                        for opt in options.get("method_options", {}).values()
-                    )
-                )
-            ):
-                logger.info(
-                    "Logging a partitioned model with a table function without specifying `target_platforms`. "
-                    'Default to `target_platforms=["WAREHOUSE"]`.'
-                )
-                platforms = [target_platform.TargetPlatform.WAREHOUSE]
+        reconciler = model_parameter_reconciler.ModelParameterReconciler(
+            session=self._model_ops._session,
+            database_name=self._database_name,
+            schema_name=self._schema_name,
+            conda_dependencies=conda_dependencies,
+            pip_requirements=pip_requirements,
+            target_platforms=target_platforms,
+            artifact_repository_map=artifact_repository_map,
+            options=options,
+            python_version=python_version,
+            statement_params=statement_params,
+        )
 
-            # Default the target platform to SPCS if not specified when running in ML runtime
-            if not platforms and env.IN_ML_RUNTIME:
-                logger.info(
-                    "Logging the model on Container Runtime for ML without specifying `target_platforms`. "
-                    'Default to `target_platforms=["SNOWPARK_CONTAINER_SERVICES"]`.'
-                )
-                platforms = [target_platform.TargetPlatform.SNOWPARK_CONTAINER_SERVICES]
-
-        if artifact_repository_map:
-            for channel, artifact_repository_name in artifact_repository_map.items():
-                db_id, schema_id, repo_id = sql_identifier.parse_fully_qualified_name(artifact_repository_name)
-
-                artifact_repository_map[channel] = sql_identifier.get_fully_qualified_name(
-                    db_id,
-                    schema_id,
-                    repo_id,
-                    self._database_name,
-                    self._schema_name,
-                )
+        model_params = reconciler.reconcile()
+
+        # Use reconciled parameters
+        artifact_repository_map = model_params.artifact_repository_map
+        save_location = model_params.save_location
 
         logger.info("Start packaging and uploading your model. It might take some time based on the size of the model.")
-        if progress_status:
-            progress_status.update("packaging model...")
-            progress_status.increment()
-
-        # Extract save_location from options if present
-        save_location = None
-        if options and "save_location" in options:
-            save_location = options.get("save_location")
+        progress_status.update("packaging model...")
+        progress_status.increment()
+
+        if save_location:
             logger.info(f"Model will be saved to local directory: {save_location}")
 
         mc = model_composer.ModelComposer(
@@ -280,9 +253,8 @@ class ModelManager:
             save_location=save_location,
         )
 
-        if progress_status:
-            progress_status.update("creating model manifest...")
-            progress_status.increment()
+        progress_status.update("creating model manifest...")
+        progress_status.increment()
 
         model_metadata: model_meta.ModelMetadata = mc.save(
             name=model_name_id.resolved(),
@@ -293,19 +265,18 @@ class ModelManager:
             pip_requirements=pip_requirements,
             artifact_repository_map=artifact_repository_map,
             resource_constraint=resource_constraint,
-            target_platforms=platforms,
+            target_platforms=model_params.target_platforms,
             python_version=python_version,
             user_files=user_files,
             code_paths=code_paths,
             ext_modules=ext_modules,
-            options=options,
+            options=model_params.options,
             task=task,
             experiment_info=experiment_info,
         )
 
-        if progress_status:
-            progress_status.update("uploading model files...")
-            progress_status.increment()
+        progress_status.update("uploading model files...")
+        progress_status.increment()
         statement_params = telemetry.add_statement_params_custom_tags(
             statement_params, model_metadata.telemetry_metadata()
         )
@@ -313,10 +284,8 @@ class ModelManager:
             statement_params, {"model_version_name": version_name_id}
         )
 
-        logger.info("Start creating MODEL object for you in the Snowflake.")
-        if progress_status:
-            progress_status.update("creating model object in Snowflake...")
-            progress_status.increment()
+        progress_status.update("creating model object in Snowflake...")
+        progress_status.increment()
 
         self._model_ops.create_from_stage(
             composed_model=mc,
@@ -343,9 +312,8 @@ class ModelManager:
             version_name=version_name_id,
         )
 
-        if progress_status:
-            progress_status.update("setting model metadata...")
-            progress_status.increment()
+        progress_status.update("setting model metadata...")
+        progress_status.increment()
 
         if comment:
             mv.comment = comment
@@ -360,8 +328,7 @@ class ModelManager:
                 statement_params=statement_params,
             )
 
-        if progress_status:
-            progress_status.update("model logged successfully!")
+        progress_status.update("model logged successfully!")
 
         return mv
 

snowflake/ml/registry/_manager/model_parameter_reconciler.py

@@ -0,0 +1,294 @@
+import warnings
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from absl.logging import logging
+from packaging import requirements
+
+from snowflake.ml import version as snowml_version
+from snowflake.ml._internal import env, env as snowml_env, env_utils
+from snowflake.ml._internal.exceptions import error_codes, exceptions
+from snowflake.ml._internal.utils import sql_identifier
+from snowflake.ml.model import target_platform, type_hints as model_types
+from snowflake.ml.model._model_composer.model_manifest import model_manifest_schema
+from snowflake.snowpark import Session
+from snowflake.snowpark._internal import utils as snowpark_utils
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ReconciledParameters:
+    """Holds the reconciled and validated parameters after processing."""
+
+    conda_dependencies: Optional[list[str]] = None
+    pip_requirements: Optional[list[str]] = None
+    target_platforms: Optional[list[model_types.TargetPlatform]] = None
+    artifact_repository_map: Optional[dict[str, str]] = None
+    options: Optional[model_types.ModelSaveOption] = None
+    save_location: Optional[str] = None
+
+
+class ModelParameterReconciler:
+    """Centralizes all complex log_model parameter validation, transformation, and reconciliation logic."""
+
+    def __init__(
+        self,
+        session: Session,
+        database_name: sql_identifier.SqlIdentifier,
+        schema_name: sql_identifier.SqlIdentifier,
+        conda_dependencies: Optional[list[str]] = None,
+        pip_requirements: Optional[list[str]] = None,
+        target_platforms: Optional[list[model_types.SupportedTargetPlatformType]] = None,
+        artifact_repository_map: Optional[dict[str, str]] = None,
+        options: Optional[model_types.ModelSaveOption] = None,
+        python_version: Optional[str] = None,
+        statement_params: Optional[dict[str, str]] = None,
+    ) -> None:
+        self._session = session
+        self._database_name = database_name
+        self._schema_name = schema_name
+        self._conda_dependencies = conda_dependencies
+        self._pip_requirements = pip_requirements
+        self._target_platforms = target_platforms
+        self._artifact_repository_map = artifact_repository_map
+        self._options = options
+        self._python_version = python_version
+        self._statement_params = statement_params
+
+    def reconcile(self) -> ReconciledParameters:
+        """Perform all parameter reconciliation and return clean parameters."""
+
+        reconciled_artifact_repository_map = self._reconcile_artifact_repository_map()
+        reconciled_save_location = self._extract_save_location()
+
+        self._validate_pip_requirements_warehouse_compatibility(reconciled_artifact_repository_map)
+
+        reconciled_target_platforms = self._reconcile_target_platforms()
+        reconciled_options = self._reconcile_explainability_options(reconciled_target_platforms)
+        reconciled_options = self._reconcile_relax_version(reconciled_options, reconciled_target_platforms)
+
+        return ReconciledParameters(
+            conda_dependencies=self._conda_dependencies,
+            pip_requirements=self._pip_requirements,
+            target_platforms=reconciled_target_platforms,
+            artifact_repository_map=reconciled_artifact_repository_map,
+            options=reconciled_options,
+            save_location=reconciled_save_location,
+        )
+
+    def _reconcile_artifact_repository_map(self) -> Optional[dict[str, str]]:
+        """Transform artifact_repository_map to use fully qualified names."""
+        if not self._artifact_repository_map:
+            return None
+
+        transformed_map = {}
+
+        for channel, artifact_repository_name in self._artifact_repository_map.items():
+            db_id, schema_id, repo_id = sql_identifier.parse_fully_qualified_name(artifact_repository_name)
+
+            transformed_map[channel] = sql_identifier.get_fully_qualified_name(
+                db_id,
+                schema_id,
+                repo_id,
+                self._database_name,
+                self._schema_name,
+            )
+
+        return transformed_map
+
+    def _extract_save_location(self) -> Optional[str]:
+        """Extract save_location from options."""
+        if self._options and "save_location" in self._options:
+            return self._options.get("save_location")
+
+        return None
+
+    def _reconcile_target_platforms(self) -> Optional[list[model_types.TargetPlatform]]:
+        """Reconcile target platforms with proper defaulting logic."""
+        # User specified target platforms are defaulted to None and will not show up in the generated manifest.
+        if self._target_platforms:
+            # Convert any string target platforms to TargetPlatform objects
+            return [model_types.TargetPlatform(platform) for platform in self._target_platforms]
+
+        # Default the target platform to warehouse if not specified and any table function exists
+        if self._has_table_function():
+            logger.info(
+                "Logging a partitioned model with a table function without specifying `target_platforms`. "
+                'Default to `target_platforms=["WAREHOUSE"]`.'
+            )
+            return [target_platform.TargetPlatform.WAREHOUSE]
+
+        # Default the target platform to SPCS if not specified when running in ML runtime
+        if env.IN_ML_RUNTIME:
+            logger.info(
+                "Logging the model on Container Runtime for ML without specifying `target_platforms`. "
+                'Default to `target_platforms=["SNOWPARK_CONTAINER_SERVICES"]`.'
+            )
+            return [target_platform.TargetPlatform.SNOWPARK_CONTAINER_SERVICES]
+
+        return None
+
+    def _has_table_function(self) -> bool:
+        """Check if any table function exists in options."""
+        if self._options is None:
+            return False
+
+        if self._options.get("function_type") == model_manifest_schema.ModelMethodFunctionTypes.TABLE_FUNCTION.value:
+            return True
+
+        for opt in self._options.get("method_options", {}).values():
+            if opt.get("function_type") == model_manifest_schema.ModelMethodFunctionTypes.TABLE_FUNCTION.value:
+                return True
+
+        return False
+
+    def _validate_pip_requirements_warehouse_compatibility(
+        self, artifact_repository_map: Optional[dict[str, str]]
+    ) -> None:
+        """Validate pip_requirements compatibility with warehouse deployment."""
+        if self._pip_requirements and not artifact_repository_map and self._targets_warehouse(self._target_platforms):
+            warnings.warn(
+                "Models logged specifying `pip_requirements` cannot be executed in a Snowflake Warehouse "
+                "without specifying `artifact_repository_map`. This model can be run in Snowpark Container "
+                "Services. See https://docs.snowflake.com/en/developer-guide/snowflake-ml/model-registry/container.",
+                category=UserWarning,
+                stacklevel=1,
+            )
+
+    @staticmethod
+    def _targets_warehouse(target_platforms: Optional[list[model_types.SupportedTargetPlatformType]]) -> bool:
+        """Returns True if warehouse is a target platform (None defaults to True)."""
+        return (
+            target_platforms is None
+            or model_types.TargetPlatform.WAREHOUSE in target_platforms
+            or "WAREHOUSE" in target_platforms
+        )
+
+    def _reconcile_explainability_options(
+        self, target_platforms: Optional[list[model_types.TargetPlatform]]
+    ) -> model_types.ModelSaveOption:
+        """Reconcile explainability settings and embed_local_ml_library based on warehouse runnability."""
+        options = self._options.copy() if self._options else model_types.BaseModelSaveOption()
+
+        conda_dep_dict = env_utils.validate_conda_dependency_string_list(self._conda_dependencies or [])
+
+        enable_explainability = options.get("enable_explainability", None)
+
+        # Handle case where user explicitly disabled explainability
+        if enable_explainability is False:
+            return self._handle_embed_local_ml_library(options, target_platforms)
+
+        target_platform_set = set(target_platforms) if target_platforms else set()
+
+        is_warehouse_runnable = self._is_warehouse_runnable(conda_dep_dict)
+        only_spcs = target_platform_set == set(target_platform.SNOWPARK_CONTAINER_SERVICES_ONLY)
+        has_both_platforms = target_platform_set == set(target_platform.BOTH_WAREHOUSE_AND_SNOWPARK_CONTAINER_SERVICES)
+
+        # Handle case where user explicitly requested explainability
+        if enable_explainability:
+            if only_spcs or not is_warehouse_runnable:
+                raise ValueError(
+                    "`enable_explainability` cannot be set to True when the model is not runnable in WH "
+                    "or the target platforms include SPCS."
+                )
+            elif has_both_platforms:
+                warnings.warn(
+                    ("Explain function will only be available for model deployed to warehouse."),
+                    category=UserWarning,
+                    stacklevel=2,
+                )
+
+        # Handle case where explainability is not specified (None) - set default behavior
+        if enable_explainability is None:
+            if only_spcs or not is_warehouse_runnable:
+                options["enable_explainability"] = False
+
+        return self._handle_embed_local_ml_library(options, target_platforms)
+
+    def _handle_embed_local_ml_library(
+        self, options: model_types.ModelSaveOption, target_platforms: Optional[list[model_types.TargetPlatform]]
+    ) -> model_types.ModelSaveOption:
+        """Handle embed_local_ml_library logic."""
+        if not snowpark_utils.is_in_stored_procedure() and target_platforms != [  # type: ignore[no-untyped-call]
+            model_types.TargetPlatform.SNOWPARK_CONTAINER_SERVICES  # no information schema check for SPCS-only models
+        ]:
+            snowml_matched_versions = env_utils.get_matched_package_versions_in_information_schema(
+                self._session,
+                reqs=[requirements.Requirement(f"{env_utils.SNOWPARK_ML_PKG_NAME}=={snowml_version.VERSION}")],
+                python_version=self._python_version or snowml_env.PYTHON_VERSION,
+                statement_params=self._statement_params,
+            ).get(env_utils.SNOWPARK_ML_PKG_NAME, [])
+
+            if len(snowml_matched_versions) < 1 and not options.get("embed_local_ml_library", False):
+                logging.info(
+                    f"Local snowflake-ml-python library has version {snowml_version.VERSION},"
+                    " which is not available in the Snowflake server, embedding local ML library automatically."
+                )
+                options["embed_local_ml_library"] = True
+
+        return options
+
+    def _is_warehouse_runnable(self, conda_dep_dict: dict[str, list[Any]]) -> bool:
+        """Check if model can run in warehouse based on conda channels and pip requirements."""
+        # If pip requirements are present but no artifact repository map, model cannot run in warehouse
+        if self._pip_requirements and not self._artifact_repository_map:
+            return False
+
+        # If no conda dependencies, model can run in warehouse
+        if not conda_dep_dict:
+            return True
+
+        # Check if all conda channels are warehouse-compatible
+        warehouse_compatible_channels = {env_utils.DEFAULT_CHANNEL_NAME, env_utils.SNOWFLAKE_CONDA_CHANNEL_URL}
+        for channel in conda_dep_dict:
+            if channel not in warehouse_compatible_channels:
+                return False
+
+        return True
+
+    def _reconcile_relax_version(
+        self,
+        options: model_types.ModelSaveOption,
+        target_platforms: Optional[list[model_types.TargetPlatform]],
+    ) -> model_types.ModelSaveOption:
+        """Reconcile relax_version setting based on pip requirements and target platforms."""
+        target_platform_set = set(target_platforms) if target_platforms else set()
+        has_pip_requirements = bool(self._pip_requirements)
+        only_spcs = target_platform_set == set(target_platform.SNOWPARK_CONTAINER_SERVICES_ONLY)
+
+        if "relax_version" not in options:
+            if has_pip_requirements or only_spcs:
+                logger.info(
+                    "Setting `relax_version=False` as this model will run in Snowpark Container Services "
+                    "or in Warehouse with a specified artifact_repository_map where exact version "
+                    " specifications will be honored."
+                )
+                relax_version = False
+            else:
+                warnings.warn(
+                    (
+                        "`relax_version` is not set and therefore defaulted to True. Dependency version constraints"
+                        " relaxed from ==x.y.z to >=x.y, <(x+1). To use specific dependency versions for compatibility,"
+                        " reproducibility, etc., set `options={'relax_version': False}` when logging the model."
+                    ),
+                    category=UserWarning,
+                    stacklevel=2,
+                )
+                relax_version = True
+            options["relax_version"] = relax_version
+            return options
+
+        # Handle case where relax_version is already set
+        relax_version = options["relax_version"]
+        if relax_version and (has_pip_requirements or only_spcs):
+            raise exceptions.SnowflakeMLException(
+                error_code=error_codes.INVALID_ARGUMENT,
+                original_exception=ValueError(
+                    "Setting `relax_version=True` is only allowed for models to be run in Warehouse with "
+                    "Snowflake Conda Channel dependencies. It cannot be used with pip requirements or when "
+                    "targeting only Snowpark Container Services."
+                ),
+            )
+
+        return options

snowflake/ml/registry/registry.py

@@ -1,4 +1,3 @@
-import warnings
 from types import ModuleType
 from typing import Any, Optional, Union, overload
 
@@ -442,15 +441,6 @@ class Registry:
             if task is not type_hints.Task.UNKNOWN:
                 raise ValueError("`task` cannot be specified when calling log_model with a ModelVersion.")
 
-        if pip_requirements and not artifact_repository_map and self._targets_warehouse(target_platforms):
-            warnings.warn(
-                "Models logged specifying `pip_requirements` cannot be executed in a Snowflake Warehouse "
-                "without specifying `artifact_repository_map`. This model can be run in Snowpark Container "
-                "Services. See https://docs.snowflake.com/en/developer-guide/snowflake-ml/model-registry/container.",
-                category=UserWarning,
-                stacklevel=1,
-            )
-
         registry_event_handler = event_handler.ModelEventHandler()
         with registry_event_handler.status("Logging model", total=6) as status:
             # Step 1: Validation and setup
@@ -662,12 +652,3 @@ class Registry:
         if not self.enable_monitoring:
             raise ValueError(_MODEL_MONITORING_DISABLED_ERROR)
         self._model_monitor_manager.delete_monitor(name)
-
-    @staticmethod
-    def _targets_warehouse(target_platforms: Optional[list[type_hints.SupportedTargetPlatformType]]) -> bool:
-        """Returns True if warehouse is a target platform (None defaults to True)."""
-        return (
-            target_platforms is None
-            or type_hints.TargetPlatform.WAREHOUSE in target_platforms
-            or "WAREHOUSE" in target_platforms
-        )

snowflake/ml/version.py

@@ -1,2 +1,2 @@
 # This is parsed by regex in conda recipe meta file. Make sure not to break it.
-VERSION = "1.9.2"
+VERSION = "1.11.0"