snowflake-ml-python 1.7.0__py3-none-any.whl → 1.7.2__py3-none-any.whl

snowflake/ml/monitoring/_manager/model_monitor_manager.py

@@ -1,66 +1,18 @@
+import json
 from typing import Any, Dict, List, Optional
 
 from snowflake import snowpark
-from snowflake.ml._internal import telemetry
-from snowflake.ml._internal.utils import db_utils, sql_identifier
+from snowflake.ml._internal.utils import sql_identifier
 from snowflake.ml.model import type_hints
 from snowflake.ml.model._client.model import model_version_impl
-from snowflake.ml.model._model_composer.model_manifest import model_manifest_schema
 from snowflake.ml.monitoring import model_monitor
 from snowflake.ml.monitoring._client import model_monitor_sql_client
-from snowflake.ml.monitoring.entities import (
-    model_monitor_config,
-    model_monitor_interval,
-)
+from snowflake.ml.monitoring.entities import model_monitor_config
 from snowflake.snowpark import session
 
 
-def _validate_name_constraints(model_version: model_version_impl.ModelVersion) -> None:
-    system_table_prefixes = [
-        model_monitor_sql_client._SNOWML_MONITORING_TABLE_NAME_PREFIX,
-        model_monitor_sql_client._SNOWML_MONITORING_ACCURACY_TABLE_NAME_PREFIX,
-    ]
-
-    max_allowed_model_name_and_version_length = (
-        db_utils.MAX_IDENTIFIER_LENGTH - max(len(prefix) for prefix in system_table_prefixes) - 1
-    )  # -1 includes '_' between model_name + model_version
-    if len(model_version.model_name) + len(model_version.version_name) > max_allowed_model_name_and_version_length:
-        error_msg = f"Model name and version name exceeds maximum length of {max_allowed_model_name_and_version_length}"
-        raise ValueError(error_msg)
-
-
 class ModelMonitorManager:
-    """Class to manage internal operations for Model Monitor workflows."""  # TODO: Move to Registry.
-
-    @staticmethod
-    def setup(session: session.Session, database_name: str, schema_name: str) -> None:
-        """Static method to set up schema for Model Monitoring resources.
-
-        Args:
-            session: The Snowpark Session to connect with Snowflake.
-            database_name: The name of the database. If None, the current database of the session
-                will be used. Defaults to None.
-            schema_name: The name of the schema. If None, the current schema of the session
-                will be used. If there is no active schema, the PUBLIC schema will be used. Defaults to None.
-        """
-        statement_params = telemetry.get_statement_params(
-            project=telemetry.TelemetryProject.MLOPS.value,
-            subproject=telemetry.TelemetrySubProject.MONITORING.value,
-        )
-        database_name_id = sql_identifier.SqlIdentifier(database_name)
-        schema_name_id = sql_identifier.SqlIdentifier(schema_name)
-        model_monitor_sql_client.ModelMonitorSQLClient.initialize_monitoring_schema(
-            session, database_name_id, schema_name_id, statement_params=statement_params
-        )
-
-    def _fetch_task_from_model_version(
-        self,
-        model_version: model_version_impl.ModelVersion,
-    ) -> type_hints.Task:
-        task = model_version.get_model_task()
-        if task == type_hints.Task.UNKNOWN:
-            raise ValueError("Registry model must be logged with task in order to be monitored.")
-        return task
+    """Class to manage internal operations for Model Monitor workflows."""
 
     def __init__(
         self,
@@ -68,7 +20,6 @@ class ModelMonitorManager:
         database_name: sql_identifier.SqlIdentifier,
         schema_name: sql_identifier.SqlIdentifier,
         *,
-        create_if_not_exists: bool = False,
         statement_params: Optional[Dict[str, Any]] = None,
     ) -> None:
         """
@@ -79,233 +30,165 @@ class ModelMonitorManager:
             session: The Snowpark Session to connect with Snowflake.
             database_name: The name of the database.
             schema_name: The name of the schema.
-            create_if_not_exists: Flag whether to initialize resources in the schema needed for Model Monitoring.
             statement_params: Optional set of statement params.
-
-        Raises:
-            ValueError: When there is no specified or active database in the session.
         """
         self._database_name = database_name
         self._schema_name = schema_name
         self.statement_params = statement_params
+
         self._model_monitor_client = model_monitor_sql_client.ModelMonitorSQLClient(
             session,
             database_name=self._database_name,
             schema_name=self._schema_name,
         )
-        if create_if_not_exists:
-            model_monitor_sql_client.ModelMonitorSQLClient.initialize_monitoring_schema(
-                session, self._database_name, self._schema_name, self.statement_params
-            )
-        elif not self._model_monitor_client._validate_is_initialized():
-            raise ValueError(
-                "Monitoring has not been setup. Set create_if_not_exists or call ModelMonitorManager.setup"
-            )
 
-    def _get_and_validate_model_function_from_model_version(
+    def _validate_task_from_model_version(
+        self,
+        model_version: model_version_impl.ModelVersion,
+    ) -> type_hints.Task:
+        task = model_version.get_model_task()
+        if task == type_hints.Task.UNKNOWN:
+            raise ValueError("Registry model must be logged with task in order to be monitored.")
+        return task
+
+    def _validate_model_function_from_model_version(
         self, function: str, model_version: model_version_impl.ModelVersion
-    ) -> model_manifest_schema.ModelFunctionInfo:
+    ) -> None:
         functions = model_version.show_functions()
         for f in functions:
             if f["target_method"] == function:
-                return f
+                return
         existing_target_methods = {f["target_method"] for f in functions}
         raise ValueError(
             f"Function with name {function} does not exist in the given model version. "
             f"Found: {existing_target_methods}."
         )
 
-    def _validate_monitor_config_or_raise(
-        self,
-        table_config: model_monitor_config.ModelMonitorTableConfig,
-        model_monitor_config: model_monitor_config.ModelMonitorConfig,
-    ) -> None:
-        """Validate provided config for model monitor.
-
-        Args:
-            table_config: Config for model monitor tables.
-            model_monitor_config: Config for ModelMonitor.
-
-        Raises:
-            ValueError: If warehouse provided does not exist.
-        """
-
-        # Validate naming will not exceed 255 chars
-        _validate_name_constraints(model_monitor_config.model_version)
-
-        if len(table_config.prediction_columns) != len(table_config.label_columns):
-            raise ValueError("Prediction and Label column names must be of the same length.")
-        # output and ground cols are list to keep interface extensible.
-        # for prpr only one label and one output col will be supported
-        if len(table_config.prediction_columns) != 1 or len(table_config.label_columns) != 1:
-            raise ValueError("Multiple Output columns are not supported in monitoring")
-
-        # Validate warehouse exists.
-        warehouse_name_id = sql_identifier.SqlIdentifier(model_monitor_config.background_compute_warehouse_name)
-        self._model_monitor_client.validate_monitor_warehouse(warehouse_name_id, statement_params=self.statement_params)
-
-        # Validate refresh interval.
-        try:
-            num_units, time_units = model_monitor_config.refresh_interval.strip().split(" ")
-            int(num_units)  # try to cast
-            if time_units.lower() not in {"seconds", "minutes", "hours", "days"}:
-                raise ValueError(
-                    """Invalid time unit in refresh interval. Provide '<num> <seconds | minutes | hours | days>'.
-See https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#required-parameters for more info."""
-                )
-        except Exception as e:  # TODO: Link to DT page.
-            raise ValueError(
-                f"""Failed to parse refresh interval with exception {e}.
-                Provide '<num> <seconds | minutes | hours | days>'.
-See https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#required-parameters for more info."""
-            )
+    def _build_column_list_from_input(self, columns: Optional[List[str]]) -> List[sql_identifier.SqlIdentifier]:
+        return [sql_identifier.SqlIdentifier(column_name) for column_name in columns] if columns else []
 
     def add_monitor(
         self,
         name: str,
-        table_config: model_monitor_config.ModelMonitorTableConfig,
+        source_config: model_monitor_config.ModelMonitorSourceConfig,
         model_monitor_config: model_monitor_config.ModelMonitorConfig,
-        *,
-        add_dashboard_udtfs: bool = False,
     ) -> model_monitor.ModelMonitor:
         """Add a new Model Monitor.
 
         Args:
             name: Name of Model Monitor to create.
-            table_config: Configuration options for the source table used in ModelMonitor.
+            source_config: Configuration options for the source table used in ModelMonitor.
             model_monitor_config: Configuration options of ModelMonitor.
-            add_dashboard_udtfs: Add UDTFs useful for creating a dashboard.
 
         Returns:
             The newly added ModelMonitor object.
         """
-        # Validates configuration or raise.
-        self._validate_monitor_config_or_raise(table_config, model_monitor_config)
-        model_function = self._get_and_validate_model_function_from_model_version(
+        warehouse_name_id = sql_identifier.SqlIdentifier(model_monitor_config.background_compute_warehouse_name)
+        self._model_monitor_client.validate_monitor_warehouse(warehouse_name_id, statement_params=self.statement_params)
+        self._validate_model_function_from_model_version(
             model_monitor_config.model_function_name, model_monitor_config.model_version
         )
-        monitor_refresh_interval = model_monitor_interval.ModelMonitorRefreshInterval(
-            model_monitor_config.refresh_interval
+        self._validate_task_from_model_version(model_monitor_config.model_version)
+        monitor_database_name_id, monitor_schema_name_id, monitor_name_id = sql_identifier.parse_fully_qualified_name(
+            name
         )
-        name_id = sql_identifier.SqlIdentifier(name)
-        source_table_name_id = sql_identifier.SqlIdentifier(table_config.source_table)
-        prediction_columns = [
-            sql_identifier.SqlIdentifier(column_name) for column_name in table_config.prediction_columns
-        ]
-        label_columns = [sql_identifier.SqlIdentifier(column_name) for column_name in table_config.label_columns]
-        id_columns = [sql_identifier.SqlIdentifier(column_name) for column_name in table_config.id_columns]
-        ts_column = sql_identifier.SqlIdentifier(table_config.timestamp_column)
+        source_database_name_id, source_schema_name_id, source_name_id = sql_identifier.parse_fully_qualified_name(
+            source_config.source
+        )
+        baseline_database_name_id, baseline_schema_name_id, baseline_name_id = (
+            sql_identifier.parse_fully_qualified_name(source_config.baseline)
+            if source_config.baseline
+            else (None, None, None)
+        )
+        model_database_name_id, model_schema_name_id, model_name_id = sql_identifier.parse_fully_qualified_name(
+            model_monitor_config.model_version.fully_qualified_model_name
+        )
+
+        prediction_score_columns = self._build_column_list_from_input(source_config.prediction_score_columns)
+        prediction_class_columns = self._build_column_list_from_input(source_config.prediction_class_columns)
+        actual_score_columns = self._build_column_list_from_input(source_config.actual_score_columns)
+        actual_class_columns = self._build_column_list_from_input(source_config.actual_class_columns)
+
+        id_columns = [sql_identifier.SqlIdentifier(column_name) for column_name in source_config.id_columns]
+        ts_column = sql_identifier.SqlIdentifier(source_config.timestamp_column)
 
         # Validate source table
-        self._model_monitor_client.validate_source_table(
-            source_table_name=source_table_name_id,
+        self._model_monitor_client.validate_source(
+            source_database=source_database_name_id,
+            source_schema=source_schema_name_id,
+            source=source_name_id,
             timestamp_column=ts_column,
-            prediction_columns=prediction_columns,
-            label_columns=label_columns,
+            prediction_score_columns=prediction_score_columns,
+            prediction_class_columns=prediction_class_columns,
+            actual_score_columns=actual_score_columns,
+            actual_class_columns=actual_class_columns,
             id_columns=id_columns,
-            model_function=model_function,
         )
 
-        task = self._fetch_task_from_model_version(model_version=model_monitor_config.model_version)
-        score_type = self._model_monitor_client.get_score_type(task, source_table_name_id, prediction_columns)
-
-        # Insert monitoring metadata for new model version.
-        self._model_monitor_client.create_monitor_on_model_version(
-            monitor_name=name_id,
-            source_table_name=source_table_name_id,
-            fully_qualified_model_name=model_monitor_config.model_version.fully_qualified_model_name,
+        self._model_monitor_client.create_model_monitor(
+            monitor_database=monitor_database_name_id,
+            monitor_schema=monitor_schema_name_id,
+            monitor_name=monitor_name_id,
+            source_database=source_database_name_id,
+            source_schema=source_schema_name_id,
+            source=source_name_id,
+            model_database=model_database_name_id,
+            model_schema=model_schema_name_id,
+            model_name=model_name_id,
             version_name=sql_identifier.SqlIdentifier(model_monitor_config.model_version.version_name),
             function_name=model_monitor_config.model_function_name,
+            warehouse_name=warehouse_name_id,
             timestamp_column=ts_column,
-            prediction_columns=prediction_columns,
-            label_columns=label_columns,
             id_columns=id_columns,
-            task=task,
-            statement_params=self.statement_params,
-        )
-
-        # Create Dynamic tables for model monitor.
-        self._model_monitor_client.create_dynamic_tables_for_monitor(
-            model_name=sql_identifier.SqlIdentifier(model_monitor_config.model_version.model_name),
-            model_version_name=sql_identifier.SqlIdentifier(model_monitor_config.model_version.version_name),
-            task=task,
-            source_table_name=source_table_name_id,
-            refresh_interval=monitor_refresh_interval,
+            prediction_score_columns=prediction_score_columns,
+            prediction_class_columns=prediction_class_columns,
+            actual_score_columns=actual_score_columns,
+            actual_class_columns=actual_class_columns,
+            refresh_interval=model_monitor_config.refresh_interval,
             aggregation_window=model_monitor_config.aggregation_window,
-            warehouse_name=sql_identifier.SqlIdentifier(model_monitor_config.background_compute_warehouse_name),
-            timestamp_column=sql_identifier.SqlIdentifier(table_config.timestamp_column),
-            id_columns=id_columns,
-            prediction_columns=prediction_columns,
-            label_columns=label_columns,
-            score_type=score_type,
-        )
-
-        # Initialize baseline table.
-        self._model_monitor_client.initialize_baseline_table(
-            model_name=sql_identifier.SqlIdentifier(model_monitor_config.model_version.model_name),
-            version_name=sql_identifier.SqlIdentifier(model_monitor_config.model_version.version_name),
-            source_table_name=table_config.source_table,
-            columns_to_drop=[ts_column, *id_columns],
+            baseline_database=baseline_database_name_id,
+            baseline_schema=baseline_schema_name_id,
+            baseline=baseline_name_id,
             statement_params=self.statement_params,
         )
-
-        # Add udtfs helpful for dashboard queries.
-        # TODO(apgupta) Make this true by default.
-        if add_dashboard_udtfs:
-            self._model_monitor_client.add_dashboard_udtfs(
-                name_id,
-                model_name=sql_identifier.SqlIdentifier(model_monitor_config.model_version.model_name),
-                model_version_name=sql_identifier.SqlIdentifier(model_monitor_config.model_version.version_name),
-                task=task,
-                score_type=score_type,
-                output_columns=prediction_columns,
-                ground_truth_columns=label_columns,
-            )
-
         return model_monitor.ModelMonitor._ref(
             model_monitor_client=self._model_monitor_client,
-            name=name_id,
-            fully_qualified_model_name=model_monitor_config.model_version.fully_qualified_model_name,
-            version_name=sql_identifier.SqlIdentifier(model_monitor_config.model_version.version_name),
-            function_name=sql_identifier.SqlIdentifier(model_monitor_config.model_function_name),
-            prediction_columns=prediction_columns,
-            label_columns=label_columns,
+            name=monitor_name_id,
         )
 
     def get_monitor_by_model_version(
         self, model_version: model_version_impl.ModelVersion
     ) -> model_monitor.ModelMonitor:
-        fq_model_name = model_version.fully_qualified_model_name
-        version_name = sql_identifier.SqlIdentifier(model_version.version_name)
-        if self._model_monitor_client.validate_existence(fq_model_name, version_name, self.statement_params):
-            model_db, model_schema, model_name = sql_identifier.parse_fully_qualified_name(fq_model_name)
-            if model_db is None or model_schema is None:
-                raise ValueError("Failed to parse model name")
-
-            model_monitor_params: model_monitor_sql_client._ModelMonitorParams = (
-                self._model_monitor_client.get_model_monitor_by_model_version(
-                    model_db=model_db,
-                    model_schema=model_schema,
-                    model_name=model_name,
-                    version_name=version_name,
-                    statement_params=self.statement_params,
-                )
-            )
-            return model_monitor.ModelMonitor._ref(
-                model_monitor_client=self._model_monitor_client,
-                name=sql_identifier.SqlIdentifier(model_monitor_params["monitor_name"]),
-                fully_qualified_model_name=fq_model_name,
-                version_name=version_name,
-                function_name=sql_identifier.SqlIdentifier(model_monitor_params["function_name"]),
-                prediction_columns=model_monitor_params["prediction_columns"],
-                label_columns=model_monitor_params["label_columns"],
-            )
+        """Get a Model Monitor by Model Version.
 
-        else:
-            raise ValueError(
-                f"ModelMonitor not found for model version {model_version.model_name} - {model_version.version_name}"
+        Args:
+            model_version: ModelVersion to retrieve Model Monitor for.
+
+        Returns:
+            The fetched ModelMonitor.
+
+        Raises:
+            ValueError: If model monitor is not found.
+        """
+        rows = self._model_monitor_client.show_model_monitors(statement_params=self.statement_params)
+
+        def model_match_fn(model_details: Dict[str, str]) -> bool:
+            return (
+                model_details[model_monitor_sql_client.MODEL_JSON_MODEL_NAME_FIELD] == model_version.model_name
+                and model_details[model_monitor_sql_client.MODEL_JSON_VERSION_NAME_FIELD] == model_version.version_name
             )
 
+        rows = [row for row in rows if model_match_fn(json.loads(row[model_monitor_sql_client.MODEL_JSON_COL_NAME]))]
+        if len(rows) == 0:
+            raise ValueError("Unable to find model monitor for the given model version.")
+        if len(rows) > 1:
+            raise ValueError("Found multiple model monitors for the given model version.")
+
+        return model_monitor.ModelMonitor._ref(
+            model_monitor_client=self._model_monitor_client,
+            name=sql_identifier.SqlIdentifier(rows[0]["name"]),
+        )
+
     def get_monitor(self, name: str) -> model_monitor.ModelMonitor:
         """Get a Model Monitor from the Registry
 
@@ -318,25 +201,18 @@ See https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#require
         Returns:
             The fetched ModelMonitor.
         """
-        name_id = sql_identifier.SqlIdentifier(name)
+        database_name_id, schema_name_id, monitor_name_id = sql_identifier.parse_fully_qualified_name(name)
 
         if not self._model_monitor_client.validate_existence_by_name(
-            monitor_name=name_id,
+            database_name=database_name_id,
+            schema_name=schema_name_id,
+            monitor_name=monitor_name_id,
             statement_params=self.statement_params,
         ):
             raise ValueError(f"Unable to find model monitor '{name}'")
-        model_monitor_params: model_monitor_sql_client._ModelMonitorParams = (
-            self._model_monitor_client.get_model_monitor_by_name(name_id, statement_params=self.statement_params)
-        )
-
         return model_monitor.ModelMonitor._ref(
             model_monitor_client=self._model_monitor_client,
-            name=name_id,
-            fully_qualified_model_name=model_monitor_params["fully_qualified_model_name"],
-            version_name=sql_identifier.SqlIdentifier(model_monitor_params["version_name"]),
-            function_name=sql_identifier.SqlIdentifier(model_monitor_params["function_name"]),
-            prediction_columns=model_monitor_params["prediction_columns"],
-            label_columns=model_monitor_params["label_columns"],
+            name=monitor_name_id,
         )
 
     def show_model_monitors(self) -> List[snowpark.Row]:
@@ -345,7 +221,7 @@ See https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#require
         Returns:
             List of snowpark.Row containing metadata for each model monitor.
         """
-        return self._model_monitor_client.get_all_model_monitor_metadata()
+        return self._model_monitor_client.show_model_monitors(statement_params=self.statement_params)
 
     def delete_monitor(self, name: str) -> None:
         """Delete a Model Monitor from the Registry
@@ -353,10 +229,10 @@ See https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#require
         Args:
             name: Name of the Model Monitor to delete.
         """
-        name_id = sql_identifier.SqlIdentifier(name)
-        monitor_params = self._model_monitor_client.get_model_monitor_by_name(name_id)
-        _, _, model = sql_identifier.parse_fully_qualified_name(monitor_params["fully_qualified_model_name"])
-        version = sql_identifier.SqlIdentifier(monitor_params["version_name"])
-        self._model_monitor_client.delete_monitor_metadata(name_id)
-        self._model_monitor_client.delete_baseline_table(model, version)
-        self._model_monitor_client.delete_dynamic_tables(model, version)
+        database_name_id, schema_name_id, monitor_name_id = sql_identifier.parse_fully_qualified_name(name)
+        self._model_monitor_client.drop_model_monitor(
+            database_name=database_name_id,
+            schema_name=schema_name_id,
+            monitor_name=monitor_name_id,
+            statement_params=self.statement_params,
+        )

snowflake/ml/monitoring/entities/model_monitor_config.py

@@ -1,28 +1,54 @@
 from dataclasses import dataclass
-from typing import List
+from typing import List, Optional
 
 from snowflake.ml.model._client.model import model_version_impl
-from snowflake.ml.monitoring.entities import model_monitor_interval
 
 
 @dataclass
-class ModelMonitorTableConfig:
-    source_table: str
+class ModelMonitorSourceConfig:
+    """Configuration for the source of data to be monitored."""
+
+    source: str
+    """Name of table or view containing monitoring data."""
+
     timestamp_column: str
-    prediction_columns: List[str]
-    label_columns: List[str]
+    """Name of column in the source containing timestamp."""
+
     id_columns: List[str]
+    """List of columns in the source containing unique identifiers."""
+
+    prediction_score_columns: Optional[List[str]] = None
+    """List of columns in the source containing prediction scores.
+    Can be regression scores for regression models and probability scores for classification models."""
+
+    prediction_class_columns: Optional[List[str]] = None
+    """List of columns in the source containing prediction classes for classification models."""
+
+    actual_score_columns: Optional[List[str]] = None
+    """List of columns in the source containing actual scores."""
+
+    actual_class_columns: Optional[List[str]] = None
+    """List of columns in the source containing actual classes for classification models."""
+
+    baseline: Optional[str] = None
+    """Name of table containing the baseline data."""
 
 
 @dataclass
 class ModelMonitorConfig:
+    """Configuration for the Model Monitor."""
+
     model_version: model_version_impl.ModelVersion
+    """Model version to monitor."""
 
-    # Python model function name
     model_function_name: str
+    """Function name in the model to monitor."""
+
     background_compute_warehouse_name: str
-    # TODO: Add support for pythonic notion of time.
-    refresh_interval: str = model_monitor_interval.ModelMonitorRefreshInterval.DAILY
-    aggregation_window: model_monitor_interval.ModelMonitorAggregationWindow = (
-        model_monitor_interval.ModelMonitorAggregationWindow.WINDOW_1_DAY
-    )
+    """Name of the warehouse to use for background compute."""
+
+    refresh_interval: str = "1 hour"
+    """Interval at which to refresh the monitoring data."""
+
+    aggregation_window: str = "1 day"
+    """Window for aggregating monitoring data."""