mlrun 1.7.0rc5__py3-none-any.whl → 1.7.0rc7__py3-none-any.whl

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -13,13 +13,17 @@
 # limitations under the License.
 
 from dataclasses import dataclass
-from typing import Final, Optional, Protocol
+from typing import Final, Optional, Protocol, cast
 
 import numpy as np
-from pandas import DataFrame, Timestamp
+from pandas import DataFrame, Series, Timestamp
 
+import mlrun.artifacts
+import mlrun.common.model_monitoring.helpers
+import mlrun.model_monitoring.features_drift_table as mm_drift_table
 from mlrun.common.schemas.model_monitoring.constants import (
     MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME,
+    EventFieldType,
     ResultKindApp,
     ResultStatusApp,
 )
@@ -27,7 +31,7 @@ from mlrun.model_monitoring.application import (
     ModelMonitoringApplicationBase,
     ModelMonitoringApplicationResult,
 )
-from mlrun.model_monitoring.batch import (
+from mlrun.model_monitoring.metrics.histogram_distance import (
     HellingerDistance,
     HistogramDistanceMetric,
     KullbackLeiblerDivergence,
@@ -115,31 +119,24 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
 
     def _compute_metrics_per_feature(
         self, sample_df_stats: DataFrame, feature_stats: DataFrame
-    ) -> dict[type[HistogramDistanceMetric], list[float]]:
+    ) -> DataFrame:
         """Compute the metrics for the different features and labels"""
-        metrics_per_feature: dict[type[HistogramDistanceMetric], list[float]] = {
-            metric_class: [] for metric_class in self.metrics
-        }
+        metrics_per_feature = DataFrame(
+            columns=[metric_class.NAME for metric_class in self.metrics]
+        )
 
-        for (sample_feat, sample_hist), (reference_feat, reference_hist) in zip(
-            sample_df_stats.items(), feature_stats.items()
-        ):
-            assert sample_feat == reference_feat, "The features do not match"
+        for feature_name in feature_stats:
+            sample_hist = np.asarray(sample_df_stats[feature_name])
+            reference_hist = np.asarray(feature_stats[feature_name])
             self.context.logger.info(
-                "Computing metrics for feature", feature_name=sample_feat
+                "Computing metrics for feature", feature_name=feature_name
             )
-            sample_arr = np.asarray(sample_hist)
-            reference_arr = np.asarray(reference_hist)
-            for metric in self.metrics:
-                metric_name = metric.NAME
-                self.context.logger.debug(
-                    "Computing data drift metric",
-                    metric_name=metric_name,
-                    feature_name=sample_feat,
-                )
-                metrics_per_feature[metric].append(
-                    metric(distrib_t=sample_arr, distrib_u=reference_arr).compute()
-                )
+            metrics_per_feature.loc[feature_name] = {  # pyright: ignore[reportCallIssue,reportArgumentType]
+                metric.NAME: metric(
+                    distrib_t=sample_hist, distrib_u=reference_hist
+                ).compute()
+                for metric in self.metrics
+            }
         self.context.logger.info("Finished computing the metrics")
 
         return metrics_per_feature
@@ -147,37 +144,37 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
     def _add_general_drift_result(
         self, results: list[ModelMonitoringApplicationResult], value: float
     ) -> None:
+        """Add the general drift result to the results list and log it"""
+        status = self._value_classifier.value_to_status(value)
         results.append(
             ModelMonitoringApplicationResult(
                 name="general_drift",
                 value=value,
                 kind=self.METRIC_KIND,
-                status=self._value_classifier.value_to_status(value),
+                status=status,
             )
         )
 
     def _get_results(
-        self, metrics_per_feature: dict[type[HistogramDistanceMetric], list[float]]
+        self, metrics_per_feature: DataFrame
     ) -> list[ModelMonitoringApplicationResult]:
         """Average the metrics over the features and add the status"""
         results: list[ModelMonitoringApplicationResult] = []
-        hellinger_tvd_values: list[float] = []
-        for metric_class, metric_values in metrics_per_feature.items():
-            self.context.logger.debug(
-                "Averaging metric over the features", metric_name=metric_class.NAME
-            )
-            value = np.mean(metric_values)
-            if metric_class == KullbackLeiblerDivergence:
+
+        self.context.logger.debug("Averaging metrics over the features")
+        metrics_mean = metrics_per_feature.mean().to_dict()
+
+        self.context.logger.debug("Creating the results")
+        for name, value in metrics_mean.items():
+            if name == KullbackLeiblerDivergence.NAME:
                 # This metric is not bounded from above [0, inf).
                 # No status is currently reported for KL divergence
                 status = ResultStatusApp.irrelevant
             else:
                 status = self._value_classifier.value_to_status(value)
-            if metric_class in self._REQUIRED_METRICS:
-                hellinger_tvd_values.append(value)
             results.append(
                 ModelMonitoringApplicationResult(
-                    name=f"{metric_class.NAME}_mean",
+                    name=f"{name}_mean",
                     value=value,
                     kind=self.METRIC_KIND,
                     status=status,
@@ -185,16 +182,102 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
             )
 
         self._add_general_drift_result(
-            results=results, value=np.mean(hellinger_tvd_values)
+            results=results,
+            value=np.mean(
+                [
+                    metrics_mean[HellingerDistance.NAME],
+                    metrics_mean[TotalVarianceDistance.NAME],
+                ]
+            ),
         )
 
+        self.context.logger.info("Finished with the results")
         return results
 
+    @staticmethod
+    def _remove_timestamp_feature(
+        sample_set_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+    ) -> mlrun.common.model_monitoring.helpers.FeatureStats:
+        """
+        Drop the 'timestamp' feature if it exists, as it is irrelevant
+        in the plotly artifact
+        """
+        sample_set_statistics = mlrun.common.model_monitoring.helpers.FeatureStats(
+            sample_set_statistics.copy()
+        )
+        if EventFieldType.TIMESTAMP in sample_set_statistics:
+            del sample_set_statistics[EventFieldType.TIMESTAMP]
+        return sample_set_statistics
+
+    def _log_json_artifact(self, drift_per_feature_values: Series) -> None:
+        """Log the drift values as a JSON artifact"""
+        self.context.logger.debug("Logging drift value per feature JSON artifact")
+        self.context.log_artifact(
+            mlrun.artifacts.Artifact(
+                body=drift_per_feature_values.to_json(),
+                format="json",
+                key="features_drift_results",
+            )
+        )
+        self.context.logger.debug("Logged JSON artifact successfully")
+
+    def _log_plotly_table_artifact(
+        self,
+        sample_set_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+        inputs_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+        metrics_per_feature: DataFrame,
+        drift_per_feature_values: Series,
+    ) -> None:
+        """Log the Plotly drift table artifact"""
+        self.context.logger.debug(
+            "Feature stats",
+            sample_set_statistics=sample_set_statistics,
+            inputs_statistics=inputs_statistics,
+        )
+
+        self.context.logger.debug("Computing drift results per feature")
+        drift_results = {
+            cast(str, key): (self._value_classifier.value_to_status(value), value)
+            for key, value in drift_per_feature_values.items()
+        }
+        self.context.logger.debug("Logging plotly artifact")
+        self.context.log_artifact(
+            mm_drift_table.FeaturesDriftTablePlot().produce(
+                sample_set_statistics=sample_set_statistics,
+                inputs_statistics=inputs_statistics,
+                metrics=metrics_per_feature.T.to_dict(),
+                drift_results=drift_results,
+            )
+        )
+        self.context.logger.debug("Logged plotly artifact successfully")
+
+    def _log_drift_artifacts(
+        self,
+        sample_set_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+        inputs_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+        metrics_per_feature: DataFrame,
+        log_json_artifact: bool = True,
+    ) -> None:
+        """Log JSON and Plotly drift data per feature artifacts"""
+        drift_per_feature_values = metrics_per_feature[
+            [HellingerDistance.NAME, TotalVarianceDistance.NAME]
+        ].mean(axis=1)
+
+        if log_json_artifact:
+            self._log_json_artifact(drift_per_feature_values)
+
+        self._log_plotly_table_artifact(
+            sample_set_statistics=self._remove_timestamp_feature(sample_set_statistics),
+            inputs_statistics=inputs_statistics,
+            metrics_per_feature=metrics_per_feature,
+            drift_per_feature_values=drift_per_feature_values,
+        )
+
     def do_tracking(
         self,
         application_name: str,
-        sample_df_stats: DataFrame,
-        feature_stats: DataFrame,
+        sample_df_stats: mlrun.common.model_monitoring.helpers.FeatureStats,
+        feature_stats: mlrun.common.model_monitoring.helpers.FeatureStats,
         sample_df: DataFrame,
         start_infer_time: Timestamp,
         end_infer_time: Timestamp,
@@ -210,7 +293,14 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         """
         self.context.logger.debug("Starting to run the application")
         metrics_per_feature = self._compute_metrics_per_feature(
-            sample_df_stats=sample_df_stats, feature_stats=feature_stats
+            sample_df_stats=self.dict_to_histogram(sample_df_stats),
+            feature_stats=self.dict_to_histogram(feature_stats),
+        )
+        self.context.logger.debug("Saving artifacts")
+        self._log_drift_artifacts(
+            inputs_statistics=feature_stats,
+            sample_set_statistics=sample_df_stats,
+            metrics_per_feature=metrics_per_feature,
         )
         self.context.logger.debug("Computing average per metric")
         results = self._get_results(metrics_per_feature)

mlrun/model_monitoring/controller.py

@@ -21,25 +21,24 @@ from collections.abc import Iterator
 from typing import Any, NamedTuple, Optional, Union, cast
 
 import nuclio
-from v3io.dataplane.response import HttpResponseError
 
 import mlrun
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.data_types.infer
 import mlrun.feature_store as fstore
+import mlrun.model_monitoring.db.stores
 from mlrun.common.model_monitoring.helpers import FeatureStats, pad_features_hist
 from mlrun.datastore import get_stream_pusher
 from mlrun.datastore.targets import ParquetTarget
 from mlrun.errors import err_to_str
-from mlrun.model_monitoring.batch import calculate_inputs_statistics
 from mlrun.model_monitoring.helpers import (
     _BatchDict,
     batch_dict2timedelta,
+    calculate_inputs_statistics,
     get_monitoring_parquet_path,
     get_stream_path,
 )
-from mlrun.utils import create_logger, datetime_now, logger
-from mlrun.utils.v3io_clients import get_v3io_client
+from mlrun.utils import datetime_now, logger
 
 
 class _Interval(NamedTuple):
@@ -48,8 +47,6 @@ class _Interval(NamedTuple):
 
 
 class _BatchWindow:
-    V3IO_CONTAINER_FORMAT = "users/pipelines/{project}/monitoring-schedules/functions"
-
     def __init__(
         self,
         project: str,
@@ -65,27 +62,22 @@ class _BatchWindow:
         All the time values are in seconds.
         The start and stop time are in seconds since the epoch.
         """
+        self.project = project
         self._endpoint = endpoint
         self._application = application
         self._first_request = first_request
-        self._kv_storage = get_v3io_client(
-            endpoint=mlrun.mlconf.v3io_api,
-            # Avoid noisy warning logs before the KV table is created
-            logger=create_logger(name="v3io_client", level="error"),
-        ).kv
-        self._v3io_container = self.V3IO_CONTAINER_FORMAT.format(project=project)
         self._stop = last_updated
         self._step = timedelta_seconds
+        self._db = mlrun.model_monitoring.get_store_object(project=self.project)
         self._start = self._get_last_analyzed()
 
     def _get_last_analyzed(self) -> Optional[int]:
         try:
-            data = self._kv_storage.get(
-                container=self._v3io_container,
-                table_path=self._endpoint,
-                key=self._application,
+            last_analyzed = self._db.get_last_analyzed(
+                endpoint_id=self._endpoint,
+                application_name=self._application,
             )
-        except HttpResponseError as err:
+        except mlrun.errors.MLRunNotFoundError:
             logger.info(
                 "No last analyzed time was found for this endpoint and "
                 "application, as this is probably the first time this "
@@ -96,7 +88,7 @@ class _BatchWindow:
                 first_request=self._first_request,
                 last_updated=self._stop,
             )
-            logger.debug("Error while getting last analyzed time", err=err)
+
             if self._first_request and self._stop:
                 # TODO : Change the timedelta according to the policy.
                 first_period_in_seconds = max(
@@ -108,7 +100,6 @@ class _BatchWindow:
                 )
             return self._first_request
 
-        last_analyzed = data.output.item[mm_constants.SchedulingKeys.LAST_ANALYZED]
         logger.info(
             "Got the last analyzed time for this endpoint and application",
             endpoint=self._endpoint,
@@ -124,11 +115,11 @@ class _BatchWindow:
             application=self._application,
             last_analyzed=last_analyzed,
         )
-        self._kv_storage.put(
-            container=self._v3io_container,
-            table_path=self._endpoint,
-            key=self._application,
-            attributes={mm_constants.SchedulingKeys.LAST_ANALYZED: last_analyzed},
+
+        self._db.update_last_analyzed(
+            endpoint_id=self._endpoint,
+            application_name=self._application,
+            last_analyzed=last_analyzed,
         )
 
     def get_intervals(
@@ -301,7 +292,7 @@ class MonitoringApplicationController:
             f"Initializing {self.__class__.__name__}", project=project
         )
 
-        self.db = mlrun.model_monitoring.get_model_endpoint_store(project=project)
+        self.db = mlrun.model_monitoring.get_store_object(project=project)
 
         self._batch_window_generator = _BatchWindowGenerator(
             batch_dict=json.loads(
@@ -359,7 +350,12 @@ class MonitoringApplicationController:
                     {
                         app.metadata.name
                         for app in monitoring_functions
-                        if app.status.state == "ready"
+                        if (
+                            app.status.state == "ready"
+                            # workaround for the default app, as its `status.state` is `None`
+                            or app.metadata.name
+                            == mm_constants.MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME
+                        )
                     }
                 )
             if not applications_names:
@@ -367,6 +363,10 @@ class MonitoringApplicationController:
                     "No monitoring functions found", project=self.project
                 )
                 return
+            self.context.logger.info(
+                "Starting to iterate over the applications",
+                applications=applications_names,
+            )
 
         except Exception as e:
             self.context.logger.error(
@@ -445,13 +445,6 @@ class MonitoringApplicationController:
             m_fs = fstore.get_feature_set(
                 endpoint[mm_constants.EventFieldType.FEATURE_SET_URI]
             )
-            labels = endpoint[mm_constants.EventFieldType.LABEL_NAMES]
-            if labels:
-                if isinstance(labels, str):
-                    labels = json.loads(labels)
-                for label in labels:
-                    if label not in list(m_fs.spec.features.keys()):
-                        m_fs.add_feature(fstore.Feature(name=label, value_type="float"))
 
             for application in applications_names:
                 batch_window = batch_window_generator.get_batch_window(

mlrun/model_monitoring/db/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .stores import ObjectStoreFactory, get_store_object
+from .stores.base import StoreBase

mlrun/model_monitoring/{stores → db/stores}/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Iguazio
+# Copyright 2024 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,60 +16,54 @@
 
 import enum
 import typing
+import warnings
 
 import mlrun.common.schemas.secret
 import mlrun.errors
 
-from .model_endpoint_store import ModelEndpointStore
+from .base import StoreBase
 
 
-class ModelEndpointStoreType(enum.Enum):
-    """Enum class to handle the different store type values for saving a model endpoint record."""
+class ObjectStoreFactory(enum.Enum):
+    """Enum class to handle the different store type values for saving model monitoring records."""
 
     v3io_nosql = "v3io-nosql"
     SQL = "sql"
 
-    def to_endpoint_store(
+    def to_object_store(
         self,
         project: str,
         access_key: str = None,
-        endpoint_store_connection: str = None,
         secret_provider: typing.Callable = None,
-    ) -> ModelEndpointStore:
+    ) -> StoreBase:
         """
-        Return a ModelEndpointStore object based on the provided enum value.
-
-        :param project:                    The name of the project.
-        :param access_key:                 Access key with permission to the DB table. Note that if access key is None
-                                           and the endpoint target is from type KV then the access key will be
-                                           retrieved from the environment variable.
-        :param endpoint_store_connection: A valid connection string for model endpoint target. Contains several
-                                          key-value pairs that required for the database connection.
-                                          e.g. A root user with password 1234, tries to connect a schema called
-                                          mlrun within a local MySQL DB instance:
-                                          'mysql+pymysql://root:1234@localhost:3306/mlrun'.
+        Return a StoreBase object based on the provided enum value.
+
+        :param project:                   The name of the project.
+        :param access_key:                Access key with permission to the DB table. Note that if access key is None
+                                          and the endpoint target is from type KV then the access key will be
+                                          retrieved from the environment variable.
         :param secret_provider:           An optional secret provider to get the connection string secret.
 
-        :return: `ModelEndpointStore` object.
+        :return: `StoreBase` object.
 
         """
 
-        if self.value == ModelEndpointStoreType.v3io_nosql.value:
-            from .kv_model_endpoint_store import KVModelEndpointStore
+        if self == self.v3io_nosql:
+            from mlrun.model_monitoring.db.stores.v3io_kv.kv_store import KVStoreBase
 
             # Get V3IO access key from env
             access_key = access_key or mlrun.mlconf.get_v3io_access_key()
 
-            return KVModelEndpointStore(project=project, access_key=access_key)
+            return KVStoreBase(project=project, access_key=access_key)
 
         # Assuming SQL store target if store type is not KV.
         # Update these lines once there are more than two store target types.
 
-        from .sql_model_endpoint_store import SQLModelEndpointStore
+        from mlrun.model_monitoring.db.stores.sqldb.sql_store import SQLStoreBase
 
-        return SQLModelEndpointStore(
+        return SQLStoreBase(
             project=project,
-            sql_connection_string=endpoint_store_connection,
             secret_provider=secret_provider,
         )
 
@@ -88,7 +82,24 @@ def get_model_endpoint_store(
     project: str,
     access_key: str = None,
     secret_provider: typing.Callable = None,
-) -> ModelEndpointStore:
+) -> StoreBase:
+    # Leaving here for backwards compatibility
+    warnings.warn(
+        "The 'get_model_endpoint_store' function is deprecated and will be removed in 1.9.0. "
+        "Please use `get_store_object` instead.",
+        # TODO: remove in 1.9.0
+        FutureWarning,
+    )
+    return get_store_object(
+        project=project, access_key=access_key, secret_provider=secret_provider
+    )
+
+
+def get_store_object(
+    project: str,
+    access_key: str = None,
+    secret_provider: typing.Callable = None,
+) -> StoreBase:
     """
     Getting the DB target type based on mlrun.config.model_endpoint_monitoring.store_type.
 
@@ -96,16 +107,14 @@ def get_model_endpoint_store(
     :param access_key:      Access key with permission to the DB table.
     :param secret_provider: An optional secret provider to get the connection string secret.
 
-    :return: `ModelEndpointStore` object. Using this object, the user can apply different operations on the
-             model endpoint record such as write, update, get and delete.
+    :return: `StoreBase` object. Using this object, the user can apply different operations on the
+             model monitoring record such as write, update, get and delete a model endpoint.
     """
 
-    # Get store type value from ModelEndpointStoreType enum class
-    model_endpoint_store_type = ModelEndpointStoreType(
-        mlrun.mlconf.model_endpoint_monitoring.store_type
-    )
+    # Get store type value from ObjectStoreFactory enum class
+    store_type = ObjectStoreFactory(mlrun.mlconf.model_endpoint_monitoring.store_type)
 
-    # Convert into model endpoint store target object
-    return model_endpoint_store_type.to_endpoint_store(
+    # Convert into store target object
+    return store_type.to_object_store(
         project=project, access_key=access_key, secret_provider=secret_provider
     )

mlrun/model_monitoring/db/stores/base/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .store import StoreBase

mlrun/model_monitoring/{stores/model_endpoint_store.py → db/stores/base/store.py}

@@ -1,4 +1,4 @@
-# Copyright 2023 Iguazio
+# Copyright 2024 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,22 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 import typing
 from abc import ABC, abstractmethod
 
 
-class ModelEndpointStore(ABC):
+class StoreBase(ABC):
     """
-    An abstract class to handle the model endpoint in the DB target.
+    An abstract class to handle the store object in the DB target.
     """
 
     def __init__(self, project: str):
         """
-        Initialize a new model endpoint target.
+        Initialize a new store target.
 
-        :param project:             The name of the project.
+        :param project: The name of the project.
         """
         self.project = project
 
@@ -143,3 +142,45 @@ class ModelEndpointStore(ABC):
         """
 
         pass
+
+    @abstractmethod
+    def write_application_result(self, event: dict[str, typing.Any]):
+        """
+        Write a new application result event in the target table.
+
+        :param event: An event dictionary that represents the application result, should be corresponded to the
+                      schema defined in the :py:class:`~mlrun.common.schemas.model_monitoring.constants.WriterEvent`
+                      object.
+        """
+        pass
+
+    @abstractmethod
+    def get_last_analyzed(self, endpoint_id: str, application_name: str) -> int:
+        """
+        Get the last analyzed time for the provided model endpoint and application.
+
+        :param endpoint_id:      The unique id of the model endpoint.
+        :param application_name: Registered application name.
+
+        :return: Timestamp as a Unix time.
+        :raise:  MLRunNotFoundError if last analyzed value is not found.
+        """
+        pass
+
+    @abstractmethod
+    def update_last_analyzed(
+        self,
+        endpoint_id: str,
+        application_name: str,
+        last_analyzed: int,
+    ):
+        """
+        Update the last analyzed time for the provided model endpoint and application.
+
+        :param endpoint_id:      The unique id of the model endpoint.
+        :param application_name: Registered application name.
+        :param last_analyzed:    Timestamp as a Unix time that represents the last analyzed time of a certain
+                                 application and model endpoint.
+
+        """
+        pass

mlrun/model_monitoring/db/stores/sqldb/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.