mlrun 1.7.0rc28__py3-none-any.whl → 1.7.0rc55__py3-none-any.whl

mlrun/feature_store/retrieval/conversion.py

@@ -1,271 +0,0 @@
-# Copyright 2024 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import warnings
-from collections import Counter
-
-# Copied from https://github.com/apache/spark/blob/v3.2.3/python/pyspark/sql/pandas/conversion.py, with
-# np.bool -> bool and np.object -> object fix backported from pyspark v3.3.3.
-
-
-class PandasConversionMixin:
-    """
-    Min-in for the conversion from Spark to pandas. Currently, only :class:`DataFrame`
-    can use this class.
-    """
-
-    def toPandas(self):
-        """
-        Returns the contents of this :class:`DataFrame` as Pandas ``pandas.DataFrame``.
-
-        This is only available if Pandas is installed and available.
-
-        .. versionadded:: 1.3.0
-
-        Notes
-        -----
-        This method should only be used if the resulting Pandas's :class:`DataFrame` is
-        expected to be small, as all the data is loaded into the driver's memory.
-
-        Usage with spark.sql.execution.arrow.pyspark.enabled=True is experimental.
-
-        Examples
-        --------
-        >>> df.toPandas()  # doctest: +SKIP
-           age   name
-        0    2  Alice
-        1    5    Bob
-        """
-        from pyspark.sql.dataframe import DataFrame
-
-        assert isinstance(self, DataFrame)
-
-        from pyspark.sql.pandas.utils import require_minimum_pandas_version
-
-        require_minimum_pandas_version()
-
-        import numpy as np
-        import pandas as pd
-        from pyspark.sql.types import (
-            BooleanType,
-            IntegralType,
-            MapType,
-            TimestampType,
-        )
-
-        timezone = self.sql_ctx._conf.sessionLocalTimeZone()
-
-        if self.sql_ctx._conf.arrowPySparkEnabled():
-            use_arrow = True
-            try:
-                from pyspark.sql.pandas.types import to_arrow_schema
-                from pyspark.sql.pandas.utils import require_minimum_pyarrow_version
-
-                require_minimum_pyarrow_version()
-                to_arrow_schema(self.schema)
-            except Exception as e:
-                if self.sql_ctx._conf.arrowPySparkFallbackEnabled():
-                    msg = (
-                        "toPandas attempted Arrow optimization because "
-                        "'spark.sql.execution.arrow.pyspark.enabled' is set to true; however, "
-                        f"failed by the reason below:\n  {e}\n"
-                        "Attempting non-optimization as "
-                        "'spark.sql.execution.arrow.pyspark.fallback.enabled' is set to "
-                        "true."
-                    )
-                    warnings.warn(msg)
-                    use_arrow = False
-                else:
-                    msg = (
-                        "toPandas attempted Arrow optimization because "
-                        "'spark.sql.execution.arrow.pyspark.enabled' is set to true, but has "
-                        "reached the error below and will not continue because automatic fallback "
-                        "with 'spark.sql.execution.arrow.pyspark.fallback.enabled' has been set to "
-                        f"false.\n  {e}"
-                    )
-                    warnings.warn(msg)
-                    raise
-
-            # Try to use Arrow optimization when the schema is supported and the required version
-            # of PyArrow is found, if 'spark.sql.execution.arrow.pyspark.enabled' is enabled.
-            if use_arrow:
-                try:
-                    import pyarrow
-                    from pyspark.sql.pandas.types import (
-                        _check_series_localize_timestamps,
-                        _convert_map_items_to_dict,
-                    )
-
-                    # Rename columns to avoid duplicated column names.
-                    tmp_column_names = [f"col_{i}" for i in range(len(self.columns))]
-                    self_destruct = self.sql_ctx._conf.arrowPySparkSelfDestructEnabled()
-                    batches = self.toDF(*tmp_column_names)._collect_as_arrow(
-                        split_batches=self_destruct
-                    )
-                    if len(batches) > 0:
-                        table = pyarrow.Table.from_batches(batches)
-                        # Ensure only the table has a reference to the batches, so that
-                        # self_destruct (if enabled) is effective
-                        del batches
-                        # Pandas DataFrame created from PyArrow uses datetime64[ns] for date type
-                        # values, but we should use datetime.date to match the behavior with when
-                        # Arrow optimization is disabled.
-                        pandas_options = {"date_as_object": True}
-                        if self_destruct:
-                            # Configure PyArrow to use as little memory as possible:
-                            # self_destruct - free columns as they are converted
-                            # split_blocks - create a separate Pandas block for each column
-                            # use_threads - convert one column at a time
-                            pandas_options.update(
-                                {
-                                    "self_destruct": True,
-                                    "split_blocks": True,
-                                    "use_threads": False,
-                                }
-                            )
-                        pdf = table.to_pandas(**pandas_options)
-                        # Rename back to the original column names.
-                        pdf.columns = self.columns
-                        for field in self.schema:
-                            if isinstance(field.dataType, TimestampType):
-                                pdf[field.name] = _check_series_localize_timestamps(
-                                    pdf[field.name], timezone
-                                )
-                            elif isinstance(field.dataType, MapType):
-                                pdf[field.name] = _convert_map_items_to_dict(
-                                    pdf[field.name]
-                                )
-                        return pdf
-                    else:
-                        return pd.DataFrame.from_records([], columns=self.columns)
-                except Exception as e:
-                    # We might have to allow fallback here as well but multiple Spark jobs can
-                    # be executed. So, simply fail in this case for now.
-                    msg = (
-                        "toPandas attempted Arrow optimization because "
-                        "'spark.sql.execution.arrow.pyspark.enabled' is set to true, but has "
-                        "reached the error below and can not continue. Note that "
-                        "'spark.sql.execution.arrow.pyspark.fallback.enabled' does not have an "
-                        "effect on failures in the middle of "
-                        f"computation.\n  {e}"
-                    )
-                    warnings.warn(msg)
-                    raise
-
-        # Below is toPandas without Arrow optimization.
-        pdf = pd.DataFrame.from_records(self.collect(), columns=self.columns)
-        column_counter = Counter(self.columns)
-
-        dtype = [None] * len(self.schema)
-        for field_idx, field in enumerate(self.schema):
-            # For duplicate column name, we use `iloc` to access it.
-            if column_counter[field.name] > 1:
-                pandas_col = pdf.iloc[:, field_idx]
-            else:
-                pandas_col = pdf[field.name]
-
-            pandas_type = PandasConversionMixin._to_corrected_pandas_type(
-                field.dataType
-            )
-            # SPARK-21766: if an integer field is nullable and has null values, it can be
-            # inferred by pandas as float column. Once we convert the column with NaN back
-            # to integer type e.g., np.int16, we will hit exception. So we use the inferred
-            # float type, not the corrected type from the schema in this case.
-            if pandas_type is not None and not (
-                isinstance(field.dataType, IntegralType)
-                and field.nullable
-                and pandas_col.isnull().any()
-            ):
-                dtype[field_idx] = pandas_type
-            # Ensure we fall back to nullable numpy types, even when whole column is null:
-            if isinstance(field.dataType, IntegralType) and pandas_col.isnull().any():
-                dtype[field_idx] = np.float64
-            if isinstance(field.dataType, BooleanType) and pandas_col.isnull().any():
-                dtype[field_idx] = object
-
-        df = pd.DataFrame()
-        for index, t in enumerate(dtype):
-            column_name = self.schema[index].name
-
-            # For duplicate column name, we use `iloc` to access it.
-            if column_counter[column_name] > 1:
-                series = pdf.iloc[:, index]
-            else:
-                series = pdf[column_name]
-
-            if t is not None:
-                series = series.astype(t, copy=False)
-
-            # `insert` API makes copy of data, we only do it for Series of duplicate column names.
-            # `pdf.iloc[:, index] = pdf.iloc[:, index]...` doesn't always work because `iloc` could
-            # return a view or a copy depending by context.
-            if column_counter[column_name] > 1:
-                df.insert(index, column_name, series, allow_duplicates=True)
-            else:
-                df[column_name] = series
-
-        pdf = df
-
-        if timezone is None:
-            return pdf
-        else:
-            from pyspark.sql.pandas.types import (
-                _check_series_convert_timestamps_local_tz,
-            )
-
-            for field in self.schema:
-                # TODO: handle nested timestamps, such as ArrayType(TimestampType())?
-                if isinstance(field.dataType, TimestampType):
-                    pdf[field.name] = _check_series_convert_timestamps_local_tz(
-                        pdf[field.name], timezone
-                    )
-            return pdf
-
-    @staticmethod
-    def _to_corrected_pandas_type(dt):
-        """
-        When converting Spark SQL records to Pandas :class:`DataFrame`, the inferred data type
-        may be wrong. This method gets the corrected data type for Pandas if that type may be
-        inferred incorrectly.
-        """
-        import numpy as np
-        from pyspark.sql.types import (
-            BooleanType,
-            ByteType,
-            DoubleType,
-            FloatType,
-            IntegerType,
-            LongType,
-            ShortType,
-            TimestampType,
-        )
-
-        if type(dt) == ByteType:
-            return np.int8
-        elif type(dt) == ShortType:
-            return np.int16
-        elif type(dt) == IntegerType:
-            return np.int32
-        elif type(dt) == LongType:
-            return np.int64
-        elif type(dt) == FloatType:
-            return np.float32
-        elif type(dt) == DoubleType:
-            return np.float64
-        elif type(dt) == BooleanType:
-            return bool
-        elif type(dt) == TimestampType:
-            return np.datetime64
-        else:
-            return None

mlrun/model_monitoring/controller_handler.py

@@ -1,37 +0,0 @@
-# Copyright 2023 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import nuclio
-
-import mlrun
-from mlrun.model_monitoring.controller import MonitoringApplicationController
-
-
-def handler(context: nuclio.Context, event: nuclio.Event) -> None:
-    """
-    Run model monitoring application processor
-
-    :param context: the Nuclio context
-    :param event:   trigger event
-    """
-    context.user_data.monitor_app_controller.run(event)
-
-
-def init_context(context):
-    mlrun_context = mlrun.get_or_create_ctx("model_monitoring_controller")
-    mlrun_context.logger.info("Initialize monitoring app controller")
-    monitor_app_controller = MonitoringApplicationController(
-        mlrun_context=mlrun_context,
-        project=mlrun_context.project,
-    )
-    setattr(context.user_data, "monitor_app_controller", monitor_app_controller)

mlrun/model_monitoring/evidently_application.py

@@ -1,20 +0,0 @@
-# Copyright 2023 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# TODO : delete this file in 1.9.0
-from mlrun.model_monitoring.applications import (  # noqa: F401
-    _HAS_EVIDENTLY,
-    SUPPORTED_EVIDENTLY_VERSION,
-    EvidentlyModelMonitoringApplicationBase,
-)

mlrun/model_monitoring/prometheus.py

@@ -1,216 +0,0 @@
-# Copyright 2023 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import prometheus_client
-
-from mlrun.common.schemas.model_monitoring import EventFieldType, PrometheusMetric
-
-# Memory path for Prometheus registry file
-_registry_path = "/tmp/prom-reg.txt"
-
-# Initializing Promethues metric collector registry
-_registry: prometheus_client.CollectorRegistry = prometheus_client.CollectorRegistry()
-
-# The following real-time metrics are being updated through the monitoring stream graph steps
-_prediction_counter: prometheus_client.Counter = prometheus_client.Counter(
-    name=PrometheusMetric.PREDICTIONS_TOTAL,
-    documentation="Counter for total predictions",
-    registry=_registry,
-    labelnames=[
-        EventFieldType.PROJECT,
-        EventFieldType.ENDPOINT_ID,
-        EventFieldType.MODEL,
-        EventFieldType.ENDPOINT_TYPE,
-    ],
-)
-_model_latency: prometheus_client.Summary = prometheus_client.Summary(
-    name=PrometheusMetric.MODEL_LATENCY_SECONDS,
-    documentation="Summary for for model latency",
-    registry=_registry,
-    labelnames=[
-        EventFieldType.PROJECT,
-        EventFieldType.ENDPOINT_ID,
-        EventFieldType.MODEL,
-        EventFieldType.ENDPOINT_TYPE,
-    ],
-)
-_income_features: prometheus_client.Gauge = prometheus_client.Gauge(
-    name=PrometheusMetric.INCOME_FEATURES,
-    documentation="Samples of features and predictions",
-    registry=_registry,
-    labelnames=[
-        EventFieldType.PROJECT,
-        EventFieldType.ENDPOINT_ID,
-        EventFieldType.METRIC,
-    ],
-)
-_error_counter: prometheus_client.Counter = prometheus_client.Counter(
-    name=PrometheusMetric.ERRORS_TOTAL,
-    documentation="Counter for total errors",
-    registry=_registry,
-    labelnames=[
-        EventFieldType.PROJECT,
-        EventFieldType.ENDPOINT_ID,
-        EventFieldType.MODEL,
-    ],
-)
-
-# The following metrics are being updated through the model monitoring batch job
-_batch_metrics: prometheus_client.Gauge = prometheus_client.Gauge(
-    name=PrometheusMetric.DRIFT_METRICS,
-    documentation="Results from the batch drift analysis",
-    registry=_registry,
-    labelnames=[
-        EventFieldType.PROJECT,
-        EventFieldType.ENDPOINT_ID,
-        EventFieldType.METRIC,
-    ],
-)
-_drift_status: prometheus_client.Enum = prometheus_client.Enum(
-    name=PrometheusMetric.DRIFT_STATUS,
-    documentation="Drift status of the model endpoint",
-    registry=_registry,
-    states=["NO_DRIFT", "DRIFT_DETECTED", "POSSIBLE_DRIFT"],
-    labelnames=[EventFieldType.PROJECT, EventFieldType.ENDPOINT_ID],
-)
-
-
-def _write_registry(func):
-    def wrapper(*args, **kwargs):
-        global _registry
-        """A wrapper function to update the registry file each time a metric has been updated"""
-        func(*args, **kwargs)
-        prometheus_client.write_to_textfile(path=_registry_path, registry=_registry)
-
-    return wrapper
-
-
-@_write_registry
-def write_predictions_and_latency_metrics(
-    project: str, endpoint_id: str, latency: int, model_name: str, endpoint_type: int
-):
-    """
-    Update the prediction counter and the latency value of the provided model endpoint within Prometheus registry.
-    Please note that while the prediction counter is ALWAYS increasing by 1,the latency summary metric is being
-    increased by the event latency time. Grafana dashboard will query the average latency time by dividing the total
-    latency value by the total amount of predictions.
-
-    :param project:       Project name.
-    :param endpoint_id:   Model endpoint unique id.
-    :param latency:       Latency time (microsecond) in which the event has been processed through the model server.
-    :param model_name:    Model name which will be used by Grafana for displaying the results by model.
-    :param endpoint_type: Endpoint type that is represented by an int (possible values: 1,2,3) corresponding to the
-                          Enum class :py:class:`~mlrun.common.schemas.model_monitoring.EndpointType`.
-    """
-
-    # Increase the prediction counter by 1
-    _prediction_counter.labels(
-        project=project,
-        endpoint_id=endpoint_id,
-        model=model_name,
-        endpoint_type=endpoint_type,
-    ).inc(1)
-
-    # Increase the latency value according to the provided latency of the current event
-    _model_latency.labels(
-        project=project,
-        endpoint_id=endpoint_id,
-        model=model_name,
-        endpoint_type=endpoint_type,
-    ).observe(latency)
-
-
-@_write_registry
-def write_income_features(project: str, endpoint_id: str, features: dict[str, float]):
-    """Update a sample of features.
-
-    :param project:     Project name.
-    :param endpoint_id: Model endpoint unique id.
-    :param features:    Dictionary in which the key is a feature name and the value is a float number.
-
-
-    """
-
-    for metric in features:
-        _income_features.labels(
-            project=project, endpoint_id=endpoint_id, metric=metric
-        ).set(value=features[metric])
-
-
-@_write_registry
-def write_drift_metrics(project: str, endpoint_id: str, metric: str, value: float):
-    """Update drift metrics that have been calculated through the monitoring batch job
-
-    :param project:     Project name.
-    :param endpoint_id: Model endpoint unique id.
-    :param metric:      Metric name (e.g. TVD, Hellinger).
-    :param value:       Metric value as a float.
-
-    """
-
-    _batch_metrics.labels(project=project, endpoint_id=endpoint_id, metric=metric).set(
-        value=value
-    )
-
-
-@_write_registry
-def write_drift_status(project: str, endpoint_id: str, drift_status: str):
-    """
-    Update the drift status enum for a specific model endpoint.
-
-    :param project:      Project name.
-    :param endpoint_id:  Model endpoint unique id.
-    :param drift_status: Drift status value, can be one of the following: 'NO_DRIFT', 'DRIFT_DETECTED', or
-                         'POSSIBLE_DRIFT'.
-    """
-
-    _drift_status.labels(project=project, endpoint_id=endpoint_id).state(drift_status)
-
-
-@_write_registry
-def write_errors(project: str, endpoint_id: str, model_name: str):
-    """
-    Update the error counter for a specific model endpoint.
-
-    :param project:     Project name.
-    :param endpoint_id: Model endpoint unique id.
-    :param model_name:  Model name. Will be used by Grafana to show the amount of errors per model by time.
-    """
-
-    _error_counter.labels(
-        project=project, endpoint_id=endpoint_id, model=model_name
-    ).inc(1)
-
-
-def get_registry() -> str:
-    """Returns the parsed registry file according to the exposition format of Prometheus."""
-
-    # Read the registry file (note that the text is stored in UTF-8 format)
-    f = open(_registry_path)
-    lines = f.read()
-    f.close()
-
-    # Reset part of the metrics to avoid a repeating scraping of the same value
-    clean_metrics()
-
-    return lines
-
-
-@_write_registry
-def clean_metrics():
-    """Clean the income features values. As these results are relevant only for a certain timestamp, we will remove
-    them from the global registry after they have been scraped by Prometheus."""
-
-    _income_features.clear()