mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/feature_store/steps.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import math
 import re
 import uuid
 import warnings
@@ -40,7 +41,9 @@ def get_engine(first_event):
 class MLRunStep(MapClass):
     def __init__(self, **kwargs):
         """Abstract class for mlrun step.
-        Can be used in pandas/storey/spark feature set ingestion"""
+        Can be used in pandas/storey/spark feature set ingestion. Extend this class and implement the relevant
+        `_do_XXX` methods to support the required execution engines.
+        """
         super().__init__(**kwargs)
         self._engine_to_do_method = {
             "pandas": self._do_pandas,
@@ -51,23 +54,41 @@ class MLRunStep(MapClass):
     def do(self, event):
         """
         This method defines the do method of this class according to the first event type.
+
+        .. warning::
+            When extending this class, do not override this method; only override the `_do_XXX` methods.
         """
         engine = get_engine(event)
         self.do = self._engine_to_do_method.get(engine, None)
         if self.do is None:
-            raise mlrun.errors.InvalidArgummentError(
+            raise mlrun.errors.MLRunInvalidArgumentError(
                 f"Unrecognized engine: {engine}. Available engines are: pandas, spark and storey"
             )
 
         return self.do(event)
 
     def _do_pandas(self, event):
+        """
+        The execution method for pandas engine.
+
+        :param event: Incoming event, a `pandas.DataFrame` object.
+        """
         raise NotImplementedError
 
     def _do_storey(self, event):
+        """
+        The execution method for storey engine.
+
+        :param event: Incoming event, a dictionary or `storey.Event` object, depending on the `full_event` value.
+        """
         raise NotImplementedError
 
     def _do_spark(self, event):
+        """
+        The execution method for spark engine.
+
+        :param event: Incoming event, a `pyspark.sql.DataFrame` object.
+        """
         raise NotImplementedError
 
 
@@ -136,7 +157,7 @@ class MapValues(StepToDict, MLRunStep):
 
     def __init__(
         self,
-        mapping: Dict[str, Dict[str, Any]],
+        mapping: Dict[str, Dict[Union[str, int, bool], Any]],
         with_original_features: bool = False,
         suffix: str = "mapped",
         **kwargs,
@@ -226,34 +247,130 @@ class MapValues(StepToDict, MLRunStep):
     def _do_spark(self, event):
         from itertools import chain
 
-        from pyspark.sql.functions import col, create_map, lit, when
+        from pyspark.sql.functions import col, create_map, isnan, isnull, lit, when
+        from pyspark.sql.types import DecimalType, DoubleType, FloatType
+        from pyspark.sql.utils import AnalysisException
 
+        df = event
+        source_column_names = df.columns
         for column, column_map in self.mapping.items():
             new_column_name = self._get_feature_name(column)
-            if "ranges" not in column_map:
+            if not self.get_ranges_key() in column_map:
+                if column not in source_column_names:
+                    continue
                 mapping_expr = create_map([lit(x) for x in chain(*column_map.items())])
-                event = event.withColumn(
-                    new_column_name, mapping_expr.getItem(col(column))
-                )
+                try:
+                    df = df.withColumn(
+                        new_column_name,
+                        when(
+                            col(column).isin(list(column_map.keys())),
+                            mapping_expr.getItem(col(column)),
+                        ).otherwise(col(column)),
+                    )
+                #  if failed to use otherwise it is probably because the new column has different type
+                #  then the original column.
+                #  we will try to replace the values without using 'otherwise'.
+                except AnalysisException:
+                    df = df.withColumn(
+                        new_column_name, mapping_expr.getItem(col(column))
+                    )
+                    col_type = df.schema[column].dataType
+                    new_col_type = df.schema[new_column_name].dataType
+                    #  in order to avoid exception at isna on non-decimal/float columns -
+                    #  we need to check their types before filtering.
+                    if isinstance(col_type, (FloatType, DoubleType, DecimalType)):
+                        column_filter = (~isnull(col(column))) & (~isnan(col(column)))
+                    else:
+                        column_filter = ~isnull(col(column))
+                    if isinstance(new_col_type, (FloatType, DoubleType, DecimalType)):
+                        new_column_filter = isnull(col(new_column_name)) | isnan(
+                            col(new_column_name)
+                        )
+                    else:
+                        #  we need to check that every value replaced if we changed column type - except None or NaN.
+                        new_column_filter = isnull(col(new_column_name))
+                    mapping_to_null = [
+                        k
+                        for k, v in column_map.items()
+                        if v is None
+                        or (
+                            isinstance(v, (float, np.float64, np.float32, np.float16))
+                            and math.isnan(v)
+                        )
+                    ]
+                    turned_to_none_values = df.filter(
+                        column_filter & new_column_filter
+                    ).filter(~col(column).isin(mapping_to_null))
+
+                    if len(turned_to_none_values.head(1)) > 0:
+                        raise mlrun.errors.MLRunInvalidArgumentError(
+                            f"MapValues - mapping that changes column type must change all values accordingly,"
+                            f" which is not the case for column '{column}'"
+                        )
             else:
                 for val, val_range in column_map["ranges"].items():
                     min_val = val_range[0] if val_range[0] != "-inf" else -np.inf
                     max_val = val_range[1] if val_range[1] != "inf" else np.inf
                     otherwise = ""
-                    if new_column_name in event.columns:
-                        otherwise = event[new_column_name]
-                    event = event.withColumn(
+                    if new_column_name in df.columns:
+                        otherwise = df[new_column_name]
+                    df = df.withColumn(
                         new_column_name,
                         when(
-                            (event[column] < max_val) & (event[column] >= min_val),
+                            (df[column] < max_val) & (df[column] >= min_val),
                             lit(val),
                         ).otherwise(otherwise),
                     )
 
         if not self.with_original_features:
-            event = event.select(*self.mapping.keys())
+            df = df.select(*self.mapping.keys())
 
-        return event
+        return df
+
+    @classmethod
+    def validate_args(cls, feature_set, **kwargs):
+        mapping = kwargs.get("mapping", [])
+        for column, column_map in mapping.items():
+            if not cls.get_ranges_key() in column_map:
+                types = set(
+                    type(val)
+                    for val in column_map.values()
+                    if type(val) is not None
+                    and not (
+                        isinstance(val, (float, np.float64, np.float32, np.float16))
+                        and math.isnan(val)
+                    )
+                )
+            else:
+                if len(column_map) > 1:
+                    raise mlrun.errors.MLRunInvalidArgumentError(
+                        f"MapValues - mapping values of the same column can not combine ranges and "
+                        f"single replacement, which is the case for column '{column}'"
+                    )
+                ranges_dict = column_map[cls.get_ranges_key()]
+                types = set()
+                for ranges_mapping_values in ranges_dict.values():
+                    range_types = set(
+                        type(val)
+                        for val in ranges_mapping_values
+                        if type(val) is not None
+                        and val != "-inf"
+                        and val != "inf"
+                        and not (
+                            isinstance(val, (float, np.float64, np.float32, np.float16))
+                            and math.isnan(val)
+                        )
+                    )
+                    types = types.union(range_types)
+            if len(types) > 1:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"MapValues - mapping values of the same column must be in the"
+                    f" same type, which was not the case for Column '{column}'"
+                )
+
+    @staticmethod
+    def get_ranges_key():
+        return "ranges"
 
 
 class Imputer(StepToDict, MLRunStep):

mlrun/features.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ import math
 import re
 from typing import Dict, List, Optional, Union
 
-from .data_types import ValueType
+from .data_types import ValueType, python_type_to_value_type
 from .errors import MLRunRuntimeError, err_to_str
 from .model import ModelObj
 
@@ -105,7 +105,12 @@ class Feature(ModelObj):
         :param labels:      a set of key/value labels (tags)
         """
         self.name = name or ""
-        self.value_type = ValueType(value_type) if value_type else ValueType.STRING
+        if isinstance(value_type, ValueType):
+            self.value_type = value_type
+        elif value_type is not None:
+            self.value_type = python_type_to_value_type(value_type)
+        else:
+            self.value_type = ValueType.STRING
         self.dims = dims
         self.description = description
         self.default = default

mlrun/frameworks/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_common/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_common/artifacts_library.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_common/mlrun_interface.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_common/model_handler.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_common/plan.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_common/producer.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_common/utils.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_dl_common/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_dl_common/loggers/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_dl_common/loggers/logger.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_dl_common/loggers/mlrun_logger.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_dl_common/model_handler.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_dl_common/utils.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/artifacts_library.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/loggers/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/loggers/logger.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/loggers/mlrun_logger.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/model_handler.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/pkl_model_server.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from typing import Any, Dict
+
 import numpy as np
 import pandas as pd
 from cloudpickle import load
@@ -56,3 +58,13 @@ class PickleModelServer(V2ModelServer):
         y_pred: np.ndarray = self.model.predict(x)
 
         return y_pred.tolist()
+
+    def explain(self, request: Dict[str, Any]) -> str:
+        """
+        Returns a string listing the model that is being served in this serving function and the function name.
+
+        :param request: A given request.
+
+        :return: Explanation string.
+        """
+        return f"A model server named '{self.name}'"

mlrun/frameworks/_ml_common/plan.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/plans/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,7 +33,6 @@ class CalibrationCurvePlan(MLPlotPlan):
 
     def __init__(
         self,
-        normalize: bool = False,
         n_bins: int = 5,
         strategy: str = "uniform",
     ):
@@ -43,14 +42,11 @@ class CalibrationCurvePlan(MLPlotPlan):
         To read more about the parameters, head to the SciKit-Learn docs at:
         https://scikit-learn.org/stable/modules/generated/sklearn.calibration.calibration_curve.html
 
-        :param normalize: Whether the probabilities needs to be normalized into the [0, 1] interval, i.e. is not a
-                          proper probability.
         :param n_bins:    Number of bins to discretize the [0, 1] interval.
         :param strategy:  Strategy used to define the widths of the bins. Can be on of {‘uniform’, ‘quantile’}.
                           Default: "uniform".
         """
         # Store the parameters:
-        self._normalize = normalize
         self._n_bins = n_bins
         self._strategy = strategy
 
@@ -94,7 +90,6 @@ class CalibrationCurvePlan(MLPlotPlan):
             y,
             y_pred[:, -1],  # Take only the second class probabilities (1, not 0).
             n_bins=self._n_bins,
-            normalize=self._normalize,
             strategy=self._strategy,
         )
 

mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/plans/dataset_plan.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/plans/feature_importance_plan.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/plans/roc_curve_plan.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/producer.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/_ml_common/utils.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/auto_mlrun/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/auto_mlrun/auto_mlrun.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/huggingface/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/huggingface/model_server.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/callbacks/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/callbacks/callback.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/callbacks/logging_callback.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/model_handler.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/model_server.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/lgbm/utils.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/frameworks/onnx/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.