mlrun 1.7.0rc6__py3-none-any.whl → 1.7.0rc9__py3-none-any.whl

mlrun/feature_store/retrieval/spark_merger.py

@@ -24,6 +24,32 @@ from .base import BaseMerger
 from .conversion import PandasConversionMixin
 
 
+def spark_df_to_pandas(spark_df):
+    # as of pyspark 3.2.3, toPandas fails to convert timestamps unless we work around the issue
+    # when we upgrade pyspark, we should check whether this workaround is still necessary
+    # see https://stackoverflow.com/questions/76389694/transforming-pyspark-to-pandas-dataframe
+    if semver.parse(pd.__version__)["major"] >= 2:
+        import pyspark.sql.functions as pyspark_functions
+
+        type_conversion_dict = {}
+        for field in spark_df.schema.fields:
+            if str(field.dataType) == "TimestampType":
+                spark_df = spark_df.withColumn(
+                    field.name,
+                    pyspark_functions.date_format(
+                        pyspark_functions.to_timestamp(field.name),
+                        "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS",
+                    ),
+                )
+                type_conversion_dict[field.name] = "datetime64[ns]"
+        df = PandasConversionMixin.toPandas(spark_df)
+        if type_conversion_dict:
+            df = df.astype(type_conversion_dict)
+        return df
+    else:
+        return PandasConversionMixin.toPandas(spark_df)
+
+
 class SparkFeatureMerger(BaseMerger):
     engine = "spark"
     support_offline = True
@@ -166,29 +192,7 @@ class SparkFeatureMerger(BaseMerger):
     def get_df(self, to_pandas=True):
         if to_pandas:
             if self._pandas_df is None:
-                df = self._result_df
-                # as of pyspark 3.2.3, toPandas fails to convert timestamps unless we work around the issue
-                # when we upgrade pyspark, we should check whether this workaround is still necessary
-                # see https://stackoverflow.com/questions/76389694/transforming-pyspark-to-pandas-dataframe
-                if semver.parse(pd.__version__)["major"] >= 2:
-                    import pyspark.sql.functions as pyspark_functions
-
-                    type_conversion_dict = {}
-                    for field in df.schema.fields:
-                        if str(field.dataType) == "TimestampType":
-                            df = df.withColumn(
-                                field.name,
-                                pyspark_functions.date_format(
-                                    pyspark_functions.to_timestamp(field.name),
-                                    "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS",
-                                ),
-                            )
-                            type_conversion_dict[field.name] = "datetime64[ns]"
-                    df = PandasConversionMixin.toPandas(df)
-                    if type_conversion_dict:
-                        df = df.astype(type_conversion_dict)
-                else:
-                    df = PandasConversionMixin.toPandas(df)
+                df = spark_df_to_pandas(self._result_df)
                 self._pandas_df = df
                 self._set_indexes(self._pandas_df)
             return self._pandas_df

mlrun/frameworks/tf_keras/callbacks/logging_callback.py

@@ -17,7 +17,7 @@ from typing import Callable, Union
 import numpy as np
 import tensorflow as tf
 from tensorflow import Tensor, Variable
-from tensorflow.keras.callbacks import Callback
+from tensorflow.python.keras.callbacks import Callback
 
 import mlrun
 

mlrun/frameworks/tf_keras/mlrun_interface.py

@@ -19,7 +19,8 @@ from typing import Union
 
 import tensorflow as tf
 from tensorflow import keras
-from tensorflow.keras.callbacks import (
+from tensorflow.keras.optimizers import Optimizer
+from tensorflow.python.keras.callbacks import (
     BaseLogger,
     Callback,
     CSVLogger,
@@ -27,7 +28,6 @@ from tensorflow.keras.callbacks import (
     ProgbarLogger,
     TensorBoard,
 )
-from tensorflow.keras.optimizers import Optimizer
 
 import mlrun
 

mlrun/kfpops.py

@@ -103,7 +103,7 @@ def write_kfpmeta(struct):
             with open(path, "w") as fp:
                 fp.write(str(val))
         except Exception as exc:
-            logger.warning("Failed writing to temp file. Ignoring", exc=repr(exc))
+            logger.warning("Failed writing to temp file. Ignoring", exc=err_to_str(exc))
             pass
 
     text = "# Run Report\n"
@@ -112,10 +112,7 @@ def write_kfpmeta(struct):
 
     text += "## Metadata\n```yaml\n" + dict_to_yaml(struct) + "```\n"
 
-    metadata = {
-        "outputs": output_artifacts
-        + [{"type": "markdown", "storage": "inline", "source": text}]
-    }
+    metadata = {"outputs": [{"type": "markdown", "storage": "inline", "source": text}]}
     with open(os.path.join(KFPMETA_DIR, "mlpipeline-ui-metadata.json"), "w") as f:
         json.dump(metadata, f)
 

mlrun/launcher/base.py

@@ -353,7 +353,7 @@ class BaseLauncher(abc.ABC):
             or {}
         )
         state_thresholds = (
-            mlrun.config.config.function.spec.state_thresholds.default.to_dict()
+            mlrun.mlconf.function.spec.state_thresholds.default.to_dict()
             | state_thresholds
         )
         run.spec.state_thresholds = state_thresholds or run.spec.state_thresholds

mlrun/launcher/client.py

@@ -47,7 +47,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
         If build is needed, set the image as the base_image for the build.
         If image is not given set the default one.
         """
-        if runtime.kind in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
+        if runtime.kind in mlrun.runtimes.RuntimeKinds.pure_nuclio_deployed_runtimes():
             return
 
         require_build = runtime.requires_build()
@@ -129,7 +129,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
             logger.info("no returned result (job may still be in progress)")
             results_tbl.append(run.to_dict())
 
-        if mlrun.utils.is_ipython and mlrun.config.config.ipython_widget:
+        if mlrun.utils.is_ipython and mlrun.mlconf.ipython_widget:
             results_tbl.show()
             print()
             ui_url = mlrun.utils.get_ui_url(project, uid)

mlrun/model.py

@@ -71,6 +71,7 @@ class ModelObj:
             return new_type.from_dict(param)
         return param
 
+    @mlrun.utils.filter_warnings("ignore", FutureWarning)
     def to_dict(
         self, fields: list = None, exclude: list = None, strip: bool = False
     ) -> dict:

mlrun/model_monitoring/__init__.py

@@ -15,7 +15,7 @@
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 # for backwards compatibility
 
+from .db import get_store_object
 from .helpers import get_stream_path
 from .model_endpoint import ModelEndpoint
-from .stores import ModelEndpointStore, ModelEndpointStoreType, get_model_endpoint_store
 from .tracking_policy import TrackingPolicy