mlrun 1.7.0rc7__py3-none-any.whl → 1.7.0rc11__py3-none-any.whl

mlrun/frameworks/pytorch/__init__.py

@@ -112,7 +112,7 @@ def train(
 
                                             {
                                                 "/.../custom_optimizer.py": "optimizer",
-                                                "/.../custom_layers.py": ["layer1", "layer2"]
+                                                "/.../custom_layers.py": ["layer1", "layer2"],
                                             }
 
                                         All the paths will be accessed from the given 'custom_objects_directory',
@@ -264,7 +264,7 @@ def evaluate(
 
                                          {
                                              "/.../custom_optimizer.py": "optimizer",
-                                             "/.../custom_layers.py": ["layer1", "layer2"]
+                                             "/.../custom_layers.py": ["layer1", "layer2"],
                                          }
 
                                      All the paths will be accessed from the given 'custom_objects_directory', meaning

mlrun/frameworks/sklearn/__init__.py

@@ -92,7 +92,7 @@ def apply_mlrun(
 
                                          {
                                              "/.../custom_model.py": "MyModel",
-                                             "/.../custom_objects.py": ["object1", "object2"]
+                                             "/.../custom_objects.py": ["object1", "object2"],
                                          }
 
                                      All the paths will be accessed from the given 'custom_objects_directory', meaning

mlrun/frameworks/tf_keras/__init__.py

@@ -85,7 +85,7 @@ def apply_mlrun(
 
                                             {
                                                 "/.../custom_optimizer.py": "optimizer",
-                                                "/.../custom_layers.py": ["layer1", "layer2"]
+                                                "/.../custom_layers.py": ["layer1", "layer2"],
                                             }
 
                                         All the paths will be accessed from the given 'custom_objects_directory',

mlrun/frameworks/tf_keras/callbacks/logging_callback.py

@@ -17,7 +17,7 @@ from typing import Callable, Union
 import numpy as np
 import tensorflow as tf
 from tensorflow import Tensor, Variable
-from tensorflow.keras.callbacks import Callback
+from tensorflow.python.keras.callbacks import Callback
 
 import mlrun
 

mlrun/frameworks/tf_keras/mlrun_interface.py

@@ -19,7 +19,8 @@ from typing import Union
 
 import tensorflow as tf
 from tensorflow import keras
-from tensorflow.keras.callbacks import (
+from tensorflow.keras.optimizers import Optimizer
+from tensorflow.python.keras.callbacks import (
     BaseLogger,
     Callback,
     CSVLogger,
@@ -27,7 +28,6 @@ from tensorflow.keras.callbacks import (
     ProgbarLogger,
     TensorBoard,
 )
-from tensorflow.keras.optimizers import Optimizer
 
 import mlrun
 

mlrun/frameworks/xgboost/__init__.py

@@ -90,7 +90,7 @@ def apply_mlrun(
 
                                          {
                                              "/.../custom_model.py": "MyModel",
-                                             "/.../custom_objects.py": ["object1", "object2"]
+                                             "/.../custom_objects.py": ["object1", "object2"],
                                          }
 
                                      All the paths will be accessed from the given 'custom_objects_directory', meaning

mlrun/kfpops.py

@@ -103,7 +103,7 @@ def write_kfpmeta(struct):
             with open(path, "w") as fp:
                 fp.write(str(val))
         except Exception as exc:
-            logger.warning("Failed writing to temp file. Ignoring", exc=repr(exc))
+            logger.warning("Failed writing to temp file. Ignoring", exc=err_to_str(exc))
             pass
 
     text = "# Run Report\n"
@@ -112,10 +112,7 @@ def write_kfpmeta(struct):
 
     text += "## Metadata\n```yaml\n" + dict_to_yaml(struct) + "```\n"
 
-    metadata = {
-        "outputs": output_artifacts
-        + [{"type": "markdown", "storage": "inline", "source": text}]
-    }
+    metadata = {"outputs": [{"type": "markdown", "storage": "inline", "source": text}]}
     with open(os.path.join(KFPMETA_DIR, "mlpipeline-ui-metadata.json"), "w") as f:
         json.dump(metadata, f)
 

mlrun/launcher/base.py

@@ -353,7 +353,7 @@ class BaseLauncher(abc.ABC):
             or {}
         )
         state_thresholds = (
-            mlrun.config.config.function.spec.state_thresholds.default.to_dict()
+            mlrun.mlconf.function.spec.state_thresholds.default.to_dict()
             | state_thresholds
         )
         run.spec.state_thresholds = state_thresholds or run.spec.state_thresholds

mlrun/launcher/client.py

@@ -47,7 +47,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
         If build is needed, set the image as the base_image for the build.
         If image is not given set the default one.
         """
-        if runtime.kind in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
+        if runtime.kind in mlrun.runtimes.RuntimeKinds.pure_nuclio_deployed_runtimes():
             return
 
         require_build = runtime.requires_build()
@@ -129,7 +129,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
             logger.info("no returned result (job may still be in progress)")
             results_tbl.append(run.to_dict())
 
-        if mlrun.utils.is_ipython and mlrun.config.config.ipython_widget:
+        if mlrun.utils.is_ipython and mlrun.mlconf.ipython_widget:
             results_tbl.show()
             print()
             ui_url = mlrun.utils.get_ui_url(project, uid)

mlrun/model.py

@@ -931,7 +931,7 @@ class RunSpec(ModelObj):
 
         >>> run_spec.inputs = {
         ...     "my_input": "...",
-        ...     "my_hinted_input : pandas.DataFrame": "..."
+        ...     "my_hinted_input : pandas.DataFrame": "...",
         ... }
 
         :param inputs: The inputs to set.
@@ -1275,7 +1275,7 @@ class RunTemplate(ModelObj):
 
         example::
 
-            grid_params = {"p1": [2,4,1], "p2": [10,20]}
+            grid_params = {"p1": [2, 4, 1], "p2": [10, 20]}
             task = mlrun.new_task("grid-search")
             task.with_hyper_params(grid_params, selector="max.accuracy")
         """

mlrun/model_monitoring/application.py

@@ -93,7 +93,11 @@ class ModelMonitoringApplicationBase(StepToDict, ABC):
                 endpoint_id: str,
                 output_stream_uri: str,
             ) -> ModelMonitoringApplicationResult:
-                self.context.log_artifact(TableArtifact("sample_df_stats", df=self.dict_to_histogram(sample_df_stats)))
+                self.context.log_artifact(
+                    TableArtifact(
+                        "sample_df_stats", df=self.dict_to_histogram(sample_df_stats)
+                    )
+                )
                 return ModelMonitoringApplicationResult(
                     name="data_drift_test",
                     value=0.5,
@@ -101,6 +105,7 @@ class ModelMonitoringApplicationBase(StepToDict, ABC):
                     status=mm_constant.ResultStatusApp.detected,
                 )
 
+
         # mlrun: end-code
     """
 
@@ -203,7 +208,11 @@ class ModelMonitoringApplicationBase(StepToDict, ABC):
             json.loads(event[mm_constant.ApplicationEvent.FEATURE_STATS]),
             ParquetTarget(
                 path=event[mm_constant.ApplicationEvent.SAMPLE_PARQUET_PATH]
-            ).as_df(start_time=start_time, end_time=end_time, time_column="timestamp"),
+            ).as_df(
+                start_time=start_time,
+                end_time=end_time,
+                time_column=mm_constant.FeatureSetFeatures.time_stamp(),
+            ),
             start_time,
             end_time,
             pd.Timestamp(event[mm_constant.ApplicationEvent.LAST_REQUEST]),

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -22,8 +22,8 @@ import mlrun.artifacts
 import mlrun.common.model_monitoring.helpers
 import mlrun.model_monitoring.features_drift_table as mm_drift_table
 from mlrun.common.schemas.model_monitoring.constants import (
-    MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME,
     EventFieldType,
+    HistogramDataDriftApplicationConstants,
     ResultKindApp,
     ResultStatusApp,
 )
@@ -94,7 +94,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
     and the status is returned.
     """
 
-    NAME: Final[str] = MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME
+    NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
     METRIC_KIND: Final[ResultKindApp] = ResultKindApp.data_drift
 
     _REQUIRED_METRICS = {HellingerDistance, TotalVarianceDistance}
@@ -148,7 +148,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         status = self._value_classifier.value_to_status(value)
         results.append(
             ModelMonitoringApplicationResult(
-                name="general_drift",
+                name=HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
                 value=value,
                 kind=self.METRIC_KIND,
                 status=status,

mlrun/model_monitoring/controller.py

@@ -354,7 +354,7 @@ class MonitoringApplicationController:
                             app.status.state == "ready"
                             # workaround for the default app, as its `status.state` is `None`
                             or app.metadata.name
-                            == mm_constants.MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME
+                            == mm_constants.HistogramDataDriftApplicationConstants.NAME
                         )
                     }
                 )
@@ -502,8 +502,7 @@ class MonitoringApplicationController:
 
                     # Get the current stats:
                     current_stats = calculate_inputs_statistics(
-                        sample_set_statistics=feature_stats,
-                        inputs=df,
+                        sample_set_statistics=feature_stats, inputs=df
                     )
 
                     cls._push_to_applications(

mlrun/model_monitoring/helpers.py

@@ -42,7 +42,7 @@ class _BatchDict(typing.TypedDict):
 def get_stream_path(
     project: str = None,
     function_name: str = mm_constants.MonitoringFunctionNames.STREAM,
-):
+) -> str:
     """
     Get stream path from the project secret. If wasn't set, take it from the system configurations
 
@@ -61,6 +61,8 @@ def get_stream_path(
         function_name=function_name,
     )
 
+    if isinstance(stream_uri, list):  # ML-6043 - user side gets only the new stream uri
+        stream_uri = stream_uri[1]  # get new stream path, under projects
     return mlrun.common.model_monitoring.helpers.parse_monitoring_stream_path(
         stream_uri=stream_uri, project=project, function_name=function_name
     )

mlrun/model_monitoring/stream_processing.py

@@ -352,7 +352,6 @@ class EventStreamProcessor:
                     rate="10/m",
                     time_col=EventFieldType.TIMESTAMP,
                     container=self.tsdb_container,
-                    access_key=self.v3io_access_key,
                     v3io_frames=self.v3io_framesd,
                     infer_columns_from_data=True,
                     index_cols=[

mlrun/model_monitoring/writer.py

@@ -23,6 +23,7 @@ from v3io_frames.errors import Error as V3IOFramesError
 from v3io_frames.frames_pb2 import IGNORE
 
 import mlrun.common.model_monitoring
+import mlrun.common.schemas.alert as alert_constants
 import mlrun.model_monitoring
 import mlrun.model_monitoring.db.stores
 import mlrun.utils.v3io_clients
@@ -171,6 +172,29 @@ class ModelMonitoringWriter(StepToDict):
                 event=event,
             )
 
+    @staticmethod
+    def _generate_event_on_drift(
+        uid: str, drift_status: str, drift_value: float, project_name: str
+    ):
+        if (
+            drift_status == ResultStatusApp.detected
+            or drift_status == ResultStatusApp.potential_detection
+        ):
+            entity = {
+                "kind": alert_constants.EventEntityKind.MODEL,
+                "project": project_name,
+                "id": uid,
+            }
+            event_kind = (
+                alert_constants.EventKind.DRIFT_DETECTED
+                if drift_status == ResultStatusApp.detected
+                else alert_constants.EventKind.DRIFT_SUSPECTED
+            )
+            event_data = mlrun.common.schemas.Event(
+                kind=event_kind, entity=entity, value=drift_value
+            )
+            mlrun.get_run_db().generate_event(event_kind, event_data)
+
     @staticmethod
     def _reconstruct_event(event: _RawEvent) -> _AppResultEvent:
         """
@@ -201,4 +225,12 @@ class ModelMonitoringWriter(StepToDict):
         self._update_tsdb(event)
         self._update_kv_db(event)
         _Notifier(event=event, notification_pusher=self._custom_notifier).notify()
+
+        if mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.enabled:
+            self._generate_event_on_drift(
+                event[WriterEvent.ENDPOINT_ID],
+                event[WriterEvent.RESULT_STATUS],
+                event[WriterEvent.RESULT_VALUE],
+                self.project,
+            )
         logger.info("Completed event DB writes")

mlrun/package/packagers_manager.py

@@ -92,6 +92,7 @@ class PackagersManager:
             from mlrun import Packager
             from x import XPackager
 
+
             class YPackager(Packager):
                 pass
 

mlrun/platforms/__init__.py

@@ -48,7 +48,7 @@ def watch_stream(
 
     example::
 
-        watch_stream('v3io:///users/admin/mystream')
+        watch_stream("v3io:///users/admin/mystream")
 
     :param url:        stream url
     :param shard_ids:  range or list of shard IDs

mlrun/platforms/other.py

@@ -33,7 +33,7 @@ def mount_pvc(pvc_name=None, volume_name="pipeline", volume_mount_path="/mnt/pip
     Usage::
 
         train = train_op(...)
-        train.apply(mount_pvc('claim-name', 'pipeline', '/mnt/pipeline'))
+        train.apply(mount_pvc("claim-name", "pipeline", "/mnt/pipeline"))
     """
     if "MLRUN_PVC_MOUNT" in os.environ:
         mount = os.environ.get("MLRUN_PVC_MOUNT")

mlrun/projects/operations.py

@@ -95,8 +95,11 @@ def run_function(
         MODEL_CLASS = "sklearn.ensemble.RandomForestClassifier"
         DATA_PATH = "s3://bigdata/data.parquet"
         function = mlrun.import_function("hub://auto-trainer")
-        run1 = run_function(function, params={"label_columns": LABELS, "model_class": MODEL_CLASS},
-                                      inputs={"dataset": DATA_PATH})
+        run1 = run_function(
+            function,
+            params={"label_columns": LABELS, "model_class": MODEL_CLASS},
+            inputs={"dataset": DATA_PATH},
+        )
 
     example (use with project)::
 
@@ -115,8 +118,12 @@ def run_function(
         @dsl.pipeline(name="test pipeline", description="test")
         def my_pipe(url=""):
             run1 = run_function("loaddata", params={"url": url}, outputs=["data"])
-            run2 = run_function("train", params={"label_columns": LABELS, "model_class": MODEL_CLASS},
-                                         inputs={"dataset": run1.outputs["data"]})
+            run2 = run_function(
+                "train",
+                params={"label_columns": LABELS, "model_class": MODEL_CLASS},
+                inputs={"dataset": run1.outputs["data"]},
+            )
+
 
         project.run(workflow_handler=my_pipe, arguments={"param1": 7})
 

mlrun/projects/pipelines.py

@@ -1071,7 +1071,7 @@ def load_and_run(
                 )
 
             except Exception as exc:
-                logger.error("Failed to send slack notification", exc=exc)
+                logger.error("Failed to send slack notification", exc=err_to_str(exc))
 
         raise error