mlrun 1.7.0rc42__py3-none-any.whl → 1.7.0rc44__py3-none-any.whl

mlrun/frameworks/_common/plan.py

@@ -11,12 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
 from abc import ABC, abstractmethod
 
 import mlrun
 from mlrun.artifacts import Artifact
-from mlrun.utils.helpers import is_ipython
+from mlrun.utils.helpers import is_jupyter
 
 
 class Plan(ABC):
@@ -84,7 +84,7 @@ class Plan(ABC):
             return
 
         # Call the correct display method according to the kernel:
-        if is_ipython:
+        if is_jupyter:
             self._gui_display()
         else:
             self._cli_display()

mlrun/frameworks/_ml_common/plan.py

@@ -16,7 +16,7 @@ import json
 from abc import ABC, abstractmethod
 from enum import Enum
 
-from IPython.core.display import HTML, display
+from IPython.display import HTML, display
 
 import mlrun
 

mlrun/frameworks/parallel_coordinates.py

@@ -18,8 +18,7 @@ from typing import Union
 
 import numpy as np
 import pandas as pd
-from IPython.core.display import HTML
-from IPython.display import display
+from IPython.display import HTML, display
 from pandas.api.types import is_numeric_dtype, is_string_dtype
 
 import mlrun
@@ -216,7 +215,7 @@ def _show_and_export_html(html: str, show=None, filename=None, runs_list=None):
                 fp.write("</body></html>")
             else:
                 fp.write(html)
-    if show or (show is None and mlrun.utils.is_ipython):
+    if show or (show is None and mlrun.utils.is_jupyter):
         display(HTML(html))
         if runs_list and len(runs_list) <= max_table_rows:
             display(HTML(html_table))

mlrun/k8s_utils.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import re
+import warnings
 
 import kubernetes.client
 
@@ -133,7 +134,7 @@ def sanitize_label_value(value: str) -> str:
     return re.sub(r"([^a-zA-Z0-9_.-]|^[^a-zA-Z0-9]|[^a-zA-Z0-9]$)", "-", value[:63])
 
 
-def verify_label_key(key: str):
+def verify_label_key(key: str, allow_k8s_prefix: bool = False):
     """
     Verify that the label key is valid for Kubernetes.
     Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
@@ -146,6 +147,10 @@ def verify_label_key(key: str):
         name = parts[0]
     elif len(parts) == 2:
         prefix, name = parts
+        if len(name) == 0:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Label key name cannot be empty when a prefix is set"
+            )
         if len(prefix) == 0:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Label key prefix cannot be empty"
@@ -173,7 +178,13 @@ def verify_label_key(key: str):
         mlrun.utils.regex.qualified_name,
     )
 
-    if key.startswith("k8s.io/") or key.startswith("kubernetes.io/"):
+    # Allow the use of Kubernetes reserved prefixes ('k8s.io/' or 'kubernetes.io/')
+    # only when setting node selectors, not when adding new labels.
+    if (
+        key.startswith("k8s.io/")
+        or key.startswith("kubernetes.io/")
+        and not allow_k8s_prefix
+    ):
         raise mlrun.errors.MLRunInvalidArgumentError(
             "Labels cannot start with 'k8s.io/' or 'kubernetes.io/'"
         )
@@ -185,3 +196,38 @@ def verify_label_value(value, label_key):
         value,
         mlrun.utils.regex.label_value,
     )
+
+
+def validate_node_selectors(
+    node_selectors: dict[str, str], raise_on_error: bool = True
+) -> bool:
+    """
+    Ensures that user-defined node selectors adhere to Kubernetes label standards:
+    - Validates that each key conforms to Kubernetes naming conventions, with specific rules for name and prefix.
+    - Ensures values comply with Kubernetes label value rules.
+    - If raise_on_error is True, raises errors for invalid selectors.
+    - If raise_on_error is False, logs warnings for invalid selectors.
+    """
+
+    # Helper function for handling errors or warnings
+    def handle_invalid(message):
+        if raise_on_error:
+            raise
+        else:
+            warnings.warn(
+                f"{message}\n"
+                f"The node selector you’ve set does not meet the validation rules for the current Kubernetes version. "
+                f"Please note that invalid node selectors may cause issues with function scheduling."
+            )
+
+    node_selectors = node_selectors or {}
+    for key, value in node_selectors.items():
+        try:
+            verify_label_key(key, allow_k8s_prefix=True)
+            verify_label_value(value, label_key=key)
+        except mlrun.errors.MLRunInvalidArgumentError as err:
+            # An error or warning is raised by handle_invalid due to validation failure.
+            # Returning False indicates validation failed, allowing us to exit the function.
+            handle_invalid(str(err))
+            return False
+    return True

mlrun/launcher/client.py

@@ -14,7 +14,7 @@
 import abc
 from typing import Optional
 
-import IPython
+import IPython.display
 
 import mlrun.common.constants as mlrun_constants
 import mlrun.errors
@@ -22,7 +22,7 @@ import mlrun.launcher.base as launcher
 import mlrun.lists
 import mlrun.model
 import mlrun.runtimes
-from mlrun.utils import logger
+import mlrun.utils
 
 
 class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
@@ -128,10 +128,10 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
         if result:
             results_tbl.append(result)
         else:
-            logger.info("no returned result (job may still be in progress)")
+            mlrun.utils.logger.info("no returned result (job may still be in progress)")
             results_tbl.append(run.to_dict())
 
-        if mlrun.utils.is_ipython and mlrun.mlconf.ipython_widget:
+        if mlrun.utils.is_jupyter and mlrun.mlconf.ipython_widget:
             results_tbl.show()
             print()
             ui_url = mlrun.utils.get_ui_url(project, uid)
@@ -147,9 +147,9 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
             project_flag = f"-p {project}" if project else ""
             info_cmd = f"mlrun get run {uid} {project_flag}"
             logs_cmd = f"mlrun logs {uid} {project_flag}"
-            logger.info(
+            mlrun.utils.logger.info(
                 "To track results use the CLI", info_cmd=info_cmd, logs_cmd=logs_cmd
             )
             ui_url = mlrun.utils.get_ui_url(project, uid)
             if ui_url:
-                logger.info("Or click for UI", ui_url=ui_url)
+                mlrun.utils.logger.info("Or click for UI", ui_url=ui_url)

mlrun/model.py

@@ -681,7 +681,8 @@ class ImageBuilder(ModelObj):
 class Notification(ModelObj):
     """Notification object
 
-    :param kind: notification implementation kind - slack, webhook, etc.
+    :param kind: notification implementation kind - slack, webhook, etc. See
+        :py:class:`mlrun.common.schemas.notification.NotificationKind`
     :param name: for logging and identification
     :param message: message content in the notification
     :param severity: severity to display in the notification

mlrun/model_monitoring/controller.py

@@ -219,7 +219,7 @@ class _BatchWindowGenerator:
             # If the endpoint does not have a stream, `last_updated` should be
             # the minimum between the current time and the last updated time.
             # This compensates for the bumping mechanism - see
-            # `bump_model_endpoint_last_request`.
+            # `update_model_endpoint_last_request`.
             last_updated = min(int(datetime_now().timestamp()), last_updated)
             logger.debug(
                 "The endpoint does not have a stream", last_updated=last_updated

mlrun/model_monitoring/db/stores/sqldb/sql_store.py

@@ -588,7 +588,11 @@ class SQLStoreBase(StoreBase):
 
         for endpoint_dict in endpoints:
             endpoint_id = endpoint_dict[mm_schemas.EventFieldType.UID]
-
+            logger.debug(
+                "Deleting model endpoint resources from the SQL tables",
+                endpoint_id=endpoint_id,
+                project=self.project,
+            )
             # Delete last analyzed records
             self._delete_last_analyzed(endpoint_id=endpoint_id)
 
@@ -598,6 +602,16 @@ class SQLStoreBase(StoreBase):
 
             # Delete model endpoint record
             self.delete_model_endpoint(endpoint_id=endpoint_id)
+            logger.debug(
+                "Successfully deleted model endpoint resources",
+                endpoint_id=endpoint_id,
+                project=self.project,
+            )
+
+        logger.debug(
+            "Successfully deleted model monitoring endpoints resources from the SQL tables",
+            project=self.project,
+        )
 
     def get_model_endpoint_metrics(
         self, endpoint_id: str, type: mm_schemas.ModelEndpointMonitoringMetricType

mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py

@@ -305,10 +305,22 @@ class KVStoreBase(StoreBase):
                 endpoint_id = endpoint_dict[mm_schemas.EventFieldType.ENDPOINT_ID]
             else:
                 endpoint_id = endpoint_dict[mm_schemas.EventFieldType.UID]
+
+            logger.debug(
+                "Deleting model endpoint resources from the V3IO KV table",
+                endpoint_id=endpoint_id,
+                project=self.project,
+            )
+
             self.delete_model_endpoint(
                 endpoint_id,
             )
 
+        logger.debug(
+            "Successfully deleted model monitoring endpoints from the V3IO KV table",
+            project=self.project,
+        )
+
         # Delete remain records in the KV
         all_records = self.client.kv.new_cursor(
             container=self.container,

mlrun/model_monitoring/db/tsdb/tdengine/schemas.py

@@ -163,8 +163,8 @@ class TDEngineSchema:
     @staticmethod
     def _get_records_query(
         table: str,
-        start: datetime,
-        end: datetime,
+        start: datetime.datetime,
+        end: datetime.datetime,
         columns_to_filter: list[str] = None,
         filter_query: Optional[str] = None,
         interval: Optional[str] = None,

mlrun/model_monitoring/helpers.py

@@ -63,7 +63,6 @@ def get_stream_path(
     )
 
     if not stream_uri or stream_uri == "v3io":
-        # TODO : remove the first part of this condition in 1.9.0
         stream_uri = mlrun.mlconf.get_model_monitoring_file_target_path(
             project=project,
             kind=mm_constants.FileTargetKind.STREAM,
@@ -71,8 +70,6 @@ def get_stream_path(
             function_name=function_name,
         )
 
-    if isinstance(stream_uri, list):  # ML-6043 - user side gets only the new stream uri
-        stream_uri = stream_uri[1]  # get new stream path, under projects
     return mlrun.common.model_monitoring.helpers.parse_monitoring_stream_path(
         stream_uri=stream_uri, project=project, function_name=function_name
     )
@@ -179,7 +176,7 @@ def _get_monitoring_time_window_from_controller_run(
 def update_model_endpoint_last_request(
     project: str,
     model_endpoint: ModelEndpoint,
-    current_request: datetime,
+    current_request: datetime.datetime,
     db: "RunDBInterface",
 ) -> None:
     """
@@ -190,7 +187,8 @@ def update_model_endpoint_last_request(
     :param current_request: current request time
     :param db:              DB interface.
     """
-    if model_endpoint.spec.stream_path != "":
+    is_model_server_endpoint = model_endpoint.spec.stream_path != ""
+    if is_model_server_endpoint:
         current_request = current_request.isoformat()
         logger.info(
             "Update model endpoint last request time (EP with serving)",
@@ -204,12 +202,13 @@ def update_model_endpoint_last_request(
             endpoint_id=model_endpoint.metadata.uid,
             attributes={mm_constants.EventFieldType.LAST_REQUEST: current_request},
         )
-    else:
+    else:  # model endpoint without any serving function - close the window "manually"
         try:
             time_window = _get_monitoring_time_window_from_controller_run(project, db)
         except mlrun.errors.MLRunNotFoundError:
-            logger.debug(
-                "Not bumping model endpoint last request time - the monitoring controller isn't deployed yet"
+            logger.warn(
+                "Not bumping model endpoint last request time - the monitoring controller isn't deployed yet.\n"
+                "Call `project.enable_model_monitoring()` first."
             )
             return
 
@@ -265,13 +264,6 @@ def calculate_inputs_statistics(
                 counts.tolist(),
                 bins.tolist(),
             ]
-        elif "hist" in inputs_statistics[feature]:
-            # Comply with the other common features' histogram length
-            mlrun.common.model_monitoring.helpers.pad_hist(
-                mlrun.common.model_monitoring.helpers.Histogram(
-                    inputs_statistics[feature]["hist"]
-                )
-            )
         else:
             # If the feature is not in the sample set and doesn't have a histogram, remove it from the statistics:
             inputs_statistics.pop(feature)

mlrun/model_monitoring/writer.py

@@ -160,7 +160,9 @@ class ModelMonitoringWriter(StepToDict):
             event_kind = f"{event_kind}_detected"
         else:
             event_kind = f"{event_kind}_suspected"
-        return alert_objects.EventKind(value=event_kind)
+        return alert_objects.EventKind(
+            value=mlrun.utils.helpers.normalize_name(event_kind)
+        )
 
     @staticmethod
     def _reconstruct_event(event: _RawEvent) -> tuple[_AppResultEvent, WriterEventKind]:
@@ -258,9 +260,13 @@ class ModelMonitoringWriter(StepToDict):
                 "data drift app",
                 endpoint_id=endpoint_id,
             )
+            attributes = json.loads(event[ResultData.RESULT_EXTRA_DATA])
+            attributes[EventFieldType.DRIFT_STATUS] = str(
+                attributes[EventFieldType.DRIFT_STATUS]
+            )
             self._app_result_store.update_model_endpoint(
                 endpoint_id=endpoint_id,
-                attributes=json.loads(event[ResultData.RESULT_EXTRA_DATA]),
+                attributes=attributes,
             )
 
         logger.info("Model monitoring writer finished handling event")

mlrun/projects/pipelines.py

@@ -80,6 +80,7 @@ class WorkflowSpec(mlrun.model.ModelObj):
         schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
         cleanup_ttl: typing.Optional[int] = None,
         image: typing.Optional[str] = None,
+        workflow_runner_node_selector: typing.Optional[dict[str, str]] = None,
     ):
         self.engine = engine
         self.code = code
@@ -93,6 +94,7 @@ class WorkflowSpec(mlrun.model.ModelObj):
         self._tmp_path = None
         self.schedule = schedule
         self.image = image
+        self.workflow_runner_node_selector = workflow_runner_node_selector
 
     def get_source_file(self, context=""):
         if not self.code and not self.path: