mlrun 1.10.0rc13__py3-none-any.whl → 1.10.0rc15__py3-none-any.whl

mlrun/db/httpdb.py

@@ -757,7 +757,7 @@ class HTTPRunDB(RunDBInterface):
         )
         if response.status_code == http.HTTPStatus.ACCEPTED:
             background_task = mlrun.common.schemas.BackgroundTask(**response.json())
-            return self._wait_for_background_task_to_reach_terminal_state(
+            return self.wait_for_background_task_to_reach_terminal_state(
                 background_task.metadata.name, project=project
             )
         return None
@@ -784,7 +784,7 @@ class HTTPRunDB(RunDBInterface):
         )
         if response.status_code == http.HTTPStatus.ACCEPTED:
             background_task = mlrun.common.schemas.BackgroundTask(**response.json())
-            background_task = self._wait_for_background_task_to_reach_terminal_state(
+            background_task = self.wait_for_background_task_to_reach_terminal_state(
                 background_task.metadata.name, project=project
             )
             if (
@@ -839,7 +839,7 @@ class HTTPRunDB(RunDBInterface):
         )
         if response.status_code == http.HTTPStatus.ACCEPTED:
             background_task = mlrun.common.schemas.BackgroundTask(**response.json())
-            background_task = self._wait_for_background_task_to_reach_terminal_state(
+            background_task = self.wait_for_background_task_to_reach_terminal_state(
                 background_task.metadata.name, project=project
             )
             if (
@@ -1485,7 +1485,7 @@ class HTTPRunDB(RunDBInterface):
                 "Function is being deleted", project_name=project, function_name=name
             )
             background_task = mlrun.common.schemas.BackgroundTask(**response.json())
-            background_task = self._wait_for_background_task_to_reach_terminal_state(
+            background_task = self.wait_for_background_task_to_reach_terminal_state(
                 background_task.metadata.name, project=project
             )
             if (
@@ -3274,7 +3274,7 @@ class HTTPRunDB(RunDBInterface):
         if response.status_code == http.HTTPStatus.ACCEPTED:
             logger.info("Waiting for project to be deleted", project_name=name)
             background_task = mlrun.common.schemas.BackgroundTask(**response.json())
-            background_task = self._wait_for_background_task_to_reach_terminal_state(
+            background_task = self.wait_for_background_task_to_reach_terminal_state(
                 background_task.metadata.name
             )
             if (
@@ -3387,7 +3387,7 @@ class HTTPRunDB(RunDBInterface):
             _verify_project_in_terminal_state,
         )
 
-    def _wait_for_background_task_to_reach_terminal_state(
+    def wait_for_background_task_to_reach_terminal_state(
         self, name: str, project: str = ""
     ) -> mlrun.common.schemas.BackgroundTask:
         def _verify_background_task_in_terminal_state():
@@ -3408,6 +3408,7 @@ class HTTPRunDB(RunDBInterface):
             logger,
             False,
             _verify_background_task_in_terminal_state,
+            fatal_exceptions=(mlrun.errors.MLRunAccessDeniedError,),
         )
 
     def create_project_secrets(
@@ -4082,7 +4083,7 @@ class HTTPRunDB(RunDBInterface):
                 **response.json()
             ).background_tasks
             for task in background_tasks:
-                task = self._wait_for_background_task_to_reach_terminal_state(
+                task = self.wait_for_background_task_to_reach_terminal_state(
                     task.metadata.name, project=project
                 )
                 if (
@@ -4119,7 +4120,7 @@ class HTTPRunDB(RunDBInterface):
                 **response.json()
             ).background_tasks
             for task in background_tasks:
-                task = self._wait_for_background_task_to_reach_terminal_state(
+                task = self.wait_for_background_task_to_reach_terminal_state(
                     task.metadata.name, project=project
                 )
                 if (
@@ -5158,6 +5159,38 @@ class HTTPRunDB(RunDBInterface):
         response = self.api_call("GET", endpoint_path, error_message)
         return mlrun.common.schemas.ProjectSummary(**response.json())
 
+    def get_drift_over_time(
+        self,
+        project: str,
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
+    ) -> mlrun.common.schemas.model_monitoring.ModelEndpointDriftValues:
+        """
+        Get drift counts over time for the project.
+
+        This method returns a list of tuples, each representing a time-interval (in a granularity set by the
+        duration of the given time range) and the number of suspected drifts and detected drifts in that interval.
+        For a range of 6 hours or less, the granularity is 10 minute, for a range of 2 hours to 72 hours, the
+        granularity is 1 hour, and for a range of more than 72 hours, the granularity is 24 hours.
+
+        :param project: The name of the project for which to retrieve drift counts.
+        :param start: Start time of the range to retrieve drift counts from.
+        :param end: End time of the range to retrieve drift counts from.
+
+        :return: A ModelEndpointDriftValues object containing the drift counts over time.
+        """
+        endpoint_path = f"projects/{project}/model-endpoints/drift-over-time"
+        error_message = f"Failed retrieving drift data for {project}"
+        response = self.api_call(
+            method="GET",
+            path=endpoint_path,
+            error=error_message,
+            params={"start": start, "end": end},
+        )
+        return mlrun.common.schemas.model_monitoring.ModelEndpointDriftValues(
+            **response.json()
+        )
+
     @staticmethod
     def _parse_labels(
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]],
@@ -5478,7 +5511,7 @@ class HTTPRunDB(RunDBInterface):
     def _wait_for_background_task_from_response(self, response):
         if response.status_code == http.HTTPStatus.ACCEPTED:
             background_task = mlrun.common.schemas.BackgroundTask(**response.json())
-            return self._wait_for_background_task_to_reach_terminal_state(
+            return self.wait_for_background_task_to_reach_terminal_state(
                 background_task.metadata.name
             )
         return None

mlrun/db/nopdb.py

@@ -980,3 +980,11 @@ class NopDB(RunDBInterface):
 
     def get_project_summary(self, project: str):
         pass
+
+    def get_drift_over_time(
+        self,
+        project: str,
+        start: Optional[datetime.datetime] = None,
+        end: Optional[datetime.datetime] = None,
+    ) -> mlrun.common.schemas.model_monitoring.ModelEndpointDriftValues:
+        pass

mlrun/execution.py

@@ -94,6 +94,7 @@ class MLClientCtx:
         self._state_thresholds = {}
         self._retry_spec = {}
         self._retry_count = None
+        self._retries = []
 
         self._labels = {}
         self._annotations = {}
@@ -468,6 +469,7 @@ class MLClientCtx:
             for key, uri in status.get("artifact_uris", {}).items():
                 self._artifacts_manager.artifact_uris[key] = uri
             self._retry_count = status.get("retry_count", self._retry_count)
+            self._retries = status.get("retries", self._retries)
             # if run is a retry, the state needs to move to running
             if include_status:
                 self._state = status.get("state", self._state)
@@ -911,7 +913,7 @@ class MLClientCtx:
     def log_llm_prompt(
         self,
         key,
-        prompt_string: Optional[str] = None,
+        prompt_template: Optional[list[dict]] = None,
         prompt_path: Optional[str] = None,
         prompt_legend: Optional[dict] = None,
         model_artifact: Union[ModelArtifact, str] = None,
@@ -935,7 +937,7 @@ class MLClientCtx:
             # Log an inline prompt
             context.log_llm_prompt(
                 key="qa-prompt",
-                prompt_string="Q: {question}",
+                prompt_template=[{"role: "user", "content": "question with {place_holder}"}],
                 model_artifact=model,
                 prompt_legend={"question": "user_input"},
                 model_configuration={"temperature": 0.7, "max_tokens": 128},
@@ -943,10 +945,16 @@ class MLClientCtx:
             )
 
         :param key: Unique name of the artifact.
-        :param prompt_string: Raw prompt text as a string. Cannot be used with `prompt_path`.
+        :param prompt_template: Raw prompt list of dicts -
+         [{"role": "system", "content": "You are a {profession} advisor"},
+         "role": "user", "content": "I need your help with {profession}"]. only "role" and "content" keys allow in any
+         str format (upper/lower case), keys will be modified to lower case.
+         Cannot be used with `prompt_path`.
         :param prompt_path: Path to a file containing the prompt content. Cannot be used with `prompt_string`.
         :param prompt_legend: A dictionary where each key is a placeholder in the prompt (e.g., ``{user_name}``)
-               and the value is a description or explanation of what that placeholder represents.
+               and the value is a dictionary holding two keys, "field", "description". "field" points to the field in
+               the event where the value of the place-holder inside the event, if None or not exist will be replaced
+               with the place-holder name. "description" will point to explanation of what that placeholder represents.
                Useful for documenting and clarifying dynamic parts of the prompt.
         :param model_artifact: Reference to the parent model (either `ModelArtifact` or model URI string).
         :param model_configuration: Dictionary of generation parameters (e.g., temperature, max_tokens).
@@ -961,15 +969,15 @@ class MLClientCtx:
         :returns: The logged `LLMPromptArtifact` object.
         """
 
-        if not prompt_string and not prompt_path:
+        if not prompt_template and not prompt_path:
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "Either 'prompt_string' or 'prompt_path' must be provided"
+                "Either 'prompt_template' or 'prompt_path' must be provided"
             )
 
         llm_prompt = LLMPromptArtifact(
             key=key,
             project=self.project or "",
-            prompt_string=prompt_string,
+            prompt_template=prompt_template,
             prompt_path=prompt_path,
             prompt_legend=prompt_legend,
             model_artifact=model_artifact,
@@ -1267,6 +1275,7 @@ class MLClientCtx:
                 "start_time": to_date_str(self._start_time),
                 "last_update": to_date_str(self._last_update),
                 "retry_count": self._retry_count,
+                "retries": self._retries,
             },
         }
 

mlrun/model.py

@@ -1375,6 +1375,7 @@ class RunStatus(ModelObj):
         notifications: Optional[dict[str, Notification]] = None,
         artifact_uris: Optional[dict[str, str]] = None,
         retry_count: Optional[int] = None,
+        retries: Optional[list[dict]] = None,
     ):
         self.state = state or "created"
         self.status_text = status_text
@@ -1393,6 +1394,7 @@ class RunStatus(ModelObj):
         # Artifact key -> URI mapping, since the full artifacts are not stored in the runs DB table
         self._artifact_uris = artifact_uris or {}
         self._retry_count = retry_count or None
+        self._retries = retries or []
 
     @classmethod
     def from_dict(
@@ -1461,6 +1463,19 @@ class RunStatus(ModelObj):
         """
         self._retry_count = retry_count
 
+    @property
+    def retries(self) -> list[dict]:
+        """List of metadata for each retry attempt."""
+        return self._retries
+
+    @retries.setter
+    def retries(self, retries: list[dict]):
+        """
+        Set the list of retry attempt metadata.
+        :param retries: A list of dictionaries, each representing a retry attempt.
+        """
+        self._retries = retries
+
     def is_failed(self) -> Optional[bool]:
         """
         This method returns whether a run has failed.

mlrun/model_monitoring/__init__.py

@@ -15,4 +15,5 @@
 from mlrun.common.schemas import ModelEndpoint, ModelEndpointList
 
 from .db import get_tsdb_connector
+from .db._schedules import delete_model_monitoring_schedules_user_folder
 from .helpers import get_stream_path

mlrun/model_monitoring/applications/base.py

@@ -17,7 +17,7 @@ import socket
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from collections.abc import Iterator
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
 from datetime import datetime, timedelta
 from typing import Any, Literal, Optional, Union, cast
 
@@ -31,6 +31,7 @@ import mlrun.errors
 import mlrun.model_monitoring.api as mm_api
 import mlrun.model_monitoring.applications.context as mm_context
 import mlrun.model_monitoring.applications.results as mm_results
+import mlrun.model_monitoring.db._schedules as mm_schedules
 import mlrun.model_monitoring.helpers as mm_helpers
 from mlrun.serving.utils import MonitoringApplicationToDict
 from mlrun.utils import logger
@@ -183,14 +184,27 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         cls,
         *,
         write_output: bool,
+        application_name: str,
+        artifact_path: str,
         stream_profile: Optional[ds_profile.DatastoreProfile],
         project: "mlrun.MlrunProject",
-    ) -> Iterator[dict[str, list[tuple]]]:
+    ) -> Iterator[
+        tuple[
+            dict[str, list[tuple]],
+            Optional[mm_schedules.ModelMonitoringSchedulesFileApplication],
+        ]
+    ]:
         endpoints_output: dict[str, list[tuple]] = defaultdict(list)
+        application_schedules = nullcontext()
         if write_output:
             cls._check_writer_is_up(project)
+            application_schedules = (
+                mm_schedules.ModelMonitoringSchedulesFileApplication(
+                    artifact_path, application=application_name
+                )
+            )
         try:
-            yield endpoints_output
+            yield endpoints_output, application_schedules.__enter__()
         finally:
             if write_output:
                 logger.debug(
@@ -218,6 +232,12 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                     endpoints_output=endpoints_output,
                 )
 
+                logger.debug(
+                    "Saving the application schedules",
+                    application_name=application_name,
+                )
+                application_schedules.__exit__(None, None, None)
+
     def _handler(
         self,
         context: "mlrun.MLClientCtx",
@@ -230,6 +250,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         end: Optional[str] = None,
         base_period: Optional[int] = None,
         write_output: bool = False,
+        allow_unordered_data: bool = False,
         stream_profile: Optional[ds_profile.DatastoreProfile] = None,
     ):
         """
@@ -250,6 +271,8 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                 "working with endpoints, without any custom data-frame input"
             )
 
+        application_name = self.__class__.__name__
+
         feature_stats = (
             mm_api.get_sample_set_statistics(reference_data)
             if reference_data is not None
@@ -257,8 +280,12 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         )
 
         with self._push_to_writer(
-            write_output=write_output, stream_profile=stream_profile, project=project
-        ) as endpoints_output:
+            write_output=write_output,
+            stream_profile=stream_profile,
+            application_name=application_name,
+            artifact_path=context.artifact_path,
+            project=project,
+        ) as (endpoints_output, application_schedules):
 
             def call_do_tracking(event: Optional[dict] = None):
                 nonlocal endpoints_output
@@ -268,7 +295,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                 monitoring_context = (
                     mm_context.MonitoringApplicationContext._from_ml_ctx(
                         event=event,
-                        application_name=self.__class__.__name__,
+                        application_name=application_name,
                         context=context,
                         project=project,
                         sample_df=sample_data,
@@ -285,10 +312,16 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                 resolved_endpoints = self._handle_endpoints_type_evaluate(
                     project=project, endpoints=endpoints
                 )
-                for window_start, window_end in self._window_generator(
-                    start, end, base_period
-                ):
-                    for endpoint_name, endpoint_id in resolved_endpoints:
+                for endpoint_name, endpoint_id in resolved_endpoints:
+                    for window_start, window_end in self._window_generator(
+                        start=start,
+                        end=end,
+                        base_period=base_period,
+                        application_schedules=application_schedules,
+                        endpoint_id=endpoint_id,
+                        application_name=application_name,
+                        allow_unordered_data=allow_unordered_data,
+                    ):
                         result = call_do_tracking(
                             event={
                                 mm_constants.ApplicationEvent.ENDPOINT_NAME: endpoint_name,
@@ -370,8 +403,103 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             )
 
     @staticmethod
+    def _validate_and_get_window_length(
+        *, base_period: int, start_dt: datetime, end_dt: datetime
+    ) -> timedelta:
+        if not isinstance(base_period, int) or base_period <= 0:
+            raise mlrun.errors.MLRunValueError(
+                "`base_period` must be a nonnegative integer - the number of minutes in a monitoring window"
+            )
+
+        window_length = timedelta(minutes=base_period)
+
+        full_interval_length = end_dt - start_dt
+        remainder = full_interval_length % window_length
+        if remainder:
+            if full_interval_length < window_length:
+                extra_msg = (
+                    "The `base_period` is longer than the difference between `end` and `start`: "
+                    f"{full_interval_length}. Consider not specifying `base_period`."
+                )
+            else:
+                extra_msg = (
+                    f"Consider changing the `end` time to `end`={end_dt - remainder}"
+                )
+            raise mlrun.errors.MLRunValueError(
+                "The difference between `end` and `start` must be a multiple of `base_period`: "
+                f"`base_period`={window_length}, `start`={start_dt}, `end`={end_dt}. "
+                f"{extra_msg}"
+            )
+        return window_length
+
+    @staticmethod
+    def _validate_monotonically_increasing_data(
+        *,
+        application_schedules: Optional[
+            mm_schedules.ModelMonitoringSchedulesFileApplication
+        ],
+        endpoint_id: str,
+        start_dt: datetime,
+        end_dt: datetime,
+        base_period: Optional[int],
+        application_name: str,
+        allow_unordered_data: bool,
+    ) -> datetime:
+        """Make sure that the (app, endpoint) pair doesn't write output before the last analyzed window"""
+        if application_schedules:
+            last_analyzed = application_schedules.get_endpoint_last_analyzed(
+                endpoint_id
+            )
+            if last_analyzed:
+                if start_dt < last_analyzed:
+                    if allow_unordered_data:
+                        if last_analyzed < end_dt and base_period is None:
+                            logger.warn(
+                                "Setting the start time to last_analyzed since the original start time precedes "
+                                "last_analyzed",
+                                original_start=start_dt,
+                                new_start=last_analyzed,
+                                application_name=application_name,
+                                endpoint_id=endpoint_id,
+                            )
+                            start_dt = last_analyzed
+                        else:
+                            raise mlrun.errors.MLRunValueError(
+                                "The start time for the application and endpoint precedes the last analyzed time: "
+                                f"{start_dt=}, {last_analyzed=}, {application_name=}, {endpoint_id=}. "
+                                "Writing data out of order is not supported, and the start time could not be "
+                                "dynamically reset, as last_analyzed is later than the given end time or that "
+                                f"base_period was specified ({end_dt=}, {base_period=})."
+                            )
+                    else:
+                        raise mlrun.errors.MLRunValueError(
+                            "The start time for the application and endpoint precedes the last analyzed time: "
+                            f"{start_dt=}, {last_analyzed=}, {application_name=}, {endpoint_id=}. "
+                            "Writing data out of order is not supported. You should change the start time to "
+                            f"'{last_analyzed}' or later."
+                        )
+            else:
+                logger.debug(
+                    "The application is running on the endpoint for the first time",
+                    endpoint_id=endpoint_id,
+                    start_dt=start_dt,
+                    application_name=application_name,
+                )
+        return start_dt
+
+    @classmethod
     def _window_generator(
-        start: Optional[str], end: Optional[str], base_period: Optional[int]
+        cls,
+        *,
+        start: Optional[str],
+        end: Optional[str],
+        base_period: Optional[int],
+        application_schedules: Optional[
+            mm_schedules.ModelMonitoringSchedulesFileApplication
+        ],
+        endpoint_id: str,
+        application_name: str,
+        allow_unordered_data: bool,
     ) -> Iterator[tuple[Optional[datetime], Optional[datetime]]]:
         if start is None or end is None:
             # A single window based on the `sample_data` input - see `_handler`.
@@ -381,20 +509,36 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         start_dt = datetime.fromisoformat(start)
         end_dt = datetime.fromisoformat(end)
 
+        start_dt = cls._validate_monotonically_increasing_data(
+            application_schedules=application_schedules,
+            endpoint_id=endpoint_id,
+            start_dt=start_dt,
+            end_dt=end_dt,
+            base_period=base_period,
+            application_name=application_name,
+            allow_unordered_data=allow_unordered_data,
+        )
+
         if base_period is None:
             yield start_dt, end_dt
+            if application_schedules:
+                application_schedules.update_endpoint_last_analyzed(
+                    endpoint_uid=endpoint_id, last_analyzed=end_dt
+                )
             return
 
-        if not isinstance(base_period, int) or base_period <= 0:
-            raise mlrun.errors.MLRunValueError(
-                "`base_period` must be a nonnegative integer - the number of minutes in a monitoring window"
-            )
+        window_length = cls._validate_and_get_window_length(
+            base_period=base_period, start_dt=start_dt, end_dt=end_dt
+        )
 
-        window_length = timedelta(minutes=base_period)
         current_start_time = start_dt
         while current_start_time < end_dt:
             current_end_time = min(current_start_time + window_length, end_dt)
             yield current_start_time, current_end_time
+            if application_schedules:
+                application_schedules.update_endpoint_last_analyzed(
+                    endpoint_uid=endpoint_id, last_analyzed=current_end_time
+                )
             current_start_time = current_end_time
 
     @classmethod
@@ -484,6 +628,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         * ``end``, ``datetime``
         * ``base_period``, ``int``
         * ``write_output``, ``bool``
+        * ``allow_unordered_data``, ``bool``
 
         For Git sources, add the source archive to the returned job and change the handler:
 
@@ -567,6 +712,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         end: Optional[datetime] = None,
         base_period: Optional[int] = None,
         write_output: bool = False,
+        allow_unordered_data: bool = False,
         stream_profile: Optional[ds_profile.DatastoreProfile] = None,
     ) -> "mlrun.RunObject":
         """
@@ -608,6 +754,8 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         :param start:             The start time of the endpoint's data, not included.
                                   If you want the model endpoint's data at ``start`` included, you need to subtract a
                                   small ``datetime.timedelta`` from it.
+                                  Make sure to include the time zone when constructing `datetime.datetime` objects
+                                  manually.
         :param end:               The end time of the endpoint's data, included.
                                   Please note: when ``start`` and ``end`` are set, they create a left-open time interval
                                   ("window") :math:`(\\operatorname{start}, \\operatorname{end}]` that excludes the
@@ -616,17 +764,24 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                                   taken in the window's data.
         :param base_period:       The window length in minutes. If ``None``, the whole window from ``start`` to ``end``
                                   is taken. If an integer is specified, the application is run from ``start`` to ``end``
-                                  in ``base_period`` length windows, except for the last window that ends at ``end`` and
-                                  therefore may be shorter:
+                                  in ``base_period`` length windows:
                                   :math:`(\\operatorname{start}, \\operatorname{start} + \\operatorname{base\\_period}],
                                   (\\operatorname{start} + \\operatorname{base\\_period},
                                   \\operatorname{start} + 2\\cdot\\operatorname{base\\_period}],
                                   ..., (\\operatorname{start} +
-                                  m\\cdot\\operatorname{base\\_period}, \\operatorname{end}]`,
-                                  where :math:`m` is some positive integer.
+                                  (m - 1)\\cdot\\operatorname{base\\_period}, \\operatorname{end}]`,
+                                  where :math:`m` is a positive integer and :math:`\\operatorname{end} =
+                                  \\operatorname{start} + m\\cdot\\operatorname{base\\_period}`.
+                                  Please note that the difference between ``end`` and ``start`` must be a multiple of
+                                  ``base_period``.
         :param write_output:      Whether to write the results and metrics to the time-series DB. Can be ``True`` only
                                   if ``endpoints`` are passed.
                                   Note: the model monitoring infrastructure must be up for the writing to work.
+        :param allow_unordered_data: Relevant only when writing outputs to the database. When ``False``, and the
+                                     requested ``start`` time precedes the ``end`` time of a previous run that also
+                                     wrote to the database - an error is raised.
+                                     If ``True``, when the previously described situation occurs, the relevant time
+                                     window is cut so that it starts at the earliest possible time after ``start``.
         :param stream_profile:    The stream datastore profile. It should be provided only when running locally and
                                   writing the outputs to the database (i.e., when both ``run_local`` and
                                   ``write_output`` are set to ``True``).
@@ -666,6 +821,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                 params["end"] = end.isoformat() if isinstance(end, datetime) else end
                 params["base_period"] = base_period
                 params["write_output"] = write_output
+                params["allow_unordered_data"] = allow_unordered_data
                 if stream_profile:
                     if not run_local:
                         raise mlrun.errors.MLRunValueError(