mlrun 1.10.0rc16__py3-none-any.whl → 1.10.0rc17__py3-none-any.whl

mlrun/common/constants.py

@@ -84,6 +84,8 @@ class MLRunInternalLabels:
     original_workflow_id = "original-workflow-id"
     workflow_id = "workflow-id"
     retrying = "retrying"
+    rerun_counter = "rerun-counter"
+    rerun_index = "rerun-index"
 
     owner = "owner"
     v3io_user = "v3io_user"

mlrun/common/schemas/__init__.py

@@ -133,6 +133,7 @@ from .k8s import NodeSelectorOperator, Resources, ResourceSpec
 from .memory_reports import MostCommonObjectTypesReport, ObjectTypeReport
 from .model_monitoring import (
     DriftStatus,
+    EndpointMode,
     EndpointType,
     EndpointUID,
     EventFieldType,

mlrun/common/schemas/model_monitoring/__init__.py

@@ -16,6 +16,7 @@ from .constants import (
     INTERSECT_DICT_KEYS,
     ApplicationEvent,
     DriftStatus,
+    EndpointMode,
     EndpointType,
     EndpointUID,
     EventFieldType,

mlrun/common/schemas/model_monitoring/constants.py

@@ -205,6 +205,11 @@ class ControllerEvent(MonitoringStrEnum):
     FIRST_REQUEST = "first_request"
     FEATURE_SET_URI = "feature_set_uri"
     ENDPOINT_TYPE = "endpoint_type"
+
+    # first_timestamp and last_timestamp are used to batch completed events
+    FIRST_TIMESTAMP = "first_timestamp"
+    LAST_TIMESTAMP = "last_timestamp"
+
     ENDPOINT_POLICY = "endpoint_policy"
     # Note: currently under endpoint policy we will have a dictionary including the keys: "application_names"
     # "base_period", and "updated_endpoint" stand for when the MEP was updated
@@ -219,6 +224,7 @@ class ControllerEventEndpointPolicy(MonitoringStrEnum):
 class ControllerEventKind(MonitoringStrEnum):
     NOP_EVENT = "nop_event"
     REGULAR_EVENT = "regular_event"
+    BATCH_COMPLETE = "batch_complete"
 
 
 class MetricData(MonitoringStrEnum):
@@ -320,6 +326,19 @@ class EndpointType(IntEnum):
     def top_level_list(cls):
         return [cls.NODE_EP, cls.ROUTER, cls.BATCH_EP]
 
+    @classmethod
+    def real_time_list(cls):
+        return [cls.NODE_EP, cls.ROUTER, cls.LEAF_EP]
+
+    @classmethod
+    def batch_list(cls):
+        return [cls.BATCH_EP]
+
+
+class EndpointMode(StrEnum):
+    REAL_TIME = "real_time"
+    BATCH = "batch"
+
 
 class MonitoringFunctionNames(MonitoringStrEnum):
     STREAM = "model-monitoring-stream"

mlrun/common/schemas/serving.py

@@ -47,3 +47,6 @@ class MonitoringData(StrEnum):
 class ModelsData(enum.Enum):
     MODEL_CLASS = 0
     MODEL_PARAMETERS = 1
+
+
+MAX_BATCH_JOB_DURATION = "1w"

mlrun/common/schemas/workflow.py

@@ -53,6 +53,7 @@ class RerunWorkflowRequest(pydantic.v1.BaseModel):
     workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
     original_workflow_runner_uid: typing.Optional[str] = None
     original_workflow_name: typing.Optional[str] = None
+    rerun_index: typing.Optional[int] = None
 
 
 class WorkflowResponse(pydantic.v1.BaseModel):

mlrun/config.py

@@ -406,11 +406,7 @@ default_config = {
                 #
                 # if set to "nil" or "none", nothing would be set
                 "modes": (
-                    "STRICT_TRANS_TABLES"
-                    ",NO_ZERO_IN_DATE"
-                    ",NO_ZERO_DATE"
-                    ",ERROR_FOR_DIVISION_BY_ZERO"
-                    ",NO_ENGINE_SUBSTITUTION",
+                    "STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION"
                 )
             },
         },

mlrun/db/base.py

@@ -741,6 +741,7 @@ class RunDBInterface(ABC):
         tsdb_metrics: bool = False,
         metric_list: Optional[list[str]] = None,
         top_level: bool = False,
+        mode: Optional[mlrun.common.schemas.EndpointMode] = None,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
     ) -> mlrun.common.schemas.ModelEndpointList:

mlrun/db/httpdb.py

@@ -3813,6 +3813,7 @@ class HTTPRunDB(RunDBInterface):
         tsdb_metrics: bool = False,
         metric_list: Optional[list[str]] = None,
         top_level: bool = False,
+        mode: mm_constants.EndpointMode = None,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
     ) -> mlrun.common.schemas.ModelEndpointList:
@@ -3833,6 +3834,8 @@ class HTTPRunDB(RunDBInterface):
                                 If tsdb_metrics=False, this parameter will be ignored and no tsdb metrics
                                 will be included.
         :param top_level:       Whether to return only top level model endpoints.
+        :param mode:            Specifies the mode of the model endpoint. Can be "real-time", "batch", or both if set
+                                to None.
         :param uids:            A list of unique ids to filter by.
         :param latest_only:     Whether to return only the latest model endpoint version.
         :return:                A list of model endpoints.
@@ -3856,6 +3859,7 @@ class HTTPRunDB(RunDBInterface):
                 "tsdb-metrics": tsdb_metrics,
                 "metric": metric_list,
                 "top-level": top_level,
+                "mode": mode,
                 "uid": uids,
                 "latest-only": latest_only,
             },

mlrun/model_monitoring/controller.py

@@ -12,13 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import collections
 import concurrent.futures
 import datetime
 import json
 import os
 import traceback
-from collections import OrderedDict
 from collections.abc import Iterator
 from contextlib import AbstractContextManager
 from types import TracebackType
@@ -29,20 +27,17 @@ import pandas as pd
 
 import mlrun
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
-import mlrun.feature_store as fstore
 import mlrun.model_monitoring
 import mlrun.model_monitoring.db._schedules as schedules
 import mlrun.model_monitoring.helpers
 import mlrun.platforms.iguazio
-from mlrun.common.schemas import EndpointType
 from mlrun.common.schemas.model_monitoring.constants import (
     ControllerEvent,
     ControllerEventEndpointPolicy,
-    ControllerEventKind,
 )
 from mlrun.errors import err_to_str
 from mlrun.model_monitoring.helpers import batch_dict2timedelta
-from mlrun.utils import datetime_now, logger
+from mlrun.utils import logger
 
 _SECONDS_IN_DAY = int(datetime.timedelta(days=1).total_seconds())
 _SECONDS_IN_MINUTE = 60
@@ -62,6 +57,7 @@ class _BatchWindow:
         timedelta_seconds: int,
         last_updated: int,
         first_request: int,
+        endpoint_mode: mm_constants.EndpointMode = mm_constants.EndpointMode.REAL_TIME,
     ) -> None:
         """
         Initialize a batch window object that handles the batch interval time range
@@ -74,6 +70,7 @@ class _BatchWindow:
         self._stop = last_updated
         self._step = timedelta_seconds
         self._db = schedules_file
+        self._endpoint_mode = endpoint_mode
         self._start = self._get_last_analyzed()
 
     def _get_saved_last_analyzed(self) -> Optional[int]:
@@ -85,10 +82,20 @@ class _BatchWindow:
         )
 
     def _get_initial_last_analyzed(self) -> int:
+        if self._endpoint_mode == mm_constants.EndpointMode.BATCH:
+            logger.info(
+                "No last analyzed time was found for this endpoint and application, as this is "
+                "probably the first time this application is running. Initializing last analyzed "
+                "to the start of the batch time",
+                application=self._application,
+                start_batch_time=self._first_request,
+            )
+            return self._first_request
         logger.info(
             "No last analyzed time was found for this endpoint and application, as this is "
             "probably the first time this application is running. Initializing last analyzed "
-            "to the latest between first request time or last update time minus one day",
+            "to the latest between first request the latest between first request time or last "
+            "update time minus one day",
             application=self._application,
             first_request=self._first_request,
             last_updated=self._stop,
@@ -103,6 +110,9 @@ class _BatchWindow:
     def _get_last_analyzed(self) -> int:
         saved_last_analyzed = self._get_saved_last_analyzed()
         if saved_last_analyzed is not None:
+            if self._endpoint_mode == mm_constants.EndpointMode.BATCH:
+                # Use the maximum between the saved last analyzed and the start of the batch
+                return max(saved_last_analyzed, self._first_request)
             return saved_last_analyzed
         else:
             last_analyzed = self._get_initial_last_analyzed()
@@ -113,6 +123,7 @@ class _BatchWindow:
     def get_intervals(self) -> Iterator[_Interval]:
         """Generate the batch interval time ranges."""
         entered = False
+        last_analyzed = None
         # Iterate timestamp from start until timestamp <= stop - step
         # so that the last interval will end at (timestamp + step) <= stop.
         # Add 1 to stop - step to get <= and not <.
@@ -134,6 +145,40 @@ class _BatchWindow:
                 last_analyzed=last_analyzed,
             )
 
+        if self._endpoint_mode == mm_constants.EndpointMode.BATCH:
+            # If the endpoint is a batch endpoint, we need to update the last analyzed time
+            # to the end of the batch time.
+            if last_analyzed:
+                if last_analyzed < self._stop:
+                    # If the last analyzed time is earlier than the stop time,
+                    # yield the final partial interval from last_analyzed to stop
+                    yield _Interval(
+                        datetime.datetime.fromtimestamp(
+                            last_analyzed, tz=datetime.timezone.utc
+                        ),
+                        datetime.datetime.fromtimestamp(
+                            self._stop, tz=datetime.timezone.utc
+                        ),
+                    )
+            else:
+                # The time span between the start and end of the batch is shorter than the step,
+                # so we need to yield a partial interval covering that range.
+                yield _Interval(
+                    datetime.datetime.fromtimestamp(
+                        self._start, tz=datetime.timezone.utc
+                    ),
+                    datetime.datetime.fromtimestamp(
+                        self._stop, tz=datetime.timezone.utc
+                    ),
+                )
+
+            self._update_last_analyzed(last_analyzed=self._stop)
+            logger.debug(
+                "Updated the last analyzed time for this endpoint and application to the end of the batch time",
+                application=self._application,
+                last_analyzed=self._stop,
+            )
+
         if not entered:
             logger.debug(
                 "All the data is set, but no complete intervals were found. "
@@ -183,28 +228,25 @@ class _BatchWindowGenerator(AbstractContextManager):
 
     @classmethod
     def _get_last_updated_time(
-        cls, last_request: datetime.datetime, not_batch_endpoint: bool
+        cls,
+        last_request: datetime.datetime,
+        endpoint_mode: mm_constants.EndpointMode,
     ) -> int:
         """
         Get the last updated time of a model endpoint.
         """
-        last_updated = int(
-            last_request.timestamp()
-            - cast(
-                float,
-                mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs,
-            )
-        )
-        if not not_batch_endpoint:
-            # If the endpoint does not have a stream, `last_updated` should be
-            # the minimum between the current time and the last updated time.
-            # This compensates for the bumping mechanism - see
-            # `update_model_endpoint_last_request`.
-            last_updated = min(int(datetime_now().timestamp()), last_updated)
-            logger.debug(
-                "The endpoint does not have a stream", last_updated=last_updated
+
+        if endpoint_mode == mm_constants.EndpointMode.REAL_TIME:
+            last_updated = int(
+                last_request.timestamp()
+                - cast(
+                    float,
+                    mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs,
+                )
             )
-        return last_updated
+
+            return last_updated
+        return int(last_request.timestamp())
 
     def get_intervals(
         self,
@@ -212,19 +254,21 @@ class _BatchWindowGenerator(AbstractContextManager):
         application: str,
         first_request: datetime.datetime,
         last_request: datetime.datetime,
-        not_batch_endpoint: bool,
+        endpoint_mode: mm_constants.EndpointMode,
     ) -> Iterator[_Interval]:
         """
         Get the batch window for a specific endpoint and application.
         `first_request` and `last_request` are the timestamps of the first request and last
         request to the endpoint, respectively. They are guaranteed to be nonempty at this point.
         """
+
         self.batch_window = _BatchWindow(
             schedules_file=self._schedules_file,
             application=application,
             timedelta_seconds=self._timedelta,
-            last_updated=self._get_last_updated_time(last_request, not_batch_endpoint),
+            last_updated=self._get_last_updated_time(last_request, endpoint_mode),
             first_request=int(first_request.timestamp()),
+            endpoint_mode=endpoint_mode,
         )
         yield from self.batch_window.get_intervals()
 
@@ -247,8 +291,6 @@ class MonitoringApplicationController:
     Note that the MonitoringApplicationController object requires access keys along with valid project configurations.
     """
 
-    _MAX_FEATURE_SET_PER_WORKER = 1000
-
     def __init__(self) -> None:
         """Initialize Monitoring Application Controller"""
         self.project = cast(str, mlrun.mlconf.active_project)
@@ -282,9 +324,6 @@ class MonitoringApplicationController:
                 mlrun.platforms.iguazio.KafkaOutputStream,
             ],
         ] = {}
-        self.feature_sets: OrderedDict[str, mlrun.feature_store.FeatureSet] = (
-            collections.OrderedDict()
-        )
         self.tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
             project=self.project
         )
@@ -421,7 +460,6 @@ class MonitoringApplicationController:
                 last_request=endpoint.status.last_request,
                 first_request=endpoint.status.first_request,
                 endpoint_type=endpoint.metadata.endpoint_type,
-                feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
             )
         return False
 
@@ -477,24 +515,67 @@ class MonitoringApplicationController:
         try:
             project_name = event[ControllerEvent.PROJECT]
             endpoint_id = event[ControllerEvent.ENDPOINT_ID]
-            endpoint_name = event[ControllerEvent.ENDPOINT_NAME]
-            applications_names = event[ControllerEvent.ENDPOINT_POLICY][
-                ControllerEventEndpointPolicy.MONITORING_APPLICATIONS
-            ]
 
-            not_batch_endpoint = (
-                event[ControllerEvent.ENDPOINT_TYPE] != EndpointType.BATCH_EP
-            )
+            if (
+                event[ControllerEvent.KIND]
+                == mm_constants.ControllerEventKind.BATCH_COMPLETE
+            ):
+                monitoring_functions = (
+                    self.project_obj.list_model_monitoring_functions()
+                )
+                if monitoring_functions:
+                    applications_names = list(
+                        {app.metadata.name for app in monitoring_functions}
+                    )
+                    last_stream_timestamp = datetime.datetime.fromisoformat(
+                        event[ControllerEvent.LAST_TIMESTAMP]
+                    )
+                    first_request = datetime.datetime.fromisoformat(
+                        event[ControllerEvent.FIRST_TIMESTAMP]
+                    )
+                    endpoint_mode = mm_constants.EndpointMode.BATCH
+                    model_endpoint = self.project_obj.list_model_endpoints(
+                        uids=[endpoint_id],
+                        latest_only=True,
+                    ).endpoints
+
+                    if not model_endpoint:
+                        logger.error(
+                            "Batch model endpoint not found",
+                            endpoint_id=endpoint_id,
+                            project=project_name,
+                        )
+                        return
+
+                    endpoint_name = model_endpoint[0].metadata.name
+                    endpoint_updated = model_endpoint[0].metadata.updated.isoformat()
+
+                else:
+                    logger.info("No monitoring functions found", project=self.project)
+                    return
+
+            else:
+                endpoint_name = event[ControllerEvent.ENDPOINT_NAME]
+                applications_names = event[ControllerEvent.ENDPOINT_POLICY][
+                    ControllerEventEndpointPolicy.MONITORING_APPLICATIONS
+                ]
+                last_stream_timestamp = datetime.datetime.fromisoformat(
+                    event[ControllerEvent.TIMESTAMP]
+                )
+                first_request = datetime.datetime.fromisoformat(
+                    event[ControllerEvent.FIRST_REQUEST]
+                )
+
+                endpoint_updated = event[ControllerEvent.ENDPOINT_POLICY][
+                    ControllerEventEndpointPolicy.ENDPOINT_UPDATED
+                ]
+
+                endpoint_mode = mm_constants.EndpointMode.REAL_TIME
 
             logger.info(
-                "Starting analyzing for", timestamp=event[ControllerEvent.TIMESTAMP]
-            )
-            last_stream_timestamp = datetime.datetime.fromisoformat(
-                event[ControllerEvent.TIMESTAMP]
-            )
-            first_request = datetime.datetime.fromisoformat(
-                event[ControllerEvent.FIRST_REQUEST]
+                "Starting to analyze", timestamp=last_stream_timestamp.isoformat()
             )
+
             with _BatchWindowGenerator(
                 project=project_name,
                 endpoint_id=endpoint_id,
@@ -506,42 +587,20 @@ class MonitoringApplicationController:
                         end_infer_time,
                     ) in batch_window_generator.get_intervals(
                         application=application,
-                        not_batch_endpoint=not_batch_endpoint,
                         first_request=first_request,
                         last_request=last_stream_timestamp,
+                        endpoint_mode=endpoint_mode,
                     ):
                         data_in_window = False
-                        if not_batch_endpoint:
-                            # Serving endpoint - get the relevant window data from the TSDB
-                            prediction_metric = self.tsdb_connector.read_predictions(
-                                start=start_infer_time,
-                                end=end_infer_time,
-                                endpoint_id=endpoint_id,
-                            )
-                            if prediction_metric.data:
-                                data_in_window = True
-                        else:
-                            if endpoint_id not in self.feature_sets:
-                                self.feature_sets[endpoint_id] = fstore.get_feature_set(
-                                    event[ControllerEvent.FEATURE_SET_URI]
-                                )
-                            self.feature_sets.move_to_end(endpoint_id, last=False)
-                            if (
-                                len(self.feature_sets)
-                                > self._MAX_FEATURE_SET_PER_WORKER
-                            ):
-                                self.feature_sets.popitem(last=True)
-                            m_fs = self.feature_sets.get(endpoint_id)
-
-                            # Batch endpoint - get the relevant window data from the parquet target
-                            df = m_fs.to_dataframe(
-                                start_time=start_infer_time,
-                                end_time=end_infer_time,
-                                time_column=mm_constants.EventFieldType.TIMESTAMP,
-                                storage_options=self.storage_options,
-                            )
-                            if len(df) > 0:
-                                data_in_window = True
+                        # Serving endpoint - get the relevant window data from the TSDB
+                        prediction_metric = self.tsdb_connector.read_predictions(
+                            start=start_infer_time,
+                            end=end_infer_time,
+                            endpoint_id=endpoint_id,
+                        )
+                        if prediction_metric.data:
+                            data_in_window = True
+
                         if not data_in_window:
                             logger.info(
                                 "No data found for the given interval",
@@ -564,49 +623,47 @@ class MonitoringApplicationController:
                                 project=project_name,
                                 applications_names=[application],
                                 model_monitoring_access_key=self.model_monitoring_access_key,
-                                endpoint_updated=event[ControllerEvent.ENDPOINT_POLICY][
-                                    ControllerEventEndpointPolicy.ENDPOINT_UPDATED
-                                ],
+                                endpoint_updated=endpoint_updated,
                             )
-                base_period = event[ControllerEvent.ENDPOINT_POLICY][
-                    ControllerEventEndpointPolicy.BASE_PERIOD
-                ]
-                current_time = mlrun.utils.datetime_now()
+
                 if (
-                    self._should_send_nop_event(
+                    event[ControllerEvent.KIND]
+                    == mm_constants.ControllerEventKind.REGULAR_EVENT
+                ):
+                    base_period = event[ControllerEvent.ENDPOINT_POLICY][
+                        ControllerEventEndpointPolicy.BASE_PERIOD
+                    ]
+                    current_time = mlrun.utils.datetime_now()
+                    if self._should_send_nop_event(
                         base_period,
                         batch_window_generator.get_min_last_analyzed(),
                         current_time,
-                    )
-                    and event[ControllerEvent.KIND] != ControllerEventKind.NOP_EVENT
-                ):
-                    event = {
-                        ControllerEvent.KIND: mm_constants.ControllerEventKind.NOP_EVENT,
-                        ControllerEvent.PROJECT: project_name,
-                        ControllerEvent.ENDPOINT_ID: endpoint_id,
-                        ControllerEvent.ENDPOINT_NAME: endpoint_name,
-                        ControllerEvent.TIMESTAMP: current_time.isoformat(
-                            timespec="microseconds"
-                        ),
-                        ControllerEvent.ENDPOINT_POLICY: event[
-                            ControllerEvent.ENDPOINT_POLICY
-                        ],
-                        ControllerEvent.ENDPOINT_TYPE: event[
-                            ControllerEvent.ENDPOINT_TYPE
-                        ],
-                        ControllerEvent.FEATURE_SET_URI: event[
-                            ControllerEvent.FEATURE_SET_URI
-                        ],
-                        ControllerEvent.FIRST_REQUEST: event[
-                            ControllerEvent.FIRST_REQUEST
-                        ],
-                    }
-                    self._push_to_main_stream(
-                        event=event,
-                        endpoint_id=endpoint_id,
-                    )
+                    ):
+                        event = {
+                            ControllerEvent.KIND: mm_constants.ControllerEventKind.NOP_EVENT,
+                            ControllerEvent.PROJECT: project_name,
+                            ControllerEvent.ENDPOINT_ID: endpoint_id,
+                            ControllerEvent.ENDPOINT_NAME: endpoint_name,
+                            ControllerEvent.TIMESTAMP: current_time.isoformat(
+                                timespec="microseconds"
+                            ),
+                            ControllerEvent.ENDPOINT_POLICY: event[
+                                ControllerEvent.ENDPOINT_POLICY
+                            ],
+                            ControllerEvent.ENDPOINT_TYPE: event[
+                                ControllerEvent.ENDPOINT_TYPE
+                            ],
+                            ControllerEvent.FIRST_REQUEST: event[
+                                ControllerEvent.FIRST_REQUEST
+                            ],
+                        }
+                        self._push_to_main_stream(
+                            event=event,
+                            endpoint_id=endpoint_id,
+                        )
             logger.info(
-                "Finish analyze for", timestamp=event[ControllerEvent.TIMESTAMP]
+                "Finish analyze for",
+                timestamp=last_stream_timestamp,
             )
 
         except Exception:
@@ -674,7 +731,9 @@ class MonitoringApplicationController:
         """
         logger.info("Starting monitoring controller chief")
         applications_names = []
-        endpoints = self.project_obj.list_model_endpoints(tsdb_metrics=False).endpoints
+        endpoints = self.project_obj.list_model_endpoints(
+            tsdb_metrics=False, mode=mm_constants.EndpointMode.REAL_TIME
+        ).endpoints
         last_request_dict = self.tsdb_connector.get_last_request(
             endpoint_ids=[mep.metadata.uid for mep in endpoints]
         )
@@ -783,7 +842,6 @@ class MonitoringApplicationController:
                     sep=" ", timespec="microseconds"
                 ),
                 endpoint_type=endpoint.metadata.endpoint_type,
-                feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
                 endpoint_policy=json.dumps(policy),
             )
             policy[ControllerEventEndpointPolicy.ENDPOINT_UPDATED] = (
@@ -801,7 +859,6 @@ class MonitoringApplicationController:
                     sep=" ", timespec="microseconds"
                 ),
                 endpoint_type=endpoint.metadata.endpoint_type.value,
-                feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
                 endpoint_policy=policy,
             )
 
@@ -814,7 +871,6 @@ class MonitoringApplicationController:
         timestamp: str,
         first_request: str,
         endpoint_type: int,
-        feature_set_uri: str,
         endpoint_policy: dict[str, Any],
     ) -> None:
         """
@@ -827,7 +883,6 @@ class MonitoringApplicationController:
         :param endpoint_id: endpoint id string
         :param endpoint_name: the endpoint name string
         :param endpoint_type: Enum of the endpoint type
-        :param feature_set_uri: the feature set uri string
         """
         event = {
             ControllerEvent.KIND.value: kind,
@@ -837,7 +892,6 @@ class MonitoringApplicationController:
             ControllerEvent.TIMESTAMP.value: timestamp,
             ControllerEvent.FIRST_REQUEST.value: first_request,
             ControllerEvent.ENDPOINT_TYPE.value: endpoint_type,
-            ControllerEvent.FEATURE_SET_URI.value: feature_set_uri,
             ControllerEvent.ENDPOINT_POLICY.value: endpoint_policy,
         }
         logger.info(

mlrun/model_monitoring/stream_processing.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import asyncio
 import datetime
 import typing
 
@@ -134,6 +134,9 @@ class EventStreamProcessor:
            the default parquet path is under mlrun.mlconf.model_endpoint_monitoring.user_space. Note that if you are
            using CE, the parquet target path is based on the defined MLRun artifact path.
 
+        In a separate branch, "batch complete" events are forwarded to the controller stream with an intentional delay,
+        to allow for data to first be written to parquet.
+
         :param fn: A serving function.
         :param tsdb_connector: Time series database connector.
         :param controller_stream_uri: The controller stream URI. Runs on server api pod so needed to be provided as
@@ -145,6 +148,20 @@ class EventStreamProcessor:
             fn.set_topology(mlrun.serving.states.StepKinds.flow, engine="async"),
         )
 
+        # forward back complete events to controller
+        graph.add_step(
+            "storey.Filter",
+            "FilterBatchComplete",
+            _fn="(event.get('kind') == 'batch_complete')",
+        )
+
+        graph.add_step(
+            "Delay",
+            name="BatchDelay",
+            after="FilterBatchComplete",
+            delay=self.parquet_batching_timeout_secs + 5,  # add margin
+        )
+
         # split the graph between event with error vs valid event
         graph.add_step(
             "storey.Filter",
@@ -261,7 +278,7 @@ class EventStreamProcessor:
                 "controller_stream",
                 path=stream_uri,
                 sharding_func=ControllerEvent.ENDPOINT_ID,
-                after="ForwardNOP",
+                after=["ForwardNOP", "BatchDelay"],
                 # Force using the pipeline key instead of the one in the profile in case of v3io profile.
                 # In case of Kafka, this parameter will be ignored.
                 alternative_v3io_access_key="V3IO_ACCESS_KEY",
@@ -309,6 +326,16 @@ class ProcessBeforeParquet(mlrun.feature_store.steps.MapClass):
         return event
 
 
+class Delay(mlrun.feature_store.steps.MapClass):
+    def __init__(self, delay: int, **kwargs):
+        super().__init__(**kwargs)
+        self._delay = delay
+
+    async def do(self, event):
+        await asyncio.sleep(self._delay)
+        return event
+
+
 class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
     def __init__(
         self,

mlrun/projects/project.py

@@ -3901,6 +3901,7 @@ class MlrunProject(ModelObj):
         start: Optional[datetime.datetime] = None,
         end: Optional[datetime.datetime] = None,
         top_level: bool = False,
+        mode: Optional[mlrun.common.schemas.EndpointMode] = None,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
         tsdb_metrics: bool = False,
@@ -3916,8 +3917,9 @@ class MlrunProject(ModelObj):
         5) function_tag
         6) labels
         7) top level
-        8) uids
-        9) start and end time, corresponding to the `created` field.
+        8) mode
+        9) uids
+        10) start and end time, corresponding to the `created` field.
         By default, when no filters are applied, all available endpoints for the given project will be listed.
 
         In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
@@ -3937,6 +3939,8 @@ class MlrunProject(ModelObj):
         :param start:           The start time to filter by.Corresponding to the `created` field.
         :param end:             The end time to filter by. Corresponding to the `created` field.
         :param top_level:       If true will return only routers and endpoint that are NOT children of any router.
+        :param mode:            Specifies the mode of the model endpoint. Can be "real-time", "batch", or both if set
+                                to None.
         :param uids:            If passed will return a list `ModelEndpoint` object with uid in uids.
         :param tsdb_metrics:    When True, the time series metrics will be added to the output
                                 of the resulting.
@@ -3958,6 +3962,7 @@ class MlrunProject(ModelObj):
             start=start,
             end=end,
             top_level=top_level,
+            mode=mode,
             uids=uids,
             latest_only=latest_only,
             tsdb_metrics=tsdb_metrics,

mlrun/serving/server.py

@@ -22,8 +22,10 @@ import os
 import socket
 import traceback
 import uuid
+from datetime import datetime, timezone
 from typing import Any, Optional, Union
 
+import pandas as pd
 import storey
 from nuclio import Context as NuclioContext
 from nuclio.request import Logger as NuclioLogger
@@ -40,6 +42,7 @@ from mlrun.secrets import SecretsStore
 
 from ..common.helpers import parse_versioned_object_uri
 from ..common.schemas.model_monitoring.constants import FileTargetKind
+from ..common.schemas.serving import MAX_BATCH_JOB_DURATION
 from ..datastore import DataItem, get_stream_pusher
 from ..datastore.store_resources import ResourceCache
 from ..errors import MLRunInvalidArgumentError
@@ -561,6 +564,7 @@ def v2_serving_init(context, namespace=None):
 async def async_execute_graph(
     context: MLClientCtx,
     data: DataItem,
+    timestamp_column: Optional[str],
     batching: bool,
     batch_size: Optional[int],
     read_as_lists: bool,
@@ -605,10 +609,43 @@ async def async_execute_graph(
                 f"(status='{task_state}')"
             )
 
+    df = data.as_df()
+
+    if df.empty:
+        context.logger.warn("Job terminated due to empty inputs (0 rows)")
+        return []
+
+    track_models = spec.get("track_models")
+
+    if track_models and timestamp_column:
+        context.logger.info(f"Sorting dataframe by {timestamp_column}")
+        df[timestamp_column] = pd.to_datetime(  # in case it's a string
+            df[timestamp_column]
+        )
+        df.sort_values(by=timestamp_column, inplace=True)
+        if len(df) > 1:
+            start_time = df[timestamp_column].iloc[0]
+            end_time = df[timestamp_column].iloc[-1]
+            time_range = end_time - start_time
+            start_time = start_time.isoformat()
+            end_time = end_time.isoformat()
+            # TODO: tie this to the controller's base period
+            if time_range > pd.Timedelta(MAX_BATCH_JOB_DURATION):
+                raise mlrun.errors.MLRunRuntimeError(
+                    f"Dataframe time range is too long: {time_range}. "
+                    "Please disable tracking or reduce the input dataset's time range below the defined limit "
+                    f"of {MAX_BATCH_JOB_DURATION}."
+                )
+        else:
+            start_time = end_time = df["timestamp"].iloc[0].isoformat()
+    else:
+        # end time will be set from clock time when the batch completes
+        start_time = datetime.now(tz=timezone.utc).isoformat()
+
     server.graph = add_system_steps_to_graph(
         server.project,
         copy.deepcopy(server.graph),
-        spec.get("track_models"),
+        track_models,
         context,
         spec,
         pause_until_background_task_completion=False,  # we've already awaited it
@@ -633,19 +670,28 @@ async def async_execute_graph(
     if server.verbose:
         context.logger.info(server.to_yaml())
 
-    df = data.as_df()
-
-    responses = []
-
     async def run(body):
         event = storey.Event(id=index, body=body)
-        response = await server.run(event, context)
-        responses.append(response)
+        if timestamp_column:
+            if batching:
+                # we use the first row in the batch to determine the timestamp for the whole batch
+                body = body[0]
+            if not isinstance(body, dict):
+                raise mlrun.errors.MLRunRuntimeError(
+                    f"When timestamp_column=True, event body must be a dict – got {type(body).__name__} instead"
+                )
+            if timestamp_column not in body:
+                raise mlrun.errors.MLRunRuntimeError(
+                    f"Event body '{body}' did not contain timestamp column '{timestamp_column}'"
+                )
+            event._original_timestamp = body[timestamp_column]
+        return await server.run(event, context)
 
     if batching and not batch_size:
         batch_size = len(df)
 
     batch = []
+    tasks = []
     for index, row in df.iterrows():
         data = row.to_list() if read_as_lists else row.to_dict()
         if nest_under_inputs:
@@ -653,24 +699,56 @@ async def async_execute_graph(
         if batching:
             batch.append(data)
             if len(batch) == batch_size:
-                await run(batch)
+                tasks.append(asyncio.create_task(run(batch)))
                 batch = []
         else:
-            await run(data)
+            tasks.append(asyncio.create_task(run(data)))
 
     if batch:
-        await run(batch)
+        tasks.append(asyncio.create_task(run(batch)))
+
+    responses = await asyncio.gather(*tasks)
 
     termination_result = server.wait_for_completion()
     if asyncio.iscoroutine(termination_result):
         await termination_result
 
+    model_endpoint_uids = spec.get("model_endpoint_uids", [])
+
+    # needed for output_stream to be created
+    server = GraphServer.from_dict(spec)
+    server.init_states(None, namespace)
+
+    batch_completion_time = datetime.now(tz=timezone.utc).isoformat()
+
+    if not timestamp_column:
+        end_time = batch_completion_time
+
+    mm_stream_record = dict(
+        kind="batch_complete",
+        project=context.project,
+        first_timestamp=start_time,
+        last_timestamp=end_time,
+        batch_completion_time=batch_completion_time,
+    )
+    output_stream = server.context.stream.output_stream
+    for mep_uid in spec.get("model_endpoint_uids", []):
+        mm_stream_record["endpoint_id"] = mep_uid
+        output_stream.push(mm_stream_record, partition_key=mep_uid)
+
+    context.logger.info(
+        f"Job completed processing {len(df)} rows",
+        timestamp_column=timestamp_column,
+        model_endpoint_uids=model_endpoint_uids,
+    )
+
     return responses
 
 
 def execute_graph(
     context: MLClientCtx,
     data: DataItem,
+    timestamp_column: Optional[str] = None,
     batching: bool = False,
     batch_size: Optional[int] = None,
     read_as_lists: bool = False,
@@ -681,6 +759,9 @@ def execute_graph(
 
     :param context: The job's execution client context.
     :param data: The input data to the job, to be pushed into the graph row by row, or in batches.
+    :param timestamp_column: The name of the column that will be used as the timestamp for model monitoring purposes.
+        when timestamp_column is used in conjunction with batching, the first timestamp will be used for the entire
+        batch.
     :param batching: Whether to push one or more batches into the graph rather than row by row.
     :param batch_size: The number of rows to push per batch. If not set, and batching=True, the entire dataset will
         be pushed into the graph in one batch.
@@ -691,7 +772,13 @@ def execute_graph(
     """
     return asyncio.run(
         async_execute_graph(
-            context, data, batching, batch_size, read_as_lists, nest_under_inputs
+            context,
+            data,
+            timestamp_column,
+            batching,
+            batch_size,
+            read_as_lists,
+            nest_under_inputs,
         )
     )
 

mlrun/serving/states.py

@@ -1754,9 +1754,10 @@ class ModelRunnerStep(MonitoredStep):
         except (
             mlrun.errors.MLRunNotFoundError,
             mlrun.errors.MLRunInvalidArgumentError,
-        ):
+        ) as ex:
             logger.warning(
-                f"Model endpoint not found, using default output schema for model {name}"
+                f"Model endpoint not found, using default output schema for model {name}",
+                error=f"{type(ex).__name__}: {ex}",
             )
         return output_schema
 
@@ -1768,22 +1769,6 @@ class ModelRunnerStep(MonitoredStep):
         )
         if isinstance(monitoring_data, dict):
             for model in monitoring_data:
-                monitoring_data[model][schemas.MonitoringData.OUTPUTS] = (
-                    monitoring_data.get(model, {}).get(schemas.MonitoringData.OUTPUTS)
-                    or self._get_model_endpoint_output_schema(
-                        name=model,
-                        project=self.context.project if self.context else None,
-                        uid=monitoring_data.get(model, {}).get(
-                            mlrun.common.schemas.MonitoringData.MODEL_ENDPOINT_UID
-                        ),
-                    )
-                )
-                # Prevent calling _get_model_output_schema for same model more than once
-                self.class_args[
-                    mlrun.common.schemas.ModelRunnerStepData.MONITORING_DATA
-                ][model][schemas.MonitoringData.OUTPUTS] = monitoring_data[model][
-                    schemas.MonitoringData.OUTPUTS
-                ]
                 monitoring_data[model][schemas.MonitoringData.INPUT_PATH] = split_path(
                     monitoring_data[model][schemas.MonitoringData.INPUT_PATH]
                 )
@@ -1791,6 +1776,10 @@ class ModelRunnerStep(MonitoredStep):
                     monitoring_data[model][schemas.MonitoringData.RESULT_PATH]
                 )
             return monitoring_data
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Monitoring data must be a dictionary."
+            )
 
     def init_object(self, context, namespace, mode="sync", reset=False, **extra_kwargs):
         self.context = context

mlrun/serving/system_steps.py

@@ -104,7 +104,7 @@ class MonitoringPreProcessor(storey.MapClass):
     @staticmethod
     def transpose_by_key(
         data: dict, schema: Optional[Union[str, list[str]]] = None
-    ) -> Union[list[float], list[list[float]]]:
+    ) -> Union[list[Any], list[list[Any]]]:
         """
         Transpose values from a dictionary by keys.
 
@@ -146,7 +146,11 @@ class MonitoringPreProcessor(storey.MapClass):
         else:
             keys = schema
 
-        values = [data[key] for key in keys]
+        values = [data[key] for key in keys if key in data]
+        if len(values) != len(keys):
+            raise mlrun.MLRunInvalidArgumentError(
+                f"Schema keys {keys} do not match the data keys {list(data.keys())}."
+            )
 
         # Detect if all are scalars ie: int,float,str
         all_scalars = all(not isinstance(v, (list, tuple, np.ndarray)) for v in values)
@@ -158,9 +162,9 @@ class MonitoringPreProcessor(storey.MapClass):
             )
 
         if all_scalars:
-            transposed = np.array([values])
+            transposed = np.array([values], dtype=object)
         elif all_lists and len(keys) > 1:
-            arrays = [np.array(v) for v in values]
+            arrays = [np.array(v, dtype=object) for v in values]
             mat = np.stack(arrays, axis=0)
             transposed = mat.T
         else:
@@ -192,6 +196,12 @@ class MonitoringPreProcessor(storey.MapClass):
                     request, resp = self.reconstruct_request_resp_fields(
                         event, model, monitoring_data[model]
                     )
+                    if hasattr(event, "_original_timestamp"):
+                        when = event._original_timestamp
+                    else:
+                        when = event._metadata.get(model, {}).get(
+                            mm_schemas.StreamProcessingEvent.WHEN
+                        )
                     monitoring_event_list.append(
                         {
                             mm_schemas.StreamProcessingEvent.MODEL: model,
@@ -201,9 +211,7 @@ class MonitoringPreProcessor(storey.MapClass):
                             mm_schemas.StreamProcessingEvent.MICROSEC: event._metadata.get(
                                 model, {}
                             ).get(mm_schemas.StreamProcessingEvent.MICROSEC),
-                            mm_schemas.StreamProcessingEvent.WHEN: event._metadata.get(
-                                model, {}
-                            ).get(mm_schemas.StreamProcessingEvent.WHEN),
+                            mm_schemas.StreamProcessingEvent.WHEN: when,
                             mm_schemas.StreamProcessingEvent.ENDPOINT_ID: monitoring_data[
                                 model
                             ].get(
@@ -236,6 +244,10 @@ class MonitoringPreProcessor(storey.MapClass):
             request, resp = self.reconstruct_request_resp_fields(
                 event, model, monitoring_data[model]
             )
+            if hasattr(event, "_original_timestamp"):
+                when = event._original_timestamp
+            else:
+                when = event._metadata.get(mm_schemas.StreamProcessingEvent.WHEN)
             monitoring_event_list.append(
                 {
                     mm_schemas.StreamProcessingEvent.MODEL: model,
@@ -245,9 +257,7 @@ class MonitoringPreProcessor(storey.MapClass):
                     mm_schemas.StreamProcessingEvent.MICROSEC: event._metadata.get(
                         mm_schemas.StreamProcessingEvent.MICROSEC
                     ),
-                    mm_schemas.StreamProcessingEvent.WHEN: event._metadata.get(
-                        mm_schemas.StreamProcessingEvent.WHEN
-                    ),
+                    mm_schemas.StreamProcessingEvent.WHEN: when,
                     mm_schemas.StreamProcessingEvent.ENDPOINT_ID: monitoring_data[
                         model
                     ].get(mlrun.common.schemas.MonitoringData.MODEL_ENDPOINT_UID),

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "78045e1e85e7c81eee93682240c4ebe7b22fa67c",
-  "version": "1.10.0-rc16"
+  "git_commit": "2e72ac795fbd5a2a1422956417d742f8dd9bcdb5",
+  "version": "1.10.0-rc17"
 }

{mlrun-1.10.0rc16.dist-info → mlrun-1.10.0rc17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc16
+Version: 1.10.0rc17
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.10.0rc16.dist-info → mlrun-1.10.0rc17.dist-info}/RECORD

@@ -1,6 +1,6 @@
 mlrun/__init__.py,sha256=Y_AFhZV1hEx4vfiO-cyjup0aLGcp6R0SeL75GqLFQrc,7514
 mlrun/__main__.py,sha256=wQNaxW7QsqFBtWffnPkw-497fnpsrQzUnscBQQAP_UM,48364
-mlrun/config.py,sha256=_-WgQPZeSbvd51ndG9qFAaV6TXdDaEgZa9W9B0Imvq8,72399
+mlrun/config.py,sha256=c3F899B20Xe_wpO7cg9G5jEM7iMxfK92LaNxPKnQTyg,72306
 mlrun/errors.py,sha256=bAk0t_qmCxQSPNK0TugOAfA5R6f0G6OYvEvXUWSJ_5U,9062
 mlrun/execution.py,sha256=dJ4PFwg5AlDHbCL2Q9dVDjWA_i64UTq2qBiF8kTU9tw,56922
 mlrun/features.py,sha256=jMEXo6NB36A6iaxNEJWzdtYwUmglYD90OIKTIEeWhE8,15841
@@ -23,7 +23,7 @@ mlrun/artifacts/manager.py,sha256=_cDNCS7wwmFIsucJ2uOgHxZQECmIGb8Wye64b6oLgKU,16
 mlrun/artifacts/model.py,sha256=8EVaD70SOkTohQIWqkDk0MEwskdofxs3wJTgspa2sho,25615
 mlrun/artifacts/plots.py,sha256=wmaxVXiAPSCyn3M7pIlcBu9pP3O8lrq0Ewx6iHRDF9s,4238
 mlrun/common/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
-mlrun/common/constants.py,sha256=bkWWccWZJHB08mkSrrnWuCFXZWdNKf4Q9SwuLowQrVM,4059
+mlrun/common/constants.py,sha256=BDxV8kAAf8D2W-gsa7IP8HJe5da-sPwVUfiodQ1O7kI,4127
 mlrun/common/helpers.py,sha256=DIdqs_eN3gO5bZ8iFobIvx8cEiOxYxhFIyut6-O69T0,1385
 mlrun/common/secrets.py,sha256=8g9xtIw-9DGcwiZRT62a5ozSQM-aYo8yK5Ghey9WM0g,5179
 mlrun/common/types.py,sha256=1gxThbmC0Vd0U1ffIkEwz4T4S7JOgHt70rvw8TCO21c,1073
@@ -41,7 +41,7 @@ mlrun/common/formatters/run.py,sha256=LlqhhVY4dAp5y17k_sWBtHaJogdNdtJWF0iO9sX-bU
 mlrun/common/model_monitoring/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
 mlrun/common/model_monitoring/helpers.py,sha256=AkuHz4u318MEP4ebxmNWlNXh6HiNLrI5oF7QvJiJkYc,2707
 mlrun/common/runtimes/constants.py,sha256=CGMHE2gdsNHXNsa-u3eL0o8sQmDs6PN5FLpMlCDClns,12218
-mlrun/common/schemas/__init__.py,sha256=Mx_N6UtivKP-FQeIvVu6n0unkT7w461lahyNJbaOL_I,5468
+mlrun/common/schemas/__init__.py,sha256=HbnF4nd6jIGbej9Qj6WYjajIr44f3vYNpgWyeLk7N6I,5486
 mlrun/common/schemas/alert.py,sha256=u6INAHBhQIfm-mMsGqDJo1_JDN6gOuWZa-8fOU-aOUE,10182
 mlrun/common/schemas/api_gateway.py,sha256=bgC3vXbyb1SVwsSZkLXtEoQLCe_QDKpIhAVX3X_HWW4,7126
 mlrun/common/schemas/artifact.py,sha256=JojMRRa4n0Rge2olGOpUyp348hkTGsMEnvUBRSoo4oE,4310
@@ -71,11 +71,11 @@ mlrun/common/schemas/runs.py,sha256=yKY29ByTS4SruWQyPpDNFGulMrcT9Ms-3lnwBUDp3us,
 mlrun/common/schemas/runtime_resource.py,sha256=TybJmCHJXmm1z3s5J1dd89TeFE6lG5t7vjcrf1R9YfE,1568
 mlrun/common/schemas/schedule.py,sha256=L7z9Lp06-xmFmdp0q5PypCU_DCl6zZIyQTVoJa01gfM,4291
 mlrun/common/schemas/secret.py,sha256=Td2UAeWHSAdA4nIP3rQv_PIVKVqcBnCnK6xjr528tS8,1486
-mlrun/common/schemas/serving.py,sha256=-3U45YLtmVWMZrx4R8kaPgFGoJ4JmD7RE3nydpYNTz8,1359
+mlrun/common/schemas/serving.py,sha256=4ek9JZDagkdeXyfkX6P6xp4deUNSf_kqXUaXcKSuv-g,1391
 mlrun/common/schemas/tag.py,sha256=1wqEiAujsElojWb3qmuyfcaLFjXSNAAQdafkDx7fkn0,891
-mlrun/common/schemas/workflow.py,sha256=GS6RNp66v7iV2flBMXZs4R5mQr15KJx6PMXFlAjyOmc,2496
-mlrun/common/schemas/model_monitoring/__init__.py,sha256=lQkWiDZagEmZd7pNE_-ySVJEzTjEzH-JS6OKZPmJiVk,1907
-mlrun/common/schemas/model_monitoring/constants.py,sha256=yjTaSGiRs0zYIE20QSuJuMNnS5iuJpnV1wBiq7leVpg,13238
+mlrun/common/schemas/workflow.py,sha256=Y-FHJnxs5c86yetuOAPdEJPkne__tLPCxjSXSb4lrjo,2541
+mlrun/common/schemas/model_monitoring/__init__.py,sha256=FqFiFIDcylquQdY0XTBamB5kMzMrMFEpVYM_ecsVfLg,1925
+mlrun/common/schemas/model_monitoring/constants.py,sha256=spujh8a2GOb7pm8LksAQwndxPWSsjeyRub3ZVICdgNI,13685
 mlrun/common/schemas/model_monitoring/functions.py,sha256=GpfSGp05D87wEKemECD3USL368pvnAM2WfS-nef5qOg,2210
 mlrun/common/schemas/model_monitoring/grafana.py,sha256=THQlLfPBevBksta8p5OaIsBaJtsNSXexLvHrDxOaVns,2095
 mlrun/common/schemas/model_monitoring/model_endpoints.py,sha256=aCrVqgoJsUEwvDJ84YFabDSy78CHcVBV3b0RdWj4JUw,13250
@@ -115,9 +115,9 @@ mlrun/datastore/wasbfs/__init__.py,sha256=s5Ul-0kAhYqFjKDR2X0O2vDGDbLQQduElb32Ev
 mlrun/datastore/wasbfs/fs.py,sha256=ge8NK__5vTcFT-krI155_8RDUywQw4SIRX6BWATXy9Q,6299
 mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
 mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
-mlrun/db/base.py,sha256=afej-ZtJiBBv0MtZx2B1MH1ZRXno4s94aL_Qs5u-WCA,32044
+mlrun/db/base.py,sha256=QUncUfRYep_2Bsui7y-DduAEm8qdgIlWIYmrns-S7HA,32110
 mlrun/db/factory.py,sha256=yP2vVmveUE7LYTCHbS6lQIxP9rW--zdISWuPd_I3d_4,2111
-mlrun/db/httpdb.py,sha256=MiXRygol2qMIMVLDOB89pouR8oICDaR510rVpZtPc4w,239160
+mlrun/db/httpdb.py,sha256=oSzc4zigDgu9S2Vs7Lkz_NY1Oc78OGyHN-xeV19nVtk,239397
 mlrun/db/nopdb.py,sha256=kRWKEEI9LUskI3mp2ofTdAWVLov-99-nSMdaqhi3XT8,28194
 mlrun/feature_store/__init__.py,sha256=SlI845bWt6xX34SXunHHqhmFAR9-5v2ak8N-qpcAPGo,1328
 mlrun/feature_store/api.py,sha256=qKj5Tk6prTab6XWatWhBuPRVp0eJEctoxRMN2wz48vA,32168
@@ -226,10 +226,10 @@ mlrun/launcher/local.py,sha256=3gv-IQYoIChSmRaZ0vLUh0Tu26oLMCx9GbBYh4fWygQ,12161
 mlrun/launcher/remote.py,sha256=zFXE52Cq_7EkC8lfNKT0ceIbye0CfFiundF7O1YU4Xw,7810
 mlrun/model_monitoring/__init__.py,sha256=qDQnncjya9XPTlfvGyfWsZWiXc-glGZrrNja-5QmCZk,782
 mlrun/model_monitoring/api.py,sha256=lAsUp-gzqw8D1cpHVGA2_nPMYn5R4jdxk9UaGOiQ8fE,25945
-mlrun/model_monitoring/controller.py,sha256=CQxK9Sq5k8XonvVBQnSimakpTwMMAyqT5mOaG534MaM,37660
+mlrun/model_monitoring/controller.py,sha256=FVckATzREAzldj68D1KxcnKSdilgKUDRdqhRUf9XpWU,39592
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
 mlrun/model_monitoring/helpers.py,sha256=0xhIYKzhaBrgyjLiA_ekCZsXzi3GBXpLyG40Bhj-PTY,23596
-mlrun/model_monitoring/stream_processing.py,sha256=Gu3TQzYoNjbreZYI73-F49QpYrod9RZOyGSgininBsA,33373
+mlrun/model_monitoring/stream_processing.py,sha256=Mzn9Pelcblw8UzOFLGKb9oXOX0tkP2aoPcFbjtfHhcA,34247
 mlrun/model_monitoring/writer.py,sha256=rGRFzSOkqZWvD3Y6sVk2H1Gepfnkzkp9ce00PsApTLo,8288
 mlrun/model_monitoring/applications/__init__.py,sha256=MaH_n4GiqqQvSkntM5yQ7_FCANtM_IfgK-IJTdo4G_E,757
 mlrun/model_monitoring/applications/_application_steps.py,sha256=t9LDIqQUGE10cyjyhlg0QqN1yVx0apD1HpERYLJfm8U,7409
@@ -277,7 +277,7 @@ mlrun/platforms/iguazio.py,sha256=6VBTq8eQ3mzT96tzjYhAtcMQ2VjF4x8LpIPW5DAcX2Q,13
 mlrun/projects/__init__.py,sha256=hdCOA6_fp8X4qGGGT7Bj7sPbkM1PayWuaVZL0DkpuZw,1240
 mlrun/projects/operations.py,sha256=Rc__P5ucNAY2G-lHc2LrnZs15PUbNFt8-NqNNT2Bjpk,20623
 mlrun/projects/pipelines.py,sha256=nGDzBABEOqoe9sWbax4SfF8CVLgrvK0NLWBadzEthVE,52219
-mlrun/projects/project.py,sha256=B2KCd5bAGMeHOGhEdVr5NKtTOigL6O-A-Bbs2FLvokY,253884
+mlrun/projects/project.py,sha256=a75Sj1lYzWNggTXIKxerSwy52YqNciGvrT2k-ddRmkQ,254149
 mlrun/runtimes/__init__.py,sha256=8cqrYKy1a0_87XG7V_p96untQ4t8RocadM4LVEEN1JM,9029
 mlrun/runtimes/base.py,sha256=FVEooeQMpwxIK2iW1R0FNbC5P1sZ_efKtJcsdNSYNmc,38266
 mlrun/runtimes/daskjob.py,sha256=kR5sDQtXtXY_VGn5Y3mapjEEB5P6Lj30pSrPe1DqsAg,20077
@@ -311,10 +311,10 @@ mlrun/serving/__init__.py,sha256=nriJAcVn5aatwU03T7SsE6ngJEGTxr3wIGt4WuvCCzY,139
 mlrun/serving/merger.py,sha256=pfOQoozUyObCTpqXAMk94PmhZefn4bBrKufO3MKnkAc,6193
 mlrun/serving/remote.py,sha256=Igha2FipK3-6rV_PZ1K464kTbiTu8rhc6SMm-HiEJ6o,18817
 mlrun/serving/routers.py,sha256=SmBOlHn7rT2gWTa-W8f16UB0UthgIFc4D1cPOZAA9ss,54003
-mlrun/serving/server.py,sha256=aFhPlIauww1e2OleZdrQxRiqMmEsoHEdTSujnAZeNe4,35202
+mlrun/serving/server.py,sha256=_P_SR4_7YKqruVzzDHgSPHWlNLGPG5-ksSUwuGhnmjg,38851
 mlrun/serving/serving_wrapper.py,sha256=UL9hhWCfMPcTJO_XrkvNaFvck1U1E7oS8trTZyak0cA,835
-mlrun/serving/states.py,sha256=ZKhlPRlyvbRaGqNueR74IP3q7n8_k-4lYzq0QhPp88Q,124767
-mlrun/serving/system_steps.py,sha256=G-vFdFXGDMZECasT831GxAayitCORNfpxPPRgK5a86w,17960
+mlrun/serving/states.py,sha256=nrwQcWC0q6A6xFSvTWvaDmtSVgv8Fva9TxCuBHGwZHs,124062
+mlrun/serving/system_steps.py,sha256=tCxkJ54peOzRTMaqvHQCbcwx0ITqZkSpGXbtpRUEfzU,18463
 mlrun/serving/utils.py,sha256=Zbfqm8TKNcTE8zRBezVBzpvR2WKeKeIRN7otNIaiYEc,4170
 mlrun/serving/v1_serving.py,sha256=c6J_MtpE-Tqu00-6r4eJOCO6rUasHDal9W2eBIcrl50,11853
 mlrun/serving/v2_serving.py,sha256=257LVOvWxV0KjeY0-Kxro6YgKmPu2QzNne2IORlXi5E,25434
@@ -347,11 +347,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=kfhogR5keR7Zjh0VCjJNK3NR5_yXT7Cv-x9GdOUW4Z8,7294
 mlrun/utils/notifications/notification/webhook.py,sha256=zxh8CAlbPnTazsk6r05X5TKwqUZVOH5KBU2fJbzQlG4,5330
 mlrun/utils/version/__init__.py,sha256=YnzE6tlf24uOQ8y7Z7l96QLAI6-QEii7-77g8ynmzy0,613
-mlrun/utils/version/version.json,sha256=yzjgOoSOXuxZFECc2IfKerszx07aKIlhF7wENYuFF_E,90
+mlrun/utils/version/version.json,sha256=V37lX2OyNEzNPMQsoXG-KqHjDgxsQv-dFqRr2AHbFCk,90
 mlrun/utils/version/version.py,sha256=M2hVhRrgkN3SxacZHs3ZqaOsqAA7B6a22ne324IQ1HE,1877
-mlrun-1.10.0rc16.dist-info/licenses/LICENSE,sha256=zTiv1CxWNkOk1q8eJS1G_8oD4gWpWLwWxj_Agcsi8Os,11337
-mlrun-1.10.0rc16.dist-info/METADATA,sha256=U1uo_EFf7k8D102Lq0z_EQ1K2AOCjwU5iNQuLVvCVjM,26195
-mlrun-1.10.0rc16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mlrun-1.10.0rc16.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.10.0rc16.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.10.0rc16.dist-info/RECORD,,
+mlrun-1.10.0rc17.dist-info/licenses/LICENSE,sha256=zTiv1CxWNkOk1q8eJS1G_8oD4gWpWLwWxj_Agcsi8Os,11337
+mlrun-1.10.0rc17.dist-info/METADATA,sha256=D6HN7BHzSm7-exb9kSE5N2h4m_dPOSsQc-PoM_5avj4,26195
+mlrun-1.10.0rc17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mlrun-1.10.0rc17.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.10.0rc17.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.10.0rc17.dist-info/RECORD,,