mlrun 1.10.0rc16__py3-none-any.whl → 1.10.0rc18__py3-none-any.whl

mlrun/serving/server.py

@@ -22,8 +22,10 @@ import os
 import socket
 import traceback
 import uuid
+from datetime import datetime, timezone
 from typing import Any, Optional, Union
 
+import pandas as pd
 import storey
 from nuclio import Context as NuclioContext
 from nuclio.request import Logger as NuclioLogger
@@ -40,6 +42,7 @@ from mlrun.secrets import SecretsStore
 
 from ..common.helpers import parse_versioned_object_uri
 from ..common.schemas.model_monitoring.constants import FileTargetKind
+from ..common.schemas.serving import MAX_BATCH_JOB_DURATION
 from ..datastore import DataItem, get_stream_pusher
 from ..datastore.store_resources import ResourceCache
 from ..errors import MLRunInvalidArgumentError
@@ -561,6 +564,7 @@ def v2_serving_init(context, namespace=None):
 async def async_execute_graph(
     context: MLClientCtx,
     data: DataItem,
+    timestamp_column: Optional[str],
     batching: bool,
     batch_size: Optional[int],
     read_as_lists: bool,
@@ -605,10 +609,43 @@ async def async_execute_graph(
                 f"(status='{task_state}')"
             )
 
+    df = data.as_df()
+
+    if df.empty:
+        context.logger.warn("Job terminated due to empty inputs (0 rows)")
+        return []
+
+    track_models = spec.get("track_models")
+
+    if track_models and timestamp_column:
+        context.logger.info(f"Sorting dataframe by {timestamp_column}")
+        df[timestamp_column] = pd.to_datetime(  # in case it's a string
+            df[timestamp_column]
+        )
+        df.sort_values(by=timestamp_column, inplace=True)
+        if len(df) > 1:
+            start_time = df[timestamp_column].iloc[0]
+            end_time = df[timestamp_column].iloc[-1]
+            time_range = end_time - start_time
+            start_time = start_time.isoformat()
+            end_time = end_time.isoformat()
+            # TODO: tie this to the controller's base period
+            if time_range > pd.Timedelta(MAX_BATCH_JOB_DURATION):
+                raise mlrun.errors.MLRunRuntimeError(
+                    f"Dataframe time range is too long: {time_range}. "
+                    "Please disable tracking or reduce the input dataset's time range below the defined limit "
+                    f"of {MAX_BATCH_JOB_DURATION}."
+                )
+        else:
+            start_time = end_time = df["timestamp"].iloc[0].isoformat()
+    else:
+        # end time will be set from clock time when the batch completes
+        start_time = datetime.now(tz=timezone.utc).isoformat()
+
     server.graph = add_system_steps_to_graph(
         server.project,
         copy.deepcopy(server.graph),
-        spec.get("track_models"),
+        track_models,
         context,
         spec,
         pause_until_background_task_completion=False,  # we've already awaited it
@@ -633,19 +670,28 @@ async def async_execute_graph(
     if server.verbose:
         context.logger.info(server.to_yaml())
 
-    df = data.as_df()
-
-    responses = []
-
     async def run(body):
         event = storey.Event(id=index, body=body)
-        response = await server.run(event, context)
-        responses.append(response)
+        if timestamp_column:
+            if batching:
+                # we use the first row in the batch to determine the timestamp for the whole batch
+                body = body[0]
+            if not isinstance(body, dict):
+                raise mlrun.errors.MLRunRuntimeError(
+                    f"When timestamp_column=True, event body must be a dict – got {type(body).__name__} instead"
+                )
+            if timestamp_column not in body:
+                raise mlrun.errors.MLRunRuntimeError(
+                    f"Event body '{body}' did not contain timestamp column '{timestamp_column}'"
+                )
+            event._original_timestamp = body[timestamp_column]
+        return await server.run(event, context)
 
     if batching and not batch_size:
         batch_size = len(df)
 
     batch = []
+    tasks = []
     for index, row in df.iterrows():
         data = row.to_list() if read_as_lists else row.to_dict()
         if nest_under_inputs:
@@ -653,24 +699,56 @@ async def async_execute_graph(
         if batching:
             batch.append(data)
             if len(batch) == batch_size:
-                await run(batch)
+                tasks.append(asyncio.create_task(run(batch)))
                 batch = []
         else:
-            await run(data)
+            tasks.append(asyncio.create_task(run(data)))
 
     if batch:
-        await run(batch)
+        tasks.append(asyncio.create_task(run(batch)))
+
+    responses = await asyncio.gather(*tasks)
 
     termination_result = server.wait_for_completion()
     if asyncio.iscoroutine(termination_result):
         await termination_result
 
+    model_endpoint_uids = spec.get("model_endpoint_uids", [])
+
+    # needed for output_stream to be created
+    server = GraphServer.from_dict(spec)
+    server.init_states(None, namespace)
+
+    batch_completion_time = datetime.now(tz=timezone.utc).isoformat()
+
+    if not timestamp_column:
+        end_time = batch_completion_time
+
+    mm_stream_record = dict(
+        kind="batch_complete",
+        project=context.project,
+        first_timestamp=start_time,
+        last_timestamp=end_time,
+        batch_completion_time=batch_completion_time,
+    )
+    output_stream = server.context.stream.output_stream
+    for mep_uid in spec.get("model_endpoint_uids", []):
+        mm_stream_record["endpoint_id"] = mep_uid
+        output_stream.push(mm_stream_record, partition_key=mep_uid)
+
+    context.logger.info(
+        f"Job completed processing {len(df)} rows",
+        timestamp_column=timestamp_column,
+        model_endpoint_uids=model_endpoint_uids,
+    )
+
     return responses
 
 
 def execute_graph(
     context: MLClientCtx,
     data: DataItem,
+    timestamp_column: Optional[str] = None,
     batching: bool = False,
     batch_size: Optional[int] = None,
     read_as_lists: bool = False,
@@ -681,6 +759,9 @@ def execute_graph(
 
     :param context: The job's execution client context.
     :param data: The input data to the job, to be pushed into the graph row by row, or in batches.
+    :param timestamp_column: The name of the column that will be used as the timestamp for model monitoring purposes.
+        when timestamp_column is used in conjunction with batching, the first timestamp will be used for the entire
+        batch.
     :param batching: Whether to push one or more batches into the graph rather than row by row.
     :param batch_size: The number of rows to push per batch. If not set, and batching=True, the entire dataset will
         be pushed into the graph in one batch.
@@ -691,7 +772,13 @@ def execute_graph(
     """
     return asyncio.run(
         async_execute_graph(
-            context, data, batching, batch_size, read_as_lists, nest_under_inputs
+            context,
+            data,
+            timestamp_column,
+            batching,
+            batch_size,
+            read_as_lists,
+            nest_under_inputs,
         )
     )
 

mlrun/serving/states.py

@@ -1206,7 +1206,7 @@ class Model(storey.ParallelExecutionRunnable, ModelObj):
 
 class LLModel(Model):
     def __init__(
-        self, name: str, input_path: Optional[Union[str, list[str]]], **kwargs
+        self, name: str, input_path: Optional[Union[str, list[str]]] = None, **kwargs
     ):
         super().__init__(name, **kwargs)
         self._input_path = split_path(input_path)
@@ -1754,9 +1754,10 @@ class ModelRunnerStep(MonitoredStep):
         except (
             mlrun.errors.MLRunNotFoundError,
             mlrun.errors.MLRunInvalidArgumentError,
-        ):
+        ) as ex:
             logger.warning(
-                f"Model endpoint not found, using default output schema for model {name}"
+                f"Model endpoint not found, using default output schema for model {name}",
+                error=f"{type(ex).__name__}: {ex}",
             )
         return output_schema
 
@@ -1768,22 +1769,6 @@ class ModelRunnerStep(MonitoredStep):
         )
         if isinstance(monitoring_data, dict):
             for model in monitoring_data:
-                monitoring_data[model][schemas.MonitoringData.OUTPUTS] = (
-                    monitoring_data.get(model, {}).get(schemas.MonitoringData.OUTPUTS)
-                    or self._get_model_endpoint_output_schema(
-                        name=model,
-                        project=self.context.project if self.context else None,
-                        uid=monitoring_data.get(model, {}).get(
-                            mlrun.common.schemas.MonitoringData.MODEL_ENDPOINT_UID
-                        ),
-                    )
-                )
-                # Prevent calling _get_model_output_schema for same model more than once
-                self.class_args[
-                    mlrun.common.schemas.ModelRunnerStepData.MONITORING_DATA
-                ][model][schemas.MonitoringData.OUTPUTS] = monitoring_data[model][
-                    schemas.MonitoringData.OUTPUTS
-                ]
                 monitoring_data[model][schemas.MonitoringData.INPUT_PATH] = split_path(
                     monitoring_data[model][schemas.MonitoringData.INPUT_PATH]
                 )
@@ -1791,6 +1776,10 @@ class ModelRunnerStep(MonitoredStep):
                     monitoring_data[model][schemas.MonitoringData.RESULT_PATH]
                 )
             return monitoring_data
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Monitoring data must be a dictionary."
+            )
 
     def init_object(self, context, namespace, mode="sync", reset=False, **extra_kwargs):
         self.context = context

mlrun/serving/system_steps.py

@@ -104,7 +104,7 @@ class MonitoringPreProcessor(storey.MapClass):
     @staticmethod
     def transpose_by_key(
         data: dict, schema: Optional[Union[str, list[str]]] = None
-    ) -> Union[list[float], list[list[float]]]:
+    ) -> Union[list[Any], list[list[Any]]]:
         """
         Transpose values from a dictionary by keys.
 
@@ -146,7 +146,11 @@ class MonitoringPreProcessor(storey.MapClass):
         else:
             keys = schema
 
-        values = [data[key] for key in keys]
+        values = [data[key] for key in keys if key in data]
+        if len(values) != len(keys):
+            raise mlrun.MLRunInvalidArgumentError(
+                f"Schema keys {keys} do not match the data keys {list(data.keys())}."
+            )
 
         # Detect if all are scalars ie: int,float,str
         all_scalars = all(not isinstance(v, (list, tuple, np.ndarray)) for v in values)
@@ -158,9 +162,9 @@ class MonitoringPreProcessor(storey.MapClass):
             )
 
         if all_scalars:
-            transposed = np.array([values])
+            transposed = np.array([values], dtype=object)
         elif all_lists and len(keys) > 1:
-            arrays = [np.array(v) for v in values]
+            arrays = [np.array(v, dtype=object) for v in values]
             mat = np.stack(arrays, axis=0)
             transposed = mat.T
         else:
@@ -192,6 +196,12 @@ class MonitoringPreProcessor(storey.MapClass):
                     request, resp = self.reconstruct_request_resp_fields(
                         event, model, monitoring_data[model]
                     )
+                    if hasattr(event, "_original_timestamp"):
+                        when = event._original_timestamp
+                    else:
+                        when = event._metadata.get(model, {}).get(
+                            mm_schemas.StreamProcessingEvent.WHEN
+                        )
                     monitoring_event_list.append(
                         {
                             mm_schemas.StreamProcessingEvent.MODEL: model,
@@ -201,9 +211,7 @@ class MonitoringPreProcessor(storey.MapClass):
                             mm_schemas.StreamProcessingEvent.MICROSEC: event._metadata.get(
                                 model, {}
                             ).get(mm_schemas.StreamProcessingEvent.MICROSEC),
-                            mm_schemas.StreamProcessingEvent.WHEN: event._metadata.get(
-                                model, {}
-                            ).get(mm_schemas.StreamProcessingEvent.WHEN),
+                            mm_schemas.StreamProcessingEvent.WHEN: when,
                             mm_schemas.StreamProcessingEvent.ENDPOINT_ID: monitoring_data[
                                 model
                             ].get(
@@ -236,6 +244,10 @@ class MonitoringPreProcessor(storey.MapClass):
             request, resp = self.reconstruct_request_resp_fields(
                 event, model, monitoring_data[model]
             )
+            if hasattr(event, "_original_timestamp"):
+                when = event._original_timestamp
+            else:
+                when = event._metadata.get(mm_schemas.StreamProcessingEvent.WHEN)
             monitoring_event_list.append(
                 {
                     mm_schemas.StreamProcessingEvent.MODEL: model,
@@ -245,9 +257,7 @@ class MonitoringPreProcessor(storey.MapClass):
                     mm_schemas.StreamProcessingEvent.MICROSEC: event._metadata.get(
                         mm_schemas.StreamProcessingEvent.MICROSEC
                     ),
-                    mm_schemas.StreamProcessingEvent.WHEN: event._metadata.get(
-                        mm_schemas.StreamProcessingEvent.WHEN
-                    ),
+                    mm_schemas.StreamProcessingEvent.WHEN: when,
                     mm_schemas.StreamProcessingEvent.ENDPOINT_ID: monitoring_data[
                         model
                     ].get(mlrun.common.schemas.MonitoringData.MODEL_ENDPOINT_UID),

mlrun/utils/helpers.py

@@ -800,7 +800,12 @@ def remove_tag_from_artifact_uri(uri: str) -> Optional[str]:
         "store://key:tag" => "store://key"
         "store://models/remote-model-project/my_model#0@tree" => unchanged (no tag)
     """
-    return re.sub(r"(?<=/[^/:]\+):[^@^:\s]+(?=(@|\^|$))", "", uri)
+    add_store = False
+    if mlrun.datastore.is_store_uri(uri):
+        uri = uri.removeprefix(DB_SCHEMA + "://")
+        add_store = True
+    uri = re.sub(r"(#[^:@\s]*)?:[^@^:\s]+(?=(@|\^|$))", lambda m: m.group(1) or "", uri)
+    return uri if not add_store else DB_SCHEMA + "://" + uri
 
 
 def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:

mlrun/utils/logger.py

@@ -393,12 +393,14 @@ def resolve_formatter_by_kind(
 
 
 def create_test_logger(name: str = "mlrun", stream: IO[str] = stdout) -> Logger:
-    return create_logger(
+    logger = create_logger(
         level="debug",
         formatter_kind=FormatterKinds.HUMAN_EXTENDED.name,
         name=name,
         stream=stream,
     )
+    logger._logger.propagate = True  # pass records up to pytest’s handler
+    return logger
 
 
 def create_logger(

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "78045e1e85e7c81eee93682240c4ebe7b22fa67c",
-  "version": "1.10.0-rc16"
+  "git_commit": "ef58f2d7ada676ba266aaf084b7a46714b95ba4e",
+  "version": "1.10.0-rc18"
 }

{mlrun-1.10.0rc16.dist-info → mlrun-1.10.0rc18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc16
+Version: 1.10.0rc18
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -44,7 +44,7 @@ Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<2024.7,>=2023.9.2
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.10.8
+Requires-Dist: storey~=1.10.9
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2

{mlrun-1.10.0rc16.dist-info → mlrun-1.10.0rc18.dist-info}/RECORD

@@ -1,6 +1,6 @@
 mlrun/__init__.py,sha256=Y_AFhZV1hEx4vfiO-cyjup0aLGcp6R0SeL75GqLFQrc,7514
 mlrun/__main__.py,sha256=wQNaxW7QsqFBtWffnPkw-497fnpsrQzUnscBQQAP_UM,48364
-mlrun/config.py,sha256=_-WgQPZeSbvd51ndG9qFAaV6TXdDaEgZa9W9B0Imvq8,72399
+mlrun/config.py,sha256=XAAb68MwEHpuPddPMtKBULtFk0hI9YC25DniYQk1DKk,72853
 mlrun/errors.py,sha256=bAk0t_qmCxQSPNK0TugOAfA5R6f0G6OYvEvXUWSJ_5U,9062
 mlrun/execution.py,sha256=dJ4PFwg5AlDHbCL2Q9dVDjWA_i64UTq2qBiF8kTU9tw,56922
 mlrun/features.py,sha256=jMEXo6NB36A6iaxNEJWzdtYwUmglYD90OIKTIEeWhE8,15841
@@ -8,7 +8,7 @@ mlrun/k8s_utils.py,sha256=mMnGyouHoJC93ZD2KGf9neJM1pD7mR9IXLnHOEwYVTQ,21469
 mlrun/lists.py,sha256=OlaV2QIFUzmenad9kxNJ3k4whlDyxI3zFbGwr6vpC5Y,8561
 mlrun/model.py,sha256=wHtM8LylSOEFk6Hxl95CVm8DOPhofjsANYdIvKHH6dw,88956
 mlrun/render.py,sha256=5DlhD6JtzHgmj5RVlpaYiHGhX84Q7qdi4RCEUj2UMgw,13195
-mlrun/run.py,sha256=_ban8NoNWfQHps3QIVyWh_Hn2S6usNrFBTUMObaeueo,46904
+mlrun/run.py,sha256=6I4I_88Slpj27WzGnoq0c02SNpDaGsJllC8Y-2BED0E,46951
 mlrun/secrets.py,sha256=dZPdkc_zzfscVQepOHUwmzFqnBavDCBXV9DQoH_eIYM,7800
 mlrun/alerts/__init__.py,sha256=0gtG1BG0DXxFrXegIkjbM1XEN4sP9ODo0ucXrNld1hU,601
 mlrun/alerts/alert.py,sha256=QQFZGydQbx9RvAaSiaH-ALQZVcDKQX5lgizqj_rXW2k,15948
@@ -23,14 +23,14 @@ mlrun/artifacts/manager.py,sha256=_cDNCS7wwmFIsucJ2uOgHxZQECmIGb8Wye64b6oLgKU,16
 mlrun/artifacts/model.py,sha256=8EVaD70SOkTohQIWqkDk0MEwskdofxs3wJTgspa2sho,25615
 mlrun/artifacts/plots.py,sha256=wmaxVXiAPSCyn3M7pIlcBu9pP3O8lrq0Ewx6iHRDF9s,4238
 mlrun/common/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
-mlrun/common/constants.py,sha256=bkWWccWZJHB08mkSrrnWuCFXZWdNKf4Q9SwuLowQrVM,4059
+mlrun/common/constants.py,sha256=BDxV8kAAf8D2W-gsa7IP8HJe5da-sPwVUfiodQ1O7kI,4127
 mlrun/common/helpers.py,sha256=DIdqs_eN3gO5bZ8iFobIvx8cEiOxYxhFIyut6-O69T0,1385
 mlrun/common/secrets.py,sha256=8g9xtIw-9DGcwiZRT62a5ozSQM-aYo8yK5Ghey9WM0g,5179
 mlrun/common/types.py,sha256=1gxThbmC0Vd0U1ffIkEwz4T4S7JOgHt70rvw8TCO21c,1073
 mlrun/common/db/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
 mlrun/common/db/dialects.py,sha256=QN9bx7CTo32IIdJ2J3ZrsX8IUdp_BPxBtl0LyjMEC9g,868
 mlrun/common/formatters/__init__.py,sha256=au7S3M3wa9964RpQhFSvflk5-i5SWMeb3kek8Gvt4kg,889
-mlrun/common/formatters/artifact.py,sha256=NT2RPfReUGWVQ-Rhv1zexgXfLuKI5FvtneJDhihZK5g,1462
+mlrun/common/formatters/artifact.py,sha256=JRs1B3nEvTdn4h69FLHjZZY_Dy40E9-BgvxfjcL_DKU,1503
 mlrun/common/formatters/base.py,sha256=85vQ0t4ZfqCs8b8QV1RLfRcEvANztKvTvsa__sD3zTo,4099
 mlrun/common/formatters/feature_set.py,sha256=SmSSiNYXZtmzQDCvQJhIpArtjOInmDN0g1xFqrDdWsw,1534
 mlrun/common/formatters/function.py,sha256=H8bHilWSn1NXm5WWlFH2dx8slomCGm3e8ghsmWOOh48,1493
@@ -41,7 +41,7 @@ mlrun/common/formatters/run.py,sha256=LlqhhVY4dAp5y17k_sWBtHaJogdNdtJWF0iO9sX-bU
 mlrun/common/model_monitoring/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
 mlrun/common/model_monitoring/helpers.py,sha256=AkuHz4u318MEP4ebxmNWlNXh6HiNLrI5oF7QvJiJkYc,2707
 mlrun/common/runtimes/constants.py,sha256=CGMHE2gdsNHXNsa-u3eL0o8sQmDs6PN5FLpMlCDClns,12218
-mlrun/common/schemas/__init__.py,sha256=Mx_N6UtivKP-FQeIvVu6n0unkT7w461lahyNJbaOL_I,5468
+mlrun/common/schemas/__init__.py,sha256=HbnF4nd6jIGbej9Qj6WYjajIr44f3vYNpgWyeLk7N6I,5486
 mlrun/common/schemas/alert.py,sha256=u6INAHBhQIfm-mMsGqDJo1_JDN6gOuWZa-8fOU-aOUE,10182
 mlrun/common/schemas/api_gateway.py,sha256=bgC3vXbyb1SVwsSZkLXtEoQLCe_QDKpIhAVX3X_HWW4,7126
 mlrun/common/schemas/artifact.py,sha256=JojMRRa4n0Rge2olGOpUyp348hkTGsMEnvUBRSoo4oE,4310
@@ -71,11 +71,11 @@ mlrun/common/schemas/runs.py,sha256=yKY29ByTS4SruWQyPpDNFGulMrcT9Ms-3lnwBUDp3us,
 mlrun/common/schemas/runtime_resource.py,sha256=TybJmCHJXmm1z3s5J1dd89TeFE6lG5t7vjcrf1R9YfE,1568
 mlrun/common/schemas/schedule.py,sha256=L7z9Lp06-xmFmdp0q5PypCU_DCl6zZIyQTVoJa01gfM,4291
 mlrun/common/schemas/secret.py,sha256=Td2UAeWHSAdA4nIP3rQv_PIVKVqcBnCnK6xjr528tS8,1486
-mlrun/common/schemas/serving.py,sha256=-3U45YLtmVWMZrx4R8kaPgFGoJ4JmD7RE3nydpYNTz8,1359
+mlrun/common/schemas/serving.py,sha256=4ek9JZDagkdeXyfkX6P6xp4deUNSf_kqXUaXcKSuv-g,1391
 mlrun/common/schemas/tag.py,sha256=1wqEiAujsElojWb3qmuyfcaLFjXSNAAQdafkDx7fkn0,891
-mlrun/common/schemas/workflow.py,sha256=GS6RNp66v7iV2flBMXZs4R5mQr15KJx6PMXFlAjyOmc,2496
-mlrun/common/schemas/model_monitoring/__init__.py,sha256=lQkWiDZagEmZd7pNE_-ySVJEzTjEzH-JS6OKZPmJiVk,1907
-mlrun/common/schemas/model_monitoring/constants.py,sha256=yjTaSGiRs0zYIE20QSuJuMNnS5iuJpnV1wBiq7leVpg,13238
+mlrun/common/schemas/workflow.py,sha256=Y-FHJnxs5c86yetuOAPdEJPkne__tLPCxjSXSb4lrjo,2541
+mlrun/common/schemas/model_monitoring/__init__.py,sha256=FqFiFIDcylquQdY0XTBamB5kMzMrMFEpVYM_ecsVfLg,1925
+mlrun/common/schemas/model_monitoring/constants.py,sha256=WhuUWgOwk91BI0dP5c1rm6X_W0V4UBV3l6KvRNHHE-E,13898
 mlrun/common/schemas/model_monitoring/functions.py,sha256=GpfSGp05D87wEKemECD3USL368pvnAM2WfS-nef5qOg,2210
 mlrun/common/schemas/model_monitoring/grafana.py,sha256=THQlLfPBevBksta8p5OaIsBaJtsNSXexLvHrDxOaVns,2095
 mlrun/common/schemas/model_monitoring/model_endpoints.py,sha256=aCrVqgoJsUEwvDJ84YFabDSy78CHcVBV3b0RdWj4JUw,13250
@@ -88,8 +88,8 @@ mlrun/datastore/__init__.py,sha256=UKGHkUfj5A9dqTiOeW4UScvwv0vq91NS_xs3-_QA7Jk,5
 mlrun/datastore/alibaba_oss.py,sha256=E0t0-e9Me2t2Mux2LWdC9riOG921TgNjhoy897JJX7o,4932
 mlrun/datastore/azure_blob.py,sha256=3LG7tOTwT97ZFBmyq-sfAIe5_SkuFgisRQtipv4kKUw,12779
 mlrun/datastore/base.py,sha256=yLdnFCL2k_rcasdbxXjnQr7Lwm-A79LnW9AITtn9-p4,25450
-mlrun/datastore/datastore.py,sha256=IbAUi2GZesWcmqPimURZD9iP5GpCB8se7GYA9afkAjA,13236
-mlrun/datastore/datastore_profile.py,sha256=fRR01v9mqC40UML49-6rdx-28b2FaRLj7iZeWJgSiT0,22842
+mlrun/datastore/datastore.py,sha256=gOlMyPDelD9CRieoraDPYf1NNig_GrQRuuQxLmRq8Bo,13298
+mlrun/datastore/datastore_profile.py,sha256=Y4VtaatIK4UXuTdpffCkAcsCBSxj5KOgnX7KlL-Yds8,23803
 mlrun/datastore/dbfs_store.py,sha256=CJwst1598qxiu63-Qa0c3e5E8LjeCv1XbMyWI7A6irY,6560
 mlrun/datastore/filestore.py,sha256=OcykjzhbUAZ6_Cb9bGAXRL2ngsOpxXSb4rR0lyogZtM,3773
 mlrun/datastore/google_cloud_storage.py,sha256=NREwZT7BCI0HfmOGkjpy5S3psiL_rgQSi43MaazJcKk,8711
@@ -105,19 +105,20 @@ mlrun/datastore/spark_utils.py,sha256=dn0RWpYzee-M8UZw-NVuHAdqlNAZ7VO-fNtI8ZiDky
 mlrun/datastore/store_resources.py,sha256=s2794zqkzy_mjRMvRedDNs_tycTLoF8wxTqsWRQphCE,6839
 mlrun/datastore/storeytargets.py,sha256=TvHbY3XS0qOg8ImXW1ET61UnjUPcYJbfSGAga3vgC-o,6492
 mlrun/datastore/targets.py,sha256=8dRnLy1wBYJbVyommYkpGeztdT1CsfFHZY6Zh7o8X-Q,79165
-mlrun/datastore/utils.py,sha256=7qhGMRos-IgV2o-dywWbMvCGQDlbRs_CZcwfvTxZTvw,11798
+mlrun/datastore/utils.py,sha256=jxvq4lgQfgqb7dwKe4Kp51fYCCyOvitEdIfV2mzlqxg,11936
 mlrun/datastore/v3io.py,sha256=sMn5473k_bXyIJovNf0rahbVHRmO0YPdOwIhbs06clg,8201
 mlrun/datastore/vectorstore.py,sha256=k-yom5gfw20hnVG0Rg7aBEehuXwvAloZwn0cx0VGals,11708
 mlrun/datastore/model_provider/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
-mlrun/datastore/model_provider/model_provider.py,sha256=eDkTdwFsVxSFzTkYUwyXfl34d5IIulcQrLRjxWM2BFc,7911
-mlrun/datastore/model_provider/openai_provider.py,sha256=KmuE9i3EbeOc7QACvFLwnU2n3i_PmIw429BaZC0Y9Zw,8463
+mlrun/datastore/model_provider/huggingface_provider.py,sha256=KTmErt_WHhAV9t8803_iCHoa7jO-0y-7bch7KMTMDKo,7264
+mlrun/datastore/model_provider/model_provider.py,sha256=dJIc1R0wbExsk-uzWgpt9w_FK027mr4lTBxYpQx-icY,8083
+mlrun/datastore/model_provider/openai_provider.py,sha256=Tl3HXLDOtHkp54rb0ZCddlCWU-gUruhFJwSC6ocbv1Y,9157
 mlrun/datastore/wasbfs/__init__.py,sha256=s5Ul-0kAhYqFjKDR2X0O2vDGDbLQQduElb32Ev56Te4,1343
 mlrun/datastore/wasbfs/fs.py,sha256=ge8NK__5vTcFT-krI155_8RDUywQw4SIRX6BWATXy9Q,6299
 mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
 mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
-mlrun/db/base.py,sha256=afej-ZtJiBBv0MtZx2B1MH1ZRXno4s94aL_Qs5u-WCA,32044
+mlrun/db/base.py,sha256=QUncUfRYep_2Bsui7y-DduAEm8qdgIlWIYmrns-S7HA,32110
 mlrun/db/factory.py,sha256=yP2vVmveUE7LYTCHbS6lQIxP9rW--zdISWuPd_I3d_4,2111
-mlrun/db/httpdb.py,sha256=MiXRygol2qMIMVLDOB89pouR8oICDaR510rVpZtPc4w,239160
+mlrun/db/httpdb.py,sha256=oSzc4zigDgu9S2Vs7Lkz_NY1Oc78OGyHN-xeV19nVtk,239397
 mlrun/db/nopdb.py,sha256=kRWKEEI9LUskI3mp2ofTdAWVLov-99-nSMdaqhi3XT8,28194
 mlrun/feature_store/__init__.py,sha256=SlI845bWt6xX34SXunHHqhmFAR9-5v2ak8N-qpcAPGo,1328
 mlrun/feature_store/api.py,sha256=qKj5Tk6prTab6XWatWhBuPRVp0eJEctoxRMN2wz48vA,32168
@@ -225,18 +226,18 @@ mlrun/launcher/factory.py,sha256=RW7mfzEFi8fR0M-4W1JQg1iq3_muUU6OTqT_3l4Ubrk,233
 mlrun/launcher/local.py,sha256=3gv-IQYoIChSmRaZ0vLUh0Tu26oLMCx9GbBYh4fWygQ,12161
 mlrun/launcher/remote.py,sha256=zFXE52Cq_7EkC8lfNKT0ceIbye0CfFiundF7O1YU4Xw,7810
 mlrun/model_monitoring/__init__.py,sha256=qDQnncjya9XPTlfvGyfWsZWiXc-glGZrrNja-5QmCZk,782
-mlrun/model_monitoring/api.py,sha256=lAsUp-gzqw8D1cpHVGA2_nPMYn5R4jdxk9UaGOiQ8fE,25945
-mlrun/model_monitoring/controller.py,sha256=CQxK9Sq5k8XonvVBQnSimakpTwMMAyqT5mOaG534MaM,37660
+mlrun/model_monitoring/api.py,sha256=9UsBIy8LYeAOoiIDTYthuD0mx1TYZxeGgaEM_H2qBkM,26092
+mlrun/model_monitoring/controller.py,sha256=FVckATzREAzldj68D1KxcnKSdilgKUDRdqhRUf9XpWU,39592
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
 mlrun/model_monitoring/helpers.py,sha256=0xhIYKzhaBrgyjLiA_ekCZsXzi3GBXpLyG40Bhj-PTY,23596
-mlrun/model_monitoring/stream_processing.py,sha256=Gu3TQzYoNjbreZYI73-F49QpYrod9RZOyGSgininBsA,33373
+mlrun/model_monitoring/stream_processing.py,sha256=Mzn9Pelcblw8UzOFLGKb9oXOX0tkP2aoPcFbjtfHhcA,34247
 mlrun/model_monitoring/writer.py,sha256=rGRFzSOkqZWvD3Y6sVk2H1Gepfnkzkp9ce00PsApTLo,8288
 mlrun/model_monitoring/applications/__init__.py,sha256=MaH_n4GiqqQvSkntM5yQ7_FCANtM_IfgK-IJTdo4G_E,757
 mlrun/model_monitoring/applications/_application_steps.py,sha256=t9LDIqQUGE10cyjyhlg0QqN1yVx0apD1HpERYLJfm8U,7409
-mlrun/model_monitoring/applications/base.py,sha256=nkJK0X3-7xEjcWGjtfLmvMomi5F14mmWs_M0_L8op5o,40547
+mlrun/model_monitoring/applications/base.py,sha256=tfxXcE7WOvPEd68b6gbZWFG-8gkeeSaRRN0G0HYn0C8,43932
 mlrun/model_monitoring/applications/context.py,sha256=fAGFNCyNhSnVJPSIeJxv-XmEL2JhDmjK5Ouog9qyvdc,17035
 mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=2qgfFmrpHf-x0_EaHD-0T28piwSQzw-HH71aV1GwbZs,15389
-mlrun/model_monitoring/applications/results.py,sha256=_qmj6TWT0SR2bi7gUyRKBU418eGgGoLW2_hTJ7S-ock,5782
+mlrun/model_monitoring/applications/results.py,sha256=LfBQOmkpKGvVGNrcj5QiXsRIG2IRgcv_Xqe4QJBmauk,5699
 mlrun/model_monitoring/applications/evidently/__init__.py,sha256=-DqdPnBSrjZhFvKOu_Ie3MiFvlur9sPTZpZ1u0_1AE8,690
 mlrun/model_monitoring/applications/evidently/base.py,sha256=shH9YwuFrGNWy1IDAbv622l-GE4o1z_u1bqhqTyTHDA,5661
 mlrun/model_monitoring/db/__init__.py,sha256=r47xPGZpIfMuv8J3PQCZTSqVPMhUta4sSJCZFKcS7FM,644
@@ -277,7 +278,7 @@ mlrun/platforms/iguazio.py,sha256=6VBTq8eQ3mzT96tzjYhAtcMQ2VjF4x8LpIPW5DAcX2Q,13
 mlrun/projects/__init__.py,sha256=hdCOA6_fp8X4qGGGT7Bj7sPbkM1PayWuaVZL0DkpuZw,1240
 mlrun/projects/operations.py,sha256=Rc__P5ucNAY2G-lHc2LrnZs15PUbNFt8-NqNNT2Bjpk,20623
 mlrun/projects/pipelines.py,sha256=nGDzBABEOqoe9sWbax4SfF8CVLgrvK0NLWBadzEthVE,52219
-mlrun/projects/project.py,sha256=B2KCd5bAGMeHOGhEdVr5NKtTOigL6O-A-Bbs2FLvokY,253884
+mlrun/projects/project.py,sha256=a75Sj1lYzWNggTXIKxerSwy52YqNciGvrT2k-ddRmkQ,254149
 mlrun/runtimes/__init__.py,sha256=8cqrYKy1a0_87XG7V_p96untQ4t8RocadM4LVEEN1JM,9029
 mlrun/runtimes/base.py,sha256=FVEooeQMpwxIK2iW1R0FNbC5P1sZ_efKtJcsdNSYNmc,38266
 mlrun/runtimes/daskjob.py,sha256=kR5sDQtXtXY_VGn5Y3mapjEEB5P6Lj30pSrPe1DqsAg,20077
@@ -311,10 +312,10 @@ mlrun/serving/__init__.py,sha256=nriJAcVn5aatwU03T7SsE6ngJEGTxr3wIGt4WuvCCzY,139
 mlrun/serving/merger.py,sha256=pfOQoozUyObCTpqXAMk94PmhZefn4bBrKufO3MKnkAc,6193
 mlrun/serving/remote.py,sha256=Igha2FipK3-6rV_PZ1K464kTbiTu8rhc6SMm-HiEJ6o,18817
 mlrun/serving/routers.py,sha256=SmBOlHn7rT2gWTa-W8f16UB0UthgIFc4D1cPOZAA9ss,54003
-mlrun/serving/server.py,sha256=aFhPlIauww1e2OleZdrQxRiqMmEsoHEdTSujnAZeNe4,35202
+mlrun/serving/server.py,sha256=_P_SR4_7YKqruVzzDHgSPHWlNLGPG5-ksSUwuGhnmjg,38851
 mlrun/serving/serving_wrapper.py,sha256=UL9hhWCfMPcTJO_XrkvNaFvck1U1E7oS8trTZyak0cA,835
-mlrun/serving/states.py,sha256=ZKhlPRlyvbRaGqNueR74IP3q7n8_k-4lYzq0QhPp88Q,124767
-mlrun/serving/system_steps.py,sha256=G-vFdFXGDMZECasT831GxAayitCORNfpxPPRgK5a86w,17960
+mlrun/serving/states.py,sha256=-eo1IGLm96Kd0bTrwj211nUcwomMizQ6MxrqtURNxAg,124069
+mlrun/serving/system_steps.py,sha256=tCxkJ54peOzRTMaqvHQCbcwx0ITqZkSpGXbtpRUEfzU,18463
 mlrun/serving/utils.py,sha256=Zbfqm8TKNcTE8zRBezVBzpvR2WKeKeIRN7otNIaiYEc,4170
 mlrun/serving/v1_serving.py,sha256=c6J_MtpE-Tqu00-6r4eJOCO6rUasHDal9W2eBIcrl50,11853
 mlrun/serving/v2_serving.py,sha256=257LVOvWxV0KjeY0-Kxro6YgKmPu2QzNne2IORlXi5E,25434
@@ -328,9 +329,9 @@ mlrun/utils/async_http.py,sha256=8Olx8TNNeXB07nEGwlqhEgFgnFAD71vBU_bqaA9JW-w,122
 mlrun/utils/azure_vault.py,sha256=IEFizrDGDbAaoWwDr1WoA88S_EZ0T--vjYtY-i0cvYQ,3450
 mlrun/utils/clones.py,sha256=qbAGyEbSvlewn3Tw_DpQZP9z6MGzFhSaZfI1CblX8Fg,7515
 mlrun/utils/condition_evaluator.py,sha256=-nGfRmZzivn01rHTroiGY4rqEv8T1irMyhzxEei-sKc,1897
-mlrun/utils/helpers.py,sha256=ympa9GWzF_NMNemKSas29s8I0-2seHGhfErFT1b-6tY,82419
+mlrun/utils/helpers.py,sha256=fMlwtYBzUK98nCTFCA5FGm1imqIdpJi0CuAPmO10YZs,82641
 mlrun/utils/http.py,sha256=5ZU2VpokaUM_DT3HBSqTm8xjUqTPjZN5fKkSIvKlTl0,8704
-mlrun/utils/logger.py,sha256=RG0m1rx6gfkJ-2C1r_p41MMpPiaDYqaYM2lYHDlNZEU,14767
+mlrun/utils/logger.py,sha256=uaCgI_ezzaXf7nJDCy-1Nrjds8vSXqDbzmjmb3IyCQo,14864
 mlrun/utils/regex.py,sha256=FcRwWD8x9X3HLhCCU2F0AVKTFah784Pr7ZAe3a02jw8,5199
 mlrun/utils/retryer.py,sha256=SHddxyNdUjIyvNJ3idTDyBzXARihCSuo3zWlZj6fqB0,7852
 mlrun/utils/singleton.py,sha256=fNOfAUtha6OPCV_M1umWnGD0iabnnRwBke9otIspv30,868
@@ -347,11 +348,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=kfhogR5keR7Zjh0VCjJNK3NR5_yXT7Cv-x9GdOUW4Z8,7294
 mlrun/utils/notifications/notification/webhook.py,sha256=zxh8CAlbPnTazsk6r05X5TKwqUZVOH5KBU2fJbzQlG4,5330
 mlrun/utils/version/__init__.py,sha256=YnzE6tlf24uOQ8y7Z7l96QLAI6-QEii7-77g8ynmzy0,613
-mlrun/utils/version/version.json,sha256=yzjgOoSOXuxZFECc2IfKerszx07aKIlhF7wENYuFF_E,90
+mlrun/utils/version/version.json,sha256=1hAlhHbCP3fWtgQa3l5zk4ROJntlDhuVxmDS5HRBMiY,90
 mlrun/utils/version/version.py,sha256=M2hVhRrgkN3SxacZHs3ZqaOsqAA7B6a22ne324IQ1HE,1877
-mlrun-1.10.0rc16.dist-info/licenses/LICENSE,sha256=zTiv1CxWNkOk1q8eJS1G_8oD4gWpWLwWxj_Agcsi8Os,11337
-mlrun-1.10.0rc16.dist-info/METADATA,sha256=U1uo_EFf7k8D102Lq0z_EQ1K2AOCjwU5iNQuLVvCVjM,26195
-mlrun-1.10.0rc16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mlrun-1.10.0rc16.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.10.0rc16.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.10.0rc16.dist-info/RECORD,,
+mlrun-1.10.0rc18.dist-info/licenses/LICENSE,sha256=zTiv1CxWNkOk1q8eJS1G_8oD4gWpWLwWxj_Agcsi8Os,11337
+mlrun-1.10.0rc18.dist-info/METADATA,sha256=Rx-oVfq1W7TA73agpQeaFDhrevZ_RYVTXG6ormcXWvk,26195
+mlrun-1.10.0rc18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mlrun-1.10.0rc18.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.10.0rc18.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.10.0rc18.dist-info/RECORD,,