mlrun 1.10.0rc27__py3-none-any.whl → 1.10.0rc28__py3-none-any.whl

mlrun/config.py

@@ -255,7 +255,8 @@ default_config = {
             },
             "runtimes": {
                 "dask": "600",
-                "dask_cluster_start": "300",
+                # cluster start might take some time in case k8s needs to spin up new nodes
+                "dask_cluster_start": "600",
             },
             "push_notifications": "60",
         },

mlrun/db/httpdb.py

@@ -5200,7 +5200,7 @@ class HTTPRunDB(RunDBInterface):
 
         :return: A ModelEndpointDriftValues object containing the drift counts over time.
         """
-        endpoint_path = f"projects/{project}/model-endpoints/drift-over-time"
+        endpoint_path = f"projects/{project}/model-monitoring/drift-over-time"
         error_message = f"Failed retrieving drift data for {project}"
         response = self.api_call(
             method="GET",

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -1499,20 +1499,51 @@ class V3IOTSDBConnector(TSDBConnector):
     ) -> mm_schemas.ModelEndpointDriftValues:
         table = mm_schemas.V3IOTSDBTables.APP_RESULTS
         start, end, interval = self._prepare_aligned_start_end(start, end)
-
-        # get per time-interval x endpoint_id combination the max result status
         df = self._get_records(
             table=table,
             start=start,
             end=end,
-            interval=interval,
-            sliding_window_step=interval,
             columns=[mm_schemas.ResultData.RESULT_STATUS],
-            agg_funcs=["max"],
-            group_by=mm_schemas.WriterEvent.ENDPOINT_ID,
         )
+        df = self._aggregate_raw_drift_data(df, start, end, interval)
         if df.empty:
             return mm_schemas.ModelEndpointDriftValues(values=[])
         df = df[df[f"max({mm_schemas.ResultData.RESULT_STATUS})"] >= 1]
-        df = df.reset_index(names="_wstart")
         return self._df_to_drift_data(df)
+
+    @staticmethod
+    def _aggregate_raw_drift_data(
+        df: pd.DataFrame, start: datetime, end: datetime, interval: str
+    ) -> pd.DataFrame:
+        if df.empty:
+            return df
+        if not isinstance(df.index, pd.DatetimeIndex):
+            raise TypeError("Expected a DatetimeIndex on the DataFrame (time index).")
+        df[EventFieldType.ENDPOINT_ID] = (
+            df[EventFieldType.ENDPOINT_ID].astype("string").str.strip()
+        )  # remove extra data carried by the category dtype
+        window = df.loc[
+            (df.index >= start) & (df.index < end),
+            [mm_schemas.ResultData.RESULT_STATUS, EventFieldType.ENDPOINT_ID],
+        ]
+        out = (
+            window.groupby(
+                [
+                    EventFieldType.ENDPOINT_ID,
+                    pd.Grouper(
+                        freq=interval, origin=start, label="left", closed="left"
+                    ),
+                ]
+                # align to start, [start, end) intervals
+            )[mm_schemas.ResultData.RESULT_STATUS]
+            .max()
+            .reset_index()
+            .rename(
+                columns={
+                    mm_schemas.ResultData.RESULT_STATUS: f"max({mm_schemas.ResultData.RESULT_STATUS})"
+                }
+            )
+        )
+        return out.rename(
+            columns={"time": "_wstart"}
+        )  # rename datetime column to _wstart to align with the tdengine result

mlrun/model_monitoring/helpers.py

@@ -659,3 +659,26 @@ def get_start_end(
         )
 
     return start, end
+
+
+def validate_time_range(
+    start: Optional[datetime.datetime] = None, end: Optional[datetime.datetime] = None
+) -> tuple[datetime.datetime, datetime.datetime]:
+    """
+    validate start and end parameters and set default values if needed.
+    :param start:       Either None or datetime, None is handled as datetime.now(tz=timezone.utc) - timedelta(days=1)
+    :param end:         Either None or datetime, None is handled as datetime.now(tz=timezone.utc)
+    :return:            start datetime, end datetime
+    """
+    end = end or mlrun.utils.helpers.datetime_now()
+    start = start or (end - datetime.timedelta(days=1))
+    if start.tzinfo is None or end.tzinfo is None:
+        raise mlrun.errors.MLRunInvalidArgumentTypeError(
+            "Custom start and end times must contain the timezone."
+        )
+    if start > end:
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            "The start time must be before the end time. Note that if end time is not provided, "
+            "the current time is used by default."
+        )
+    return start, end

mlrun/runtimes/base.py

@@ -443,9 +443,11 @@ class BaseRuntime(ModelObj):
         :param runobj: Run context object (RunObject) with run metadata and status
         :return: Dictionary with all the variables that could be parsed
         """
+        active_project = self.metadata.project or config.active_project
         runtime_env = {
-            mlrun_constants.MLRUN_ACTIVE_PROJECT: self.metadata.project
-            or config.active_project
+            mlrun_constants.MLRUN_ACTIVE_PROJECT: active_project,
+            # TODO: Remove this in 1.12.0 as MLRUN_DEFAULT_PROJECT is deprecated and should not be injected anymore
+            "MLRUN_DEFAULT_PROJECT": active_project,
         }
         if runobj:
             runtime_env["MLRUN_EXEC_CONFIG"] = runobj.to_json(

mlrun/utils/helpers.py

@@ -45,6 +45,7 @@ import pytz
 import semver
 import yaml
 from dateutil import parser
+from orjson import orjson
 from pandas import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
@@ -1214,52 +1215,58 @@ def get_workflow_url(
 
 
 def get_kfp_list_runs_filter(
-    project_name: Optional[str] = None,
-    end_date: Optional[str] = None,
     start_date: Optional[str] = None,
+    end_date: Optional[str] = None,
+    filter_: Optional[str] = None,
+    experiment_ids: Optional[list[str]] = None,
 ) -> str:
     """
-    Generates a filter for listing Kubeflow Pipelines (KFP) runs.
-
-    :param project_name: The name of the project. If "*", it won't filter by project.
-    :param end_date: The latest creation date for filtering runs (ISO 8601 format).
-    :param start_date: The earliest creation date for filtering runs (ISO 8601 format).
-    :return: A JSON-formatted filter string for KFP.
+    Generate a filter for KFP runs based on start and end dates, and experiment IDs.
     """
+    existing_filter_object = json.loads(filter_) if filter_ else {"predicates": []}
+    preserved_predicates = [
+        predicate
+        for predicate in existing_filter_object.get("predicates", [])
+        if predicate.get("key") != "name"
+    ]
 
-    # KFP filter operation codes
-    kfp_less_than_or_equal_op = 7  # '<='
-    kfp_greater_than_or_equal_op = 5  # '>='
-    kfp_substring_op = 9  # Substring match
-
-    filters = {"predicates": []}
-
+    new_predicates = []
     if end_date:
-        filters["predicates"].append(
+        new_predicates.append(
             {
-                "key": "created_at",
-                "op": kfp_less_than_or_equal_op,
+                "key": mlrun_pipelines.models.FilterFields.CREATED_AT,
+                "op": mlrun_pipelines.models.FilterOperations.LESS_THAN_EQUALS.value,
                 "timestamp_value": end_date,
             }
         )
 
-    if project_name and project_name != "*":
-        filters["predicates"].append(
+    if start_date:
+        new_predicates.append(
             {
-                "key": "name",
-                "op": kfp_substring_op,
-                "string_value": project_name,
+                "key": mlrun_pipelines.models.FilterFields.CREATED_AT,
+                "op": mlrun_pipelines.models.FilterOperations.GREATER_THAN_EQUALS.value,
+                "timestamp_value": start_date,
             }
         )
-    if start_date:
-        filters["predicates"].append(
+
+    if experiment_ids and all(experiment_ids):
+        new_predicates.append(
             {
-                "key": "created_at",
-                "op": kfp_greater_than_or_equal_op,
-                "timestamp_value": start_date,
+                "key": mlrun_pipelines.models.FilterFields.EXPERIMENT_ID,
+                "op": mlrun_pipelines.models.FilterOperations.IN.value,
+                "string_values": {"values": experiment_ids},
             }
         )
-    return json.dumps(filters)
+
+    final_filter_object = {"predicates": preserved_predicates + new_predicates}
+    if not final_filter_object["predicates"]:
+        return ""
+
+    logger.debug(
+        "Generated KFP runs filter",
+        filter_object_with_predicates=final_filter_object,
+    )
+    return orjson.dumps(final_filter_object).decode()
 
 
 def validate_and_convert_date(date_input: str) -> str:

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "8b5d7492c858f675bcf8fce28d536c9af13a8d86",
-  "version": "1.10.0-rc27"
+  "git_commit": "ec44cbf076b74e7961c4597ce7b9d92dcaf87f90",
+  "version": "1.10.0-rc28"
 }

{mlrun-1.10.0rc27.dist-info → mlrun-1.10.0rc28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc27
+Version: 1.10.0rc28
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.10.0rc27.dist-info → mlrun-1.10.0rc28.dist-info}/RECORD

@@ -1,6 +1,6 @@
 mlrun/__init__.py,sha256=JYy9uteFFNPbPoC0geDEPhaLrfiqTijxUhLZSToAky4,8029
 mlrun/__main__.py,sha256=wQNaxW7QsqFBtWffnPkw-497fnpsrQzUnscBQQAP_UM,48364
-mlrun/config.py,sha256=3FWf5jbJDJrwfipSXoiNWOQ5A2Vy4FcZl7Ai8Pv4yIg,73057
+mlrun/config.py,sha256=F1PDI88t2cFujGnDr4YslEBzG6SckBKjUSdkxFX3zUE,73149
 mlrun/errors.py,sha256=bAk0t_qmCxQSPNK0TugOAfA5R6f0G6OYvEvXUWSJ_5U,9062
 mlrun/execution.py,sha256=wkmT1k0QROgGJFMBIsYUsJaqEF2bkqaYVzp_ZQb527Q,58814
 mlrun/features.py,sha256=jMEXo6NB36A6iaxNEJWzdtYwUmglYD90OIKTIEeWhE8,15841
@@ -118,7 +118,7 @@ mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
 mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
 mlrun/db/base.py,sha256=D4P8jhsp4j3ZPg2tKRTgFgm4hrGAx7kVnmio9qfJZDI,32295
 mlrun/db/factory.py,sha256=yP2vVmveUE7LYTCHbS6lQIxP9rW--zdISWuPd_I3d_4,2111
-mlrun/db/httpdb.py,sha256=-Jl7WsZDFcTSpiSmBrutLoMPGALDjpDYI899i3F9hcw,238788
+mlrun/db/httpdb.py,sha256=q0AggpqHdG5oLukZQUsUOXG26eeXlrvdcTacueNbabY,238789
 mlrun/db/nopdb.py,sha256=SZqCaCnaijT8-vivdVjj0VvZcZyqzat4YFFhOJlrTtI,28661
 mlrun/feature_store/__init__.py,sha256=SlI845bWt6xX34SXunHHqhmFAR9-5v2ak8N-qpcAPGo,1328
 mlrun/feature_store/api.py,sha256=qKj5Tk6prTab6XWatWhBuPRVp0eJEctoxRMN2wz48vA,32168
@@ -229,7 +229,7 @@ mlrun/model_monitoring/__init__.py,sha256=qDQnncjya9XPTlfvGyfWsZWiXc-glGZrrNja-5
 mlrun/model_monitoring/api.py,sha256=k0eOm-vW8z2u05PwMK2PI2mSAplK0xGIrUe_XWk7mRM,27000
 mlrun/model_monitoring/controller.py,sha256=2XOkOZRB03K9ph6TH-ICspHga-GQOURL0C8-0GTHaTY,43961
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
-mlrun/model_monitoring/helpers.py,sha256=VQ_afyhqzFMOqPdqpkpX3md_afjlkFtsUWIg28zxp5g,23596
+mlrun/model_monitoring/helpers.py,sha256=50oFqgIc5xFHYPIVgq3M-Gbr7epqAI5NgHmvOeMy52U,24667
 mlrun/model_monitoring/stream_processing.py,sha256=bryYO3D0cC10MAQ-liHxUZ79MrL-VFXCb7KNyj6bl-8,34655
 mlrun/model_monitoring/writer.py,sha256=rGRFzSOkqZWvD3Y6sVk2H1Gepfnkzkp9ce00PsApTLo,8288
 mlrun/model_monitoring/applications/__init__.py,sha256=BwlmRELlFJf2b2YMyv5kUSHNe8--OyqWhDgRlT8a_8g,779
@@ -253,7 +253,7 @@ mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py,sha256=dtkaHaWKWE
 mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=Vj8eWZ6jxXs9nTlo5Du1jJjYutwSNp4ZtztvKsnrr4M,51333
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=sNQFj6qyJx5eSBKRC3gyTc1cfh1l2IkRpPtuZwtzCW0,6844
-mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=fLzZrtZvJQk22eBT-zUzJfvbrlFwyBB3FbTSgivdLNQ,60242
+mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=3GNMudOpedhu_OId4Gp-r9nj1gtqh_353yn2gWta-BY,61459
 mlrun/model_monitoring/metrics/__init__.py,sha256=6CsTXAxeLbbf8yfCADTaxmiavqwrLEdYFJ-qc5kgDAY,569
 mlrun/model_monitoring/metrics/histogram_distance.py,sha256=E9_WIl2vd6qNvoHVHoFcnuQk3ekbFWOdi8aU7sHrfk4,4724
 mlrun/package/__init__.py,sha256=v7VDyK9kDOOuDvFo4oiGV2fx-vM1KL7fdN9pGLakhUQ,7008
@@ -280,7 +280,7 @@ mlrun/projects/operations.py,sha256=Rc__P5ucNAY2G-lHc2LrnZs15PUbNFt8-NqNNT2Bjpk,
 mlrun/projects/pipelines.py,sha256=ZOfuIEHOXfuc4qAkuWvbWhCjP6kqpLkv-yBBaY9RXhg,52219
 mlrun/projects/project.py,sha256=xbh6qXfU9ckkDSTcd-LyPWxHtHzz6gsqDI63ZZQPuhs,256413
 mlrun/runtimes/__init__.py,sha256=8cqrYKy1a0_87XG7V_p96untQ4t8RocadM4LVEEN1JM,9029
-mlrun/runtimes/base.py,sha256=8AMJcTnm9_LSEKLlmmbEzGSCRAoCh3vAQudKzuIVXhY,38285
+mlrun/runtimes/base.py,sha256=LyrF2SmyNN2uBRL-5bUzFxTm186pXywjnFmu5mhtNtA,38483
 mlrun/runtimes/daskjob.py,sha256=IN6gKKrmCIjWooj5FgFm-pAb2i7ra1ERRzClfu_rYGI,20102
 mlrun/runtimes/funcdoc.py,sha256=zRFHrJsV8rhDLJwoUhcfZ7Cs0j-tQ76DxwUqdXV_Wyc,9810
 mlrun/runtimes/function_reference.py,sha256=fnMKUEieKgy4JyVLhFpDtr6JvKgOaQP8F_K2H3-Pk9U,5030
@@ -329,7 +329,7 @@ mlrun/utils/async_http.py,sha256=8Olx8TNNeXB07nEGwlqhEgFgnFAD71vBU_bqaA9JW-w,122
 mlrun/utils/azure_vault.py,sha256=IEFizrDGDbAaoWwDr1WoA88S_EZ0T--vjYtY-i0cvYQ,3450
 mlrun/utils/clones.py,sha256=qbAGyEbSvlewn3Tw_DpQZP9z6MGzFhSaZfI1CblX8Fg,7515
 mlrun/utils/condition_evaluator.py,sha256=-nGfRmZzivn01rHTroiGY4rqEv8T1irMyhzxEei-sKc,1897
-mlrun/utils/helpers.py,sha256=cN_cVAaJh5T2xE_KgiXTWlXDLM_waIi8PDV3NBj6ioE,83104
+mlrun/utils/helpers.py,sha256=dCISzUwBjGztqsl27-WrznT485nZa7ZliLF_WXw2i_4,83457
 mlrun/utils/http.py,sha256=5ZU2VpokaUM_DT3HBSqTm8xjUqTPjZN5fKkSIvKlTl0,8704
 mlrun/utils/logger.py,sha256=uaCgI_ezzaXf7nJDCy-1Nrjds8vSXqDbzmjmb3IyCQo,14864
 mlrun/utils/regex.py,sha256=FcRwWD8x9X3HLhCCU2F0AVKTFah784Pr7ZAe3a02jw8,5199
@@ -348,11 +348,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=wSu_7W0EnGLBNwIgWCYEeTP8j9SPAMPDBnfUcPnVZYA,7299
 mlrun/utils/notifications/notification/webhook.py,sha256=FM5-LQAKAVJKp37MRzR3SsejalcnpM6r_9Oe7znxZEA,5313
 mlrun/utils/version/__init__.py,sha256=YnzE6tlf24uOQ8y7Z7l96QLAI6-QEii7-77g8ynmzy0,613
-mlrun/utils/version/version.json,sha256=4XxnjcO5M_JMzeQSt05p2f7kyBOjoyz8_C6-mXrlqUs,90
+mlrun/utils/version/version.json,sha256=8LqPKs3owy8BezmiKv4mJzJ7Q6092ElXwEN7UTAst3U,90
 mlrun/utils/version/version.py,sha256=M2hVhRrgkN3SxacZHs3ZqaOsqAA7B6a22ne324IQ1HE,1877
-mlrun-1.10.0rc27.dist-info/licenses/LICENSE,sha256=zTiv1CxWNkOk1q8eJS1G_8oD4gWpWLwWxj_Agcsi8Os,11337
-mlrun-1.10.0rc27.dist-info/METADATA,sha256=2-9lgsReCFdnK-z8X5FCT2rdbRvTrkUGErsiib9H1To,26017
-mlrun-1.10.0rc27.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mlrun-1.10.0rc27.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.10.0rc27.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.10.0rc27.dist-info/RECORD,,
+mlrun-1.10.0rc28.dist-info/licenses/LICENSE,sha256=zTiv1CxWNkOk1q8eJS1G_8oD4gWpWLwWxj_Agcsi8Os,11337
+mlrun-1.10.0rc28.dist-info/METADATA,sha256=whhfF5YIKLUim-iWuo6lB4V9mzSCh99eaYisXKLAnEI,26017
+mlrun-1.10.0rc28.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mlrun-1.10.0rc28.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.10.0rc28.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.10.0rc28.dist-info/RECORD,,