mlrun 1.10.0rc27__tar.gz → 1.10.0rc28__tar.gz

{mlrun-1.10.0rc27 → mlrun-1.10.0rc28}/Makefile

@@ -656,8 +656,6 @@ test: clean ## Run mlrun tests
 	--ignore=server/py/services/api/migrations \
 	") && \
 	PER_PYTHON_VERSION_IGNORE_TEST_FLAGS=$(if $(filter $(MLRUN_PYTHON_VERSION),3.11),$$(echo "\
-		--ignore=server/py/services/api/tests/unit/api/test_pipelines.py \
-		--ignore=server/py/services/api/tests/unit/crud/test_pipelines.py \
 		--ignore=tests/serving/test_remote.py \
 		--ignore=tests/projects/test_remote_pipeline.py \
 		--ignore=tests/frameworks/tf_keras/test_tf_keras.py \

{mlrun-1.10.0rc27/mlrun.egg-info → mlrun-1.10.0rc28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc27
+Version: 1.10.0rc28
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.10.0rc27 → mlrun-1.10.0rc28}/mlrun/config.py

@@ -255,7 +255,8 @@ default_config = {
             },
             "runtimes": {
                 "dask": "600",
-                "dask_cluster_start": "300",
+                # cluster start might take some time in case k8s needs to spin up new nodes
+                "dask_cluster_start": "600",
             },
             "push_notifications": "60",
         },

{mlrun-1.10.0rc27 → mlrun-1.10.0rc28}/mlrun/db/httpdb.py

@@ -5200,7 +5200,7 @@ class HTTPRunDB(RunDBInterface):
 
         :return: A ModelEndpointDriftValues object containing the drift counts over time.
         """
-        endpoint_path = f"projects/{project}/model-endpoints/drift-over-time"
+        endpoint_path = f"projects/{project}/model-monitoring/drift-over-time"
         error_message = f"Failed retrieving drift data for {project}"
         response = self.api_call(
             method="GET",

{mlrun-1.10.0rc27 → mlrun-1.10.0rc28}/mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -1499,20 +1499,51 @@ class V3IOTSDBConnector(TSDBConnector):
     ) -> mm_schemas.ModelEndpointDriftValues:
         table = mm_schemas.V3IOTSDBTables.APP_RESULTS
         start, end, interval = self._prepare_aligned_start_end(start, end)
-
-        # get per time-interval x endpoint_id combination the max result status
         df = self._get_records(
             table=table,
             start=start,
             end=end,
-            interval=interval,
-            sliding_window_step=interval,
             columns=[mm_schemas.ResultData.RESULT_STATUS],
-            agg_funcs=["max"],
-            group_by=mm_schemas.WriterEvent.ENDPOINT_ID,
         )
+        df = self._aggregate_raw_drift_data(df, start, end, interval)
         if df.empty:
             return mm_schemas.ModelEndpointDriftValues(values=[])
         df = df[df[f"max({mm_schemas.ResultData.RESULT_STATUS})"] >= 1]
-        df = df.reset_index(names="_wstart")
         return self._df_to_drift_data(df)
+
+    @staticmethod
+    def _aggregate_raw_drift_data(
+        df: pd.DataFrame, start: datetime, end: datetime, interval: str
+    ) -> pd.DataFrame:
+        if df.empty:
+            return df
+        if not isinstance(df.index, pd.DatetimeIndex):
+            raise TypeError("Expected a DatetimeIndex on the DataFrame (time index).")
+        df[EventFieldType.ENDPOINT_ID] = (
+            df[EventFieldType.ENDPOINT_ID].astype("string").str.strip()
+        )  # remove extra data carried by the category dtype
+        window = df.loc[
+            (df.index >= start) & (df.index < end),
+            [mm_schemas.ResultData.RESULT_STATUS, EventFieldType.ENDPOINT_ID],
+        ]
+        out = (
+            window.groupby(
+                [
+                    EventFieldType.ENDPOINT_ID,
+                    pd.Grouper(
+                        freq=interval, origin=start, label="left", closed="left"
+                    ),
+                ]
+                # align to start, [start, end) intervals
+            )[mm_schemas.ResultData.RESULT_STATUS]
+            .max()
+            .reset_index()
+            .rename(
+                columns={
+                    mm_schemas.ResultData.RESULT_STATUS: f"max({mm_schemas.ResultData.RESULT_STATUS})"
+                }
+            )
+        )
+        return out.rename(
+            columns={"time": "_wstart"}
+        )  # rename datetime column to _wstart to align with the tdengine result

{mlrun-1.10.0rc27 → mlrun-1.10.0rc28}/mlrun/model_monitoring/helpers.py

@@ -659,3 +659,26 @@ def get_start_end(
         )
 
     return start, end
+
+
+def validate_time_range(
+    start: Optional[datetime.datetime] = None, end: Optional[datetime.datetime] = None
+) -> tuple[datetime.datetime, datetime.datetime]:
+    """
+    validate start and end parameters and set default values if needed.
+    :param start:       Either None or datetime, None is handled as datetime.now(tz=timezone.utc) - timedelta(days=1)
+    :param end:         Either None or datetime, None is handled as datetime.now(tz=timezone.utc)
+    :return:            start datetime, end datetime
+    """
+    end = end or mlrun.utils.helpers.datetime_now()
+    start = start or (end - datetime.timedelta(days=1))
+    if start.tzinfo is None or end.tzinfo is None:
+        raise mlrun.errors.MLRunInvalidArgumentTypeError(
+            "Custom start and end times must contain the timezone."
+        )
+    if start > end:
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            "The start time must be before the end time. Note that if end time is not provided, "
+            "the current time is used by default."
+        )
+    return start, end

{mlrun-1.10.0rc27 → mlrun-1.10.0rc28}/mlrun/runtimes/base.py

@@ -443,9 +443,11 @@ class BaseRuntime(ModelObj):
         :param runobj: Run context object (RunObject) with run metadata and status
         :return: Dictionary with all the variables that could be parsed
         """
+        active_project = self.metadata.project or config.active_project
         runtime_env = {
-            mlrun_constants.MLRUN_ACTIVE_PROJECT: self.metadata.project
-            or config.active_project
+            mlrun_constants.MLRUN_ACTIVE_PROJECT: active_project,
+            # TODO: Remove this in 1.12.0 as MLRUN_DEFAULT_PROJECT is deprecated and should not be injected anymore
+            "MLRUN_DEFAULT_PROJECT": active_project,
         }
         if runobj:
             runtime_env["MLRUN_EXEC_CONFIG"] = runobj.to_json(

{mlrun-1.10.0rc27 → mlrun-1.10.0rc28}/mlrun/utils/helpers.py

@@ -45,6 +45,7 @@ import pytz
 import semver
 import yaml
 from dateutil import parser
+from orjson import orjson
 from pandas import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
@@ -1214,52 +1215,58 @@ def get_workflow_url(
 
 
 def get_kfp_list_runs_filter(
-    project_name: Optional[str] = None,
-    end_date: Optional[str] = None,
     start_date: Optional[str] = None,
+    end_date: Optional[str] = None,
+    filter_: Optional[str] = None,
+    experiment_ids: Optional[list[str]] = None,
 ) -> str:
     """
-    Generates a filter for listing Kubeflow Pipelines (KFP) runs.
-
-    :param project_name: The name of the project. If "*", it won't filter by project.
-    :param end_date: The latest creation date for filtering runs (ISO 8601 format).
-    :param start_date: The earliest creation date for filtering runs (ISO 8601 format).
-    :return: A JSON-formatted filter string for KFP.
+    Generate a filter for KFP runs based on start and end dates, and experiment IDs.
     """
+    existing_filter_object = json.loads(filter_) if filter_ else {"predicates": []}
+    preserved_predicates = [
+        predicate
+        for predicate in existing_filter_object.get("predicates", [])
+        if predicate.get("key") != "name"
+    ]
 
-    # KFP filter operation codes
-    kfp_less_than_or_equal_op = 7  # '<='
-    kfp_greater_than_or_equal_op = 5  # '>='
-    kfp_substring_op = 9  # Substring match
-
-    filters = {"predicates": []}
-
+    new_predicates = []
     if end_date:
-        filters["predicates"].append(
+        new_predicates.append(
             {
-                "key": "created_at",
-                "op": kfp_less_than_or_equal_op,
+                "key": mlrun_pipelines.models.FilterFields.CREATED_AT,
+                "op": mlrun_pipelines.models.FilterOperations.LESS_THAN_EQUALS.value,
                 "timestamp_value": end_date,
             }
         )
 
-    if project_name and project_name != "*":
-        filters["predicates"].append(
+    if start_date:
+        new_predicates.append(
             {
-                "key": "name",
-                "op": kfp_substring_op,
-                "string_value": project_name,
+                "key": mlrun_pipelines.models.FilterFields.CREATED_AT,
+                "op": mlrun_pipelines.models.FilterOperations.GREATER_THAN_EQUALS.value,
+                "timestamp_value": start_date,
             }
         )
-    if start_date:
-        filters["predicates"].append(
+
+    if experiment_ids and all(experiment_ids):
+        new_predicates.append(
             {
-                "key": "created_at",
-                "op": kfp_greater_than_or_equal_op,
-                "timestamp_value": start_date,
+                "key": mlrun_pipelines.models.FilterFields.EXPERIMENT_ID,
+                "op": mlrun_pipelines.models.FilterOperations.IN.value,
+                "string_values": {"values": experiment_ids},
             }
         )
-    return json.dumps(filters)
+
+    final_filter_object = {"predicates": preserved_predicates + new_predicates}
+    if not final_filter_object["predicates"]:
+        return ""
+
+    logger.debug(
+        "Generated KFP runs filter",
+        filter_object_with_predicates=final_filter_object,
+    )
+    return orjson.dumps(final_filter_object).decode()
 
 
 def validate_and_convert_date(date_input: str) -> str:

mlrun-1.10.0rc28/mlrun/utils/version/version.json

@@ -0,0 +1,4 @@
+{
+  "git_commit": "ec44cbf076b74e7961c4597ce7b9d92dcaf87f90",
+  "version": "1.10.0-rc28"
+}

{mlrun-1.10.0rc27 → mlrun-1.10.0rc28/mlrun.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc27
+Version: 1.10.0rc28
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

mlrun-1.10.0rc27/mlrun/utils/version/version.json

@@ -1,4 +0,0 @@
-{
-  "git_commit": "8b5d7492c858f675bcf8fce28d536c9af13a8d86",
-  "version": "1.10.0-rc27"
-}