mlrun 1.10.0rc4__py3-none-any.whl → 1.10.0rc6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (36) hide show
  1. mlrun/artifacts/model.py +3 -3
  2. mlrun/common/schemas/model_monitoring/__init__.py +1 -0
  3. mlrun/common/schemas/model_monitoring/constants.py +14 -2
  4. mlrun/common/schemas/model_monitoring/functions.py +66 -0
  5. mlrun/common/schemas/project.py +3 -0
  6. mlrun/config.py +7 -4
  7. mlrun/db/base.py +13 -0
  8. mlrun/db/httpdb.py +47 -0
  9. mlrun/db/nopdb.py +12 -0
  10. mlrun/launcher/client.py +23 -0
  11. mlrun/model_monitoring/applications/base.py +9 -5
  12. mlrun/model_monitoring/db/tsdb/base.py +30 -0
  13. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +118 -50
  14. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +117 -24
  15. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +106 -15
  16. mlrun/projects/project.py +40 -1
  17. mlrun/runtimes/base.py +0 -27
  18. mlrun/runtimes/daskjob.py +12 -5
  19. mlrun/runtimes/databricks_job/databricks_runtime.py +0 -2
  20. mlrun/runtimes/mpijob/abstract.py +0 -2
  21. mlrun/runtimes/mpijob/v1.py +0 -2
  22. mlrun/runtimes/nuclio/application/application.py +0 -5
  23. mlrun/runtimes/nuclio/function.py +0 -11
  24. mlrun/runtimes/nuclio/serving.py +0 -6
  25. mlrun/runtimes/pod.py +1 -3
  26. mlrun/runtimes/remotesparkjob.py +0 -2
  27. mlrun/runtimes/sparkjob/spark3job.py +0 -2
  28. mlrun/serving/states.py +16 -18
  29. mlrun/utils/helpers.py +17 -1
  30. mlrun/utils/version/version.json +2 -2
  31. {mlrun-1.10.0rc4.dist-info → mlrun-1.10.0rc6.dist-info}/METADATA +2 -1
  32. {mlrun-1.10.0rc4.dist-info → mlrun-1.10.0rc6.dist-info}/RECORD +36 -35
  33. {mlrun-1.10.0rc4.dist-info → mlrun-1.10.0rc6.dist-info}/WHEEL +0 -0
  34. {mlrun-1.10.0rc4.dist-info → mlrun-1.10.0rc6.dist-info}/entry_points.txt +0 -0
  35. {mlrun-1.10.0rc4.dist-info → mlrun-1.10.0rc6.dist-info}/licenses/LICENSE +0 -0
  36. {mlrun-1.10.0rc4.dist-info → mlrun-1.10.0rc6.dist-info}/top_level.txt +0 -0
@@ -12,8 +12,8 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
+ import threading
15
16
  from datetime import datetime, timedelta
16
- from threading import Lock
17
17
  from typing import Callable, Final, Literal, Optional, Union
18
18
 
19
19
  import pandas as pd
@@ -32,8 +32,8 @@ from mlrun.model_monitoring.db.tsdb.tdengine.tdengine_connection import (
32
32
  from mlrun.model_monitoring.helpers import get_invocations_fqn
33
33
  from mlrun.utils import logger
34
34
 
35
- _connection = None
36
- _connection_lock = Lock()
35
+ # Thread-local storage for connections
36
+ _thread_local = threading.local()
37
37
 
38
38
 
39
39
  class TDEngineTimestampPrecision(mlrun.common.types.StrEnum):
@@ -76,16 +76,15 @@ class TDEngineConnector(TSDBConnector):
76
76
 
77
77
  @property
78
78
  def connection(self) -> TDEngineConnection:
79
- global _connection
80
-
81
- if _connection:
82
- return _connection
83
-
84
- with _connection_lock:
85
- if not _connection:
86
- _connection = self._create_connection()
87
-
88
- return _connection
79
+ if not hasattr(_thread_local, "connection"):
80
+ _thread_local.connection = self._create_connection()
81
+ logger.debug(
82
+ "Created new TDEngine connection for thread",
83
+ project=self.project,
84
+ thread_name=threading.current_thread().name,
85
+ thread_id=threading.get_ident(),
86
+ )
87
+ return _thread_local.connection
89
88
 
90
89
  def _create_connection(self) -> TDEngineConnection:
91
90
  """Establish a connection to the TSDB server."""
@@ -204,14 +203,27 @@ class TDEngineConnector(TSDBConnector):
204
203
  return datetime.fromisoformat(val) if isinstance(val, str) else val
205
204
 
206
205
  @staticmethod
207
- def _get_endpoint_filter(endpoint_id: Union[str, list[str]]) -> str:
208
- if isinstance(endpoint_id, str):
209
- return f"endpoint_id='{endpoint_id}'"
210
- elif isinstance(endpoint_id, list):
211
- return f"endpoint_id IN({str(endpoint_id)[1:-1]}) "
206
+ def _generate_filter_query(
207
+ filter_column: str, filter_values: Union[str, list[Union[str, int]]]
208
+ ) -> Optional[str]:
209
+ """
210
+ Generate a filter query for TDEngine based on the provided column and values.
211
+
212
+ :param filter_column: The column to filter by.
213
+ :param filter_values: A single value or a list of values to filter by.
214
+
215
+ :return: A string representing the filter query.
216
+ :raise: MLRunInvalidArgumentError if the filter values are not of type string or list.
217
+ """
218
+
219
+ if isinstance(filter_values, str):
220
+ return f"{filter_column}='{filter_values}'"
221
+ elif isinstance(filter_values, list):
222
+ return f"{filter_column} IN ({', '.join(repr(v) for v in filter_values)}) "
212
223
  else:
213
224
  raise mlrun.errors.MLRunInvalidArgumentError(
214
- "Invalid 'endpoint_id' filter: must be a string or a list."
225
+ f"Invalid filter values {filter_values}: must be a string or a list, "
226
+ f"got {type(filter_values).__name__}; filter values: {filter_values}"
215
227
  )
216
228
 
217
229
  def _drop_database_query(self) -> str:
@@ -673,7 +685,10 @@ class TDEngineConnector(TSDBConnector):
673
685
  start: Optional[datetime] = None,
674
686
  end: Optional[datetime] = None,
675
687
  ) -> pd.DataFrame:
676
- filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
688
+ filter_query = self._generate_filter_query(
689
+ filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
690
+ filter_values=endpoint_ids,
691
+ )
677
692
  start, end = self._get_start_end(start, end)
678
693
  df = self._get_records(
679
694
  table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
@@ -714,7 +729,10 @@ class TDEngineConnector(TSDBConnector):
714
729
  end: Optional[datetime] = None,
715
730
  get_raw: bool = False,
716
731
  ) -> pd.DataFrame:
717
- filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
732
+ filter_query = self._generate_filter_query(
733
+ filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
734
+ filter_values=endpoint_ids,
735
+ )
718
736
  start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
719
737
  start, end = self._get_start_end(start, end)
720
738
  df = self._get_records(
@@ -741,6 +759,72 @@ class TDEngineConnector(TSDBConnector):
741
759
  df.dropna(inplace=True)
742
760
  return df
743
761
 
762
+ def count_results_by_status(
763
+ self,
764
+ start: Optional[Union[datetime, str]] = None,
765
+ end: Optional[Union[datetime, str]] = None,
766
+ endpoint_ids: Optional[Union[str, list[str]]] = None,
767
+ application_names: Optional[Union[str, list[str]]] = None,
768
+ result_status_list: Optional[list[int]] = None,
769
+ ) -> dict[tuple[str, int], int]:
770
+ filter_query = ""
771
+ now = mlrun.utils.datetime_now()
772
+ start = start or (now - timedelta(hours=24))
773
+ end = end or now
774
+ if endpoint_ids:
775
+ filter_query = self._generate_filter_query(
776
+ filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
777
+ filter_values=endpoint_ids,
778
+ )
779
+ if application_names:
780
+ app_filter_query = self._generate_filter_query(
781
+ filter_column=mm_schemas.ApplicationEvent.APPLICATION_NAME,
782
+ filter_values=application_names,
783
+ )
784
+ if filter_query:
785
+ filter_query += f" AND {app_filter_query}"
786
+ else:
787
+ filter_query = app_filter_query
788
+ if result_status_list:
789
+ status_filter_query = self._generate_filter_query(
790
+ filter_column=mm_schemas.ResultData.RESULT_STATUS,
791
+ filter_values=result_status_list,
792
+ )
793
+ if filter_query:
794
+ filter_query += f" AND {status_filter_query}"
795
+ else:
796
+ filter_query = status_filter_query
797
+
798
+ df = self._get_records(
799
+ table=self.tables[mm_schemas.TDEngineSuperTables.APP_RESULTS].super_table,
800
+ start=start,
801
+ end=end,
802
+ columns=[
803
+ mm_schemas.WriterEvent.APPLICATION_NAME,
804
+ mm_schemas.ResultData.RESULT_STATUS,
805
+ mm_schemas.ResultData.RESULT_VALUE,
806
+ ],
807
+ filter_query=filter_query,
808
+ timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
809
+ group_by=[
810
+ mm_schemas.WriterEvent.APPLICATION_NAME,
811
+ mm_schemas.ResultData.RESULT_STATUS,
812
+ ],
813
+ agg_funcs=["count"],
814
+ preform_agg_columns=[mm_schemas.ResultData.RESULT_VALUE],
815
+ )
816
+ if df.empty:
817
+ return {}
818
+
819
+ # Convert DataFrame to a dictionary
820
+ return {
821
+ (
822
+ row[mm_schemas.WriterEvent.APPLICATION_NAME],
823
+ row[mm_schemas.ResultData.RESULT_STATUS],
824
+ ): row["count(result_value)"]
825
+ for _, row in df.iterrows()
826
+ }
827
+
744
828
  def get_metrics_metadata(
745
829
  self,
746
830
  endpoint_id: Union[str, list[str]],
@@ -757,7 +841,10 @@ class TDEngineConnector(TSDBConnector):
757
841
  mm_schemas.MetricData.METRIC_NAME,
758
842
  mm_schemas.EventFieldType.ENDPOINT_ID,
759
843
  ],
760
- filter_query=self._get_endpoint_filter(endpoint_id=endpoint_id),
844
+ filter_query=self._generate_filter_query(
845
+ filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
846
+ filter_values=endpoint_id,
847
+ ),
761
848
  timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
762
849
  group_by=[
763
850
  mm_schemas.WriterEvent.APPLICATION_NAME,
@@ -795,7 +882,10 @@ class TDEngineConnector(TSDBConnector):
795
882
  mm_schemas.ResultData.RESULT_KIND,
796
883
  mm_schemas.EventFieldType.ENDPOINT_ID,
797
884
  ],
798
- filter_query=self._get_endpoint_filter(endpoint_id=endpoint_id),
885
+ filter_query=self._generate_filter_query(
886
+ filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
887
+ filter_values=endpoint_id,
888
+ ),
799
889
  timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
800
890
  group_by=[
801
891
  mm_schemas.WriterEvent.APPLICATION_NAME,
@@ -824,7 +914,10 @@ class TDEngineConnector(TSDBConnector):
824
914
  end: Optional[datetime] = None,
825
915
  get_raw: bool = False,
826
916
  ) -> pd.DataFrame:
827
- filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
917
+ filter_query = self._generate_filter_query(
918
+ filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
919
+ filter_values=endpoint_ids,
920
+ )
828
921
  filter_query += f"AND {mm_schemas.EventFieldType.ERROR_TYPE} = '{mm_schemas.EventFieldType.INFER_ERROR}'"
829
922
  start, end = self._get_start_end(start, end)
830
923
  df = self._get_records(
@@ -417,6 +417,7 @@ class V3IOTSDBConnector(TSDBConnector):
417
417
  mm_schemas.WriterEvent.END_INFER_TIME,
418
418
  mm_schemas.WriterEvent.ENDPOINT_ID,
419
419
  mm_schemas.WriterEvent.APPLICATION_NAME,
420
+ mm_schemas.WriterEvent.ENDPOINT_NAME,
420
421
  ]
421
422
 
422
423
  if kind == mm_schemas.WriterEventKind.METRIC:
@@ -694,22 +695,26 @@ class V3IOTSDBConnector(TSDBConnector):
694
695
  )
695
696
 
696
697
  @staticmethod
697
- def _get_endpoint_filter(endpoint_id: Union[str, list[str]]) -> Optional[str]:
698
- if isinstance(endpoint_id, str):
699
- return f"endpoint_id=='{endpoint_id}'"
700
- elif isinstance(endpoint_id, list):
701
- if len(endpoint_id) > V3IO_FRAMESD_MEPS_LIMIT:
698
+ def _generate_filter_query(
699
+ filter_key: str, filter_values: Union[str, list[str]]
700
+ ) -> Optional[str]:
701
+ if isinstance(filter_values, str):
702
+ return f"{filter_key}=='{filter_values}'"
703
+ elif isinstance(filter_values, list):
704
+ if len(filter_values) > V3IO_FRAMESD_MEPS_LIMIT:
702
705
  logger.info(
703
- "The number of endpoint ids exceeds the v3io-engine filter-expression limit, "
704
- "retrieving all the model endpoints from the db.",
706
+ "The number of filter values exceeds the v3io-engine filter-expression limit, "
707
+ "retrieving all the values from the db.",
708
+ filter_key=filter_key,
705
709
  limit=V3IO_FRAMESD_MEPS_LIMIT,
706
- amount=len(endpoint_id),
710
+ amount=len(filter_values),
707
711
  )
708
712
  return None
709
- return f"endpoint_id IN({str(endpoint_id)[1:-1]}) "
713
+ return f"{filter_key} IN ({', '.join(repr(v) for v in filter_values)}) "
710
714
  else:
711
715
  raise mlrun.errors.MLRunInvalidArgumentError(
712
- f"Invalid 'endpoint_id' filter: must be a string or a list, endpoint_id: {endpoint_id}"
716
+ f"Invalid filter key {filter_key}: must be a string or a list, got {type(filter_values).__name__}; "
717
+ f"filter values: {filter_values}"
713
718
  )
714
719
 
715
720
  def read_metrics_data(
@@ -946,7 +951,11 @@ class V3IOTSDBConnector(TSDBConnector):
946
951
  end: Optional[datetime] = None,
947
952
  get_raw: bool = False,
948
953
  ) -> Union[pd.DataFrame, list[v3io_frames.client.RawFrame]]:
949
- filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
954
+ filter_query = self._generate_filter_query(
955
+ filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
956
+ filter_values=endpoint_ids,
957
+ )
958
+
950
959
  start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
951
960
  start, end = self._get_start_end(start, end)
952
961
  res = self._get_records(
@@ -976,7 +985,10 @@ class V3IOTSDBConnector(TSDBConnector):
976
985
  end: Optional[datetime] = None,
977
986
  ) -> pd.DataFrame:
978
987
  start, end = self._get_start_end(start, end)
979
- filter_query = self._get_endpoint_filter(endpoint_id=endpoint_id)
988
+ filter_query = self._generate_filter_query(
989
+ filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
990
+ filter_values=endpoint_id,
991
+ )
980
992
  df = self._get_records(
981
993
  table=mm_schemas.V3IOTSDBTables.METRICS,
982
994
  start=start,
@@ -998,7 +1010,10 @@ class V3IOTSDBConnector(TSDBConnector):
998
1010
  end: Optional[datetime] = None,
999
1011
  ) -> pd.DataFrame:
1000
1012
  start, end = self._get_start_end(start, end)
1001
- filter_query = self._get_endpoint_filter(endpoint_id=endpoint_id)
1013
+ filter_query = self._generate_filter_query(
1014
+ filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
1015
+ filter_values=endpoint_id,
1016
+ )
1002
1017
  df = self._get_records(
1003
1018
  table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
1004
1019
  start=start,
@@ -1025,7 +1040,10 @@ class V3IOTSDBConnector(TSDBConnector):
1025
1040
  end: Optional[datetime] = None,
1026
1041
  get_raw: bool = False,
1027
1042
  ) -> Union[pd.DataFrame, list[v3io_frames.client.RawFrame]]:
1028
- filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
1043
+ filter_query = self._generate_filter_query(
1044
+ filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
1045
+ filter_values=endpoint_ids,
1046
+ )
1029
1047
  if filter_query:
1030
1048
  filter_query += f"AND {mm_schemas.EventFieldType.ERROR_TYPE} == '{mm_schemas.EventFieldType.INFER_ERROR}'"
1031
1049
  else:
@@ -1062,7 +1080,10 @@ class V3IOTSDBConnector(TSDBConnector):
1062
1080
  end: Optional[datetime] = None,
1063
1081
  get_raw: bool = False,
1064
1082
  ) -> Union[pd.DataFrame, list[v3io_frames.client.RawFrame]]:
1065
- filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
1083
+ filter_query = self._generate_filter_query(
1084
+ filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
1085
+ filter_values=endpoint_ids,
1086
+ )
1066
1087
  start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
1067
1088
  start, end = self._get_start_end(start, end)
1068
1089
  res = self._get_records(
@@ -1177,3 +1198,73 @@ class V3IOTSDBConnector(TSDBConnector):
1177
1198
  mep.status.last_request = last_request_dictionary.get(
1178
1199
  uid, mep.status.last_request
1179
1200
  )
1201
+
1202
+ def count_results_by_status(
1203
+ self,
1204
+ start: Optional[Union[datetime, str]] = None,
1205
+ end: Optional[Union[datetime, str]] = None,
1206
+ endpoint_ids: Optional[Union[str, list[str]]] = None,
1207
+ application_names: Optional[Union[str, list[str]]] = None,
1208
+ result_status_list: Optional[list[int]] = None,
1209
+ ) -> dict[tuple[str, int], int]:
1210
+ now = mlrun.utils.datetime_now()
1211
+ start = start or (now - timedelta(hours=24))
1212
+ end = end or now
1213
+ filter_query = ""
1214
+ if endpoint_ids:
1215
+ filter_query = self._generate_filter_query(
1216
+ filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
1217
+ filter_values=endpoint_ids,
1218
+ )
1219
+ if application_names:
1220
+ app_filter_query = self._generate_filter_query(
1221
+ filter_key=mm_schemas.ApplicationEvent.APPLICATION_NAME,
1222
+ filter_values=application_names,
1223
+ )
1224
+ if filter_query:
1225
+ filter_query += f" AND {app_filter_query}"
1226
+ else:
1227
+ filter_query = app_filter_query
1228
+
1229
+ df = self._get_records(
1230
+ table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
1231
+ start=start,
1232
+ end=end,
1233
+ columns=[
1234
+ mm_schemas.ResultData.RESULT_VALUE,
1235
+ mm_schemas.ResultData.RESULT_STATUS,
1236
+ ],
1237
+ filter_query=filter_query,
1238
+ )
1239
+
1240
+ # filter result status
1241
+ if result_status_list and not df.empty:
1242
+ df = df[df[mm_schemas.ResultData.RESULT_STATUS].isin(result_status_list)]
1243
+
1244
+ if df.empty:
1245
+ return {}
1246
+ else:
1247
+ # convert application name to lower case
1248
+ df[mm_schemas.ApplicationEvent.APPLICATION_NAME] = df[
1249
+ mm_schemas.ApplicationEvent.APPLICATION_NAME
1250
+ ].str.lower()
1251
+
1252
+ df = (
1253
+ df[
1254
+ [
1255
+ mm_schemas.ApplicationEvent.APPLICATION_NAME,
1256
+ mm_schemas.ResultData.RESULT_STATUS,
1257
+ mm_schemas.ResultData.RESULT_VALUE,
1258
+ ]
1259
+ ]
1260
+ .groupby(
1261
+ [
1262
+ mm_schemas.ApplicationEvent.APPLICATION_NAME,
1263
+ mm_schemas.ResultData.RESULT_STATUS,
1264
+ ],
1265
+ observed=True,
1266
+ )
1267
+ .count()
1268
+ )
1269
+
1270
+ return df[mm_schemas.ResultData.RESULT_VALUE].to_dict()
mlrun/projects/project.py CHANGED
@@ -1409,7 +1409,10 @@ class MlrunProject(ModelObj):
1409
1409
  https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
1410
1410
  Note that "local" engine does not support this argument
1411
1411
  :param ttl: Pipeline ttl in secs (after that the pods will be removed)
1412
- :param image: Image for workflow runner job, only for scheduled and remote workflows
1412
+ :param image: Image for workflow runner job, only for scheduled and remote workflows.
1413
+ The image must have mlrun[kfp] installed which requires python 3.9.
1414
+ Therefore, the project default image will not be used for the workflow,
1415
+ and the image must be specified explicitly.
1413
1416
  :param args: Argument values (key=value, ..)
1414
1417
  """
1415
1418
 
@@ -4971,6 +4974,42 @@ class MlrunProject(ModelObj):
4971
4974
  labels=model_monitoring_labels_list,
4972
4975
  )
4973
4976
 
4977
+ def get_monitoring_function_summaries(
4978
+ self,
4979
+ start: Optional[datetime.datetime] = None,
4980
+ end: Optional[datetime.datetime] = None,
4981
+ names: Optional[Union[list[str], str]] = None,
4982
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
4983
+ include_stats: bool = False,
4984
+ include_infra: bool = True,
4985
+ ) -> list[mlrun.common.schemas.model_monitoring.FunctionSummary]:
4986
+ """Get monitoring function summaries for the specified project.
4987
+ :param start: Start time for filtering the results (optional).
4988
+ :param end: End time for filtering the results (optional).
4989
+ :param names: List of function names to filter by (optional).
4990
+ :param labels: Labels to filter by (optional).
4991
+ :param include_stats: Whether to include statistics in the response (default is False).
4992
+ :param include_infra: whether to include model monitoring infrastructure functions (default is True).
4993
+ :return: A list of FunctionSummary objects containing information about the monitoring functions.
4994
+ """
4995
+
4996
+ if start is not None and end is not None:
4997
+ if start.tzinfo is None or end.tzinfo is None:
4998
+ raise mlrun.errors.MLRunInvalidArgumentTypeError(
4999
+ "Custom start and end times must contain the timezone."
5000
+ )
5001
+
5002
+ db = mlrun.db.get_run_db(secrets=self._secrets)
5003
+ return db.get_monitoring_function_summaries(
5004
+ project=self.metadata.name,
5005
+ start=start,
5006
+ end=end,
5007
+ names=names,
5008
+ labels=labels,
5009
+ include_stats=include_stats,
5010
+ include_infra=include_infra,
5011
+ )
5012
+
4974
5013
  def list_runs(
4975
5014
  self,
4976
5015
  name: Optional[str] = None,
mlrun/runtimes/base.py CHANGED
@@ -74,7 +74,6 @@ spec_fields = [
74
74
  "pythonpath",
75
75
  "disable_auto_mount",
76
76
  "allow_empty_resources",
77
- "clone_target_dir",
78
77
  "reset_on_run",
79
78
  ]
80
79
 
@@ -117,7 +116,6 @@ class FunctionSpec(ModelObj):
117
116
  default_handler=None,
118
117
  pythonpath=None,
119
118
  disable_auto_mount=False,
120
- clone_target_dir=None,
121
119
  ):
122
120
  self.command = command or ""
123
121
  self.image = image or ""
@@ -134,9 +132,6 @@ class FunctionSpec(ModelObj):
134
132
  self.entry_points = entry_points or {}
135
133
  self.disable_auto_mount = disable_auto_mount
136
134
  self.allow_empty_resources = None
137
- # The build.source is cloned/extracted to the specified clone_target_dir
138
- # if a relative path is specified, it will be enriched with a temp dir path
139
- self._clone_target_dir = clone_target_dir or None
140
135
 
141
136
  @property
142
137
  def build(self) -> ImageBuilder:
@@ -146,28 +141,6 @@ class FunctionSpec(ModelObj):
146
141
  def build(self, build):
147
142
  self._build = self._verify_dict(build, "build", ImageBuilder)
148
143
 
149
- @property
150
- def clone_target_dir(self):
151
- # TODO: remove this property in 1.10.0
152
- if self.build.source_code_target_dir:
153
- warnings.warn(
154
- "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.10.0. "
155
- "Use spec.build.source_code_target_dir instead.",
156
- FutureWarning,
157
- )
158
- return self.build.source_code_target_dir
159
-
160
- @clone_target_dir.setter
161
- def clone_target_dir(self, clone_target_dir):
162
- # TODO: remove this property in 1.10.0
163
- if clone_target_dir:
164
- warnings.warn(
165
- "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.10.0. "
166
- "Use spec.build.source_code_target_dir instead.",
167
- FutureWarning,
168
- )
169
- self.build.source_code_target_dir = clone_target_dir
170
-
171
144
  def enrich_function_preemption_spec(self):
172
145
  pass
173
146
 
mlrun/runtimes/daskjob.py CHANGED
@@ -91,7 +91,6 @@ class DaskSpec(KubeResourceSpec):
91
91
  tolerations=None,
92
92
  preemption_mode=None,
93
93
  security_context=None,
94
- clone_target_dir=None,
95
94
  state_thresholds=None,
96
95
  ):
97
96
  super().__init__(
@@ -121,7 +120,6 @@ class DaskSpec(KubeResourceSpec):
121
120
  tolerations=tolerations,
122
121
  preemption_mode=preemption_mode,
123
122
  security_context=security_context,
124
- clone_target_dir=clone_target_dir,
125
123
  state_thresholds=state_thresholds,
126
124
  )
127
125
  self.args = args
@@ -192,7 +190,9 @@ class DaskCluster(KubejobRuntime):
192
190
  super().__init__(spec, metadata)
193
191
  self._cluster = None
194
192
  self.use_remote = not mlrun.k8s_utils.is_running_inside_kubernetes_cluster()
195
- self.spec.build.base_image = self.spec.build.base_image or "daskdev/dask:latest"
193
+ self.spec.build.base_image = (
194
+ self.spec.build.base_image or mlrun.mlconf.default_base_image
195
+ )
196
196
 
197
197
  @property
198
198
  def spec(self) -> DaskSpec:
@@ -248,14 +248,18 @@ class DaskCluster(KubejobRuntime):
248
248
  if not self.is_deployed():
249
249
  raise RunError(
250
250
  "Function image is not built/ready, use .deploy()"
251
- " method first, or set base dask image (daskdev/dask:latest)"
251
+ " method first, or set base dask image to mlrun/mlrun"
252
252
  )
253
253
 
254
254
  self.save(versioned=False)
255
255
  background_task = db.start_function(func_url=self._function_uri())
256
256
  if watch:
257
257
  now = datetime.datetime.utcnow()
258
- timeout = now + datetime.timedelta(minutes=10)
258
+ timeout = now + datetime.timedelta(
259
+ seconds=int(
260
+ mlrun.mlconf.background_tasks.default_timeouts.runtimes.dask_cluster_start
261
+ )
262
+ )
259
263
  while now < timeout:
260
264
  background_task = db.get_project_background_task(
261
265
  background_task.metadata.project, background_task.metadata.name
@@ -282,6 +286,9 @@ class DaskCluster(KubejobRuntime):
282
286
  return
283
287
  time.sleep(5)
284
288
  now = datetime.datetime.utcnow()
289
+ raise mlrun.errors.MLRunTimeoutError(
290
+ "Timeout waiting for Dask cluster to start"
291
+ )
285
292
 
286
293
  def close(self, running=True):
287
294
  from dask.distributed import default_client
@@ -82,7 +82,6 @@ class DatabricksSpec(pod.KubeResourceSpec):
82
82
  tolerations=None,
83
83
  preemption_mode=None,
84
84
  security_context=None,
85
- clone_target_dir=None,
86
85
  state_thresholds=None,
87
86
  ):
88
87
  super().__init__(
@@ -112,7 +111,6 @@ class DatabricksSpec(pod.KubeResourceSpec):
112
111
  tolerations=tolerations,
113
112
  preemption_mode=preemption_mode,
114
113
  security_context=security_context,
115
- clone_target_dir=clone_target_dir,
116
114
  state_thresholds=state_thresholds,
117
115
  )
118
116
  self._termination_grace_period_seconds = 60
@@ -53,7 +53,6 @@ class MPIResourceSpec(KubeResourceSpec):
53
53
  tolerations=None,
54
54
  preemption_mode=None,
55
55
  security_context=None,
56
- clone_target_dir=None,
57
56
  state_thresholds=None,
58
57
  ):
59
58
  super().__init__(
@@ -83,7 +82,6 @@ class MPIResourceSpec(KubeResourceSpec):
83
82
  tolerations=tolerations,
84
83
  preemption_mode=preemption_mode,
85
84
  security_context=security_context,
86
- clone_target_dir=clone_target_dir,
87
85
  state_thresholds=state_thresholds,
88
86
  )
89
87
  self.mpi_args = mpi_args or [
@@ -48,7 +48,6 @@ class MPIV1ResourceSpec(MPIResourceSpec):
48
48
  tolerations=None,
49
49
  preemption_mode=None,
50
50
  security_context=None,
51
- clone_target_dir=None,
52
51
  state_thresholds=None,
53
52
  ):
54
53
  super().__init__(
@@ -79,7 +78,6 @@ class MPIV1ResourceSpec(MPIResourceSpec):
79
78
  tolerations=tolerations,
80
79
  preemption_mode=preemption_mode,
81
80
  security_context=security_context,
82
- clone_target_dir=clone_target_dir,
83
81
  state_thresholds=state_thresholds,
84
82
  )
85
83
  self.clean_pod_policy = clean_pod_policy or MPIJobV1CleanPodPolicies.default()
@@ -76,7 +76,6 @@ class ApplicationSpec(NuclioSpec):
76
76
  security_context=None,
77
77
  service_type=None,
78
78
  add_templated_ingress_host_mode=None,
79
- clone_target_dir=None,
80
79
  state_thresholds=None,
81
80
  disable_default_http_trigger=None,
82
81
  internal_application_port=None,
@@ -119,7 +118,6 @@ class ApplicationSpec(NuclioSpec):
119
118
  security_context=security_context,
120
119
  service_type=service_type,
121
120
  add_templated_ingress_host_mode=add_templated_ingress_host_mode,
122
- clone_target_dir=clone_target_dir,
123
121
  state_thresholds=state_thresholds,
124
122
  disable_default_http_trigger=disable_default_http_trigger,
125
123
  )
@@ -274,7 +272,6 @@ class ApplicationRuntime(RemoteRuntime):
274
272
  project="",
275
273
  tag="",
276
274
  verbose=False,
277
- auth_info: schemas.AuthInfo = None,
278
275
  builder_env: typing.Optional[dict] = None,
279
276
  force_build: bool = False,
280
277
  with_mlrun=None,
@@ -291,7 +288,6 @@ class ApplicationRuntime(RemoteRuntime):
291
288
  :param project: Project name
292
289
  :param tag: Function tag
293
290
  :param verbose: Set True for verbose logging
294
- :param auth_info: Service AuthInfo (deprecated and ignored)
295
291
  :param builder_env: Env vars dict for source archive config/credentials
296
292
  e.g. builder_env={"GIT_TOKEN": token}
297
293
  :param force_build: Set True for force building the application image
@@ -332,7 +328,6 @@ class ApplicationRuntime(RemoteRuntime):
332
328
  project=project,
333
329
  tag=tag,
334
330
  verbose=verbose,
335
- auth_info=auth_info,
336
331
  builder_env=builder_env,
337
332
  )
338
333
  logger.info(