mlrun 1.10.0rc5__py3-none-any.whl → 1.10.0rc6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/artifacts/model.py +3 -3
- mlrun/common/schemas/model_monitoring/__init__.py +1 -0
- mlrun/common/schemas/model_monitoring/constants.py +14 -2
- mlrun/common/schemas/model_monitoring/functions.py +66 -0
- mlrun/common/schemas/project.py +3 -0
- mlrun/config.py +3 -3
- mlrun/db/base.py +13 -0
- mlrun/db/httpdb.py +47 -0
- mlrun/db/nopdb.py +12 -0
- mlrun/launcher/client.py +23 -0
- mlrun/model_monitoring/db/tsdb/base.py +30 -0
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +118 -50
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +117 -24
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +106 -15
- mlrun/projects/project.py +40 -1
- mlrun/runtimes/base.py +0 -27
- mlrun/runtimes/daskjob.py +4 -4
- mlrun/runtimes/databricks_job/databricks_runtime.py +0 -2
- mlrun/runtimes/mpijob/abstract.py +0 -2
- mlrun/runtimes/mpijob/v1.py +0 -2
- mlrun/runtimes/nuclio/application/application.py +0 -5
- mlrun/runtimes/nuclio/function.py +0 -11
- mlrun/runtimes/nuclio/serving.py +0 -6
- mlrun/runtimes/pod.py +1 -3
- mlrun/runtimes/remotesparkjob.py +0 -2
- mlrun/runtimes/sparkjob/spark3job.py +0 -2
- mlrun/serving/states.py +16 -18
- mlrun/utils/helpers.py +15 -0
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.10.0rc5.dist-info → mlrun-1.10.0rc6.dist-info}/METADATA +2 -1
- {mlrun-1.10.0rc5.dist-info → mlrun-1.10.0rc6.dist-info}/RECORD +35 -34
- {mlrun-1.10.0rc5.dist-info → mlrun-1.10.0rc6.dist-info}/WHEEL +0 -0
- {mlrun-1.10.0rc5.dist-info → mlrun-1.10.0rc6.dist-info}/entry_points.txt +0 -0
- {mlrun-1.10.0rc5.dist-info → mlrun-1.10.0rc6.dist-info}/licenses/LICENSE +0 -0
- {mlrun-1.10.0rc5.dist-info → mlrun-1.10.0rc6.dist-info}/top_level.txt +0 -0
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
import threading
|
|
15
16
|
from datetime import datetime, timedelta
|
|
16
|
-
from threading import Lock
|
|
17
17
|
from typing import Callable, Final, Literal, Optional, Union
|
|
18
18
|
|
|
19
19
|
import pandas as pd
|
|
@@ -32,8 +32,8 @@ from mlrun.model_monitoring.db.tsdb.tdengine.tdengine_connection import (
|
|
|
32
32
|
from mlrun.model_monitoring.helpers import get_invocations_fqn
|
|
33
33
|
from mlrun.utils import logger
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
# Thread-local storage for connections
|
|
36
|
+
_thread_local = threading.local()
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
class TDEngineTimestampPrecision(mlrun.common.types.StrEnum):
|
|
@@ -76,16 +76,15 @@ class TDEngineConnector(TSDBConnector):
|
|
|
76
76
|
|
|
77
77
|
@property
|
|
78
78
|
def connection(self) -> TDEngineConnection:
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
return _connection
|
|
79
|
+
if not hasattr(_thread_local, "connection"):
|
|
80
|
+
_thread_local.connection = self._create_connection()
|
|
81
|
+
logger.debug(
|
|
82
|
+
"Created new TDEngine connection for thread",
|
|
83
|
+
project=self.project,
|
|
84
|
+
thread_name=threading.current_thread().name,
|
|
85
|
+
thread_id=threading.get_ident(),
|
|
86
|
+
)
|
|
87
|
+
return _thread_local.connection
|
|
89
88
|
|
|
90
89
|
def _create_connection(self) -> TDEngineConnection:
|
|
91
90
|
"""Establish a connection to the TSDB server."""
|
|
@@ -204,14 +203,27 @@ class TDEngineConnector(TSDBConnector):
|
|
|
204
203
|
return datetime.fromisoformat(val) if isinstance(val, str) else val
|
|
205
204
|
|
|
206
205
|
@staticmethod
|
|
207
|
-
def
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
206
|
+
def _generate_filter_query(
|
|
207
|
+
filter_column: str, filter_values: Union[str, list[Union[str, int]]]
|
|
208
|
+
) -> Optional[str]:
|
|
209
|
+
"""
|
|
210
|
+
Generate a filter query for TDEngine based on the provided column and values.
|
|
211
|
+
|
|
212
|
+
:param filter_column: The column to filter by.
|
|
213
|
+
:param filter_values: A single value or a list of values to filter by.
|
|
214
|
+
|
|
215
|
+
:return: A string representing the filter query.
|
|
216
|
+
:raise: MLRunInvalidArgumentError if the filter values are not of type string or list.
|
|
217
|
+
"""
|
|
218
|
+
|
|
219
|
+
if isinstance(filter_values, str):
|
|
220
|
+
return f"{filter_column}='{filter_values}'"
|
|
221
|
+
elif isinstance(filter_values, list):
|
|
222
|
+
return f"{filter_column} IN ({', '.join(repr(v) for v in filter_values)}) "
|
|
212
223
|
else:
|
|
213
224
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
214
|
-
"Invalid
|
|
225
|
+
f"Invalid filter values {filter_values}: must be a string or a list, "
|
|
226
|
+
f"got {type(filter_values).__name__}; filter values: {filter_values}"
|
|
215
227
|
)
|
|
216
228
|
|
|
217
229
|
def _drop_database_query(self) -> str:
|
|
@@ -673,7 +685,10 @@ class TDEngineConnector(TSDBConnector):
|
|
|
673
685
|
start: Optional[datetime] = None,
|
|
674
686
|
end: Optional[datetime] = None,
|
|
675
687
|
) -> pd.DataFrame:
|
|
676
|
-
filter_query = self.
|
|
688
|
+
filter_query = self._generate_filter_query(
|
|
689
|
+
filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
|
|
690
|
+
filter_values=endpoint_ids,
|
|
691
|
+
)
|
|
677
692
|
start, end = self._get_start_end(start, end)
|
|
678
693
|
df = self._get_records(
|
|
679
694
|
table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
|
|
@@ -714,7 +729,10 @@ class TDEngineConnector(TSDBConnector):
|
|
|
714
729
|
end: Optional[datetime] = None,
|
|
715
730
|
get_raw: bool = False,
|
|
716
731
|
) -> pd.DataFrame:
|
|
717
|
-
filter_query = self.
|
|
732
|
+
filter_query = self._generate_filter_query(
|
|
733
|
+
filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
|
|
734
|
+
filter_values=endpoint_ids,
|
|
735
|
+
)
|
|
718
736
|
start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
|
|
719
737
|
start, end = self._get_start_end(start, end)
|
|
720
738
|
df = self._get_records(
|
|
@@ -741,6 +759,72 @@ class TDEngineConnector(TSDBConnector):
|
|
|
741
759
|
df.dropna(inplace=True)
|
|
742
760
|
return df
|
|
743
761
|
|
|
762
|
+
def count_results_by_status(
|
|
763
|
+
self,
|
|
764
|
+
start: Optional[Union[datetime, str]] = None,
|
|
765
|
+
end: Optional[Union[datetime, str]] = None,
|
|
766
|
+
endpoint_ids: Optional[Union[str, list[str]]] = None,
|
|
767
|
+
application_names: Optional[Union[str, list[str]]] = None,
|
|
768
|
+
result_status_list: Optional[list[int]] = None,
|
|
769
|
+
) -> dict[tuple[str, int], int]:
|
|
770
|
+
filter_query = ""
|
|
771
|
+
now = mlrun.utils.datetime_now()
|
|
772
|
+
start = start or (now - timedelta(hours=24))
|
|
773
|
+
end = end or now
|
|
774
|
+
if endpoint_ids:
|
|
775
|
+
filter_query = self._generate_filter_query(
|
|
776
|
+
filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
|
|
777
|
+
filter_values=endpoint_ids,
|
|
778
|
+
)
|
|
779
|
+
if application_names:
|
|
780
|
+
app_filter_query = self._generate_filter_query(
|
|
781
|
+
filter_column=mm_schemas.ApplicationEvent.APPLICATION_NAME,
|
|
782
|
+
filter_values=application_names,
|
|
783
|
+
)
|
|
784
|
+
if filter_query:
|
|
785
|
+
filter_query += f" AND {app_filter_query}"
|
|
786
|
+
else:
|
|
787
|
+
filter_query = app_filter_query
|
|
788
|
+
if result_status_list:
|
|
789
|
+
status_filter_query = self._generate_filter_query(
|
|
790
|
+
filter_column=mm_schemas.ResultData.RESULT_STATUS,
|
|
791
|
+
filter_values=result_status_list,
|
|
792
|
+
)
|
|
793
|
+
if filter_query:
|
|
794
|
+
filter_query += f" AND {status_filter_query}"
|
|
795
|
+
else:
|
|
796
|
+
filter_query = status_filter_query
|
|
797
|
+
|
|
798
|
+
df = self._get_records(
|
|
799
|
+
table=self.tables[mm_schemas.TDEngineSuperTables.APP_RESULTS].super_table,
|
|
800
|
+
start=start,
|
|
801
|
+
end=end,
|
|
802
|
+
columns=[
|
|
803
|
+
mm_schemas.WriterEvent.APPLICATION_NAME,
|
|
804
|
+
mm_schemas.ResultData.RESULT_STATUS,
|
|
805
|
+
mm_schemas.ResultData.RESULT_VALUE,
|
|
806
|
+
],
|
|
807
|
+
filter_query=filter_query,
|
|
808
|
+
timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
|
|
809
|
+
group_by=[
|
|
810
|
+
mm_schemas.WriterEvent.APPLICATION_NAME,
|
|
811
|
+
mm_schemas.ResultData.RESULT_STATUS,
|
|
812
|
+
],
|
|
813
|
+
agg_funcs=["count"],
|
|
814
|
+
preform_agg_columns=[mm_schemas.ResultData.RESULT_VALUE],
|
|
815
|
+
)
|
|
816
|
+
if df.empty:
|
|
817
|
+
return {}
|
|
818
|
+
|
|
819
|
+
# Convert DataFrame to a dictionary
|
|
820
|
+
return {
|
|
821
|
+
(
|
|
822
|
+
row[mm_schemas.WriterEvent.APPLICATION_NAME],
|
|
823
|
+
row[mm_schemas.ResultData.RESULT_STATUS],
|
|
824
|
+
): row["count(result_value)"]
|
|
825
|
+
for _, row in df.iterrows()
|
|
826
|
+
}
|
|
827
|
+
|
|
744
828
|
def get_metrics_metadata(
|
|
745
829
|
self,
|
|
746
830
|
endpoint_id: Union[str, list[str]],
|
|
@@ -757,7 +841,10 @@ class TDEngineConnector(TSDBConnector):
|
|
|
757
841
|
mm_schemas.MetricData.METRIC_NAME,
|
|
758
842
|
mm_schemas.EventFieldType.ENDPOINT_ID,
|
|
759
843
|
],
|
|
760
|
-
filter_query=self.
|
|
844
|
+
filter_query=self._generate_filter_query(
|
|
845
|
+
filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
|
|
846
|
+
filter_values=endpoint_id,
|
|
847
|
+
),
|
|
761
848
|
timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
|
|
762
849
|
group_by=[
|
|
763
850
|
mm_schemas.WriterEvent.APPLICATION_NAME,
|
|
@@ -795,7 +882,10 @@ class TDEngineConnector(TSDBConnector):
|
|
|
795
882
|
mm_schemas.ResultData.RESULT_KIND,
|
|
796
883
|
mm_schemas.EventFieldType.ENDPOINT_ID,
|
|
797
884
|
],
|
|
798
|
-
filter_query=self.
|
|
885
|
+
filter_query=self._generate_filter_query(
|
|
886
|
+
filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
|
|
887
|
+
filter_values=endpoint_id,
|
|
888
|
+
),
|
|
799
889
|
timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
|
|
800
890
|
group_by=[
|
|
801
891
|
mm_schemas.WriterEvent.APPLICATION_NAME,
|
|
@@ -824,7 +914,10 @@ class TDEngineConnector(TSDBConnector):
|
|
|
824
914
|
end: Optional[datetime] = None,
|
|
825
915
|
get_raw: bool = False,
|
|
826
916
|
) -> pd.DataFrame:
|
|
827
|
-
filter_query = self.
|
|
917
|
+
filter_query = self._generate_filter_query(
|
|
918
|
+
filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
|
|
919
|
+
filter_values=endpoint_ids,
|
|
920
|
+
)
|
|
828
921
|
filter_query += f"AND {mm_schemas.EventFieldType.ERROR_TYPE} = '{mm_schemas.EventFieldType.INFER_ERROR}'"
|
|
829
922
|
start, end = self._get_start_end(start, end)
|
|
830
923
|
df = self._get_records(
|
|
@@ -417,6 +417,7 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
417
417
|
mm_schemas.WriterEvent.END_INFER_TIME,
|
|
418
418
|
mm_schemas.WriterEvent.ENDPOINT_ID,
|
|
419
419
|
mm_schemas.WriterEvent.APPLICATION_NAME,
|
|
420
|
+
mm_schemas.WriterEvent.ENDPOINT_NAME,
|
|
420
421
|
]
|
|
421
422
|
|
|
422
423
|
if kind == mm_schemas.WriterEventKind.METRIC:
|
|
@@ -694,22 +695,26 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
694
695
|
)
|
|
695
696
|
|
|
696
697
|
@staticmethod
|
|
697
|
-
def
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
698
|
+
def _generate_filter_query(
|
|
699
|
+
filter_key: str, filter_values: Union[str, list[str]]
|
|
700
|
+
) -> Optional[str]:
|
|
701
|
+
if isinstance(filter_values, str):
|
|
702
|
+
return f"{filter_key}=='{filter_values}'"
|
|
703
|
+
elif isinstance(filter_values, list):
|
|
704
|
+
if len(filter_values) > V3IO_FRAMESD_MEPS_LIMIT:
|
|
702
705
|
logger.info(
|
|
703
|
-
"The number of
|
|
704
|
-
"retrieving all the
|
|
706
|
+
"The number of filter values exceeds the v3io-engine filter-expression limit, "
|
|
707
|
+
"retrieving all the values from the db.",
|
|
708
|
+
filter_key=filter_key,
|
|
705
709
|
limit=V3IO_FRAMESD_MEPS_LIMIT,
|
|
706
|
-
amount=len(
|
|
710
|
+
amount=len(filter_values),
|
|
707
711
|
)
|
|
708
712
|
return None
|
|
709
|
-
return f"
|
|
713
|
+
return f"{filter_key} IN ({', '.join(repr(v) for v in filter_values)}) "
|
|
710
714
|
else:
|
|
711
715
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
712
|
-
f"Invalid
|
|
716
|
+
f"Invalid filter key {filter_key}: must be a string or a list, got {type(filter_values).__name__}; "
|
|
717
|
+
f"filter values: {filter_values}"
|
|
713
718
|
)
|
|
714
719
|
|
|
715
720
|
def read_metrics_data(
|
|
@@ -946,7 +951,11 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
946
951
|
end: Optional[datetime] = None,
|
|
947
952
|
get_raw: bool = False,
|
|
948
953
|
) -> Union[pd.DataFrame, list[v3io_frames.client.RawFrame]]:
|
|
949
|
-
filter_query = self.
|
|
954
|
+
filter_query = self._generate_filter_query(
|
|
955
|
+
filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
|
|
956
|
+
filter_values=endpoint_ids,
|
|
957
|
+
)
|
|
958
|
+
|
|
950
959
|
start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
|
|
951
960
|
start, end = self._get_start_end(start, end)
|
|
952
961
|
res = self._get_records(
|
|
@@ -976,7 +985,10 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
976
985
|
end: Optional[datetime] = None,
|
|
977
986
|
) -> pd.DataFrame:
|
|
978
987
|
start, end = self._get_start_end(start, end)
|
|
979
|
-
filter_query = self.
|
|
988
|
+
filter_query = self._generate_filter_query(
|
|
989
|
+
filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
|
|
990
|
+
filter_values=endpoint_id,
|
|
991
|
+
)
|
|
980
992
|
df = self._get_records(
|
|
981
993
|
table=mm_schemas.V3IOTSDBTables.METRICS,
|
|
982
994
|
start=start,
|
|
@@ -998,7 +1010,10 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
998
1010
|
end: Optional[datetime] = None,
|
|
999
1011
|
) -> pd.DataFrame:
|
|
1000
1012
|
start, end = self._get_start_end(start, end)
|
|
1001
|
-
filter_query = self.
|
|
1013
|
+
filter_query = self._generate_filter_query(
|
|
1014
|
+
filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
|
|
1015
|
+
filter_values=endpoint_id,
|
|
1016
|
+
)
|
|
1002
1017
|
df = self._get_records(
|
|
1003
1018
|
table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
|
|
1004
1019
|
start=start,
|
|
@@ -1025,7 +1040,10 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
1025
1040
|
end: Optional[datetime] = None,
|
|
1026
1041
|
get_raw: bool = False,
|
|
1027
1042
|
) -> Union[pd.DataFrame, list[v3io_frames.client.RawFrame]]:
|
|
1028
|
-
filter_query = self.
|
|
1043
|
+
filter_query = self._generate_filter_query(
|
|
1044
|
+
filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
|
|
1045
|
+
filter_values=endpoint_ids,
|
|
1046
|
+
)
|
|
1029
1047
|
if filter_query:
|
|
1030
1048
|
filter_query += f"AND {mm_schemas.EventFieldType.ERROR_TYPE} == '{mm_schemas.EventFieldType.INFER_ERROR}'"
|
|
1031
1049
|
else:
|
|
@@ -1062,7 +1080,10 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
1062
1080
|
end: Optional[datetime] = None,
|
|
1063
1081
|
get_raw: bool = False,
|
|
1064
1082
|
) -> Union[pd.DataFrame, list[v3io_frames.client.RawFrame]]:
|
|
1065
|
-
filter_query = self.
|
|
1083
|
+
filter_query = self._generate_filter_query(
|
|
1084
|
+
filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
|
|
1085
|
+
filter_values=endpoint_ids,
|
|
1086
|
+
)
|
|
1066
1087
|
start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
|
|
1067
1088
|
start, end = self._get_start_end(start, end)
|
|
1068
1089
|
res = self._get_records(
|
|
@@ -1177,3 +1198,73 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
1177
1198
|
mep.status.last_request = last_request_dictionary.get(
|
|
1178
1199
|
uid, mep.status.last_request
|
|
1179
1200
|
)
|
|
1201
|
+
|
|
1202
|
+
def count_results_by_status(
|
|
1203
|
+
self,
|
|
1204
|
+
start: Optional[Union[datetime, str]] = None,
|
|
1205
|
+
end: Optional[Union[datetime, str]] = None,
|
|
1206
|
+
endpoint_ids: Optional[Union[str, list[str]]] = None,
|
|
1207
|
+
application_names: Optional[Union[str, list[str]]] = None,
|
|
1208
|
+
result_status_list: Optional[list[int]] = None,
|
|
1209
|
+
) -> dict[tuple[str, int], int]:
|
|
1210
|
+
now = mlrun.utils.datetime_now()
|
|
1211
|
+
start = start or (now - timedelta(hours=24))
|
|
1212
|
+
end = end or now
|
|
1213
|
+
filter_query = ""
|
|
1214
|
+
if endpoint_ids:
|
|
1215
|
+
filter_query = self._generate_filter_query(
|
|
1216
|
+
filter_key=mm_schemas.ApplicationEvent.ENDPOINT_ID,
|
|
1217
|
+
filter_values=endpoint_ids,
|
|
1218
|
+
)
|
|
1219
|
+
if application_names:
|
|
1220
|
+
app_filter_query = self._generate_filter_query(
|
|
1221
|
+
filter_key=mm_schemas.ApplicationEvent.APPLICATION_NAME,
|
|
1222
|
+
filter_values=application_names,
|
|
1223
|
+
)
|
|
1224
|
+
if filter_query:
|
|
1225
|
+
filter_query += f" AND {app_filter_query}"
|
|
1226
|
+
else:
|
|
1227
|
+
filter_query = app_filter_query
|
|
1228
|
+
|
|
1229
|
+
df = self._get_records(
|
|
1230
|
+
table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
|
|
1231
|
+
start=start,
|
|
1232
|
+
end=end,
|
|
1233
|
+
columns=[
|
|
1234
|
+
mm_schemas.ResultData.RESULT_VALUE,
|
|
1235
|
+
mm_schemas.ResultData.RESULT_STATUS,
|
|
1236
|
+
],
|
|
1237
|
+
filter_query=filter_query,
|
|
1238
|
+
)
|
|
1239
|
+
|
|
1240
|
+
# filter result status
|
|
1241
|
+
if result_status_list and not df.empty:
|
|
1242
|
+
df = df[df[mm_schemas.ResultData.RESULT_STATUS].isin(result_status_list)]
|
|
1243
|
+
|
|
1244
|
+
if df.empty:
|
|
1245
|
+
return {}
|
|
1246
|
+
else:
|
|
1247
|
+
# convert application name to lower case
|
|
1248
|
+
df[mm_schemas.ApplicationEvent.APPLICATION_NAME] = df[
|
|
1249
|
+
mm_schemas.ApplicationEvent.APPLICATION_NAME
|
|
1250
|
+
].str.lower()
|
|
1251
|
+
|
|
1252
|
+
df = (
|
|
1253
|
+
df[
|
|
1254
|
+
[
|
|
1255
|
+
mm_schemas.ApplicationEvent.APPLICATION_NAME,
|
|
1256
|
+
mm_schemas.ResultData.RESULT_STATUS,
|
|
1257
|
+
mm_schemas.ResultData.RESULT_VALUE,
|
|
1258
|
+
]
|
|
1259
|
+
]
|
|
1260
|
+
.groupby(
|
|
1261
|
+
[
|
|
1262
|
+
mm_schemas.ApplicationEvent.APPLICATION_NAME,
|
|
1263
|
+
mm_schemas.ResultData.RESULT_STATUS,
|
|
1264
|
+
],
|
|
1265
|
+
observed=True,
|
|
1266
|
+
)
|
|
1267
|
+
.count()
|
|
1268
|
+
)
|
|
1269
|
+
|
|
1270
|
+
return df[mm_schemas.ResultData.RESULT_VALUE].to_dict()
|
mlrun/projects/project.py
CHANGED
|
@@ -1409,7 +1409,10 @@ class MlrunProject(ModelObj):
|
|
|
1409
1409
|
https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
|
|
1410
1410
|
Note that "local" engine does not support this argument
|
|
1411
1411
|
:param ttl: Pipeline ttl in secs (after that the pods will be removed)
|
|
1412
|
-
:param image: Image for workflow runner job, only for scheduled and remote workflows
|
|
1412
|
+
:param image: Image for workflow runner job, only for scheduled and remote workflows.
|
|
1413
|
+
The image must have mlrun[kfp] installed which requires python 3.9.
|
|
1414
|
+
Therefore, the project default image will not be used for the workflow,
|
|
1415
|
+
and the image must be specified explicitly.
|
|
1413
1416
|
:param args: Argument values (key=value, ..)
|
|
1414
1417
|
"""
|
|
1415
1418
|
|
|
@@ -4971,6 +4974,42 @@ class MlrunProject(ModelObj):
|
|
|
4971
4974
|
labels=model_monitoring_labels_list,
|
|
4972
4975
|
)
|
|
4973
4976
|
|
|
4977
|
+
def get_monitoring_function_summaries(
|
|
4978
|
+
self,
|
|
4979
|
+
start: Optional[datetime.datetime] = None,
|
|
4980
|
+
end: Optional[datetime.datetime] = None,
|
|
4981
|
+
names: Optional[Union[list[str], str]] = None,
|
|
4982
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
4983
|
+
include_stats: bool = False,
|
|
4984
|
+
include_infra: bool = True,
|
|
4985
|
+
) -> list[mlrun.common.schemas.model_monitoring.FunctionSummary]:
|
|
4986
|
+
"""Get monitoring function summaries for the specified project.
|
|
4987
|
+
:param start: Start time for filtering the results (optional).
|
|
4988
|
+
:param end: End time for filtering the results (optional).
|
|
4989
|
+
:param names: List of function names to filter by (optional).
|
|
4990
|
+
:param labels: Labels to filter by (optional).
|
|
4991
|
+
:param include_stats: Whether to include statistics in the response (default is False).
|
|
4992
|
+
:param include_infra: whether to include model monitoring infrastructure functions (default is True).
|
|
4993
|
+
:return: A list of FunctionSummary objects containing information about the monitoring functions.
|
|
4994
|
+
"""
|
|
4995
|
+
|
|
4996
|
+
if start is not None and end is not None:
|
|
4997
|
+
if start.tzinfo is None or end.tzinfo is None:
|
|
4998
|
+
raise mlrun.errors.MLRunInvalidArgumentTypeError(
|
|
4999
|
+
"Custom start and end times must contain the timezone."
|
|
5000
|
+
)
|
|
5001
|
+
|
|
5002
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
5003
|
+
return db.get_monitoring_function_summaries(
|
|
5004
|
+
project=self.metadata.name,
|
|
5005
|
+
start=start,
|
|
5006
|
+
end=end,
|
|
5007
|
+
names=names,
|
|
5008
|
+
labels=labels,
|
|
5009
|
+
include_stats=include_stats,
|
|
5010
|
+
include_infra=include_infra,
|
|
5011
|
+
)
|
|
5012
|
+
|
|
4974
5013
|
def list_runs(
|
|
4975
5014
|
self,
|
|
4976
5015
|
name: Optional[str] = None,
|
mlrun/runtimes/base.py
CHANGED
|
@@ -74,7 +74,6 @@ spec_fields = [
|
|
|
74
74
|
"pythonpath",
|
|
75
75
|
"disable_auto_mount",
|
|
76
76
|
"allow_empty_resources",
|
|
77
|
-
"clone_target_dir",
|
|
78
77
|
"reset_on_run",
|
|
79
78
|
]
|
|
80
79
|
|
|
@@ -117,7 +116,6 @@ class FunctionSpec(ModelObj):
|
|
|
117
116
|
default_handler=None,
|
|
118
117
|
pythonpath=None,
|
|
119
118
|
disable_auto_mount=False,
|
|
120
|
-
clone_target_dir=None,
|
|
121
119
|
):
|
|
122
120
|
self.command = command or ""
|
|
123
121
|
self.image = image or ""
|
|
@@ -134,9 +132,6 @@ class FunctionSpec(ModelObj):
|
|
|
134
132
|
self.entry_points = entry_points or {}
|
|
135
133
|
self.disable_auto_mount = disable_auto_mount
|
|
136
134
|
self.allow_empty_resources = None
|
|
137
|
-
# The build.source is cloned/extracted to the specified clone_target_dir
|
|
138
|
-
# if a relative path is specified, it will be enriched with a temp dir path
|
|
139
|
-
self._clone_target_dir = clone_target_dir or None
|
|
140
135
|
|
|
141
136
|
@property
|
|
142
137
|
def build(self) -> ImageBuilder:
|
|
@@ -146,28 +141,6 @@ class FunctionSpec(ModelObj):
|
|
|
146
141
|
def build(self, build):
|
|
147
142
|
self._build = self._verify_dict(build, "build", ImageBuilder)
|
|
148
143
|
|
|
149
|
-
@property
|
|
150
|
-
def clone_target_dir(self):
|
|
151
|
-
# TODO: remove this property in 1.10.0
|
|
152
|
-
if self.build.source_code_target_dir:
|
|
153
|
-
warnings.warn(
|
|
154
|
-
"The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.10.0. "
|
|
155
|
-
"Use spec.build.source_code_target_dir instead.",
|
|
156
|
-
FutureWarning,
|
|
157
|
-
)
|
|
158
|
-
return self.build.source_code_target_dir
|
|
159
|
-
|
|
160
|
-
@clone_target_dir.setter
|
|
161
|
-
def clone_target_dir(self, clone_target_dir):
|
|
162
|
-
# TODO: remove this property in 1.10.0
|
|
163
|
-
if clone_target_dir:
|
|
164
|
-
warnings.warn(
|
|
165
|
-
"The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.10.0. "
|
|
166
|
-
"Use spec.build.source_code_target_dir instead.",
|
|
167
|
-
FutureWarning,
|
|
168
|
-
)
|
|
169
|
-
self.build.source_code_target_dir = clone_target_dir
|
|
170
|
-
|
|
171
144
|
def enrich_function_preemption_spec(self):
|
|
172
145
|
pass
|
|
173
146
|
|
mlrun/runtimes/daskjob.py
CHANGED
|
@@ -91,7 +91,6 @@ class DaskSpec(KubeResourceSpec):
|
|
|
91
91
|
tolerations=None,
|
|
92
92
|
preemption_mode=None,
|
|
93
93
|
security_context=None,
|
|
94
|
-
clone_target_dir=None,
|
|
95
94
|
state_thresholds=None,
|
|
96
95
|
):
|
|
97
96
|
super().__init__(
|
|
@@ -121,7 +120,6 @@ class DaskSpec(KubeResourceSpec):
|
|
|
121
120
|
tolerations=tolerations,
|
|
122
121
|
preemption_mode=preemption_mode,
|
|
123
122
|
security_context=security_context,
|
|
124
|
-
clone_target_dir=clone_target_dir,
|
|
125
123
|
state_thresholds=state_thresholds,
|
|
126
124
|
)
|
|
127
125
|
self.args = args
|
|
@@ -192,7 +190,9 @@ class DaskCluster(KubejobRuntime):
|
|
|
192
190
|
super().__init__(spec, metadata)
|
|
193
191
|
self._cluster = None
|
|
194
192
|
self.use_remote = not mlrun.k8s_utils.is_running_inside_kubernetes_cluster()
|
|
195
|
-
self.spec.build.base_image =
|
|
193
|
+
self.spec.build.base_image = (
|
|
194
|
+
self.spec.build.base_image or mlrun.mlconf.default_base_image
|
|
195
|
+
)
|
|
196
196
|
|
|
197
197
|
@property
|
|
198
198
|
def spec(self) -> DaskSpec:
|
|
@@ -248,7 +248,7 @@ class DaskCluster(KubejobRuntime):
|
|
|
248
248
|
if not self.is_deployed():
|
|
249
249
|
raise RunError(
|
|
250
250
|
"Function image is not built/ready, use .deploy()"
|
|
251
|
-
" method first, or set base dask image
|
|
251
|
+
" method first, or set base dask image to mlrun/mlrun"
|
|
252
252
|
)
|
|
253
253
|
|
|
254
254
|
self.save(versioned=False)
|
|
@@ -82,7 +82,6 @@ class DatabricksSpec(pod.KubeResourceSpec):
|
|
|
82
82
|
tolerations=None,
|
|
83
83
|
preemption_mode=None,
|
|
84
84
|
security_context=None,
|
|
85
|
-
clone_target_dir=None,
|
|
86
85
|
state_thresholds=None,
|
|
87
86
|
):
|
|
88
87
|
super().__init__(
|
|
@@ -112,7 +111,6 @@ class DatabricksSpec(pod.KubeResourceSpec):
|
|
|
112
111
|
tolerations=tolerations,
|
|
113
112
|
preemption_mode=preemption_mode,
|
|
114
113
|
security_context=security_context,
|
|
115
|
-
clone_target_dir=clone_target_dir,
|
|
116
114
|
state_thresholds=state_thresholds,
|
|
117
115
|
)
|
|
118
116
|
self._termination_grace_period_seconds = 60
|
|
@@ -53,7 +53,6 @@ class MPIResourceSpec(KubeResourceSpec):
|
|
|
53
53
|
tolerations=None,
|
|
54
54
|
preemption_mode=None,
|
|
55
55
|
security_context=None,
|
|
56
|
-
clone_target_dir=None,
|
|
57
56
|
state_thresholds=None,
|
|
58
57
|
):
|
|
59
58
|
super().__init__(
|
|
@@ -83,7 +82,6 @@ class MPIResourceSpec(KubeResourceSpec):
|
|
|
83
82
|
tolerations=tolerations,
|
|
84
83
|
preemption_mode=preemption_mode,
|
|
85
84
|
security_context=security_context,
|
|
86
|
-
clone_target_dir=clone_target_dir,
|
|
87
85
|
state_thresholds=state_thresholds,
|
|
88
86
|
)
|
|
89
87
|
self.mpi_args = mpi_args or [
|
mlrun/runtimes/mpijob/v1.py
CHANGED
|
@@ -48,7 +48,6 @@ class MPIV1ResourceSpec(MPIResourceSpec):
|
|
|
48
48
|
tolerations=None,
|
|
49
49
|
preemption_mode=None,
|
|
50
50
|
security_context=None,
|
|
51
|
-
clone_target_dir=None,
|
|
52
51
|
state_thresholds=None,
|
|
53
52
|
):
|
|
54
53
|
super().__init__(
|
|
@@ -79,7 +78,6 @@ class MPIV1ResourceSpec(MPIResourceSpec):
|
|
|
79
78
|
tolerations=tolerations,
|
|
80
79
|
preemption_mode=preemption_mode,
|
|
81
80
|
security_context=security_context,
|
|
82
|
-
clone_target_dir=clone_target_dir,
|
|
83
81
|
state_thresholds=state_thresholds,
|
|
84
82
|
)
|
|
85
83
|
self.clean_pod_policy = clean_pod_policy or MPIJobV1CleanPodPolicies.default()
|
|
@@ -76,7 +76,6 @@ class ApplicationSpec(NuclioSpec):
|
|
|
76
76
|
security_context=None,
|
|
77
77
|
service_type=None,
|
|
78
78
|
add_templated_ingress_host_mode=None,
|
|
79
|
-
clone_target_dir=None,
|
|
80
79
|
state_thresholds=None,
|
|
81
80
|
disable_default_http_trigger=None,
|
|
82
81
|
internal_application_port=None,
|
|
@@ -119,7 +118,6 @@ class ApplicationSpec(NuclioSpec):
|
|
|
119
118
|
security_context=security_context,
|
|
120
119
|
service_type=service_type,
|
|
121
120
|
add_templated_ingress_host_mode=add_templated_ingress_host_mode,
|
|
122
|
-
clone_target_dir=clone_target_dir,
|
|
123
121
|
state_thresholds=state_thresholds,
|
|
124
122
|
disable_default_http_trigger=disable_default_http_trigger,
|
|
125
123
|
)
|
|
@@ -274,7 +272,6 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
274
272
|
project="",
|
|
275
273
|
tag="",
|
|
276
274
|
verbose=False,
|
|
277
|
-
auth_info: schemas.AuthInfo = None,
|
|
278
275
|
builder_env: typing.Optional[dict] = None,
|
|
279
276
|
force_build: bool = False,
|
|
280
277
|
with_mlrun=None,
|
|
@@ -291,7 +288,6 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
291
288
|
:param project: Project name
|
|
292
289
|
:param tag: Function tag
|
|
293
290
|
:param verbose: Set True for verbose logging
|
|
294
|
-
:param auth_info: Service AuthInfo (deprecated and ignored)
|
|
295
291
|
:param builder_env: Env vars dict for source archive config/credentials
|
|
296
292
|
e.g. builder_env={"GIT_TOKEN": token}
|
|
297
293
|
:param force_build: Set True for force building the application image
|
|
@@ -332,7 +328,6 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
332
328
|
project=project,
|
|
333
329
|
tag=tag,
|
|
334
330
|
verbose=verbose,
|
|
335
|
-
auth_info=auth_info,
|
|
336
331
|
builder_env=builder_env,
|
|
337
332
|
)
|
|
338
333
|
logger.info(
|
|
@@ -16,7 +16,6 @@ import asyncio
|
|
|
16
16
|
import copy
|
|
17
17
|
import json
|
|
18
18
|
import typing
|
|
19
|
-
import warnings
|
|
20
19
|
from datetime import datetime
|
|
21
20
|
from time import sleep
|
|
22
21
|
|
|
@@ -153,7 +152,6 @@ class NuclioSpec(KubeResourceSpec):
|
|
|
153
152
|
security_context=None,
|
|
154
153
|
service_type=None,
|
|
155
154
|
add_templated_ingress_host_mode=None,
|
|
156
|
-
clone_target_dir=None,
|
|
157
155
|
state_thresholds=None,
|
|
158
156
|
disable_default_http_trigger=None,
|
|
159
157
|
):
|
|
@@ -184,7 +182,6 @@ class NuclioSpec(KubeResourceSpec):
|
|
|
184
182
|
tolerations=tolerations,
|
|
185
183
|
preemption_mode=preemption_mode,
|
|
186
184
|
security_context=security_context,
|
|
187
|
-
clone_target_dir=clone_target_dir,
|
|
188
185
|
state_thresholds=state_thresholds,
|
|
189
186
|
)
|
|
190
187
|
|
|
@@ -609,7 +606,6 @@ class RemoteRuntime(KubeResource):
|
|
|
609
606
|
project="",
|
|
610
607
|
tag="",
|
|
611
608
|
verbose=False,
|
|
612
|
-
auth_info: AuthInfo = None,
|
|
613
609
|
builder_env: typing.Optional[dict] = None,
|
|
614
610
|
force_build: bool = False,
|
|
615
611
|
):
|
|
@@ -618,16 +614,9 @@ class RemoteRuntime(KubeResource):
|
|
|
618
614
|
:param project: project name
|
|
619
615
|
:param tag: function tag
|
|
620
616
|
:param verbose: set True for verbose logging
|
|
621
|
-
:param auth_info: service AuthInfo (deprecated and ignored)
|
|
622
617
|
:param builder_env: env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
|
|
623
618
|
:param force_build: set True for force building the image
|
|
624
619
|
"""
|
|
625
|
-
if auth_info:
|
|
626
|
-
# TODO: remove in 1.10.0
|
|
627
|
-
warnings.warn(
|
|
628
|
-
"'auth_info' is deprecated for nuclio runtimes in 1.7.0 and will be removed in 1.10.0",
|
|
629
|
-
FutureWarning,
|
|
630
|
-
)
|
|
631
620
|
|
|
632
621
|
old_http_session = getattr(self, "_http_session", None)
|
|
633
622
|
if old_http_session:
|