mlrun 1.8.0rc1__py3-none-any.whl → 1.8.0rc3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +5 -7
- mlrun/__main__.py +1 -1
- mlrun/artifacts/__init__.py +1 -0
- mlrun/artifacts/document.py +313 -0
- mlrun/artifacts/manager.py +2 -0
- mlrun/common/formatters/project.py +9 -0
- mlrun/common/schemas/__init__.py +4 -0
- mlrun/common/schemas/alert.py +31 -18
- mlrun/common/schemas/api_gateway.py +3 -3
- mlrun/common/schemas/artifact.py +7 -7
- mlrun/common/schemas/auth.py +6 -4
- mlrun/common/schemas/background_task.py +7 -7
- mlrun/common/schemas/client_spec.py +2 -2
- mlrun/common/schemas/clusterization_spec.py +2 -2
- mlrun/common/schemas/common.py +5 -5
- mlrun/common/schemas/constants.py +15 -0
- mlrun/common/schemas/datastore_profile.py +1 -1
- mlrun/common/schemas/feature_store.py +9 -9
- mlrun/common/schemas/frontend_spec.py +4 -4
- mlrun/common/schemas/function.py +10 -10
- mlrun/common/schemas/hub.py +1 -1
- mlrun/common/schemas/k8s.py +3 -3
- mlrun/common/schemas/memory_reports.py +3 -3
- mlrun/common/schemas/model_monitoring/grafana.py +1 -1
- mlrun/common/schemas/model_monitoring/model_endpoint_v2.py +1 -1
- mlrun/common/schemas/model_monitoring/model_endpoints.py +1 -1
- mlrun/common/schemas/notification.py +18 -3
- mlrun/common/schemas/object.py +1 -1
- mlrun/common/schemas/pagination.py +4 -4
- mlrun/common/schemas/partition.py +16 -1
- mlrun/common/schemas/pipeline.py +2 -2
- mlrun/common/schemas/project.py +22 -17
- mlrun/common/schemas/runs.py +2 -2
- mlrun/common/schemas/runtime_resource.py +5 -5
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/schemas/secret.py +1 -1
- mlrun/common/schemas/tag.py +3 -3
- mlrun/common/schemas/workflow.py +5 -5
- mlrun/config.py +23 -1
- mlrun/datastore/datastore_profile.py +38 -19
- mlrun/datastore/vectorstore.py +186 -0
- mlrun/db/base.py +58 -6
- mlrun/db/httpdb.py +267 -15
- mlrun/db/nopdb.py +44 -5
- mlrun/execution.py +47 -1
- mlrun/model.py +2 -2
- mlrun/model_monitoring/applications/results.py +2 -2
- mlrun/model_monitoring/db/tsdb/base.py +2 -2
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +37 -13
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +32 -40
- mlrun/model_monitoring/helpers.py +4 -10
- mlrun/model_monitoring/stream_processing.py +14 -11
- mlrun/platforms/__init__.py +44 -13
- mlrun/projects/__init__.py +6 -1
- mlrun/projects/pipelines.py +184 -55
- mlrun/projects/project.py +309 -33
- mlrun/run.py +4 -1
- mlrun/runtimes/base.py +2 -1
- mlrun/runtimes/mounts.py +572 -0
- mlrun/runtimes/nuclio/function.py +1 -2
- mlrun/runtimes/pod.py +82 -18
- mlrun/runtimes/remotesparkjob.py +1 -1
- mlrun/runtimes/sparkjob/spark3job.py +1 -1
- mlrun/utils/clones.py +1 -1
- mlrun/utils/helpers.py +12 -2
- mlrun/utils/logger.py +2 -2
- mlrun/utils/notifications/notification/__init__.py +22 -19
- mlrun/utils/notifications/notification/base.py +12 -12
- mlrun/utils/notifications/notification/console.py +6 -6
- mlrun/utils/notifications/notification/git.py +6 -6
- mlrun/utils/notifications/notification/ipython.py +6 -6
- mlrun/utils/notifications/notification/mail.py +149 -0
- mlrun/utils/notifications/notification/slack.py +6 -6
- mlrun/utils/notifications/notification/webhook.py +6 -6
- mlrun/utils/notifications/notification_pusher.py +20 -12
- mlrun/utils/regex.py +2 -0
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc3.dist-info}/METADATA +190 -186
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc3.dist-info}/RECORD +83 -79
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc3.dist-info}/WHEEL +1 -1
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc3.dist-info}/LICENSE +0 -0
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc3.dist-info}/entry_points.txt +0 -0
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc3.dist-info}/top_level.txt +0 -0
mlrun/db/base.py
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
import datetime
|
|
16
16
|
from abc import ABC, abstractmethod
|
|
17
|
-
from typing import Optional, Union
|
|
17
|
+
from typing import Literal, Optional, Union
|
|
18
18
|
|
|
19
19
|
from deprecated import deprecated
|
|
20
20
|
|
|
@@ -23,6 +23,7 @@ import mlrun.common
|
|
|
23
23
|
import mlrun.common.formatters
|
|
24
24
|
import mlrun.common.runtimes.constants
|
|
25
25
|
import mlrun.common.schemas
|
|
26
|
+
import mlrun.common.schemas.model_monitoring.model_endpoints as mm_endpoints
|
|
26
27
|
import mlrun.model_monitoring
|
|
27
28
|
|
|
28
29
|
|
|
@@ -157,6 +158,16 @@ class RunDBInterface(ABC):
|
|
|
157
158
|
tree: Optional[str] = None,
|
|
158
159
|
format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
|
|
159
160
|
limit: Optional[int] = None,
|
|
161
|
+
partition_by: Optional[
|
|
162
|
+
Union[mlrun.common.schemas.ArtifactPartitionByField, str]
|
|
163
|
+
] = None,
|
|
164
|
+
rows_per_partition: int = 1,
|
|
165
|
+
partition_sort_by: Optional[
|
|
166
|
+
Union[mlrun.common.schemas.SortField, str]
|
|
167
|
+
] = mlrun.common.schemas.SortField.updated,
|
|
168
|
+
partition_order: Union[
|
|
169
|
+
mlrun.common.schemas.OrderType, str
|
|
170
|
+
] = mlrun.common.schemas.OrderType.desc,
|
|
160
171
|
):
|
|
161
172
|
pass
|
|
162
173
|
|
|
@@ -213,11 +224,13 @@ class RunDBInterface(ABC):
|
|
|
213
224
|
def list_functions(
|
|
214
225
|
self,
|
|
215
226
|
name: Optional[str] = None,
|
|
216
|
-
project: Optional[str] =
|
|
217
|
-
tag: Optional[str] =
|
|
227
|
+
project: Optional[str] = None,
|
|
228
|
+
tag: Optional[str] = None,
|
|
229
|
+
kind: Optional[str] = None,
|
|
218
230
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
219
|
-
|
|
220
|
-
|
|
231
|
+
format_: mlrun.common.formatters.FunctionFormat = mlrun.common.formatters.FunctionFormat.full,
|
|
232
|
+
since: Optional[datetime.datetime] = None,
|
|
233
|
+
until: Optional[datetime.datetime] = None,
|
|
221
234
|
):
|
|
222
235
|
pass
|
|
223
236
|
|
|
@@ -306,6 +319,14 @@ class RunDBInterface(ABC):
|
|
|
306
319
|
kind="artifact", identifiers=artifact_identifiers
|
|
307
320
|
)
|
|
308
321
|
|
|
322
|
+
def get_model_endpoint_monitoring_metrics(
|
|
323
|
+
self,
|
|
324
|
+
project: str,
|
|
325
|
+
endpoint_id: str,
|
|
326
|
+
type: Literal["results", "metrics", "all"] = "all",
|
|
327
|
+
) -> list[mm_endpoints.ModelEndpointMonitoringMetric]:
|
|
328
|
+
pass
|
|
329
|
+
|
|
309
330
|
@abstractmethod
|
|
310
331
|
def delete_project(
|
|
311
332
|
self,
|
|
@@ -666,7 +687,9 @@ class RunDBInterface(ABC):
|
|
|
666
687
|
start: str = "now-1h",
|
|
667
688
|
end: str = "now",
|
|
668
689
|
metrics: Optional[list[str]] = None,
|
|
669
|
-
|
|
690
|
+
top_level: bool = False,
|
|
691
|
+
uids: Optional[list[str]] = None,
|
|
692
|
+
) -> list[mlrun.model_monitoring.model_endpoint.ModelEndpoint]:
|
|
670
693
|
pass
|
|
671
694
|
|
|
672
695
|
@abstractmethod
|
|
@@ -832,6 +855,35 @@ class RunDBInterface(ABC):
|
|
|
832
855
|
def list_alert_templates(self):
|
|
833
856
|
pass
|
|
834
857
|
|
|
858
|
+
@abstractmethod
|
|
859
|
+
def list_alert_activations(
|
|
860
|
+
self,
|
|
861
|
+
project: Optional[str] = None,
|
|
862
|
+
name: Optional[str] = None,
|
|
863
|
+
since: Optional[datetime.datetime] = None,
|
|
864
|
+
until: Optional[datetime.datetime] = None,
|
|
865
|
+
entity: Optional[str] = None,
|
|
866
|
+
severity: Optional[
|
|
867
|
+
list[Union[mlrun.common.schemas.alert.AlertSeverity, str]]
|
|
868
|
+
] = None,
|
|
869
|
+
entity_kind: Optional[
|
|
870
|
+
Union[mlrun.common.schemas.alert.EventEntityKind, str]
|
|
871
|
+
] = None,
|
|
872
|
+
event_kind: Optional[Union[mlrun.common.schemas.alert.EventKind, str]] = None,
|
|
873
|
+
):
|
|
874
|
+
pass
|
|
875
|
+
|
|
876
|
+
@abstractmethod
|
|
877
|
+
def paginated_list_alert_activations(
|
|
878
|
+
self,
|
|
879
|
+
*args,
|
|
880
|
+
page: Optional[int] = None,
|
|
881
|
+
page_size: Optional[int] = None,
|
|
882
|
+
page_token: Optional[str] = None,
|
|
883
|
+
**kwargs,
|
|
884
|
+
):
|
|
885
|
+
pass
|
|
886
|
+
|
|
835
887
|
@abstractmethod
|
|
836
888
|
def get_builder_status(
|
|
837
889
|
self,
|
mlrun/db/httpdb.py
CHANGED
|
@@ -22,18 +22,20 @@ import warnings
|
|
|
22
22
|
from copy import deepcopy
|
|
23
23
|
from datetime import datetime, timedelta
|
|
24
24
|
from os import path, remove
|
|
25
|
-
from typing import Optional, Union
|
|
25
|
+
from typing import Literal, Optional, Union
|
|
26
26
|
from urllib.parse import urlparse
|
|
27
27
|
|
|
28
|
-
import pydantic
|
|
28
|
+
import pydantic.v1
|
|
29
29
|
import requests
|
|
30
30
|
import semver
|
|
31
|
+
from pydantic import parse_obj_as
|
|
31
32
|
|
|
32
33
|
import mlrun
|
|
33
34
|
import mlrun.common.constants
|
|
34
35
|
import mlrun.common.formatters
|
|
35
36
|
import mlrun.common.runtimes
|
|
36
37
|
import mlrun.common.schemas
|
|
38
|
+
import mlrun.common.schemas.model_monitoring.model_endpoints as mm_endpoints
|
|
37
39
|
import mlrun.common.types
|
|
38
40
|
import mlrun.model_monitoring.model_endpoint
|
|
39
41
|
import mlrun.platforms
|
|
@@ -856,8 +858,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
856
858
|
:param last_update_time_from: Filter by run last update time in ``(last_update_time_from,
|
|
857
859
|
last_update_time_to)``.
|
|
858
860
|
:param last_update_time_to: Filter by run last update time in ``(last_update_time_from, last_update_time_to)``.
|
|
859
|
-
:param partition_by: Field to group results by.
|
|
860
|
-
|
|
861
|
+
:param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
|
|
862
|
+
parameter must be provided as well.
|
|
861
863
|
:param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
|
|
862
864
|
to return per group. Default value is 1.
|
|
863
865
|
:param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
|
|
@@ -1135,6 +1137,16 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1135
1137
|
mlrun.common.formatters.ArtifactFormat
|
|
1136
1138
|
] = mlrun.common.formatters.ArtifactFormat.full,
|
|
1137
1139
|
limit: Optional[int] = None,
|
|
1140
|
+
partition_by: Optional[
|
|
1141
|
+
Union[mlrun.common.schemas.ArtifactPartitionByField, str]
|
|
1142
|
+
] = None,
|
|
1143
|
+
rows_per_partition: int = 1,
|
|
1144
|
+
partition_sort_by: Optional[
|
|
1145
|
+
Union[mlrun.common.schemas.SortField, str]
|
|
1146
|
+
] = mlrun.common.schemas.SortField.updated,
|
|
1147
|
+
partition_order: Union[
|
|
1148
|
+
mlrun.common.schemas.OrderType, str
|
|
1149
|
+
] = mlrun.common.schemas.OrderType.desc,
|
|
1138
1150
|
) -> ArtifactList:
|
|
1139
1151
|
"""List artifacts filtered by various parameters.
|
|
1140
1152
|
|
|
@@ -1176,6 +1188,13 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1176
1188
|
is a workflow id (artifact was created as part of a workflow).
|
|
1177
1189
|
:param format_: The format in which to return the artifacts. Default is 'full'.
|
|
1178
1190
|
:param limit: Maximum number of artifacts to return.
|
|
1191
|
+
:param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
|
|
1192
|
+
parameter must be provided as well.
|
|
1193
|
+
:param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
|
|
1194
|
+
to return per group. Default value is 1.
|
|
1195
|
+
:param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
|
|
1196
|
+
Currently the only allowed values are `created` and `updated`.
|
|
1197
|
+
:param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
|
|
1179
1198
|
"""
|
|
1180
1199
|
|
|
1181
1200
|
artifacts, _ = self._list_artifacts(
|
|
@@ -1193,6 +1212,10 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1193
1212
|
producer_uri=producer_uri,
|
|
1194
1213
|
format_=format_,
|
|
1195
1214
|
limit=limit,
|
|
1215
|
+
partition_by=partition_by,
|
|
1216
|
+
rows_per_partition=rows_per_partition,
|
|
1217
|
+
partition_sort_by=partition_sort_by,
|
|
1218
|
+
partition_order=partition_order,
|
|
1196
1219
|
return_all=True,
|
|
1197
1220
|
)
|
|
1198
1221
|
return artifacts
|
|
@@ -1392,6 +1415,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1392
1415
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
1393
1416
|
since: Optional[datetime] = None,
|
|
1394
1417
|
until: Optional[datetime] = None,
|
|
1418
|
+
kind: Optional[str] = None,
|
|
1419
|
+
format_: mlrun.common.formatters.FunctionFormat = mlrun.common.formatters.FunctionFormat.full,
|
|
1395
1420
|
):
|
|
1396
1421
|
"""Retrieve a list of functions, filtered by specific criteria.
|
|
1397
1422
|
|
|
@@ -1407,13 +1432,17 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1407
1432
|
the specified key-value pairs or key existence.
|
|
1408
1433
|
:param since: Return functions updated after this date (as datetime object).
|
|
1409
1434
|
:param until: Return functions updated before this date (as datetime object).
|
|
1435
|
+
:param kind: Return only functions of a specific kind.
|
|
1436
|
+
:param format_: The format in which to return the functions. Default is 'full'.
|
|
1410
1437
|
:returns: List of function objects (as dictionary).
|
|
1411
1438
|
"""
|
|
1412
1439
|
functions, _ = self._list_functions(
|
|
1413
1440
|
name=name,
|
|
1414
1441
|
project=project,
|
|
1415
1442
|
tag=tag,
|
|
1443
|
+
kind=kind,
|
|
1416
1444
|
labels=labels,
|
|
1445
|
+
format_=format_,
|
|
1417
1446
|
since=since,
|
|
1418
1447
|
until=until,
|
|
1419
1448
|
return_all=True,
|
|
@@ -2463,7 +2492,6 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2463
2492
|
|
|
2464
2493
|
@staticmethod
|
|
2465
2494
|
def _generate_partition_by_params(
|
|
2466
|
-
partition_by_cls,
|
|
2467
2495
|
partition_by,
|
|
2468
2496
|
rows_per_partition,
|
|
2469
2497
|
sort_by,
|
|
@@ -2543,7 +2571,6 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2543
2571
|
if partition_by:
|
|
2544
2572
|
params.update(
|
|
2545
2573
|
self._generate_partition_by_params(
|
|
2546
|
-
mlrun.common.schemas.FeatureStorePartitionByField,
|
|
2547
2574
|
partition_by,
|
|
2548
2575
|
rows_per_partition,
|
|
2549
2576
|
partition_sort_by,
|
|
@@ -2768,7 +2795,6 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2768
2795
|
if partition_by:
|
|
2769
2796
|
params.update(
|
|
2770
2797
|
self._generate_partition_by_params(
|
|
2771
|
-
mlrun.common.schemas.FeatureStorePartitionByField,
|
|
2772
2798
|
partition_by,
|
|
2773
2799
|
rows_per_partition,
|
|
2774
2800
|
partition_sort_by,
|
|
@@ -3359,6 +3385,37 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3359
3385
|
params=params,
|
|
3360
3386
|
)
|
|
3361
3387
|
|
|
3388
|
+
def get_model_endpoint_monitoring_metrics(
|
|
3389
|
+
self,
|
|
3390
|
+
project: str,
|
|
3391
|
+
endpoint_id: str,
|
|
3392
|
+
type: Literal["results", "metrics", "all"] = "all",
|
|
3393
|
+
) -> list[mm_endpoints.ModelEndpointMonitoringMetric]:
|
|
3394
|
+
"""Get application metrics/results by endpoint id and project.
|
|
3395
|
+
|
|
3396
|
+
:param project: The name of the project.
|
|
3397
|
+
:param endpoint_id: The unique id of the model endpoint.
|
|
3398
|
+
:param type: The type of the metrics to return. "all" means "results" and "metrics".
|
|
3399
|
+
|
|
3400
|
+
:return: A list of the application metrics or/and results for this model endpoint.
|
|
3401
|
+
"""
|
|
3402
|
+
path = f"projects/{project}/model-endpoints/{endpoint_id}/metrics"
|
|
3403
|
+
params = {"type": type}
|
|
3404
|
+
error_message = (
|
|
3405
|
+
f"Failed to get model endpoint monitoring metrics,"
|
|
3406
|
+
f" endpoint_id: {endpoint_id}, project: {project}"
|
|
3407
|
+
)
|
|
3408
|
+
response = self.api_call(
|
|
3409
|
+
mlrun.common.types.HTTPMethod.GET,
|
|
3410
|
+
path,
|
|
3411
|
+
error_message,
|
|
3412
|
+
params=params,
|
|
3413
|
+
)
|
|
3414
|
+
monitoring_metrics = response.json()
|
|
3415
|
+
return parse_obj_as(
|
|
3416
|
+
list[mm_endpoints.ModelEndpointMonitoringMetric], monitoring_metrics
|
|
3417
|
+
)
|
|
3418
|
+
|
|
3362
3419
|
def create_user_secrets(
|
|
3363
3420
|
self,
|
|
3364
3421
|
user: str,
|
|
@@ -3536,6 +3593,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3536
3593
|
`m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
|
|
3537
3594
|
:param top_level: if true will return only routers and endpoint that are NOT children of any router
|
|
3538
3595
|
:param uids: if passed will return a list `ModelEndpoint` object with uid in uids
|
|
3596
|
+
|
|
3597
|
+
:returns: Returns a list of `ModelEndpoint` objects.
|
|
3539
3598
|
"""
|
|
3540
3599
|
|
|
3541
3600
|
path = f"projects/{project}/model-endpoints"
|
|
@@ -4183,12 +4242,21 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4183
4242
|
"operations/migrations",
|
|
4184
4243
|
"Failed triggering migrations",
|
|
4185
4244
|
)
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4245
|
+
return self._wait_for_background_task_from_response(response)
|
|
4246
|
+
|
|
4247
|
+
def refresh_smtp_configuration(
|
|
4248
|
+
self,
|
|
4249
|
+
) -> Optional[mlrun.common.schemas.BackgroundTask]:
|
|
4250
|
+
"""Refresh smtp configuration and wait for the task to finish
|
|
4251
|
+
|
|
4252
|
+
:returns: :py:class:`~mlrun.common.schemas.BackgroundTask`.
|
|
4253
|
+
"""
|
|
4254
|
+
response = self.api_call(
|
|
4255
|
+
"POST",
|
|
4256
|
+
"operations/refresh-smtp-configuration",
|
|
4257
|
+
"Failed refreshing smtp configuration",
|
|
4258
|
+
)
|
|
4259
|
+
return self._wait_for_background_task_from_response(response)
|
|
4192
4260
|
|
|
4193
4261
|
def set_run_notifications(
|
|
4194
4262
|
self,
|
|
@@ -4625,6 +4693,112 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4625
4693
|
results.append(mlrun.common.schemas.AlertTemplate(**item))
|
|
4626
4694
|
return results
|
|
4627
4695
|
|
|
4696
|
+
def list_alert_activations(
|
|
4697
|
+
self,
|
|
4698
|
+
project: Optional[str] = None,
|
|
4699
|
+
name: Optional[str] = None,
|
|
4700
|
+
since: Optional[datetime] = None,
|
|
4701
|
+
until: Optional[datetime] = None,
|
|
4702
|
+
entity: Optional[str] = None,
|
|
4703
|
+
severity: Optional[
|
|
4704
|
+
list[Union[mlrun.common.schemas.alert.AlertSeverity, str]]
|
|
4705
|
+
] = None,
|
|
4706
|
+
entity_kind: Optional[
|
|
4707
|
+
Union[mlrun.common.schemas.alert.EventEntityKind, str]
|
|
4708
|
+
] = None,
|
|
4709
|
+
event_kind: Optional[Union[mlrun.common.schemas.alert.EventKind, str]] = None,
|
|
4710
|
+
) -> list[mlrun.common.schemas.AlertActivation]:
|
|
4711
|
+
"""
|
|
4712
|
+
Retrieve a list of all alert activations.
|
|
4713
|
+
|
|
4714
|
+
:param project: The project name to filter by. If None, results are not filtered by project.
|
|
4715
|
+
:param name: The alert name to filter by. Supports exact matching or partial matching if prefixed with `~`.
|
|
4716
|
+
:param since: Filters for alert activations occurring after this timestamp.
|
|
4717
|
+
:param until: Filters for alert activations occurring before this timestamp.
|
|
4718
|
+
:param entity: The entity ID to filter by. Supports wildcard matching if prefixed with `~`.
|
|
4719
|
+
:param severity: A list of severity levels to filter by (e.g., ["high", "low"]).
|
|
4720
|
+
:param entity_kind: The kind of entity (e.g., "job", "endpoint") to filter by.
|
|
4721
|
+
:param event_kind: The kind of event (e.g., ""data-drift-detected"", "failed") to filter by.
|
|
4722
|
+
|
|
4723
|
+
:returns: A list of alert activations matching the provided filters.
|
|
4724
|
+
"""
|
|
4725
|
+
|
|
4726
|
+
alert_activations, _ = self._list_alert_activations(
|
|
4727
|
+
project=project,
|
|
4728
|
+
name=name,
|
|
4729
|
+
since=since,
|
|
4730
|
+
until=until,
|
|
4731
|
+
entity=entity,
|
|
4732
|
+
severity=severity,
|
|
4733
|
+
entity_kind=entity_kind,
|
|
4734
|
+
event_kind=event_kind,
|
|
4735
|
+
return_all=True,
|
|
4736
|
+
)
|
|
4737
|
+
return alert_activations
|
|
4738
|
+
|
|
4739
|
+
def paginated_list_alert_activations(
|
|
4740
|
+
self,
|
|
4741
|
+
*args,
|
|
4742
|
+
page: Optional[int] = None,
|
|
4743
|
+
page_size: Optional[int] = None,
|
|
4744
|
+
page_token: Optional[str] = None,
|
|
4745
|
+
**kwargs,
|
|
4746
|
+
) -> tuple[list, Optional[str]]:
|
|
4747
|
+
"""List alerts activations with support for pagination and various filtering options.
|
|
4748
|
+
|
|
4749
|
+
This method retrieves a paginated list of alert activations based on the specified filter parameters.
|
|
4750
|
+
Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
|
|
4751
|
+
will return a list of alert activations that match the filtering criteria provided.
|
|
4752
|
+
|
|
4753
|
+
For detailed information about the parameters, refer to the list_alert_activations method:
|
|
4754
|
+
See :py:func:`~list_alert_activations` for more details.
|
|
4755
|
+
|
|
4756
|
+
Examples::
|
|
4757
|
+
|
|
4758
|
+
# Fetch first page of alert activations with page size of 5
|
|
4759
|
+
alert_activations, token = db.paginated_list_alert_activations(
|
|
4760
|
+
project="my-project", page_size=5
|
|
4761
|
+
)
|
|
4762
|
+
# Fetch next page using the pagination token from the previous response
|
|
4763
|
+
alert_activations, token = db.paginated_list_alert_activations(
|
|
4764
|
+
project="my-project", page_token=token
|
|
4765
|
+
)
|
|
4766
|
+
# Fetch alert activations for a specific page (e.g., page 3)
|
|
4767
|
+
alert_activations, token = db.paginated_list_alert_activations(
|
|
4768
|
+
project="my-project", page=3, page_size=5
|
|
4769
|
+
)
|
|
4770
|
+
|
|
4771
|
+
# Automatically iterate over all pages without explicitly specifying the page number
|
|
4772
|
+
alert_activations = []
|
|
4773
|
+
token = None
|
|
4774
|
+
while True:
|
|
4775
|
+
page_alert_activations, token = db.paginated_list_alert_activations(
|
|
4776
|
+
project="my-project", page_token=token, page_size=5
|
|
4777
|
+
)
|
|
4778
|
+
alert_activations.extend(page_alert_activations)
|
|
4779
|
+
|
|
4780
|
+
# If token is None and page_alert_activations is empty, we've reached the end (no more activations).
|
|
4781
|
+
# If token is None and page_alert_activations is not empty, we've fetched the last page of activations.
|
|
4782
|
+
if not token:
|
|
4783
|
+
break
|
|
4784
|
+
print(f"Total alert activations retrieved: {len(alert_activations)}")
|
|
4785
|
+
|
|
4786
|
+
:param page: The page number to retrieve. If not provided, the next page will be retrieved.
|
|
4787
|
+
:param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
|
|
4788
|
+
:param page_token: A pagination token used to retrieve the next page of results. Should not be provided
|
|
4789
|
+
for the first request.
|
|
4790
|
+
|
|
4791
|
+
:returns: A tuple containing the list of alert activations and an optional `page_token` for pagination.
|
|
4792
|
+
"""
|
|
4793
|
+
return self._list_alert_activations(
|
|
4794
|
+
*args,
|
|
4795
|
+
page=page,
|
|
4796
|
+
page_size=page_size,
|
|
4797
|
+
page_token=page_token,
|
|
4798
|
+
return_all=False,
|
|
4799
|
+
**kwargs,
|
|
4800
|
+
)
|
|
4801
|
+
|
|
4628
4802
|
@staticmethod
|
|
4629
4803
|
def _parse_labels(
|
|
4630
4804
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]],
|
|
@@ -4641,7 +4815,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4641
4815
|
"""
|
|
4642
4816
|
try:
|
|
4643
4817
|
return mlrun.common.schemas.common.LabelsModel(labels=labels).labels
|
|
4644
|
-
except pydantic.error_wrappers.ValidationError as exc:
|
|
4818
|
+
except pydantic.v1.error_wrappers.ValidationError as exc:
|
|
4645
4819
|
raise mlrun.errors.MLRunValueError(
|
|
4646
4820
|
"Invalid labels format. Must be a dictionary of strings, a list of strings, "
|
|
4647
4821
|
"or a comma-separated string."
|
|
@@ -4665,6 +4839,16 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4665
4839
|
mlrun.common.formatters.ArtifactFormat
|
|
4666
4840
|
] = mlrun.common.formatters.ArtifactFormat.full,
|
|
4667
4841
|
limit: Optional[int] = None,
|
|
4842
|
+
partition_by: Optional[
|
|
4843
|
+
Union[mlrun.common.schemas.ArtifactPartitionByField, str]
|
|
4844
|
+
] = None,
|
|
4845
|
+
rows_per_partition: int = 1,
|
|
4846
|
+
partition_sort_by: Optional[
|
|
4847
|
+
Union[mlrun.common.schemas.SortField, str]
|
|
4848
|
+
] = mlrun.common.schemas.SortField.updated,
|
|
4849
|
+
partition_order: Union[
|
|
4850
|
+
mlrun.common.schemas.OrderType, str
|
|
4851
|
+
] = mlrun.common.schemas.OrderType.desc,
|
|
4668
4852
|
page: Optional[int] = None,
|
|
4669
4853
|
page_size: Optional[int] = None,
|
|
4670
4854
|
page_token: Optional[str] = None,
|
|
@@ -4694,6 +4878,15 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4694
4878
|
"page-token": page_token,
|
|
4695
4879
|
}
|
|
4696
4880
|
|
|
4881
|
+
if partition_by:
|
|
4882
|
+
params.update(
|
|
4883
|
+
self._generate_partition_by_params(
|
|
4884
|
+
partition_by,
|
|
4885
|
+
rows_per_partition,
|
|
4886
|
+
partition_sort_by,
|
|
4887
|
+
partition_order,
|
|
4888
|
+
)
|
|
4889
|
+
)
|
|
4697
4890
|
error = "list artifacts"
|
|
4698
4891
|
endpoint_path = f"projects/{project}/artifacts"
|
|
4699
4892
|
|
|
@@ -4719,7 +4912,9 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4719
4912
|
name: Optional[str] = None,
|
|
4720
4913
|
project: Optional[str] = None,
|
|
4721
4914
|
tag: Optional[str] = None,
|
|
4915
|
+
kind: Optional[str] = None,
|
|
4722
4916
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
4917
|
+
format_: Optional[str] = None,
|
|
4723
4918
|
since: Optional[datetime] = None,
|
|
4724
4919
|
until: Optional[datetime] = None,
|
|
4725
4920
|
page: Optional[int] = None,
|
|
@@ -4734,9 +4929,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4734
4929
|
params = {
|
|
4735
4930
|
"name": name,
|
|
4736
4931
|
"tag": tag,
|
|
4932
|
+
"kind": kind,
|
|
4737
4933
|
"label": labels,
|
|
4738
4934
|
"since": datetime_to_iso(since),
|
|
4739
4935
|
"until": datetime_to_iso(until),
|
|
4936
|
+
"format": format_,
|
|
4740
4937
|
"page": page,
|
|
4741
4938
|
"page-size": page_size,
|
|
4742
4939
|
"page-token": page_token,
|
|
@@ -4852,7 +5049,6 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4852
5049
|
if partition_by:
|
|
4853
5050
|
params.update(
|
|
4854
5051
|
self._generate_partition_by_params(
|
|
4855
|
-
mlrun.common.schemas.RunPartitionByField,
|
|
4856
5052
|
partition_by,
|
|
4857
5053
|
rows_per_partition,
|
|
4858
5054
|
partition_sort_by,
|
|
@@ -4870,6 +5066,62 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4870
5066
|
paginated_responses, token = self.process_paginated_responses(responses, "runs")
|
|
4871
5067
|
return RunList(paginated_responses), token
|
|
4872
5068
|
|
|
5069
|
+
def _list_alert_activations(
|
|
5070
|
+
self,
|
|
5071
|
+
project: Optional[str] = None,
|
|
5072
|
+
name: Optional[str] = None,
|
|
5073
|
+
since: Optional[datetime] = None,
|
|
5074
|
+
until: Optional[datetime] = None,
|
|
5075
|
+
entity: Optional[str] = None,
|
|
5076
|
+
severity: Optional[
|
|
5077
|
+
list[Union[mlrun.common.schemas.alert.AlertSeverity, str]]
|
|
5078
|
+
] = None,
|
|
5079
|
+
entity_kind: Optional[
|
|
5080
|
+
Union[mlrun.common.schemas.alert.EventEntityKind, str]
|
|
5081
|
+
] = None,
|
|
5082
|
+
event_kind: Optional[Union[mlrun.common.schemas.alert.EventKind, str]] = None,
|
|
5083
|
+
page: Optional[int] = None,
|
|
5084
|
+
page_size: Optional[int] = None,
|
|
5085
|
+
page_token: Optional[str] = None,
|
|
5086
|
+
return_all: bool = False,
|
|
5087
|
+
) -> tuple[list[mlrun.common.schemas.AlertActivation], Optional[str]]:
|
|
5088
|
+
project = project or config.default_project
|
|
5089
|
+
params = {
|
|
5090
|
+
"name": name,
|
|
5091
|
+
"since": datetime_to_iso(since),
|
|
5092
|
+
"until": datetime_to_iso(until),
|
|
5093
|
+
"entity": entity,
|
|
5094
|
+
"severity": severity,
|
|
5095
|
+
"entity-kind": entity_kind,
|
|
5096
|
+
"event-kind": event_kind,
|
|
5097
|
+
"page": page,
|
|
5098
|
+
"page-size": page_size,
|
|
5099
|
+
"page-token": page_token,
|
|
5100
|
+
}
|
|
5101
|
+
error = "list alert activations"
|
|
5102
|
+
path = f"projects/{project}/alert-activations"
|
|
5103
|
+
|
|
5104
|
+
# Fetch the responses, either one page or all based on `return_all`
|
|
5105
|
+
responses = self.paginated_api_call(
|
|
5106
|
+
"GET", path, error, params=params, return_all=return_all
|
|
5107
|
+
)
|
|
5108
|
+
paginated_responses, token = self.process_paginated_responses(
|
|
5109
|
+
responses, "activations"
|
|
5110
|
+
)
|
|
5111
|
+
paginated_results = [
|
|
5112
|
+
mlrun.common.schemas.AlertActivation(**item) for item in paginated_responses
|
|
5113
|
+
]
|
|
5114
|
+
|
|
5115
|
+
return paginated_results, token
|
|
5116
|
+
|
|
5117
|
+
def _wait_for_background_task_from_response(self, response):
|
|
5118
|
+
if response.status_code == http.HTTPStatus.ACCEPTED:
|
|
5119
|
+
background_task = mlrun.common.schemas.BackgroundTask(**response.json())
|
|
5120
|
+
return self._wait_for_background_task_to_reach_terminal_state(
|
|
5121
|
+
background_task.metadata.name
|
|
5122
|
+
)
|
|
5123
|
+
return None
|
|
5124
|
+
|
|
4873
5125
|
|
|
4874
5126
|
def _as_json(obj):
|
|
4875
5127
|
fn = getattr(obj, "to_json", None)
|
mlrun/db/nopdb.py
CHANGED
|
@@ -182,6 +182,16 @@ class NopDB(RunDBInterface):
|
|
|
182
182
|
tree: Optional[str] = None,
|
|
183
183
|
format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
|
|
184
184
|
limit: Optional[int] = None,
|
|
185
|
+
partition_by: Optional[
|
|
186
|
+
Union[mlrun.common.schemas.ArtifactPartitionByField, str]
|
|
187
|
+
] = None,
|
|
188
|
+
rows_per_partition: int = 1,
|
|
189
|
+
partition_sort_by: Optional[
|
|
190
|
+
Union[mlrun.common.schemas.SortField, str]
|
|
191
|
+
] = mlrun.common.schemas.SortField.updated,
|
|
192
|
+
partition_order: Union[
|
|
193
|
+
mlrun.common.schemas.OrderType, str
|
|
194
|
+
] = mlrun.common.schemas.OrderType.desc,
|
|
185
195
|
):
|
|
186
196
|
return mlrun.lists.ArtifactList()
|
|
187
197
|
|
|
@@ -230,12 +240,14 @@ class NopDB(RunDBInterface):
|
|
|
230
240
|
|
|
231
241
|
def list_functions(
|
|
232
242
|
self,
|
|
233
|
-
name=None,
|
|
234
|
-
project=
|
|
235
|
-
tag=
|
|
243
|
+
name: Optional[str] = None,
|
|
244
|
+
project: Optional[str] = None,
|
|
245
|
+
tag: Optional[str] = None,
|
|
246
|
+
kind: Optional[str] = None,
|
|
236
247
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
237
|
-
|
|
238
|
-
|
|
248
|
+
format_: mlrun.common.formatters.FunctionFormat = mlrun.common.formatters.FunctionFormat.full,
|
|
249
|
+
since: Optional[datetime.datetime] = None,
|
|
250
|
+
until: Optional[datetime.datetime] = None,
|
|
239
251
|
):
|
|
240
252
|
return []
|
|
241
253
|
|
|
@@ -863,3 +875,30 @@ class NopDB(RunDBInterface):
|
|
|
863
875
|
|
|
864
876
|
def list_alert_templates(self):
|
|
865
877
|
pass
|
|
878
|
+
|
|
879
|
+
def list_alert_activations(
|
|
880
|
+
self,
|
|
881
|
+
project: Optional[str] = None,
|
|
882
|
+
name: Optional[str] = None,
|
|
883
|
+
since: Optional[datetime.datetime] = None,
|
|
884
|
+
until: Optional[datetime.datetime] = None,
|
|
885
|
+
entity: Optional[str] = None,
|
|
886
|
+
severity: Optional[
|
|
887
|
+
list[Union[mlrun.common.schemas.alert.AlertSeverity, str]]
|
|
888
|
+
] = None,
|
|
889
|
+
entity_kind: Optional[
|
|
890
|
+
Union[mlrun.common.schemas.alert.EventEntityKind, str]
|
|
891
|
+
] = None,
|
|
892
|
+
event_kind: Optional[Union[mlrun.common.schemas.alert.EventKind, str]] = None,
|
|
893
|
+
):
|
|
894
|
+
pass
|
|
895
|
+
|
|
896
|
+
def paginated_list_alert_activations(
|
|
897
|
+
self,
|
|
898
|
+
*args,
|
|
899
|
+
page: Optional[int] = None,
|
|
900
|
+
page_size: Optional[int] = None,
|
|
901
|
+
page_token: Optional[str] = None,
|
|
902
|
+
**kwargs,
|
|
903
|
+
):
|
|
904
|
+
pass
|
mlrun/execution.py
CHANGED
|
@@ -26,7 +26,13 @@ from dateutil import parser
|
|
|
26
26
|
import mlrun
|
|
27
27
|
import mlrun.common.constants as mlrun_constants
|
|
28
28
|
import mlrun.common.formatters
|
|
29
|
-
from mlrun.artifacts import
|
|
29
|
+
from mlrun.artifacts import (
|
|
30
|
+
Artifact,
|
|
31
|
+
DatasetArtifact,
|
|
32
|
+
DocumentArtifact,
|
|
33
|
+
DocumentLoaderSpec,
|
|
34
|
+
ModelArtifact,
|
|
35
|
+
)
|
|
30
36
|
from mlrun.datastore.store_resources import get_store_resource
|
|
31
37
|
from mlrun.errors import MLRunInvalidArgumentError
|
|
32
38
|
|
|
@@ -861,6 +867,46 @@ class MLClientCtx:
|
|
|
861
867
|
self._update_run()
|
|
862
868
|
return item
|
|
863
869
|
|
|
870
|
+
def log_document(
|
|
871
|
+
self,
|
|
872
|
+
key: str,
|
|
873
|
+
artifact_path: Optional[str] = None,
|
|
874
|
+
document_loader: DocumentLoaderSpec = DocumentLoaderSpec(),
|
|
875
|
+
tag: str = "",
|
|
876
|
+
upload: Optional[bool] = False,
|
|
877
|
+
labels: Optional[dict[str, str]] = None,
|
|
878
|
+
**kwargs,
|
|
879
|
+
) -> DocumentArtifact:
|
|
880
|
+
"""
|
|
881
|
+
Log a document as an artifact.
|
|
882
|
+
|
|
883
|
+
:param key: Artifact key
|
|
884
|
+
:param target_path: Path to the local file
|
|
885
|
+
:param artifact_path: Target path for artifact storage
|
|
886
|
+
:param document_loader: Spec to use to load the artifact as langchain document
|
|
887
|
+
:param tag: Version tag
|
|
888
|
+
:param upload: Whether to upload the artifact
|
|
889
|
+
:param labels: Key-value labels
|
|
890
|
+
:param kwargs: Additional keyword arguments
|
|
891
|
+
:return: DocumentArtifact object
|
|
892
|
+
"""
|
|
893
|
+
doc_artifact = DocumentArtifact(
|
|
894
|
+
key=key,
|
|
895
|
+
document_loader=document_loader,
|
|
896
|
+
**kwargs,
|
|
897
|
+
)
|
|
898
|
+
|
|
899
|
+
item = self._artifacts_manager.log_artifact(
|
|
900
|
+
self,
|
|
901
|
+
doc_artifact,
|
|
902
|
+
artifact_path=extend_artifact_path(artifact_path, self.artifact_path),
|
|
903
|
+
tag=tag,
|
|
904
|
+
upload=upload,
|
|
905
|
+
labels=labels,
|
|
906
|
+
)
|
|
907
|
+
self._update_run()
|
|
908
|
+
return item
|
|
909
|
+
|
|
864
910
|
def get_cached_artifact(self, key):
|
|
865
911
|
"""Return a logged artifact from cache (for potential updates)"""
|
|
866
912
|
warnings.warn(
|