mlrun 1.10.0rc13__py3-none-any.whl → 1.10.0rc42__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +22 -2
- mlrun/artifacts/base.py +0 -31
- mlrun/artifacts/document.py +6 -1
- mlrun/artifacts/llm_prompt.py +123 -25
- mlrun/artifacts/manager.py +0 -5
- mlrun/artifacts/model.py +3 -3
- mlrun/common/constants.py +10 -1
- mlrun/common/formatters/artifact.py +1 -0
- mlrun/common/model_monitoring/helpers.py +86 -0
- mlrun/common/schemas/__init__.py +3 -0
- mlrun/common/schemas/auth.py +2 -0
- mlrun/common/schemas/function.py +10 -0
- mlrun/common/schemas/hub.py +30 -18
- mlrun/common/schemas/model_monitoring/__init__.py +3 -0
- mlrun/common/schemas/model_monitoring/constants.py +30 -6
- mlrun/common/schemas/model_monitoring/functions.py +14 -5
- mlrun/common/schemas/model_monitoring/model_endpoints.py +21 -0
- mlrun/common/schemas/pipeline.py +1 -1
- mlrun/common/schemas/serving.py +3 -0
- mlrun/common/schemas/workflow.py +3 -1
- mlrun/common/secrets.py +22 -1
- mlrun/config.py +33 -11
- mlrun/datastore/__init__.py +11 -3
- mlrun/datastore/azure_blob.py +162 -47
- mlrun/datastore/datastore.py +9 -4
- mlrun/datastore/datastore_profile.py +61 -5
- mlrun/datastore/model_provider/huggingface_provider.py +363 -0
- mlrun/datastore/model_provider/mock_model_provider.py +87 -0
- mlrun/datastore/model_provider/model_provider.py +230 -65
- mlrun/datastore/model_provider/openai_provider.py +295 -42
- mlrun/datastore/s3.py +24 -2
- mlrun/datastore/storeytargets.py +2 -3
- mlrun/datastore/utils.py +15 -3
- mlrun/db/base.py +47 -19
- mlrun/db/httpdb.py +120 -56
- mlrun/db/nopdb.py +38 -10
- mlrun/execution.py +70 -19
- mlrun/hub/__init__.py +15 -0
- mlrun/hub/module.py +181 -0
- mlrun/k8s_utils.py +105 -16
- mlrun/launcher/base.py +13 -6
- mlrun/launcher/local.py +15 -0
- mlrun/model.py +24 -3
- mlrun/model_monitoring/__init__.py +1 -0
- mlrun/model_monitoring/api.py +66 -27
- mlrun/model_monitoring/applications/__init__.py +1 -1
- mlrun/model_monitoring/applications/base.py +509 -117
- mlrun/model_monitoring/applications/context.py +2 -4
- mlrun/model_monitoring/applications/results.py +4 -7
- mlrun/model_monitoring/controller.py +239 -101
- mlrun/model_monitoring/db/_schedules.py +116 -33
- mlrun/model_monitoring/db/_stats.py +4 -3
- mlrun/model_monitoring/db/tsdb/base.py +100 -9
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +11 -6
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +191 -50
- mlrun/model_monitoring/db/tsdb/tdengine/writer_graph_steps.py +51 -0
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +17 -4
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +259 -40
- mlrun/model_monitoring/helpers.py +54 -9
- mlrun/model_monitoring/stream_processing.py +45 -14
- mlrun/model_monitoring/writer.py +220 -1
- mlrun/platforms/__init__.py +3 -2
- mlrun/platforms/iguazio.py +7 -3
- mlrun/projects/operations.py +6 -1
- mlrun/projects/pipelines.py +46 -26
- mlrun/projects/project.py +166 -58
- mlrun/run.py +94 -17
- mlrun/runtimes/__init__.py +18 -0
- mlrun/runtimes/base.py +14 -6
- mlrun/runtimes/daskjob.py +7 -0
- mlrun/runtimes/local.py +5 -2
- mlrun/runtimes/mounts.py +20 -2
- mlrun/runtimes/mpijob/abstract.py +6 -0
- mlrun/runtimes/mpijob/v1.py +6 -0
- mlrun/runtimes/nuclio/__init__.py +1 -0
- mlrun/runtimes/nuclio/application/application.py +149 -17
- mlrun/runtimes/nuclio/function.py +76 -27
- mlrun/runtimes/nuclio/serving.py +97 -15
- mlrun/runtimes/pod.py +234 -21
- mlrun/runtimes/remotesparkjob.py +6 -0
- mlrun/runtimes/sparkjob/spark3job.py +6 -0
- mlrun/runtimes/utils.py +49 -11
- mlrun/secrets.py +54 -13
- mlrun/serving/__init__.py +2 -0
- mlrun/serving/remote.py +79 -6
- mlrun/serving/routers.py +23 -41
- mlrun/serving/server.py +320 -80
- mlrun/serving/states.py +725 -157
- mlrun/serving/steps.py +62 -0
- mlrun/serving/system_steps.py +200 -119
- mlrun/serving/v2_serving.py +9 -10
- mlrun/utils/helpers.py +288 -88
- mlrun/utils/logger.py +3 -1
- mlrun/utils/notifications/notification/base.py +18 -0
- mlrun/utils/notifications/notification/git.py +2 -4
- mlrun/utils/notifications/notification/slack.py +2 -4
- mlrun/utils/notifications/notification/webhook.py +2 -5
- mlrun/utils/notifications/notification_pusher.py +1 -1
- mlrun/utils/retryer.py +15 -2
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.10.0rc13.dist-info → mlrun-1.10.0rc42.dist-info}/METADATA +45 -51
- {mlrun-1.10.0rc13.dist-info → mlrun-1.10.0rc42.dist-info}/RECORD +106 -101
- mlrun/api/schemas/__init__.py +0 -259
- {mlrun-1.10.0rc13.dist-info → mlrun-1.10.0rc42.dist-info}/WHEEL +0 -0
- {mlrun-1.10.0rc13.dist-info → mlrun-1.10.0rc42.dist-info}/entry_points.txt +0 -0
- {mlrun-1.10.0rc13.dist-info → mlrun-1.10.0rc42.dist-info}/licenses/LICENSE +0 -0
- {mlrun-1.10.0rc13.dist-info → mlrun-1.10.0rc42.dist-info}/top_level.txt +0 -0
mlrun/db/httpdb.py
CHANGED
|
@@ -24,6 +24,7 @@ from datetime import datetime, timedelta
|
|
|
24
24
|
from os import environ, path, remove
|
|
25
25
|
from typing import Literal, Optional, Union
|
|
26
26
|
from urllib.parse import urlparse
|
|
27
|
+
from uuid import UUID
|
|
27
28
|
|
|
28
29
|
import pydantic.v1
|
|
29
30
|
import requests
|
|
@@ -44,6 +45,7 @@ import mlrun.runtimes.nuclio.api_gateway
|
|
|
44
45
|
import mlrun.runtimes.nuclio.function
|
|
45
46
|
import mlrun.utils
|
|
46
47
|
from mlrun.alerts.alert import AlertConfig
|
|
48
|
+
from mlrun.common.schemas.hub import HubSourceType
|
|
47
49
|
from mlrun.db.auth_utils import OAuthClientIDTokenProvider, StaticTokenProvider
|
|
48
50
|
from mlrun.errors import MLRunInvalidArgumentError, err_to_str
|
|
49
51
|
from mlrun.secrets import get_secret_or_env
|
|
@@ -757,7 +759,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
757
759
|
)
|
|
758
760
|
if response.status_code == http.HTTPStatus.ACCEPTED:
|
|
759
761
|
background_task = mlrun.common.schemas.BackgroundTask(**response.json())
|
|
760
|
-
return self.
|
|
762
|
+
return self.wait_for_background_task_to_reach_terminal_state(
|
|
761
763
|
background_task.metadata.name, project=project
|
|
762
764
|
)
|
|
763
765
|
return None
|
|
@@ -784,7 +786,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
784
786
|
)
|
|
785
787
|
if response.status_code == http.HTTPStatus.ACCEPTED:
|
|
786
788
|
background_task = mlrun.common.schemas.BackgroundTask(**response.json())
|
|
787
|
-
background_task = self.
|
|
789
|
+
background_task = self.wait_for_background_task_to_reach_terminal_state(
|
|
788
790
|
background_task.metadata.name, project=project
|
|
789
791
|
)
|
|
790
792
|
if (
|
|
@@ -839,7 +841,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
839
841
|
)
|
|
840
842
|
if response.status_code == http.HTTPStatus.ACCEPTED:
|
|
841
843
|
background_task = mlrun.common.schemas.BackgroundTask(**response.json())
|
|
842
|
-
background_task = self.
|
|
844
|
+
background_task = self.wait_for_background_task_to_reach_terminal_state(
|
|
843
845
|
background_task.metadata.name, project=project
|
|
844
846
|
)
|
|
845
847
|
if (
|
|
@@ -1485,7 +1487,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1485
1487
|
"Function is being deleted", project_name=project, function_name=name
|
|
1486
1488
|
)
|
|
1487
1489
|
background_task = mlrun.common.schemas.BackgroundTask(**response.json())
|
|
1488
|
-
background_task = self.
|
|
1490
|
+
background_task = self.wait_for_background_task_to_reach_terminal_state(
|
|
1489
1491
|
background_task.metadata.name, project=project
|
|
1490
1492
|
)
|
|
1491
1493
|
if (
|
|
@@ -2554,50 +2556,6 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2554
2556
|
resp = self.api_call("GET", path, error_message)
|
|
2555
2557
|
return FeatureSet.from_dict(resp.json())
|
|
2556
2558
|
|
|
2557
|
-
def list_features(
|
|
2558
|
-
self,
|
|
2559
|
-
project: Optional[str] = None,
|
|
2560
|
-
name: Optional[str] = None,
|
|
2561
|
-
tag: Optional[str] = None,
|
|
2562
|
-
entities: Optional[list[str]] = None,
|
|
2563
|
-
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
2564
|
-
) -> list[dict]:
|
|
2565
|
-
"""List feature-sets which contain specific features. This function may return multiple versions of the same
|
|
2566
|
-
feature-set if a specific tag is not requested. Note that the various filters of this function actually
|
|
2567
|
-
refer to the feature-set object containing the features, not to the features themselves.
|
|
2568
|
-
|
|
2569
|
-
:param project: Project which contains these features.
|
|
2570
|
-
:param name: Name of the feature to look for. The name is used in a like query, and is not case-sensitive. For
|
|
2571
|
-
example, looking for ``feat`` will return features which are named ``MyFeature`` as well as ``defeat``.
|
|
2572
|
-
:param tag: Return feature-sets which contain the features looked for, and are tagged with the specific tag.
|
|
2573
|
-
:param entities: Return only feature-sets which contain an entity whose name is contained in this list.
|
|
2574
|
-
:param labels: Filter feature-sets by label key-value pairs or key existence. This can be provided as:
|
|
2575
|
-
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
2576
|
-
or `{"label": None}` to check for key existence.
|
|
2577
|
-
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
2578
|
-
or just `"label"` for key existence.
|
|
2579
|
-
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
2580
|
-
the specified key-value pairs or key existence.
|
|
2581
|
-
:returns: A list of mapping from feature to a digest of the feature-set, which contains the feature-set
|
|
2582
|
-
meta-data. Multiple entries may be returned for any specific feature due to multiple tags or versions
|
|
2583
|
-
of the feature-set.
|
|
2584
|
-
"""
|
|
2585
|
-
|
|
2586
|
-
project = project or config.active_project
|
|
2587
|
-
labels = self._parse_labels(labels)
|
|
2588
|
-
params = {
|
|
2589
|
-
"name": name,
|
|
2590
|
-
"tag": tag,
|
|
2591
|
-
"entity": entities or [],
|
|
2592
|
-
"label": labels,
|
|
2593
|
-
}
|
|
2594
|
-
|
|
2595
|
-
path = f"projects/{project}/features"
|
|
2596
|
-
|
|
2597
|
-
error_message = f"Failed listing features, project: {project}, query: {params}"
|
|
2598
|
-
resp = self.api_call("GET", path, error_message, params=params)
|
|
2599
|
-
return resp.json()["features"]
|
|
2600
|
-
|
|
2601
2559
|
def list_features_v2(
|
|
2602
2560
|
self,
|
|
2603
2561
|
project: Optional[str] = None,
|
|
@@ -3274,7 +3232,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3274
3232
|
if response.status_code == http.HTTPStatus.ACCEPTED:
|
|
3275
3233
|
logger.info("Waiting for project to be deleted", project_name=name)
|
|
3276
3234
|
background_task = mlrun.common.schemas.BackgroundTask(**response.json())
|
|
3277
|
-
background_task = self.
|
|
3235
|
+
background_task = self.wait_for_background_task_to_reach_terminal_state(
|
|
3278
3236
|
background_task.metadata.name
|
|
3279
3237
|
)
|
|
3280
3238
|
if (
|
|
@@ -3387,7 +3345,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3387
3345
|
_verify_project_in_terminal_state,
|
|
3388
3346
|
)
|
|
3389
3347
|
|
|
3390
|
-
def
|
|
3348
|
+
def wait_for_background_task_to_reach_terminal_state(
|
|
3391
3349
|
self, name: str, project: str = ""
|
|
3392
3350
|
) -> mlrun.common.schemas.BackgroundTask:
|
|
3393
3351
|
def _verify_background_task_in_terminal_state():
|
|
@@ -3408,6 +3366,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3408
3366
|
logger,
|
|
3409
3367
|
False,
|
|
3410
3368
|
_verify_background_task_in_terminal_state,
|
|
3369
|
+
fatal_exceptions=(mlrun.errors.MLRunAccessDeniedError,),
|
|
3411
3370
|
)
|
|
3412
3371
|
|
|
3413
3372
|
def create_project_secrets(
|
|
@@ -3622,7 +3581,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3622
3581
|
intersection {"intersect_metrics":[], "intersect_results":[]}
|
|
3623
3582
|
:return: A dictionary of application metrics and/or results for the model endpoints formatted by events_format.
|
|
3624
3583
|
"""
|
|
3625
|
-
path = f"projects/{project}/model-
|
|
3584
|
+
path = f"projects/{project}/model-monitoring/metrics"
|
|
3626
3585
|
params = {
|
|
3627
3586
|
"type": type,
|
|
3628
3587
|
"endpoint-id": endpoint_ids,
|
|
@@ -3812,6 +3771,9 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3812
3771
|
tsdb_metrics: bool = False,
|
|
3813
3772
|
metric_list: Optional[list[str]] = None,
|
|
3814
3773
|
top_level: bool = False,
|
|
3774
|
+
modes: Optional[
|
|
3775
|
+
Union[mm_constants.EndpointMode, list[mm_constants.EndpointMode]]
|
|
3776
|
+
] = None,
|
|
3815
3777
|
uids: Optional[list[str]] = None,
|
|
3816
3778
|
latest_only: bool = False,
|
|
3817
3779
|
) -> mlrun.common.schemas.ModelEndpointList:
|
|
@@ -3832,6 +3794,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3832
3794
|
If tsdb_metrics=False, this parameter will be ignored and no tsdb metrics
|
|
3833
3795
|
will be included.
|
|
3834
3796
|
:param top_level: Whether to return only top level model endpoints.
|
|
3797
|
+
:param modes: Specifies the modes of the model endpoints. Can be "real-time" (0), "batch" (1),
|
|
3798
|
+
"batch_legacy" (2). If set to None, all are included.
|
|
3835
3799
|
:param uids: A list of unique ids to filter by.
|
|
3836
3800
|
:param latest_only: Whether to return only the latest model endpoint version.
|
|
3837
3801
|
:return: A list of model endpoints.
|
|
@@ -3840,6 +3804,13 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3840
3804
|
labels = self._parse_labels(labels)
|
|
3841
3805
|
if names and isinstance(names, str):
|
|
3842
3806
|
names = [names]
|
|
3807
|
+
if modes:
|
|
3808
|
+
# Ensure backward compatibility with Python 3.9 clients by converting IntEnum modes to integer values
|
|
3809
|
+
modes = (
|
|
3810
|
+
[modes.value]
|
|
3811
|
+
if isinstance(modes, mm_constants.EndpointMode)
|
|
3812
|
+
else [mode.value for mode in modes]
|
|
3813
|
+
)
|
|
3843
3814
|
response = self.api_call(
|
|
3844
3815
|
method=mlrun.common.types.HTTPMethod.GET,
|
|
3845
3816
|
path=path,
|
|
@@ -3855,6 +3826,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3855
3826
|
"tsdb-metrics": tsdb_metrics,
|
|
3856
3827
|
"metric": metric_list,
|
|
3857
3828
|
"top-level": top_level,
|
|
3829
|
+
"mode": modes,
|
|
3858
3830
|
"uid": uids,
|
|
3859
3831
|
"latest-only": latest_only,
|
|
3860
3832
|
},
|
|
@@ -3963,6 +3935,13 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3963
3935
|
raise MLRunInvalidArgumentError(
|
|
3964
3936
|
"Either endpoint_uid or function_name and function_tag must be provided"
|
|
3965
3937
|
)
|
|
3938
|
+
if uid:
|
|
3939
|
+
try:
|
|
3940
|
+
UUID(uid)
|
|
3941
|
+
except (ValueError, TypeError):
|
|
3942
|
+
raise MLRunInvalidArgumentError(
|
|
3943
|
+
"endpoint_id must be a valid UUID string"
|
|
3944
|
+
)
|
|
3966
3945
|
|
|
3967
3946
|
def update_model_monitoring_controller(
|
|
3968
3947
|
self,
|
|
@@ -4082,7 +4061,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4082
4061
|
**response.json()
|
|
4083
4062
|
).background_tasks
|
|
4084
4063
|
for task in background_tasks:
|
|
4085
|
-
task = self.
|
|
4064
|
+
task = self.wait_for_background_task_to_reach_terminal_state(
|
|
4086
4065
|
task.metadata.name, project=project
|
|
4087
4066
|
)
|
|
4088
4067
|
if (
|
|
@@ -4110,7 +4089,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4110
4089
|
response = self.api_call(
|
|
4111
4090
|
method=mlrun.common.types.HTTPMethod.DELETE,
|
|
4112
4091
|
path=f"projects/{project}/model-monitoring/functions",
|
|
4113
|
-
params={"
|
|
4092
|
+
params={"function": functions},
|
|
4114
4093
|
)
|
|
4115
4094
|
deletion_failed = False
|
|
4116
4095
|
if response.status_code == http.HTTPStatus.ACCEPTED:
|
|
@@ -4119,7 +4098,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4119
4098
|
**response.json()
|
|
4120
4099
|
).background_tasks
|
|
4121
4100
|
for task in background_tasks:
|
|
4122
|
-
task = self.
|
|
4101
|
+
task = self.wait_for_background_task_to_reach_terminal_state(
|
|
4123
4102
|
task.metadata.name, project=project
|
|
4124
4103
|
)
|
|
4125
4104
|
if (
|
|
@@ -4152,6 +4131,26 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4152
4131
|
params={**credentials, "replace_creds": replace_creds},
|
|
4153
4132
|
)
|
|
4154
4133
|
|
|
4134
|
+
def delete_model_monitoring_metrics(
|
|
4135
|
+
self,
|
|
4136
|
+
project: str,
|
|
4137
|
+
application_name: str,
|
|
4138
|
+
endpoint_ids: Optional[list[str]] = None,
|
|
4139
|
+
) -> None:
|
|
4140
|
+
"""
|
|
4141
|
+
Delete model endpoints metrics values.
|
|
4142
|
+
|
|
4143
|
+
:param project: The name of the project.
|
|
4144
|
+
:param application_name: The name of the application.
|
|
4145
|
+
:param endpoint_ids: The unique IDs of the model endpoints to delete metrics values from. If none is
|
|
4146
|
+
provided, the metrics values will be deleted from all project's model endpoints.
|
|
4147
|
+
"""
|
|
4148
|
+
self.api_call(
|
|
4149
|
+
method=mlrun.common.types.HTTPMethod.DELETE,
|
|
4150
|
+
path=f"projects/{project}/model-monitoring/metrics",
|
|
4151
|
+
params={"endpoint-id": endpoint_ids, "application-name": application_name},
|
|
4152
|
+
)
|
|
4153
|
+
|
|
4155
4154
|
def get_monitoring_function_summaries(
|
|
4156
4155
|
self,
|
|
4157
4156
|
project: str,
|
|
@@ -4318,6 +4317,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4318
4317
|
item_name: Optional[str] = None,
|
|
4319
4318
|
tag: Optional[str] = None,
|
|
4320
4319
|
version: Optional[str] = None,
|
|
4320
|
+
item_type: HubSourceType = HubSourceType.functions,
|
|
4321
4321
|
) -> list[mlrun.common.schemas.hub.IndexedHubSource]:
|
|
4322
4322
|
"""
|
|
4323
4323
|
List hub sources in the MLRun DB.
|
|
@@ -4325,6 +4325,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4325
4325
|
:param item_name: Sources contain this item will be returned, If not provided all sources will be returned.
|
|
4326
4326
|
:param tag: Item tag to filter by, supported only if item name is provided.
|
|
4327
4327
|
:param version: Item version to filter by, supported only if item name is provided and tag is not.
|
|
4328
|
+
:param item_type: Item type to filter by, supported only if item name is provided.
|
|
4328
4329
|
|
|
4329
4330
|
:returns: List of indexed hub sources.
|
|
4330
4331
|
"""
|
|
@@ -4332,6 +4333,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4332
4333
|
params = {}
|
|
4333
4334
|
if item_name:
|
|
4334
4335
|
params["item-name"] = normalize_name(item_name)
|
|
4336
|
+
params["item-type"] = item_type
|
|
4335
4337
|
if tag:
|
|
4336
4338
|
params["tag"] = tag
|
|
4337
4339
|
if version:
|
|
@@ -4370,6 +4372,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4370
4372
|
version: Optional[str] = None,
|
|
4371
4373
|
tag: Optional[str] = None,
|
|
4372
4374
|
force_refresh: bool = False,
|
|
4375
|
+
object_type: HubSourceType = HubSourceType.functions,
|
|
4373
4376
|
):
|
|
4374
4377
|
"""
|
|
4375
4378
|
Retrieve the item catalog for a specified hub source.
|
|
@@ -4382,6 +4385,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4382
4385
|
rather than rely on cached information which may exist from previous get requests. For example,
|
|
4383
4386
|
if the source was re-built,
|
|
4384
4387
|
this will make the server get the updated information. Default is ``False``.
|
|
4388
|
+
:param object_type: Type of object to retrieve from the hub source (e.g: functions, modules).
|
|
4385
4389
|
:returns: :py:class:`~mlrun.common.schemas.hub.HubCatalog` object, which is essentially a list
|
|
4386
4390
|
of :py:class:`~mlrun.common.schemas.hub.HubItem` entries.
|
|
4387
4391
|
"""
|
|
@@ -4390,6 +4394,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4390
4394
|
"version": version,
|
|
4391
4395
|
"tag": tag,
|
|
4392
4396
|
"force-refresh": force_refresh,
|
|
4397
|
+
"object_type": object_type,
|
|
4393
4398
|
}
|
|
4394
4399
|
response = self.api_call(method="GET", path=path, params=params)
|
|
4395
4400
|
return mlrun.common.schemas.HubCatalog(**response.json())
|
|
@@ -4401,6 +4406,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4401
4406
|
version: Optional[str] = None,
|
|
4402
4407
|
tag: str = "latest",
|
|
4403
4408
|
force_refresh: bool = False,
|
|
4409
|
+
item_type: HubSourceType = HubSourceType.functions,
|
|
4404
4410
|
):
|
|
4405
4411
|
"""
|
|
4406
4412
|
Retrieve a specific hub item.
|
|
@@ -4412,6 +4418,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4412
4418
|
:param force_refresh: Make the server fetch the information from the actual hub
|
|
4413
4419
|
source, rather than
|
|
4414
4420
|
rely on cached information. Default is ``False``.
|
|
4421
|
+
:param item_type: The type of item to retrieve from the hub source (e.g: functions, modules).
|
|
4415
4422
|
:returns: :py:class:`~mlrun.common.schemas.hub.HubItem`.
|
|
4416
4423
|
"""
|
|
4417
4424
|
path = (f"hub/sources/{source_name}/items/{item_name}",)
|
|
@@ -4419,6 +4426,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4419
4426
|
"version": version,
|
|
4420
4427
|
"tag": tag,
|
|
4421
4428
|
"force-refresh": force_refresh,
|
|
4429
|
+
"item_type": item_type,
|
|
4422
4430
|
}
|
|
4423
4431
|
response = self.api_call(method="GET", path=path, params=params)
|
|
4424
4432
|
return mlrun.common.schemas.HubItem(**response.json())
|
|
@@ -4430,6 +4438,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4430
4438
|
asset_name: str,
|
|
4431
4439
|
version: Optional[str] = None,
|
|
4432
4440
|
tag: str = "latest",
|
|
4441
|
+
item_type: HubSourceType = HubSourceType.functions,
|
|
4433
4442
|
):
|
|
4434
4443
|
"""
|
|
4435
4444
|
Get hub asset from item.
|
|
@@ -4439,13 +4448,14 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4439
4448
|
:param asset_name: Name of the asset to retrieve.
|
|
4440
4449
|
:param version: Get a specific version of the item. Default is ``None``.
|
|
4441
4450
|
:param tag: Get a specific version of the item identified by tag. Default is ``latest``.
|
|
4442
|
-
|
|
4451
|
+
:param item_type: The type of item to retrieve from the hub source (e.g: functions, modules).
|
|
4443
4452
|
:returns: http response with the asset in the content attribute
|
|
4444
4453
|
"""
|
|
4445
4454
|
path = f"hub/sources/{source_name}/items/{item_name}/assets/{asset_name}"
|
|
4446
4455
|
params = {
|
|
4447
4456
|
"version": version,
|
|
4448
4457
|
"tag": tag,
|
|
4458
|
+
"item_type": item_type,
|
|
4449
4459
|
}
|
|
4450
4460
|
response = self.api_call(method="GET", path=path, params=params)
|
|
4451
4461
|
return response
|
|
@@ -4740,6 +4750,28 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4740
4750
|
)
|
|
4741
4751
|
return mlrun.common.schemas.GetWorkflowResponse(**response.json())
|
|
4742
4752
|
|
|
4753
|
+
def set_run_retrying_status(
|
|
4754
|
+
self, project: str, name: str, run_id: str, retrying: bool = False
|
|
4755
|
+
):
|
|
4756
|
+
"""
|
|
4757
|
+
Toggle the “retrying” label on a workflow-runner run.
|
|
4758
|
+
|
|
4759
|
+
This will POST to the workflows endpoint to either add or remove the
|
|
4760
|
+
`retrying` flag on a specific run, which prevents parallel retries.
|
|
4761
|
+
|
|
4762
|
+
:param project: The project name under which the workflow is defined.
|
|
4763
|
+
:param name: The workflow name (as in the URL path).
|
|
4764
|
+
:param run_id: The UID of the workflow-runner run to update.
|
|
4765
|
+
:param retrying: True to add the `retrying` label, False to remove it.
|
|
4766
|
+
|
|
4767
|
+
:raises MLRunHTTPError: If the HTTP request fails or returns an error status.
|
|
4768
|
+
"""
|
|
4769
|
+
path = f"projects/{project}/workflows/{name}/runs/{run_id}/set-retry-status"
|
|
4770
|
+
params = {"retrying": retrying}
|
|
4771
|
+
self.api_call(
|
|
4772
|
+
"POST", path, f"set retrying on {project}/{run_id}", params=params
|
|
4773
|
+
)
|
|
4774
|
+
|
|
4743
4775
|
def load_project(
|
|
4744
4776
|
self,
|
|
4745
4777
|
name: str,
|
|
@@ -5158,6 +5190,38 @@ class HTTPRunDB(RunDBInterface):
|
|
|
5158
5190
|
response = self.api_call("GET", endpoint_path, error_message)
|
|
5159
5191
|
return mlrun.common.schemas.ProjectSummary(**response.json())
|
|
5160
5192
|
|
|
5193
|
+
def get_drift_over_time(
|
|
5194
|
+
self,
|
|
5195
|
+
project: str,
|
|
5196
|
+
start: Optional[datetime] = None,
|
|
5197
|
+
end: Optional[datetime] = None,
|
|
5198
|
+
) -> mlrun.common.schemas.model_monitoring.ModelEndpointDriftValues:
|
|
5199
|
+
"""
|
|
5200
|
+
Get drift counts over time for the project.
|
|
5201
|
+
|
|
5202
|
+
This method returns a list of tuples, each representing a time-interval (in a granularity set by the
|
|
5203
|
+
duration of the given time range) and the number of suspected drifts and detected drifts in that interval.
|
|
5204
|
+
For a range of 6 hours or less, the granularity is 10 minute, for a range of 2 hours to 72 hours, the
|
|
5205
|
+
granularity is 1 hour, and for a range of more than 72 hours, the granularity is 24 hours.
|
|
5206
|
+
|
|
5207
|
+
:param project: The name of the project for which to retrieve drift counts.
|
|
5208
|
+
:param start: Start time of the range to retrieve drift counts from.
|
|
5209
|
+
:param end: End time of the range to retrieve drift counts from.
|
|
5210
|
+
|
|
5211
|
+
:return: A ModelEndpointDriftValues object containing the drift counts over time.
|
|
5212
|
+
"""
|
|
5213
|
+
endpoint_path = f"projects/{project}/model-monitoring/drift-over-time"
|
|
5214
|
+
error_message = f"Failed retrieving drift data for {project}"
|
|
5215
|
+
response = self.api_call(
|
|
5216
|
+
method="GET",
|
|
5217
|
+
path=endpoint_path,
|
|
5218
|
+
error=error_message,
|
|
5219
|
+
params={"start": start, "end": end},
|
|
5220
|
+
)
|
|
5221
|
+
return mlrun.common.schemas.model_monitoring.ModelEndpointDriftValues(
|
|
5222
|
+
**response.json()
|
|
5223
|
+
)
|
|
5224
|
+
|
|
5161
5225
|
@staticmethod
|
|
5162
5226
|
def _parse_labels(
|
|
5163
5227
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]],
|
|
@@ -5478,7 +5542,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
5478
5542
|
def _wait_for_background_task_from_response(self, response):
|
|
5479
5543
|
if response.status_code == http.HTTPStatus.ACCEPTED:
|
|
5480
5544
|
background_task = mlrun.common.schemas.BackgroundTask(**response.json())
|
|
5481
|
-
return self.
|
|
5545
|
+
return self.wait_for_background_task_to_reach_terminal_state(
|
|
5482
5546
|
background_task.metadata.name
|
|
5483
5547
|
)
|
|
5484
5548
|
return None
|
mlrun/db/nopdb.py
CHANGED
|
@@ -72,6 +72,11 @@ class NopDB(RunDBInterface):
|
|
|
72
72
|
def update_run(self, updates: dict, uid, project="", iter=0):
|
|
73
73
|
pass
|
|
74
74
|
|
|
75
|
+
def set_run_retrying_status(
|
|
76
|
+
self, project: str, name: str, run_id: str, retrying: bool
|
|
77
|
+
):
|
|
78
|
+
pass
|
|
79
|
+
|
|
75
80
|
def abort_run(self, uid, project="", iter=0, timeout=45, status_text=""):
|
|
76
81
|
pass
|
|
77
82
|
|
|
@@ -371,16 +376,6 @@ class NopDB(RunDBInterface):
|
|
|
371
376
|
) -> dict:
|
|
372
377
|
pass
|
|
373
378
|
|
|
374
|
-
def list_features(
|
|
375
|
-
self,
|
|
376
|
-
project: str,
|
|
377
|
-
name: Optional[str] = None,
|
|
378
|
-
tag: Optional[str] = None,
|
|
379
|
-
entities: Optional[list[str]] = None,
|
|
380
|
-
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
381
|
-
) -> mlrun.common.schemas.FeaturesOutput:
|
|
382
|
-
pass
|
|
383
|
-
|
|
384
379
|
def list_features_v2(
|
|
385
380
|
self,
|
|
386
381
|
project: str,
|
|
@@ -631,6 +626,9 @@ class NopDB(RunDBInterface):
|
|
|
631
626
|
tsdb_metrics: bool = False,
|
|
632
627
|
metric_list: Optional[list[str]] = None,
|
|
633
628
|
top_level: bool = False,
|
|
629
|
+
modes: Optional[
|
|
630
|
+
Union[mm_constants.EndpointMode, list[mm_constants.EndpointMode]]
|
|
631
|
+
] = None,
|
|
634
632
|
uids: Optional[list[str]] = None,
|
|
635
633
|
latest_only: bool = False,
|
|
636
634
|
) -> mlrun.common.schemas.ModelEndpointList:
|
|
@@ -677,6 +675,7 @@ class NopDB(RunDBInterface):
|
|
|
677
675
|
item_name: Optional[str] = None,
|
|
678
676
|
tag: Optional[str] = None,
|
|
679
677
|
version: Optional[str] = None,
|
|
678
|
+
item_type: mlrun.common.schemas.hub.HubSourceType = mlrun.common.schemas.hub.HubSourceType.functions,
|
|
680
679
|
):
|
|
681
680
|
pass
|
|
682
681
|
|
|
@@ -693,6 +692,7 @@ class NopDB(RunDBInterface):
|
|
|
693
692
|
version: Optional[str] = None,
|
|
694
693
|
tag: Optional[str] = None,
|
|
695
694
|
force_refresh: bool = False,
|
|
695
|
+
object_type: mlrun.common.schemas.hub.HubSourceType = mlrun.common.schemas.hub.HubSourceType.functions,
|
|
696
696
|
):
|
|
697
697
|
pass
|
|
698
698
|
|
|
@@ -704,6 +704,18 @@ class NopDB(RunDBInterface):
|
|
|
704
704
|
version: Optional[str] = None,
|
|
705
705
|
tag: str = "latest",
|
|
706
706
|
force_refresh: bool = False,
|
|
707
|
+
item_type: mlrun.common.schemas.hub.HubSourceType = mlrun.common.schemas.hub.HubSourceType.functions,
|
|
708
|
+
):
|
|
709
|
+
pass
|
|
710
|
+
|
|
711
|
+
def get_hub_asset(
|
|
712
|
+
self,
|
|
713
|
+
source_name: str,
|
|
714
|
+
item_name: str,
|
|
715
|
+
asset_name: str,
|
|
716
|
+
version: Optional[str] = None,
|
|
717
|
+
tag: str = "latest",
|
|
718
|
+
item_type: mlrun.common.schemas.hub.HubSourceType = mlrun.common.schemas.hub.HubSourceType.functions,
|
|
707
719
|
):
|
|
708
720
|
pass
|
|
709
721
|
|
|
@@ -890,6 +902,14 @@ class NopDB(RunDBInterface):
|
|
|
890
902
|
) -> None:
|
|
891
903
|
pass
|
|
892
904
|
|
|
905
|
+
def delete_model_monitoring_metrics(
|
|
906
|
+
self,
|
|
907
|
+
project: str,
|
|
908
|
+
application_name: str,
|
|
909
|
+
endpoint_ids: Optional[list[str]] = None,
|
|
910
|
+
) -> None:
|
|
911
|
+
pass
|
|
912
|
+
|
|
893
913
|
def get_monitoring_function_summaries(
|
|
894
914
|
self,
|
|
895
915
|
project: str,
|
|
@@ -980,3 +1000,11 @@ class NopDB(RunDBInterface):
|
|
|
980
1000
|
|
|
981
1001
|
def get_project_summary(self, project: str):
|
|
982
1002
|
pass
|
|
1003
|
+
|
|
1004
|
+
def get_drift_over_time(
|
|
1005
|
+
self,
|
|
1006
|
+
project: str,
|
|
1007
|
+
start: Optional[datetime.datetime] = None,
|
|
1008
|
+
end: Optional[datetime.datetime] = None,
|
|
1009
|
+
) -> mlrun.common.schemas.model_monitoring.ModelEndpointDriftValues:
|
|
1010
|
+
pass
|
mlrun/execution.py
CHANGED
|
@@ -94,6 +94,7 @@ class MLClientCtx:
|
|
|
94
94
|
self._state_thresholds = {}
|
|
95
95
|
self._retry_spec = {}
|
|
96
96
|
self._retry_count = None
|
|
97
|
+
self._retries = []
|
|
97
98
|
|
|
98
99
|
self._labels = {}
|
|
99
100
|
self._annotations = {}
|
|
@@ -468,6 +469,7 @@ class MLClientCtx:
|
|
|
468
469
|
for key, uri in status.get("artifact_uris", {}).items():
|
|
469
470
|
self._artifacts_manager.artifact_uris[key] = uri
|
|
470
471
|
self._retry_count = status.get("retry_count", self._retry_count)
|
|
472
|
+
self._retries = status.get("retries", self._retries)
|
|
471
473
|
# if run is a retry, the state needs to move to running
|
|
472
474
|
if include_status:
|
|
473
475
|
self._state = status.get("state", self._state)
|
|
@@ -911,11 +913,11 @@ class MLClientCtx:
|
|
|
911
913
|
def log_llm_prompt(
|
|
912
914
|
self,
|
|
913
915
|
key,
|
|
914
|
-
|
|
916
|
+
prompt_template: Optional[list[dict]] = None,
|
|
915
917
|
prompt_path: Optional[str] = None,
|
|
916
918
|
prompt_legend: Optional[dict] = None,
|
|
917
919
|
model_artifact: Union[ModelArtifact, str] = None,
|
|
918
|
-
|
|
920
|
+
invocation_config: Optional[dict] = None,
|
|
919
921
|
description: Optional[str] = None,
|
|
920
922
|
target_path: Optional[str] = None,
|
|
921
923
|
artifact_path: Optional[str] = None,
|
|
@@ -932,27 +934,75 @@ class MLClientCtx:
|
|
|
932
934
|
|
|
933
935
|
Examples::
|
|
934
936
|
|
|
935
|
-
# Log an inline prompt
|
|
937
|
+
# Log directly with an inline prompt template
|
|
936
938
|
context.log_llm_prompt(
|
|
937
|
-
key="
|
|
938
|
-
|
|
939
|
+
key="customer_support_prompt",
|
|
940
|
+
prompt_template=[
|
|
941
|
+
{
|
|
942
|
+
"role": "system",
|
|
943
|
+
"content": "You are a helpful customer support assistant.",
|
|
944
|
+
},
|
|
945
|
+
{
|
|
946
|
+
"role": "user",
|
|
947
|
+
"content": "The customer reports: {issue_description}",
|
|
948
|
+
},
|
|
949
|
+
],
|
|
950
|
+
prompt_legend={
|
|
951
|
+
"issue_description": {
|
|
952
|
+
"field": "user_issue",
|
|
953
|
+
"description": "Detailed description of the customer's issue",
|
|
954
|
+
},
|
|
955
|
+
"solution": {
|
|
956
|
+
"field": "proposed_solution",
|
|
957
|
+
"description": "Suggested fix for the customer's issue",
|
|
958
|
+
},
|
|
959
|
+
},
|
|
939
960
|
model_artifact=model,
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
tag="
|
|
961
|
+
invocation_config={"temperature": 0.5, "max_tokens": 200},
|
|
962
|
+
description="Prompt for handling customer support queries",
|
|
963
|
+
tag="support-v1",
|
|
964
|
+
labels={"domain": "support"},
|
|
965
|
+
)
|
|
966
|
+
|
|
967
|
+
# Log a prompt from file
|
|
968
|
+
context.log_llm_prompt(
|
|
969
|
+
key="qa_prompt",
|
|
970
|
+
prompt_path="prompts/template.json",
|
|
971
|
+
prompt_legend={
|
|
972
|
+
"question": {
|
|
973
|
+
"field": "user_question",
|
|
974
|
+
"description": "The actual question asked by the user",
|
|
975
|
+
}
|
|
976
|
+
},
|
|
977
|
+
model_artifact=model,
|
|
978
|
+
invocation_config={"temperature": 0.7, "max_tokens": 256},
|
|
979
|
+
description="Q&A prompt template with user-provided question",
|
|
980
|
+
tag="v2",
|
|
981
|
+
labels={"task": "qa", "stage": "experiment"},
|
|
943
982
|
)
|
|
944
983
|
|
|
945
984
|
:param key: Unique name of the artifact.
|
|
946
|
-
:param
|
|
947
|
-
|
|
985
|
+
:param prompt_template: Raw prompt list of dicts -
|
|
986
|
+
[{"role": "system", "content": "You are a {profession} advisor"},
|
|
987
|
+
"role": "user", "content": "I need your help with {profession}"]. only "role" and "content" keys allow in any
|
|
988
|
+
str format (upper/lower case), keys will be modified to lower case.
|
|
989
|
+
Cannot be used with `prompt_path`.
|
|
990
|
+
:param prompt_path: Path to a JSON file containing the prompt template.
|
|
991
|
+
Cannot be used together with `prompt_template`.
|
|
992
|
+
The file should define a list of dictionaries in the same format
|
|
993
|
+
supported by `prompt_template`.
|
|
948
994
|
:param prompt_legend: A dictionary where each key is a placeholder in the prompt (e.g., ``{user_name}``)
|
|
949
|
-
and the value is a description
|
|
995
|
+
and the value is a dictionary holding two keys, "field", "description". "field" points to the field in
|
|
996
|
+
the event where the value of the place-holder inside the event, if None or not exist will be replaced
|
|
997
|
+
with the place-holder name. "description" will point to explanation of what that placeholder represents.
|
|
950
998
|
Useful for documenting and clarifying dynamic parts of the prompt.
|
|
951
999
|
:param model_artifact: Reference to the parent model (either `ModelArtifact` or model URI string).
|
|
952
|
-
:param
|
|
953
|
-
:param description:
|
|
954
|
-
:param target_path:
|
|
955
|
-
:param artifact_path:
|
|
1000
|
+
:param invocation_config: Dictionary of generation parameters (e.g., temperature, max_tokens).
|
|
1001
|
+
:param description: Optional description of the prompt.
|
|
1002
|
+
:param target_path: Absolute target path (instead of using artifact_path + local_path)
|
|
1003
|
+
:param artifact_path: Target artifact path (when not using the default)
|
|
1004
|
+
To define a subpath under the default location use:
|
|
1005
|
+
`artifact_path=context.artifact_subpath('data')`
|
|
956
1006
|
:param tag: Tag/version to assign to the prompt artifact.
|
|
957
1007
|
:param labels: Labels to tag the artifact (e.g., list or dict of key-value pairs).
|
|
958
1008
|
:param upload: Whether to upload the artifact to the store (defaults to True).
|
|
@@ -961,19 +1011,19 @@ class MLClientCtx:
|
|
|
961
1011
|
:returns: The logged `LLMPromptArtifact` object.
|
|
962
1012
|
"""
|
|
963
1013
|
|
|
964
|
-
if not
|
|
1014
|
+
if not prompt_template and not prompt_path:
|
|
965
1015
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
966
|
-
"Either '
|
|
1016
|
+
"Either 'prompt_template' or 'prompt_path' must be provided"
|
|
967
1017
|
)
|
|
968
1018
|
|
|
969
1019
|
llm_prompt = LLMPromptArtifact(
|
|
970
1020
|
key=key,
|
|
971
1021
|
project=self.project or "",
|
|
972
|
-
|
|
1022
|
+
prompt_template=prompt_template,
|
|
973
1023
|
prompt_path=prompt_path,
|
|
974
1024
|
prompt_legend=prompt_legend,
|
|
975
1025
|
model_artifact=model_artifact,
|
|
976
|
-
|
|
1026
|
+
invocation_config=invocation_config,
|
|
977
1027
|
target_path=target_path,
|
|
978
1028
|
description=description,
|
|
979
1029
|
**kwargs,
|
|
@@ -1267,6 +1317,7 @@ class MLClientCtx:
|
|
|
1267
1317
|
"start_time": to_date_str(self._start_time),
|
|
1268
1318
|
"last_update": to_date_str(self._last_update),
|
|
1269
1319
|
"retry_count": self._retry_count,
|
|
1320
|
+
"retries": self._retries,
|
|
1270
1321
|
},
|
|
1271
1322
|
}
|
|
1272
1323
|
|
mlrun/hub/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright 2025 Iguazio
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from .module import get_hub_module, import_module
|