mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +14 -12
- mlrun/__main__.py +3 -3
- mlrun/alerts/alert.py +19 -12
- mlrun/artifacts/__init__.py +0 -2
- mlrun/artifacts/base.py +34 -11
- mlrun/artifacts/dataset.py +16 -16
- mlrun/artifacts/manager.py +13 -13
- mlrun/artifacts/model.py +66 -53
- mlrun/common/constants.py +6 -0
- mlrun/common/formatters/__init__.py +1 -0
- mlrun/common/formatters/feature_set.py +1 -0
- mlrun/common/formatters/function.py +1 -0
- mlrun/common/formatters/model_endpoint.py +30 -0
- mlrun/common/formatters/pipeline.py +1 -2
- mlrun/common/model_monitoring/__init__.py +0 -3
- mlrun/common/model_monitoring/helpers.py +1 -1
- mlrun/common/runtimes/constants.py +1 -2
- mlrun/common/schemas/__init__.py +4 -2
- mlrun/common/schemas/artifact.py +0 -6
- mlrun/common/schemas/common.py +50 -0
- mlrun/common/schemas/model_monitoring/__init__.py +8 -1
- mlrun/common/schemas/model_monitoring/constants.py +62 -12
- mlrun/common/schemas/model_monitoring/model_endpoint_v2.py +149 -0
- mlrun/common/schemas/model_monitoring/model_endpoints.py +21 -5
- mlrun/common/schemas/partition.py +122 -0
- mlrun/config.py +43 -15
- mlrun/data_types/__init__.py +0 -2
- mlrun/data_types/data_types.py +0 -1
- mlrun/data_types/infer.py +3 -1
- mlrun/data_types/spark.py +4 -4
- mlrun/data_types/to_pandas.py +2 -11
- mlrun/datastore/__init__.py +0 -2
- mlrun/datastore/alibaba_oss.py +4 -1
- mlrun/datastore/azure_blob.py +4 -1
- mlrun/datastore/base.py +12 -4
- mlrun/datastore/datastore.py +9 -3
- mlrun/datastore/datastore_profile.py +1 -1
- mlrun/datastore/dbfs_store.py +4 -1
- mlrun/datastore/filestore.py +4 -1
- mlrun/datastore/google_cloud_storage.py +4 -1
- mlrun/datastore/hdfs.py +4 -1
- mlrun/datastore/inmem.py +4 -1
- mlrun/datastore/redis.py +4 -1
- mlrun/datastore/s3.py +4 -1
- mlrun/datastore/sources.py +51 -49
- mlrun/datastore/store_resources.py +0 -2
- mlrun/datastore/targets.py +22 -23
- mlrun/datastore/utils.py +2 -2
- mlrun/datastore/v3io.py +4 -1
- mlrun/datastore/wasbfs/fs.py +13 -12
- mlrun/db/base.py +126 -62
- mlrun/db/factory.py +3 -0
- mlrun/db/httpdb.py +767 -231
- mlrun/db/nopdb.py +126 -57
- mlrun/errors.py +2 -2
- mlrun/execution.py +55 -29
- mlrun/feature_store/__init__.py +0 -2
- mlrun/feature_store/api.py +40 -40
- mlrun/feature_store/common.py +9 -9
- mlrun/feature_store/feature_set.py +20 -18
- mlrun/feature_store/feature_vector.py +27 -24
- mlrun/feature_store/retrieval/base.py +14 -9
- mlrun/feature_store/retrieval/job.py +2 -1
- mlrun/feature_store/steps.py +2 -2
- mlrun/features.py +30 -13
- mlrun/frameworks/__init__.py +1 -2
- mlrun/frameworks/_common/__init__.py +1 -2
- mlrun/frameworks/_common/artifacts_library.py +2 -2
- mlrun/frameworks/_common/mlrun_interface.py +10 -6
- mlrun/frameworks/_common/model_handler.py +29 -27
- mlrun/frameworks/_common/producer.py +3 -1
- mlrun/frameworks/_dl_common/__init__.py +1 -2
- mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
- mlrun/frameworks/_ml_common/__init__.py +1 -2
- mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
- mlrun/frameworks/_ml_common/model_handler.py +21 -21
- mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
- mlrun/frameworks/auto_mlrun/__init__.py +1 -2
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
- mlrun/frameworks/huggingface/__init__.py +1 -2
- mlrun/frameworks/huggingface/model_server.py +9 -9
- mlrun/frameworks/lgbm/__init__.py +47 -44
- mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
- mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
- mlrun/frameworks/lgbm/model_handler.py +15 -11
- mlrun/frameworks/lgbm/model_server.py +11 -7
- mlrun/frameworks/lgbm/utils.py +2 -2
- mlrun/frameworks/onnx/__init__.py +1 -2
- mlrun/frameworks/onnx/dataset.py +3 -3
- mlrun/frameworks/onnx/mlrun_interface.py +2 -2
- mlrun/frameworks/onnx/model_handler.py +7 -5
- mlrun/frameworks/onnx/model_server.py +8 -6
- mlrun/frameworks/parallel_coordinates.py +11 -11
- mlrun/frameworks/pytorch/__init__.py +22 -23
- mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
- mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
- mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
- mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
- mlrun/frameworks/pytorch/model_handler.py +21 -17
- mlrun/frameworks/pytorch/model_server.py +13 -9
- mlrun/frameworks/sklearn/__init__.py +19 -18
- mlrun/frameworks/sklearn/estimator.py +2 -2
- mlrun/frameworks/sklearn/metric.py +3 -3
- mlrun/frameworks/sklearn/metrics_library.py +8 -6
- mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
- mlrun/frameworks/sklearn/model_handler.py +4 -3
- mlrun/frameworks/tf_keras/__init__.py +11 -12
- mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
- mlrun/frameworks/tf_keras/model_handler.py +17 -13
- mlrun/frameworks/tf_keras/model_server.py +12 -8
- mlrun/frameworks/xgboost/__init__.py +19 -18
- mlrun/frameworks/xgboost/model_handler.py +13 -9
- mlrun/launcher/base.py +3 -4
- mlrun/launcher/local.py +1 -1
- mlrun/launcher/remote.py +1 -1
- mlrun/lists.py +4 -3
- mlrun/model.py +108 -44
- mlrun/model_monitoring/__init__.py +1 -2
- mlrun/model_monitoring/api.py +6 -6
- mlrun/model_monitoring/applications/_application_steps.py +13 -15
- mlrun/model_monitoring/applications/histogram_data_drift.py +41 -15
- mlrun/model_monitoring/applications/results.py +55 -3
- mlrun/model_monitoring/controller.py +185 -223
- mlrun/model_monitoring/db/_schedules.py +156 -0
- mlrun/model_monitoring/db/_stats.py +189 -0
- mlrun/model_monitoring/db/stores/__init__.py +1 -1
- mlrun/model_monitoring/db/stores/base/store.py +6 -65
- mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -25
- mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -97
- mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +2 -58
- mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -15
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +6 -257
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +9 -271
- mlrun/model_monitoring/db/tsdb/base.py +74 -22
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +66 -35
- mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +284 -51
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +35 -17
- mlrun/model_monitoring/helpers.py +97 -1
- mlrun/model_monitoring/model_endpoint.py +4 -2
- mlrun/model_monitoring/stream_processing.py +2 -2
- mlrun/model_monitoring/tracking_policy.py +10 -3
- mlrun/model_monitoring/writer.py +47 -26
- mlrun/package/__init__.py +3 -6
- mlrun/package/context_handler.py +1 -1
- mlrun/package/packager.py +12 -9
- mlrun/package/packagers/__init__.py +0 -2
- mlrun/package/packagers/default_packager.py +14 -11
- mlrun/package/packagers/numpy_packagers.py +16 -7
- mlrun/package/packagers/pandas_packagers.py +18 -18
- mlrun/package/packagers/python_standard_library_packagers.py +25 -11
- mlrun/package/packagers_manager.py +31 -14
- mlrun/package/utils/__init__.py +0 -3
- mlrun/package/utils/_pickler.py +6 -6
- mlrun/platforms/__init__.py +3 -3
- mlrun/platforms/iguazio.py +4 -1
- mlrun/projects/__init__.py +1 -6
- mlrun/projects/operations.py +27 -27
- mlrun/projects/pipelines.py +85 -215
- mlrun/projects/project.py +444 -158
- mlrun/run.py +9 -9
- mlrun/runtimes/__init__.py +1 -3
- mlrun/runtimes/base.py +13 -10
- mlrun/runtimes/daskjob.py +9 -9
- mlrun/runtimes/generators.py +2 -1
- mlrun/runtimes/kubejob.py +4 -5
- mlrun/runtimes/mpijob/__init__.py +0 -2
- mlrun/runtimes/mpijob/abstract.py +7 -6
- mlrun/runtimes/nuclio/api_gateway.py +7 -7
- mlrun/runtimes/nuclio/application/application.py +11 -11
- mlrun/runtimes/nuclio/function.py +14 -13
- mlrun/runtimes/nuclio/serving.py +9 -9
- mlrun/runtimes/pod.py +74 -29
- mlrun/runtimes/remotesparkjob.py +3 -2
- mlrun/runtimes/sparkjob/__init__.py +0 -2
- mlrun/runtimes/sparkjob/spark3job.py +21 -11
- mlrun/runtimes/utils.py +6 -5
- mlrun/serving/merger.py +6 -4
- mlrun/serving/remote.py +18 -17
- mlrun/serving/routers.py +27 -27
- mlrun/serving/server.py +1 -1
- mlrun/serving/states.py +76 -71
- mlrun/serving/utils.py +13 -2
- mlrun/serving/v1_serving.py +3 -2
- mlrun/serving/v2_serving.py +4 -4
- mlrun/track/__init__.py +1 -1
- mlrun/track/tracker.py +2 -2
- mlrun/track/trackers/mlflow_tracker.py +6 -5
- mlrun/utils/async_http.py +1 -1
- mlrun/utils/helpers.py +72 -28
- mlrun/utils/logger.py +104 -2
- mlrun/utils/notifications/notification/base.py +23 -4
- mlrun/utils/notifications/notification/console.py +1 -1
- mlrun/utils/notifications/notification/git.py +6 -6
- mlrun/utils/notifications/notification/ipython.py +5 -4
- mlrun/utils/notifications/notification/slack.py +1 -1
- mlrun/utils/notifications/notification/webhook.py +13 -17
- mlrun/utils/notifications/notification_pusher.py +23 -19
- mlrun/utils/regex.py +1 -1
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0rc1.dist-info}/METADATA +186 -186
- mlrun-1.8.0rc1.dist-info/RECORD +356 -0
- {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0rc1.dist-info}/WHEEL +1 -1
- mlrun-1.7.2rc3.dist-info/RECORD +0 -351
- {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0rc1.dist-info}/LICENSE +0 -0
- {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0rc1.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0rc1.dist-info}/top_level.txt +0 -0
mlrun/db/httpdb.py
CHANGED
|
@@ -25,11 +25,12 @@ from os import path, remove
|
|
|
25
25
|
from typing import Optional, Union
|
|
26
26
|
from urllib.parse import urlparse
|
|
27
27
|
|
|
28
|
+
import pydantic
|
|
28
29
|
import requests
|
|
29
30
|
import semver
|
|
30
|
-
from mlrun_pipelines.utils import compile_pipeline
|
|
31
31
|
|
|
32
32
|
import mlrun
|
|
33
|
+
import mlrun.common.constants
|
|
33
34
|
import mlrun.common.formatters
|
|
34
35
|
import mlrun.common.runtimes
|
|
35
36
|
import mlrun.common.schemas
|
|
@@ -43,6 +44,7 @@ import mlrun.utils
|
|
|
43
44
|
from mlrun.alerts.alert import AlertConfig
|
|
44
45
|
from mlrun.db.auth_utils import OAuthClientIDTokenProvider, StaticTokenProvider
|
|
45
46
|
from mlrun.errors import MLRunInvalidArgumentError, err_to_str
|
|
47
|
+
from mlrun_pipelines.utils import compile_pipeline
|
|
46
48
|
|
|
47
49
|
from ..artifacts import Artifact
|
|
48
50
|
from ..config import config
|
|
@@ -169,7 +171,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
169
171
|
return f"{cls}({self.base_url!r})"
|
|
170
172
|
|
|
171
173
|
@staticmethod
|
|
172
|
-
def get_api_path_prefix(version: str = None) -> str:
|
|
174
|
+
def get_api_path_prefix(version: Optional[str] = None) -> str:
|
|
173
175
|
"""
|
|
174
176
|
:param version: API version to use, None (the default) will mean to use the default value from mlrun.config,
|
|
175
177
|
for un-versioned api set an empty string.
|
|
@@ -182,7 +184,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
182
184
|
)
|
|
183
185
|
return api_version_path
|
|
184
186
|
|
|
185
|
-
def get_base_api_url(self, path: str, version: str = None) -> str:
|
|
187
|
+
def get_base_api_url(self, path: str, version: Optional[str] = None) -> str:
|
|
186
188
|
path_prefix = self.get_api_path_prefix(version)
|
|
187
189
|
url = f"{self.base_url}/{path_prefix}/{path}"
|
|
188
190
|
return url
|
|
@@ -310,9 +312,26 @@ class HTTPRunDB(RunDBInterface):
|
|
|
310
312
|
headers=None,
|
|
311
313
|
timeout=45,
|
|
312
314
|
version=None,
|
|
315
|
+
return_all=False,
|
|
313
316
|
) -> typing.Generator[requests.Response, None, None]:
|
|
314
317
|
"""
|
|
315
|
-
Calls the
|
|
318
|
+
Calls the API with pagination and yields each page of the response.
|
|
319
|
+
|
|
320
|
+
Depending on the `return_all` parameter:
|
|
321
|
+
- If `return_all` is `True`, fetches and yields all pages of results.
|
|
322
|
+
- If `return_all` is False, only a single page of results is fetched and yielded.
|
|
323
|
+
|
|
324
|
+
:param method: The HTTP method (GET, POST, etc.).
|
|
325
|
+
:param path: The API endpoint path.
|
|
326
|
+
:param error: Error message used for debugging if the request fails.
|
|
327
|
+
:param params: The parameters to pass for the API request, including filters.
|
|
328
|
+
:param body: The body of the request.
|
|
329
|
+
:param json: The JSON payload for the request.
|
|
330
|
+
:param headers: Custom headers for the request.
|
|
331
|
+
:param timeout: Timeout for the request.
|
|
332
|
+
:param version: API version, optional.
|
|
333
|
+
:param return_all: If `True`, fetches all pages and returns them in one shot. If `False`, returns only
|
|
334
|
+
the requested page or the next page.
|
|
316
335
|
"""
|
|
317
336
|
|
|
318
337
|
def _api_call(_params):
|
|
@@ -328,38 +347,50 @@ class HTTPRunDB(RunDBInterface):
|
|
|
328
347
|
version=version,
|
|
329
348
|
)
|
|
330
349
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
if
|
|
337
|
-
|
|
338
|
-
return
|
|
350
|
+
page_params = deepcopy(params) or {}
|
|
351
|
+
|
|
352
|
+
if page_params.get("page-token") is None and page_params.get("page") is None:
|
|
353
|
+
page_params["page"] = 1
|
|
354
|
+
|
|
355
|
+
if page_params.get("page-size") is None:
|
|
356
|
+
page_params["page-size"] = config.httpdb.pagination.default_page_size
|
|
339
357
|
|
|
340
|
-
|
|
341
|
-
params_with_page_token["page-token"] = page_token
|
|
342
|
-
while page_token:
|
|
343
|
-
yield response
|
|
344
|
-
try:
|
|
345
|
-
response = _api_call(params_with_page_token)
|
|
346
|
-
except mlrun.errors.MLRunNotFoundError:
|
|
347
|
-
# pagination token expired
|
|
348
|
-
break
|
|
358
|
+
response = _api_call(page_params)
|
|
349
359
|
|
|
360
|
+
# Yield only a single page of results
|
|
361
|
+
yield response
|
|
362
|
+
|
|
363
|
+
if return_all:
|
|
350
364
|
page_token = response.json().get("pagination", {}).get("page-token", None)
|
|
351
365
|
|
|
366
|
+
while page_token:
|
|
367
|
+
try:
|
|
368
|
+
# Use the page token to get the next page.
|
|
369
|
+
# No need to supply any other parameters as the token informs the pagination cache
|
|
370
|
+
# which parameters to use.
|
|
371
|
+
response = _api_call({"page-token": page_token})
|
|
372
|
+
except mlrun.errors.MLRunNotFoundError:
|
|
373
|
+
# pagination token expired, we've reached the last page
|
|
374
|
+
break
|
|
375
|
+
|
|
376
|
+
yield response
|
|
377
|
+
page_token = (
|
|
378
|
+
response.json().get("pagination", {}).get("page-token", None)
|
|
379
|
+
)
|
|
380
|
+
|
|
352
381
|
@staticmethod
|
|
353
382
|
def process_paginated_responses(
|
|
354
383
|
responses: typing.Generator[requests.Response, None, None], key: str = "data"
|
|
355
|
-
) -> list[typing.Any]:
|
|
384
|
+
) -> tuple[list[typing.Any], Optional[str]]:
|
|
356
385
|
"""
|
|
357
386
|
Processes the paginated responses and returns the combined data
|
|
358
387
|
"""
|
|
359
388
|
data = []
|
|
389
|
+
page_token = None
|
|
360
390
|
for response in responses:
|
|
391
|
+
page_token = response.json().get("pagination", {}).get("page-token", None)
|
|
361
392
|
data.extend(response.json().get(key, []))
|
|
362
|
-
return data
|
|
393
|
+
return data, page_token
|
|
363
394
|
|
|
364
395
|
def _init_session(self, retry_on_post: bool = False):
|
|
365
396
|
return mlrun.utils.HTTPSessionWithRetry(
|
|
@@ -768,7 +799,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
768
799
|
name: Optional[str] = None,
|
|
769
800
|
uid: Optional[Union[str, list[str]]] = None,
|
|
770
801
|
project: Optional[str] = None,
|
|
771
|
-
labels: Optional[Union[str, list[str]]] = None,
|
|
802
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
772
803
|
state: Optional[
|
|
773
804
|
mlrun.common.runtimes.constants.RunStates
|
|
774
805
|
] = None, # Backward compatibility
|
|
@@ -807,9 +838,13 @@ class HTTPRunDB(RunDBInterface):
|
|
|
807
838
|
:param name: Name of the run to retrieve.
|
|
808
839
|
:param uid: Unique ID of the run, or a list of run UIDs.
|
|
809
840
|
:param project: Project that the runs belongs to.
|
|
810
|
-
:param labels:
|
|
811
|
-
|
|
812
|
-
|
|
841
|
+
:param labels: Filter runs by label key-value pairs or key existence. This can be provided as:
|
|
842
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
843
|
+
or `{"label": None}` to check for key existence.
|
|
844
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
845
|
+
or just `"label"` for key existence.
|
|
846
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
847
|
+
the specified key-value pairs or key existence.
|
|
813
848
|
:param state: Deprecated - List only runs whose state is specified (will be removed in 1.9.0)
|
|
814
849
|
:param states: List only runs whose state is one of the provided states.
|
|
815
850
|
:param sort: Whether to sort the result according to their start time. Otherwise, results will be
|
|
@@ -832,81 +867,95 @@ class HTTPRunDB(RunDBInterface):
|
|
|
832
867
|
limit.
|
|
833
868
|
:param with_notifications: Return runs with notifications, and join them to the response. Default is `False`.
|
|
834
869
|
"""
|
|
870
|
+
runs, _ = self._list_runs(
|
|
871
|
+
name=name,
|
|
872
|
+
uid=uid,
|
|
873
|
+
project=project,
|
|
874
|
+
labels=labels,
|
|
875
|
+
state=state,
|
|
876
|
+
states=states,
|
|
877
|
+
sort=sort,
|
|
878
|
+
last=last,
|
|
879
|
+
iter=iter,
|
|
880
|
+
start_time_from=start_time_from,
|
|
881
|
+
start_time_to=start_time_to,
|
|
882
|
+
last_update_time_from=last_update_time_from,
|
|
883
|
+
last_update_time_to=last_update_time_to,
|
|
884
|
+
partition_by=partition_by,
|
|
885
|
+
rows_per_partition=rows_per_partition,
|
|
886
|
+
partition_sort_by=partition_sort_by,
|
|
887
|
+
partition_order=partition_order,
|
|
888
|
+
max_partitions=max_partitions,
|
|
889
|
+
with_notifications=with_notifications,
|
|
890
|
+
return_all=True,
|
|
891
|
+
)
|
|
892
|
+
return runs
|
|
835
893
|
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
warnings.warn(
|
|
846
|
-
"'last' is deprecated and will be removed in 1.8.0.",
|
|
847
|
-
FutureWarning,
|
|
848
|
-
)
|
|
894
|
+
def paginated_list_runs(
|
|
895
|
+
self,
|
|
896
|
+
*args,
|
|
897
|
+
page: Optional[int] = None,
|
|
898
|
+
page_size: Optional[int] = None,
|
|
899
|
+
page_token: Optional[str] = None,
|
|
900
|
+
**kwargs,
|
|
901
|
+
) -> tuple[RunList, Optional[str]]:
|
|
902
|
+
"""List runs with support for pagination and various filtering options.
|
|
849
903
|
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
"'state' is deprecated and will be removed in 1.9.0. Use 'states' instead.",
|
|
854
|
-
FutureWarning,
|
|
855
|
-
)
|
|
904
|
+
This method retrieves a paginated list of runs based on the specified filter parameters.
|
|
905
|
+
Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
|
|
906
|
+
will return a list of runs that match the filtering criteria provided.
|
|
856
907
|
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
and not uid
|
|
860
|
-
and not labels
|
|
861
|
-
and not state
|
|
862
|
-
and not states
|
|
863
|
-
and not last
|
|
864
|
-
and not start_time_from
|
|
865
|
-
and not start_time_to
|
|
866
|
-
and not last_update_time_from
|
|
867
|
-
and not last_update_time_to
|
|
868
|
-
and not partition_by
|
|
869
|
-
and not partition_sort_by
|
|
870
|
-
and not iter
|
|
871
|
-
):
|
|
872
|
-
# default to last week on no filter
|
|
873
|
-
start_time_from = datetime.now() - timedelta(days=7)
|
|
874
|
-
partition_by = mlrun.common.schemas.RunPartitionByField.project_and_name
|
|
875
|
-
partition_sort_by = mlrun.common.schemas.SortField.updated
|
|
908
|
+
For detailed information about the parameters, refer to the list_runs method:
|
|
909
|
+
See :py:func:`~list_runs` for more details.
|
|
876
910
|
|
|
877
|
-
|
|
878
|
-
"name": name,
|
|
879
|
-
"uid": uid,
|
|
880
|
-
"label": labels or [],
|
|
881
|
-
"state": mlrun.utils.helpers.as_list(state)
|
|
882
|
-
if state is not None
|
|
883
|
-
else states or None,
|
|
884
|
-
"sort": bool2str(sort),
|
|
885
|
-
"iter": bool2str(iter),
|
|
886
|
-
"start_time_from": datetime_to_iso(start_time_from),
|
|
887
|
-
"start_time_to": datetime_to_iso(start_time_to),
|
|
888
|
-
"last_update_time_from": datetime_to_iso(last_update_time_from),
|
|
889
|
-
"last_update_time_to": datetime_to_iso(last_update_time_to),
|
|
890
|
-
"with-notifications": with_notifications,
|
|
891
|
-
}
|
|
911
|
+
Examples::
|
|
892
912
|
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
913
|
+
# Fetch first page of runs with page size of 5
|
|
914
|
+
runs, token = db.paginated_list_runs(project="my-project", page_size=5)
|
|
915
|
+
# Fetch next page using the pagination token from the previous response
|
|
916
|
+
runs, token = db.paginated_list_runs(project="my-project", page_token=token)
|
|
917
|
+
# Fetch runs for a specific page (e.g., page 3)
|
|
918
|
+
runs, token = db.paginated_list_runs(project="my-project", page=3, page_size=5)
|
|
919
|
+
|
|
920
|
+
# Automatically iterate over all pages without explicitly specifying the page number
|
|
921
|
+
runs = []
|
|
922
|
+
token = None
|
|
923
|
+
while True:
|
|
924
|
+
page_runs, token = db.paginated_list_runs(
|
|
925
|
+
project="my-project", page_token=token, page_size=5
|
|
902
926
|
)
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
927
|
+
runs.extend(page_runs)
|
|
928
|
+
|
|
929
|
+
# If token is None and page_runs is empty, we've reached the end (no more runs).
|
|
930
|
+
# If token is None and page_runs is not empty, we've fetched the last page of runs.
|
|
931
|
+
if not token:
|
|
932
|
+
break
|
|
933
|
+
print(f"Total runs retrieved: {len(runs)}")
|
|
934
|
+
|
|
935
|
+
:param page: The page number to retrieve. If not provided, the next page will be retrieved.
|
|
936
|
+
:param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
|
|
937
|
+
:param page_token: A pagination token used to retrieve the next page of results. Should not be provided
|
|
938
|
+
for the first request.
|
|
939
|
+
|
|
940
|
+
:returns: A tuple containing the list of runs and an optional `page_token` for pagination.
|
|
941
|
+
"""
|
|
942
|
+
return self._list_runs(
|
|
943
|
+
*args,
|
|
944
|
+
page=page,
|
|
945
|
+
page_size=page_size,
|
|
946
|
+
page_token=page_token,
|
|
947
|
+
return_all=False,
|
|
948
|
+
**kwargs,
|
|
949
|
+
)
|
|
908
950
|
|
|
909
|
-
def del_runs(
|
|
951
|
+
def del_runs(
|
|
952
|
+
self,
|
|
953
|
+
name: Optional[str] = None,
|
|
954
|
+
project: Optional[str] = None,
|
|
955
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
956
|
+
state: Optional[mlrun.common.runtimes.constants.RunStates] = None,
|
|
957
|
+
days_ago: int = 0,
|
|
958
|
+
):
|
|
910
959
|
"""Delete a group of runs identified by the parameters of the function.
|
|
911
960
|
|
|
912
961
|
Example::
|
|
@@ -915,16 +964,23 @@ class HTTPRunDB(RunDBInterface):
|
|
|
915
964
|
|
|
916
965
|
:param name: Name of the task which the runs belong to.
|
|
917
966
|
:param project: Project to which the runs belong.
|
|
918
|
-
:param labels: Filter runs
|
|
967
|
+
:param labels: Filter runs by label key-value pairs or key existence. This can be provided as:
|
|
968
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
969
|
+
or `{"label": None}` to check for key existence.
|
|
970
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
971
|
+
or just `"label"` for key existence.
|
|
972
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
973
|
+
the specified key-value pairs or key existence.
|
|
919
974
|
:param state: Filter only runs which are in this state.
|
|
920
975
|
:param days_ago: Filter runs whose start time is newer than this parameter.
|
|
921
976
|
"""
|
|
922
977
|
|
|
923
978
|
project = project or config.default_project
|
|
979
|
+
labels = self._parse_labels(labels)
|
|
924
980
|
params = {
|
|
925
981
|
"name": name,
|
|
926
982
|
"project": project,
|
|
927
|
-
"label": labels
|
|
983
|
+
"label": labels,
|
|
928
984
|
"state": state,
|
|
929
985
|
"days_ago": str(days_ago),
|
|
930
986
|
}
|
|
@@ -1028,7 +1084,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1028
1084
|
deletion_strategy: mlrun.common.schemas.artifact.ArtifactsDeletionStrategies = (
|
|
1029
1085
|
mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
|
|
1030
1086
|
),
|
|
1031
|
-
secrets: dict = None,
|
|
1087
|
+
secrets: Optional[dict] = None,
|
|
1032
1088
|
iter=None,
|
|
1033
1089
|
):
|
|
1034
1090
|
"""Delete an artifact.
|
|
@@ -1063,29 +1119,29 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1063
1119
|
|
|
1064
1120
|
def list_artifacts(
|
|
1065
1121
|
self,
|
|
1066
|
-
name=None,
|
|
1067
|
-
project=None,
|
|
1068
|
-
tag=None,
|
|
1069
|
-
labels: Optional[Union[dict[str, str], list[str]]] = None,
|
|
1122
|
+
name: Optional[str] = None,
|
|
1123
|
+
project: Optional[str] = None,
|
|
1124
|
+
tag: Optional[str] = None,
|
|
1125
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
1070
1126
|
since: Optional[datetime] = None,
|
|
1071
1127
|
until: Optional[datetime] = None,
|
|
1072
|
-
iter: int = None,
|
|
1128
|
+
iter: Optional[int] = None,
|
|
1073
1129
|
best_iteration: bool = False,
|
|
1074
|
-
kind: str = None,
|
|
1130
|
+
kind: Optional[str] = None,
|
|
1075
1131
|
category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
|
|
1076
|
-
tree: str = None,
|
|
1077
|
-
producer_uri: str = None,
|
|
1132
|
+
tree: Optional[str] = None,
|
|
1133
|
+
producer_uri: Optional[str] = None,
|
|
1078
1134
|
format_: Optional[
|
|
1079
1135
|
mlrun.common.formatters.ArtifactFormat
|
|
1080
1136
|
] = mlrun.common.formatters.ArtifactFormat.full,
|
|
1081
|
-
limit: int = None,
|
|
1137
|
+
limit: Optional[int] = None,
|
|
1082
1138
|
) -> ArtifactList:
|
|
1083
1139
|
"""List artifacts filtered by various parameters.
|
|
1084
1140
|
|
|
1085
1141
|
Examples::
|
|
1086
1142
|
|
|
1087
1143
|
# Show latest version of all artifacts in project
|
|
1088
|
-
latest_artifacts = db.list_artifacts(
|
|
1144
|
+
latest_artifacts = db.list_artifacts(tag="latest", project="iris")
|
|
1089
1145
|
# check different artifact versions for a specific artifact
|
|
1090
1146
|
result_versions = db.list_artifacts("results", tag="*", project="iris")
|
|
1091
1147
|
# Show artifacts with label filters - both uploaded and of binary type
|
|
@@ -1098,8 +1154,13 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1098
1154
|
``my_Name_1`` or ``surname``.
|
|
1099
1155
|
:param project: Project name.
|
|
1100
1156
|
:param tag: Return artifacts assigned this tag.
|
|
1101
|
-
:param labels:
|
|
1102
|
-
|
|
1157
|
+
:param labels: Filter artifacts by label key-value pairs or key existence. This can be provided as:
|
|
1158
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
1159
|
+
or `{"label": None}` to check for key existence.
|
|
1160
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
1161
|
+
or just `"label"` for key existence.
|
|
1162
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
1163
|
+
the specified key-value pairs or key existence.
|
|
1103
1164
|
:param since: Return artifacts updated after this date (as datetime object).
|
|
1104
1165
|
:param until: Return artifacts updated before this date (as datetime object).
|
|
1105
1166
|
:param iter: Return artifacts from a specific iteration (where ``iter=0`` means the root iteration). If
|
|
@@ -1117,36 +1178,97 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1117
1178
|
:param limit: Maximum number of artifacts to return.
|
|
1118
1179
|
"""
|
|
1119
1180
|
|
|
1120
|
-
|
|
1181
|
+
artifacts, _ = self._list_artifacts(
|
|
1182
|
+
name=name,
|
|
1183
|
+
project=project,
|
|
1184
|
+
tag=tag,
|
|
1185
|
+
labels=labels,
|
|
1186
|
+
since=since,
|
|
1187
|
+
until=until,
|
|
1188
|
+
iter=iter,
|
|
1189
|
+
best_iteration=best_iteration,
|
|
1190
|
+
kind=kind,
|
|
1191
|
+
category=category,
|
|
1192
|
+
tree=tree,
|
|
1193
|
+
producer_uri=producer_uri,
|
|
1194
|
+
format_=format_,
|
|
1195
|
+
limit=limit,
|
|
1196
|
+
return_all=True,
|
|
1197
|
+
)
|
|
1198
|
+
return artifacts
|
|
1121
1199
|
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1200
|
+
def paginated_list_artifacts(
|
|
1201
|
+
self,
|
|
1202
|
+
*args,
|
|
1203
|
+
page: Optional[int] = None,
|
|
1204
|
+
page_size: Optional[int] = None,
|
|
1205
|
+
page_token: Optional[str] = None,
|
|
1206
|
+
**kwargs,
|
|
1207
|
+
) -> tuple[ArtifactList, Optional[str]]:
|
|
1208
|
+
"""List artifacts with support for pagination and various filtering options.
|
|
1125
1209
|
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1210
|
+
This method retrieves a paginated list of artifacts based on the specified filter parameters.
|
|
1211
|
+
Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
|
|
1212
|
+
will return a list of artifacts that match the filtering criteria provided.
|
|
1213
|
+
|
|
1214
|
+
For detailed information about the parameters, refer to the list_artifacts method:
|
|
1215
|
+
See :py:func:`~list_artifacts` for more details.
|
|
1216
|
+
|
|
1217
|
+
Examples::
|
|
1218
|
+
|
|
1219
|
+
# Fetch first page of artifacts with page size of 5
|
|
1220
|
+
artifacts, token = db.paginated_list_artifacts(
|
|
1221
|
+
project="my-project", page_size=5
|
|
1222
|
+
)
|
|
1223
|
+
# Fetch next page using the pagination token from the previous response
|
|
1224
|
+
artifacts, token = db.paginated_list_artifacts(
|
|
1225
|
+
project="my-project", page_token=token
|
|
1226
|
+
)
|
|
1227
|
+
# Fetch artifacts for a specific page (e.g., page 3)
|
|
1228
|
+
artifacts, token = db.paginated_list_artifacts(
|
|
1229
|
+
project="my-project", page=3, page_size=5
|
|
1230
|
+
)
|
|
1231
|
+
|
|
1232
|
+
# Automatically iterate over all pages without explicitly specifying the page number
|
|
1233
|
+
artifacts = []
|
|
1234
|
+
token = None
|
|
1235
|
+
while True:
|
|
1236
|
+
page_artifacts, token = db.paginated_list_artifacts(
|
|
1237
|
+
project="my-project", page_token=token, page_size=5
|
|
1238
|
+
)
|
|
1239
|
+
artifacts.extend(page_artifacts)
|
|
1240
|
+
|
|
1241
|
+
# If token is None and page_artifacts is empty, we've reached the end (no more artifacts).
|
|
1242
|
+
# If token is None and page_artifacts is not empty, we've fetched the last page of artifacts.
|
|
1243
|
+
if not token:
|
|
1244
|
+
break
|
|
1245
|
+
print(f"Total artifacts retrieved: {len(artifacts)}")
|
|
1246
|
+
|
|
1247
|
+
:param page: The page number to retrieve. If not provided, the next page will be retrieved.
|
|
1248
|
+
:param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
|
|
1249
|
+
:param page_token: A pagination token used to retrieve the next page of results. Should not be provided
|
|
1250
|
+
for the first request.
|
|
1251
|
+
|
|
1252
|
+
:returns: A tuple containing the list of artifacts and an optional `page_token` for pagination.
|
|
1253
|
+
"""
|
|
1254
|
+
|
|
1255
|
+
return self._list_artifacts(
|
|
1256
|
+
*args,
|
|
1257
|
+
page=page,
|
|
1258
|
+
page_size=page_size,
|
|
1259
|
+
page_token=page_token,
|
|
1260
|
+
return_all=False,
|
|
1261
|
+
**kwargs,
|
|
1262
|
+
)
|
|
1147
1263
|
|
|
1148
1264
|
def del_artifacts(
|
|
1149
|
-
self,
|
|
1265
|
+
self,
|
|
1266
|
+
name: Optional[str] = None,
|
|
1267
|
+
project: Optional[str] = None,
|
|
1268
|
+
tag: Optional[str] = None,
|
|
1269
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
1270
|
+
days_ago=0,
|
|
1271
|
+
tree: Optional[str] = None,
|
|
1150
1272
|
):
|
|
1151
1273
|
"""Delete artifacts referenced by the parameters.
|
|
1152
1274
|
|
|
@@ -1154,15 +1276,24 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1154
1276
|
:py:func:`~list_artifacts` for more details.
|
|
1155
1277
|
:param project: Project that artifacts belong to.
|
|
1156
1278
|
:param tag: Choose artifacts who are assigned this tag.
|
|
1157
|
-
:param labels:
|
|
1279
|
+
:param labels: Filter artifacts by label key-value pairs or key existence. This can be provided as:
|
|
1280
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
1281
|
+
or `{"label": None}` to check for key existence.
|
|
1282
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
1283
|
+
or just `"label"` for key existence.
|
|
1284
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
1285
|
+
the specified key-value pairs or key existence.
|
|
1158
1286
|
:param days_ago: This parameter is deprecated and not used.
|
|
1287
|
+
:param tree: Delete artifacts filtered by tree.
|
|
1159
1288
|
"""
|
|
1160
1289
|
project = project or config.default_project
|
|
1290
|
+
labels = self._parse_labels(labels)
|
|
1291
|
+
|
|
1161
1292
|
params = {
|
|
1162
1293
|
"name": name,
|
|
1163
1294
|
"tag": tag,
|
|
1164
1295
|
"tree": tree,
|
|
1165
|
-
"label": labels
|
|
1296
|
+
"label": labels,
|
|
1166
1297
|
"days_ago": str(days_ago),
|
|
1167
1298
|
}
|
|
1168
1299
|
error = "del artifacts"
|
|
@@ -1254,30 +1385,104 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1254
1385
|
)
|
|
1255
1386
|
|
|
1256
1387
|
def list_functions(
|
|
1257
|
-
self,
|
|
1388
|
+
self,
|
|
1389
|
+
name: Optional[str] = None,
|
|
1390
|
+
project: Optional[str] = None,
|
|
1391
|
+
tag: Optional[str] = None,
|
|
1392
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
1393
|
+
since: Optional[datetime] = None,
|
|
1394
|
+
until: Optional[datetime] = None,
|
|
1258
1395
|
):
|
|
1259
1396
|
"""Retrieve a list of functions, filtered by specific criteria.
|
|
1260
1397
|
|
|
1261
1398
|
:param name: Return only functions with a specific name.
|
|
1262
1399
|
:param project: Return functions belonging to this project. If not specified, the default project is used.
|
|
1263
1400
|
:param tag: Return function versions with specific tags. To return only tagged functions, set tag to ``"*"``.
|
|
1264
|
-
:param labels:
|
|
1401
|
+
:param labels: Filter functions by label key-value pairs or key existence. This can be provided as:
|
|
1402
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
1403
|
+
or `{"label": None}` to check for key existence.
|
|
1404
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
1405
|
+
or just `"label"` for key existence.
|
|
1406
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
1407
|
+
the specified key-value pairs or key existence.
|
|
1265
1408
|
:param since: Return functions updated after this date (as datetime object).
|
|
1266
1409
|
:param until: Return functions updated before this date (as datetime object).
|
|
1267
1410
|
:returns: List of function objects (as dictionary).
|
|
1268
1411
|
"""
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1412
|
+
functions, _ = self._list_functions(
|
|
1413
|
+
name=name,
|
|
1414
|
+
project=project,
|
|
1415
|
+
tag=tag,
|
|
1416
|
+
labels=labels,
|
|
1417
|
+
since=since,
|
|
1418
|
+
until=until,
|
|
1419
|
+
return_all=True,
|
|
1420
|
+
)
|
|
1421
|
+
return functions
|
|
1422
|
+
|
|
1423
|
+
def paginated_list_functions(
|
|
1424
|
+
self,
|
|
1425
|
+
*args,
|
|
1426
|
+
page: Optional[int] = None,
|
|
1427
|
+
page_size: Optional[int] = None,
|
|
1428
|
+
page_token: Optional[str] = None,
|
|
1429
|
+
**kwargs,
|
|
1430
|
+
) -> tuple[list[dict], Optional[str]]:
|
|
1431
|
+
"""List functions with support for pagination and various filtering options.
|
|
1432
|
+
|
|
1433
|
+
This method retrieves a paginated list of functions based on the specified filter parameters.
|
|
1434
|
+
Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
|
|
1435
|
+
will return a list of functions that match the filtering criteria provided.
|
|
1436
|
+
|
|
1437
|
+
For detailed information about the parameters, refer to the list_functions method:
|
|
1438
|
+
See :py:func:`~list_functions` for more details.
|
|
1439
|
+
|
|
1440
|
+
Examples::
|
|
1441
|
+
|
|
1442
|
+
# Fetch first page of functions with page size of 5
|
|
1443
|
+
functions, token = db.paginated_list_functions(
|
|
1444
|
+
project="my-project", page_size=5
|
|
1445
|
+
)
|
|
1446
|
+
# Fetch next page using the pagination token from the previous response
|
|
1447
|
+
functions, token = db.paginated_list_functions(
|
|
1448
|
+
project="my-project", page_token=token
|
|
1449
|
+
)
|
|
1450
|
+
# Fetch functions for a specific page (e.g., page 3)
|
|
1451
|
+
functions, token = db.paginated_list_functions(
|
|
1452
|
+
project="my-project", page=3, page_size=5
|
|
1453
|
+
)
|
|
1454
|
+
|
|
1455
|
+
# Automatically iterate over all pages without explicitly specifying the page number
|
|
1456
|
+
functions = []
|
|
1457
|
+
token = None
|
|
1458
|
+
while True:
|
|
1459
|
+
page_functions, token = db.paginated_list_functions(
|
|
1460
|
+
project="my-project", page_token=token, page_size=5
|
|
1461
|
+
)
|
|
1462
|
+
functions.extend(page_functions)
|
|
1463
|
+
|
|
1464
|
+
# If token is None and page_functions is empty, we've reached the end (no more functions).
|
|
1465
|
+
# If token is None and page_functions is not empty, we've fetched the last page of functions.
|
|
1466
|
+
if not token:
|
|
1467
|
+
break
|
|
1468
|
+
print(f"Total functions retrieved: {len(functions)}")
|
|
1469
|
+
|
|
1470
|
+
:param page: The page number to retrieve. If not provided, the next page will be retrieved.
|
|
1471
|
+
:param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
|
|
1472
|
+
:param page_token: A pagination token used to retrieve the next page of results. Should not be provided
|
|
1473
|
+
for the first request.
|
|
1474
|
+
|
|
1475
|
+
:returns: A tuple containing the list of functions objects (as dictionary) and an optional
|
|
1476
|
+
`page_token` for pagination.
|
|
1477
|
+
"""
|
|
1478
|
+
return self._list_functions(
|
|
1479
|
+
*args,
|
|
1480
|
+
page=page,
|
|
1481
|
+
page_size=page_size,
|
|
1482
|
+
page_token=page_token,
|
|
1483
|
+
return_all=False,
|
|
1484
|
+
**kwargs,
|
|
1485
|
+
)
|
|
1281
1486
|
|
|
1282
1487
|
def list_runtime_resources(
|
|
1283
1488
|
self,
|
|
@@ -1352,7 +1557,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1352
1557
|
kind: Optional[str] = None,
|
|
1353
1558
|
object_id: Optional[str] = None,
|
|
1354
1559
|
force: bool = False,
|
|
1355
|
-
grace_period: int = None,
|
|
1560
|
+
grace_period: Optional[int] = None,
|
|
1356
1561
|
) -> mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput:
|
|
1357
1562
|
"""Delete all runtime resources which are in terminal state.
|
|
1358
1563
|
|
|
@@ -1464,7 +1669,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1464
1669
|
def list_schedules(
|
|
1465
1670
|
self,
|
|
1466
1671
|
project: str,
|
|
1467
|
-
name: str = None,
|
|
1672
|
+
name: Optional[str] = None,
|
|
1468
1673
|
kind: mlrun.common.schemas.ScheduleKinds = None,
|
|
1469
1674
|
include_last_run: bool = False,
|
|
1470
1675
|
) -> mlrun.common.schemas.SchedulesOutput:
|
|
@@ -1636,6 +1841,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1636
1841
|
logs: bool = True,
|
|
1637
1842
|
last_log_timestamp: float = 0.0,
|
|
1638
1843
|
verbose: bool = False,
|
|
1844
|
+
events_offset: int = 0,
|
|
1639
1845
|
):
|
|
1640
1846
|
"""Retrieve the status of a build operation currently in progress.
|
|
1641
1847
|
|
|
@@ -1645,6 +1851,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1645
1851
|
:param last_log_timestamp: Last timestamp of logs that were already retrieved. Function will return only logs
|
|
1646
1852
|
later than this parameter.
|
|
1647
1853
|
:param verbose: Add verbose logs into the output.
|
|
1854
|
+
:param events_offset: Offset into the build events to retrieve events from.
|
|
1648
1855
|
|
|
1649
1856
|
:returns: The following parameters:
|
|
1650
1857
|
|
|
@@ -1661,6 +1868,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1661
1868
|
"tag": func.metadata.tag,
|
|
1662
1869
|
"logs": bool2str(logs),
|
|
1663
1870
|
"offset": str(offset),
|
|
1871
|
+
"events_offset": str(events_offset),
|
|
1664
1872
|
"last_log_timestamp": str(last_log_timestamp),
|
|
1665
1873
|
"verbose": bool2str(verbose),
|
|
1666
1874
|
}
|
|
@@ -1673,6 +1881,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1673
1881
|
logger.warning(f"failed resp, {resp.text}")
|
|
1674
1882
|
raise RunDBError("bad function build response")
|
|
1675
1883
|
|
|
1884
|
+
deploy_status_text_kind = mlrun.common.constants.DeployStatusTextKind.logs
|
|
1676
1885
|
if resp.headers:
|
|
1677
1886
|
func.status.state = resp.headers.get("x-mlrun-function-status", "")
|
|
1678
1887
|
last_log_timestamp = float(
|
|
@@ -1691,13 +1900,20 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1691
1900
|
if function_image:
|
|
1692
1901
|
func.spec.image = function_image
|
|
1693
1902
|
|
|
1903
|
+
deploy_status_text_kind = resp.headers.get(
|
|
1904
|
+
"deploy_status_text_kind",
|
|
1905
|
+
mlrun.common.constants.DeployStatusTextKind.logs,
|
|
1906
|
+
)
|
|
1907
|
+
|
|
1694
1908
|
text = ""
|
|
1695
1909
|
if resp.content:
|
|
1696
1910
|
text = resp.content.decode()
|
|
1697
|
-
return text, last_log_timestamp
|
|
1911
|
+
return text, last_log_timestamp, deploy_status_text_kind
|
|
1698
1912
|
|
|
1699
1913
|
def start_function(
|
|
1700
|
-
self,
|
|
1914
|
+
self,
|
|
1915
|
+
func_url: Optional[str] = None,
|
|
1916
|
+
function: "mlrun.runtimes.BaseRuntime" = None,
|
|
1701
1917
|
) -> mlrun.common.schemas.BackgroundTask:
|
|
1702
1918
|
"""Execute a function remotely, Used for ``dask`` functions.
|
|
1703
1919
|
|
|
@@ -1939,14 +2155,14 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1939
2155
|
def list_pipelines(
|
|
1940
2156
|
self,
|
|
1941
2157
|
project: str,
|
|
1942
|
-
namespace: str = None,
|
|
2158
|
+
namespace: Optional[str] = None,
|
|
1943
2159
|
sort_by: str = "",
|
|
1944
2160
|
page_token: str = "",
|
|
1945
2161
|
filter_: str = "",
|
|
1946
2162
|
format_: Union[
|
|
1947
2163
|
str, mlrun.common.formatters.PipelineFormat
|
|
1948
2164
|
] = mlrun.common.formatters.PipelineFormat.metadata_only,
|
|
1949
|
-
page_size: int = None,
|
|
2165
|
+
page_size: Optional[int] = None,
|
|
1950
2166
|
) -> mlrun.common.schemas.PipelinesOutput:
|
|
1951
2167
|
"""Retrieve a list of KFP pipelines. This function can be invoked to get all pipelines from all projects,
|
|
1952
2168
|
by specifying ``project=*``, in which case pagination can be used and the various sorting and pagination
|
|
@@ -1988,12 +2204,12 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1988
2204
|
def get_pipeline(
|
|
1989
2205
|
self,
|
|
1990
2206
|
run_id: str,
|
|
1991
|
-
namespace: str = None,
|
|
2207
|
+
namespace: Optional[str] = None,
|
|
1992
2208
|
timeout: int = 30,
|
|
1993
2209
|
format_: Union[
|
|
1994
2210
|
str, mlrun.common.formatters.PipelineFormat
|
|
1995
2211
|
] = mlrun.common.formatters.PipelineFormat.summary,
|
|
1996
|
-
project: str = None,
|
|
2212
|
+
project: Optional[str] = None,
|
|
1997
2213
|
):
|
|
1998
2214
|
"""Retrieve details of a specific pipeline using its run ID (as provided when the pipeline was executed)."""
|
|
1999
2215
|
|
|
@@ -2061,7 +2277,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2061
2277
|
return resp.json()
|
|
2062
2278
|
|
|
2063
2279
|
def get_feature_set(
|
|
2064
|
-
self,
|
|
2280
|
+
self,
|
|
2281
|
+
name: str,
|
|
2282
|
+
project: str = "",
|
|
2283
|
+
tag: Optional[str] = None,
|
|
2284
|
+
uid: Optional[str] = None,
|
|
2065
2285
|
) -> FeatureSet:
|
|
2066
2286
|
"""Retrieve a ~mlrun.feature_store.FeatureSet` object. If both ``tag`` and ``uid`` are not specified, then
|
|
2067
2287
|
the object tagged ``latest`` will be retrieved.
|
|
@@ -2081,11 +2301,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2081
2301
|
|
|
2082
2302
|
def list_features(
|
|
2083
2303
|
self,
|
|
2084
|
-
project: str,
|
|
2085
|
-
name: str = None,
|
|
2086
|
-
tag: str = None,
|
|
2087
|
-
entities: list[str] = None,
|
|
2088
|
-
labels: list[str] = None,
|
|
2304
|
+
project: Optional[str] = None,
|
|
2305
|
+
name: Optional[str] = None,
|
|
2306
|
+
tag: Optional[str] = None,
|
|
2307
|
+
entities: Optional[list[str]] = None,
|
|
2308
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
2089
2309
|
) -> list[dict]:
|
|
2090
2310
|
"""List feature-sets which contain specific features. This function may return multiple versions of the same
|
|
2091
2311
|
feature-set if a specific tag is not requested. Note that the various filters of this function actually
|
|
@@ -2096,18 +2316,25 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2096
2316
|
example, looking for ``feat`` will return features which are named ``MyFeature`` as well as ``defeat``.
|
|
2097
2317
|
:param tag: Return feature-sets which contain the features looked for, and are tagged with the specific tag.
|
|
2098
2318
|
:param entities: Return only feature-sets which contain an entity whose name is contained in this list.
|
|
2099
|
-
:param labels:
|
|
2319
|
+
:param labels: Filter feature-sets by label key-value pairs or key existence. This can be provided as:
|
|
2320
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
2321
|
+
or `{"label": None}` to check for key existence.
|
|
2322
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
2323
|
+
or just `"label"` for key existence.
|
|
2324
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
2325
|
+
the specified key-value pairs or key existence.
|
|
2100
2326
|
:returns: A list of mapping from feature to a digest of the feature-set, which contains the feature-set
|
|
2101
2327
|
meta-data. Multiple entries may be returned for any specific feature due to multiple tags or versions
|
|
2102
2328
|
of the feature-set.
|
|
2103
2329
|
"""
|
|
2104
2330
|
|
|
2105
2331
|
project = project or config.default_project
|
|
2332
|
+
labels = self._parse_labels(labels)
|
|
2106
2333
|
params = {
|
|
2107
2334
|
"name": name,
|
|
2108
2335
|
"tag": tag,
|
|
2109
2336
|
"entity": entities or [],
|
|
2110
|
-
"label": labels
|
|
2337
|
+
"label": labels,
|
|
2111
2338
|
}
|
|
2112
2339
|
|
|
2113
2340
|
path = f"projects/{project}/features"
|
|
@@ -2118,11 +2345,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2118
2345
|
|
|
2119
2346
|
def list_features_v2(
|
|
2120
2347
|
self,
|
|
2121
|
-
project: str,
|
|
2122
|
-
name: str = None,
|
|
2123
|
-
tag: str = None,
|
|
2124
|
-
entities: list[str] = None,
|
|
2125
|
-
labels: list[str] = None,
|
|
2348
|
+
project: Optional[str] = None,
|
|
2349
|
+
name: Optional[str] = None,
|
|
2350
|
+
tag: Optional[str] = None,
|
|
2351
|
+
entities: Optional[list[str]] = None,
|
|
2352
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
2126
2353
|
) -> dict[str, list[dict]]:
|
|
2127
2354
|
"""List feature-sets which contain specific features. This function may return multiple versions of the same
|
|
2128
2355
|
feature-set if a specific tag is not requested. Note that the various filters of this function actually
|
|
@@ -2133,16 +2360,23 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2133
2360
|
example, looking for ``feat`` will return features which are named ``MyFeature`` as well as ``defeat``.
|
|
2134
2361
|
:param tag: Return feature-sets which contain the features looked for, and are tagged with the specific tag.
|
|
2135
2362
|
:param entities: Return only feature-sets which contain an entity whose name is contained in this list.
|
|
2136
|
-
:param labels:
|
|
2363
|
+
:param labels: Filter feature-sets by label key-value pairs or key existence. This can be provided as:
|
|
2364
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
2365
|
+
or `{"label": None}` to check for key existence.
|
|
2366
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
2367
|
+
or just `"label"` for key existence.
|
|
2368
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
2369
|
+
the specified key-value pairs or key existence.
|
|
2137
2370
|
:returns: A list of features, and a list of their corresponding feature sets.
|
|
2138
2371
|
"""
|
|
2139
2372
|
|
|
2140
2373
|
project = project or config.default_project
|
|
2374
|
+
labels = self._parse_labels(labels)
|
|
2141
2375
|
params = {
|
|
2142
2376
|
"name": name,
|
|
2143
2377
|
"tag": tag,
|
|
2144
2378
|
"entity": entities or [],
|
|
2145
|
-
"label": labels
|
|
2379
|
+
"label": labels,
|
|
2146
2380
|
}
|
|
2147
2381
|
|
|
2148
2382
|
path = f"projects/{project}/features"
|
|
@@ -2153,21 +2387,34 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2153
2387
|
|
|
2154
2388
|
def list_entities(
|
|
2155
2389
|
self,
|
|
2156
|
-
project: str,
|
|
2157
|
-
name: str = None,
|
|
2158
|
-
tag: str = None,
|
|
2159
|
-
labels: list[str] = None,
|
|
2390
|
+
project: Optional[str] = None,
|
|
2391
|
+
name: Optional[str] = None,
|
|
2392
|
+
tag: Optional[str] = None,
|
|
2393
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
2160
2394
|
) -> list[dict]:
|
|
2161
2395
|
"""Retrieve a list of entities and their mapping to the containing feature-sets. This function is similar
|
|
2162
2396
|
to the :py:func:`~list_features` function, and uses the same logic. However, the entities are matched
|
|
2163
2397
|
against the name rather than the features.
|
|
2398
|
+
|
|
2399
|
+
:param project: The project containing the entities.
|
|
2400
|
+
:param name: The name of the entities to retrieve.
|
|
2401
|
+
:param tag: The tag of the specific entity version to retrieve.
|
|
2402
|
+
:param labels: Filter entities by label key-value pairs or key existence. This can be provided as:
|
|
2403
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
2404
|
+
or `{"label": None}` to check for key existence.
|
|
2405
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
2406
|
+
or just `"label"` for key existence.
|
|
2407
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
2408
|
+
the specified key-value pairs or key existence.
|
|
2409
|
+
:returns: A list of entities.
|
|
2164
2410
|
"""
|
|
2165
2411
|
|
|
2166
2412
|
project = project or config.default_project
|
|
2413
|
+
labels = self._parse_labels(labels)
|
|
2167
2414
|
params = {
|
|
2168
2415
|
"name": name,
|
|
2169
2416
|
"tag": tag,
|
|
2170
|
-
"label": labels
|
|
2417
|
+
"label": labels,
|
|
2171
2418
|
}
|
|
2172
2419
|
|
|
2173
2420
|
path = f"projects/{project}/entities"
|
|
@@ -2178,21 +2425,34 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2178
2425
|
|
|
2179
2426
|
def list_entities_v2(
|
|
2180
2427
|
self,
|
|
2181
|
-
project: str,
|
|
2182
|
-
name: str = None,
|
|
2183
|
-
tag: str = None,
|
|
2184
|
-
labels: list[str] = None,
|
|
2428
|
+
project: Optional[str] = None,
|
|
2429
|
+
name: Optional[str] = None,
|
|
2430
|
+
tag: Optional[str] = None,
|
|
2431
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
2185
2432
|
) -> dict[str, list[dict]]:
|
|
2186
2433
|
"""Retrieve a list of entities and their mapping to the containing feature-sets. This function is similar
|
|
2187
2434
|
to the :py:func:`~list_features_v2` function, and uses the same logic. However, the entities are matched
|
|
2188
2435
|
against the name rather than the features.
|
|
2436
|
+
|
|
2437
|
+
:param project: The project containing the entities.
|
|
2438
|
+
:param name: The name of the entities to retrieve.
|
|
2439
|
+
:param tag: The tag of the specific entity version to retrieve.
|
|
2440
|
+
:param labels: Filter entities by label key-value pairs or key existence. This can be provided as:
|
|
2441
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
2442
|
+
or `{"label": None}` to check for key existence.
|
|
2443
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
2444
|
+
or just `"label"` for key existence.
|
|
2445
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
2446
|
+
the specified key-value pairs or key existence.
|
|
2447
|
+
:returns: A list of entities.
|
|
2189
2448
|
"""
|
|
2190
2449
|
|
|
2191
2450
|
project = project or config.default_project
|
|
2451
|
+
labels = self._parse_labels(labels)
|
|
2192
2452
|
params = {
|
|
2193
2453
|
"name": name,
|
|
2194
2454
|
"tag": tag,
|
|
2195
|
-
"label": labels
|
|
2455
|
+
"label": labels,
|
|
2196
2456
|
}
|
|
2197
2457
|
|
|
2198
2458
|
path = f"projects/{project}/entities"
|
|
@@ -2222,13 +2482,13 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2222
2482
|
|
|
2223
2483
|
def list_feature_sets(
|
|
2224
2484
|
self,
|
|
2225
|
-
project: str =
|
|
2226
|
-
name: str = None,
|
|
2227
|
-
tag: str = None,
|
|
2228
|
-
state: str = None,
|
|
2229
|
-
entities: list[str] = None,
|
|
2230
|
-
features: list[str] = None,
|
|
2231
|
-
labels: list[str] = None,
|
|
2485
|
+
project: Optional[str] = None,
|
|
2486
|
+
name: Optional[str] = None,
|
|
2487
|
+
tag: Optional[str] = None,
|
|
2488
|
+
state: Optional[str] = None,
|
|
2489
|
+
entities: Optional[list[str]] = None,
|
|
2490
|
+
features: Optional[list[str]] = None,
|
|
2491
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
2232
2492
|
partition_by: Union[
|
|
2233
2493
|
mlrun.common.schemas.FeatureStorePartitionByField, str
|
|
2234
2494
|
] = None,
|
|
@@ -2249,7 +2509,13 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2249
2509
|
:param state: Match feature-sets with a specific state.
|
|
2250
2510
|
:param entities: Match feature-sets which contain entities whose name is in this list.
|
|
2251
2511
|
:param features: Match feature-sets which contain features whose name is in this list.
|
|
2252
|
-
:param labels:
|
|
2512
|
+
:param labels: Filter feature-sets by label key-value pairs or key existence. This can be provided as:
|
|
2513
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
2514
|
+
or `{"label": None}` to check for key existence.
|
|
2515
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
2516
|
+
or just `"label"` for key existence.
|
|
2517
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
2518
|
+
the specified key-value pairs or key existence.
|
|
2253
2519
|
:param partition_by: Field to group results by. Only allowed value is `name`. When `partition_by` is specified,
|
|
2254
2520
|
the `partition_sort_by` parameter must be provided as well.
|
|
2255
2521
|
:param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
|
|
@@ -2264,14 +2530,14 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2264
2530
|
"""
|
|
2265
2531
|
|
|
2266
2532
|
project = project or config.default_project
|
|
2267
|
-
|
|
2533
|
+
labels = self._parse_labels(labels)
|
|
2268
2534
|
params = {
|
|
2269
2535
|
"name": name,
|
|
2270
2536
|
"state": state,
|
|
2271
2537
|
"tag": tag,
|
|
2272
2538
|
"entity": entities or [],
|
|
2273
2539
|
"feature": features or [],
|
|
2274
|
-
"label": labels
|
|
2540
|
+
"label": labels,
|
|
2275
2541
|
"format": format_,
|
|
2276
2542
|
}
|
|
2277
2543
|
if partition_by:
|
|
@@ -2436,7 +2702,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2436
2702
|
return resp.json()
|
|
2437
2703
|
|
|
2438
2704
|
def get_feature_vector(
|
|
2439
|
-
self,
|
|
2705
|
+
self,
|
|
2706
|
+
name: str,
|
|
2707
|
+
project: str = "",
|
|
2708
|
+
tag: Optional[str] = None,
|
|
2709
|
+
uid: Optional[str] = None,
|
|
2440
2710
|
) -> FeatureVector:
|
|
2441
2711
|
"""Return a specific feature-vector referenced by its tag or uid. If none are provided, ``latest`` tag will
|
|
2442
2712
|
be used."""
|
|
@@ -2450,11 +2720,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2450
2720
|
|
|
2451
2721
|
def list_feature_vectors(
|
|
2452
2722
|
self,
|
|
2453
|
-
project: str =
|
|
2454
|
-
name: str = None,
|
|
2455
|
-
tag: str = None,
|
|
2456
|
-
state: str = None,
|
|
2457
|
-
labels: list[str] = None,
|
|
2723
|
+
project: Optional[str] = None,
|
|
2724
|
+
name: Optional[str] = None,
|
|
2725
|
+
tag: Optional[str] = None,
|
|
2726
|
+
state: Optional[str] = None,
|
|
2727
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
2458
2728
|
partition_by: Union[
|
|
2459
2729
|
mlrun.common.schemas.FeatureStorePartitionByField, str
|
|
2460
2730
|
] = None,
|
|
@@ -2470,7 +2740,13 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2470
2740
|
:param name: Name of feature-vector to match. This is a like query, and is case-insensitive.
|
|
2471
2741
|
:param tag: Match feature-vectors with specific tag.
|
|
2472
2742
|
:param state: Match feature-vectors with a specific state.
|
|
2473
|
-
:param labels:
|
|
2743
|
+
:param labels: Filter feature-vectors by label key-value pairs or key existence. This can be provided as:
|
|
2744
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
2745
|
+
or `{"label": None}` to check for key existence.
|
|
2746
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
2747
|
+
or just `"label"` for key existence.
|
|
2748
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
2749
|
+
the specified key-value pairs or key existence.
|
|
2474
2750
|
:param partition_by: Field to group results by. Only allowed value is `name`. When `partition_by` is specified,
|
|
2475
2751
|
the `partition_sort_by` parameter must be provided as well.
|
|
2476
2752
|
:param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
|
|
@@ -2482,12 +2758,12 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2482
2758
|
"""
|
|
2483
2759
|
|
|
2484
2760
|
project = project or config.default_project
|
|
2485
|
-
|
|
2761
|
+
labels = self._parse_labels(labels)
|
|
2486
2762
|
params = {
|
|
2487
2763
|
"name": name,
|
|
2488
2764
|
"state": state,
|
|
2489
2765
|
"tag": tag,
|
|
2490
|
-
"label": labels
|
|
2766
|
+
"label": labels,
|
|
2491
2767
|
}
|
|
2492
2768
|
if partition_by:
|
|
2493
2769
|
params.update(
|
|
@@ -2699,11 +2975,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2699
2975
|
|
|
2700
2976
|
def list_projects(
|
|
2701
2977
|
self,
|
|
2702
|
-
owner: str = None,
|
|
2978
|
+
owner: Optional[str] = None,
|
|
2703
2979
|
format_: Union[
|
|
2704
2980
|
str, mlrun.common.formatters.ProjectFormat
|
|
2705
2981
|
] = mlrun.common.formatters.ProjectFormat.name_only,
|
|
2706
|
-
labels: list[str] = None,
|
|
2982
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
2707
2983
|
state: Union[str, mlrun.common.schemas.ProjectState] = None,
|
|
2708
2984
|
) -> list[Union[mlrun.projects.MlrunProject, str]]:
|
|
2709
2985
|
"""Return a list of the existing projects, potentially filtered by specific criteria.
|
|
@@ -2715,15 +2991,22 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2715
2991
|
- ``minimal`` - Return minimal project objects (minimization happens in the BE).
|
|
2716
2992
|
- ``full`` - Return full project objects.
|
|
2717
2993
|
|
|
2718
|
-
:param labels: Filter by
|
|
2994
|
+
:param labels: Filter projects by label key-value pairs or key existence. This can be provided as:
|
|
2995
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
2996
|
+
or `{"label": None}` to check for key existence.
|
|
2997
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
2998
|
+
or just `"label"` for key existence.
|
|
2999
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
3000
|
+
the specified key-value pairs or key existence.
|
|
2719
3001
|
:param state: Filter by project's state. Can be either ``online`` or ``archived``.
|
|
2720
3002
|
"""
|
|
3003
|
+
labels = self._parse_labels(labels)
|
|
2721
3004
|
|
|
2722
3005
|
params = {
|
|
2723
3006
|
"owner": owner,
|
|
2724
3007
|
"state": state,
|
|
2725
3008
|
"format": format_,
|
|
2726
|
-
"label": labels
|
|
3009
|
+
"label": labels,
|
|
2727
3010
|
}
|
|
2728
3011
|
|
|
2729
3012
|
error_message = f"Failed listing projects, query: {params}"
|
|
@@ -2919,7 +3202,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2919
3202
|
provider: Union[
|
|
2920
3203
|
str, mlrun.common.schemas.SecretProviderName
|
|
2921
3204
|
] = mlrun.common.schemas.SecretProviderName.kubernetes,
|
|
2922
|
-
secrets: dict = None,
|
|
3205
|
+
secrets: Optional[dict] = None,
|
|
2923
3206
|
):
|
|
2924
3207
|
"""Create project-context secrets using either ``vault`` or ``kubernetes`` provider.
|
|
2925
3208
|
When using with Vault, this will create needed Vault structures for storing secrets in project-context, and
|
|
@@ -2963,11 +3246,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2963
3246
|
def list_project_secrets(
|
|
2964
3247
|
self,
|
|
2965
3248
|
project: str,
|
|
2966
|
-
token: str = None,
|
|
3249
|
+
token: Optional[str] = None,
|
|
2967
3250
|
provider: Union[
|
|
2968
3251
|
str, mlrun.common.schemas.SecretProviderName
|
|
2969
3252
|
] = mlrun.common.schemas.SecretProviderName.kubernetes,
|
|
2970
|
-
secrets: list[str] = None,
|
|
3253
|
+
secrets: Optional[list[str]] = None,
|
|
2971
3254
|
) -> mlrun.common.schemas.SecretsData:
|
|
2972
3255
|
"""Retrieve project-context secrets from Vault.
|
|
2973
3256
|
|
|
@@ -3010,7 +3293,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3010
3293
|
provider: Union[
|
|
3011
3294
|
str, mlrun.common.schemas.SecretProviderName
|
|
3012
3295
|
] = mlrun.common.schemas.SecretProviderName.kubernetes,
|
|
3013
|
-
token: str = None,
|
|
3296
|
+
token: Optional[str] = None,
|
|
3014
3297
|
) -> mlrun.common.schemas.SecretKeysData:
|
|
3015
3298
|
"""Retrieve project-context secret keys from Vault or Kubernetes.
|
|
3016
3299
|
|
|
@@ -3056,7 +3339,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3056
3339
|
provider: Union[
|
|
3057
3340
|
str, mlrun.common.schemas.SecretProviderName
|
|
3058
3341
|
] = mlrun.common.schemas.SecretProviderName.kubernetes,
|
|
3059
|
-
secrets: list[str] = None,
|
|
3342
|
+
secrets: Optional[list[str]] = None,
|
|
3060
3343
|
):
|
|
3061
3344
|
"""Delete project-context secrets from Kubernetes.
|
|
3062
3345
|
|
|
@@ -3082,7 +3365,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3082
3365
|
provider: Union[
|
|
3083
3366
|
str, mlrun.common.schemas.SecretProviderName
|
|
3084
3367
|
] = mlrun.common.schemas.SecretProviderName.vault,
|
|
3085
|
-
secrets: dict = None,
|
|
3368
|
+
secrets: Optional[dict] = None,
|
|
3086
3369
|
):
|
|
3087
3370
|
"""Create user-context secret in Vault. Please refer to :py:func:`create_project_secrets` for more details
|
|
3088
3371
|
and status of this functionality.
|
|
@@ -3213,7 +3496,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3213
3496
|
project: str,
|
|
3214
3497
|
model: Optional[str] = None,
|
|
3215
3498
|
function: Optional[str] = None,
|
|
3216
|
-
labels: list[str] = None,
|
|
3499
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
3217
3500
|
start: str = "now-1h",
|
|
3218
3501
|
end: str = "now",
|
|
3219
3502
|
metrics: Optional[list[str]] = None,
|
|
@@ -3237,8 +3520,13 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3237
3520
|
:param project: The name of the project
|
|
3238
3521
|
:param model: The name of the model to filter by
|
|
3239
3522
|
:param function: The name of the function to filter by
|
|
3240
|
-
:param labels:
|
|
3241
|
-
|
|
3523
|
+
:param labels: Filter model endpoints by label key-value pairs or key existence. This can be provided as:
|
|
3524
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
3525
|
+
or `{"label": None}` to check for key existence.
|
|
3526
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
3527
|
+
or just `"label"` for key existence.
|
|
3528
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
3529
|
+
the specified key-value pairs or key existence.
|
|
3242
3530
|
:param metrics: A list of metrics to return for each endpoint, read more in 'TimeMetric'
|
|
3243
3531
|
:param start: The start time of the metrics. Can be represented by a string containing an RFC 3339 time, a
|
|
3244
3532
|
Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
|
|
@@ -3251,9 +3539,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3251
3539
|
"""
|
|
3252
3540
|
|
|
3253
3541
|
path = f"projects/{project}/model-endpoints"
|
|
3254
|
-
|
|
3255
|
-
if labels and isinstance(labels, dict):
|
|
3256
|
-
labels = [f"{key}={value}" for key, value in labels.items()]
|
|
3542
|
+
labels = self._parse_labels(labels)
|
|
3257
3543
|
|
|
3258
3544
|
response = self.api_call(
|
|
3259
3545
|
method="GET",
|
|
@@ -3261,7 +3547,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3261
3547
|
params={
|
|
3262
3548
|
"model": model,
|
|
3263
3549
|
"function": function,
|
|
3264
|
-
"label": labels
|
|
3550
|
+
"label": labels,
|
|
3265
3551
|
"start": start,
|
|
3266
3552
|
"end": end,
|
|
3267
3553
|
"metric": metrics or [],
|
|
@@ -3441,7 +3727,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3441
3727
|
delete_stream_function: bool = False,
|
|
3442
3728
|
delete_histogram_data_drift_app: bool = True,
|
|
3443
3729
|
delete_user_applications: bool = False,
|
|
3444
|
-
user_application_list: list[str] = None,
|
|
3730
|
+
user_application_list: Optional[list[str]] = None,
|
|
3445
3731
|
) -> bool:
|
|
3446
3732
|
"""
|
|
3447
3733
|
Disable model monitoring application controller, writer, stream, histogram data drift application
|
|
@@ -3714,8 +4000,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3714
4000
|
def get_hub_catalog(
|
|
3715
4001
|
self,
|
|
3716
4002
|
source_name: str,
|
|
3717
|
-
version: str = None,
|
|
3718
|
-
tag: str = None,
|
|
4003
|
+
version: Optional[str] = None,
|
|
4004
|
+
tag: Optional[str] = None,
|
|
3719
4005
|
force_refresh: bool = False,
|
|
3720
4006
|
):
|
|
3721
4007
|
"""
|
|
@@ -3745,7 +4031,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3745
4031
|
self,
|
|
3746
4032
|
source_name: str,
|
|
3747
4033
|
item_name: str,
|
|
3748
|
-
version: str = None,
|
|
4034
|
+
version: Optional[str] = None,
|
|
3749
4035
|
tag: str = "latest",
|
|
3750
4036
|
force_refresh: bool = False,
|
|
3751
4037
|
):
|
|
@@ -3775,7 +4061,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3775
4061
|
source_name: str,
|
|
3776
4062
|
item_name: str,
|
|
3777
4063
|
asset_name: str,
|
|
3778
|
-
version: str = None,
|
|
4064
|
+
version: Optional[str] = None,
|
|
3779
4065
|
tag: str = "latest",
|
|
3780
4066
|
):
|
|
3781
4067
|
"""
|
|
@@ -3908,7 +4194,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3908
4194
|
self,
|
|
3909
4195
|
project: str,
|
|
3910
4196
|
run_uid: str,
|
|
3911
|
-
notifications: list[mlrun.model.Notification] = None,
|
|
4197
|
+
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
3912
4198
|
):
|
|
3913
4199
|
"""
|
|
3914
4200
|
Set notifications on a run. This will override any existing notifications on the run.
|
|
@@ -3934,7 +4220,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3934
4220
|
self,
|
|
3935
4221
|
project: str,
|
|
3936
4222
|
schedule_name: str,
|
|
3937
|
-
notifications: list[mlrun.model.Notification] = None,
|
|
4223
|
+
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
3938
4224
|
):
|
|
3939
4225
|
"""
|
|
3940
4226
|
Set notifications on a schedule. This will override any existing notifications on the schedule.
|
|
@@ -3960,7 +4246,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3960
4246
|
self,
|
|
3961
4247
|
notification_objects: list[mlrun.model.Notification],
|
|
3962
4248
|
run_uid: str,
|
|
3963
|
-
project: str = None,
|
|
4249
|
+
project: Optional[str] = None,
|
|
3964
4250
|
mask_params: bool = True,
|
|
3965
4251
|
):
|
|
3966
4252
|
"""
|
|
@@ -3994,7 +4280,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3994
4280
|
source: Optional[str] = None,
|
|
3995
4281
|
run_name: Optional[str] = None,
|
|
3996
4282
|
namespace: Optional[str] = None,
|
|
3997
|
-
notifications: list[mlrun.model.Notification] = None,
|
|
4283
|
+
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
3998
4284
|
) -> mlrun.common.schemas.WorkflowResponse:
|
|
3999
4285
|
"""
|
|
4000
4286
|
Submitting workflow for a remote execution.
|
|
@@ -4216,6 +4502,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4216
4502
|
alert_name: str,
|
|
4217
4503
|
alert_data: Union[dict, AlertConfig],
|
|
4218
4504
|
project="",
|
|
4505
|
+
force_reset: bool = False,
|
|
4219
4506
|
) -> AlertConfig:
|
|
4220
4507
|
"""
|
|
4221
4508
|
Create/modify an alert.
|
|
@@ -4223,6 +4510,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4223
4510
|
:param alert_name: The name of the alert.
|
|
4224
4511
|
:param alert_data: The data of the alert.
|
|
4225
4512
|
:param project: The project that the alert belongs to.
|
|
4513
|
+
:param force_reset: If True and the alert already exists, the alert would be reset.
|
|
4226
4514
|
:returns: The created/modified alert.
|
|
4227
4515
|
"""
|
|
4228
4516
|
if not alert_data:
|
|
@@ -4247,7 +4535,10 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4247
4535
|
|
|
4248
4536
|
alert_data = alert_instance.to_dict()
|
|
4249
4537
|
body = _as_json(alert_data)
|
|
4250
|
-
|
|
4538
|
+
params = {"force_reset": bool2str(force_reset)} if force_reset else {}
|
|
4539
|
+
response = self.api_call(
|
|
4540
|
+
"PUT", endpoint_path, error_message, params=params, body=body
|
|
4541
|
+
)
|
|
4251
4542
|
return AlertConfig.from_dict(response.json())
|
|
4252
4543
|
|
|
4253
4544
|
def get_alert_config(self, alert_name: str, project="") -> AlertConfig:
|
|
@@ -4334,6 +4625,251 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4334
4625
|
results.append(mlrun.common.schemas.AlertTemplate(**item))
|
|
4335
4626
|
return results
|
|
4336
4627
|
|
|
4628
|
+
@staticmethod
|
|
4629
|
+
def _parse_labels(
|
|
4630
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]],
|
|
4631
|
+
):
|
|
4632
|
+
"""
|
|
4633
|
+
Parse labels to support providing a dictionary from the SDK,
|
|
4634
|
+
which may not be directly supported in the endpoints.
|
|
4635
|
+
|
|
4636
|
+
:param labels: The labels to parse, which can be a dictionary, a list of strings,
|
|
4637
|
+
or a comma-separated string. This function converts them into a list
|
|
4638
|
+
of labels in the format 'key=value' or 'key'.
|
|
4639
|
+
:return: A list of parsed labels in the format 'key=value' or 'key'.
|
|
4640
|
+
:raises MLRunValueError: If the labels format is invalid.
|
|
4641
|
+
"""
|
|
4642
|
+
try:
|
|
4643
|
+
return mlrun.common.schemas.common.LabelsModel(labels=labels).labels
|
|
4644
|
+
except pydantic.error_wrappers.ValidationError as exc:
|
|
4645
|
+
raise mlrun.errors.MLRunValueError(
|
|
4646
|
+
"Invalid labels format. Must be a dictionary of strings, a list of strings, "
|
|
4647
|
+
"or a comma-separated string."
|
|
4648
|
+
) from exc
|
|
4649
|
+
|
|
4650
|
+
def _list_artifacts(
|
|
4651
|
+
self,
|
|
4652
|
+
name: Optional[str] = None,
|
|
4653
|
+
project: Optional[str] = None,
|
|
4654
|
+
tag: Optional[str] = None,
|
|
4655
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
4656
|
+
since: Optional[datetime] = None,
|
|
4657
|
+
until: Optional[datetime] = None,
|
|
4658
|
+
iter: Optional[int] = None,
|
|
4659
|
+
best_iteration: bool = False,
|
|
4660
|
+
kind: Optional[str] = None,
|
|
4661
|
+
category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
|
|
4662
|
+
tree: Optional[str] = None,
|
|
4663
|
+
producer_uri: Optional[str] = None,
|
|
4664
|
+
format_: Optional[
|
|
4665
|
+
mlrun.common.formatters.ArtifactFormat
|
|
4666
|
+
] = mlrun.common.formatters.ArtifactFormat.full,
|
|
4667
|
+
limit: Optional[int] = None,
|
|
4668
|
+
page: Optional[int] = None,
|
|
4669
|
+
page_size: Optional[int] = None,
|
|
4670
|
+
page_token: Optional[str] = None,
|
|
4671
|
+
return_all: bool = False,
|
|
4672
|
+
) -> tuple[ArtifactList, Optional[str]]:
|
|
4673
|
+
"""Handles list artifacts, both paginated and not."""
|
|
4674
|
+
|
|
4675
|
+
project = project or config.default_project
|
|
4676
|
+
labels = self._parse_labels(labels)
|
|
4677
|
+
|
|
4678
|
+
params = {
|
|
4679
|
+
"name": name,
|
|
4680
|
+
"tag": tag,
|
|
4681
|
+
"label": labels,
|
|
4682
|
+
"iter": iter,
|
|
4683
|
+
"best-iteration": best_iteration,
|
|
4684
|
+
"kind": kind,
|
|
4685
|
+
"category": category,
|
|
4686
|
+
"tree": tree,
|
|
4687
|
+
"format": format_,
|
|
4688
|
+
"producer_uri": producer_uri,
|
|
4689
|
+
"since": datetime_to_iso(since),
|
|
4690
|
+
"until": datetime_to_iso(until),
|
|
4691
|
+
"limit": limit,
|
|
4692
|
+
"page": page,
|
|
4693
|
+
"page-size": page_size,
|
|
4694
|
+
"page-token": page_token,
|
|
4695
|
+
}
|
|
4696
|
+
|
|
4697
|
+
error = "list artifacts"
|
|
4698
|
+
endpoint_path = f"projects/{project}/artifacts"
|
|
4699
|
+
|
|
4700
|
+
# Fetch the responses, either one page or all based on `return_all`
|
|
4701
|
+
responses = self.paginated_api_call(
|
|
4702
|
+
"GET",
|
|
4703
|
+
endpoint_path,
|
|
4704
|
+
error,
|
|
4705
|
+
params=params,
|
|
4706
|
+
version="v2",
|
|
4707
|
+
return_all=return_all,
|
|
4708
|
+
)
|
|
4709
|
+
paginated_responses, token = self.process_paginated_responses(
|
|
4710
|
+
responses, "artifacts"
|
|
4711
|
+
)
|
|
4712
|
+
|
|
4713
|
+
values = ArtifactList(paginated_responses)
|
|
4714
|
+
values.tag = tag
|
|
4715
|
+
return values, token
|
|
4716
|
+
|
|
4717
|
+
def _list_functions(
|
|
4718
|
+
self,
|
|
4719
|
+
name: Optional[str] = None,
|
|
4720
|
+
project: Optional[str] = None,
|
|
4721
|
+
tag: Optional[str] = None,
|
|
4722
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
4723
|
+
since: Optional[datetime] = None,
|
|
4724
|
+
until: Optional[datetime] = None,
|
|
4725
|
+
page: Optional[int] = None,
|
|
4726
|
+
page_size: Optional[int] = None,
|
|
4727
|
+
page_token: Optional[str] = None,
|
|
4728
|
+
return_all: bool = False,
|
|
4729
|
+
) -> tuple[list, Optional[str]]:
|
|
4730
|
+
"""Handles list functions, both paginated and not."""
|
|
4731
|
+
|
|
4732
|
+
project = project or config.default_project
|
|
4733
|
+
labels = self._parse_labels(labels)
|
|
4734
|
+
params = {
|
|
4735
|
+
"name": name,
|
|
4736
|
+
"tag": tag,
|
|
4737
|
+
"label": labels,
|
|
4738
|
+
"since": datetime_to_iso(since),
|
|
4739
|
+
"until": datetime_to_iso(until),
|
|
4740
|
+
"page": page,
|
|
4741
|
+
"page-size": page_size,
|
|
4742
|
+
"page-token": page_token,
|
|
4743
|
+
}
|
|
4744
|
+
error = "list functions"
|
|
4745
|
+
path = f"projects/{project}/functions"
|
|
4746
|
+
|
|
4747
|
+
# Fetch the responses, either one page or all based on `return_all`
|
|
4748
|
+
responses = self.paginated_api_call(
|
|
4749
|
+
"GET", path, error, params=params, return_all=return_all
|
|
4750
|
+
)
|
|
4751
|
+
paginated_responses, token = self.process_paginated_responses(
|
|
4752
|
+
responses, "funcs"
|
|
4753
|
+
)
|
|
4754
|
+
return paginated_responses, token
|
|
4755
|
+
|
|
4756
|
+
def _list_runs(
|
|
4757
|
+
self,
|
|
4758
|
+
name: Optional[str] = None,
|
|
4759
|
+
uid: Optional[Union[str, list[str]]] = None,
|
|
4760
|
+
project: Optional[str] = None,
|
|
4761
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
4762
|
+
state: Optional[
|
|
4763
|
+
mlrun.common.runtimes.constants.RunStates
|
|
4764
|
+
] = None, # Backward compatibility
|
|
4765
|
+
states: typing.Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
|
|
4766
|
+
sort: bool = True,
|
|
4767
|
+
last: int = 0,
|
|
4768
|
+
iter: bool = False,
|
|
4769
|
+
start_time_from: Optional[datetime] = None,
|
|
4770
|
+
start_time_to: Optional[datetime] = None,
|
|
4771
|
+
last_update_time_from: Optional[datetime] = None,
|
|
4772
|
+
last_update_time_to: Optional[datetime] = None,
|
|
4773
|
+
partition_by: Optional[
|
|
4774
|
+
Union[mlrun.common.schemas.RunPartitionByField, str]
|
|
4775
|
+
] = None,
|
|
4776
|
+
rows_per_partition: int = 1,
|
|
4777
|
+
partition_sort_by: Optional[Union[mlrun.common.schemas.SortField, str]] = None,
|
|
4778
|
+
partition_order: Union[
|
|
4779
|
+
mlrun.common.schemas.OrderType, str
|
|
4780
|
+
] = mlrun.common.schemas.OrderType.desc,
|
|
4781
|
+
max_partitions: int = 0,
|
|
4782
|
+
with_notifications: bool = False,
|
|
4783
|
+
page: Optional[int] = None,
|
|
4784
|
+
page_size: Optional[int] = None,
|
|
4785
|
+
page_token: Optional[str] = None,
|
|
4786
|
+
return_all: bool = False,
|
|
4787
|
+
) -> tuple[RunList, Optional[str]]:
|
|
4788
|
+
"""Handles list runs, both paginated and not."""
|
|
4789
|
+
|
|
4790
|
+
project = project or config.default_project
|
|
4791
|
+
if with_notifications:
|
|
4792
|
+
logger.warning(
|
|
4793
|
+
"Local run notifications are not persisted in the DB, therefore local runs will not be returned when "
|
|
4794
|
+
"using the `with_notifications` flag."
|
|
4795
|
+
)
|
|
4796
|
+
|
|
4797
|
+
if last:
|
|
4798
|
+
# TODO: Remove this in 1.8.0
|
|
4799
|
+
warnings.warn(
|
|
4800
|
+
"'last' is deprecated and will be removed in 1.8.0.",
|
|
4801
|
+
FutureWarning,
|
|
4802
|
+
)
|
|
4803
|
+
|
|
4804
|
+
if state:
|
|
4805
|
+
# TODO: Remove this in 1.9.0
|
|
4806
|
+
warnings.warn(
|
|
4807
|
+
"'state' is deprecated and will be removed in 1.9.0. Use 'states' instead.",
|
|
4808
|
+
FutureWarning,
|
|
4809
|
+
)
|
|
4810
|
+
|
|
4811
|
+
labels = self._parse_labels(labels)
|
|
4812
|
+
|
|
4813
|
+
if (
|
|
4814
|
+
not name
|
|
4815
|
+
and not uid
|
|
4816
|
+
and not labels
|
|
4817
|
+
and not state
|
|
4818
|
+
and not states
|
|
4819
|
+
and not last
|
|
4820
|
+
and not start_time_from
|
|
4821
|
+
and not start_time_to
|
|
4822
|
+
and not last_update_time_from
|
|
4823
|
+
and not last_update_time_to
|
|
4824
|
+
and not partition_by
|
|
4825
|
+
and not partition_sort_by
|
|
4826
|
+
and not iter
|
|
4827
|
+
):
|
|
4828
|
+
# default to last week on no filter
|
|
4829
|
+
start_time_from = datetime.now() - timedelta(days=7)
|
|
4830
|
+
partition_by = mlrun.common.schemas.RunPartitionByField.project_and_name
|
|
4831
|
+
partition_sort_by = mlrun.common.schemas.SortField.updated
|
|
4832
|
+
|
|
4833
|
+
params = {
|
|
4834
|
+
"name": name,
|
|
4835
|
+
"uid": uid,
|
|
4836
|
+
"label": labels,
|
|
4837
|
+
"state": mlrun.utils.helpers.as_list(state)
|
|
4838
|
+
if state is not None
|
|
4839
|
+
else states or None,
|
|
4840
|
+
"sort": bool2str(sort),
|
|
4841
|
+
"iter": bool2str(iter),
|
|
4842
|
+
"start_time_from": datetime_to_iso(start_time_from),
|
|
4843
|
+
"start_time_to": datetime_to_iso(start_time_to),
|
|
4844
|
+
"last_update_time_from": datetime_to_iso(last_update_time_from),
|
|
4845
|
+
"last_update_time_to": datetime_to_iso(last_update_time_to),
|
|
4846
|
+
"with-notifications": with_notifications,
|
|
4847
|
+
"page": page,
|
|
4848
|
+
"page-size": page_size,
|
|
4849
|
+
"page-token": page_token,
|
|
4850
|
+
}
|
|
4851
|
+
|
|
4852
|
+
if partition_by:
|
|
4853
|
+
params.update(
|
|
4854
|
+
self._generate_partition_by_params(
|
|
4855
|
+
mlrun.common.schemas.RunPartitionByField,
|
|
4856
|
+
partition_by,
|
|
4857
|
+
rows_per_partition,
|
|
4858
|
+
partition_sort_by,
|
|
4859
|
+
partition_order,
|
|
4860
|
+
max_partitions,
|
|
4861
|
+
)
|
|
4862
|
+
)
|
|
4863
|
+
error = "list runs"
|
|
4864
|
+
_path = self._path_of("runs", project)
|
|
4865
|
+
|
|
4866
|
+
# Fetch the responses, either one page or all based on `return_all`
|
|
4867
|
+
responses = self.paginated_api_call(
|
|
4868
|
+
"GET", _path, error, params=params, return_all=return_all
|
|
4869
|
+
)
|
|
4870
|
+
paginated_responses, token = self.process_paginated_responses(responses, "runs")
|
|
4871
|
+
return RunList(paginated_responses), token
|
|
4872
|
+
|
|
4337
4873
|
|
|
4338
4874
|
def _as_json(obj):
|
|
4339
4875
|
fn = getattr(obj, "to_json", None)
|