mlrun 1.7.1rc4__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (257) hide show
  1. mlrun/__init__.py +23 -21
  2. mlrun/__main__.py +3 -3
  3. mlrun/alerts/alert.py +148 -14
  4. mlrun/artifacts/__init__.py +1 -2
  5. mlrun/artifacts/base.py +46 -12
  6. mlrun/artifacts/dataset.py +16 -16
  7. mlrun/artifacts/document.py +334 -0
  8. mlrun/artifacts/manager.py +15 -13
  9. mlrun/artifacts/model.py +66 -53
  10. mlrun/common/constants.py +7 -0
  11. mlrun/common/formatters/__init__.py +1 -0
  12. mlrun/common/formatters/feature_set.py +1 -0
  13. mlrun/common/formatters/function.py +1 -0
  14. mlrun/{model_monitoring/db/stores/base/__init__.py → common/formatters/model_endpoint.py} +16 -1
  15. mlrun/common/formatters/pipeline.py +1 -2
  16. mlrun/common/formatters/project.py +9 -0
  17. mlrun/common/model_monitoring/__init__.py +0 -5
  18. mlrun/common/model_monitoring/helpers.py +1 -29
  19. mlrun/common/runtimes/constants.py +1 -2
  20. mlrun/common/schemas/__init__.py +6 -2
  21. mlrun/common/schemas/alert.py +111 -19
  22. mlrun/common/schemas/api_gateway.py +3 -3
  23. mlrun/common/schemas/artifact.py +11 -7
  24. mlrun/common/schemas/auth.py +6 -4
  25. mlrun/common/schemas/background_task.py +7 -7
  26. mlrun/common/schemas/client_spec.py +2 -3
  27. mlrun/common/schemas/clusterization_spec.py +2 -2
  28. mlrun/common/schemas/common.py +53 -3
  29. mlrun/common/schemas/constants.py +15 -0
  30. mlrun/common/schemas/datastore_profile.py +1 -1
  31. mlrun/common/schemas/feature_store.py +9 -9
  32. mlrun/common/schemas/frontend_spec.py +4 -4
  33. mlrun/common/schemas/function.py +10 -10
  34. mlrun/common/schemas/hub.py +1 -1
  35. mlrun/common/schemas/k8s.py +3 -3
  36. mlrun/common/schemas/memory_reports.py +3 -3
  37. mlrun/common/schemas/model_monitoring/__init__.py +2 -1
  38. mlrun/common/schemas/model_monitoring/constants.py +66 -14
  39. mlrun/common/schemas/model_monitoring/grafana.py +1 -1
  40. mlrun/common/schemas/model_monitoring/model_endpoints.py +91 -147
  41. mlrun/common/schemas/notification.py +24 -3
  42. mlrun/common/schemas/object.py +1 -1
  43. mlrun/common/schemas/pagination.py +4 -4
  44. mlrun/common/schemas/partition.py +137 -0
  45. mlrun/common/schemas/pipeline.py +2 -2
  46. mlrun/common/schemas/project.py +25 -17
  47. mlrun/common/schemas/runs.py +2 -2
  48. mlrun/common/schemas/runtime_resource.py +5 -5
  49. mlrun/common/schemas/schedule.py +1 -1
  50. mlrun/common/schemas/secret.py +1 -1
  51. mlrun/common/schemas/tag.py +3 -3
  52. mlrun/common/schemas/workflow.py +5 -5
  53. mlrun/config.py +67 -10
  54. mlrun/data_types/__init__.py +0 -2
  55. mlrun/data_types/infer.py +3 -1
  56. mlrun/data_types/spark.py +2 -1
  57. mlrun/datastore/__init__.py +0 -2
  58. mlrun/datastore/alibaba_oss.py +4 -1
  59. mlrun/datastore/azure_blob.py +4 -1
  60. mlrun/datastore/base.py +12 -4
  61. mlrun/datastore/datastore.py +9 -3
  62. mlrun/datastore/datastore_profile.py +79 -20
  63. mlrun/datastore/dbfs_store.py +4 -1
  64. mlrun/datastore/filestore.py +4 -1
  65. mlrun/datastore/google_cloud_storage.py +4 -1
  66. mlrun/datastore/hdfs.py +4 -1
  67. mlrun/datastore/inmem.py +4 -1
  68. mlrun/datastore/redis.py +4 -1
  69. mlrun/datastore/s3.py +4 -1
  70. mlrun/datastore/sources.py +52 -51
  71. mlrun/datastore/store_resources.py +0 -2
  72. mlrun/datastore/targets.py +21 -21
  73. mlrun/datastore/utils.py +2 -2
  74. mlrun/datastore/v3io.py +4 -1
  75. mlrun/datastore/vectorstore.py +194 -0
  76. mlrun/datastore/wasbfs/fs.py +13 -12
  77. mlrun/db/base.py +208 -82
  78. mlrun/db/factory.py +0 -3
  79. mlrun/db/httpdb.py +1237 -386
  80. mlrun/db/nopdb.py +201 -74
  81. mlrun/errors.py +2 -2
  82. mlrun/execution.py +136 -50
  83. mlrun/feature_store/__init__.py +0 -2
  84. mlrun/feature_store/api.py +41 -40
  85. mlrun/feature_store/common.py +9 -9
  86. mlrun/feature_store/feature_set.py +20 -18
  87. mlrun/feature_store/feature_vector.py +27 -24
  88. mlrun/feature_store/retrieval/base.py +14 -9
  89. mlrun/feature_store/retrieval/job.py +2 -1
  90. mlrun/feature_store/steps.py +2 -2
  91. mlrun/features.py +30 -13
  92. mlrun/frameworks/__init__.py +1 -2
  93. mlrun/frameworks/_common/__init__.py +1 -2
  94. mlrun/frameworks/_common/artifacts_library.py +2 -2
  95. mlrun/frameworks/_common/mlrun_interface.py +10 -6
  96. mlrun/frameworks/_common/model_handler.py +29 -27
  97. mlrun/frameworks/_common/producer.py +3 -1
  98. mlrun/frameworks/_dl_common/__init__.py +1 -2
  99. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
  100. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
  101. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
  102. mlrun/frameworks/_ml_common/__init__.py +1 -2
  103. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
  104. mlrun/frameworks/_ml_common/model_handler.py +21 -21
  105. mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
  106. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
  107. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  108. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  109. mlrun/frameworks/auto_mlrun/__init__.py +1 -2
  110. mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
  111. mlrun/frameworks/huggingface/__init__.py +1 -2
  112. mlrun/frameworks/huggingface/model_server.py +9 -9
  113. mlrun/frameworks/lgbm/__init__.py +47 -44
  114. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
  115. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
  116. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
  117. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
  118. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
  119. mlrun/frameworks/lgbm/model_handler.py +15 -11
  120. mlrun/frameworks/lgbm/model_server.py +11 -7
  121. mlrun/frameworks/lgbm/utils.py +2 -2
  122. mlrun/frameworks/onnx/__init__.py +1 -2
  123. mlrun/frameworks/onnx/dataset.py +3 -3
  124. mlrun/frameworks/onnx/mlrun_interface.py +2 -2
  125. mlrun/frameworks/onnx/model_handler.py +7 -5
  126. mlrun/frameworks/onnx/model_server.py +8 -6
  127. mlrun/frameworks/parallel_coordinates.py +11 -11
  128. mlrun/frameworks/pytorch/__init__.py +22 -23
  129. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
  130. mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
  131. mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
  132. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
  133. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
  134. mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
  135. mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
  136. mlrun/frameworks/pytorch/model_handler.py +21 -17
  137. mlrun/frameworks/pytorch/model_server.py +13 -9
  138. mlrun/frameworks/sklearn/__init__.py +19 -18
  139. mlrun/frameworks/sklearn/estimator.py +2 -2
  140. mlrun/frameworks/sklearn/metric.py +3 -3
  141. mlrun/frameworks/sklearn/metrics_library.py +8 -6
  142. mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
  143. mlrun/frameworks/sklearn/model_handler.py +4 -3
  144. mlrun/frameworks/tf_keras/__init__.py +11 -12
  145. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
  146. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
  147. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
  148. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
  149. mlrun/frameworks/tf_keras/model_handler.py +17 -13
  150. mlrun/frameworks/tf_keras/model_server.py +12 -8
  151. mlrun/frameworks/xgboost/__init__.py +19 -18
  152. mlrun/frameworks/xgboost/model_handler.py +13 -9
  153. mlrun/launcher/base.py +3 -4
  154. mlrun/launcher/local.py +1 -1
  155. mlrun/launcher/remote.py +1 -1
  156. mlrun/lists.py +4 -3
  157. mlrun/model.py +117 -46
  158. mlrun/model_monitoring/__init__.py +4 -4
  159. mlrun/model_monitoring/api.py +61 -59
  160. mlrun/model_monitoring/applications/_application_steps.py +17 -17
  161. mlrun/model_monitoring/applications/base.py +165 -6
  162. mlrun/model_monitoring/applications/context.py +88 -37
  163. mlrun/model_monitoring/applications/evidently_base.py +1 -2
  164. mlrun/model_monitoring/applications/histogram_data_drift.py +43 -21
  165. mlrun/model_monitoring/applications/results.py +55 -3
  166. mlrun/model_monitoring/controller.py +207 -239
  167. mlrun/model_monitoring/db/__init__.py +0 -2
  168. mlrun/model_monitoring/db/_schedules.py +156 -0
  169. mlrun/model_monitoring/db/_stats.py +189 -0
  170. mlrun/model_monitoring/db/tsdb/base.py +78 -25
  171. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +90 -16
  172. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
  173. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +279 -59
  174. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
  175. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +78 -17
  176. mlrun/model_monitoring/helpers.py +152 -49
  177. mlrun/model_monitoring/stream_processing.py +99 -283
  178. mlrun/model_monitoring/tracking_policy.py +10 -3
  179. mlrun/model_monitoring/writer.py +48 -36
  180. mlrun/package/__init__.py +3 -6
  181. mlrun/package/context_handler.py +1 -1
  182. mlrun/package/packager.py +12 -9
  183. mlrun/package/packagers/__init__.py +0 -2
  184. mlrun/package/packagers/default_packager.py +14 -11
  185. mlrun/package/packagers/numpy_packagers.py +16 -7
  186. mlrun/package/packagers/pandas_packagers.py +18 -18
  187. mlrun/package/packagers/python_standard_library_packagers.py +25 -11
  188. mlrun/package/packagers_manager.py +31 -14
  189. mlrun/package/utils/__init__.py +0 -3
  190. mlrun/package/utils/_pickler.py +6 -6
  191. mlrun/platforms/__init__.py +47 -16
  192. mlrun/platforms/iguazio.py +4 -1
  193. mlrun/projects/operations.py +27 -27
  194. mlrun/projects/pipelines.py +75 -38
  195. mlrun/projects/project.py +865 -206
  196. mlrun/run.py +53 -10
  197. mlrun/runtimes/__init__.py +1 -3
  198. mlrun/runtimes/base.py +15 -11
  199. mlrun/runtimes/daskjob.py +9 -9
  200. mlrun/runtimes/generators.py +2 -1
  201. mlrun/runtimes/kubejob.py +4 -5
  202. mlrun/runtimes/mounts.py +572 -0
  203. mlrun/runtimes/mpijob/__init__.py +0 -2
  204. mlrun/runtimes/mpijob/abstract.py +7 -6
  205. mlrun/runtimes/nuclio/api_gateway.py +7 -7
  206. mlrun/runtimes/nuclio/application/application.py +11 -11
  207. mlrun/runtimes/nuclio/function.py +19 -17
  208. mlrun/runtimes/nuclio/serving.py +18 -11
  209. mlrun/runtimes/pod.py +154 -45
  210. mlrun/runtimes/remotesparkjob.py +3 -2
  211. mlrun/runtimes/sparkjob/__init__.py +0 -2
  212. mlrun/runtimes/sparkjob/spark3job.py +21 -11
  213. mlrun/runtimes/utils.py +6 -5
  214. mlrun/serving/merger.py +6 -4
  215. mlrun/serving/remote.py +18 -17
  216. mlrun/serving/routers.py +185 -172
  217. mlrun/serving/server.py +7 -1
  218. mlrun/serving/states.py +97 -78
  219. mlrun/serving/utils.py +13 -2
  220. mlrun/serving/v1_serving.py +3 -2
  221. mlrun/serving/v2_serving.py +74 -65
  222. mlrun/track/__init__.py +1 -1
  223. mlrun/track/tracker.py +2 -2
  224. mlrun/track/trackers/mlflow_tracker.py +6 -5
  225. mlrun/utils/async_http.py +1 -1
  226. mlrun/utils/clones.py +1 -1
  227. mlrun/utils/helpers.py +66 -18
  228. mlrun/utils/logger.py +106 -4
  229. mlrun/utils/notifications/notification/__init__.py +22 -19
  230. mlrun/utils/notifications/notification/base.py +33 -14
  231. mlrun/utils/notifications/notification/console.py +6 -6
  232. mlrun/utils/notifications/notification/git.py +11 -11
  233. mlrun/utils/notifications/notification/ipython.py +10 -9
  234. mlrun/utils/notifications/notification/mail.py +176 -0
  235. mlrun/utils/notifications/notification/slack.py +6 -6
  236. mlrun/utils/notifications/notification/webhook.py +6 -6
  237. mlrun/utils/notifications/notification_pusher.py +86 -44
  238. mlrun/utils/regex.py +3 -1
  239. mlrun/utils/version/version.json +2 -2
  240. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/METADATA +191 -186
  241. mlrun-1.8.0rc8.dist-info/RECORD +347 -0
  242. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/WHEEL +1 -1
  243. mlrun/model_monitoring/db/stores/__init__.py +0 -136
  244. mlrun/model_monitoring/db/stores/base/store.py +0 -213
  245. mlrun/model_monitoring/db/stores/sqldb/__init__.py +0 -13
  246. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -71
  247. mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -190
  248. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +0 -103
  249. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -40
  250. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +0 -659
  251. mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +0 -13
  252. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +0 -726
  253. mlrun/model_monitoring/model_endpoint.py +0 -118
  254. mlrun-1.7.1rc4.dist-info/RECORD +0 -351
  255. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/LICENSE +0 -0
  256. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/entry_points.txt +0 -0
  257. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/top_level.txt +0 -0
mlrun/db/httpdb.py CHANGED
@@ -22,19 +22,21 @@ import warnings
22
22
  from copy import deepcopy
23
23
  from datetime import datetime, timedelta
24
24
  from os import path, remove
25
- from typing import Optional, Union
25
+ from typing import Literal, Optional, Union
26
26
  from urllib.parse import urlparse
27
27
 
28
+ import pydantic.v1
28
29
  import requests
29
30
  import semver
30
- from mlrun_pipelines.utils import compile_pipeline
31
+ from pydantic.v1 import parse_obj_as
31
32
 
32
33
  import mlrun
34
+ import mlrun.common.constants
33
35
  import mlrun.common.formatters
34
36
  import mlrun.common.runtimes
35
37
  import mlrun.common.schemas
38
+ import mlrun.common.schemas.model_monitoring.model_endpoints as mm_endpoints
36
39
  import mlrun.common.types
37
- import mlrun.model_monitoring.model_endpoint
38
40
  import mlrun.platforms
39
41
  import mlrun.projects
40
42
  import mlrun.runtimes.nuclio.api_gateway
@@ -43,8 +45,10 @@ import mlrun.utils
43
45
  from mlrun.alerts.alert import AlertConfig
44
46
  from mlrun.db.auth_utils import OAuthClientIDTokenProvider, StaticTokenProvider
45
47
  from mlrun.errors import MLRunInvalidArgumentError, err_to_str
48
+ from mlrun_pipelines.utils import compile_pipeline
46
49
 
47
50
  from ..artifacts import Artifact
51
+ from ..common.schemas import AlertActivations
48
52
  from ..config import config
49
53
  from ..datastore.datastore_profile import DatastoreProfile2Json
50
54
  from ..feature_store import FeatureSet, FeatureVector
@@ -169,7 +173,7 @@ class HTTPRunDB(RunDBInterface):
169
173
  return f"{cls}({self.base_url!r})"
170
174
 
171
175
  @staticmethod
172
- def get_api_path_prefix(version: str = None) -> str:
176
+ def get_api_path_prefix(version: Optional[str] = None) -> str:
173
177
  """
174
178
  :param version: API version to use, None (the default) will mean to use the default value from mlrun.config,
175
179
  for un-versioned api set an empty string.
@@ -182,7 +186,7 @@ class HTTPRunDB(RunDBInterface):
182
186
  )
183
187
  return api_version_path
184
188
 
185
- def get_base_api_url(self, path: str, version: str = None) -> str:
189
+ def get_base_api_url(self, path: str, version: Optional[str] = None) -> str:
186
190
  path_prefix = self.get_api_path_prefix(version)
187
191
  url = f"{self.base_url}/{path_prefix}/{path}"
188
192
  return url
@@ -310,9 +314,26 @@ class HTTPRunDB(RunDBInterface):
310
314
  headers=None,
311
315
  timeout=45,
312
316
  version=None,
317
+ return_all=False,
313
318
  ) -> typing.Generator[requests.Response, None, None]:
314
319
  """
315
- Calls the api with pagination, yielding each page of the response
320
+ Calls the API with pagination and yields each page of the response.
321
+
322
+ Depending on the `return_all` parameter:
323
+ - If `return_all` is `True`, fetches and yields all pages of results.
324
+ - If `return_all` is False, only a single page of results is fetched and yielded.
325
+
326
+ :param method: The HTTP method (GET, POST, etc.).
327
+ :param path: The API endpoint path.
328
+ :param error: Error message used for debugging if the request fails.
329
+ :param params: The parameters to pass for the API request, including filters.
330
+ :param body: The body of the request.
331
+ :param json: The JSON payload for the request.
332
+ :param headers: Custom headers for the request.
333
+ :param timeout: Timeout for the request.
334
+ :param version: API version, optional.
335
+ :param return_all: If `True`, fetches all pages and returns them in one shot. If `False`, returns only
336
+ the requested page or the next page.
316
337
  """
317
338
 
318
339
  def _api_call(_params):
@@ -328,38 +349,50 @@ class HTTPRunDB(RunDBInterface):
328
349
  version=version,
329
350
  )
330
351
 
331
- first_page_params = deepcopy(params) or {}
332
- first_page_params["page"] = 1
333
- first_page_params["page-size"] = config.httpdb.pagination.default_page_size
334
- response = _api_call(first_page_params)
335
- page_token = response.json().get("pagination", {}).get("page-token")
336
- if not page_token:
337
- yield response
338
- return
352
+ page_params = deepcopy(params) or {}
353
+
354
+ if page_params.get("page-token") is None and page_params.get("page") is None:
355
+ page_params["page"] = 1
356
+
357
+ if page_params.get("page-size") is None:
358
+ page_params["page-size"] = config.httpdb.pagination.default_page_size
359
+
360
+ response = _api_call(page_params)
339
361
 
340
- params_with_page_token = deepcopy(params) or {}
341
- params_with_page_token["page-token"] = page_token
342
- while page_token:
343
- yield response
344
- try:
345
- response = _api_call(params_with_page_token)
346
- except mlrun.errors.MLRunNotFoundError:
347
- # pagination token expired
348
- break
362
+ # Yield only a single page of results
363
+ yield response
349
364
 
365
+ if return_all:
350
366
  page_token = response.json().get("pagination", {}).get("page-token", None)
351
367
 
368
+ while page_token:
369
+ try:
370
+ # Use the page token to get the next page.
371
+ # No need to supply any other parameters as the token informs the pagination cache
372
+ # which parameters to use.
373
+ response = _api_call({"page-token": page_token})
374
+ except mlrun.errors.MLRunNotFoundError:
375
+ # pagination token expired, we've reached the last page
376
+ break
377
+
378
+ yield response
379
+ page_token = (
380
+ response.json().get("pagination", {}).get("page-token", None)
381
+ )
382
+
352
383
  @staticmethod
353
384
  def process_paginated_responses(
354
385
  responses: typing.Generator[requests.Response, None, None], key: str = "data"
355
- ) -> list[typing.Any]:
386
+ ) -> tuple[list[typing.Any], Optional[str]]:
356
387
  """
357
388
  Processes the paginated responses and returns the combined data
358
389
  """
359
390
  data = []
391
+ page_token = None
360
392
  for response in responses:
393
+ page_token = response.json().get("pagination", {}).get("page-token", None)
361
394
  data.extend(response.json().get(key, []))
362
- return data
395
+ return data, page_token
363
396
 
364
397
  def _init_session(self, retry_on_post: bool = False):
365
398
  return mlrun.utils.HTTPSessionWithRetry(
@@ -525,10 +558,6 @@ class HTTPRunDB(RunDBInterface):
525
558
  server_cfg.get("external_platform_tracking")
526
559
  or config.external_platform_tracking
527
560
  )
528
- config.model_endpoint_monitoring.endpoint_store_connection = (
529
- server_cfg.get("model_endpoint_monitoring_endpoint_store_connection")
530
- or config.model_endpoint_monitoring.endpoint_store_connection
531
- )
532
561
  config.model_endpoint_monitoring.tsdb_connection = (
533
562
  server_cfg.get("model_monitoring_tsdb_connection")
534
563
  or config.model_endpoint_monitoring.tsdb_connection
@@ -768,7 +797,7 @@ class HTTPRunDB(RunDBInterface):
768
797
  name: Optional[str] = None,
769
798
  uid: Optional[Union[str, list[str]]] = None,
770
799
  project: Optional[str] = None,
771
- labels: Optional[Union[str, list[str]]] = None,
800
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
772
801
  state: Optional[
773
802
  mlrun.common.runtimes.constants.RunStates
774
803
  ] = None, # Backward compatibility
@@ -807,9 +836,13 @@ class HTTPRunDB(RunDBInterface):
807
836
  :param name: Name of the run to retrieve.
808
837
  :param uid: Unique ID of the run, or a list of run UIDs.
809
838
  :param project: Project that the runs belongs to.
810
- :param labels: A list of labels to filter by. Label filters work by either filtering a specific value
811
- of a label (i.e. list("key=value")) or by looking for the existence of a given
812
- key (i.e. "key").
839
+ :param labels: Filter runs by label key-value pairs or key existence. This can be provided as:
840
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
841
+ or `{"label": None}` to check for key existence.
842
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
843
+ or just `"label"` for key existence.
844
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
845
+ the specified key-value pairs or key existence.
813
846
  :param state: Deprecated - List only runs whose state is specified (will be removed in 1.9.0)
814
847
  :param states: List only runs whose state is one of the provided states.
815
848
  :param sort: Whether to sort the result according to their start time. Otherwise, results will be
@@ -821,8 +854,8 @@ class HTTPRunDB(RunDBInterface):
821
854
  :param last_update_time_from: Filter by run last update time in ``(last_update_time_from,
822
855
  last_update_time_to)``.
823
856
  :param last_update_time_to: Filter by run last update time in ``(last_update_time_from, last_update_time_to)``.
824
- :param partition_by: Field to group results by. Only allowed value is `name`. When `partition_by` is specified,
825
- the `partition_sort_by` parameter must be provided as well.
857
+ :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
858
+ parameter must be provided as well.
826
859
  :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
827
860
  to return per group. Default value is 1.
828
861
  :param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
@@ -832,81 +865,95 @@ class HTTPRunDB(RunDBInterface):
832
865
  limit.
833
866
  :param with_notifications: Return runs with notifications, and join them to the response. Default is `False`.
834
867
  """
868
+ runs, _ = self._list_runs(
869
+ name=name,
870
+ uid=uid,
871
+ project=project,
872
+ labels=labels,
873
+ state=state,
874
+ states=states,
875
+ sort=sort,
876
+ last=last,
877
+ iter=iter,
878
+ start_time_from=start_time_from,
879
+ start_time_to=start_time_to,
880
+ last_update_time_from=last_update_time_from,
881
+ last_update_time_to=last_update_time_to,
882
+ partition_by=partition_by,
883
+ rows_per_partition=rows_per_partition,
884
+ partition_sort_by=partition_sort_by,
885
+ partition_order=partition_order,
886
+ max_partitions=max_partitions,
887
+ with_notifications=with_notifications,
888
+ return_all=True,
889
+ )
890
+ return runs
835
891
 
836
- project = project or config.default_project
837
- if with_notifications:
838
- logger.warning(
839
- "Local run notifications are not persisted in the DB, therefore local runs will not be returned when "
840
- "using the `with_notifications` flag."
841
- )
842
-
843
- if last:
844
- # TODO: Remove this in 1.8.0
845
- warnings.warn(
846
- "'last' is deprecated and will be removed in 1.8.0.",
847
- FutureWarning,
848
- )
892
+ def paginated_list_runs(
893
+ self,
894
+ *args,
895
+ page: Optional[int] = None,
896
+ page_size: Optional[int] = None,
897
+ page_token: Optional[str] = None,
898
+ **kwargs,
899
+ ) -> tuple[RunList, Optional[str]]:
900
+ """List runs with support for pagination and various filtering options.
849
901
 
850
- if state:
851
- # TODO: Remove this in 1.9.0
852
- warnings.warn(
853
- "'state' is deprecated and will be removed in 1.9.0. Use 'states' instead.",
854
- FutureWarning,
855
- )
902
+ This method retrieves a paginated list of runs based on the specified filter parameters.
903
+ Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
904
+ will return a list of runs that match the filtering criteria provided.
856
905
 
857
- if (
858
- not name
859
- and not uid
860
- and not labels
861
- and not state
862
- and not states
863
- and not last
864
- and not start_time_from
865
- and not start_time_to
866
- and not last_update_time_from
867
- and not last_update_time_to
868
- and not partition_by
869
- and not partition_sort_by
870
- and not iter
871
- ):
872
- # default to last week on no filter
873
- start_time_from = datetime.now() - timedelta(days=7)
874
- partition_by = mlrun.common.schemas.RunPartitionByField.project_and_name
875
- partition_sort_by = mlrun.common.schemas.SortField.updated
906
+ For detailed information about the parameters, refer to the list_runs method:
907
+ See :py:func:`~list_runs` for more details.
876
908
 
877
- params = {
878
- "name": name,
879
- "uid": uid,
880
- "label": labels or [],
881
- "state": mlrun.utils.helpers.as_list(state)
882
- if state is not None
883
- else states or None,
884
- "sort": bool2str(sort),
885
- "iter": bool2str(iter),
886
- "start_time_from": datetime_to_iso(start_time_from),
887
- "start_time_to": datetime_to_iso(start_time_to),
888
- "last_update_time_from": datetime_to_iso(last_update_time_from),
889
- "last_update_time_to": datetime_to_iso(last_update_time_to),
890
- "with-notifications": with_notifications,
891
- }
909
+ Examples::
892
910
 
893
- if partition_by:
894
- params.update(
895
- self._generate_partition_by_params(
896
- mlrun.common.schemas.RunPartitionByField,
897
- partition_by,
898
- rows_per_partition,
899
- partition_sort_by,
900
- partition_order,
901
- max_partitions,
911
+ # Fetch first page of runs with page size of 5
912
+ runs, token = db.paginated_list_runs(project="my-project", page_size=5)
913
+ # Fetch next page using the pagination token from the previous response
914
+ runs, token = db.paginated_list_runs(project="my-project", page_token=token)
915
+ # Fetch runs for a specific page (e.g., page 3)
916
+ runs, token = db.paginated_list_runs(project="my-project", page=3, page_size=5)
917
+
918
+ # Automatically iterate over all pages without explicitly specifying the page number
919
+ runs = []
920
+ token = None
921
+ while True:
922
+ page_runs, token = db.paginated_list_runs(
923
+ project="my-project", page_token=token, page_size=5
902
924
  )
903
- )
904
- error = "list runs"
905
- _path = self._path_of("runs", project)
906
- responses = self.paginated_api_call("GET", _path, error, params=params)
907
- return RunList(self.process_paginated_responses(responses, "runs"))
925
+ runs.extend(page_runs)
908
926
 
909
- def del_runs(self, name=None, project=None, labels=None, state=None, days_ago=0):
927
+ # If token is None and page_runs is empty, we've reached the end (no more runs).
928
+ # If token is None and page_runs is not empty, we've fetched the last page of runs.
929
+ if not token:
930
+ break
931
+ print(f"Total runs retrieved: {len(runs)}")
932
+
933
+ :param page: The page number to retrieve. If not provided, the next page will be retrieved.
934
+ :param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
935
+ :param page_token: A pagination token used to retrieve the next page of results. Should not be provided
936
+ for the first request.
937
+
938
+ :returns: A tuple containing the list of runs and an optional `page_token` for pagination.
939
+ """
940
+ return self._list_runs(
941
+ *args,
942
+ page=page,
943
+ page_size=page_size,
944
+ page_token=page_token,
945
+ return_all=False,
946
+ **kwargs,
947
+ )
948
+
949
+ def del_runs(
950
+ self,
951
+ name: Optional[str] = None,
952
+ project: Optional[str] = None,
953
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
954
+ state: Optional[mlrun.common.runtimes.constants.RunStates] = None,
955
+ days_ago: int = 0,
956
+ ):
910
957
  """Delete a group of runs identified by the parameters of the function.
911
958
 
912
959
  Example::
@@ -915,16 +962,23 @@ class HTTPRunDB(RunDBInterface):
915
962
 
916
963
  :param name: Name of the task which the runs belong to.
917
964
  :param project: Project to which the runs belong.
918
- :param labels: Filter runs that are labeled using these specific label values.
965
+ :param labels: Filter runs by label key-value pairs or key existence. This can be provided as:
966
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
967
+ or `{"label": None}` to check for key existence.
968
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
969
+ or just `"label"` for key existence.
970
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
971
+ the specified key-value pairs or key existence.
919
972
  :param state: Filter only runs which are in this state.
920
973
  :param days_ago: Filter runs whose start time is newer than this parameter.
921
974
  """
922
975
 
923
976
  project = project or config.default_project
977
+ labels = self._parse_labels(labels)
924
978
  params = {
925
979
  "name": name,
926
980
  "project": project,
927
- "label": labels or [],
981
+ "label": labels,
928
982
  "state": state,
929
983
  "days_ago": str(days_ago),
930
984
  }
@@ -1028,7 +1082,7 @@ class HTTPRunDB(RunDBInterface):
1028
1082
  deletion_strategy: mlrun.common.schemas.artifact.ArtifactsDeletionStrategies = (
1029
1083
  mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
1030
1084
  ),
1031
- secrets: dict = None,
1085
+ secrets: Optional[dict] = None,
1032
1086
  iter=None,
1033
1087
  ):
1034
1088
  """Delete an artifact.
@@ -1063,29 +1117,39 @@ class HTTPRunDB(RunDBInterface):
1063
1117
 
1064
1118
  def list_artifacts(
1065
1119
  self,
1066
- name=None,
1067
- project=None,
1068
- tag=None,
1069
- labels: Optional[Union[dict[str, str], list[str]]] = None,
1120
+ name: Optional[str] = None,
1121
+ project: Optional[str] = None,
1122
+ tag: Optional[str] = None,
1123
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
1070
1124
  since: Optional[datetime] = None,
1071
1125
  until: Optional[datetime] = None,
1072
- iter: int = None,
1126
+ iter: Optional[int] = None,
1073
1127
  best_iteration: bool = False,
1074
- kind: str = None,
1128
+ kind: Optional[str] = None,
1075
1129
  category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
1076
- tree: str = None,
1077
- producer_uri: str = None,
1130
+ tree: Optional[str] = None,
1131
+ producer_uri: Optional[str] = None,
1078
1132
  format_: Optional[
1079
1133
  mlrun.common.formatters.ArtifactFormat
1080
1134
  ] = mlrun.common.formatters.ArtifactFormat.full,
1081
- limit: int = None,
1135
+ limit: Optional[int] = None,
1136
+ partition_by: Optional[
1137
+ Union[mlrun.common.schemas.ArtifactPartitionByField, str]
1138
+ ] = None,
1139
+ rows_per_partition: int = 1,
1140
+ partition_sort_by: Optional[
1141
+ Union[mlrun.common.schemas.SortField, str]
1142
+ ] = mlrun.common.schemas.SortField.updated,
1143
+ partition_order: Union[
1144
+ mlrun.common.schemas.OrderType, str
1145
+ ] = mlrun.common.schemas.OrderType.desc,
1082
1146
  ) -> ArtifactList:
1083
1147
  """List artifacts filtered by various parameters.
1084
1148
 
1085
1149
  Examples::
1086
1150
 
1087
1151
  # Show latest version of all artifacts in project
1088
- latest_artifacts = db.list_artifacts("", tag="latest", project="iris")
1152
+ latest_artifacts = db.list_artifacts(tag="latest", project="iris")
1089
1153
  # check different artifact versions for a specific artifact
1090
1154
  result_versions = db.list_artifacts("results", tag="*", project="iris")
1091
1155
  # Show artifacts with label filters - both uploaded and of binary type
@@ -1098,8 +1162,13 @@ class HTTPRunDB(RunDBInterface):
1098
1162
  ``my_Name_1`` or ``surname``.
1099
1163
  :param project: Project name.
1100
1164
  :param tag: Return artifacts assigned this tag.
1101
- :param labels: Return artifacts that have these labels. Labels can either be a dictionary {"label": "value"} or
1102
- a list of "label=value" (match label key and value) or "label" (match just label key) strings.
1165
+ :param labels: Filter artifacts by label key-value pairs or key existence. This can be provided as:
1166
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
1167
+ or `{"label": None}` to check for key existence.
1168
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
1169
+ or just `"label"` for key existence.
1170
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
1171
+ the specified key-value pairs or key existence.
1103
1172
  :param since: Return artifacts updated after this date (as datetime object).
1104
1173
  :param until: Return artifacts updated before this date (as datetime object).
1105
1174
  :param iter: Return artifacts from a specific iteration (where ``iter=0`` means the root iteration). If
@@ -1115,38 +1184,110 @@ class HTTPRunDB(RunDBInterface):
1115
1184
  is a workflow id (artifact was created as part of a workflow).
1116
1185
  :param format_: The format in which to return the artifacts. Default is 'full'.
1117
1186
  :param limit: Maximum number of artifacts to return.
1187
+ :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
1188
+ parameter must be provided as well.
1189
+ :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
1190
+ to return per group. Default value is 1.
1191
+ :param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
1192
+ Currently the only allowed values are `created` and `updated`.
1193
+ :param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
1118
1194
  """
1119
1195
 
1120
- project = project or config.default_project
1196
+ artifacts, _ = self._list_artifacts(
1197
+ name=name,
1198
+ project=project,
1199
+ tag=tag,
1200
+ labels=labels,
1201
+ since=since,
1202
+ until=until,
1203
+ iter=iter,
1204
+ best_iteration=best_iteration,
1205
+ kind=kind,
1206
+ category=category,
1207
+ tree=tree,
1208
+ producer_uri=producer_uri,
1209
+ format_=format_,
1210
+ limit=limit,
1211
+ partition_by=partition_by,
1212
+ rows_per_partition=rows_per_partition,
1213
+ partition_sort_by=partition_sort_by,
1214
+ partition_order=partition_order,
1215
+ return_all=True,
1216
+ )
1217
+ return artifacts
1121
1218
 
1122
- labels = labels or []
1123
- if isinstance(labels, dict):
1124
- labels = [f"{key}={value}" for key, value in labels.items()]
1219
+ def paginated_list_artifacts(
1220
+ self,
1221
+ *args,
1222
+ page: Optional[int] = None,
1223
+ page_size: Optional[int] = None,
1224
+ page_token: Optional[str] = None,
1225
+ **kwargs,
1226
+ ) -> tuple[ArtifactList, Optional[str]]:
1227
+ """List artifacts with support for pagination and various filtering options.
1125
1228
 
1126
- params = {
1127
- "name": name,
1128
- "tag": tag,
1129
- "label": labels,
1130
- "iter": iter,
1131
- "best-iteration": best_iteration,
1132
- "kind": kind,
1133
- "category": category,
1134
- "tree": tree,
1135
- "format": format_,
1136
- "producer_uri": producer_uri,
1137
- "limit": limit,
1138
- "since": datetime_to_iso(since),
1139
- "until": datetime_to_iso(until),
1140
- }
1141
- error = "list artifacts"
1142
- endpoint_path = f"projects/{project}/artifacts"
1143
- resp = self.api_call("GET", endpoint_path, error, params=params, version="v2")
1144
- values = ArtifactList(resp.json()["artifacts"])
1145
- values.tag = tag
1146
- return values
1229
+ This method retrieves a paginated list of artifacts based on the specified filter parameters.
1230
+ Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
1231
+ will return a list of artifacts that match the filtering criteria provided.
1232
+
1233
+ For detailed information about the parameters, refer to the list_artifacts method:
1234
+ See :py:func:`~list_artifacts` for more details.
1235
+
1236
+ Examples::
1237
+
1238
+ # Fetch first page of artifacts with page size of 5
1239
+ artifacts, token = db.paginated_list_artifacts(
1240
+ project="my-project", page_size=5
1241
+ )
1242
+ # Fetch next page using the pagination token from the previous response
1243
+ artifacts, token = db.paginated_list_artifacts(
1244
+ project="my-project", page_token=token
1245
+ )
1246
+ # Fetch artifacts for a specific page (e.g., page 3)
1247
+ artifacts, token = db.paginated_list_artifacts(
1248
+ project="my-project", page=3, page_size=5
1249
+ )
1250
+
1251
+ # Automatically iterate over all pages without explicitly specifying the page number
1252
+ artifacts = []
1253
+ token = None
1254
+ while True:
1255
+ page_artifacts, token = db.paginated_list_artifacts(
1256
+ project="my-project", page_token=token, page_size=5
1257
+ )
1258
+ artifacts.extend(page_artifacts)
1259
+
1260
+ # If token is None and page_artifacts is empty, we've reached the end (no more artifacts).
1261
+ # If token is None and page_artifacts is not empty, we've fetched the last page of artifacts.
1262
+ if not token:
1263
+ break
1264
+ print(f"Total artifacts retrieved: {len(artifacts)}")
1265
+
1266
+ :param page: The page number to retrieve. If not provided, the next page will be retrieved.
1267
+ :param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
1268
+ :param page_token: A pagination token used to retrieve the next page of results. Should not be provided
1269
+ for the first request.
1270
+
1271
+ :returns: A tuple containing the list of artifacts and an optional `page_token` for pagination.
1272
+ """
1273
+
1274
+ return self._list_artifacts(
1275
+ *args,
1276
+ page=page,
1277
+ page_size=page_size,
1278
+ page_token=page_token,
1279
+ return_all=False,
1280
+ **kwargs,
1281
+ )
1147
1282
 
1148
1283
  def del_artifacts(
1149
- self, name=None, project=None, tag=None, labels=None, days_ago=0, tree=None
1284
+ self,
1285
+ name: Optional[str] = None,
1286
+ project: Optional[str] = None,
1287
+ tag: Optional[str] = None,
1288
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
1289
+ days_ago=0,
1290
+ tree: Optional[str] = None,
1150
1291
  ):
1151
1292
  """Delete artifacts referenced by the parameters.
1152
1293
 
@@ -1154,15 +1295,24 @@ class HTTPRunDB(RunDBInterface):
1154
1295
  :py:func:`~list_artifacts` for more details.
1155
1296
  :param project: Project that artifacts belong to.
1156
1297
  :param tag: Choose artifacts who are assigned this tag.
1157
- :param labels: Choose artifacts which are labeled.
1298
+ :param labels: Filter artifacts by label key-value pairs or key existence. This can be provided as:
1299
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
1300
+ or `{"label": None}` to check for key existence.
1301
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
1302
+ or just `"label"` for key existence.
1303
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
1304
+ the specified key-value pairs or key existence.
1158
1305
  :param days_ago: This parameter is deprecated and not used.
1306
+ :param tree: Delete artifacts filtered by tree.
1159
1307
  """
1160
1308
  project = project or config.default_project
1309
+ labels = self._parse_labels(labels)
1310
+
1161
1311
  params = {
1162
1312
  "name": name,
1163
1313
  "tag": tag,
1164
1314
  "tree": tree,
1165
- "label": labels or [],
1315
+ "label": labels,
1166
1316
  "days_ago": str(days_ago),
1167
1317
  }
1168
1318
  error = "del artifacts"
@@ -1254,30 +1404,110 @@ class HTTPRunDB(RunDBInterface):
1254
1404
  )
1255
1405
 
1256
1406
  def list_functions(
1257
- self, name=None, project=None, tag=None, labels=None, since=None, until=None
1407
+ self,
1408
+ name: Optional[str] = None,
1409
+ project: Optional[str] = None,
1410
+ tag: Optional[str] = None,
1411
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
1412
+ since: Optional[datetime] = None,
1413
+ until: Optional[datetime] = None,
1414
+ kind: Optional[str] = None,
1415
+ format_: mlrun.common.formatters.FunctionFormat = mlrun.common.formatters.FunctionFormat.full,
1258
1416
  ):
1259
1417
  """Retrieve a list of functions, filtered by specific criteria.
1260
1418
 
1261
1419
  :param name: Return only functions with a specific name.
1262
1420
  :param project: Return functions belonging to this project. If not specified, the default project is used.
1263
1421
  :param tag: Return function versions with specific tags. To return only tagged functions, set tag to ``"*"``.
1264
- :param labels: Return functions that have specific labels assigned to them.
1422
+ :param labels: Filter functions by label key-value pairs or key existence. This can be provided as:
1423
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
1424
+ or `{"label": None}` to check for key existence.
1425
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
1426
+ or just `"label"` for key existence.
1427
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
1428
+ the specified key-value pairs or key existence.
1265
1429
  :param since: Return functions updated after this date (as datetime object).
1266
1430
  :param until: Return functions updated before this date (as datetime object).
1431
+ :param kind: Return only functions of a specific kind.
1432
+ :param format_: The format in which to return the functions. Default is 'full'.
1267
1433
  :returns: List of function objects (as dictionary).
1268
1434
  """
1269
- project = project or config.default_project
1270
- params = {
1271
- "name": name,
1272
- "tag": tag,
1273
- "label": labels or [],
1274
- "since": datetime_to_iso(since),
1275
- "until": datetime_to_iso(until),
1276
- }
1277
- error = "list functions"
1278
- path = f"projects/{project}/functions"
1279
- responses = self.paginated_api_call("GET", path, error, params=params)
1280
- return self.process_paginated_responses(responses, "funcs")
1435
+ functions, _ = self._list_functions(
1436
+ name=name,
1437
+ project=project,
1438
+ tag=tag,
1439
+ kind=kind,
1440
+ labels=labels,
1441
+ format_=format_,
1442
+ since=since,
1443
+ until=until,
1444
+ return_all=True,
1445
+ )
1446
+ return functions
1447
+
1448
+ def paginated_list_functions(
1449
+ self,
1450
+ *args,
1451
+ page: Optional[int] = None,
1452
+ page_size: Optional[int] = None,
1453
+ page_token: Optional[str] = None,
1454
+ **kwargs,
1455
+ ) -> tuple[list[dict], Optional[str]]:
1456
+ """List functions with support for pagination and various filtering options.
1457
+
1458
+ This method retrieves a paginated list of functions based on the specified filter parameters.
1459
+ Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
1460
+ will return a list of functions that match the filtering criteria provided.
1461
+
1462
+ For detailed information about the parameters, refer to the list_functions method:
1463
+ See :py:func:`~list_functions` for more details.
1464
+
1465
+ Examples::
1466
+
1467
+ # Fetch first page of functions with page size of 5
1468
+ functions, token = db.paginated_list_functions(
1469
+ project="my-project", page_size=5
1470
+ )
1471
+ # Fetch next page using the pagination token from the previous response
1472
+ functions, token = db.paginated_list_functions(
1473
+ project="my-project", page_token=token
1474
+ )
1475
+ # Fetch functions for a specific page (e.g., page 3)
1476
+ functions, token = db.paginated_list_functions(
1477
+ project="my-project", page=3, page_size=5
1478
+ )
1479
+
1480
+ # Automatically iterate over all pages without explicitly specifying the page number
1481
+ functions = []
1482
+ token = None
1483
+ while True:
1484
+ page_functions, token = db.paginated_list_functions(
1485
+ project="my-project", page_token=token, page_size=5
1486
+ )
1487
+ functions.extend(page_functions)
1488
+
1489
+ # If token is None and page_functions is empty, we've reached the end (no more functions).
1490
+ # If token is None and page_functions is not empty, we've fetched the last page of functions.
1491
+ if not token:
1492
+ break
1493
+ print(f"Total functions retrieved: {len(functions)}")
1494
+
1495
+ :param page: The page number to retrieve. If not provided, the next page will be retrieved.
1496
+ :param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
1497
+ :param page_token: A pagination token used to retrieve the next page of results. Should not be provided
1498
+ for the first request.
1499
+
1500
+ :returns: A tuple containing the list of functions objects (as dictionary) and an optional
1501
+ `page_token` for pagination.
1502
+ """
1503
+ return self._list_functions(
1504
+ *args,
1505
+ page=page,
1506
+ page_size=page_size,
1507
+ page_token=page_token,
1508
+ return_all=False,
1509
+ **kwargs,
1510
+ )
1281
1511
 
1282
1512
  def list_runtime_resources(
1283
1513
  self,
@@ -1352,7 +1582,7 @@ class HTTPRunDB(RunDBInterface):
1352
1582
  kind: Optional[str] = None,
1353
1583
  object_id: Optional[str] = None,
1354
1584
  force: bool = False,
1355
- grace_period: int = None,
1585
+ grace_period: Optional[int] = None,
1356
1586
  ) -> mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput:
1357
1587
  """Delete all runtime resources which are in terminal state.
1358
1588
 
@@ -1464,9 +1694,11 @@ class HTTPRunDB(RunDBInterface):
1464
1694
  def list_schedules(
1465
1695
  self,
1466
1696
  project: str,
1467
- name: str = None,
1697
+ name: Optional[str] = None,
1468
1698
  kind: mlrun.common.schemas.ScheduleKinds = None,
1469
1699
  include_last_run: bool = False,
1700
+ next_run_time_since: Optional[datetime] = None,
1701
+ next_run_time_until: Optional[datetime] = None,
1470
1702
  ) -> mlrun.common.schemas.SchedulesOutput:
1471
1703
  """Retrieve list of schedules of specific name or kind.
1472
1704
 
@@ -1475,10 +1707,18 @@ class HTTPRunDB(RunDBInterface):
1475
1707
  :param kind: Kind of schedule objects to retrieve, can be either ``job`` or ``pipeline``.
1476
1708
  :param include_last_run: Whether to return for each schedule returned also the results of the last run of
1477
1709
  that schedule.
1710
+ :param next_run_time_since: Return only schedules with next run time after this date.
1711
+ :param next_run_time_until: Return only schedules with next run time before this date.
1478
1712
  """
1479
1713
 
1480
1714
  project = project or config.default_project
1481
- params = {"kind": kind, "name": name, "include_last_run": include_last_run}
1715
+ params = {
1716
+ "kind": kind,
1717
+ "name": name,
1718
+ "include_last_run": include_last_run,
1719
+ "next_run_time_since": datetime_to_iso(next_run_time_since),
1720
+ "next_run_time_until": datetime_to_iso(next_run_time_until),
1721
+ }
1482
1722
  path = f"projects/{project}/schedules"
1483
1723
  error_message = f"Failed listing schedules for {project} ? {kind} {name}"
1484
1724
  resp = self.api_call("GET", path, error_message, params=params)
@@ -1636,6 +1876,7 @@ class HTTPRunDB(RunDBInterface):
1636
1876
  logs: bool = True,
1637
1877
  last_log_timestamp: float = 0.0,
1638
1878
  verbose: bool = False,
1879
+ events_offset: int = 0,
1639
1880
  ):
1640
1881
  """Retrieve the status of a build operation currently in progress.
1641
1882
 
@@ -1645,6 +1886,7 @@ class HTTPRunDB(RunDBInterface):
1645
1886
  :param last_log_timestamp: Last timestamp of logs that were already retrieved. Function will return only logs
1646
1887
  later than this parameter.
1647
1888
  :param verbose: Add verbose logs into the output.
1889
+ :param events_offset: Offset into the build events to retrieve events from.
1648
1890
 
1649
1891
  :returns: The following parameters:
1650
1892
 
@@ -1661,6 +1903,7 @@ class HTTPRunDB(RunDBInterface):
1661
1903
  "tag": func.metadata.tag,
1662
1904
  "logs": bool2str(logs),
1663
1905
  "offset": str(offset),
1906
+ "events_offset": str(events_offset),
1664
1907
  "last_log_timestamp": str(last_log_timestamp),
1665
1908
  "verbose": bool2str(verbose),
1666
1909
  }
@@ -1673,6 +1916,7 @@ class HTTPRunDB(RunDBInterface):
1673
1916
  logger.warning(f"failed resp, {resp.text}")
1674
1917
  raise RunDBError("bad function build response")
1675
1918
 
1919
+ deploy_status_text_kind = mlrun.common.constants.DeployStatusTextKind.logs
1676
1920
  if resp.headers:
1677
1921
  func.status.state = resp.headers.get("x-mlrun-function-status", "")
1678
1922
  last_log_timestamp = float(
@@ -1691,13 +1935,20 @@ class HTTPRunDB(RunDBInterface):
1691
1935
  if function_image:
1692
1936
  func.spec.image = function_image
1693
1937
 
1938
+ deploy_status_text_kind = resp.headers.get(
1939
+ "deploy_status_text_kind",
1940
+ mlrun.common.constants.DeployStatusTextKind.logs,
1941
+ )
1942
+
1694
1943
  text = ""
1695
1944
  if resp.content:
1696
1945
  text = resp.content.decode()
1697
- return text, last_log_timestamp
1946
+ return text, last_log_timestamp, deploy_status_text_kind
1698
1947
 
1699
1948
  def start_function(
1700
- self, func_url: str = None, function: "mlrun.runtimes.BaseRuntime" = None
1949
+ self,
1950
+ func_url: Optional[str] = None,
1951
+ function: "mlrun.runtimes.BaseRuntime" = None,
1701
1952
  ) -> mlrun.common.schemas.BackgroundTask:
1702
1953
  """Execute a function remotely, Used for ``dask`` functions.
1703
1954
 
@@ -1939,14 +2190,14 @@ class HTTPRunDB(RunDBInterface):
1939
2190
  def list_pipelines(
1940
2191
  self,
1941
2192
  project: str,
1942
- namespace: str = None,
2193
+ namespace: Optional[str] = None,
1943
2194
  sort_by: str = "",
1944
2195
  page_token: str = "",
1945
2196
  filter_: str = "",
1946
2197
  format_: Union[
1947
2198
  str, mlrun.common.formatters.PipelineFormat
1948
2199
  ] = mlrun.common.formatters.PipelineFormat.metadata_only,
1949
- page_size: int = None,
2200
+ page_size: Optional[int] = None,
1950
2201
  ) -> mlrun.common.schemas.PipelinesOutput:
1951
2202
  """Retrieve a list of KFP pipelines. This function can be invoked to get all pipelines from all projects,
1952
2203
  by specifying ``project=*``, in which case pagination can be used and the various sorting and pagination
@@ -1988,12 +2239,12 @@ class HTTPRunDB(RunDBInterface):
1988
2239
  def get_pipeline(
1989
2240
  self,
1990
2241
  run_id: str,
1991
- namespace: str = None,
2242
+ namespace: Optional[str] = None,
1992
2243
  timeout: int = 30,
1993
2244
  format_: Union[
1994
2245
  str, mlrun.common.formatters.PipelineFormat
1995
2246
  ] = mlrun.common.formatters.PipelineFormat.summary,
1996
- project: str = None,
2247
+ project: Optional[str] = None,
1997
2248
  ):
1998
2249
  """Retrieve details of a specific pipeline using its run ID (as provided when the pipeline was executed)."""
1999
2250
 
@@ -2015,6 +2266,75 @@ class HTTPRunDB(RunDBInterface):
2015
2266
 
2016
2267
  return resp.json()
2017
2268
 
2269
+ def retry_pipeline(
2270
+ self,
2271
+ run_id: str,
2272
+ namespace: Optional[str] = None,
2273
+ timeout: int = 30,
2274
+ project: Optional[str] = None,
2275
+ ):
2276
+ """
2277
+ Retry a specific pipeline run using its run ID. This function sends an API request
2278
+ to retry a pipeline run. If a project is specified, the run must belong to that
2279
+ project; otherwise, all projects are queried.
2280
+
2281
+ :param run_id: The unique ID of the pipeline run to retry.
2282
+ :param namespace: Kubernetes namespace where the pipeline is running. Optional.
2283
+ :param timeout: Timeout (in seconds) for the API call. Defaults to 30 seconds.
2284
+ :param project: Name of the MLRun project associated with the pipeline. Can be
2285
+ ``*`` to query across all projects. Optional.
2286
+
2287
+ :raises ValueError: Raised if the API response is not successful or contains an
2288
+ error.
2289
+
2290
+ :return: JSON response containing details of the retried pipeline run.
2291
+ """
2292
+
2293
+ params = {}
2294
+ if namespace:
2295
+ params["namespace"] = namespace
2296
+ project_path = project if project else "*"
2297
+
2298
+ resp_text = ""
2299
+ resp_code = None
2300
+ try:
2301
+ resp = self.api_call(
2302
+ "POST",
2303
+ f"projects/{project_path}/pipelines/{run_id}/retry",
2304
+ params=params,
2305
+ timeout=timeout,
2306
+ )
2307
+ resp_code = resp.status_code
2308
+ resp_text = resp.text
2309
+ if not resp.ok:
2310
+ raise mlrun.errors.MLRunHTTPError(
2311
+ f"Failed to retry pipeline run '{run_id}'. "
2312
+ f"HTTP {resp_code}: {resp_text}"
2313
+ )
2314
+ except Exception as exc:
2315
+ logger.error(
2316
+ "Retry pipeline API call encountered an error.",
2317
+ run_id=run_id,
2318
+ project=project_path,
2319
+ namespace=namespace,
2320
+ response_code=resp_code,
2321
+ response_text=resp_text,
2322
+ error=str(exc),
2323
+ )
2324
+ if isinstance(exc, mlrun.errors.MLRunHTTPError):
2325
+ raise exc # Re-raise known HTTP errors
2326
+ raise mlrun.errors.MLRunRuntimeError(
2327
+ f"Unexpected error while retrying pipeline run '{run_id}'."
2328
+ ) from exc
2329
+
2330
+ logger.info(
2331
+ "Successfully retried pipeline run",
2332
+ run_id=run_id,
2333
+ project=project_path,
2334
+ namespace=namespace,
2335
+ )
2336
+ return resp.json()
2337
+
2018
2338
  @staticmethod
2019
2339
  def _resolve_reference(tag, uid):
2020
2340
  if uid and tag:
@@ -2061,7 +2381,11 @@ class HTTPRunDB(RunDBInterface):
2061
2381
  return resp.json()
2062
2382
 
2063
2383
  def get_feature_set(
2064
- self, name: str, project: str = "", tag: str = None, uid: str = None
2384
+ self,
2385
+ name: str,
2386
+ project: str = "",
2387
+ tag: Optional[str] = None,
2388
+ uid: Optional[str] = None,
2065
2389
  ) -> FeatureSet:
2066
2390
  """Retrieve a ~mlrun.feature_store.FeatureSet` object. If both ``tag`` and ``uid`` are not specified, then
2067
2391
  the object tagged ``latest`` will be retrieved.
@@ -2081,11 +2405,11 @@ class HTTPRunDB(RunDBInterface):
2081
2405
 
2082
2406
  def list_features(
2083
2407
  self,
2084
- project: str,
2085
- name: str = None,
2086
- tag: str = None,
2087
- entities: list[str] = None,
2088
- labels: list[str] = None,
2408
+ project: Optional[str] = None,
2409
+ name: Optional[str] = None,
2410
+ tag: Optional[str] = None,
2411
+ entities: Optional[list[str]] = None,
2412
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
2089
2413
  ) -> list[dict]:
2090
2414
  """List feature-sets which contain specific features. This function may return multiple versions of the same
2091
2415
  feature-set if a specific tag is not requested. Note that the various filters of this function actually
@@ -2096,18 +2420,25 @@ class HTTPRunDB(RunDBInterface):
2096
2420
  example, looking for ``feat`` will return features which are named ``MyFeature`` as well as ``defeat``.
2097
2421
  :param tag: Return feature-sets which contain the features looked for, and are tagged with the specific tag.
2098
2422
  :param entities: Return only feature-sets which contain an entity whose name is contained in this list.
2099
- :param labels: Return only feature-sets which are labeled as requested.
2423
+ :param labels: Filter feature-sets by label key-value pairs or key existence. This can be provided as:
2424
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
2425
+ or `{"label": None}` to check for key existence.
2426
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
2427
+ or just `"label"` for key existence.
2428
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
2429
+ the specified key-value pairs or key existence.
2100
2430
  :returns: A list of mapping from feature to a digest of the feature-set, which contains the feature-set
2101
2431
  meta-data. Multiple entries may be returned for any specific feature due to multiple tags or versions
2102
2432
  of the feature-set.
2103
2433
  """
2104
2434
 
2105
2435
  project = project or config.default_project
2436
+ labels = self._parse_labels(labels)
2106
2437
  params = {
2107
2438
  "name": name,
2108
2439
  "tag": tag,
2109
2440
  "entity": entities or [],
2110
- "label": labels or [],
2441
+ "label": labels,
2111
2442
  }
2112
2443
 
2113
2444
  path = f"projects/{project}/features"
@@ -2118,11 +2449,11 @@ class HTTPRunDB(RunDBInterface):
2118
2449
 
2119
2450
  def list_features_v2(
2120
2451
  self,
2121
- project: str,
2122
- name: str = None,
2123
- tag: str = None,
2124
- entities: list[str] = None,
2125
- labels: list[str] = None,
2452
+ project: Optional[str] = None,
2453
+ name: Optional[str] = None,
2454
+ tag: Optional[str] = None,
2455
+ entities: Optional[list[str]] = None,
2456
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
2126
2457
  ) -> dict[str, list[dict]]:
2127
2458
  """List feature-sets which contain specific features. This function may return multiple versions of the same
2128
2459
  feature-set if a specific tag is not requested. Note that the various filters of this function actually
@@ -2133,16 +2464,23 @@ class HTTPRunDB(RunDBInterface):
2133
2464
  example, looking for ``feat`` will return features which are named ``MyFeature`` as well as ``defeat``.
2134
2465
  :param tag: Return feature-sets which contain the features looked for, and are tagged with the specific tag.
2135
2466
  :param entities: Return only feature-sets which contain an entity whose name is contained in this list.
2136
- :param labels: Return only feature-sets which are labeled as requested.
2467
+ :param labels: Filter feature-sets by label key-value pairs or key existence. This can be provided as:
2468
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
2469
+ or `{"label": None}` to check for key existence.
2470
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
2471
+ or just `"label"` for key existence.
2472
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
2473
+ the specified key-value pairs or key existence.
2137
2474
  :returns: A list of features, and a list of their corresponding feature sets.
2138
2475
  """
2139
2476
 
2140
2477
  project = project or config.default_project
2478
+ labels = self._parse_labels(labels)
2141
2479
  params = {
2142
2480
  "name": name,
2143
2481
  "tag": tag,
2144
2482
  "entity": entities or [],
2145
- "label": labels or [],
2483
+ "label": labels,
2146
2484
  }
2147
2485
 
2148
2486
  path = f"projects/{project}/features"
@@ -2153,21 +2491,34 @@ class HTTPRunDB(RunDBInterface):
2153
2491
 
2154
2492
  def list_entities(
2155
2493
  self,
2156
- project: str,
2157
- name: str = None,
2158
- tag: str = None,
2159
- labels: list[str] = None,
2494
+ project: Optional[str] = None,
2495
+ name: Optional[str] = None,
2496
+ tag: Optional[str] = None,
2497
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
2160
2498
  ) -> list[dict]:
2161
2499
  """Retrieve a list of entities and their mapping to the containing feature-sets. This function is similar
2162
2500
  to the :py:func:`~list_features` function, and uses the same logic. However, the entities are matched
2163
2501
  against the name rather than the features.
2502
+
2503
+ :param project: The project containing the entities.
2504
+ :param name: The name of the entities to retrieve.
2505
+ :param tag: The tag of the specific entity version to retrieve.
2506
+ :param labels: Filter entities by label key-value pairs or key existence. This can be provided as:
2507
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
2508
+ or `{"label": None}` to check for key existence.
2509
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
2510
+ or just `"label"` for key existence.
2511
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
2512
+ the specified key-value pairs or key existence.
2513
+ :returns: A list of entities.
2164
2514
  """
2165
2515
 
2166
2516
  project = project or config.default_project
2517
+ labels = self._parse_labels(labels)
2167
2518
  params = {
2168
2519
  "name": name,
2169
2520
  "tag": tag,
2170
- "label": labels or [],
2521
+ "label": labels,
2171
2522
  }
2172
2523
 
2173
2524
  path = f"projects/{project}/entities"
@@ -2178,21 +2529,34 @@ class HTTPRunDB(RunDBInterface):
2178
2529
 
2179
2530
  def list_entities_v2(
2180
2531
  self,
2181
- project: str,
2182
- name: str = None,
2183
- tag: str = None,
2184
- labels: list[str] = None,
2532
+ project: Optional[str] = None,
2533
+ name: Optional[str] = None,
2534
+ tag: Optional[str] = None,
2535
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
2185
2536
  ) -> dict[str, list[dict]]:
2186
2537
  """Retrieve a list of entities and their mapping to the containing feature-sets. This function is similar
2187
2538
  to the :py:func:`~list_features_v2` function, and uses the same logic. However, the entities are matched
2188
2539
  against the name rather than the features.
2540
+
2541
+ :param project: The project containing the entities.
2542
+ :param name: The name of the entities to retrieve.
2543
+ :param tag: The tag of the specific entity version to retrieve.
2544
+ :param labels: Filter entities by label key-value pairs or key existence. This can be provided as:
2545
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
2546
+ or `{"label": None}` to check for key existence.
2547
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
2548
+ or just `"label"` for key existence.
2549
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
2550
+ the specified key-value pairs or key existence.
2551
+ :returns: A list of entities.
2189
2552
  """
2190
2553
 
2191
2554
  project = project or config.default_project
2555
+ labels = self._parse_labels(labels)
2192
2556
  params = {
2193
2557
  "name": name,
2194
2558
  "tag": tag,
2195
- "label": labels or [],
2559
+ "label": labels,
2196
2560
  }
2197
2561
 
2198
2562
  path = f"projects/{project}/entities"
@@ -2203,7 +2567,6 @@ class HTTPRunDB(RunDBInterface):
2203
2567
 
2204
2568
  @staticmethod
2205
2569
  def _generate_partition_by_params(
2206
- partition_by_cls,
2207
2570
  partition_by,
2208
2571
  rows_per_partition,
2209
2572
  sort_by,
@@ -2222,16 +2585,16 @@ class HTTPRunDB(RunDBInterface):
2222
2585
 
2223
2586
  def list_feature_sets(
2224
2587
  self,
2225
- project: str = "",
2226
- name: str = None,
2227
- tag: str = None,
2228
- state: str = None,
2229
- entities: list[str] = None,
2230
- features: list[str] = None,
2231
- labels: list[str] = None,
2232
- partition_by: Union[
2233
- mlrun.common.schemas.FeatureStorePartitionByField, str
2234
- ] = None,
2588
+ project: Optional[str] = None,
2589
+ name: Optional[str] = None,
2590
+ tag: Optional[str] = None,
2591
+ state: Optional[str] = None,
2592
+ entities: Optional[list[str]] = None,
2593
+ features: Optional[list[str]] = None,
2594
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
2595
+ partition_by: Union[
2596
+ mlrun.common.schemas.FeatureStorePartitionByField, str
2597
+ ] = None,
2235
2598
  rows_per_partition: int = 1,
2236
2599
  partition_sort_by: Union[mlrun.common.schemas.SortField, str] = None,
2237
2600
  partition_order: Union[
@@ -2249,7 +2612,13 @@ class HTTPRunDB(RunDBInterface):
2249
2612
  :param state: Match feature-sets with a specific state.
2250
2613
  :param entities: Match feature-sets which contain entities whose name is in this list.
2251
2614
  :param features: Match feature-sets which contain features whose name is in this list.
2252
- :param labels: Match feature-sets which have these labels.
2615
+ :param labels: Filter feature-sets by label key-value pairs or key existence. This can be provided as:
2616
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
2617
+ or `{"label": None}` to check for key existence.
2618
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
2619
+ or just `"label"` for key existence.
2620
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
2621
+ the specified key-value pairs or key existence.
2253
2622
  :param partition_by: Field to group results by. Only allowed value is `name`. When `partition_by` is specified,
2254
2623
  the `partition_sort_by` parameter must be provided as well.
2255
2624
  :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
@@ -2264,20 +2633,19 @@ class HTTPRunDB(RunDBInterface):
2264
2633
  """
2265
2634
 
2266
2635
  project = project or config.default_project
2267
-
2636
+ labels = self._parse_labels(labels)
2268
2637
  params = {
2269
2638
  "name": name,
2270
2639
  "state": state,
2271
2640
  "tag": tag,
2272
2641
  "entity": entities or [],
2273
2642
  "feature": features or [],
2274
- "label": labels or [],
2643
+ "label": labels,
2275
2644
  "format": format_,
2276
2645
  }
2277
2646
  if partition_by:
2278
2647
  params.update(
2279
2648
  self._generate_partition_by_params(
2280
- mlrun.common.schemas.FeatureStorePartitionByField,
2281
2649
  partition_by,
2282
2650
  rows_per_partition,
2283
2651
  partition_sort_by,
@@ -2436,7 +2804,11 @@ class HTTPRunDB(RunDBInterface):
2436
2804
  return resp.json()
2437
2805
 
2438
2806
  def get_feature_vector(
2439
- self, name: str, project: str = "", tag: str = None, uid: str = None
2807
+ self,
2808
+ name: str,
2809
+ project: str = "",
2810
+ tag: Optional[str] = None,
2811
+ uid: Optional[str] = None,
2440
2812
  ) -> FeatureVector:
2441
2813
  """Return a specific feature-vector referenced by its tag or uid. If none are provided, ``latest`` tag will
2442
2814
  be used."""
@@ -2450,11 +2822,11 @@ class HTTPRunDB(RunDBInterface):
2450
2822
 
2451
2823
  def list_feature_vectors(
2452
2824
  self,
2453
- project: str = "",
2454
- name: str = None,
2455
- tag: str = None,
2456
- state: str = None,
2457
- labels: list[str] = None,
2825
+ project: Optional[str] = None,
2826
+ name: Optional[str] = None,
2827
+ tag: Optional[str] = None,
2828
+ state: Optional[str] = None,
2829
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
2458
2830
  partition_by: Union[
2459
2831
  mlrun.common.schemas.FeatureStorePartitionByField, str
2460
2832
  ] = None,
@@ -2470,7 +2842,13 @@ class HTTPRunDB(RunDBInterface):
2470
2842
  :param name: Name of feature-vector to match. This is a like query, and is case-insensitive.
2471
2843
  :param tag: Match feature-vectors with specific tag.
2472
2844
  :param state: Match feature-vectors with a specific state.
2473
- :param labels: Match feature-vectors which have these labels.
2845
+ :param labels: Filter feature-vectors by label key-value pairs or key existence. This can be provided as:
2846
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
2847
+ or `{"label": None}` to check for key existence.
2848
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
2849
+ or just `"label"` for key existence.
2850
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
2851
+ the specified key-value pairs or key existence.
2474
2852
  :param partition_by: Field to group results by. Only allowed value is `name`. When `partition_by` is specified,
2475
2853
  the `partition_sort_by` parameter must be provided as well.
2476
2854
  :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
@@ -2482,17 +2860,16 @@ class HTTPRunDB(RunDBInterface):
2482
2860
  """
2483
2861
 
2484
2862
  project = project or config.default_project
2485
-
2863
+ labels = self._parse_labels(labels)
2486
2864
  params = {
2487
2865
  "name": name,
2488
2866
  "state": state,
2489
2867
  "tag": tag,
2490
- "label": labels or [],
2868
+ "label": labels,
2491
2869
  }
2492
2870
  if partition_by:
2493
2871
  params.update(
2494
2872
  self._generate_partition_by_params(
2495
- mlrun.common.schemas.FeatureStorePartitionByField,
2496
2873
  partition_by,
2497
2874
  rows_per_partition,
2498
2875
  partition_sort_by,
@@ -2699,11 +3076,11 @@ class HTTPRunDB(RunDBInterface):
2699
3076
 
2700
3077
  def list_projects(
2701
3078
  self,
2702
- owner: str = None,
3079
+ owner: Optional[str] = None,
2703
3080
  format_: Union[
2704
3081
  str, mlrun.common.formatters.ProjectFormat
2705
3082
  ] = mlrun.common.formatters.ProjectFormat.name_only,
2706
- labels: list[str] = None,
3083
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
2707
3084
  state: Union[str, mlrun.common.schemas.ProjectState] = None,
2708
3085
  ) -> list[Union[mlrun.projects.MlrunProject, str]]:
2709
3086
  """Return a list of the existing projects, potentially filtered by specific criteria.
@@ -2715,15 +3092,22 @@ class HTTPRunDB(RunDBInterface):
2715
3092
  - ``minimal`` - Return minimal project objects (minimization happens in the BE).
2716
3093
  - ``full`` - Return full project objects.
2717
3094
 
2718
- :param labels: Filter by labels attached to the project.
3095
+ :param labels: Filter projects by label key-value pairs or key existence. This can be provided as:
3096
+ - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
3097
+ or `{"label": None}` to check for key existence.
3098
+ - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
3099
+ or just `"label"` for key existence.
3100
+ - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
3101
+ the specified key-value pairs or key existence.
2719
3102
  :param state: Filter by project's state. Can be either ``online`` or ``archived``.
2720
3103
  """
3104
+ labels = self._parse_labels(labels)
2721
3105
 
2722
3106
  params = {
2723
3107
  "owner": owner,
2724
3108
  "state": state,
2725
3109
  "format": format_,
2726
- "label": labels or [],
3110
+ "label": labels,
2727
3111
  }
2728
3112
 
2729
3113
  error_message = f"Failed listing projects, query: {params}"
@@ -2739,7 +3123,7 @@ class HTTPRunDB(RunDBInterface):
2739
3123
  for project_dict in response.json()["projects"]
2740
3124
  ]
2741
3125
 
2742
- def get_project(self, name: str) -> mlrun.projects.MlrunProject:
3126
+ def get_project(self, name: str) -> "mlrun.MlrunProject":
2743
3127
  """Get details for a specific project."""
2744
3128
 
2745
3129
  if not name:
@@ -2748,7 +3132,7 @@ class HTTPRunDB(RunDBInterface):
2748
3132
  path = f"projects/{name}"
2749
3133
  error_message = f"Failed retrieving project {name}"
2750
3134
  response = self.api_call("GET", path, error_message)
2751
- return mlrun.projects.MlrunProject.from_dict(response.json())
3135
+ return mlrun.MlrunProject.from_dict(response.json())
2752
3136
 
2753
3137
  def delete_project(
2754
3138
  self,
@@ -2919,7 +3303,7 @@ class HTTPRunDB(RunDBInterface):
2919
3303
  provider: Union[
2920
3304
  str, mlrun.common.schemas.SecretProviderName
2921
3305
  ] = mlrun.common.schemas.SecretProviderName.kubernetes,
2922
- secrets: dict = None,
3306
+ secrets: Optional[dict] = None,
2923
3307
  ):
2924
3308
  """Create project-context secrets using either ``vault`` or ``kubernetes`` provider.
2925
3309
  When using with Vault, this will create needed Vault structures for storing secrets in project-context, and
@@ -2963,11 +3347,11 @@ class HTTPRunDB(RunDBInterface):
2963
3347
  def list_project_secrets(
2964
3348
  self,
2965
3349
  project: str,
2966
- token: str = None,
3350
+ token: Optional[str] = None,
2967
3351
  provider: Union[
2968
3352
  str, mlrun.common.schemas.SecretProviderName
2969
3353
  ] = mlrun.common.schemas.SecretProviderName.kubernetes,
2970
- secrets: list[str] = None,
3354
+ secrets: Optional[list[str]] = None,
2971
3355
  ) -> mlrun.common.schemas.SecretsData:
2972
3356
  """Retrieve project-context secrets from Vault.
2973
3357
 
@@ -3010,7 +3394,7 @@ class HTTPRunDB(RunDBInterface):
3010
3394
  provider: Union[
3011
3395
  str, mlrun.common.schemas.SecretProviderName
3012
3396
  ] = mlrun.common.schemas.SecretProviderName.kubernetes,
3013
- token: str = None,
3397
+ token: Optional[str] = None,
3014
3398
  ) -> mlrun.common.schemas.SecretKeysData:
3015
3399
  """Retrieve project-context secret keys from Vault or Kubernetes.
3016
3400
 
@@ -3056,7 +3440,7 @@ class HTTPRunDB(RunDBInterface):
3056
3440
  provider: Union[
3057
3441
  str, mlrun.common.schemas.SecretProviderName
3058
3442
  ] = mlrun.common.schemas.SecretProviderName.kubernetes,
3059
- secrets: list[str] = None,
3443
+ secrets: Optional[list[str]] = None,
3060
3444
  ):
3061
3445
  """Delete project-context secrets from Kubernetes.
3062
3446
 
@@ -3076,13 +3460,44 @@ class HTTPRunDB(RunDBInterface):
3076
3460
  params=params,
3077
3461
  )
3078
3462
 
3463
+ def get_model_endpoint_monitoring_metrics(
3464
+ self,
3465
+ project: str,
3466
+ endpoint_id: str,
3467
+ type: Literal["results", "metrics", "all"] = "all",
3468
+ ) -> list[mm_endpoints.ModelEndpointMonitoringMetric]:
3469
+ """Get application metrics/results by endpoint id and project.
3470
+
3471
+ :param project: The name of the project.
3472
+ :param endpoint_id: The unique id of the model endpoint.
3473
+ :param type: The type of the metrics to return. "all" means "results" and "metrics".
3474
+
3475
+ :return: A list of the application metrics or/and results for this model endpoint.
3476
+ """
3477
+ path = f"projects/{project}/model-endpoints/{endpoint_id}/metrics"
3478
+ params = {"type": type}
3479
+ error_message = (
3480
+ f"Failed to get model endpoint monitoring metrics,"
3481
+ f" endpoint_id: {endpoint_id}, project: {project}"
3482
+ )
3483
+ response = self.api_call(
3484
+ mlrun.common.types.HTTPMethod.GET,
3485
+ path,
3486
+ error_message,
3487
+ params=params,
3488
+ )
3489
+ monitoring_metrics = response.json()
3490
+ return parse_obj_as(
3491
+ list[mm_endpoints.ModelEndpointMonitoringMetric], monitoring_metrics
3492
+ )
3493
+
3079
3494
  def create_user_secrets(
3080
3495
  self,
3081
3496
  user: str,
3082
3497
  provider: Union[
3083
3498
  str, mlrun.common.schemas.SecretProviderName
3084
3499
  ] = mlrun.common.schemas.SecretProviderName.vault,
3085
- secrets: dict = None,
3500
+ secrets: Optional[dict] = None,
3086
3501
  ):
3087
3502
  """Create user-context secret in Vault. Please refer to :py:func:`create_project_secrets` for more details
3088
3503
  and status of this functionality.
@@ -3164,212 +3579,180 @@ class HTTPRunDB(RunDBInterface):
3164
3579
 
3165
3580
  def create_model_endpoint(
3166
3581
  self,
3167
- project: str,
3168
- endpoint_id: str,
3169
- model_endpoint: Union[
3170
- mlrun.model_monitoring.model_endpoint.ModelEndpoint, dict
3171
- ],
3172
- ):
3582
+ model_endpoint: mlrun.common.schemas.ModelEndpoint,
3583
+ ) -> mlrun.common.schemas.ModelEndpoint:
3173
3584
  """
3174
3585
  Creates a DB record with the given model_endpoint record.
3175
3586
 
3176
- :param project: The name of the project.
3177
- :param endpoint_id: The id of the endpoint.
3178
3587
  :param model_endpoint: An object representing the model endpoint.
3179
- """
3180
3588
 
3181
- if isinstance(
3182
- model_endpoint, mlrun.model_monitoring.model_endpoint.ModelEndpoint
3183
- ):
3184
- model_endpoint = model_endpoint.to_dict()
3589
+ :return: The created model endpoint object.
3590
+ """
3185
3591
 
3186
- path = f"projects/{project}/model-endpoints/{endpoint_id}"
3187
- self.api_call(
3188
- method="POST",
3592
+ path = f"projects/{model_endpoint.metadata.project}/model-endpoints"
3593
+ response = self.api_call(
3594
+ method=mlrun.common.types.HTTPMethod.POST,
3189
3595
  path=path,
3190
- body=dict_to_json(model_endpoint),
3596
+ body=model_endpoint.json(),
3191
3597
  )
3598
+ return mlrun.common.schemas.ModelEndpoint(**response.json())
3192
3599
 
3193
3600
  def delete_model_endpoint(
3194
3601
  self,
3602
+ name: str,
3195
3603
  project: str,
3604
+ function_name: str,
3196
3605
  endpoint_id: str,
3197
3606
  ):
3198
3607
  """
3199
3608
  Deletes the DB record of a given model endpoint, project and endpoint_id are used for lookup
3200
3609
 
3610
+ :param name: The name of the model endpoint
3201
3611
  :param project: The name of the project
3612
+ :param function_name: The name of the function
3202
3613
  :param endpoint_id: The id of the endpoint
3203
3614
  """
3204
3615
 
3205
- path = f"projects/{project}/model-endpoints/{endpoint_id}"
3616
+ path = f"projects/{project}/model-endpoints/{name}"
3206
3617
  self.api_call(
3207
- method="DELETE",
3618
+ method=mlrun.common.types.HTTPMethod.DELETE,
3208
3619
  path=path,
3620
+ params={
3621
+ "function_name": function_name,
3622
+ "endpoint_id": endpoint_id,
3623
+ },
3209
3624
  )
3210
3625
 
3211
3626
  def list_model_endpoints(
3212
3627
  self,
3213
3628
  project: str,
3214
- model: Optional[str] = None,
3215
- function: Optional[str] = None,
3216
- labels: list[str] = None,
3217
- start: str = "now-1h",
3218
- end: str = "now",
3219
- metrics: Optional[list[str]] = None,
3629
+ name: Optional[str] = None,
3630
+ function_name: Optional[str] = None,
3631
+ model_name: Optional[str] = None,
3632
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
3633
+ start: Optional[datetime] = None,
3634
+ end: Optional[datetime] = None,
3635
+ tsdb_metrics: bool = True,
3220
3636
  top_level: bool = False,
3221
3637
  uids: Optional[list[str]] = None,
3222
- ) -> list[mlrun.model_monitoring.model_endpoint.ModelEndpoint]:
3638
+ latest_only: bool = False,
3639
+ ) -> mlrun.common.schemas.ModelEndpointList:
3640
+ """
3641
+ List model endpoints with optional filtering by name, function name, model name, labels, and time range.
3642
+
3643
+ :param project: The name of the project
3644
+ :param name: The name of the model endpoint
3645
+ :param function_name: The name of the function
3646
+ :param model_name: The name of the model
3647
+ :param labels: A list of labels to filter by. (see mlrun.common.schemas.LabelsModel)
3648
+ :param start: The start time to filter by.Corresponding to the `created` field.
3649
+ :param end: The end time to filter by. Corresponding to the `created` field.
3650
+ :param tsdb_metrics: Whether to include metrics from the time series DB.
3651
+ :param top_level: Whether to return only top level model endpoints.
3652
+ :param uids: A list of unique ids to filter by.
3653
+ :param latest_only: Whether to return only the latest model endpoint version.
3654
+ :return: A list of model endpoints.
3223
3655
  """
3224
- Returns a list of `ModelEndpoint` objects. Each `ModelEndpoint` object represents the current state of a
3225
- model endpoint. This functions supports filtering by the following parameters:
3226
- 1) model
3227
- 2) function
3228
- 3) labels
3229
- 4) top level
3230
- 5) uids
3231
- By default, when no filters are applied, all available endpoints for the given project will be listed.
3232
-
3233
- In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
3234
- and depends on the 'start' and 'end' parameters. By default, when the metrics parameter is None, no metrics are
3235
- added to the output of this function.
3236
-
3237
- :param project: The name of the project
3238
- :param model: The name of the model to filter by
3239
- :param function: The name of the function to filter by
3240
- :param labels: A list of labels to filter by. Label filters work by either filtering a specific value of a
3241
- label (i.e. list("key=value")) or by looking for the existence of a given key (i.e. "key")
3242
- :param metrics: A list of metrics to return for each endpoint, read more in 'TimeMetric'
3243
- :param start: The start time of the metrics. Can be represented by a string containing an RFC 3339 time, a
3244
- Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
3245
- `m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
3246
- :param end: The end time of the metrics. Can be represented by a string containing an RFC 3339 time, a
3247
- Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
3248
- `m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
3249
- :param top_level: if true will return only routers and endpoint that are NOT children of any router
3250
- :param uids: if passed will return a list `ModelEndpoint` object with uid in uids
3251
- """
3252
-
3253
3656
  path = f"projects/{project}/model-endpoints"
3254
-
3255
- if labels and isinstance(labels, dict):
3256
- labels = [f"{key}={value}" for key, value in labels.items()]
3657
+ labels = self._parse_labels(labels)
3257
3658
 
3258
3659
  response = self.api_call(
3259
- method="GET",
3660
+ method=mlrun.common.types.HTTPMethod.GET,
3260
3661
  path=path,
3261
3662
  params={
3262
- "model": model,
3263
- "function": function,
3264
- "label": labels or [],
3265
- "start": start,
3266
- "end": end,
3267
- "metric": metrics or [],
3663
+ "name": name,
3664
+ "model_name": model_name,
3665
+ "function_name": function_name,
3666
+ "label": labels,
3667
+ "start": datetime_to_iso(start),
3668
+ "end": datetime_to_iso(end),
3669
+ "tsdb_metrics": tsdb_metrics,
3268
3670
  "top-level": top_level,
3269
3671
  "uid": uids,
3672
+ "latest_only": latest_only,
3270
3673
  },
3271
3674
  )
3272
3675
 
3273
- # Generate a list of a model endpoint dictionaries
3274
- model_endpoints = response.json()["endpoints"]
3275
- if model_endpoints:
3276
- return [
3277
- mlrun.model_monitoring.model_endpoint.ModelEndpoint.from_dict(obj)
3278
- for obj in model_endpoints
3279
- ]
3280
- return []
3676
+ return mlrun.common.schemas.ModelEndpointList(**response.json())
3281
3677
 
3282
3678
  def get_model_endpoint(
3283
3679
  self,
3680
+ name: str,
3284
3681
  project: str,
3285
- endpoint_id: str,
3286
- start: Optional[str] = None,
3287
- end: Optional[str] = None,
3288
- metrics: Optional[list[str]] = None,
3682
+ function_name: Optional[str] = None,
3683
+ # TODO: function_tag
3684
+ endpoint_id: Optional[str] = None,
3685
+ tsdb_metrics: bool = True,
3289
3686
  feature_analysis: bool = False,
3290
- ) -> mlrun.model_monitoring.model_endpoint.ModelEndpoint:
3687
+ ) -> mlrun.common.schemas.ModelEndpoint:
3291
3688
  """
3292
3689
  Returns a single `ModelEndpoint` object with additional metrics and feature related data.
3293
3690
 
3691
+ :param name: The name of the model endpoint
3294
3692
  :param project: The name of the project
3295
- :param endpoint_id: The unique id of the model endpoint.
3296
- :param start: The start time of the metrics. Can be represented by a string containing an
3297
- RFC 3339 time, a Unix timestamp in milliseconds, a relative time
3298
- (`'now'` or `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours,
3299
- `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
3300
- :param end: The end time of the metrics. Can be represented by a string containing an
3301
- RFC 3339 time, a Unix timestamp in milliseconds, a relative time
3302
- (`'now'` or `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours,
3303
- `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
3304
- :param metrics: A list of metrics to return for the model endpoint. There are pre-defined
3305
- metrics for model endpoints such as predictions_per_second and
3306
- latency_avg_5m but also custom metrics defined by the user. Please note that
3307
- these metrics are stored in the time series DB and the results will be
3308
- appeared under model_endpoint.spec.metrics.
3309
- :param feature_analysis: When True, the base feature statistics and current feature statistics will
3310
- be added to the output of the resulting object.
3311
-
3312
- :returns: A `ModelEndpoint` object.
3313
- """
3314
-
3315
- path = f"projects/{project}/model-endpoints/{endpoint_id}"
3693
+ :param function_name: The name of the function
3694
+ :param endpoint_id: The id of the endpoint
3695
+ :param tsdb_metrics: Whether to include metrics from the time series DB.
3696
+ :param feature_analysis: Whether to include feature analysis data (feature_stats,
3697
+ current_stats & drift_measures).
3698
+
3699
+ :return: A `ModelEndpoint` object.
3700
+ """
3701
+
3702
+ path = f"projects/{project}/model-endpoints/{name}"
3316
3703
  response = self.api_call(
3317
- method="GET",
3704
+ method=mlrun.common.types.HTTPMethod.GET,
3318
3705
  path=path,
3319
3706
  params={
3320
- "start": start,
3321
- "end": end,
3322
- "metric": metrics or [],
3707
+ "function_name": function_name,
3708
+ "endpoint_id": endpoint_id,
3709
+ "tsdb_metrics": tsdb_metrics,
3323
3710
  "feature_analysis": feature_analysis,
3324
3711
  },
3325
3712
  )
3326
3713
 
3327
- return mlrun.model_monitoring.model_endpoint.ModelEndpoint.from_dict(
3328
- response.json()
3329
- )
3714
+ return mlrun.common.schemas.ModelEndpoint(**response.json())
3330
3715
 
3331
3716
  def patch_model_endpoint(
3332
3717
  self,
3718
+ name: str,
3333
3719
  project: str,
3334
- endpoint_id: str,
3335
3720
  attributes: dict,
3336
- ):
3337
- """
3338
- Updates model endpoint with provided attributes.
3339
-
3340
- :param project: The name of the project.
3341
- :param endpoint_id: The id of the endpoint.
3342
- :param attributes: Dictionary of attributes that will be used for update the model endpoint. The keys
3343
- of this dictionary should exist in the target table. Note that the values should be from type string or from
3344
- a valid numerical type such as int or float. More details about the model endpoint available attributes can
3345
- be found under :py:class:`~mlrun.common.schemas.ModelEndpoint`.
3346
-
3347
- Example::
3348
-
3349
- # Generate current stats for two features
3350
- current_stats = {'tvd_sum': 2.2,
3351
- 'tvd_mean': 0.5,
3352
- 'hellinger_sum': 3.6,
3353
- 'hellinger_mean': 0.9,
3354
- 'kld_sum': 24.2,
3355
- 'kld_mean': 6.0,
3356
- 'f1': {'tvd': 0.5, 'hellinger': 1.0, 'kld': 6.4},
3357
- 'f2': {'tvd': 0.5, 'hellinger': 1.0, 'kld': 6.5}}
3358
-
3359
- # Create attributes dictionary according to the required format
3360
- attributes = {`current_stats`: json.dumps(current_stats),
3361
- `drift_status`: "DRIFT_DETECTED"}
3362
-
3721
+ function_name: Optional[str] = None,
3722
+ endpoint_id: Optional[str] = None,
3723
+ ) -> mlrun.common.schemas.ModelEndpoint:
3363
3724
  """
3725
+ Updates a model endpoint with the given attributes.
3364
3726
 
3365
- attributes = {"attributes": _as_json(attributes)}
3366
- path = f"projects/{project}/model-endpoints/{endpoint_id}"
3367
- self.api_call(
3368
- method="PATCH",
3727
+ :param name: The name of the model endpoint
3728
+ :param project: The name of the project
3729
+ :param attributes: The attributes to update
3730
+ :param function_name: The name of the function
3731
+ :param endpoint_id: The id of the endpoint
3732
+ :return: The updated `ModelEndpoint` object.
3733
+ """
3734
+ attributes_keys = list(attributes.keys())
3735
+ attributes["name"] = name
3736
+ attributes["project"] = project
3737
+ attributes["uid"] = endpoint_id or ""
3738
+ model_endpoint = mlrun.common.schemas.ModelEndpoint.from_flat_dict(attributes)
3739
+ path = f"projects/{project}/model-endpoints"
3740
+ logger.info(
3741
+ "Patching model endpoint",
3742
+ attributes_keys=attributes_keys,
3743
+ model_endpoint=model_endpoint,
3744
+ )
3745
+ response = self.api_call(
3746
+ method=mlrun.common.types.HTTPMethod.PATCH,
3369
3747
  path=path,
3370
- params=attributes,
3748
+ params={
3749
+ "attribute-key": attributes_keys,
3750
+ },
3751
+ body=model_endpoint.json(),
3371
3752
  )
3372
3753
 
3754
+ return mlrun.common.schemas.ModelEndpoint(**response.json())
3755
+
3373
3756
  def update_model_monitoring_controller(
3374
3757
  self,
3375
3758
  project: str,
@@ -3441,7 +3824,7 @@ class HTTPRunDB(RunDBInterface):
3441
3824
  delete_stream_function: bool = False,
3442
3825
  delete_histogram_data_drift_app: bool = True,
3443
3826
  delete_user_applications: bool = False,
3444
- user_application_list: list[str] = None,
3827
+ user_application_list: Optional[list[str]] = None,
3445
3828
  ) -> bool:
3446
3829
  """
3447
3830
  Disable model monitoring application controller, writer, stream, histogram data drift application
@@ -3714,8 +4097,8 @@ class HTTPRunDB(RunDBInterface):
3714
4097
  def get_hub_catalog(
3715
4098
  self,
3716
4099
  source_name: str,
3717
- version: str = None,
3718
- tag: str = None,
4100
+ version: Optional[str] = None,
4101
+ tag: Optional[str] = None,
3719
4102
  force_refresh: bool = False,
3720
4103
  ):
3721
4104
  """
@@ -3745,7 +4128,7 @@ class HTTPRunDB(RunDBInterface):
3745
4128
  self,
3746
4129
  source_name: str,
3747
4130
  item_name: str,
3748
- version: str = None,
4131
+ version: Optional[str] = None,
3749
4132
  tag: str = "latest",
3750
4133
  force_refresh: bool = False,
3751
4134
  ):
@@ -3775,7 +4158,7 @@ class HTTPRunDB(RunDBInterface):
3775
4158
  source_name: str,
3776
4159
  item_name: str,
3777
4160
  asset_name: str,
3778
- version: str = None,
4161
+ version: Optional[str] = None,
3779
4162
  tag: str = "latest",
3780
4163
  ):
3781
4164
  """
@@ -3897,18 +4280,27 @@ class HTTPRunDB(RunDBInterface):
3897
4280
  "operations/migrations",
3898
4281
  "Failed triggering migrations",
3899
4282
  )
3900
- if response.status_code == http.HTTPStatus.ACCEPTED:
3901
- background_task = mlrun.common.schemas.BackgroundTask(**response.json())
3902
- return self._wait_for_background_task_to_reach_terminal_state(
3903
- background_task.metadata.name
3904
- )
3905
- return None
4283
+ return self._wait_for_background_task_from_response(response)
4284
+
4285
+ def refresh_smtp_configuration(
4286
+ self,
4287
+ ) -> Optional[mlrun.common.schemas.BackgroundTask]:
4288
+ """Refresh smtp configuration and wait for the task to finish
4289
+
4290
+ :returns: :py:class:`~mlrun.common.schemas.BackgroundTask`.
4291
+ """
4292
+ response = self.api_call(
4293
+ "POST",
4294
+ "operations/refresh-smtp-configuration",
4295
+ "Failed refreshing smtp configuration",
4296
+ )
4297
+ return self._wait_for_background_task_from_response(response)
3906
4298
 
3907
4299
  def set_run_notifications(
3908
4300
  self,
3909
4301
  project: str,
3910
4302
  run_uid: str,
3911
- notifications: list[mlrun.model.Notification] = None,
4303
+ notifications: Optional[list[mlrun.model.Notification]] = None,
3912
4304
  ):
3913
4305
  """
3914
4306
  Set notifications on a run. This will override any existing notifications on the run.
@@ -3934,7 +4326,7 @@ class HTTPRunDB(RunDBInterface):
3934
4326
  self,
3935
4327
  project: str,
3936
4328
  schedule_name: str,
3937
- notifications: list[mlrun.model.Notification] = None,
4329
+ notifications: Optional[list[mlrun.model.Notification]] = None,
3938
4330
  ):
3939
4331
  """
3940
4332
  Set notifications on a schedule. This will override any existing notifications on the schedule.
@@ -3960,7 +4352,7 @@ class HTTPRunDB(RunDBInterface):
3960
4352
  self,
3961
4353
  notification_objects: list[mlrun.model.Notification],
3962
4354
  run_uid: str,
3963
- project: str = None,
4355
+ project: Optional[str] = None,
3964
4356
  mask_params: bool = True,
3965
4357
  ):
3966
4358
  """
@@ -3994,7 +4386,7 @@ class HTTPRunDB(RunDBInterface):
3994
4386
  source: Optional[str] = None,
3995
4387
  run_name: Optional[str] = None,
3996
4388
  namespace: Optional[str] = None,
3997
- notifications: list[mlrun.model.Notification] = None,
4389
+ notifications: Optional[list[mlrun.model.Notification]] = None,
3998
4390
  ) -> mlrun.common.schemas.WorkflowResponse:
3999
4391
  """
4000
4392
  Submitting workflow for a remote execution.
@@ -4216,6 +4608,7 @@ class HTTPRunDB(RunDBInterface):
4216
4608
  alert_name: str,
4217
4609
  alert_data: Union[dict, AlertConfig],
4218
4610
  project="",
4611
+ force_reset: bool = False,
4219
4612
  ) -> AlertConfig:
4220
4613
  """
4221
4614
  Create/modify an alert.
@@ -4223,6 +4616,7 @@ class HTTPRunDB(RunDBInterface):
4223
4616
  :param alert_name: The name of the alert.
4224
4617
  :param alert_data: The data of the alert.
4225
4618
  :param project: The project that the alert belongs to.
4619
+ :param force_reset: If True and the alert already exists, the alert would be reset.
4226
4620
  :returns: The created/modified alert.
4227
4621
  """
4228
4622
  if not alert_data:
@@ -4247,7 +4641,10 @@ class HTTPRunDB(RunDBInterface):
4247
4641
 
4248
4642
  alert_data = alert_instance.to_dict()
4249
4643
  body = _as_json(alert_data)
4250
- response = self.api_call("PUT", endpoint_path, error_message, body=body)
4644
+ params = {"force_reset": bool2str(force_reset)} if force_reset else {}
4645
+ response = self.api_call(
4646
+ "PUT", endpoint_path, error_message, params=params, body=body
4647
+ )
4251
4648
  return AlertConfig.from_dict(response.json())
4252
4649
 
4253
4650
  def get_alert_config(self, alert_name: str, project="") -> AlertConfig:
@@ -4334,6 +4731,460 @@ class HTTPRunDB(RunDBInterface):
4334
4731
  results.append(mlrun.common.schemas.AlertTemplate(**item))
4335
4732
  return results
4336
4733
 
4734
+ def list_alert_activations(
4735
+ self,
4736
+ project: Optional[str] = None,
4737
+ name: Optional[str] = None,
4738
+ since: Optional[datetime] = None,
4739
+ until: Optional[datetime] = None,
4740
+ entity: Optional[str] = None,
4741
+ severity: Optional[
4742
+ list[Union[mlrun.common.schemas.alert.AlertSeverity, str]]
4743
+ ] = None,
4744
+ entity_kind: Optional[
4745
+ Union[mlrun.common.schemas.alert.EventEntityKind, str]
4746
+ ] = None,
4747
+ event_kind: Optional[Union[mlrun.common.schemas.alert.EventKind, str]] = None,
4748
+ ) -> mlrun.common.schemas.AlertActivations:
4749
+ """
4750
+ Retrieve a list of all alert activations.
4751
+
4752
+ :param project: The project name to filter by. If None, results are not filtered by project.
4753
+ :param name: The alert name to filter by. Supports exact matching or partial matching if prefixed with `~`.
4754
+ :param since: Filters for alert activations occurring after this timestamp.
4755
+ :param until: Filters for alert activations occurring before this timestamp.
4756
+ :param entity: The entity ID to filter by. Supports wildcard matching if prefixed with `~`.
4757
+ :param severity: A list of severity levels to filter by (e.g., ["high", "low"]).
4758
+ :param entity_kind: The kind of entity (e.g., "job", "endpoint") to filter by.
4759
+ :param event_kind: The kind of event (e.g., ""data-drift-detected"", "failed") to filter by.
4760
+
4761
+ :returns: A list of alert activations matching the provided filters.
4762
+ """
4763
+
4764
+ alert_activations, _ = self._list_alert_activations(
4765
+ project=project,
4766
+ name=name,
4767
+ since=since,
4768
+ until=until,
4769
+ entity=entity,
4770
+ severity=severity,
4771
+ entity_kind=entity_kind,
4772
+ event_kind=event_kind,
4773
+ return_all=True,
4774
+ )
4775
+ return alert_activations
4776
+
4777
+ def paginated_list_alert_activations(
4778
+ self,
4779
+ *args,
4780
+ page: Optional[int] = None,
4781
+ page_size: Optional[int] = None,
4782
+ page_token: Optional[str] = None,
4783
+ **kwargs,
4784
+ ) -> tuple[AlertActivations, Optional[str]]:
4785
+ """List alerts activations with support for pagination and various filtering options.
4786
+
4787
+ This method retrieves a paginated list of alert activations based on the specified filter parameters.
4788
+ Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
4789
+ will return a list of alert activations that match the filtering criteria provided.
4790
+
4791
+ For detailed information about the parameters, refer to the list_alert_activations method:
4792
+ See :py:func:`~list_alert_activations` for more details.
4793
+
4794
+ Examples::
4795
+
4796
+ # Fetch first page of alert activations with page size of 5
4797
+ alert_activations, token = db.paginated_list_alert_activations(
4798
+ project="my-project", page_size=5
4799
+ )
4800
+ # Fetch next page using the pagination token from the previous response
4801
+ alert_activations, token = db.paginated_list_alert_activations(
4802
+ project="my-project", page_token=token
4803
+ )
4804
+ # Fetch alert activations for a specific page (e.g., page 3)
4805
+ alert_activations, token = db.paginated_list_alert_activations(
4806
+ project="my-project", page=3, page_size=5
4807
+ )
4808
+
4809
+ # Automatically iterate over all pages without explicitly specifying the page number
4810
+ alert_activations = []
4811
+ token = None
4812
+ while True:
4813
+ page_alert_activations, token = db.paginated_list_alert_activations(
4814
+ project="my-project", page_token=token, page_size=5
4815
+ )
4816
+ alert_activations.extend(page_alert_activations)
4817
+
4818
+ # If token is None and page_alert_activations is empty, we've reached the end (no more activations).
4819
+ # If token is None and page_alert_activations is not empty, we've fetched the last page of activations.
4820
+ if not token:
4821
+ break
4822
+ print(f"Total alert activations retrieved: {len(alert_activations)}")
4823
+
4824
+ :param page: The page number to retrieve. If not provided, the next page will be retrieved.
4825
+ :param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
4826
+ :param page_token: A pagination token used to retrieve the next page of results. Should not be provided
4827
+ for the first request.
4828
+
4829
+ :returns: A tuple containing the list of alert activations and an optional `page_token` for pagination.
4830
+ """
4831
+ return self._list_alert_activations(
4832
+ *args,
4833
+ page=page,
4834
+ page_size=page_size,
4835
+ page_token=page_token,
4836
+ return_all=False,
4837
+ **kwargs,
4838
+ )
4839
+
4840
+ def get_project_summary(
4841
+ self, project: Optional[str] = None
4842
+ ) -> mlrun.common.schemas.ProjectSummary:
4843
+ """
4844
+ Retrieve the summary of a project.
4845
+
4846
+ :param project: Project name for which the summary belongs.
4847
+ :returns: A summary of the project.
4848
+ """
4849
+ project = project or config.default_project
4850
+
4851
+ endpoint_path = f"project-summaries/{project}"
4852
+ error_message = f"Failed retrieving project summary for {project}"
4853
+ response = self.api_call("GET", endpoint_path, error_message)
4854
+ return mlrun.common.schemas.ProjectSummary(**response.json())
4855
+
4856
+ @staticmethod
4857
+ def _parse_labels(
4858
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]],
4859
+ ):
4860
+ """
4861
+ Parse labels to support providing a dictionary from the SDK,
4862
+ which may not be directly supported in the endpoints.
4863
+
4864
+ :param labels: The labels to parse, which can be a dictionary, a list of strings,
4865
+ or a comma-separated string. This function converts them into a list
4866
+ of labels in the format 'key=value' or 'key'.
4867
+ :return: A list of parsed labels in the format 'key=value' or 'key'.
4868
+ :raises MLRunValueError: If the labels format is invalid.
4869
+ """
4870
+ try:
4871
+ return mlrun.common.schemas.common.LabelsModel(labels=labels).labels
4872
+ except pydantic.v1.error_wrappers.ValidationError as exc:
4873
+ raise mlrun.errors.MLRunValueError(
4874
+ "Invalid labels format. Must be a dictionary of strings, a list of strings, "
4875
+ "or a comma-separated string."
4876
+ ) from exc
4877
+
4878
+ def _list_artifacts(
4879
+ self,
4880
+ name: Optional[str] = None,
4881
+ project: Optional[str] = None,
4882
+ tag: Optional[str] = None,
4883
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
4884
+ since: Optional[datetime] = None,
4885
+ until: Optional[datetime] = None,
4886
+ iter: Optional[int] = None,
4887
+ best_iteration: bool = False,
4888
+ kind: Optional[str] = None,
4889
+ category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
4890
+ tree: Optional[str] = None,
4891
+ producer_uri: Optional[str] = None,
4892
+ format_: Optional[
4893
+ mlrun.common.formatters.ArtifactFormat
4894
+ ] = mlrun.common.formatters.ArtifactFormat.full,
4895
+ limit: Optional[int] = None,
4896
+ partition_by: Optional[
4897
+ Union[mlrun.common.schemas.ArtifactPartitionByField, str]
4898
+ ] = None,
4899
+ rows_per_partition: int = 1,
4900
+ partition_sort_by: Optional[
4901
+ Union[mlrun.common.schemas.SortField, str]
4902
+ ] = mlrun.common.schemas.SortField.updated,
4903
+ partition_order: Union[
4904
+ mlrun.common.schemas.OrderType, str
4905
+ ] = mlrun.common.schemas.OrderType.desc,
4906
+ page: Optional[int] = None,
4907
+ page_size: Optional[int] = None,
4908
+ page_token: Optional[str] = None,
4909
+ return_all: bool = False,
4910
+ ) -> tuple[ArtifactList, Optional[str]]:
4911
+ """Handles list artifacts, both paginated and not."""
4912
+
4913
+ project = project or config.default_project
4914
+ labels = self._parse_labels(labels)
4915
+
4916
+ params = {
4917
+ "name": name,
4918
+ "tag": tag,
4919
+ "label": labels,
4920
+ "iter": iter,
4921
+ "best-iteration": best_iteration,
4922
+ "kind": kind,
4923
+ "category": category,
4924
+ "tree": tree,
4925
+ "format": format_,
4926
+ "producer_uri": producer_uri,
4927
+ "since": datetime_to_iso(since),
4928
+ "until": datetime_to_iso(until),
4929
+ "limit": limit,
4930
+ "page": page,
4931
+ "page-size": page_size,
4932
+ "page-token": page_token,
4933
+ }
4934
+
4935
+ if partition_by:
4936
+ params.update(
4937
+ self._generate_partition_by_params(
4938
+ partition_by,
4939
+ rows_per_partition,
4940
+ partition_sort_by,
4941
+ partition_order,
4942
+ )
4943
+ )
4944
+ error = "list artifacts"
4945
+ endpoint_path = f"projects/{project}/artifacts"
4946
+
4947
+ # Fetch the responses, either one page or all based on `return_all`
4948
+ responses = self.paginated_api_call(
4949
+ "GET",
4950
+ endpoint_path,
4951
+ error,
4952
+ params=params,
4953
+ version="v2",
4954
+ return_all=return_all,
4955
+ )
4956
+ paginated_responses, token = self.process_paginated_responses(
4957
+ responses, "artifacts"
4958
+ )
4959
+
4960
+ values = ArtifactList(paginated_responses)
4961
+ values.tag = tag
4962
+ return values, token
4963
+
4964
+ def _list_functions(
4965
+ self,
4966
+ name: Optional[str] = None,
4967
+ project: Optional[str] = None,
4968
+ tag: Optional[str] = None,
4969
+ kind: Optional[str] = None,
4970
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
4971
+ format_: Optional[str] = None,
4972
+ since: Optional[datetime] = None,
4973
+ until: Optional[datetime] = None,
4974
+ page: Optional[int] = None,
4975
+ page_size: Optional[int] = None,
4976
+ page_token: Optional[str] = None,
4977
+ return_all: bool = False,
4978
+ ) -> tuple[list, Optional[str]]:
4979
+ """Handles list functions, both paginated and not."""
4980
+
4981
+ project = project or config.default_project
4982
+ labels = self._parse_labels(labels)
4983
+ params = {
4984
+ "name": name,
4985
+ "tag": tag,
4986
+ "kind": kind,
4987
+ "label": labels,
4988
+ "since": datetime_to_iso(since),
4989
+ "until": datetime_to_iso(until),
4990
+ "format": format_,
4991
+ "page": page,
4992
+ "page-size": page_size,
4993
+ "page-token": page_token,
4994
+ }
4995
+ error = "list functions"
4996
+ path = f"projects/{project}/functions"
4997
+
4998
+ # Fetch the responses, either one page or all based on `return_all`
4999
+ responses = self.paginated_api_call(
5000
+ "GET", path, error, params=params, return_all=return_all
5001
+ )
5002
+ paginated_responses, token = self.process_paginated_responses(
5003
+ responses, "funcs"
5004
+ )
5005
+ return paginated_responses, token
5006
+
5007
+ def _list_runs(
5008
+ self,
5009
+ name: Optional[str] = None,
5010
+ uid: Optional[Union[str, list[str]]] = None,
5011
+ project: Optional[str] = None,
5012
+ labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
5013
+ state: Optional[
5014
+ mlrun.common.runtimes.constants.RunStates
5015
+ ] = None, # Backward compatibility
5016
+ states: typing.Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
5017
+ sort: bool = True,
5018
+ last: int = 0,
5019
+ iter: bool = False,
5020
+ start_time_from: Optional[datetime] = None,
5021
+ start_time_to: Optional[datetime] = None,
5022
+ last_update_time_from: Optional[datetime] = None,
5023
+ last_update_time_to: Optional[datetime] = None,
5024
+ partition_by: Optional[
5025
+ Union[mlrun.common.schemas.RunPartitionByField, str]
5026
+ ] = None,
5027
+ rows_per_partition: int = 1,
5028
+ partition_sort_by: Optional[Union[mlrun.common.schemas.SortField, str]] = None,
5029
+ partition_order: Union[
5030
+ mlrun.common.schemas.OrderType, str
5031
+ ] = mlrun.common.schemas.OrderType.desc,
5032
+ max_partitions: int = 0,
5033
+ with_notifications: bool = False,
5034
+ page: Optional[int] = None,
5035
+ page_size: Optional[int] = None,
5036
+ page_token: Optional[str] = None,
5037
+ return_all: bool = False,
5038
+ ) -> tuple[RunList, Optional[str]]:
5039
+ """Handles list runs, both paginated and not."""
5040
+
5041
+ project = project or config.default_project
5042
+ if with_notifications:
5043
+ logger.warning(
5044
+ "Local run notifications are not persisted in the DB, therefore local runs will not be returned when "
5045
+ "using the `with_notifications` flag."
5046
+ )
5047
+
5048
+ if last:
5049
+ # TODO: Remove this in 1.8.0
5050
+ warnings.warn(
5051
+ "'last' is deprecated and will be removed in 1.8.0.",
5052
+ FutureWarning,
5053
+ )
5054
+
5055
+ if state:
5056
+ # TODO: Remove this in 1.9.0
5057
+ warnings.warn(
5058
+ "'state' is deprecated and will be removed in 1.9.0. Use 'states' instead.",
5059
+ FutureWarning,
5060
+ )
5061
+
5062
+ labels = self._parse_labels(labels)
5063
+
5064
+ if (
5065
+ not name
5066
+ and not uid
5067
+ and not labels
5068
+ and not state
5069
+ and not states
5070
+ and not last
5071
+ and not start_time_from
5072
+ and not start_time_to
5073
+ and not last_update_time_from
5074
+ and not last_update_time_to
5075
+ and not partition_by
5076
+ and not partition_sort_by
5077
+ and not iter
5078
+ ):
5079
+ # default to last week on no filter
5080
+ start_time_from = datetime.now() - timedelta(days=7)
5081
+ partition_by = mlrun.common.schemas.RunPartitionByField.project_and_name
5082
+ partition_sort_by = mlrun.common.schemas.SortField.updated
5083
+
5084
+ params = {
5085
+ "name": name,
5086
+ "uid": uid,
5087
+ "label": labels,
5088
+ "state": (
5089
+ mlrun.utils.helpers.as_list(state)
5090
+ if state is not None
5091
+ else states or None
5092
+ ),
5093
+ "sort": bool2str(sort),
5094
+ "iter": bool2str(iter),
5095
+ "start_time_from": datetime_to_iso(start_time_from),
5096
+ "start_time_to": datetime_to_iso(start_time_to),
5097
+ "last_update_time_from": datetime_to_iso(last_update_time_from),
5098
+ "last_update_time_to": datetime_to_iso(last_update_time_to),
5099
+ "with-notifications": with_notifications,
5100
+ "page": page,
5101
+ "page-size": page_size,
5102
+ "page-token": page_token,
5103
+ }
5104
+
5105
+ if partition_by:
5106
+ params.update(
5107
+ self._generate_partition_by_params(
5108
+ partition_by,
5109
+ rows_per_partition,
5110
+ partition_sort_by,
5111
+ partition_order,
5112
+ max_partitions,
5113
+ )
5114
+ )
5115
+ error = "list runs"
5116
+ _path = self._path_of("runs", project)
5117
+
5118
+ # Fetch the responses, either one page or all based on `return_all`
5119
+ responses = self.paginated_api_call(
5120
+ "GET", _path, error, params=params, return_all=return_all
5121
+ )
5122
+ paginated_responses, token = self.process_paginated_responses(responses, "runs")
5123
+ return RunList(paginated_responses), token
5124
+
5125
+ def _list_alert_activations(
5126
+ self,
5127
+ project: Optional[str] = None,
5128
+ name: Optional[str] = None,
5129
+ since: Optional[datetime] = None,
5130
+ until: Optional[datetime] = None,
5131
+ entity: Optional[str] = None,
5132
+ severity: Optional[
5133
+ Union[
5134
+ mlrun.common.schemas.alert.AlertSeverity,
5135
+ str,
5136
+ list[Union[mlrun.common.schemas.alert.AlertSeverity, str]],
5137
+ ]
5138
+ ] = None,
5139
+ entity_kind: Optional[
5140
+ Union[mlrun.common.schemas.alert.EventEntityKind, str]
5141
+ ] = None,
5142
+ event_kind: Optional[Union[mlrun.common.schemas.alert.EventKind, str]] = None,
5143
+ page: Optional[int] = None,
5144
+ page_size: Optional[int] = None,
5145
+ page_token: Optional[str] = None,
5146
+ return_all: bool = False,
5147
+ ) -> tuple[mlrun.common.schemas.AlertActivations, Optional[str]]:
5148
+ project = project or config.default_project
5149
+ params = {
5150
+ "name": name,
5151
+ "since": datetime_to_iso(since),
5152
+ "until": datetime_to_iso(until),
5153
+ "entity": entity,
5154
+ "severity": mlrun.utils.helpers.as_list(severity) if severity else None,
5155
+ "entity-kind": entity_kind,
5156
+ "event-kind": event_kind,
5157
+ "page": page,
5158
+ "page-size": page_size,
5159
+ "page-token": page_token,
5160
+ }
5161
+ error = "list alert activations"
5162
+ path = f"projects/{project}/alert-activations"
5163
+
5164
+ # Fetch the responses, either one page or all based on `return_all`
5165
+ responses = self.paginated_api_call(
5166
+ "GET", path, error, params=params, return_all=return_all
5167
+ )
5168
+ paginated_responses, token = self.process_paginated_responses(
5169
+ responses, "activations"
5170
+ )
5171
+ paginated_results = mlrun.common.schemas.AlertActivations(
5172
+ activations=[
5173
+ mlrun.common.schemas.AlertActivation(**item)
5174
+ for item in paginated_responses
5175
+ ]
5176
+ )
5177
+
5178
+ return paginated_results, token
5179
+
5180
+ def _wait_for_background_task_from_response(self, response):
5181
+ if response.status_code == http.HTTPStatus.ACCEPTED:
5182
+ background_task = mlrun.common.schemas.BackgroundTask(**response.json())
5183
+ return self._wait_for_background_task_to_reach_terminal_state(
5184
+ background_task.metadata.name
5185
+ )
5186
+ return None
5187
+
4337
5188
 
4338
5189
  def _as_json(obj):
4339
5190
  fn = getattr(obj, "to_json", None)