mlrun 1.10.0rc40__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (150) hide show
  1. mlrun/__init__.py +3 -2
  2. mlrun/__main__.py +0 -4
  3. mlrun/artifacts/dataset.py +2 -2
  4. mlrun/artifacts/plots.py +1 -1
  5. mlrun/{model_monitoring/db/tsdb/tdengine → auth}/__init__.py +2 -3
  6. mlrun/auth/nuclio.py +89 -0
  7. mlrun/auth/providers.py +429 -0
  8. mlrun/auth/utils.py +415 -0
  9. mlrun/common/constants.py +7 -0
  10. mlrun/common/model_monitoring/helpers.py +41 -4
  11. mlrun/common/runtimes/constants.py +28 -0
  12. mlrun/common/schemas/__init__.py +13 -3
  13. mlrun/common/schemas/alert.py +2 -2
  14. mlrun/common/schemas/api_gateway.py +3 -0
  15. mlrun/common/schemas/auth.py +10 -10
  16. mlrun/common/schemas/client_spec.py +4 -0
  17. mlrun/common/schemas/constants.py +25 -0
  18. mlrun/common/schemas/frontend_spec.py +1 -8
  19. mlrun/common/schemas/function.py +24 -0
  20. mlrun/common/schemas/hub.py +3 -2
  21. mlrun/common/schemas/model_monitoring/__init__.py +1 -1
  22. mlrun/common/schemas/model_monitoring/constants.py +2 -2
  23. mlrun/common/schemas/secret.py +17 -2
  24. mlrun/common/secrets.py +95 -1
  25. mlrun/common/types.py +10 -10
  26. mlrun/config.py +53 -15
  27. mlrun/data_types/infer.py +2 -2
  28. mlrun/datastore/__init__.py +2 -3
  29. mlrun/datastore/base.py +274 -10
  30. mlrun/datastore/datastore.py +1 -1
  31. mlrun/datastore/datastore_profile.py +49 -17
  32. mlrun/datastore/model_provider/huggingface_provider.py +6 -2
  33. mlrun/datastore/model_provider/model_provider.py +2 -2
  34. mlrun/datastore/model_provider/openai_provider.py +2 -2
  35. mlrun/datastore/s3.py +15 -16
  36. mlrun/datastore/sources.py +1 -1
  37. mlrun/datastore/store_resources.py +4 -4
  38. mlrun/datastore/storeytargets.py +16 -10
  39. mlrun/datastore/targets.py +1 -1
  40. mlrun/datastore/utils.py +16 -3
  41. mlrun/datastore/v3io.py +1 -1
  42. mlrun/db/base.py +36 -12
  43. mlrun/db/httpdb.py +316 -101
  44. mlrun/db/nopdb.py +29 -11
  45. mlrun/errors.py +4 -2
  46. mlrun/execution.py +11 -12
  47. mlrun/feature_store/api.py +1 -1
  48. mlrun/feature_store/common.py +1 -1
  49. mlrun/feature_store/feature_vector_utils.py +1 -1
  50. mlrun/feature_store/steps.py +8 -6
  51. mlrun/frameworks/_common/utils.py +3 -3
  52. mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
  53. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +2 -1
  54. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  55. mlrun/frameworks/_ml_common/utils.py +2 -1
  56. mlrun/frameworks/auto_mlrun/auto_mlrun.py +4 -3
  57. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +2 -1
  58. mlrun/frameworks/onnx/dataset.py +2 -1
  59. mlrun/frameworks/onnx/mlrun_interface.py +2 -1
  60. mlrun/frameworks/pytorch/callbacks/logging_callback.py +5 -4
  61. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +2 -1
  62. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +2 -1
  63. mlrun/frameworks/pytorch/utils.py +2 -1
  64. mlrun/frameworks/sklearn/metric.py +2 -1
  65. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +5 -4
  66. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +2 -1
  67. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +2 -1
  68. mlrun/hub/__init__.py +37 -0
  69. mlrun/hub/base.py +142 -0
  70. mlrun/hub/module.py +67 -76
  71. mlrun/hub/step.py +113 -0
  72. mlrun/launcher/base.py +2 -1
  73. mlrun/launcher/local.py +2 -1
  74. mlrun/model.py +12 -2
  75. mlrun/model_monitoring/__init__.py +0 -1
  76. mlrun/model_monitoring/api.py +2 -2
  77. mlrun/model_monitoring/applications/base.py +20 -6
  78. mlrun/model_monitoring/applications/context.py +1 -0
  79. mlrun/model_monitoring/controller.py +7 -17
  80. mlrun/model_monitoring/db/_schedules.py +2 -16
  81. mlrun/model_monitoring/db/_stats.py +2 -13
  82. mlrun/model_monitoring/db/tsdb/__init__.py +9 -7
  83. mlrun/model_monitoring/db/tsdb/base.py +2 -4
  84. mlrun/model_monitoring/db/tsdb/preaggregate.py +234 -0
  85. mlrun/model_monitoring/db/tsdb/stream_graph_steps.py +63 -0
  86. mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_metrics_queries.py +414 -0
  87. mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_predictions_queries.py +376 -0
  88. mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_results_queries.py +590 -0
  89. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_connection.py +434 -0
  90. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_connector.py +541 -0
  91. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_operations.py +808 -0
  92. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_schema.py +502 -0
  93. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_stream.py +163 -0
  94. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_stream_graph_steps.py +60 -0
  95. mlrun/model_monitoring/db/tsdb/timescaledb/utils/timescaledb_dataframe_processor.py +141 -0
  96. mlrun/model_monitoring/db/tsdb/timescaledb/utils/timescaledb_query_builder.py +585 -0
  97. mlrun/model_monitoring/db/tsdb/timescaledb/writer_graph_steps.py +73 -0
  98. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +4 -6
  99. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +147 -79
  100. mlrun/model_monitoring/features_drift_table.py +2 -1
  101. mlrun/model_monitoring/helpers.py +2 -1
  102. mlrun/model_monitoring/stream_processing.py +18 -16
  103. mlrun/model_monitoring/writer.py +4 -3
  104. mlrun/package/__init__.py +2 -1
  105. mlrun/platforms/__init__.py +0 -44
  106. mlrun/platforms/iguazio.py +1 -1
  107. mlrun/projects/operations.py +11 -10
  108. mlrun/projects/project.py +81 -82
  109. mlrun/run.py +4 -7
  110. mlrun/runtimes/__init__.py +2 -204
  111. mlrun/runtimes/base.py +89 -21
  112. mlrun/runtimes/constants.py +225 -0
  113. mlrun/runtimes/daskjob.py +4 -2
  114. mlrun/runtimes/databricks_job/databricks_runtime.py +2 -1
  115. mlrun/runtimes/mounts.py +5 -0
  116. mlrun/runtimes/nuclio/__init__.py +12 -8
  117. mlrun/runtimes/nuclio/api_gateway.py +36 -6
  118. mlrun/runtimes/nuclio/application/application.py +200 -32
  119. mlrun/runtimes/nuclio/function.py +154 -49
  120. mlrun/runtimes/nuclio/serving.py +55 -42
  121. mlrun/runtimes/pod.py +59 -10
  122. mlrun/secrets.py +46 -2
  123. mlrun/serving/__init__.py +2 -0
  124. mlrun/serving/remote.py +5 -5
  125. mlrun/serving/routers.py +3 -3
  126. mlrun/serving/server.py +46 -43
  127. mlrun/serving/serving_wrapper.py +6 -2
  128. mlrun/serving/states.py +554 -207
  129. mlrun/serving/steps.py +1 -1
  130. mlrun/serving/system_steps.py +42 -33
  131. mlrun/track/trackers/mlflow_tracker.py +29 -31
  132. mlrun/utils/helpers.py +89 -16
  133. mlrun/utils/http.py +9 -2
  134. mlrun/utils/notifications/notification/git.py +1 -1
  135. mlrun/utils/notifications/notification/mail.py +39 -16
  136. mlrun/utils/notifications/notification_pusher.py +2 -2
  137. mlrun/utils/version/version.json +2 -2
  138. mlrun/utils/version/version.py +3 -4
  139. {mlrun-1.10.0rc40.dist-info → mlrun-1.11.0rc16.dist-info}/METADATA +39 -49
  140. {mlrun-1.10.0rc40.dist-info → mlrun-1.11.0rc16.dist-info}/RECORD +144 -130
  141. mlrun/db/auth_utils.py +0 -152
  142. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +0 -343
  143. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +0 -75
  144. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +0 -281
  145. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +0 -1368
  146. mlrun/model_monitoring/db/tsdb/tdengine/writer_graph_steps.py +0 -51
  147. {mlrun-1.10.0rc40.dist-info → mlrun-1.11.0rc16.dist-info}/WHEEL +0 -0
  148. {mlrun-1.10.0rc40.dist-info → mlrun-1.11.0rc16.dist-info}/entry_points.txt +0 -0
  149. {mlrun-1.10.0rc40.dist-info → mlrun-1.11.0rc16.dist-info}/licenses/LICENSE +0 -0
  150. {mlrun-1.10.0rc40.dist-info → mlrun-1.11.0rc16.dist-info}/top_level.txt +0 -0
mlrun/db/httpdb.py CHANGED
@@ -11,9 +11,9 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
-
15
14
  import enum
16
15
  import http
16
+ import os
17
17
  import re
18
18
  import time
19
19
  import traceback
@@ -32,6 +32,7 @@ import semver
32
32
  from pydantic.v1 import parse_obj_as
33
33
 
34
34
  import mlrun
35
+ import mlrun.auth
35
36
  import mlrun.common.constants
36
37
  import mlrun.common.formatters
37
38
  import mlrun.common.runtimes
@@ -39,6 +40,7 @@ import mlrun.common.schemas
39
40
  import mlrun.common.schemas.model_monitoring.constants as mm_constants
40
41
  import mlrun.common.schemas.model_monitoring.model_endpoints as mm_endpoints
41
42
  import mlrun.common.types
43
+ import mlrun.k8s_utils
42
44
  import mlrun.platforms
43
45
  import mlrun.projects
44
46
  import mlrun.runtimes.nuclio.api_gateway
@@ -46,7 +48,6 @@ import mlrun.runtimes.nuclio.function
46
48
  import mlrun.utils
47
49
  from mlrun.alerts.alert import AlertConfig
48
50
  from mlrun.common.schemas.hub import HubSourceType
49
- from mlrun.db.auth_utils import OAuthClientIDTokenProvider, StaticTokenProvider
50
51
  from mlrun.errors import MLRunInvalidArgumentError, err_to_str
51
52
  from mlrun.secrets import get_secret_or_env
52
53
  from mlrun_pipelines.utils import compile_pipeline
@@ -126,6 +127,11 @@ class HTTPRunDB(RunDBInterface):
126
127
  r"\/?run\/.+\/.+",
127
128
  ]
128
129
 
130
+ NON_RETRIABLE_PATHS = [
131
+ # Storing user secret tokens is not idempotent — retrying the request may result inconsistent secret storage.
132
+ r"\/?user-secrets/tokens",
133
+ ]
134
+
129
135
  def __init__(self, url):
130
136
  self.server_version = ""
131
137
  self.session = None
@@ -137,43 +143,56 @@ class HTTPRunDB(RunDBInterface):
137
143
  version.Version().get_python_version()
138
144
  )
139
145
 
146
+ self.user = None
147
+ self.password = None
148
+ self.token_provider = None
149
+ self.base_url = None
150
+ self._parsed_url = None
151
+
140
152
  self._enrich_and_validate(url)
141
153
 
142
154
  def _enrich_and_validate(self, url):
143
- parsed_url = urlparse(url)
144
- scheme = parsed_url.scheme.lower()
145
- if scheme not in ("http", "https"):
146
- raise ValueError(
147
- f"Invalid URL scheme {scheme} for HTTPRunDB, only http(s) is supported"
148
- )
149
-
150
- endpoint = parsed_url.hostname
151
- if parsed_url.port:
152
- endpoint += f":{parsed_url.port}"
153
- base_url = f"{parsed_url.scheme}://{endpoint}{parsed_url.path}"
155
+ base_url, parsed_url = self._resolve_api_urls(url)
154
156
 
155
157
  self.base_url = base_url
156
- username = parsed_url.username or config.httpdb.user
157
- password = parsed_url.password or config.httpdb.password
158
+ self._parsed_url = parsed_url
159
+ self.user = parsed_url.username or config.httpdb.user
160
+ self.password = parsed_url.password or config.httpdb.password
161
+ self._init_token_provider()
162
+
163
+ def _init_token_provider(self):
164
+ """
165
+ Initialize token provider according to current config.
166
+
167
+ Must be called after `connect()` synced config from server (client-spec), since
168
+ some auth flows (e.g. Iguazio V4 OAuth token) require values fetched from the API.
169
+ """
158
170
  self.token_provider = None
159
171
 
160
172
  if config.auth_with_client_id.enabled:
161
- self.token_provider = OAuthClientIDTokenProvider(
173
+ self.token_provider = mlrun.auth.OAuthClientIDTokenProvider(
162
174
  token_endpoint=get_secret_or_env("MLRUN_AUTH_TOKEN_ENDPOINT"),
163
175
  client_id=get_secret_or_env("MLRUN_AUTH_CLIENT_ID"),
164
176
  client_secret=get_secret_or_env("MLRUN_AUTH_CLIENT_SECRET"),
165
177
  timeout=config.auth_with_client_id.request_timeout,
166
178
  )
179
+ elif config.auth_with_oauth_token.enabled:
180
+ self.token_provider = mlrun.auth.IGTokenProvider(
181
+ token_endpoint=config.auth_token_endpoint,
182
+ timeout=config.auth_with_oauth_token.request_timeout,
183
+ )
167
184
  else:
168
185
  username, password, token = mlrun.platforms.add_or_refresh_credentials(
169
- parsed_url.hostname, username, password, config.httpdb.token
186
+ self._parsed_url.hostname,
187
+ self.user,
188
+ self.password,
189
+ config.httpdb.token,
170
190
  )
191
+ self.user = username
192
+ self.password = password
171
193
 
172
194
  if token:
173
- self.token_provider = StaticTokenProvider(token)
174
-
175
- self.user = username
176
- self.password = password
195
+ self.token_provider = mlrun.auth.StaticTokenProvider(token)
177
196
 
178
197
  def __repr__(self):
179
198
  cls = self.__class__.__name__
@@ -254,8 +273,18 @@ class HTTPRunDB(RunDBInterface):
254
273
  }
255
274
  kw["cookies"] = cookies
256
275
  else:
257
- if "Authorization" not in kw.setdefault("headers", {}):
258
- kw["headers"].update({"Authorization": "Bearer " + token})
276
+ if (
277
+ mlrun.common.schemas.HeaderNames.authorization
278
+ not in kw.setdefault("headers", {})
279
+ ):
280
+ kw["headers"].update(
281
+ {
282
+ mlrun.common.schemas.HeaderNames.authorization: (
283
+ mlrun.common.schemas.AuthorizationHeaderPrefixes.bearer
284
+ + token
285
+ )
286
+ }
287
+ )
259
288
 
260
289
  if mlrun.common.schemas.HeaderNames.client_version not in kw.setdefault(
261
290
  "headers", {}
@@ -275,10 +304,13 @@ class HTTPRunDB(RunDBInterface):
275
304
  if isinstance(dict_[key], enum.Enum):
276
305
  dict_[key] = dict_[key].value
277
306
 
278
- # if the method is POST, we need to update the session with the appropriate retry policy
279
- if not self.session or method == "POST":
280
- retry_on_post = self._is_retry_on_post_allowed(method, path)
281
- self.session = self._init_session(retry_on_post)
307
+ retry_on_post = self._is_retry_on_post_allowed(method, path)
308
+
309
+ retry_on_put = self._is_retry_put_allowed(method, path)
310
+
311
+ # if the method is POST or PUT, we need to update the session with the appropriate retry policy
312
+ if not self.session or method in ("POST", "PUT"):
313
+ self.session = self._init_session(retry_on_post, retry_on_put)
282
314
 
283
315
  try:
284
316
  response = self.session.request(
@@ -394,11 +426,12 @@ class HTTPRunDB(RunDBInterface):
394
426
  data.extend(response.json().get(key, []))
395
427
  return data, page_token
396
428
 
397
- def _init_session(self, retry_on_post: bool = False):
429
+ def _init_session(self, retry_on_post: bool = False, retry_on_put: bool = True):
398
430
  return mlrun.utils.HTTPSessionWithRetry(
399
431
  retry_on_exception=config.httpdb.retry_api_call_on_exception
400
432
  == mlrun.common.schemas.HTTPSessionRetryMode.enabled.value,
401
433
  retry_on_post=retry_on_post,
434
+ retry_on_put=retry_on_put,
402
435
  )
403
436
 
404
437
  def _path_of(self, resource, project, uid=None):
@@ -420,7 +453,28 @@ class HTTPRunDB(RunDBInterface):
420
453
  re.match(regex, path) for regex in self.RETRIABLE_POST_PATHS
421
454
  )
422
455
 
423
- def connect(self, secrets=None):
456
+ def _is_retry_put_allowed(self, method: str, path: str) -> bool:
457
+ """
458
+ Determine if PUT request to the given path should be retried.
459
+
460
+ :param method: HTTP method
461
+ :param path: API path to check
462
+ :return: True if retry is allowed, False otherwise
463
+ """
464
+ if method != "PUT":
465
+ return True
466
+
467
+ # Strip query parameters and fragment if present
468
+ parsed_path = urlparse(path).path.lstrip("/")
469
+
470
+ # If the path matches a non-retriable path, do not allow retry
471
+ for regex in self.NON_RETRIABLE_PATHS:
472
+ if re.fullmatch(regex, parsed_path):
473
+ return False
474
+
475
+ return True
476
+
477
+ def connect(self, secrets=None) -> typing.Self:
424
478
  """Connect to the MLRun API server. Must be called prior to executing any other method.
425
479
  The code utilizes the URL for the API server from the configuration - ``config.dbpath``.
426
480
 
@@ -431,7 +485,8 @@ class HTTPRunDB(RunDBInterface):
431
485
  """
432
486
  # hack to allow unit tests to instantiate HTTPRunDB without a real server behind
433
487
  if "mock-server" in self.base_url:
434
- return
488
+ return self
489
+
435
490
  resp = self.api_call("GET", "client-spec", timeout=5)
436
491
  try:
437
492
  server_cfg = resp.json()
@@ -583,6 +638,44 @@ class HTTPRunDB(RunDBInterface):
583
638
  prefix,
584
639
  store_prefix_value,
585
640
  )
641
+ config.httpdb.authentication.mode = (
642
+ server_cfg.get("authentication_mode")
643
+ or config.httpdb.authentication.mode
644
+ )
645
+
646
+ # Iguazio V4 OAuth token config auto-initialization
647
+ if (
648
+ config.httpdb.authentication.mode
649
+ == mlrun.common.types.AuthenticationMode.IGUAZIO_V4.value
650
+ ):
651
+ if not config.auth_with_oauth_token.token_file:
652
+ user_token_file = os.path.expanduser("~/.igz.yml")
653
+
654
+ # runtimes
655
+ # TODO: change to os.getenv("MLRUN_RUNTIME_KIND")
656
+ # when https://github.com/mlrun/mlrun/pull/9121 is done.
657
+ if (
658
+ mlrun.k8s_utils.is_running_inside_kubernetes_cluster()
659
+ and not os.environ.get("JPY_SESSION_NAME")
660
+ ):
661
+ user_token_file = os.path.join(
662
+ mlrun.common.constants.MLRUN_JOB_AUTH_SECRET_PATH,
663
+ mlrun.common.constants.MLRUN_JOB_AUTH_SECRET_FILE,
664
+ )
665
+
666
+ config.auth_with_oauth_token.token_file = user_token_file
667
+
668
+ # if running inside kubernetes, use the internal endpoint, otherwise use the external endpoint
669
+ if mlrun.k8s_utils.is_running_inside_kubernetes_cluster():
670
+ config.auth_token_endpoint = server_cfg.get(
671
+ "oauth_internal_token_endpoint"
672
+ )
673
+ else:
674
+ config.auth_token_endpoint = server_cfg.get(
675
+ "oauth_external_token_endpoint"
676
+ )
677
+
678
+ config.auth_with_oauth_token.enabled = True
586
679
 
587
680
  except Exception as exc:
588
681
  logger.warning(
@@ -590,6 +683,12 @@ class HTTPRunDB(RunDBInterface):
590
683
  exc=err_to_str(exc),
591
684
  traceback=traceback.format_exc(),
592
685
  )
686
+
687
+ # Initialize token provider after syncing config from server
688
+ self._init_token_provider()
689
+
690
+ if config.is_iguazio_v4_mode() and config.auth_with_oauth_token.enabled:
691
+ mlrun.secrets.sync_secret_tokens()
593
692
  return self
594
693
 
595
694
  def store_log(self, uid, project="", body=None, append=False):
@@ -1233,7 +1332,6 @@ class HTTPRunDB(RunDBInterface):
1233
1332
  format_: Optional[
1234
1333
  mlrun.common.formatters.ArtifactFormat
1235
1334
  ] = mlrun.common.formatters.ArtifactFormat.full,
1236
- limit: Optional[int] = None,
1237
1335
  partition_by: Optional[
1238
1336
  Union[mlrun.common.schemas.ArtifactPartitionByField, str]
1239
1337
  ] = None,
@@ -1286,7 +1384,6 @@ class HTTPRunDB(RunDBInterface):
1286
1384
  points to a run and is used to filter artifacts by the run that produced them when the artifact producer id
1287
1385
  is a workflow id (artifact was created as part of a workflow).
1288
1386
  :param format_: The format in which to return the artifacts. Default is 'full'.
1289
- :param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
1290
1387
  :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
1291
1388
  parameter must be provided as well.
1292
1389
  :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
@@ -1310,12 +1407,11 @@ class HTTPRunDB(RunDBInterface):
1310
1407
  tree=tree,
1311
1408
  producer_uri=producer_uri,
1312
1409
  format_=format_,
1313
- limit=limit,
1314
1410
  partition_by=partition_by,
1315
1411
  rows_per_partition=rows_per_partition,
1316
1412
  partition_sort_by=partition_sort_by,
1317
1413
  partition_order=partition_order,
1318
- return_all=not limit,
1414
+ return_all=True,
1319
1415
  parent=parent,
1320
1416
  )
1321
1417
  return artifacts
@@ -2206,7 +2302,7 @@ class HTTPRunDB(RunDBInterface):
2206
2302
  :param project: The project of the pipeline
2207
2303
  :param pipeline: Pipeline function or path to .yaml/.zip pipeline file.
2208
2304
  :param arguments: A dictionary of arguments to pass to the pipeline.
2209
- :param experiment: A name to assign for the specific experiment.
2305
+ :param experiment: (deprecated) A name to assign for the specific experiment.
2210
2306
  :param run: A name for this specific run.
2211
2307
  :param namespace: Kubernetes namespace to execute the pipeline in.
2212
2308
  :param artifact_path: A path to artifacts used by this pipeline.
@@ -2215,6 +2311,13 @@ class HTTPRunDB(RunDBInterface):
2215
2311
  workflow and all its resources are deleted)
2216
2312
  :param timeout: Timeout for the API call.
2217
2313
  """
2314
+ if experiment is not None:
2315
+ warnings.warn(
2316
+ "The 'experiment' parameter is deprecated and will be removed in 1.13.0. "
2317
+ "Pipelines are automatically scoped by project.",
2318
+ # TODO: Remove this in 1.13.0
2319
+ FutureWarning,
2320
+ )
2218
2321
 
2219
2322
  if isinstance(pipeline, str):
2220
2323
  pipe_file = pipeline
@@ -3605,40 +3708,6 @@ class HTTPRunDB(RunDBInterface):
3605
3708
  )
3606
3709
  return parsed_metrics_by_endpoint
3607
3710
 
3608
- def create_user_secrets(
3609
- self,
3610
- user: str,
3611
- provider: Union[
3612
- str, mlrun.common.schemas.SecretProviderName
3613
- ] = mlrun.common.schemas.SecretProviderName.vault,
3614
- secrets: Optional[dict] = None,
3615
- ):
3616
- """Create user-context secret in Vault. Please refer to :py:func:`create_project_secrets` for more details
3617
- and status of this functionality.
3618
-
3619
- Note:
3620
- This method is currently in technical preview, and requires a HashiCorp Vault infrastructure
3621
- properly set up and connected to the MLRun API server.
3622
-
3623
- :param user: The user context for which to generate the infra and store secrets.
3624
- :param provider: The name of the secrets-provider to work with. Currently only ``vault`` is supported.
3625
- :param secrets: A set of secret values to store within the Vault.
3626
- """
3627
- path = "user-secrets"
3628
- secrets_creation_request = mlrun.common.schemas.UserSecretCreationRequest(
3629
- user=user,
3630
- provider=provider,
3631
- secrets=secrets,
3632
- )
3633
- body = secrets_creation_request.dict()
3634
- error_message = f"Failed creating user secrets - {user}"
3635
- self.api_call(
3636
- "POST",
3637
- path,
3638
- error_message,
3639
- body=dict_to_json(body),
3640
- )
3641
-
3642
3711
  @staticmethod
3643
3712
  def _validate_version_compatibility(server_version, client_version) -> bool:
3644
3713
  try:
@@ -4165,8 +4234,12 @@ class HTTPRunDB(RunDBInterface):
4165
4234
  Get monitoring function summaries for the specified project.
4166
4235
 
4167
4236
  :param project: The name of the project.
4168
- :param start: Start time for filtering the results (optional).
4169
- :param end: End time for filtering the results (optional).
4237
+ :param start: The start time of the monitoring applications’ statistics.
4238
+ If not defined, the default is 24 hours ago.
4239
+ Required timezone, applicable only when `include_stats` is set to True.
4240
+ :param end: The end time of the monitoring applications’ statistics.
4241
+ If not defined, the default is now.
4242
+ Required timezone, applicable only when `include_stats` is set to True.
4170
4243
  :param names: List of function names to filter by (optional).
4171
4244
  :param labels: Labels to filter by (optional).
4172
4245
  :param include_stats: Whether to include statistics in the response (default is False).
@@ -4207,12 +4280,14 @@ class HTTPRunDB(RunDBInterface):
4207
4280
  ) -> FunctionSummary:
4208
4281
  """
4209
4282
  Get a monitoring function summary for the specified project and function.
4210
- :param project: The name of the project.
4211
- :param function_name: The name of the function.
4212
- :param start: Start time for filtering the results (optional).
4213
- :param end: End time for filtering the results (optional).
4214
- :param include_latest_metrics: Whether to include the latest metrics in the response (default is False).
4215
4283
 
4284
+ :param project: The name of the project.
4285
+ :param function_name: The name of the function.
4286
+ :param start: The start time of the monitoring application's statistics.
4287
+ If not defined, the default is 24 hours ago. Required timezone.
4288
+ :param end: The end time of the monitoring application's statistics.
4289
+ If not defined, the default is now. Required timezone.
4290
+ :param include_latest_metrics: Whether to include the latest metrics in the response (default is False).
4216
4291
  :return: A FunctionSummary object containing information about the monitoring function.
4217
4292
  """
4218
4293
 
@@ -4421,7 +4496,7 @@ class HTTPRunDB(RunDBInterface):
4421
4496
  :param item_type: The type of item to retrieve from the hub source (e.g: functions, modules).
4422
4497
  :returns: :py:class:`~mlrun.common.schemas.hub.HubItem`.
4423
4498
  """
4424
- path = (f"hub/sources/{source_name}/items/{item_name}",)
4499
+ path = f"hub/sources/{source_name}/items/{item_name}"
4425
4500
  params = {
4426
4501
  "version": version,
4427
4502
  "tag": tag,
@@ -5222,6 +5297,152 @@ class HTTPRunDB(RunDBInterface):
5222
5297
  **response.json()
5223
5298
  )
5224
5299
 
5300
+ @mlrun.utils.iguazio_v4_only
5301
+ def store_secret_token(
5302
+ self,
5303
+ secret_token: mlrun.common.schemas.SecretToken,
5304
+ log_warning: bool = True,
5305
+ force: bool = False,
5306
+ ) -> mlrun.common.schemas.StoreSecretTokensResponse:
5307
+ """
5308
+ Store or update a single secret token in the MLRun backend.
5309
+
5310
+ Example::
5311
+
5312
+ from mlrun.common.schemas import SecretToken
5313
+
5314
+ secret = SecretToken(name="my-token", token="dummy-token")
5315
+ db.store_secret_token(secret)
5316
+
5317
+ :param secret_token: A SecretToken object with name and token fields.
5318
+ :param force: Whether to force update the token if it already exists. Defaults to False.
5319
+ :param log_warning: Whether to log a warning about local config sync. Defaults to True.
5320
+ :return: A structured response indicating which tokens were created, updated, or skipped.
5321
+ """
5322
+ if not secret_token:
5323
+ raise MLRunInvalidArgumentError("No secret token provided")
5324
+
5325
+ response = self._store_secret_tokens(
5326
+ secret_tokens=[secret_token],
5327
+ force=force,
5328
+ error=f"store user secret token {secret_token.name}",
5329
+ )
5330
+
5331
+ # Only log if the token was created or updated
5332
+ if log_warning and (
5333
+ secret_token.name in response.created_tokens
5334
+ or secret_token.name in response.updated_tokens
5335
+ ):
5336
+ logger.warning(
5337
+ f"Token '{secret_token.name}' was stored in the backend, "
5338
+ f"but the local configuration file ({mlrun.mlconf.auth_with_oauth_token.token_file}) was not "
5339
+ "updated. Update it manually or run `mlrun.sync_secret_tokens()` to sync your local environment."
5340
+ )
5341
+ # maybe the response should not be a list?
5342
+ return response
5343
+
5344
+ @mlrun.utils.iguazio_v4_only
5345
+ def store_secret_tokens(
5346
+ self,
5347
+ secret_tokens: list[mlrun.common.schemas.SecretToken],
5348
+ log_warning: bool = True,
5349
+ force: bool = False,
5350
+ ) -> mlrun.common.schemas.StoreSecretTokensResponse:
5351
+ """
5352
+ Store or update multiple secret tokens in the MLRun backend.
5353
+
5354
+ Example::
5355
+
5356
+ from mlrun.common.schemas import SecretToken
5357
+
5358
+ tokens = [
5359
+ SecretToken(name="token1", token="dummy-token-1"),
5360
+ SecretToken(name="token2", token="dummy-token-2"),
5361
+ ]
5362
+ db.store_secret_tokens(tokens)
5363
+
5364
+ :param secret_tokens: List of SecretToken objects with 'name' and 'token' fields.
5365
+ :param force: Whether to force update tokens if they already exist. Defaults to False.
5366
+ :param log_warning: Whether to log a warning about local config file sync. Defaults to True.
5367
+ :return: StoreSecretTokensResponse object indicating which tokens were created, updated, or skipped.
5368
+ """
5369
+ if not secret_tokens:
5370
+ raise MLRunInvalidArgumentError("No secret tokens provided")
5371
+
5372
+ response = self._store_secret_tokens(
5373
+ secret_tokens=secret_tokens,
5374
+ force=force,
5375
+ error="store multiple user secret tokens",
5376
+ )
5377
+
5378
+ # Only log a warning if at least one token was actually created or updated
5379
+ if log_warning and (response.created_tokens or response.updated_tokens):
5380
+ affected_tokens = response.created_tokens + response.updated_tokens
5381
+ token_names = "', '".join(affected_tokens)
5382
+ logger.warning(
5383
+ f"Tokens '{token_names}' were stored in the backend, "
5384
+ f"but the local configuration file ({mlrun.mlconf.auth_with_oauth_token.token_file}) was not "
5385
+ "updated. Update it manually or run `mlrun.sync_secret_tokens()` to sync your local environment."
5386
+ )
5387
+
5388
+ return response
5389
+
5390
+ @mlrun.utils.iguazio_v4_only
5391
+ def list_secret_tokens(
5392
+ self,
5393
+ ) -> mlrun.common.schemas.ListSecretTokensResponse:
5394
+ """
5395
+ List all secret tokens for the current user.
5396
+ """
5397
+ endpoint_path = "user-secrets/tokens"
5398
+ response = self.api_call(
5399
+ mlrun.common.types.HTTPMethod.GET,
5400
+ endpoint_path,
5401
+ "list user secret tokens",
5402
+ )
5403
+
5404
+ return mlrun.common.schemas.ListSecretTokensResponse(**response.json())
5405
+
5406
+ @mlrun.utils.iguazio_v4_only
5407
+ def revoke_secret_token(self, token_name: str) -> None:
5408
+ endpoint_path = f"user-secrets/tokens/{token_name}"
5409
+ self.api_call(
5410
+ mlrun.common.types.HTTPMethod.DELETE,
5411
+ endpoint_path,
5412
+ "delete user secret token",
5413
+ )
5414
+
5415
+ @mlrun.utils.iguazio_v4_only
5416
+ def get_secret_token(
5417
+ self,
5418
+ token_name: str,
5419
+ username: Optional[str] = None,
5420
+ ) -> mlrun.common.schemas.SecretToken:
5421
+ raise NotImplementedError(
5422
+ "Getting secret token is not supported for security reasons."
5423
+ )
5424
+
5425
+ @mlrun.utils.iguazio_v4_only
5426
+ def _store_secret_tokens(
5427
+ self,
5428
+ secret_tokens: list[mlrun.common.schemas.SecretToken],
5429
+ error: str,
5430
+ force: bool = False,
5431
+ ) -> mlrun.common.schemas.StoreSecretTokensResponse:
5432
+ body = [token.dict() for token in secret_tokens]
5433
+ params = {"force": force} # send as query param
5434
+ endpoint_path = "user-secrets/tokens"
5435
+
5436
+ response = self.api_call(
5437
+ mlrun.common.types.HTTPMethod.PUT,
5438
+ endpoint_path,
5439
+ error,
5440
+ params=params,
5441
+ body=dict_to_json(body),
5442
+ )
5443
+
5444
+ return mlrun.common.schemas.StoreSecretTokensResponse(**response.json())
5445
+
5225
5446
  @staticmethod
5226
5447
  def _parse_labels(
5227
5448
  labels: Optional[Union[str, dict[str, Optional[str]], list[str]]],
@@ -5262,7 +5483,6 @@ class HTTPRunDB(RunDBInterface):
5262
5483
  format_: Optional[
5263
5484
  mlrun.common.formatters.ArtifactFormat
5264
5485
  ] = mlrun.common.formatters.ArtifactFormat.full,
5265
- limit: Optional[int] = None,
5266
5486
  partition_by: Optional[
5267
5487
  Union[mlrun.common.schemas.ArtifactPartitionByField, str]
5268
5488
  ] = None,
@@ -5283,13 +5503,6 @@ class HTTPRunDB(RunDBInterface):
5283
5503
  project = project or config.active_project
5284
5504
  labels = self._parse_labels(labels)
5285
5505
 
5286
- if limit:
5287
- # TODO: Remove this in 1.11.0
5288
- warnings.warn(
5289
- "'limit' is deprecated and will be removed in 1.11.0. Use 'page' and 'page_size' instead.",
5290
- FutureWarning,
5291
- )
5292
-
5293
5506
  params = {
5294
5507
  "name": name,
5295
5508
  "tag": tag,
@@ -5303,7 +5516,6 @@ class HTTPRunDB(RunDBInterface):
5303
5516
  "producer_uri": producer_uri,
5304
5517
  "since": datetime_to_iso(since),
5305
5518
  "until": datetime_to_iso(until),
5306
- "limit": limit,
5307
5519
  "page": page,
5308
5520
  "page-size": page_size,
5309
5521
  "page-token": page_token,
@@ -5555,24 +5767,27 @@ class HTTPRunDB(RunDBInterface):
5555
5767
  if page_params.get("page-token") is None and page_params.get("page") is None:
5556
5768
  page_params["page"] = 1
5557
5769
  if page_params.get("page-size") is None:
5558
- page_size = config.httpdb.pagination.default_page_size
5559
-
5560
- if page_params.get("limit") is not None:
5561
- page_size = page_params["limit"]
5770
+ page_params["page-size"] = config.httpdb.pagination.default_page_size
5562
5771
 
5563
- # limit and page/page size are conflicting
5564
- page_params.pop("limit")
5565
- page_params["page-size"] = page_size
5772
+ return page_params
5566
5773
 
5567
- # this may happen only when page-size was explicitly set along with limit
5568
- # this is to ensure we will not get stopped by API on similar below validation
5569
- # but rather simply fallback to use page-size.
5570
- if page_params.get("page-size") and page_params.get("limit"):
5571
- logger.warning(
5572
- "Both 'limit' and 'page-size' are provided, using 'page-size'."
5774
+ def _resolve_api_urls(self, url: str) -> tuple[str, str]:
5775
+ """
5776
+ Resolves the base url and parsed url from the given url.
5777
+ """
5778
+ parsed_url = urlparse(url)
5779
+ scheme = parsed_url.scheme.lower()
5780
+ if scheme not in ("http", "https"):
5781
+ raise ValueError(
5782
+ f"Invalid URL scheme {scheme} for HTTPRunDB, only http(s) is supported"
5573
5783
  )
5574
- page_params.pop("limit")
5575
- return page_params
5784
+
5785
+ endpoint = parsed_url.hostname
5786
+ if parsed_url.port:
5787
+ endpoint += f":{parsed_url.port}"
5788
+ base_url = f"{parsed_url.scheme}://{endpoint}{parsed_url.path}"
5789
+
5790
+ return base_url, parsed_url
5576
5791
 
5577
5792
 
5578
5793
  def _as_json(obj):
mlrun/db/nopdb.py CHANGED
@@ -212,7 +212,6 @@ class NopDB(RunDBInterface):
212
212
  tree: Optional[str] = None,
213
213
  parent: Optional[str] = None,
214
214
  format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
215
- limit: Optional[int] = None,
216
215
  partition_by: Optional[
217
216
  Union[mlrun.common.schemas.ArtifactPartitionByField, str]
218
217
  ] = None,
@@ -583,16 +582,6 @@ class NopDB(RunDBInterface):
583
582
  ):
584
583
  pass
585
584
 
586
- def create_user_secrets(
587
- self,
588
- user: str,
589
- provider: Union[
590
- str, mlrun.common.schemas.SecretProviderName
591
- ] = mlrun.common.schemas.SecretProviderName.vault,
592
- secrets: Optional[dict] = None,
593
- ):
594
- pass
595
-
596
585
  def create_model_endpoint(
597
586
  self,
598
587
  model_endpoint: mlrun.common.schemas.ModelEndpoint,
@@ -1008,3 +997,32 @@ class NopDB(RunDBInterface):
1008
997
  end: Optional[datetime.datetime] = None,
1009
998
  ) -> mlrun.common.schemas.model_monitoring.ModelEndpointDriftValues:
1010
999
  pass
1000
+
1001
+ def store_secret_token(
1002
+ self,
1003
+ secret_token: mlrun.common.schemas.SecretToken,
1004
+ log_warning: bool = True,
1005
+ force: bool = False,
1006
+ ) -> mlrun.common.schemas.StoreSecretTokensResponse:
1007
+ pass
1008
+
1009
+ def store_secret_tokens(
1010
+ self,
1011
+ secret_tokens: list[mlrun.common.schemas.SecretToken],
1012
+ log_warning: bool = True,
1013
+ force: bool = False,
1014
+ ) -> mlrun.common.schemas.StoreSecretTokensResponse:
1015
+ pass
1016
+
1017
+ def list_secret_tokens(self) -> mlrun.common.schemas.ListSecretTokensResponse:
1018
+ pass
1019
+
1020
+ def revoke_secret_token(self, token_name: str) -> None:
1021
+ pass
1022
+
1023
+ def get_secret_token(
1024
+ self,
1025
+ token_name: str,
1026
+ username: Optional[str] = None,
1027
+ ) -> mlrun.common.schemas.SecretToken:
1028
+ pass