mlrun 1.7.0rc14__py3-none-any.whl → 1.7.0rc21__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (152) hide show
  1. mlrun/__init__.py +10 -1
  2. mlrun/__main__.py +23 -111
  3. mlrun/alerts/__init__.py +15 -0
  4. mlrun/alerts/alert.py +144 -0
  5. mlrun/api/schemas/__init__.py +4 -3
  6. mlrun/artifacts/__init__.py +8 -3
  7. mlrun/artifacts/base.py +36 -253
  8. mlrun/artifacts/dataset.py +9 -190
  9. mlrun/artifacts/manager.py +46 -42
  10. mlrun/artifacts/model.py +9 -141
  11. mlrun/artifacts/plots.py +14 -375
  12. mlrun/common/constants.py +65 -3
  13. mlrun/common/formatters/__init__.py +19 -0
  14. mlrun/{runtimes/mpijob/v1alpha1.py → common/formatters/artifact.py} +6 -14
  15. mlrun/common/formatters/base.py +113 -0
  16. mlrun/common/formatters/function.py +46 -0
  17. mlrun/common/formatters/pipeline.py +53 -0
  18. mlrun/common/formatters/project.py +51 -0
  19. mlrun/{runtimes → common/runtimes}/constants.py +32 -4
  20. mlrun/common/schemas/__init__.py +10 -5
  21. mlrun/common/schemas/alert.py +92 -11
  22. mlrun/common/schemas/api_gateway.py +56 -0
  23. mlrun/common/schemas/artifact.py +15 -5
  24. mlrun/common/schemas/auth.py +2 -0
  25. mlrun/common/schemas/client_spec.py +1 -0
  26. mlrun/common/schemas/frontend_spec.py +1 -0
  27. mlrun/common/schemas/function.py +4 -0
  28. mlrun/common/schemas/model_monitoring/__init__.py +15 -3
  29. mlrun/common/schemas/model_monitoring/constants.py +58 -7
  30. mlrun/common/schemas/model_monitoring/grafana.py +9 -5
  31. mlrun/common/schemas/model_monitoring/model_endpoints.py +86 -2
  32. mlrun/common/schemas/pipeline.py +0 -9
  33. mlrun/common/schemas/project.py +5 -11
  34. mlrun/common/types.py +1 -0
  35. mlrun/config.py +27 -9
  36. mlrun/data_types/to_pandas.py +9 -9
  37. mlrun/datastore/base.py +41 -9
  38. mlrun/datastore/datastore.py +6 -2
  39. mlrun/datastore/datastore_profile.py +56 -4
  40. mlrun/datastore/inmem.py +2 -2
  41. mlrun/datastore/redis.py +2 -2
  42. mlrun/datastore/s3.py +5 -0
  43. mlrun/datastore/sources.py +147 -7
  44. mlrun/datastore/store_resources.py +7 -7
  45. mlrun/datastore/targets.py +110 -42
  46. mlrun/datastore/utils.py +42 -0
  47. mlrun/db/base.py +54 -10
  48. mlrun/db/httpdb.py +282 -79
  49. mlrun/db/nopdb.py +52 -10
  50. mlrun/errors.py +11 -0
  51. mlrun/execution.py +24 -9
  52. mlrun/feature_store/__init__.py +0 -2
  53. mlrun/feature_store/api.py +12 -47
  54. mlrun/feature_store/feature_set.py +9 -0
  55. mlrun/feature_store/feature_vector.py +8 -0
  56. mlrun/feature_store/ingestion.py +7 -6
  57. mlrun/feature_store/retrieval/base.py +9 -4
  58. mlrun/feature_store/retrieval/conversion.py +9 -9
  59. mlrun/feature_store/retrieval/dask_merger.py +2 -0
  60. mlrun/feature_store/retrieval/job.py +9 -3
  61. mlrun/feature_store/retrieval/local_merger.py +2 -0
  62. mlrun/feature_store/retrieval/spark_merger.py +16 -0
  63. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +7 -12
  64. mlrun/frameworks/parallel_coordinates.py +2 -1
  65. mlrun/frameworks/tf_keras/__init__.py +4 -1
  66. mlrun/k8s_utils.py +10 -11
  67. mlrun/launcher/base.py +4 -3
  68. mlrun/launcher/client.py +5 -3
  69. mlrun/launcher/local.py +8 -2
  70. mlrun/launcher/remote.py +8 -2
  71. mlrun/lists.py +6 -2
  72. mlrun/model.py +45 -21
  73. mlrun/model_monitoring/__init__.py +1 -1
  74. mlrun/model_monitoring/api.py +41 -18
  75. mlrun/model_monitoring/application.py +5 -305
  76. mlrun/model_monitoring/applications/__init__.py +11 -0
  77. mlrun/model_monitoring/applications/_application_steps.py +157 -0
  78. mlrun/model_monitoring/applications/base.py +280 -0
  79. mlrun/model_monitoring/applications/context.py +214 -0
  80. mlrun/model_monitoring/applications/evidently_base.py +211 -0
  81. mlrun/model_monitoring/applications/histogram_data_drift.py +132 -91
  82. mlrun/model_monitoring/applications/results.py +99 -0
  83. mlrun/model_monitoring/controller.py +3 -1
  84. mlrun/model_monitoring/db/__init__.py +2 -0
  85. mlrun/model_monitoring/db/stores/__init__.py +0 -2
  86. mlrun/model_monitoring/db/stores/base/store.py +22 -37
  87. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +43 -21
  88. mlrun/model_monitoring/db/stores/sqldb/models/base.py +39 -8
  89. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +27 -7
  90. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +5 -0
  91. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +246 -224
  92. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +232 -216
  93. mlrun/model_monitoring/db/tsdb/__init__.py +100 -0
  94. mlrun/model_monitoring/db/tsdb/base.py +329 -0
  95. mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
  96. mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
  97. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +240 -0
  98. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +45 -0
  99. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +397 -0
  100. mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
  101. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +117 -0
  102. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +636 -0
  103. mlrun/model_monitoring/evidently_application.py +6 -118
  104. mlrun/model_monitoring/helpers.py +46 -1
  105. mlrun/model_monitoring/model_endpoint.py +3 -2
  106. mlrun/model_monitoring/stream_processing.py +57 -216
  107. mlrun/model_monitoring/writer.py +134 -124
  108. mlrun/package/utils/_formatter.py +2 -2
  109. mlrun/platforms/__init__.py +10 -9
  110. mlrun/platforms/iguazio.py +21 -202
  111. mlrun/projects/operations.py +19 -12
  112. mlrun/projects/pipelines.py +79 -102
  113. mlrun/projects/project.py +265 -103
  114. mlrun/render.py +15 -14
  115. mlrun/run.py +16 -46
  116. mlrun/runtimes/__init__.py +6 -3
  117. mlrun/runtimes/base.py +8 -7
  118. mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
  119. mlrun/runtimes/funcdoc.py +0 -28
  120. mlrun/runtimes/kubejob.py +2 -1
  121. mlrun/runtimes/local.py +5 -2
  122. mlrun/runtimes/mpijob/__init__.py +0 -20
  123. mlrun/runtimes/mpijob/v1.py +1 -1
  124. mlrun/runtimes/nuclio/api_gateway.py +194 -84
  125. mlrun/runtimes/nuclio/application/application.py +170 -8
  126. mlrun/runtimes/nuclio/function.py +39 -49
  127. mlrun/runtimes/pod.py +16 -36
  128. mlrun/runtimes/remotesparkjob.py +9 -3
  129. mlrun/runtimes/sparkjob/spark3job.py +1 -1
  130. mlrun/runtimes/utils.py +6 -45
  131. mlrun/serving/server.py +2 -1
  132. mlrun/serving/v2_serving.py +5 -1
  133. mlrun/track/tracker.py +2 -1
  134. mlrun/utils/async_http.py +25 -5
  135. mlrun/utils/helpers.py +107 -75
  136. mlrun/utils/logger.py +39 -7
  137. mlrun/utils/notifications/notification/__init__.py +14 -9
  138. mlrun/utils/notifications/notification/base.py +1 -1
  139. mlrun/utils/notifications/notification/slack.py +34 -7
  140. mlrun/utils/notifications/notification/webhook.py +1 -1
  141. mlrun/utils/notifications/notification_pusher.py +147 -16
  142. mlrun/utils/regex.py +9 -0
  143. mlrun/utils/v3io_clients.py +0 -1
  144. mlrun/utils/version/version.json +2 -2
  145. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/METADATA +14 -6
  146. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/RECORD +150 -130
  147. mlrun/kfpops.py +0 -865
  148. mlrun/platforms/other.py +0 -305
  149. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/LICENSE +0 -0
  150. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/WHEEL +0 -0
  151. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/entry_points.txt +0 -0
  152. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/top_level.txt +0 -0
mlrun/db/httpdb.py CHANGED
@@ -15,7 +15,6 @@
15
15
  import enum
16
16
  import http
17
17
  import re
18
- import tempfile
19
18
  import time
20
19
  import traceback
21
20
  import typing
@@ -26,11 +25,13 @@ from os import path, remove
26
25
  from typing import Optional, Union
27
26
  from urllib.parse import urlparse
28
27
 
29
- import kfp
30
28
  import requests
31
29
  import semver
30
+ from mlrun_pipelines.utils import compile_pipeline
32
31
 
33
32
  import mlrun
33
+ import mlrun.common.formatters
34
+ import mlrun.common.runtimes
34
35
  import mlrun.common.schemas
35
36
  import mlrun.common.types
36
37
  import mlrun.model_monitoring.model_endpoint
@@ -38,6 +39,7 @@ import mlrun.platforms
38
39
  import mlrun.projects
39
40
  import mlrun.runtimes.nuclio.api_gateway
40
41
  import mlrun.utils
42
+ from mlrun.alerts.alert import AlertConfig
41
43
  from mlrun.db.auth_utils import OAuthClientIDTokenProvider, StaticTokenProvider
42
44
  from mlrun.errors import MLRunInvalidArgumentError, err_to_str
43
45
 
@@ -51,7 +53,6 @@ from ..utils import (
51
53
  datetime_to_iso,
52
54
  dict_to_json,
53
55
  logger,
54
- new_pipe_metadata,
55
56
  normalize_name,
56
57
  version,
57
58
  )
@@ -214,7 +215,7 @@ class HTTPRunDB(RunDBInterface):
214
215
  :param version: API version to use, None (the default) will mean to use the default value from config,
215
216
  for un-versioned api set an empty string.
216
217
 
217
- :return: `requests.Response` HTTP response object
218
+ :returns: `requests.Response` HTTP response object
218
219
  """
219
220
  url = self.get_base_api_url(path, version)
220
221
  kw = {
@@ -531,6 +532,10 @@ class HTTPRunDB(RunDBInterface):
531
532
  server_cfg.get("model_endpoint_monitoring_endpoint_store_connection")
532
533
  or config.model_endpoint_monitoring.endpoint_store_connection
533
534
  )
535
+ config.model_endpoint_monitoring.tsdb_connection = (
536
+ server_cfg.get("model_monitoring_tsdb_connection")
537
+ or config.model_endpoint_monitoring.tsdb_connection
538
+ )
534
539
  config.packagers = server_cfg.get("packagers") or config.packagers
535
540
  server_data_prefixes = server_cfg.get("feature_store_data_prefixes") or {}
536
541
  for prefix in ["default", "nosql", "redisnosql"]:
@@ -590,7 +595,7 @@ class HTTPRunDB(RunDBInterface):
590
595
  if offset < 0:
591
596
  raise MLRunInvalidArgumentError("Offset cannot be negative")
592
597
  if size is None:
593
- size = int(config.httpdb.logs.pull_logs_default_size_limit)
598
+ size = int(mlrun.mlconf.httpdb.logs.pull_logs_default_size_limit)
594
599
  elif size == -1:
595
600
  logger.warning(
596
601
  "Retrieving all logs. This may be inefficient and can result in a large log."
@@ -636,33 +641,35 @@ class HTTPRunDB(RunDBInterface):
636
641
 
637
642
  state, text = self.get_log(uid, project, offset=offset)
638
643
  if text:
639
- print(text.decode(errors=config.httpdb.logs.decode.errors))
644
+ print(text.decode(errors=mlrun.mlconf.httpdb.logs.decode.errors))
640
645
  nil_resp = 0
641
646
  while True:
642
647
  offset += len(text)
643
648
  # if we get 3 nil responses in a row, increase the sleep time to 10 seconds
644
649
  # TODO: refactor this to use a conditional backoff mechanism
645
650
  if nil_resp < 3:
646
- time.sleep(int(config.httpdb.logs.pull_logs_default_interval))
651
+ time.sleep(int(mlrun.mlconf.httpdb.logs.pull_logs_default_interval))
647
652
  else:
648
653
  time.sleep(
649
- int(config.httpdb.logs.pull_logs_backoff_no_logs_default_interval)
654
+ int(
655
+ mlrun.mlconf.httpdb.logs.pull_logs_backoff_no_logs_default_interval
656
+ )
650
657
  )
651
658
  state, text = self.get_log(uid, project, offset=offset)
652
659
  if text:
653
660
  nil_resp = 0
654
661
  print(
655
- text.decode(errors=config.httpdb.logs.decode.errors),
662
+ text.decode(errors=mlrun.mlconf.httpdb.logs.decode.errors),
656
663
  end="",
657
664
  )
658
665
  else:
659
666
  nil_resp += 1
660
667
 
661
668
  if watch and state in [
662
- mlrun.runtimes.constants.RunStates.pending,
663
- mlrun.runtimes.constants.RunStates.running,
664
- mlrun.runtimes.constants.RunStates.created,
665
- mlrun.runtimes.constants.RunStates.aborting,
669
+ mlrun.common.runtimes.constants.RunStates.pending,
670
+ mlrun.common.runtimes.constants.RunStates.running,
671
+ mlrun.common.runtimes.constants.RunStates.created,
672
+ mlrun.common.runtimes.constants.RunStates.aborting,
666
673
  ]:
667
674
  continue
668
675
  else:
@@ -751,7 +758,10 @@ class HTTPRunDB(RunDBInterface):
751
758
  uid: Optional[Union[str, list[str]]] = None,
752
759
  project: Optional[str] = None,
753
760
  labels: Optional[Union[str, list[str]]] = None,
754
- state: Optional[str] = None,
761
+ state: Optional[
762
+ mlrun.common.runtimes.constants.RunStates
763
+ ] = None, # Backward compatibility
764
+ states: typing.Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
755
765
  sort: bool = True,
756
766
  last: int = 0,
757
767
  iter: bool = False,
@@ -789,7 +799,8 @@ class HTTPRunDB(RunDBInterface):
789
799
  :param labels: A list of labels to filter by. Label filters work by either filtering a specific value
790
800
  of a label (i.e. list("key=value")) or by looking for the existence of a given
791
801
  key (i.e. "key").
792
- :param state: List only runs whose state is specified.
802
+ :param state: Deprecated - List only runs whose state is specified (will be removed in 1.9.0)
803
+ :param states: List only runs whose state is one of the provided states.
793
804
  :param sort: Whether to sort the result according to their start time. Otherwise, results will be
794
805
  returned by their internal order in the DB (order will not be guaranteed).
795
806
  :param last: Deprecated - currently not used (will be removed in 1.8.0).
@@ -825,11 +836,19 @@ class HTTPRunDB(RunDBInterface):
825
836
  FutureWarning,
826
837
  )
827
838
 
839
+ if state:
840
+ # TODO: Remove this in 1.9.0
841
+ warnings.warn(
842
+ "'state' is deprecated and will be removed in 1.9.0. Use 'states' instead.",
843
+ FutureWarning,
844
+ )
845
+
828
846
  if (
829
847
  not name
830
848
  and not uid
831
849
  and not labels
832
850
  and not state
851
+ and not states
833
852
  and not last
834
853
  and not start_time_from
835
854
  and not start_time_to
@@ -848,7 +867,9 @@ class HTTPRunDB(RunDBInterface):
848
867
  "name": name,
849
868
  "uid": uid,
850
869
  "label": labels or [],
851
- "state": state,
870
+ "state": mlrun.utils.helpers.as_list(state)
871
+ if state is not None
872
+ else states or None,
852
873
  "sort": bool2str(sort),
853
874
  "iter": bool2str(iter),
854
875
  "start_time_from": datetime_to_iso(start_time_from),
@@ -975,7 +996,7 @@ class HTTPRunDB(RunDBInterface):
975
996
  error = f"read artifact {project}/{key}"
976
997
  # explicitly set artifacts format to 'full' since old servers may default to 'legacy'
977
998
  params = {
978
- "format": mlrun.common.schemas.ArtifactsFormat.full.value,
999
+ "format": mlrun.common.formatters.ArtifactFormat.full.value,
979
1000
  "tag": tag,
980
1001
  "tree": tree,
981
1002
  "uid": uid,
@@ -985,7 +1006,18 @@ class HTTPRunDB(RunDBInterface):
985
1006
  resp = self.api_call("GET", endpoint_path, error, params=params, version="v2")
986
1007
  return resp.json()
987
1008
 
988
- def del_artifact(self, key, tag=None, project="", tree=None, uid=None):
1009
+ def del_artifact(
1010
+ self,
1011
+ key,
1012
+ tag=None,
1013
+ project="",
1014
+ tree=None,
1015
+ uid=None,
1016
+ deletion_strategy: mlrun.common.schemas.artifact.ArtifactsDeletionStrategies = (
1017
+ mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
1018
+ ),
1019
+ secrets: dict = None,
1020
+ ):
989
1021
  """Delete an artifact.
990
1022
 
991
1023
  :param key: Identifying key of the artifact.
@@ -993,6 +1025,8 @@ class HTTPRunDB(RunDBInterface):
993
1025
  :param project: Project that the artifact belongs to.
994
1026
  :param tree: The tree which generated this artifact.
995
1027
  :param uid: A unique ID for this specific version of the artifact (the uid that was generated in the backend)
1028
+ :param deletion_strategy: The artifact deletion strategy types.
1029
+ :param secrets: Credentials needed to access the artifact data.
996
1030
  """
997
1031
 
998
1032
  endpoint_path = f"projects/{project}/artifacts/{key}"
@@ -1001,9 +1035,17 @@ class HTTPRunDB(RunDBInterface):
1001
1035
  "tag": tag,
1002
1036
  "tree": tree,
1003
1037
  "uid": uid,
1038
+ "deletion_strategy": deletion_strategy,
1004
1039
  }
1005
1040
  error = f"del artifact {project}/{key}"
1006
- self.api_call("DELETE", endpoint_path, error, params=params, version="v2")
1041
+ self.api_call(
1042
+ "DELETE",
1043
+ endpoint_path,
1044
+ error,
1045
+ params=params,
1046
+ version="v2",
1047
+ body=dict_to_json(secrets),
1048
+ )
1007
1049
 
1008
1050
  def list_artifacts(
1009
1051
  self,
@@ -1018,6 +1060,7 @@ class HTTPRunDB(RunDBInterface):
1018
1060
  kind: str = None,
1019
1061
  category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
1020
1062
  tree: str = None,
1063
+ producer_uri: str = None,
1021
1064
  ) -> ArtifactList:
1022
1065
  """List artifacts filtered by various parameters.
1023
1066
 
@@ -1046,9 +1089,12 @@ class HTTPRunDB(RunDBInterface):
1046
1089
  :param best_iteration: Returns the artifact which belongs to the best iteration of a given run, in the case of
1047
1090
  artifacts generated from a hyper-param run. If only a single iteration exists, will return the artifact
1048
1091
  from that iteration. If using ``best_iter``, the ``iter`` parameter must not be used.
1049
- :param kind: Return artifacts of the requested kind.
1050
- :param category: Return artifacts of the requested category.
1051
- :param tree: Return artifacts of the requested tree.
1092
+ :param kind: Return artifacts of the requested kind.
1093
+ :param category: Return artifacts of the requested category.
1094
+ :param tree: Return artifacts of the requested tree.
1095
+ :param producer_uri: Return artifacts produced by the requested producer URI. Producer URI usually
1096
+ points to a run and is used to filter artifacts by the run that produced them when the artifact producer id
1097
+ is a workflow id (artifact was created as part of a workflow).
1052
1098
  """
1053
1099
 
1054
1100
  project = project or config.default_project
@@ -1066,7 +1112,8 @@ class HTTPRunDB(RunDBInterface):
1066
1112
  "kind": kind,
1067
1113
  "category": category,
1068
1114
  "tree": tree,
1069
- "format": mlrun.common.schemas.ArtifactsFormat.full.value,
1115
+ "format": mlrun.common.formatters.ArtifactFormat.full.value,
1116
+ "producer_uri": producer_uri,
1070
1117
  }
1071
1118
  error = "list artifacts"
1072
1119
  endpoint_path = f"projects/{project}/artifacts"
@@ -1177,7 +1224,10 @@ class HTTPRunDB(RunDBInterface):
1177
1224
  == mlrun.common.schemas.BackgroundTaskState.failed
1178
1225
  ):
1179
1226
  logger.info(
1180
- "Function deletion failed", project_name=project, function_name=name
1227
+ "Function deletion failed",
1228
+ reason=background_task.status.error,
1229
+ project_name=project,
1230
+ function_name=name,
1181
1231
  )
1182
1232
 
1183
1233
  def list_functions(self, name=None, project=None, tag=None, labels=None):
@@ -1479,6 +1529,7 @@ class HTTPRunDB(RunDBInterface):
1479
1529
  ):
1480
1530
  """
1481
1531
  Deploy a Nuclio function.
1532
+
1482
1533
  :param func: Function to build.
1483
1534
  :param builder_env: Kaniko builder pod env vars dict (for config/credentials)
1484
1535
  """
@@ -1828,14 +1879,11 @@ class HTTPRunDB(RunDBInterface):
1828
1879
  if isinstance(pipeline, str):
1829
1880
  pipe_file = pipeline
1830
1881
  else:
1831
- pipe_file = tempfile.NamedTemporaryFile(suffix=".yaml", delete=False).name
1832
- conf = new_pipe_metadata(
1882
+ pipe_file = compile_pipeline(
1833
1883
  artifact_path=artifact_path,
1834
1884
  cleanup_ttl=cleanup_ttl,
1835
- op_transformers=ops,
1836
- )
1837
- kfp.compiler.Compiler().compile(
1838
- pipeline, pipe_file, type_check=False, pipeline_conf=conf
1885
+ ops=ops,
1886
+ pipeline=pipeline,
1839
1887
  )
1840
1888
 
1841
1889
  if pipe_file.endswith(".yaml"):
@@ -1890,8 +1938,8 @@ class HTTPRunDB(RunDBInterface):
1890
1938
  page_token: str = "",
1891
1939
  filter_: str = "",
1892
1940
  format_: Union[
1893
- str, mlrun.common.schemas.PipelinesFormat
1894
- ] = mlrun.common.schemas.PipelinesFormat.metadata_only,
1941
+ str, mlrun.common.formatters.PipelineFormat
1942
+ ] = mlrun.common.formatters.PipelineFormat.metadata_only,
1895
1943
  page_size: int = None,
1896
1944
  ) -> mlrun.common.schemas.PipelinesOutput:
1897
1945
  """Retrieve a list of KFP pipelines. This function can be invoked to get all pipelines from all projects,
@@ -1937,8 +1985,8 @@ class HTTPRunDB(RunDBInterface):
1937
1985
  namespace: str = None,
1938
1986
  timeout: int = 30,
1939
1987
  format_: Union[
1940
- str, mlrun.common.schemas.PipelinesFormat
1941
- ] = mlrun.common.schemas.PipelinesFormat.summary,
1988
+ str, mlrun.common.formatters.PipelineFormat
1989
+ ] = mlrun.common.formatters.PipelineFormat.summary,
1942
1990
  project: str = None,
1943
1991
  ):
1944
1992
  """Retrieve details of a specific pipeline using its run ID (as provided when the pipeline was executed)."""
@@ -2580,8 +2628,8 @@ class HTTPRunDB(RunDBInterface):
2580
2628
  self,
2581
2629
  owner: str = None,
2582
2630
  format_: Union[
2583
- str, mlrun.common.schemas.ProjectsFormat
2584
- ] = mlrun.common.schemas.ProjectsFormat.name_only,
2631
+ str, mlrun.common.formatters.ProjectFormat
2632
+ ] = mlrun.common.formatters.ProjectFormat.name_only,
2585
2633
  labels: list[str] = None,
2586
2634
  state: Union[str, mlrun.common.schemas.ProjectState] = None,
2587
2635
  ) -> list[Union[mlrun.projects.MlrunProject, str]]:
@@ -2607,7 +2655,7 @@ class HTTPRunDB(RunDBInterface):
2607
2655
 
2608
2656
  error_message = f"Failed listing projects, query: {params}"
2609
2657
  response = self.api_call("GET", "projects", error_message, params=params)
2610
- if format_ == mlrun.common.schemas.ProjectsFormat.name_only:
2658
+ if format_ == mlrun.common.formatters.ProjectFormat.name_only:
2611
2659
  # projects is just a list of strings
2612
2660
  return response.json()["projects"]
2613
2661
 
@@ -3112,14 +3160,12 @@ class HTTPRunDB(RunDBInterface):
3112
3160
  :param labels: A list of labels to filter by. Label filters work by either filtering a specific value of a
3113
3161
  label (i.e. list("key=value")) or by looking for the existence of a given key (i.e. "key")
3114
3162
  :param metrics: A list of metrics to return for each endpoint, read more in 'TimeMetric'
3115
- :param start: The start time of the metrics. Can be represented by a string containing an RFC 3339
3116
- time, a Unix timestamp in milliseconds, a relative time (`'now'` or
3117
- `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` =
3118
- days), or 0 for the earliest time.
3119
- :param end: The end time of the metrics. Can be represented by a string containing an RFC 3339
3120
- time, a Unix timestamp in milliseconds, a relative time (`'now'` or
3121
- `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` =
3122
- days), or 0 for the earliest time.
3163
+ :param start: The start time of the metrics. Can be represented by a string containing an RFC 3339 time, a
3164
+ Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
3165
+ `m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
3166
+ :param end: The end time of the metrics. Can be represented by a string containing an RFC 3339 time, a
3167
+ Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
3168
+ `m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
3123
3169
  :param top_level: if true will return only routers and endpoint that are NOT children of any router
3124
3170
  :param uids: if passed will return a list `ModelEndpoint` object with uid in uids
3125
3171
  """
@@ -3168,13 +3214,13 @@ class HTTPRunDB(RunDBInterface):
3168
3214
  :param project: The name of the project
3169
3215
  :param endpoint_id: The unique id of the model endpoint.
3170
3216
  :param start: The start time of the metrics. Can be represented by a string containing an
3171
- RFC 3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
3172
- `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or
3173
- 0 for the earliest time.
3217
+ RFC 3339 time, a Unix timestamp in milliseconds, a relative time
3218
+ (`'now'` or `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours,
3219
+ `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
3174
3220
  :param end: The end time of the metrics. Can be represented by a string containing an
3175
- RFC 3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
3176
- `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or
3177
- 0 for the earliest time.
3221
+ RFC 3339 time, a Unix timestamp in milliseconds, a relative time
3222
+ (`'now'` or `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours,
3223
+ `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
3178
3224
  :param metrics: A list of metrics to return for the model endpoint. There are pre-defined
3179
3225
  metrics for model endpoints such as predictions_per_second and
3180
3226
  latency_avg_5m but also custom metrics defined by the user. Please note that
@@ -3183,7 +3229,7 @@ class HTTPRunDB(RunDBInterface):
3183
3229
  :param feature_analysis: When True, the base feature statistics and current feature statistics will
3184
3230
  be added to the output of the resulting object.
3185
3231
 
3186
- :return: A `ModelEndpoint` object.
3232
+ :returns: A `ModelEndpoint` object.
3187
3233
  """
3188
3234
 
3189
3235
  path = f"projects/{project}/model-endpoints/{endpoint_id}"
@@ -3301,6 +3347,113 @@ class HTTPRunDB(RunDBInterface):
3301
3347
  },
3302
3348
  )
3303
3349
 
3350
+ def disable_model_monitoring(
3351
+ self,
3352
+ project: str,
3353
+ delete_resources: bool = True,
3354
+ delete_stream_function: bool = False,
3355
+ delete_histogram_data_drift_app: bool = True,
3356
+ delete_user_applications: bool = False,
3357
+ user_application_list: list[str] = None,
3358
+ ) -> bool:
3359
+ """
3360
+ Disable model monitoring application controller, writer, stream, histogram data drift application
3361
+ and the user's applications functions, according to the given params.
3362
+
3363
+ :param project: Project name.
3364
+ :param delete_resources: If True, it would delete the model monitoring controller & writer
3365
+ functions. Default True
3366
+ :param delete_stream_function: If True, it would delete model monitoring stream function,
3367
+ need to use wisely because if you're deleting this function
3368
+ this can cause data loss in case you will want to
3369
+ enable the model monitoring capability to the project.
3370
+ Default False.
3371
+ :param delete_histogram_data_drift_app: If True, it would delete the default histogram-based data drift
3372
+ application. Default False.
3373
+ :param delete_user_applications: If True, it would delete the user's model monitoring
3374
+ application according to user_application_list, Default False.
3375
+ :param user_application_list: List of the user's model monitoring application to disable.
3376
+ Default all the applications.
3377
+ Note: you have to set delete_user_applications to True
3378
+ in order to delete the desired application.
3379
+
3380
+ :returns: True if the deletion was successful, False otherwise.
3381
+ """
3382
+ response = self.api_call(
3383
+ method=mlrun.common.types.HTTPMethod.DELETE,
3384
+ path=f"projects/{project}/model-monitoring/disable-model-monitoring",
3385
+ params={
3386
+ "delete_resources": delete_resources,
3387
+ "delete_stream_function": delete_stream_function,
3388
+ "delete_histogram_data_drift_app": delete_histogram_data_drift_app,
3389
+ "delete_user_applications": delete_user_applications,
3390
+ "user_application_list": user_application_list,
3391
+ },
3392
+ )
3393
+ deletion_failed = False
3394
+ if response.status_code == http.HTTPStatus.ACCEPTED:
3395
+ if delete_resources:
3396
+ logger.info(
3397
+ "Model Monitoring is being disable",
3398
+ project_name=project,
3399
+ )
3400
+ if delete_user_applications:
3401
+ logger.info("User applications are being deleted", project_name=project)
3402
+ background_tasks = mlrun.common.schemas.BackgroundTaskList(
3403
+ **response.json()
3404
+ ).background_tasks
3405
+ for task in background_tasks:
3406
+ task = self._wait_for_background_task_to_reach_terminal_state(
3407
+ task.metadata.name, project=project
3408
+ )
3409
+ if (
3410
+ task.status.state
3411
+ == mlrun.common.schemas.BackgroundTaskState.succeeded
3412
+ ):
3413
+ continue
3414
+ elif (
3415
+ task.status.state == mlrun.common.schemas.BackgroundTaskState.failed
3416
+ ):
3417
+ deletion_failed = True
3418
+ return not deletion_failed
3419
+
3420
+ def delete_model_monitoring_function(
3421
+ self, project: str, functions: list[str]
3422
+ ) -> bool:
3423
+ """
3424
+ Delete a model monitoring application.
3425
+
3426
+ :param functions: List of the model monitoring function to delete.
3427
+ :param project: Project name.
3428
+
3429
+ :returns: True if the deletion was successful, False otherwise.
3430
+ """
3431
+ response = self.api_call(
3432
+ method=mlrun.common.types.HTTPMethod.DELETE,
3433
+ path=f"projects/{project}/model-monitoring/functions",
3434
+ params={"functions": functions},
3435
+ )
3436
+ deletion_failed = False
3437
+ if response.status_code == http.HTTPStatus.ACCEPTED:
3438
+ logger.info("User applications are being deleted", project_name=project)
3439
+ background_tasks = mlrun.common.schemas.BackgroundTaskList(
3440
+ **response.json()
3441
+ ).background_tasks
3442
+ for task in background_tasks:
3443
+ task = self._wait_for_background_task_to_reach_terminal_state(
3444
+ task.metadata.name, project=project
3445
+ )
3446
+ if (
3447
+ task.status.state
3448
+ == mlrun.common.schemas.BackgroundTaskState.succeeded
3449
+ ):
3450
+ continue
3451
+ elif (
3452
+ task.status.state == mlrun.common.schemas.BackgroundTaskState.failed
3453
+ ):
3454
+ deletion_failed = True
3455
+ return not deletion_failed
3456
+
3304
3457
  def deploy_histogram_data_drift_app(
3305
3458
  self, project: str, image: str = "mlrun/mlrun"
3306
3459
  ) -> None:
@@ -3528,7 +3681,7 @@ class HTTPRunDB(RunDBInterface):
3528
3681
  :param version: Get a specific version of the item. Default is ``None``.
3529
3682
  :param tag: Get a specific version of the item identified by tag. Default is ``latest``.
3530
3683
 
3531
- :return: http response with the asset in the content attribute
3684
+ :returns: http response with the asset in the content attribute
3532
3685
  """
3533
3686
  path = f"hub/sources/{source_name}/items/{item_name}/assets/{asset_name}"
3534
3687
  params = {
@@ -3559,9 +3712,10 @@ class HTTPRunDB(RunDBInterface):
3559
3712
  def list_api_gateways(self, project=None) -> mlrun.common.schemas.APIGatewaysOutput:
3560
3713
  """
3561
3714
  Returns a list of Nuclio api gateways
3715
+
3562
3716
  :param project: optional str parameter to filter by project, if not passed, default project value is taken
3563
3717
 
3564
- :return: :py:class:`~mlrun.common.schemas.APIGateways`.
3718
+ :returns: :py:class:`~mlrun.common.schemas.APIGateways`.
3565
3719
  """
3566
3720
  project = project or config.default_project
3567
3721
  error = "list api gateways"
@@ -3572,10 +3726,11 @@ class HTTPRunDB(RunDBInterface):
3572
3726
  def get_api_gateway(self, name, project=None) -> mlrun.common.schemas.APIGateway:
3573
3727
  """
3574
3728
  Returns an API gateway
3729
+
3575
3730
  :param name: API gateway name
3576
3731
  :param project: optional str parameter to filter by project, if not passed, default project value is taken
3577
3732
 
3578
- :return: :py:class:`~mlrun.common.schemas.APIGateway`.
3733
+ :returns: :py:class:`~mlrun.common.schemas.APIGateway`.
3579
3734
  """
3580
3735
  project = project or config.default_project
3581
3736
  error = "get api gateway"
@@ -3586,6 +3741,7 @@ class HTTPRunDB(RunDBInterface):
3586
3741
  def delete_api_gateway(self, name, project=None):
3587
3742
  """
3588
3743
  Deletes an API gateway
3744
+
3589
3745
  :param name: API gateway name
3590
3746
  :param project: Project name
3591
3747
  """
@@ -3604,11 +3760,12 @@ class HTTPRunDB(RunDBInterface):
3604
3760
  ) -> mlrun.common.schemas.APIGateway:
3605
3761
  """
3606
3762
  Stores an API Gateway.
3607
- :param api_gateway :py:class:`~mlrun.runtimes.nuclio.APIGateway`
3763
+
3764
+ :param api_gateway: :py:class:`~mlrun.runtimes.nuclio.APIGateway`
3608
3765
  or :py:class:`~mlrun.common.schemas.APIGateway`: API Gateway entity.
3609
3766
  :param project: project name. Mandatory if api_gateway is mlrun.common.schemas.APIGateway.
3610
3767
 
3611
- :return: :py:class:`~mlrun.common.schemas.APIGateway`.
3768
+ :returns: :py:class:`~mlrun.common.schemas.APIGateway`.
3612
3769
  """
3613
3770
 
3614
3771
  if isinstance(api_gateway, mlrun.runtimes.nuclio.api_gateway.APIGateway):
@@ -3626,6 +3783,7 @@ class HTTPRunDB(RunDBInterface):
3626
3783
  def trigger_migrations(self) -> Optional[mlrun.common.schemas.BackgroundTask]:
3627
3784
  """Trigger migrations (will do nothing if no migrations are needed) and wait for them to finish if actually
3628
3785
  triggered
3786
+
3629
3787
  :returns: :py:class:`~mlrun.common.schemas.BackgroundTask`.
3630
3788
  """
3631
3789
  response = self.api_call(
@@ -3648,6 +3806,7 @@ class HTTPRunDB(RunDBInterface):
3648
3806
  ):
3649
3807
  """
3650
3808
  Set notifications on a run. This will override any existing notifications on the run.
3809
+
3651
3810
  :param project: Project containing the run.
3652
3811
  :param run_uid: UID of the run.
3653
3812
  :param notifications: List of notifications to set on the run. Default is an empty list.
@@ -3673,6 +3832,7 @@ class HTTPRunDB(RunDBInterface):
3673
3832
  ):
3674
3833
  """
3675
3834
  Set notifications on a schedule. This will override any existing notifications on the schedule.
3835
+
3676
3836
  :param project: Project containing the schedule.
3677
3837
  :param schedule_name: Name of the schedule.
3678
3838
  :param notifications: List of notifications to set on the schedule. Default is an empty list.
@@ -3821,15 +3981,16 @@ class HTTPRunDB(RunDBInterface):
3821
3981
  ) -> str:
3822
3982
  """
3823
3983
  Loading a project remotely from the given source.
3984
+
3824
3985
  :param name: project name
3825
3986
  :param url: git or tar.gz or .zip sources archive path e.g.:
3826
- git://github.com/mlrun/demo-xgb-project.git
3827
- http://mysite/archived-project.zip
3828
- The git project should include the project yaml file.
3987
+ git://github.com/mlrun/demo-xgb-project.git
3988
+ http://mysite/archived-project.zip
3989
+ The git project should include the project yaml file.
3829
3990
  :param secrets: Secrets to store in project in order to load it from the provided url. For more
3830
- information see :py:func:`mlrun.load_project` function.
3991
+ information see :py:func:`mlrun.load_project` function.
3831
3992
  :param save_secrets: Whether to store secrets in the loaded project. Setting to False will cause waiting
3832
- for the process completion.
3993
+ for the process completion.
3833
3994
 
3834
3995
  :returns: The terminal state of load project process.
3835
3996
  """
@@ -3915,7 +4076,7 @@ class HTTPRunDB(RunDBInterface):
3915
4076
  logger.warning(
3916
4077
  "Building a function image to ECR and loading an S3 source to the image may require conflicting access "
3917
4078
  "keys. Only the permissions granted to the platform's configured secret will take affect "
3918
- "(see mlrun.config.config.httpdb.builder.docker_registry_secret). "
4079
+ "(see mlrun.mlconf.httpdb.builder.docker_registry_secret). "
3919
4080
  "In case the permissions are limited to ECR scope, you may use pull_at_runtime=True instead",
3920
4081
  source=func.spec.build.source,
3921
4082
  load_source_on_run=func.spec.build.load_source_on_run,
@@ -3927,9 +4088,10 @@ class HTTPRunDB(RunDBInterface):
3927
4088
  ):
3928
4089
  """
3929
4090
  Generate an event.
3930
- :param name: The name of the event.
4091
+
4092
+ :param name: The name of the event.
3931
4093
  :param event_data: The data of the event.
3932
- :param project: The project that the event belongs to.
4094
+ :param project: The project that the event belongs to.
3933
4095
  """
3934
4096
  project = project or config.default_project
3935
4097
  endpoint_path = f"projects/{project}/events/{name}"
@@ -3943,43 +4105,54 @@ class HTTPRunDB(RunDBInterface):
3943
4105
  def store_alert_config(
3944
4106
  self,
3945
4107
  alert_name: str,
3946
- alert_data: Union[dict, mlrun.common.schemas.AlertConfig],
4108
+ alert_data: Union[dict, AlertConfig],
3947
4109
  project="",
3948
- ):
4110
+ ) -> AlertConfig:
3949
4111
  """
3950
4112
  Create/modify an alert.
4113
+
3951
4114
  :param alert_name: The name of the alert.
3952
4115
  :param alert_data: The data of the alert.
3953
- :param project: the project that the alert belongs to.
3954
- :return: The created/modified alert.
4116
+ :param project: The project that the alert belongs to.
4117
+ :returns: The created/modified alert.
3955
4118
  """
3956
4119
  project = project or config.default_project
3957
4120
  endpoint_path = f"projects/{project}/alerts/{alert_name}"
3958
4121
  error_message = f"put alert {project}/alerts/{alert_name}"
3959
- if isinstance(alert_data, mlrun.common.schemas.AlertConfig):
3960
- alert_data = alert_data.dict()
4122
+ alert_instance = (
4123
+ alert_data
4124
+ if isinstance(alert_data, AlertConfig)
4125
+ else AlertConfig.from_dict(alert_data)
4126
+ )
4127
+ alert_instance.validate_required_fields()
4128
+
4129
+ alert_data = alert_instance.to_dict()
3961
4130
  body = _as_json(alert_data)
3962
4131
  response = self.api_call("PUT", endpoint_path, error_message, body=body)
3963
- return mlrun.common.schemas.AlertConfig(**response.json())
4132
+ return AlertConfig.from_dict(response.json())
3964
4133
 
3965
- def get_alert_config(self, alert_name: str, project=""):
4134
+ def get_alert_config(self, alert_name: str, project="") -> AlertConfig:
3966
4135
  """
3967
4136
  Retrieve an alert.
4137
+
3968
4138
  :param alert_name: The name of the alert to retrieve.
3969
- :param project: The project that the alert belongs to.
3970
- :return: The alert object.
4139
+ :param project: The project that the alert belongs to.
4140
+
4141
+ :returns: The alert object.
3971
4142
  """
3972
4143
  project = project or config.default_project
3973
4144
  endpoint_path = f"projects/{project}/alerts/{alert_name}"
3974
4145
  error_message = f"get alert {project}/alerts/{alert_name}"
3975
4146
  response = self.api_call("GET", endpoint_path, error_message)
3976
- return mlrun.common.schemas.AlertConfig(**response.json())
4147
+ return AlertConfig.from_dict(response.json())
3977
4148
 
3978
- def list_alerts_configs(self, project=""):
4149
+ def list_alerts_configs(self, project="") -> list[AlertConfig]:
3979
4150
  """
3980
4151
  Retrieve list of alerts of a project.
4152
+
3981
4153
  :param project: The project name.
3982
- :return: All the alerts objects of the project.
4154
+
4155
+ :returns: All the alerts objects of the project.
3983
4156
  """
3984
4157
  project = project or config.default_project
3985
4158
  endpoint_path = f"projects/{project}/alerts"
@@ -3987,7 +4160,7 @@ class HTTPRunDB(RunDBInterface):
3987
4160
  response = self.api_call("GET", endpoint_path, error_message).json()
3988
4161
  results = []
3989
4162
  for item in response:
3990
- results.append(mlrun.common.schemas.AlertConfig(**item))
4163
+ results.append(AlertConfig(**item))
3991
4164
  return results
3992
4165
 
3993
4166
  def delete_alert_config(self, alert_name: str, project=""):
@@ -4004,6 +4177,7 @@ class HTTPRunDB(RunDBInterface):
4004
4177
  def reset_alert_config(self, alert_name: str, project=""):
4005
4178
  """
4006
4179
  Reset an alert.
4180
+
4007
4181
  :param alert_name: The name of the alert to reset.
4008
4182
  :param project: The project that the alert belongs to.
4009
4183
  """
@@ -4012,6 +4186,35 @@ class HTTPRunDB(RunDBInterface):
4012
4186
  error_message = f"post alert {project}/alerts/{alert_name}/reset"
4013
4187
  self.api_call("POST", endpoint_path, error_message)
4014
4188
 
4189
+ def get_alert_template(
4190
+ self, template_name: str
4191
+ ) -> mlrun.common.schemas.AlertTemplate:
4192
+ """
4193
+ Retrieve a specific alert template.
4194
+
4195
+ :param template_name: The name of the template to retrieve.
4196
+
4197
+ :returns: The template object.
4198
+ """
4199
+ endpoint_path = f"alert-templates/{template_name}"
4200
+ error_message = f"get template alert-templates/{template_name}"
4201
+ response = self.api_call("GET", endpoint_path, error_message)
4202
+ return mlrun.common.schemas.AlertTemplate(**response.json())
4203
+
4204
+ def list_alert_templates(self) -> list[mlrun.common.schemas.AlertTemplate]:
4205
+ """
4206
+ Retrieve list of all alert templates.
4207
+
4208
+ :returns: All the alert template objects in the database.
4209
+ """
4210
+ endpoint_path = "alert-templates"
4211
+ error_message = "get templates /alert-templates"
4212
+ response = self.api_call("GET", endpoint_path, error_message).json()
4213
+ results = []
4214
+ for item in response:
4215
+ results.append(mlrun.common.schemas.AlertTemplate(**item))
4216
+ return results
4217
+
4015
4218
 
4016
4219
  def _as_json(obj):
4017
4220
  fn = getattr(obj, "to_json", None)