mlrun 1.4.0rc25__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (184) hide show
  1. mlrun/__init__.py +2 -35
  2. mlrun/__main__.py +3 -41
  3. mlrun/api/api/api.py +6 -0
  4. mlrun/api/api/endpoints/feature_store.py +0 -4
  5. mlrun/api/api/endpoints/files.py +14 -2
  6. mlrun/api/api/endpoints/frontend_spec.py +2 -1
  7. mlrun/api/api/endpoints/functions.py +95 -59
  8. mlrun/api/api/endpoints/grafana_proxy.py +9 -9
  9. mlrun/api/api/endpoints/logs.py +17 -3
  10. mlrun/api/api/endpoints/model_endpoints.py +3 -2
  11. mlrun/api/api/endpoints/pipelines.py +1 -5
  12. mlrun/api/api/endpoints/projects.py +88 -0
  13. mlrun/api/api/endpoints/runs.py +48 -6
  14. mlrun/api/api/endpoints/submit.py +2 -1
  15. mlrun/api/api/endpoints/workflows.py +355 -0
  16. mlrun/api/api/utils.py +3 -4
  17. mlrun/api/crud/__init__.py +1 -0
  18. mlrun/api/crud/client_spec.py +6 -2
  19. mlrun/api/crud/feature_store.py +5 -0
  20. mlrun/api/crud/model_monitoring/__init__.py +1 -0
  21. mlrun/api/crud/model_monitoring/deployment.py +497 -0
  22. mlrun/api/crud/model_monitoring/grafana.py +96 -42
  23. mlrun/api/crud/model_monitoring/helpers.py +159 -0
  24. mlrun/api/crud/model_monitoring/model_endpoints.py +202 -476
  25. mlrun/api/crud/notifications.py +9 -4
  26. mlrun/api/crud/pipelines.py +6 -11
  27. mlrun/api/crud/projects.py +2 -2
  28. mlrun/api/crud/runtime_resources.py +4 -3
  29. mlrun/api/crud/runtimes/nuclio/helpers.py +5 -1
  30. mlrun/api/crud/secrets.py +21 -0
  31. mlrun/api/crud/workflows.py +352 -0
  32. mlrun/api/db/base.py +16 -1
  33. mlrun/api/db/init_db.py +2 -4
  34. mlrun/api/db/session.py +1 -1
  35. mlrun/api/db/sqldb/db.py +129 -31
  36. mlrun/api/db/sqldb/models/models_mysql.py +15 -1
  37. mlrun/api/db/sqldb/models/models_sqlite.py +16 -2
  38. mlrun/api/launcher.py +38 -6
  39. mlrun/api/main.py +3 -2
  40. mlrun/api/rundb/__init__.py +13 -0
  41. mlrun/{db → api/rundb}/sqldb.py +36 -84
  42. mlrun/api/runtime_handlers/__init__.py +56 -0
  43. mlrun/api/runtime_handlers/base.py +1247 -0
  44. mlrun/api/runtime_handlers/daskjob.py +209 -0
  45. mlrun/api/runtime_handlers/kubejob.py +37 -0
  46. mlrun/api/runtime_handlers/mpijob.py +147 -0
  47. mlrun/api/runtime_handlers/remotesparkjob.py +29 -0
  48. mlrun/api/runtime_handlers/sparkjob.py +148 -0
  49. mlrun/api/schemas/__init__.py +17 -6
  50. mlrun/api/utils/builder.py +1 -4
  51. mlrun/api/utils/clients/chief.py +14 -0
  52. mlrun/api/utils/clients/iguazio.py +33 -33
  53. mlrun/api/utils/clients/nuclio.py +2 -2
  54. mlrun/api/utils/periodic.py +9 -2
  55. mlrun/api/utils/projects/follower.py +14 -7
  56. mlrun/api/utils/projects/leader.py +2 -1
  57. mlrun/api/utils/projects/remotes/nop_follower.py +2 -2
  58. mlrun/api/utils/projects/remotes/nop_leader.py +2 -2
  59. mlrun/api/utils/runtimes/__init__.py +14 -0
  60. mlrun/api/utils/runtimes/nuclio.py +43 -0
  61. mlrun/api/utils/scheduler.py +98 -15
  62. mlrun/api/utils/singletons/db.py +5 -1
  63. mlrun/api/utils/singletons/project_member.py +4 -1
  64. mlrun/api/utils/singletons/scheduler.py +1 -1
  65. mlrun/artifacts/base.py +6 -6
  66. mlrun/artifacts/dataset.py +4 -4
  67. mlrun/artifacts/manager.py +2 -3
  68. mlrun/artifacts/model.py +2 -2
  69. mlrun/artifacts/plots.py +8 -8
  70. mlrun/common/db/__init__.py +14 -0
  71. mlrun/common/helpers.py +37 -0
  72. mlrun/{mlutils → common/model_monitoring}/__init__.py +3 -2
  73. mlrun/common/model_monitoring/helpers.py +69 -0
  74. mlrun/common/schemas/__init__.py +13 -1
  75. mlrun/common/schemas/auth.py +4 -1
  76. mlrun/common/schemas/client_spec.py +1 -1
  77. mlrun/common/schemas/function.py +17 -0
  78. mlrun/common/schemas/model_monitoring/__init__.py +48 -0
  79. mlrun/common/{model_monitoring.py → schemas/model_monitoring/constants.py} +11 -23
  80. mlrun/common/schemas/model_monitoring/grafana.py +55 -0
  81. mlrun/common/schemas/{model_endpoints.py → model_monitoring/model_endpoints.py} +32 -65
  82. mlrun/common/schemas/notification.py +1 -0
  83. mlrun/common/schemas/object.py +4 -0
  84. mlrun/common/schemas/project.py +1 -0
  85. mlrun/common/schemas/regex.py +1 -1
  86. mlrun/common/schemas/runs.py +1 -8
  87. mlrun/common/schemas/schedule.py +1 -8
  88. mlrun/common/schemas/workflow.py +54 -0
  89. mlrun/config.py +45 -42
  90. mlrun/datastore/__init__.py +21 -0
  91. mlrun/datastore/base.py +1 -1
  92. mlrun/datastore/datastore.py +9 -0
  93. mlrun/datastore/dbfs_store.py +168 -0
  94. mlrun/datastore/helpers.py +18 -0
  95. mlrun/datastore/sources.py +1 -0
  96. mlrun/datastore/store_resources.py +2 -5
  97. mlrun/datastore/v3io.py +1 -2
  98. mlrun/db/__init__.py +4 -68
  99. mlrun/db/base.py +12 -0
  100. mlrun/db/factory.py +65 -0
  101. mlrun/db/httpdb.py +175 -20
  102. mlrun/db/nopdb.py +4 -2
  103. mlrun/execution.py +4 -2
  104. mlrun/feature_store/__init__.py +1 -0
  105. mlrun/feature_store/api.py +1 -2
  106. mlrun/feature_store/common.py +2 -1
  107. mlrun/feature_store/feature_set.py +1 -11
  108. mlrun/feature_store/feature_vector.py +340 -2
  109. mlrun/feature_store/ingestion.py +5 -10
  110. mlrun/feature_store/retrieval/base.py +118 -104
  111. mlrun/feature_store/retrieval/dask_merger.py +17 -10
  112. mlrun/feature_store/retrieval/job.py +4 -1
  113. mlrun/feature_store/retrieval/local_merger.py +18 -18
  114. mlrun/feature_store/retrieval/spark_merger.py +21 -14
  115. mlrun/feature_store/retrieval/storey_merger.py +22 -16
  116. mlrun/kfpops.py +3 -9
  117. mlrun/launcher/base.py +57 -53
  118. mlrun/launcher/client.py +5 -4
  119. mlrun/launcher/factory.py +24 -13
  120. mlrun/launcher/local.py +6 -6
  121. mlrun/launcher/remote.py +4 -4
  122. mlrun/lists.py +0 -11
  123. mlrun/model.py +11 -17
  124. mlrun/model_monitoring/__init__.py +2 -22
  125. mlrun/model_monitoring/features_drift_table.py +1 -1
  126. mlrun/model_monitoring/helpers.py +22 -210
  127. mlrun/model_monitoring/model_endpoint.py +1 -1
  128. mlrun/model_monitoring/model_monitoring_batch.py +127 -50
  129. mlrun/model_monitoring/prometheus.py +219 -0
  130. mlrun/model_monitoring/stores/__init__.py +16 -11
  131. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +95 -23
  132. mlrun/model_monitoring/stores/models/mysql.py +47 -29
  133. mlrun/model_monitoring/stores/models/sqlite.py +47 -29
  134. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +31 -19
  135. mlrun/model_monitoring/{stream_processing_fs.py → stream_processing.py} +206 -64
  136. mlrun/model_monitoring/tracking_policy.py +104 -0
  137. mlrun/package/packager.py +6 -8
  138. mlrun/package/packagers/default_packager.py +121 -10
  139. mlrun/package/packagers/numpy_packagers.py +1 -1
  140. mlrun/platforms/__init__.py +0 -2
  141. mlrun/platforms/iguazio.py +0 -56
  142. mlrun/projects/pipelines.py +53 -159
  143. mlrun/projects/project.py +10 -37
  144. mlrun/render.py +1 -1
  145. mlrun/run.py +8 -124
  146. mlrun/runtimes/__init__.py +6 -42
  147. mlrun/runtimes/base.py +29 -1249
  148. mlrun/runtimes/daskjob.py +2 -198
  149. mlrun/runtimes/funcdoc.py +0 -9
  150. mlrun/runtimes/function.py +25 -29
  151. mlrun/runtimes/kubejob.py +5 -29
  152. mlrun/runtimes/local.py +1 -1
  153. mlrun/runtimes/mpijob/__init__.py +2 -2
  154. mlrun/runtimes/mpijob/abstract.py +10 -1
  155. mlrun/runtimes/mpijob/v1.py +0 -76
  156. mlrun/runtimes/mpijob/v1alpha1.py +1 -74
  157. mlrun/runtimes/nuclio.py +3 -2
  158. mlrun/runtimes/pod.py +28 -18
  159. mlrun/runtimes/remotesparkjob.py +1 -15
  160. mlrun/runtimes/serving.py +14 -6
  161. mlrun/runtimes/sparkjob/__init__.py +0 -1
  162. mlrun/runtimes/sparkjob/abstract.py +4 -131
  163. mlrun/runtimes/utils.py +0 -26
  164. mlrun/serving/routers.py +7 -7
  165. mlrun/serving/server.py +11 -8
  166. mlrun/serving/states.py +7 -1
  167. mlrun/serving/v2_serving.py +6 -6
  168. mlrun/utils/helpers.py +23 -42
  169. mlrun/utils/notifications/notification/__init__.py +4 -0
  170. mlrun/utils/notifications/notification/webhook.py +61 -0
  171. mlrun/utils/notifications/notification_pusher.py +5 -25
  172. mlrun/utils/regex.py +7 -2
  173. mlrun/utils/version/version.json +2 -2
  174. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/METADATA +26 -25
  175. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/RECORD +180 -158
  176. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/WHEEL +1 -1
  177. mlrun/mlutils/data.py +0 -160
  178. mlrun/mlutils/models.py +0 -78
  179. mlrun/mlutils/plots.py +0 -902
  180. mlrun/utils/model_monitoring.py +0 -249
  181. /mlrun/{api/db/sqldb/session.py → common/db/sql_session.py} +0 -0
  182. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/LICENSE +0 -0
  183. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/entry_points.txt +0 -0
  184. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/top_level.txt +0 -0
mlrun/__init__.py CHANGED
@@ -36,14 +36,7 @@ from .errors import MLRunInvalidArgumentError, MLRunNotFoundError
36
36
  from .execution import MLClientCtx
37
37
  from .model import RunObject, RunTemplate, new_task
38
38
  from .package import ArtifactType, DefaultPackager, Packager, handler
39
- from .platforms import (
40
- VolumeMount,
41
- auto_mount,
42
- mount_v3io,
43
- mount_v3io_extended,
44
- mount_v3io_legacy,
45
- v3io_cred,
46
- )
39
+ from .platforms import VolumeMount, auto_mount, mount_v3io, v3io_cred
47
40
  from .projects import (
48
41
  ProjectMetadata,
49
42
  build_function,
@@ -66,7 +59,6 @@ from .run import (
66
59
  import_function,
67
60
  new_function,
68
61
  run_local,
69
- run_pipeline,
70
62
  wait_for_pipeline_completion,
71
63
  )
72
64
  from .runtimes import new_model_server
@@ -91,9 +83,7 @@ if "IGZ_NAMESPACE_DOMAIN" in environ:
91
83
  def set_environment(
92
84
  api_path: str = None,
93
85
  artifact_path: str = "",
94
- project: str = "",
95
86
  access_key: str = None,
96
- user_project=False,
97
87
  username: str = None,
98
88
  env_file: str = None,
99
89
  mock_functions: str = None,
@@ -109,18 +99,14 @@ def set_environment(
109
99
  example::
110
100
 
111
101
  from os import path
112
- project_name, artifact_path = set_environment(project='my-project')
102
+ project_name, artifact_path = set_environment()
113
103
  set_environment("http://localhost:8080", artifact_path="./")
114
104
  set_environment(env_file="mlrun.env")
115
105
  set_environment("<remote-service-url>", access_key="xyz", username="joe")
116
106
 
117
107
  :param api_path: location/url of mlrun api service
118
108
  :param artifact_path: path/url for storing experiment artifacts
119
- :param project: default project name (deprecated in 1.3.0 and will be removed in 1.5.0) - use project
120
- APIs such as `get_or_create_project`, `load_project` to configure the active project
121
109
  :param access_key: set the remote cluster access key (V3IO_ACCESS_KEY)
122
- :param user_project: add the current user name to the provided project name (making it unique per user)
123
- (deprecated in 1.3.0 and will be removed in 1.5.0)
124
110
  :param username: name of the user to authenticate
125
111
  :param env_file: path/url to .env file (holding MLRun config and other env vars), see: set_env_from_file()
126
112
  :param mock_functions: set to True to create local/mock functions instead of real containers,
@@ -129,14 +115,6 @@ def set_environment(
129
115
  default project name
130
116
  actual artifact path/url, can be used to create subpaths per task or group of artifacts
131
117
  """
132
- if user_project or project:
133
- warnings.warn(
134
- "'user_project' and 'project' are deprecated in 1.3.0, and will be removed in 1.5.0, use project "
135
- "APIs such as 'get_or_create_project', 'load_project' to configure the active project.",
136
- # TODO: Remove in 1.5.0
137
- FutureWarning,
138
- )
139
-
140
118
  if env_file:
141
119
  set_env_from_file(env_file)
142
120
 
@@ -160,17 +138,6 @@ def set_environment(
160
138
  if api_path:
161
139
  environ["MLRUN_DBPATH"] = mlconf.dbpath
162
140
 
163
- project = _add_username_to_project_name_if_needed(project, user_project)
164
- if project:
165
- ProjectMetadata.validate_project_name(project)
166
-
167
- mlconf.default_project = project or mlconf.default_project
168
- # We want to ensure the project exists, and verify we're authorized to work on it
169
- # if it doesn't exist this will create it (and obviously if we created it, we're authorized to work on it)
170
- # if it does exist - this will get it, which will fail if we're not authorized to work on it
171
- if project:
172
- get_or_create_project(mlconf.default_project, "./")
173
-
174
141
  if not mlconf.artifact_path and not artifact_path:
175
142
  raise ValueError(
176
143
  "default artifact_path was not configured, please specify a valid artifact_path"
mlrun/__main__.py CHANGED
@@ -17,7 +17,6 @@ import json
17
17
  import pathlib
18
18
  import socket
19
19
  import traceback
20
- import warnings
21
20
  from ast import literal_eval
22
21
  from base64 import b64decode, b64encode
23
22
  from os import environ, path, remove
@@ -33,6 +32,7 @@ import yaml
33
32
  from tabulate import tabulate
34
33
 
35
34
  import mlrun
35
+ from mlrun.common.helpers import parse_versioned_object_uri
36
36
 
37
37
  from .config import config as mlconf
38
38
  from .db import get_run_db
@@ -55,7 +55,6 @@ from .utils import (
55
55
  is_relative_path,
56
56
  list2dict,
57
57
  logger,
58
- parse_versioned_object_uri,
59
58
  run_keys,
60
59
  update_in,
61
60
  )
@@ -592,12 +591,6 @@ def build(
592
591
  default="",
593
592
  help="path/url of function yaml or function " "yaml or db://<project>/<name>[:tag]",
594
593
  )
595
- @click.option(
596
- "--dashboard",
597
- "-d",
598
- default="",
599
- help="Deprecated. Keep empty to allow auto-detect by MLRun API",
600
- )
601
594
  @click.option("--project", "-p", default="", help="project name")
602
595
  @click.option("--model", "-m", multiple=True, help="model name and path (name=path)")
603
596
  @click.option("--kind", "-k", default=None, help="runtime sub kind")
@@ -616,7 +609,6 @@ def deploy(
616
609
  spec,
617
610
  source,
618
611
  func_url,
619
- dashboard,
620
612
  project,
621
613
  model,
622
614
  tag,
@@ -677,16 +669,8 @@ def deploy(
677
669
  function.set_env(k, v)
678
670
  function.verbose = verbose
679
671
 
680
- if dashboard:
681
- warnings.warn(
682
- "'--dashboard' is deprecated in 1.3.0, and will be removed in 1.5.0, "
683
- "Keep '--dashboard' value empty to allow auto-detection by MLRun API.",
684
- # TODO: Remove in 1.5.0
685
- FutureWarning,
686
- )
687
-
688
672
  try:
689
- addr = function.deploy(dashboard=dashboard, project=project, tag=tag)
673
+ addr = function.deploy(project=project, tag=tag)
690
674
  except Exception as err:
691
675
  print(f"deploy error: {err_to_str(err)}")
692
676
  exit(1)
@@ -767,6 +751,7 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
767
751
  ]
768
752
  df["tree"] = df["tree"].apply(lambda x: f"..{x[-8:]}")
769
753
  df["hash"] = df["hash"].apply(lambda x: f"..{x[-6:]}")
754
+ df["updated"] = df["updated"].apply(time_str)
770
755
  print(tabulate(df, headers="keys"))
771
756
 
772
757
  elif kind.startswith("func"):
@@ -1004,12 +989,6 @@ def logs(uid, project, offset, db, watch):
1004
989
  @click.option(
1005
990
  "--env-file", default="", help="path to .env file to load config/variables from"
1006
991
  )
1007
- # TODO: Remove --ensure-project in 1.5.0
1008
- @click.option(
1009
- "--ensure-project",
1010
- is_flag=True,
1011
- help="ensure the project exists, if not, create project",
1012
- )
1013
992
  @click.option(
1014
993
  "--save/--no-save",
1015
994
  default=True,
@@ -1024,13 +1003,6 @@ def logs(uid, project, offset, db, watch):
1024
1003
  "https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron."
1025
1004
  "For using the pre-defined workflow's schedule, set --schedule 'true'",
1026
1005
  )
1027
- # TODO: Remove in 1.5.0
1028
- @click.option(
1029
- "--overwrite-schedule",
1030
- "-os",
1031
- is_flag=True,
1032
- help="Overwrite a schedule when submitting a new one with the same name.",
1033
- )
1034
1006
  @click.option(
1035
1007
  "--save-secrets",
1036
1008
  is_flag=True,
@@ -1068,10 +1040,8 @@ def project(
1068
1040
  local,
1069
1041
  env_file,
1070
1042
  timeout,
1071
- ensure_project,
1072
1043
  schedule,
1073
1044
  notifications,
1074
- overwrite_schedule,
1075
1045
  save_secrets,
1076
1046
  save,
1077
1047
  ):
@@ -1079,13 +1049,6 @@ def project(
1079
1049
  if env_file:
1080
1050
  mlrun.set_env_from_file(env_file)
1081
1051
 
1082
- if ensure_project:
1083
- warnings.warn(
1084
- "'ensure_project' is deprecated and will be removed in 1.5.0, use 'save' (True by default) instead. ",
1085
- # TODO: Remove this in 1.5.0
1086
- FutureWarning,
1087
- )
1088
-
1089
1052
  if db:
1090
1053
  mlconf.dbpath = db
1091
1054
 
@@ -1168,7 +1131,6 @@ def project(
1168
1131
  local=local,
1169
1132
  schedule=schedule,
1170
1133
  timeout=timeout,
1171
- overwrite=overwrite_schedule,
1172
1134
  )
1173
1135
  except Exception as err:
1174
1136
  print(traceback.format_exc())
mlrun/api/api/api.py CHANGED
@@ -41,6 +41,7 @@ from mlrun.api.api.endpoints import (
41
41
  secrets,
42
42
  submit,
43
43
  tags,
44
+ workflows,
44
45
  )
45
46
 
46
47
  api_router = APIRouter(dependencies=[Depends(mlrun.api.api.deps.verify_api_state)])
@@ -145,3 +146,8 @@ api_router.include_router(
145
146
  tags=["internal"],
146
147
  dependencies=[Depends(mlrun.api.api.deps.authenticate_request)],
147
148
  )
149
+ api_router.include_router(
150
+ workflows.router,
151
+ tags=["workflows"],
152
+ dependencies=[Depends(mlrun.api.api.deps.authenticate_request)],
153
+ )
@@ -408,10 +408,6 @@ async def ingest_feature_set(
408
408
  mlrun.common.schemas.AuthorizationAction.read,
409
409
  auth_info,
410
410
  )
411
- # Need to override the default rundb since we're in the server.
412
- # this is done so further down the flow when running the function created for ingestion we won't access the httpdb
413
- # but rather "understand" that we are running on server side and call the DB.
414
- await run_in_threadpool(feature_set._override_run_db, db_session)
415
411
 
416
412
  if ingest_parameters.targets:
417
413
  data_targets = [
@@ -31,7 +31,13 @@ from mlrun.utils import logger
31
31
  router = fastapi.APIRouter()
32
32
 
33
33
 
34
- @router.get("/files")
34
+ # TODO: Remove in 1.7.0
35
+ @router.get(
36
+ "/files",
37
+ deprecated=True,
38
+ description="'/files' and '/filestat' will be removed in 1.7.0, "
39
+ "use /projects/{project}/files instead.",
40
+ )
35
41
  def get_files(
36
42
  schema: str = "",
37
43
  objpath: str = fastapi.Query("", alias="path"),
@@ -73,7 +79,13 @@ async def get_files_with_project_secrets(
73
79
  )
74
80
 
75
81
 
76
- @router.get("/filestat")
82
+ # TODO: Remove in 1.7.0
83
+ @router.get(
84
+ "/filestat",
85
+ deprecated=True,
86
+ description="'/files' and '/filestat' will be removed in 1.7.0, "
87
+ "use /projects/{project}/filestat instead.",
88
+ )
77
89
  def get_filestat(
78
90
  schema: str = "",
79
91
  path: str = "",
@@ -20,6 +20,7 @@ import semver
20
20
  import mlrun.api.api.deps
21
21
  import mlrun.api.utils.builder
22
22
  import mlrun.api.utils.clients.iguazio
23
+ import mlrun.api.utils.runtimes.nuclio
23
24
  import mlrun.common.schemas
24
25
  import mlrun.runtimes
25
26
  import mlrun.runtimes.utils
@@ -117,7 +118,7 @@ def _resolve_feature_flags() -> mlrun.common.schemas.FeatureFlags:
117
118
  nuclio_streams = mlrun.common.schemas.NuclioStreamsFeatureFlag.disabled
118
119
 
119
120
  if mlrun.mlconf.get_parsed_igz_version() and semver.VersionInfo.parse(
120
- mlrun.runtimes.utils.resolve_nuclio_version()
121
+ mlrun.api.utils.runtimes.nuclio.resolve_nuclio_version()
121
122
  ) >= semver.VersionInfo.parse("1.7.8"):
122
123
  nuclio_streams = mlrun.common.schemas.NuclioStreamsFeatureFlag.enabled
123
124
 
@@ -34,7 +34,7 @@ from fastapi.concurrency import run_in_threadpool
34
34
  from kubernetes.client.rest import ApiException
35
35
  from sqlalchemy.orm import Session
36
36
 
37
- import mlrun.api.crud
37
+ import mlrun.api.crud.model_monitoring.deployment
38
38
  import mlrun.api.crud.runtimes.nuclio.function
39
39
  import mlrun.api.db.session
40
40
  import mlrun.api.launcher
@@ -44,19 +44,22 @@ import mlrun.api.utils.clients.chief
44
44
  import mlrun.api.utils.singletons.k8s
45
45
  import mlrun.api.utils.singletons.project_member
46
46
  import mlrun.common.model_monitoring
47
+ import mlrun.common.model_monitoring.helpers
47
48
  import mlrun.common.schemas
48
49
  from mlrun.api.api import deps
49
50
  from mlrun.api.api.utils import get_run_db_instance, log_and_raise, log_path
50
51
  from mlrun.api.crud.secrets import Secrets, SecretsClientType
51
52
  from mlrun.api.utils.builder import build_runtime
52
53
  from mlrun.api.utils.singletons.scheduler import get_scheduler
54
+ from mlrun.common.helpers import parse_versioned_object_uri
55
+ from mlrun.common.model_monitoring.helpers import parse_model_endpoint_store_prefix
53
56
  from mlrun.config import config
54
57
  from mlrun.errors import MLRunRuntimeError, err_to_str
58
+ from mlrun.model_monitoring.tracking_policy import TrackingPolicy
55
59
  from mlrun.run import new_function
56
60
  from mlrun.runtimes import RuntimeKinds, ServingRuntime, runtime_resources_map
57
61
  from mlrun.runtimes.utils import get_item_name
58
- from mlrun.utils import get_in, logger, parse_versioned_object_uri, update_in
59
- from mlrun.utils.model_monitoring import parse_model_endpoint_store_prefix
62
+ from mlrun.utils import get_in, logger, update_in
60
63
 
61
64
  router = APIRouter()
62
65
 
@@ -448,21 +451,21 @@ async def build_status(
448
451
 
449
452
 
450
453
  def _handle_job_deploy_status(
451
- db_session,
452
- fn,
453
- name,
454
- project,
455
- tag,
456
- offset,
457
- logs,
454
+ db_session: Session,
455
+ fn: dict,
456
+ name: str,
457
+ project: str,
458
+ tag: str,
459
+ offset: int,
460
+ logs: bool,
458
461
  ):
459
462
  # job deploy status
460
- state = get_in(fn, "status.state", "")
463
+ function_state = get_in(fn, "status.state", "")
461
464
  pod = get_in(fn, "status.build_pod", "")
462
465
  image = get_in(fn, "spec.build.image", "")
463
466
  out = b""
464
467
  if not pod:
465
- if state == mlrun.common.schemas.FunctionState.ready:
468
+ if function_state == mlrun.common.schemas.FunctionState.ready:
466
469
  # when the function has been built we set the created image into the `spec.image` for reference see at the
467
470
  # end of the function where we resolve if the status is ready and then set the spec.build.image to
468
471
  # spec.image
@@ -474,17 +477,19 @@ def _handle_job_deploy_status(
474
477
  content=out,
475
478
  media_type="text/plain",
476
479
  headers={
477
- "function_status": state,
480
+ "function_status": function_state,
478
481
  "function_image": image,
479
482
  "builder_pod": pod,
480
483
  },
481
484
  )
482
485
 
483
486
  # read from log file
484
- terminal_states = ["failed", "error", "ready"]
485
487
  log_file = log_path(project, f"build_{name}__{tag or 'latest'}")
486
- if state in terminal_states and log_file.exists():
487
- if state == mlrun.common.schemas.FunctionState.ready:
488
+ if (
489
+ function_state in mlrun.common.schemas.FunctionState.terminal_states()
490
+ and log_file.exists()
491
+ ):
492
+ if function_state == mlrun.common.schemas.FunctionState.ready:
488
493
  # when the function has been built we set the created image into the `spec.image` for reference see at the
489
494
  # end of the function where we resolve if the status is ready and then set the spec.build.image to
490
495
  # spec.image
@@ -500,40 +505,65 @@ def _handle_job_deploy_status(
500
505
  content=out,
501
506
  media_type="text/plain",
502
507
  headers={
503
- "x-mlrun-function-status": state,
504
- "function_status": state,
508
+ "x-mlrun-function-status": function_state,
509
+ "function_status": function_state,
505
510
  "function_image": image,
506
511
  "builder_pod": pod,
507
512
  },
508
513
  )
509
514
 
510
- # TODO: change state to pod_status
511
- state = mlrun.api.utils.singletons.k8s.get_k8s_helper(silent=False).get_pod_status(
512
- pod
515
+ build_pod_state = mlrun.api.utils.singletons.k8s.get_k8s_helper(
516
+ silent=False
517
+ ).get_pod_status(pod)
518
+ logger.debug(
519
+ "Resolved pod status",
520
+ function_name=name,
521
+ pod_status=build_pod_state,
522
+ pod_name=pod,
513
523
  )
514
- logger.info("Resolved pod status", pod_status=state, pod_name=pod)
515
524
 
516
- if state == "succeeded":
517
- logger.info("Build completed successfully")
518
- state = mlrun.common.schemas.FunctionState.ready
519
- if state in ["failed", "error"]:
520
- logger.error("Build failed", pod_name=pod, pod_status=state)
521
- state = mlrun.common.schemas.FunctionState.error
525
+ normalized_pod_function_state = (
526
+ mlrun.common.schemas.FunctionState.get_function_state_from_pod_state(
527
+ build_pod_state
528
+ )
529
+ )
530
+ if normalized_pod_function_state == mlrun.common.schemas.FunctionState.ready:
531
+ logger.info(
532
+ "Build completed successfully",
533
+ function_name=name,
534
+ pod=pod,
535
+ pod_state=build_pod_state,
536
+ )
537
+ elif normalized_pod_function_state == mlrun.common.schemas.FunctionState.error:
538
+ logger.error(
539
+ "Build failed", function_name=name, pod_name=pod, pod_status=build_pod_state
540
+ )
522
541
 
523
- if (logs and state != "pending") or state in terminal_states:
542
+ if (
543
+ (
544
+ logs
545
+ and normalized_pod_function_state
546
+ != mlrun.common.schemas.FunctionState.pending
547
+ )
548
+ or normalized_pod_function_state
549
+ in mlrun.common.schemas.FunctionState.terminal_states()
550
+ ):
524
551
  try:
525
552
  resp = mlrun.api.utils.singletons.k8s.get_k8s_helper(silent=False).logs(pod)
526
553
  except ApiException as exc:
527
554
  logger.warning(
528
555
  "Failed to get build logs",
529
556
  function_name=name,
530
- function_state=state,
557
+ function_state=normalized_pod_function_state,
531
558
  pod=pod,
532
559
  exc_info=exc,
533
560
  )
534
561
  resp = ""
535
562
 
536
- if state in terminal_states:
563
+ if (
564
+ normalized_pod_function_state
565
+ in mlrun.common.schemas.FunctionState.terminal_states()
566
+ ):
537
567
  # TODO: move to log collector
538
568
  log_file.parent.mkdir(parents=True, exist_ok=True)
539
569
  with log_file.open("wb") as fp:
@@ -543,28 +573,31 @@ def _handle_job_deploy_status(
543
573
  # begin from the offset number and then encode
544
574
  out = resp[offset:].encode()
545
575
 
546
- update_in(fn, "status.state", state)
547
- if state == mlrun.common.schemas.FunctionState.ready:
548
- update_in(fn, "spec.image", image)
576
+ # check if the previous function state is different from the current build pod state, if that is the case then
577
+ # update the function and store to the database
578
+ if function_state != normalized_pod_function_state:
579
+ update_in(fn, "status.state", normalized_pod_function_state)
549
580
 
550
- versioned = False
551
- if state == mlrun.common.schemas.FunctionState.ready:
552
- versioned = True
553
- mlrun.api.crud.Functions().store_function(
554
- db_session,
555
- fn,
556
- name,
557
- project,
558
- tag,
559
- versioned=versioned,
560
- )
581
+ versioned = False
582
+ if normalized_pod_function_state == mlrun.common.schemas.FunctionState.ready:
583
+ update_in(fn, "spec.image", image)
584
+ versioned = True
585
+
586
+ mlrun.api.crud.Functions().store_function(
587
+ db_session,
588
+ fn,
589
+ name,
590
+ project,
591
+ tag,
592
+ versioned=versioned,
593
+ )
561
594
 
562
595
  return Response(
563
596
  content=out,
564
597
  media_type="text/plain",
565
598
  headers={
566
- "x-mlrun-function-status": state,
567
- "function_status": state,
599
+ "x-mlrun-function-status": normalized_pod_function_state,
600
+ "function_status": normalized_pod_function_state,
568
601
  "function_image": image,
569
602
  "builder_pod": pod,
570
603
  },
@@ -673,9 +706,14 @@ def _build_function(
673
706
  reason=f"runtime error: {err_to_str(err)}",
674
707
  )
675
708
  try:
709
+ # connect to run db
676
710
  run_db = get_run_db_instance(db_session)
677
711
  fn.set_db_connection(run_db)
678
- mlrun.api.launcher.ServerSideLauncher.enrich_runtime(runtime=fn)
712
+
713
+ # enrich runtime with project defaults
714
+ launcher = mlrun.api.launcher.ServerSideLauncher()
715
+ launcher.enrich_runtime(runtime=fn)
716
+
679
717
  fn.save(versioned=False)
680
718
  if fn.kind in RuntimeKinds.nuclio_runtimes():
681
719
  mlrun.api.api.utils.apply_enrichment_and_validation_on_function(
@@ -695,11 +733,13 @@ def _build_function(
695
733
  model_monitoring_access_key = _process_model_monitoring_secret(
696
734
  db_session,
697
735
  fn.metadata.project,
698
- mlrun.common.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
736
+ mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
699
737
  )
700
738
 
701
- stream_path = mlrun.utils.model_monitoring.get_stream_path(
702
- project=fn.metadata.project
739
+ stream_path = (
740
+ mlrun.api.crud.model_monitoring.get_stream_path(
741
+ project=fn.metadata.project
742
+ )
703
743
  )
704
744
 
705
745
  if stream_path.startswith("v3io://"):
@@ -712,19 +752,15 @@ def _build_function(
712
752
 
713
753
  if fn.spec.tracking_policy:
714
754
  # Convert to `TrackingPolicy` object as `fn.spec.tracking_policy` is provided as a dict
715
- fn.spec.tracking_policy = (
716
- mlrun.utils.model_monitoring.TrackingPolicy.from_dict(
717
- fn.spec.tracking_policy
718
- )
755
+ fn.spec.tracking_policy = TrackingPolicy.from_dict(
756
+ fn.spec.tracking_policy
719
757
  )
720
758
  else:
721
759
  # Initialize tracking policy with default values
722
- fn.spec.tracking_policy = (
723
- mlrun.utils.model_monitoring.TrackingPolicy()
724
- )
760
+ fn.spec.tracking_policy = TrackingPolicy()
725
761
 
726
762
  # deploy both model monitoring stream and model monitoring batch job
727
- mlrun.api.crud.ModelEndpoints().deploy_monitoring_functions(
763
+ mlrun.api.crud.model_monitoring.deployment.MonitoringDeployment().deploy_monitoring_functions(
728
764
  project=fn.metadata.project,
729
765
  db_session=db_session,
730
766
  auth_info=auth_info,
@@ -21,11 +21,10 @@ from fastapi import APIRouter, Depends, Request, Response
21
21
  from fastapi.concurrency import run_in_threadpool
22
22
  from sqlalchemy.orm import Session
23
23
 
24
- import mlrun.api.crud
24
+ import mlrun.api.crud.model_monitoring.deployment
25
25
  import mlrun.api.crud.model_monitoring.grafana
26
- import mlrun.api.utils.auth.verifier
27
- import mlrun.common.model_monitoring
28
- import mlrun.common.schemas
26
+ import mlrun.api.crud.model_monitoring.helpers
27
+ import mlrun.common.schemas.model_monitoring.grafana
29
28
  from mlrun.api.api import deps
30
29
 
31
30
  router = APIRouter(prefix="/grafana-proxy/model-endpoints")
@@ -53,7 +52,7 @@ def grafana_proxy_model_endpoints_check_connection(
53
52
  connectivity.
54
53
  """
55
54
  if not mlrun.mlconf.is_ce_mode():
56
- mlrun.api.crud.ModelEndpoints().get_access_key(auth_info)
55
+ mlrun.api.crud.model_monitoring.helpers.get_access_key(auth_info)
57
56
  return Response(status_code=HTTPStatus.OK.value)
58
57
 
59
58
 
@@ -77,7 +76,7 @@ async def grafana_proxy_model_endpoints_search(
77
76
  :return: List of results. e.g. list of available project names.
78
77
  """
79
78
  if not mlrun.mlconf.is_ce_mode():
80
- mlrun.api.crud.ModelEndpoints().get_access_key(auth_info)
79
+ mlrun.api.crud.model_monitoring.helpers.get_access_key(auth_info)
81
80
  body = await request.json()
82
81
  query_parameters = mlrun.api.crud.model_monitoring.grafana.parse_search_parameters(
83
82
  body
@@ -104,8 +103,8 @@ async def grafana_proxy_model_endpoints_search(
104
103
  "/query",
105
104
  response_model=List[
106
105
  Union[
107
- mlrun.common.schemas.GrafanaTable,
108
- mlrun.common.schemas.GrafanaTimeSeriesTarget,
106
+ mlrun.common.schemas.model_monitoring.grafana.GrafanaTable,
107
+ mlrun.common.schemas.model_monitoring.grafana.GrafanaTimeSeriesTarget,
109
108
  ]
110
109
  ],
111
110
  )
@@ -114,7 +113,8 @@ async def grafana_proxy_model_endpoints_query(
114
113
  auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
115
114
  ) -> List[
116
115
  Union[
117
- mlrun.common.schemas.GrafanaTable, mlrun.common.schemas.GrafanaTimeSeriesTarget
116
+ mlrun.common.schemas.model_monitoring.grafana.GrafanaTable,
117
+ mlrun.common.schemas.model_monitoring.grafana.GrafanaTimeSeriesTarget,
118
118
  ]
119
119
  ]:
120
120
  """
@@ -21,10 +21,17 @@ import mlrun.api.crud
21
21
  import mlrun.api.utils.auth.verifier
22
22
  import mlrun.common.schemas
23
23
 
24
- router = fastapi.APIRouter(prefix="/log/{project}")
24
+ router = fastapi.APIRouter()
25
25
 
26
26
 
27
- @router.post("/{uid}")
27
+ # TODO: remove /log/{project}/{uid} in 1.7.0
28
+ @router.post(
29
+ "/log/{project}/{uid}",
30
+ deprecated=True,
31
+ description="/log/{project}/{uid} is deprecated in 1.5.0 and will be removed in 1.7.0, "
32
+ "use /projects/{project}/logs/{uid} instead",
33
+ )
34
+ @router.post("/projects/{project}/logs/{uid}")
28
35
  async def store_log(
29
36
  request: fastapi.Request,
30
37
  project: str,
@@ -52,7 +59,14 @@ async def store_log(
52
59
  return {}
53
60
 
54
61
 
55
- @router.get("/{uid}")
62
+ # TODO: remove /log/{project}/{uid} in 1.7.0
63
+ @router.get(
64
+ "/log/{project}/{uid}",
65
+ deprecated=True,
66
+ description="/log/{project}/{uid} is deprecated in 1.5.0 and will be removed in 1.7.0, "
67
+ "use /projects/{project}/logs/{uid} instead",
68
+ )
69
+ @router.get("/projects/{project}/logs/{uid}")
56
70
  async def get_log(
57
71
  project: str,
58
72
  uid: str,
@@ -23,7 +23,8 @@ from fastapi.concurrency import run_in_threadpool
23
23
  from sqlalchemy.orm import Session
24
24
 
25
25
  import mlrun.api.api.deps
26
- import mlrun.api.crud
26
+ import mlrun.api.crud.model_monitoring.deployment
27
+ import mlrun.api.crud.model_monitoring.helpers
27
28
  import mlrun.api.utils.auth.verifier
28
29
  import mlrun.common.schemas
29
30
  from mlrun.errors import MLRunConflictError
@@ -64,7 +65,7 @@ async def create_or_patch(
64
65
  auth_info,
65
66
  )
66
67
  # get_access_key will validate the needed auth (which is used later) exists in the request
67
- mlrun.api.crud.ModelEndpoints().get_access_key(auth_info)
68
+ mlrun.api.crud.model_monitoring.helpers.get_access_key(auth_info)
68
69
  if project != model_endpoint.metadata.project:
69
70
  raise MLRunConflictError(
70
71
  f"Can't store endpoint of project {model_endpoint.metadata.project} into project {project}"