mlrun 1.7.0rc4__py3-none-any.whl → 1.7.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (235) hide show
  1. mlrun/__init__.py +11 -1
  2. mlrun/__main__.py +39 -121
  3. mlrun/{datastore/helpers.py → alerts/__init__.py} +2 -5
  4. mlrun/alerts/alert.py +248 -0
  5. mlrun/api/schemas/__init__.py +4 -3
  6. mlrun/artifacts/__init__.py +8 -3
  7. mlrun/artifacts/base.py +39 -254
  8. mlrun/artifacts/dataset.py +9 -190
  9. mlrun/artifacts/manager.py +73 -46
  10. mlrun/artifacts/model.py +30 -158
  11. mlrun/artifacts/plots.py +23 -380
  12. mlrun/common/constants.py +73 -1
  13. mlrun/common/db/sql_session.py +3 -2
  14. mlrun/common/formatters/__init__.py +21 -0
  15. mlrun/common/formatters/artifact.py +46 -0
  16. mlrun/common/formatters/base.py +113 -0
  17. mlrun/common/formatters/feature_set.py +44 -0
  18. mlrun/common/formatters/function.py +46 -0
  19. mlrun/common/formatters/pipeline.py +53 -0
  20. mlrun/common/formatters/project.py +51 -0
  21. mlrun/common/formatters/run.py +29 -0
  22. mlrun/common/helpers.py +11 -1
  23. mlrun/{runtimes → common/runtimes}/constants.py +32 -4
  24. mlrun/common/schemas/__init__.py +31 -4
  25. mlrun/common/schemas/alert.py +202 -0
  26. mlrun/common/schemas/api_gateway.py +196 -0
  27. mlrun/common/schemas/artifact.py +28 -1
  28. mlrun/common/schemas/auth.py +13 -2
  29. mlrun/common/schemas/client_spec.py +2 -1
  30. mlrun/common/schemas/common.py +7 -4
  31. mlrun/common/schemas/constants.py +3 -0
  32. mlrun/common/schemas/feature_store.py +58 -28
  33. mlrun/common/schemas/frontend_spec.py +8 -0
  34. mlrun/common/schemas/function.py +11 -0
  35. mlrun/common/schemas/hub.py +7 -9
  36. mlrun/common/schemas/model_monitoring/__init__.py +21 -4
  37. mlrun/common/schemas/model_monitoring/constants.py +136 -42
  38. mlrun/common/schemas/model_monitoring/grafana.py +9 -5
  39. mlrun/common/schemas/model_monitoring/model_endpoints.py +89 -41
  40. mlrun/common/schemas/notification.py +69 -12
  41. mlrun/{runtimes/mpijob/v1alpha1.py → common/schemas/pagination.py} +10 -13
  42. mlrun/common/schemas/pipeline.py +7 -0
  43. mlrun/common/schemas/project.py +67 -16
  44. mlrun/common/schemas/runs.py +17 -0
  45. mlrun/common/schemas/schedule.py +1 -1
  46. mlrun/common/schemas/workflow.py +10 -2
  47. mlrun/common/types.py +14 -1
  48. mlrun/config.py +233 -58
  49. mlrun/data_types/data_types.py +11 -1
  50. mlrun/data_types/spark.py +5 -4
  51. mlrun/data_types/to_pandas.py +75 -34
  52. mlrun/datastore/__init__.py +8 -10
  53. mlrun/datastore/alibaba_oss.py +131 -0
  54. mlrun/datastore/azure_blob.py +131 -43
  55. mlrun/datastore/base.py +107 -47
  56. mlrun/datastore/datastore.py +17 -7
  57. mlrun/datastore/datastore_profile.py +91 -7
  58. mlrun/datastore/dbfs_store.py +3 -7
  59. mlrun/datastore/filestore.py +1 -3
  60. mlrun/datastore/google_cloud_storage.py +92 -32
  61. mlrun/datastore/hdfs.py +5 -0
  62. mlrun/datastore/inmem.py +6 -3
  63. mlrun/datastore/redis.py +3 -2
  64. mlrun/datastore/s3.py +30 -12
  65. mlrun/datastore/snowflake_utils.py +45 -0
  66. mlrun/datastore/sources.py +274 -59
  67. mlrun/datastore/spark_utils.py +30 -0
  68. mlrun/datastore/store_resources.py +9 -7
  69. mlrun/datastore/storeytargets.py +151 -0
  70. mlrun/datastore/targets.py +387 -119
  71. mlrun/datastore/utils.py +68 -5
  72. mlrun/datastore/v3io.py +28 -50
  73. mlrun/db/auth_utils.py +152 -0
  74. mlrun/db/base.py +245 -20
  75. mlrun/db/factory.py +1 -4
  76. mlrun/db/httpdb.py +909 -231
  77. mlrun/db/nopdb.py +279 -14
  78. mlrun/errors.py +35 -5
  79. mlrun/execution.py +111 -38
  80. mlrun/feature_store/__init__.py +0 -2
  81. mlrun/feature_store/api.py +46 -53
  82. mlrun/feature_store/common.py +6 -11
  83. mlrun/feature_store/feature_set.py +48 -23
  84. mlrun/feature_store/feature_vector.py +13 -2
  85. mlrun/feature_store/ingestion.py +7 -6
  86. mlrun/feature_store/retrieval/base.py +9 -4
  87. mlrun/feature_store/retrieval/dask_merger.py +2 -0
  88. mlrun/feature_store/retrieval/job.py +13 -4
  89. mlrun/feature_store/retrieval/local_merger.py +2 -0
  90. mlrun/feature_store/retrieval/spark_merger.py +24 -32
  91. mlrun/feature_store/steps.py +38 -19
  92. mlrun/features.py +6 -14
  93. mlrun/frameworks/_common/plan.py +3 -3
  94. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +7 -12
  95. mlrun/frameworks/_ml_common/plan.py +1 -1
  96. mlrun/frameworks/auto_mlrun/auto_mlrun.py +2 -2
  97. mlrun/frameworks/lgbm/__init__.py +1 -1
  98. mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
  99. mlrun/frameworks/lgbm/model_handler.py +1 -1
  100. mlrun/frameworks/parallel_coordinates.py +4 -4
  101. mlrun/frameworks/pytorch/__init__.py +2 -2
  102. mlrun/frameworks/sklearn/__init__.py +1 -1
  103. mlrun/frameworks/sklearn/mlrun_interface.py +13 -3
  104. mlrun/frameworks/tf_keras/__init__.py +5 -2
  105. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  106. mlrun/frameworks/tf_keras/mlrun_interface.py +2 -2
  107. mlrun/frameworks/xgboost/__init__.py +1 -1
  108. mlrun/k8s_utils.py +57 -12
  109. mlrun/launcher/__init__.py +1 -1
  110. mlrun/launcher/base.py +6 -5
  111. mlrun/launcher/client.py +13 -11
  112. mlrun/launcher/factory.py +1 -1
  113. mlrun/launcher/local.py +15 -5
  114. mlrun/launcher/remote.py +10 -3
  115. mlrun/lists.py +6 -2
  116. mlrun/model.py +297 -48
  117. mlrun/model_monitoring/__init__.py +1 -1
  118. mlrun/model_monitoring/api.py +152 -357
  119. mlrun/model_monitoring/applications/__init__.py +10 -0
  120. mlrun/model_monitoring/applications/_application_steps.py +190 -0
  121. mlrun/model_monitoring/applications/base.py +108 -0
  122. mlrun/model_monitoring/applications/context.py +341 -0
  123. mlrun/model_monitoring/{evidently_application.py → applications/evidently_base.py} +27 -22
  124. mlrun/model_monitoring/applications/histogram_data_drift.py +227 -91
  125. mlrun/model_monitoring/applications/results.py +99 -0
  126. mlrun/model_monitoring/controller.py +130 -303
  127. mlrun/model_monitoring/{stores/models/sqlite.py → db/__init__.py} +5 -10
  128. mlrun/model_monitoring/db/stores/__init__.py +136 -0
  129. mlrun/model_monitoring/db/stores/base/__init__.py +15 -0
  130. mlrun/model_monitoring/db/stores/base/store.py +213 -0
  131. mlrun/model_monitoring/db/stores/sqldb/__init__.py +13 -0
  132. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +71 -0
  133. mlrun/model_monitoring/db/stores/sqldb/models/base.py +190 -0
  134. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +103 -0
  135. mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py} +19 -13
  136. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +659 -0
  137. mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +13 -0
  138. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +726 -0
  139. mlrun/model_monitoring/db/tsdb/__init__.py +105 -0
  140. mlrun/model_monitoring/db/tsdb/base.py +448 -0
  141. mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
  142. mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
  143. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +298 -0
  144. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +42 -0
  145. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +522 -0
  146. mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
  147. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +158 -0
  148. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +849 -0
  149. mlrun/model_monitoring/features_drift_table.py +34 -22
  150. mlrun/model_monitoring/helpers.py +177 -39
  151. mlrun/model_monitoring/model_endpoint.py +3 -2
  152. mlrun/model_monitoring/stream_processing.py +165 -398
  153. mlrun/model_monitoring/tracking_policy.py +7 -1
  154. mlrun/model_monitoring/writer.py +161 -125
  155. mlrun/package/packagers/default_packager.py +2 -2
  156. mlrun/package/packagers_manager.py +1 -0
  157. mlrun/package/utils/_formatter.py +2 -2
  158. mlrun/platforms/__init__.py +11 -10
  159. mlrun/platforms/iguazio.py +67 -228
  160. mlrun/projects/__init__.py +6 -1
  161. mlrun/projects/operations.py +47 -20
  162. mlrun/projects/pipelines.py +396 -249
  163. mlrun/projects/project.py +1176 -406
  164. mlrun/render.py +28 -22
  165. mlrun/run.py +208 -181
  166. mlrun/runtimes/__init__.py +76 -11
  167. mlrun/runtimes/base.py +54 -24
  168. mlrun/runtimes/daskjob.py +9 -2
  169. mlrun/runtimes/databricks_job/databricks_runtime.py +1 -0
  170. mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
  171. mlrun/runtimes/funcdoc.py +1 -29
  172. mlrun/runtimes/kubejob.py +34 -128
  173. mlrun/runtimes/local.py +39 -10
  174. mlrun/runtimes/mpijob/__init__.py +0 -20
  175. mlrun/runtimes/mpijob/abstract.py +8 -8
  176. mlrun/runtimes/mpijob/v1.py +1 -1
  177. mlrun/runtimes/nuclio/__init__.py +1 -0
  178. mlrun/runtimes/nuclio/api_gateway.py +769 -0
  179. mlrun/runtimes/nuclio/application/__init__.py +15 -0
  180. mlrun/runtimes/nuclio/application/application.py +758 -0
  181. mlrun/runtimes/nuclio/application/reverse_proxy.go +95 -0
  182. mlrun/runtimes/nuclio/function.py +188 -68
  183. mlrun/runtimes/nuclio/serving.py +57 -60
  184. mlrun/runtimes/pod.py +191 -58
  185. mlrun/runtimes/remotesparkjob.py +11 -8
  186. mlrun/runtimes/sparkjob/spark3job.py +17 -18
  187. mlrun/runtimes/utils.py +40 -73
  188. mlrun/secrets.py +6 -2
  189. mlrun/serving/__init__.py +8 -1
  190. mlrun/serving/remote.py +2 -3
  191. mlrun/serving/routers.py +89 -64
  192. mlrun/serving/server.py +54 -26
  193. mlrun/serving/states.py +187 -56
  194. mlrun/serving/utils.py +19 -11
  195. mlrun/serving/v2_serving.py +136 -63
  196. mlrun/track/tracker.py +2 -1
  197. mlrun/track/trackers/mlflow_tracker.py +5 -0
  198. mlrun/utils/async_http.py +26 -6
  199. mlrun/utils/db.py +18 -0
  200. mlrun/utils/helpers.py +375 -105
  201. mlrun/utils/http.py +2 -2
  202. mlrun/utils/logger.py +75 -9
  203. mlrun/utils/notifications/notification/__init__.py +14 -10
  204. mlrun/utils/notifications/notification/base.py +48 -0
  205. mlrun/utils/notifications/notification/console.py +2 -0
  206. mlrun/utils/notifications/notification/git.py +24 -1
  207. mlrun/utils/notifications/notification/ipython.py +2 -0
  208. mlrun/utils/notifications/notification/slack.py +96 -21
  209. mlrun/utils/notifications/notification/webhook.py +63 -2
  210. mlrun/utils/notifications/notification_pusher.py +146 -16
  211. mlrun/utils/regex.py +9 -0
  212. mlrun/utils/retryer.py +3 -2
  213. mlrun/utils/v3io_clients.py +2 -3
  214. mlrun/utils/version/version.json +2 -2
  215. mlrun-1.7.2.dist-info/METADATA +390 -0
  216. mlrun-1.7.2.dist-info/RECORD +351 -0
  217. {mlrun-1.7.0rc4.dist-info → mlrun-1.7.2.dist-info}/WHEEL +1 -1
  218. mlrun/feature_store/retrieval/conversion.py +0 -271
  219. mlrun/kfpops.py +0 -868
  220. mlrun/model_monitoring/application.py +0 -310
  221. mlrun/model_monitoring/batch.py +0 -974
  222. mlrun/model_monitoring/controller_handler.py +0 -37
  223. mlrun/model_monitoring/prometheus.py +0 -216
  224. mlrun/model_monitoring/stores/__init__.py +0 -111
  225. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +0 -574
  226. mlrun/model_monitoring/stores/model_endpoint_store.py +0 -145
  227. mlrun/model_monitoring/stores/models/__init__.py +0 -27
  228. mlrun/model_monitoring/stores/models/base.py +0 -84
  229. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -382
  230. mlrun/platforms/other.py +0 -305
  231. mlrun-1.7.0rc4.dist-info/METADATA +0 -269
  232. mlrun-1.7.0rc4.dist-info/RECORD +0 -321
  233. {mlrun-1.7.0rc4.dist-info → mlrun-1.7.2.dist-info}/LICENSE +0 -0
  234. {mlrun-1.7.0rc4.dist-info → mlrun-1.7.2.dist-info}/entry_points.txt +0 -0
  235. {mlrun-1.7.0rc4.dist-info → mlrun-1.7.2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,758 @@
1
+ # Copyright 2024 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ import pathlib
15
+ import typing
16
+
17
+ import nuclio
18
+ import nuclio.auth
19
+
20
+ import mlrun.common.schemas as schemas
21
+ import mlrun.errors
22
+ import mlrun.run
23
+ from mlrun.common.runtimes.constants import NuclioIngressAddTemplatedIngressModes
24
+ from mlrun.runtimes import RemoteRuntime
25
+ from mlrun.runtimes.nuclio import min_nuclio_versions
26
+ from mlrun.runtimes.nuclio.api_gateway import (
27
+ APIGateway,
28
+ APIGatewayMetadata,
29
+ APIGatewaySpec,
30
+ )
31
+ from mlrun.runtimes.nuclio.function import NuclioSpec, NuclioStatus
32
+ from mlrun.utils import logger, update_in
33
+
34
+
35
+ class ApplicationSpec(NuclioSpec):
36
+ _dict_fields = NuclioSpec._dict_fields + [
37
+ "internal_application_port",
38
+ ]
39
+
40
+ def __init__(
41
+ self,
42
+ command=None,
43
+ args=None,
44
+ image=None,
45
+ mode=None,
46
+ entry_points=None,
47
+ description=None,
48
+ replicas=None,
49
+ min_replicas=None,
50
+ max_replicas=None,
51
+ volumes=None,
52
+ volume_mounts=None,
53
+ env=None,
54
+ resources=None,
55
+ config=None,
56
+ base_spec=None,
57
+ no_cache=None,
58
+ source=None,
59
+ image_pull_policy=None,
60
+ function_kind=None,
61
+ build=None,
62
+ service_account=None,
63
+ readiness_timeout=None,
64
+ readiness_timeout_before_failure=None,
65
+ default_handler=None,
66
+ node_name=None,
67
+ node_selector=None,
68
+ affinity=None,
69
+ disable_auto_mount=False,
70
+ priority_class_name=None,
71
+ pythonpath=None,
72
+ workdir=None,
73
+ image_pull_secret=None,
74
+ tolerations=None,
75
+ preemption_mode=None,
76
+ security_context=None,
77
+ service_type=None,
78
+ add_templated_ingress_host_mode=None,
79
+ clone_target_dir=None,
80
+ state_thresholds=None,
81
+ disable_default_http_trigger=None,
82
+ internal_application_port=None,
83
+ ):
84
+ super().__init__(
85
+ command=command,
86
+ args=args,
87
+ image=image,
88
+ mode=mode,
89
+ entry_points=entry_points,
90
+ description=description,
91
+ replicas=replicas,
92
+ min_replicas=min_replicas,
93
+ max_replicas=max_replicas,
94
+ volumes=volumes,
95
+ volume_mounts=volume_mounts,
96
+ env=env,
97
+ resources=resources,
98
+ config=config,
99
+ base_spec=base_spec,
100
+ no_cache=no_cache,
101
+ source=source,
102
+ image_pull_policy=image_pull_policy,
103
+ function_kind=function_kind,
104
+ build=build,
105
+ service_account=service_account,
106
+ readiness_timeout=readiness_timeout,
107
+ readiness_timeout_before_failure=readiness_timeout_before_failure,
108
+ default_handler=default_handler,
109
+ node_name=node_name,
110
+ node_selector=node_selector,
111
+ affinity=affinity,
112
+ disable_auto_mount=disable_auto_mount,
113
+ priority_class_name=priority_class_name,
114
+ pythonpath=pythonpath,
115
+ workdir=workdir,
116
+ image_pull_secret=image_pull_secret,
117
+ tolerations=tolerations,
118
+ preemption_mode=preemption_mode,
119
+ security_context=security_context,
120
+ service_type=service_type,
121
+ add_templated_ingress_host_mode=add_templated_ingress_host_mode,
122
+ clone_target_dir=clone_target_dir,
123
+ state_thresholds=state_thresholds,
124
+ disable_default_http_trigger=disable_default_http_trigger,
125
+ )
126
+
127
+ # Override default min/max replicas (don't assume application is stateless)
128
+ self.min_replicas = min_replicas or 1
129
+ self.max_replicas = max_replicas or 1
130
+
131
+ self.internal_application_port = (
132
+ internal_application_port
133
+ or mlrun.mlconf.function.application.default_sidecar_internal_port
134
+ )
135
+
136
+ @property
137
+ def internal_application_port(self):
138
+ return self._internal_application_port
139
+
140
+ @internal_application_port.setter
141
+ def internal_application_port(self, port):
142
+ port = int(port)
143
+ if port < 0 or port > 65535:
144
+ raise ValueError("Port must be in the range 0-65535")
145
+ self._internal_application_port = port
146
+
147
+
148
+ class ApplicationStatus(NuclioStatus):
149
+ def __init__(
150
+ self,
151
+ state=None,
152
+ nuclio_name=None,
153
+ address=None,
154
+ internal_invocation_urls=None,
155
+ external_invocation_urls=None,
156
+ build_pod=None,
157
+ container_image=None,
158
+ application_image=None,
159
+ application_source=None,
160
+ sidecar_name=None,
161
+ api_gateway_name=None,
162
+ api_gateway=None,
163
+ url=None,
164
+ ):
165
+ super().__init__(
166
+ state=state,
167
+ nuclio_name=nuclio_name,
168
+ address=address,
169
+ internal_invocation_urls=internal_invocation_urls,
170
+ external_invocation_urls=external_invocation_urls,
171
+ build_pod=build_pod,
172
+ container_image=container_image,
173
+ )
174
+ self.application_image = application_image or None
175
+ self.application_source = application_source or None
176
+ self.sidecar_name = sidecar_name or None
177
+ self.api_gateway_name = api_gateway_name or None
178
+ self.api_gateway: typing.Optional[APIGateway] = api_gateway or None
179
+ self.url = url or None
180
+
181
+
182
+ class ApplicationRuntime(RemoteRuntime):
183
+ kind = "application"
184
+ reverse_proxy_image = None
185
+
186
+ @min_nuclio_versions("1.13.1")
187
+ def __init__(self, spec=None, metadata=None):
188
+ super().__init__(spec=spec, metadata=metadata)
189
+
190
+ @property
191
+ def spec(self) -> ApplicationSpec:
192
+ return self._spec
193
+
194
+ @spec.setter
195
+ def spec(self, spec):
196
+ self._spec = self._verify_dict(spec, "spec", ApplicationSpec)
197
+
198
+ @property
199
+ def status(self) -> ApplicationStatus:
200
+ return self._status
201
+
202
+ @status.setter
203
+ def status(self, status):
204
+ self._status = self._verify_dict(status, "status", ApplicationStatus)
205
+
206
+ @property
207
+ def api_gateway(self):
208
+ return self.status.api_gateway
209
+
210
+ @api_gateway.setter
211
+ def api_gateway(self, api_gateway: APIGateway):
212
+ self.status.api_gateway = api_gateway
213
+
214
+ @property
215
+ def url(self):
216
+ if not self.status.api_gateway:
217
+ self._sync_api_gateway()
218
+ return self.status.api_gateway.invoke_url
219
+
220
+ @url.setter
221
+ def url(self, url):
222
+ self.status.url = url
223
+
224
+ def set_internal_application_port(self, port: int):
225
+ self.spec.internal_application_port = port
226
+
227
+ def pre_deploy_validation(self):
228
+ super().pre_deploy_validation()
229
+ if not self.spec.config.get("spec.sidecars"):
230
+ raise mlrun.errors.MLRunBadRequestError(
231
+ "Application spec must include a sidecar configuration"
232
+ )
233
+
234
+ sidecars = self.spec.config["spec.sidecars"]
235
+ for sidecar in sidecars:
236
+ if not sidecar.get("image"):
237
+ raise mlrun.errors.MLRunBadRequestError(
238
+ "Application sidecar spec must include an image"
239
+ )
240
+
241
+ if not sidecar.get("ports"):
242
+ raise mlrun.errors.MLRunBadRequestError(
243
+ "Application sidecar spec must include at least one port"
244
+ )
245
+
246
+ ports = sidecar["ports"]
247
+ for port in ports:
248
+ if not port.get("containerPort"):
249
+ raise mlrun.errors.MLRunBadRequestError(
250
+ "Application sidecar port spec must include a containerPort"
251
+ )
252
+
253
+ if not port.get("name"):
254
+ raise mlrun.errors.MLRunBadRequestError(
255
+ "Application sidecar port spec must include a name"
256
+ )
257
+
258
+ if not sidecar.get("command") and sidecar.get("args"):
259
+ raise mlrun.errors.MLRunBadRequestError(
260
+ "Application sidecar spec must include a command if args are provided"
261
+ )
262
+
263
+ def prepare_image_for_deploy(self):
264
+ if self.spec.build.source and self.spec.build.load_source_on_run:
265
+ logger.warning(
266
+ "Application runtime requires loading the source into the application image. "
267
+ f"Even though {self.spec.build.load_source_on_run=}, loading on build will be forced."
268
+ )
269
+ self.spec.build.load_source_on_run = False
270
+ super().prepare_image_for_deploy()
271
+
272
+ def deploy(
273
+ self,
274
+ project="",
275
+ tag="",
276
+ verbose=False,
277
+ auth_info: schemas.AuthInfo = None,
278
+ builder_env: dict = None,
279
+ force_build: bool = False,
280
+ with_mlrun=None,
281
+ skip_deployed=False,
282
+ is_kfp=False,
283
+ mlrun_version_specifier=None,
284
+ show_on_failure: bool = False,
285
+ create_default_api_gateway: bool = True,
286
+ ):
287
+ """
288
+ Deploy function, builds the application image if required (self.requires_build()) or force_build is True,
289
+ Once the image is built, the function is deployed.
290
+
291
+ :param project: Project name
292
+ :param tag: Function tag
293
+ :param verbose: Set True for verbose logging
294
+ :param auth_info: Service AuthInfo (deprecated and ignored)
295
+ :param builder_env: Env vars dict for source archive config/credentials
296
+ e.g. builder_env={"GIT_TOKEN": token}
297
+ :param force_build: Set True for force building the application image
298
+ :param with_mlrun: Add the current mlrun package to the container build
299
+ :param skip_deployed: Skip the build if we already have an image for the function
300
+ :param is_kfp: Deploy as part of a kfp pipeline
301
+ :param mlrun_version_specifier: Which mlrun package version to include (if not current)
302
+ :param show_on_failure: Show logs only in case of build failure
303
+ :param create_default_api_gateway: When deploy finishes the default API gateway will be created for the
304
+ application. Disabling this flag means that the application will not be
305
+ accessible until an API gateway is created for it.
306
+
307
+ :return: The default API gateway URL if created or True if the function is ready (deployed)
308
+ """
309
+ if (self.requires_build() and not self.spec.image) or force_build:
310
+ self._fill_credentials()
311
+ self._build_application_image(
312
+ builder_env=builder_env,
313
+ force_build=force_build,
314
+ watch=True,
315
+ with_mlrun=with_mlrun,
316
+ skip_deployed=skip_deployed,
317
+ is_kfp=is_kfp,
318
+ mlrun_version_specifier=mlrun_version_specifier,
319
+ show_on_failure=show_on_failure,
320
+ )
321
+
322
+ # This is a class method that accepts a function instance, so we pass self as the function instance
323
+ self._ensure_reverse_proxy_configurations(self)
324
+ self._configure_application_sidecar()
325
+
326
+ # We only allow accessing the application via the API Gateway
327
+ self.spec.add_templated_ingress_host_mode = (
328
+ NuclioIngressAddTemplatedIngressModes.never
329
+ )
330
+
331
+ super().deploy(
332
+ project=project,
333
+ tag=tag,
334
+ verbose=verbose,
335
+ auth_info=auth_info,
336
+ builder_env=builder_env,
337
+ )
338
+ logger.info(
339
+ "Successfully deployed function.",
340
+ )
341
+
342
+ # Restore the source in case it was removed to make nuclio not consider it when building
343
+ if not self.spec.build.source and self.status.application_source:
344
+ self.spec.build.source = self.status.application_source
345
+ self.save(versioned=False)
346
+
347
+ if create_default_api_gateway:
348
+ try:
349
+ api_gateway_name = self.resolve_default_api_gateway_name()
350
+ return self.create_api_gateway(api_gateway_name, set_as_default=True)
351
+ except Exception as exc:
352
+ logger.warning(
353
+ "Failed to create default API gateway, application may not be accessible. "
354
+ "Use the `create_api_gateway` method to make it accessible",
355
+ exc=mlrun.errors.err_to_str(exc),
356
+ )
357
+ elif not self.status.api_gateway:
358
+ logger.warning(
359
+ "Application is online but may not be accessible since default gateway creation was not requested."
360
+ "Use the `create_api_gateway` method to make it accessible."
361
+ )
362
+
363
+ return True
364
+
365
+ def with_source_archive(
366
+ self,
367
+ source,
368
+ workdir=None,
369
+ pull_at_runtime: bool = False,
370
+ target_dir: str = None,
371
+ ):
372
+ """load the code from git/tar/zip archive at build
373
+
374
+ :param source: valid absolute path or URL to git, zip, or tar file, e.g.
375
+ git://github.com/mlrun/something.git
376
+ http://some/url/file.zip
377
+ note path source must exist on the image or exist locally when run is local
378
+ (it is recommended to use 'workdir' when source is a filepath instead)
379
+ :param workdir: working dir relative to the archive root (e.g. './subdir') or absolute to the image root
380
+ :param pull_at_runtime: currently not supported, source must be loaded into the image during the build process
381
+ :param target_dir: target dir on runtime pod or repo clone / archive extraction
382
+ """
383
+ if pull_at_runtime:
384
+ logger.warning(
385
+ f"{pull_at_runtime=} is currently not supported for application runtime "
386
+ "and will be overridden to False",
387
+ pull_at_runtime=pull_at_runtime,
388
+ )
389
+
390
+ self._configure_mlrun_build_with_source(
391
+ source=source,
392
+ workdir=workdir,
393
+ pull_at_runtime=False,
394
+ target_dir=target_dir,
395
+ )
396
+
397
+ def from_image(self, image):
398
+ """
399
+ Deploy the function with an existing nuclio processor image.
400
+ This applies only for the reverse proxy and not the application image.
401
+
402
+ :param image: image name
403
+ """
404
+ super().from_image(image)
405
+ # nuclio implementation detail - when providing the image and emptying out the source code and build source,
406
+ # nuclio skips rebuilding the image and simply takes the prebuilt image
407
+ self.spec.build.functionSourceCode = ""
408
+ self.status.application_source = self.spec.build.source
409
+ self.spec.build.source = ""
410
+
411
+ # save the image in the status, so we won't repopulate the function source code
412
+ self.status.container_image = image
413
+
414
+ # ensure golang runtime and handler for the reverse proxy
415
+ self.spec.nuclio_runtime = "golang"
416
+ update_in(
417
+ self.spec.base_spec,
418
+ "spec.handler",
419
+ "main:Handler",
420
+ )
421
+
422
+ @staticmethod
423
+ def get_filename_and_handler() -> (str, str):
424
+ reverse_proxy_file_path = pathlib.Path(__file__).parent / "reverse_proxy.go"
425
+ return str(reverse_proxy_file_path), "Handler"
426
+
427
+ def create_api_gateway(
428
+ self,
429
+ name: str = None,
430
+ path: str = None,
431
+ direct_port_access: bool = False,
432
+ authentication_mode: schemas.APIGatewayAuthenticationMode = None,
433
+ authentication_creds: tuple[str, str] = None,
434
+ ssl_redirect: bool = None,
435
+ set_as_default: bool = False,
436
+ gateway_timeout: typing.Optional[int] = None,
437
+ ):
438
+ """
439
+ Create the application API gateway. Once the application is deployed, the API gateway can be created.
440
+ An application without an API gateway is not accessible.
441
+
442
+ :param name: The name of the API gateway
443
+ :param path: Optional path of the API gateway, default value is "/".
444
+ The given path should be supported by the deployed application
445
+ :param direct_port_access: Set True to allow direct port access to the application sidecar
446
+ :param authentication_mode: API Gateway authentication mode
447
+ :param authentication_creds: API Gateway basic authentication credentials as a tuple (username, password)
448
+ :param ssl_redirect: Set True to force SSL redirect, False to disable. Defaults to
449
+ mlrun.mlconf.force_api_gateway_ssl_redirect()
450
+ :param set_as_default: Set the API gateway as the default for the application (`status.api_gateway`)
451
+ :param gateway_timeout: nginx ingress timeout in sec (request timeout, when will the gateway return an
452
+ error)
453
+ :return: The API gateway URL
454
+ """
455
+ if not name:
456
+ raise mlrun.errors.MLRunInvalidArgumentError(
457
+ "API gateway name must be specified."
458
+ )
459
+
460
+ if not set_as_default and name == self.resolve_default_api_gateway_name():
461
+ raise mlrun.errors.MLRunInvalidArgumentError(
462
+ f"Non-default API gateway cannot use the default gateway name, {name=}."
463
+ )
464
+
465
+ if (
466
+ authentication_mode == schemas.APIGatewayAuthenticationMode.basic
467
+ and not authentication_creds
468
+ ):
469
+ raise mlrun.errors.MLRunInvalidArgumentError(
470
+ "Authentication credentials not provided"
471
+ )
472
+
473
+ ports = self.spec.internal_application_port if direct_port_access else []
474
+
475
+ api_gateway = APIGateway(
476
+ APIGatewayMetadata(
477
+ name=name,
478
+ namespace=self.metadata.namespace,
479
+ labels=self.metadata.labels.copy(),
480
+ ),
481
+ APIGatewaySpec(
482
+ functions=[self],
483
+ project=self.metadata.project,
484
+ path=path,
485
+ ports=mlrun.utils.helpers.as_list(ports) if ports else None,
486
+ ),
487
+ )
488
+
489
+ api_gateway.with_gateway_timeout(gateway_timeout)
490
+ if ssl_redirect is None:
491
+ ssl_redirect = mlrun.mlconf.force_api_gateway_ssl_redirect()
492
+ if ssl_redirect:
493
+ # Force ssl redirect so that the application is only accessible via https
494
+ api_gateway.with_force_ssl_redirect()
495
+
496
+ # Add authentication if required
497
+ authentication_mode = (
498
+ authentication_mode
499
+ or mlrun.mlconf.function.application.default_authentication_mode
500
+ )
501
+ if authentication_mode == schemas.APIGatewayAuthenticationMode.access_key:
502
+ api_gateway.with_access_key_auth()
503
+ elif authentication_mode == schemas.APIGatewayAuthenticationMode.basic:
504
+ api_gateway.with_basic_auth(*authentication_creds)
505
+
506
+ db = self._get_db()
507
+ api_gateway_scheme = db.store_api_gateway(
508
+ api_gateway=api_gateway.to_scheme(), project=self.metadata.project
509
+ )
510
+
511
+ if set_as_default:
512
+ self.status.api_gateway_name = api_gateway_scheme.metadata.name
513
+ self.status.api_gateway = APIGateway.from_scheme(api_gateway_scheme)
514
+ self.status.api_gateway.wait_for_readiness()
515
+ self.url = self.status.api_gateway.invoke_url
516
+ url = self.url
517
+ else:
518
+ api_gateway = APIGateway.from_scheme(api_gateway_scheme)
519
+ api_gateway.wait_for_readiness()
520
+ url = api_gateway.invoke_url
521
+ # Update application status (enriches invocation url)
522
+ self._get_state(raise_on_exception=False)
523
+
524
+ logger.info("Successfully created API gateway", url=url)
525
+ return url
526
+
527
+ def delete_api_gateway(self, name: str):
528
+ """
529
+ Delete API gateway by name.
530
+ Refreshes the application status to update api gateway and invocation URLs.
531
+ :param name: The API gateway name
532
+ """
533
+ self._get_db().delete_api_gateway(name=name, project=self.metadata.project)
534
+ if name == self.status.api_gateway_name:
535
+ self.status.api_gateway_name = None
536
+ self.status.api_gateway = None
537
+ self._get_state()
538
+
539
+ def invoke(
540
+ self,
541
+ path: str = "",
542
+ body: typing.Optional[typing.Union[str, bytes, dict]] = None,
543
+ method: str = None,
544
+ headers: dict = None,
545
+ dashboard: str = "",
546
+ force_external_address: bool = False,
547
+ auth_info: schemas.AuthInfo = None,
548
+ mock: bool = None,
549
+ credentials: tuple[str, str] = None,
550
+ **http_client_kwargs,
551
+ ):
552
+ self._sync_api_gateway()
553
+
554
+ # If the API Gateway is not ready or not set, try to invoke the function directly (without the API Gateway)
555
+ if not self.status.api_gateway:
556
+ logger.warning(
557
+ "Default API gateway is not configured, invoking function invocation URL."
558
+ )
559
+ # create a requests auth object if credentials are provided and not already set in the http client kwargs
560
+ auth = http_client_kwargs.pop("auth", None) or (
561
+ nuclio.auth.AuthInfo(
562
+ username=credentials[0], password=credentials[1]
563
+ ).to_requests_auth()
564
+ if credentials
565
+ else None
566
+ )
567
+ return super().invoke(
568
+ path,
569
+ body,
570
+ method,
571
+ headers,
572
+ dashboard,
573
+ force_external_address,
574
+ auth_info,
575
+ mock,
576
+ auth=auth,
577
+ **http_client_kwargs,
578
+ )
579
+
580
+ if not method:
581
+ method = "POST" if body else "GET"
582
+
583
+ return self.status.api_gateway.invoke(
584
+ method=method,
585
+ headers=headers,
586
+ credentials=credentials,
587
+ path=path,
588
+ body=body,
589
+ **http_client_kwargs,
590
+ )
591
+
592
+ @classmethod
593
+ def deploy_reverse_proxy_image(cls):
594
+ """
595
+ Build the reverse proxy image and save it.
596
+ The reverse proxy image is used to route requests to the application sidecar.
597
+ This is useful when you want to decrease build time by building the application image only once.
598
+
599
+ :param use_cache: Use the cache when building the image
600
+ """
601
+ # create a function that includes only the reverse proxy, without the application
602
+
603
+ reverse_proxy_func = mlrun.run.new_function(
604
+ name="reverse-proxy-temp", kind="remote"
605
+ )
606
+ # default max replicas is 4, we only need one replica for the reverse proxy
607
+ reverse_proxy_func.spec.max_replicas = 1
608
+
609
+ # the reverse proxy image should not be based on another image
610
+ reverse_proxy_func.set_config("spec.build.baseImage", None)
611
+ reverse_proxy_func.spec.image = ""
612
+ reverse_proxy_func.spec.build.base_image = ""
613
+
614
+ cls._ensure_reverse_proxy_configurations(reverse_proxy_func)
615
+ reverse_proxy_func.deploy()
616
+
617
+ # save the created container image
618
+ cls.reverse_proxy_image = reverse_proxy_func.status.container_image
619
+
620
+ # delete the function to avoid cluttering the project
621
+ mlrun.get_run_db().delete_function(
622
+ reverse_proxy_func.metadata.name, reverse_proxy_func.metadata.project
623
+ )
624
+
625
+ def resolve_default_api_gateway_name(self):
626
+ return (
627
+ f"{self.metadata.name}-{self.metadata.tag}"
628
+ if self.metadata.tag
629
+ else self.metadata.name
630
+ )
631
+
632
+ @min_nuclio_versions("1.13.1")
633
+ def disable_default_http_trigger(
634
+ self,
635
+ ):
636
+ raise mlrun.runtimes.RunError(
637
+ "Application runtime does not support disabling the default HTTP trigger"
638
+ )
639
+
640
+ @min_nuclio_versions("1.13.1")
641
+ def enable_default_http_trigger(
642
+ self,
643
+ ):
644
+ pass
645
+
646
+ def _run(self, runobj: "mlrun.RunObject", execution):
647
+ raise mlrun.runtimes.RunError(
648
+ "Application runtime .run() is not yet supported. Use .invoke() instead."
649
+ )
650
+
651
+ def _enrich_command_from_status(self):
652
+ pass
653
+
654
+ def _build_application_image(
655
+ self,
656
+ builder_env: dict = None,
657
+ force_build: bool = False,
658
+ watch=True,
659
+ with_mlrun=None,
660
+ skip_deployed=False,
661
+ is_kfp=False,
662
+ mlrun_version_specifier=None,
663
+ show_on_failure: bool = False,
664
+ ):
665
+ if not self.spec.command:
666
+ logger.warning(
667
+ "Building the application image without a command. "
668
+ "Use spec.command and spec.args to specify the application entrypoint",
669
+ command=self.spec.command,
670
+ args=self.spec.args,
671
+ )
672
+
673
+ if self.spec.build.source in [".", "./"]:
674
+ logger.info(
675
+ "The application is configured to use the project's source. "
676
+ "Application runtime requires loading the source into the application image. "
677
+ "Loading on build will be forced regardless of whether 'pull_at_runtime=True' was configured."
678
+ )
679
+
680
+ with_mlrun = self._resolve_build_with_mlrun(with_mlrun)
681
+ return self._build_image(
682
+ builder_env=builder_env,
683
+ force_build=force_build,
684
+ mlrun_version_specifier=mlrun_version_specifier,
685
+ show_on_failure=show_on_failure,
686
+ skip_deployed=skip_deployed,
687
+ watch=watch,
688
+ is_kfp=is_kfp,
689
+ with_mlrun=with_mlrun,
690
+ )
691
+
692
+ @staticmethod
693
+ def _ensure_reverse_proxy_configurations(function: RemoteRuntime):
694
+ if function.spec.build.functionSourceCode or function.status.container_image:
695
+ return
696
+
697
+ filename, handler = ApplicationRuntime.get_filename_and_handler()
698
+ name, spec, code = nuclio.build_file(
699
+ filename,
700
+ name=function.metadata.name,
701
+ handler=handler,
702
+ )
703
+ function.spec.function_handler = mlrun.utils.get_in(spec, "spec.handler")
704
+ function.spec.build.functionSourceCode = mlrun.utils.get_in(
705
+ spec, "spec.build.functionSourceCode"
706
+ )
707
+ function.spec.nuclio_runtime = mlrun.utils.get_in(spec, "spec.runtime")
708
+
709
+ # default the reverse proxy logger level to info
710
+ logger_sinks_key = "spec.loggerSinks"
711
+ if not function.spec.config.get(logger_sinks_key):
712
+ function.set_config(
713
+ logger_sinks_key, [{"level": "info", "sink": "myStdoutLoggerSink"}]
714
+ )
715
+
716
+ def _configure_application_sidecar(self):
717
+ # Save the application image in the status to allow overriding it with the reverse proxy entry point
718
+ if self.spec.image and (
719
+ not self.status.application_image
720
+ or self.spec.image != self.status.container_image
721
+ ):
722
+ self.status.application_image = self.spec.image
723
+ self.spec.image = ""
724
+
725
+ # reuse the reverse proxy image if it was built before
726
+ if (
727
+ reverse_proxy_image := self.status.container_image
728
+ or self.reverse_proxy_image
729
+ ):
730
+ self.from_image(reverse_proxy_image)
731
+
732
+ self.status.sidecar_name = f"{self.metadata.name}-sidecar"
733
+ self.with_sidecar(
734
+ name=self.status.sidecar_name,
735
+ image=self.status.application_image,
736
+ ports=self.spec.internal_application_port,
737
+ command=self.spec.command,
738
+ args=self.spec.args,
739
+ )
740
+ self.set_env("SIDECAR_PORT", self.spec.internal_application_port)
741
+ self.set_env("SIDECAR_HOST", "http://localhost")
742
+
743
+ # configure the sidecar container as the default container for logging purposes
744
+ self.metadata.annotations["kubectl.kubernetes.io/default-container"] = (
745
+ self.status.sidecar_name
746
+ )
747
+
748
+ def _sync_api_gateway(self):
749
+ if not self.status.api_gateway_name:
750
+ return
751
+
752
+ db = self._get_db()
753
+ api_gateway_scheme = db.get_api_gateway(
754
+ name=self.status.api_gateway_name, project=self.metadata.project
755
+ )
756
+ self.status.api_gateway = APIGateway.from_scheme(api_gateway_scheme)
757
+ self.status.api_gateway.wait_for_readiness()
758
+ self.url = self.status.api_gateway.invoke_url