mlrun 1.7.0rc13__py3-none-any.whl → 1.7.0rc21__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (156) hide show
  1. mlrun/__init__.py +10 -1
  2. mlrun/__main__.py +23 -111
  3. mlrun/alerts/__init__.py +15 -0
  4. mlrun/alerts/alert.py +144 -0
  5. mlrun/api/schemas/__init__.py +4 -3
  6. mlrun/artifacts/__init__.py +8 -3
  7. mlrun/artifacts/base.py +36 -253
  8. mlrun/artifacts/dataset.py +9 -190
  9. mlrun/artifacts/manager.py +46 -42
  10. mlrun/artifacts/model.py +9 -141
  11. mlrun/artifacts/plots.py +14 -375
  12. mlrun/common/constants.py +65 -3
  13. mlrun/common/formatters/__init__.py +19 -0
  14. mlrun/{runtimes/mpijob/v1alpha1.py → common/formatters/artifact.py} +6 -14
  15. mlrun/common/formatters/base.py +113 -0
  16. mlrun/common/formatters/function.py +46 -0
  17. mlrun/common/formatters/pipeline.py +53 -0
  18. mlrun/common/formatters/project.py +51 -0
  19. mlrun/{runtimes → common/runtimes}/constants.py +32 -4
  20. mlrun/common/schemas/__init__.py +10 -5
  21. mlrun/common/schemas/alert.py +92 -11
  22. mlrun/common/schemas/api_gateway.py +56 -0
  23. mlrun/common/schemas/artifact.py +15 -5
  24. mlrun/common/schemas/auth.py +2 -0
  25. mlrun/common/schemas/client_spec.py +1 -0
  26. mlrun/common/schemas/frontend_spec.py +1 -0
  27. mlrun/common/schemas/function.py +4 -0
  28. mlrun/common/schemas/model_monitoring/__init__.py +15 -3
  29. mlrun/common/schemas/model_monitoring/constants.py +58 -7
  30. mlrun/common/schemas/model_monitoring/grafana.py +9 -5
  31. mlrun/common/schemas/model_monitoring/model_endpoints.py +86 -2
  32. mlrun/common/schemas/pipeline.py +0 -9
  33. mlrun/common/schemas/project.py +6 -11
  34. mlrun/common/types.py +1 -0
  35. mlrun/config.py +36 -8
  36. mlrun/data_types/to_pandas.py +9 -9
  37. mlrun/datastore/base.py +41 -9
  38. mlrun/datastore/datastore.py +6 -2
  39. mlrun/datastore/datastore_profile.py +56 -4
  40. mlrun/datastore/hdfs.py +5 -0
  41. mlrun/datastore/inmem.py +2 -2
  42. mlrun/datastore/redis.py +2 -2
  43. mlrun/datastore/s3.py +5 -0
  44. mlrun/datastore/sources.py +147 -7
  45. mlrun/datastore/store_resources.py +7 -7
  46. mlrun/datastore/targets.py +129 -9
  47. mlrun/datastore/utils.py +42 -0
  48. mlrun/datastore/v3io.py +1 -1
  49. mlrun/db/auth_utils.py +152 -0
  50. mlrun/db/base.py +55 -11
  51. mlrun/db/httpdb.py +346 -107
  52. mlrun/db/nopdb.py +52 -10
  53. mlrun/errors.py +11 -0
  54. mlrun/execution.py +24 -9
  55. mlrun/feature_store/__init__.py +0 -2
  56. mlrun/feature_store/api.py +12 -47
  57. mlrun/feature_store/feature_set.py +9 -0
  58. mlrun/feature_store/feature_vector.py +8 -0
  59. mlrun/feature_store/ingestion.py +7 -6
  60. mlrun/feature_store/retrieval/base.py +9 -4
  61. mlrun/feature_store/retrieval/conversion.py +9 -9
  62. mlrun/feature_store/retrieval/dask_merger.py +2 -0
  63. mlrun/feature_store/retrieval/job.py +9 -3
  64. mlrun/feature_store/retrieval/local_merger.py +2 -0
  65. mlrun/feature_store/retrieval/spark_merger.py +16 -0
  66. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +7 -12
  67. mlrun/frameworks/parallel_coordinates.py +2 -1
  68. mlrun/frameworks/tf_keras/__init__.py +4 -1
  69. mlrun/k8s_utils.py +10 -11
  70. mlrun/launcher/base.py +4 -3
  71. mlrun/launcher/client.py +5 -3
  72. mlrun/launcher/local.py +8 -2
  73. mlrun/launcher/remote.py +8 -2
  74. mlrun/lists.py +6 -2
  75. mlrun/model.py +62 -20
  76. mlrun/model_monitoring/__init__.py +1 -1
  77. mlrun/model_monitoring/api.py +41 -18
  78. mlrun/model_monitoring/application.py +5 -305
  79. mlrun/model_monitoring/applications/__init__.py +11 -0
  80. mlrun/model_monitoring/applications/_application_steps.py +157 -0
  81. mlrun/model_monitoring/applications/base.py +280 -0
  82. mlrun/model_monitoring/applications/context.py +214 -0
  83. mlrun/model_monitoring/applications/evidently_base.py +211 -0
  84. mlrun/model_monitoring/applications/histogram_data_drift.py +132 -91
  85. mlrun/model_monitoring/applications/results.py +99 -0
  86. mlrun/model_monitoring/controller.py +3 -1
  87. mlrun/model_monitoring/db/__init__.py +2 -0
  88. mlrun/model_monitoring/db/stores/__init__.py +0 -2
  89. mlrun/model_monitoring/db/stores/base/store.py +22 -37
  90. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +43 -21
  91. mlrun/model_monitoring/db/stores/sqldb/models/base.py +39 -8
  92. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +27 -7
  93. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +5 -0
  94. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +246 -224
  95. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +232 -216
  96. mlrun/model_monitoring/db/tsdb/__init__.py +100 -0
  97. mlrun/model_monitoring/db/tsdb/base.py +329 -0
  98. mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
  99. mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
  100. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +240 -0
  101. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +45 -0
  102. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +397 -0
  103. mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
  104. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +117 -0
  105. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +636 -0
  106. mlrun/model_monitoring/evidently_application.py +6 -118
  107. mlrun/model_monitoring/helpers.py +46 -1
  108. mlrun/model_monitoring/model_endpoint.py +3 -2
  109. mlrun/model_monitoring/stream_processing.py +57 -216
  110. mlrun/model_monitoring/writer.py +134 -124
  111. mlrun/package/utils/_formatter.py +2 -2
  112. mlrun/platforms/__init__.py +10 -9
  113. mlrun/platforms/iguazio.py +21 -202
  114. mlrun/projects/operations.py +19 -12
  115. mlrun/projects/pipelines.py +103 -109
  116. mlrun/projects/project.py +377 -137
  117. mlrun/render.py +15 -14
  118. mlrun/run.py +16 -47
  119. mlrun/runtimes/__init__.py +6 -3
  120. mlrun/runtimes/base.py +8 -7
  121. mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
  122. mlrun/runtimes/funcdoc.py +0 -28
  123. mlrun/runtimes/kubejob.py +2 -1
  124. mlrun/runtimes/local.py +5 -2
  125. mlrun/runtimes/mpijob/__init__.py +0 -20
  126. mlrun/runtimes/mpijob/v1.py +1 -1
  127. mlrun/runtimes/nuclio/api_gateway.py +440 -208
  128. mlrun/runtimes/nuclio/application/application.py +170 -8
  129. mlrun/runtimes/nuclio/function.py +39 -49
  130. mlrun/runtimes/pod.py +21 -41
  131. mlrun/runtimes/remotesparkjob.py +9 -3
  132. mlrun/runtimes/sparkjob/spark3job.py +1 -1
  133. mlrun/runtimes/utils.py +6 -45
  134. mlrun/serving/server.py +2 -1
  135. mlrun/serving/states.py +53 -2
  136. mlrun/serving/v2_serving.py +5 -1
  137. mlrun/track/tracker.py +2 -1
  138. mlrun/utils/async_http.py +25 -5
  139. mlrun/utils/helpers.py +107 -75
  140. mlrun/utils/logger.py +39 -7
  141. mlrun/utils/notifications/notification/__init__.py +14 -9
  142. mlrun/utils/notifications/notification/base.py +1 -1
  143. mlrun/utils/notifications/notification/slack.py +61 -13
  144. mlrun/utils/notifications/notification/webhook.py +1 -1
  145. mlrun/utils/notifications/notification_pusher.py +147 -16
  146. mlrun/utils/regex.py +9 -0
  147. mlrun/utils/v3io_clients.py +0 -1
  148. mlrun/utils/version/version.json +2 -2
  149. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc21.dist-info}/METADATA +14 -6
  150. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc21.dist-info}/RECORD +154 -133
  151. mlrun/kfpops.py +0 -865
  152. mlrun/platforms/other.py +0 -305
  153. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc21.dist-info}/LICENSE +0 -0
  154. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc21.dist-info}/WHEEL +0 -0
  155. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc21.dist-info}/entry_points.txt +0 -0
  156. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc21.dist-info}/top_level.txt +0 -0
@@ -17,30 +17,30 @@ from typing import Optional, Union
17
17
  from urllib.parse import urljoin
18
18
 
19
19
  import requests
20
- from requests.auth import HTTPBasicAuth
20
+ from nuclio.auth import AuthInfo as NuclioAuthInfo
21
+ from nuclio.auth import AuthKinds as NuclioAuthKinds
21
22
 
22
23
  import mlrun
23
- import mlrun.common.schemas
24
+ import mlrun.common.constants as mlrun_constants
25
+ import mlrun.common.schemas as schemas
26
+ import mlrun.common.types
27
+ from mlrun.model import ModelObj
28
+ from mlrun.platforms.iguazio import min_iguazio_versions
29
+ from mlrun.utils import logger
24
30
 
25
- from ..utils import logger
26
- from .function import RemoteRuntime, get_fullname, min_nuclio_versions
27
- from .serving import ServingRuntime
31
+ from .function import min_nuclio_versions
28
32
 
29
- NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH = "basicAuth"
30
- NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_NONE = "none"
31
- PROJECT_NAME_LABEL = "nuclio.io/project-name"
32
33
 
33
-
34
- class APIGatewayAuthenticator(typing.Protocol):
34
+ class Authenticator(typing.Protocol):
35
35
  @property
36
36
  def authentication_mode(self) -> str:
37
- return NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_NONE
37
+ return schemas.APIGatewayAuthenticationMode.none.value
38
38
 
39
39
  @classmethod
40
- def from_scheme(cls, api_gateway_spec: mlrun.common.schemas.APIGatewaySpec):
40
+ def from_scheme(cls, api_gateway_spec: schemas.APIGatewaySpec):
41
41
  if (
42
42
  api_gateway_spec.authenticationMode
43
- == NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
43
+ == schemas.APIGatewayAuthenticationMode.basic.value
44
44
  ):
45
45
  if api_gateway_spec.authentication:
46
46
  return BasicAuth(
@@ -49,15 +49,24 @@ class APIGatewayAuthenticator(typing.Protocol):
49
49
  )
50
50
  else:
51
51
  return BasicAuth()
52
+ elif (
53
+ api_gateway_spec.authenticationMode
54
+ == schemas.APIGatewayAuthenticationMode.access_key.value
55
+ ):
56
+ return AccessKeyAuth()
52
57
  else:
53
58
  return NoneAuth()
54
59
 
55
60
  def to_scheme(
56
61
  self,
57
- ) -> Optional[dict[str, Optional[mlrun.common.schemas.APIGatewayBasicAuth]]]:
62
+ ) -> Optional[dict[str, Optional[schemas.APIGatewayBasicAuth]]]:
58
63
  return None
59
64
 
60
65
 
66
+ class APIGatewayAuthenticator(Authenticator, ModelObj):
67
+ _dict_fields = ["authentication_mode"]
68
+
69
+
61
70
  class NoneAuth(APIGatewayAuthenticator):
62
71
  """
63
72
  An API gateway authenticator with no authentication.
@@ -80,82 +89,288 @@ class BasicAuth(APIGatewayAuthenticator):
80
89
 
81
90
  @property
82
91
  def authentication_mode(self) -> str:
83
- return NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
92
+ return schemas.APIGatewayAuthenticationMode.basic.value
84
93
 
85
94
  def to_scheme(
86
95
  self,
87
- ) -> Optional[dict[str, Optional[mlrun.common.schemas.APIGatewayBasicAuth]]]:
96
+ ) -> Optional[dict[str, Optional[schemas.APIGatewayBasicAuth]]]:
88
97
  return {
89
- "basicAuth": mlrun.common.schemas.APIGatewayBasicAuth(
98
+ "basicAuth": schemas.APIGatewayBasicAuth(
90
99
  username=self._username, password=self._password
91
100
  )
92
101
  }
93
102
 
94
103
 
95
- class APIGateway:
96
- @min_nuclio_versions("1.13.1")
104
+ class AccessKeyAuth(APIGatewayAuthenticator):
105
+ """
106
+ An API gateway authenticator with access key authentication.
107
+ """
108
+
109
+ @property
110
+ def authentication_mode(self) -> str:
111
+ return schemas.APIGatewayAuthenticationMode.access_key.value
112
+
113
+
114
+ class APIGatewayMetadata(ModelObj):
115
+ _dict_fields = ["name", "namespace", "labels", "annotations", "creation_timestamp"]
116
+
97
117
  def __init__(
98
118
  self,
99
- project,
100
119
  name: str,
120
+ namespace: str = None,
121
+ labels: dict = None,
122
+ annotations: dict = None,
123
+ creation_timestamp: str = None,
124
+ ):
125
+ """
126
+ :param name: The name of the API gateway
127
+ :param namespace: The namespace of the API gateway
128
+ :param labels: The labels of the API gateway
129
+ :param annotations: The annotations of the API gateway
130
+ :param creation_timestamp: The creation timestamp of the API gateway
131
+ """
132
+ self.name = name
133
+ self.namespace = namespace
134
+ self.labels = labels or {}
135
+ self.annotations = annotations or {}
136
+ self.creation_timestamp = creation_timestamp
137
+
138
+ if not self.name:
139
+ raise mlrun.errors.MLRunInvalidArgumentError(
140
+ "API Gateway name cannot be empty"
141
+ )
142
+
143
+
144
+ class APIGatewaySpec(ModelObj):
145
+ _dict_fields = [
146
+ "functions",
147
+ "project",
148
+ "name",
149
+ "description",
150
+ "host",
151
+ "path",
152
+ "authentication",
153
+ "canary",
154
+ ]
155
+
156
+ def __init__(
157
+ self,
101
158
  functions: Union[
102
- list[str],
103
- Union[
104
- list[
105
- Union[
106
- RemoteRuntime,
107
- ServingRuntime,
108
- ]
109
- ],
110
- Union[RemoteRuntime, ServingRuntime],
159
+ list[
160
+ Union[
161
+ str,
162
+ "mlrun.runtimes.nuclio.function.RemoteRuntime",
163
+ "mlrun.runtimes.nuclio.serving.ServingRuntime",
164
+ "mlrun.runtimes.nuclio.application.ApplicationRuntime",
165
+ ]
111
166
  ],
167
+ "mlrun.runtimes.nuclio.function.RemoteRuntime",
168
+ "mlrun.runtimes.nuclio.serving.ServingRuntime",
169
+ "mlrun.runtimes.nuclio.application.ApplicationRuntime",
112
170
  ],
171
+ project: str = None,
113
172
  description: str = "",
173
+ host: str = None,
114
174
  path: str = "/",
115
175
  authentication: Optional[APIGatewayAuthenticator] = NoneAuth(),
116
- host: Optional[str] = None,
117
176
  canary: Optional[list[int]] = None,
177
+ ports: Optional[list[int]] = None,
118
178
  ):
119
179
  """
120
- Initialize the APIGateway instance.
121
-
122
- :param project: The project name
123
- :param name: The name of the API gateway
124
180
  :param functions: The list of functions associated with the API gateway
125
181
  Can be a list of function names (["my-func1", "my-func2"])
126
182
  or a list or a single entity of
127
183
  :py:class:`~mlrun.runtimes.nuclio.function.RemoteRuntime` OR
128
- :py:class:`~mlrun.runtimes.nuclio.serving.ServingRuntime`
129
-
184
+ :py:class:`~mlrun.runtimes.nuclio.serving.ServingRuntime` OR
185
+ :py:class:`~mlrun.runtimes.nuclio.application.ApplicationRuntime`
186
+ :param project: The project name
130
187
  :param description: Optional description of the API gateway
131
188
  :param path: Optional path of the API gateway, default value is "/"
132
189
  :param authentication: The authentication for the API gateway of type
133
190
  :py:class:`~mlrun.runtimes.nuclio.api_gateway.BasicAuth`
134
191
  :param host: The host of the API gateway (optional). If not set, it will be automatically generated
135
- :param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80]
192
+ :param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80] (optional)
193
+ :param ports: The ports of the API gateway, as a list of integers that correspond to the functions in the
194
+ functions list. for instance: [8050] or [8050, 8081] (optional)
136
195
  """
137
- self.functions = None
138
- self._validate(
139
- project=project,
140
- functions=functions,
141
- name=name,
142
- canary=canary,
143
- )
144
- self.project = project
145
- self.name = name
196
+ self.description = description
146
197
  self.host = host
147
-
148
198
  self.path = path
149
- self.description = description
150
- self.canary = canary
151
199
  self.authentication = authentication
200
+ self.functions = functions
201
+ self.canary = canary
202
+ self.project = project
203
+ self.ports = ports
204
+
205
+ self.validate(project=project, functions=functions, canary=canary, ports=ports)
206
+
207
+ def validate(
208
+ self,
209
+ project: str,
210
+ functions: Union[
211
+ list[
212
+ Union[
213
+ str,
214
+ "mlrun.runtimes.nuclio.function.RemoteRuntime",
215
+ "mlrun.runtimes.nuclio.serving.ServingRuntime",
216
+ "mlrun.runtimes.nuclio.application.ApplicationRuntime",
217
+ ]
218
+ ],
219
+ "mlrun.runtimes.nuclio.function.RemoteRuntime",
220
+ "mlrun.runtimes.nuclio.serving.ServingRuntime",
221
+ "mlrun.runtimes.nuclio.application.ApplicationRuntime",
222
+ ],
223
+ canary: Optional[list[int]] = None,
224
+ ports: Optional[list[int]] = None,
225
+ ):
226
+ self.functions = self._validate_functions(project=project, functions=functions)
227
+
228
+ # validating canary
229
+ if canary:
230
+ self.canary = self._validate_canary(canary)
231
+
232
+ # validating ports
233
+ if ports:
234
+ self.ports = self._validate_ports(ports)
235
+
236
+ def _validate_canary(self, canary: list[int]):
237
+ if len(self.functions) != len(canary):
238
+ raise mlrun.errors.MLRunInvalidArgumentError(
239
+ "Function and canary lists lengths do not match"
240
+ )
241
+ for canary_percent in canary:
242
+ if canary_percent < 0 or canary_percent > 100:
243
+ raise mlrun.errors.MLRunInvalidArgumentError(
244
+ "The percentage value must be in the range from 0 to 100"
245
+ )
246
+ if sum(canary) != 100:
247
+ raise mlrun.errors.MLRunInvalidArgumentError(
248
+ "The sum of canary function percents should be equal to 100"
249
+ )
250
+ return canary
251
+
252
+ def _validate_ports(self, ports):
253
+ if len(self.functions) != len(ports):
254
+ raise mlrun.errors.MLRunInvalidArgumentError(
255
+ "Function and port lists lengths do not match"
256
+ )
257
+
258
+ return ports
259
+
260
+ @staticmethod
261
+ def _validate_functions(
262
+ project: str,
263
+ functions: Union[
264
+ list[
265
+ Union[
266
+ str,
267
+ "mlrun.runtimes.nuclio.function.RemoteRuntime",
268
+ "mlrun.runtimes.nuclio.serving.ServingRuntime",
269
+ "mlrun.runtimes.nuclio.application.ApplicationRuntime",
270
+ ]
271
+ ],
272
+ "mlrun.runtimes.nuclio.function.RemoteRuntime",
273
+ "mlrun.runtimes.nuclio.serving.ServingRuntime",
274
+ "mlrun.runtimes.nuclio.application.ApplicationRuntime",
275
+ ],
276
+ ):
277
+ if not isinstance(functions, list):
278
+ functions = [functions]
279
+
280
+ # validating functions
281
+ if not 1 <= len(functions) <= 2:
282
+ raise mlrun.errors.MLRunInvalidArgumentError(
283
+ f"Gateway can be created from one or two functions, "
284
+ f"the number of functions passed is {len(functions)}"
285
+ )
286
+
287
+ function_names = []
288
+ for func in functions:
289
+ if isinstance(func, str):
290
+ # check whether the function was passed as a URI or just a name
291
+ parsed_project, function_name, _, _ = (
292
+ mlrun.common.helpers.parse_versioned_object_uri(func)
293
+ )
294
+
295
+ if parsed_project and function_name:
296
+ # check that parsed project and passed project are the same
297
+ if parsed_project != project:
298
+ raise mlrun.errors.MLRunInvalidArgumentError(
299
+ "Function doesn't belong to passed project"
300
+ )
301
+ function_uri = func
302
+ else:
303
+ function_uri = mlrun.utils.generate_object_uri(project, func)
304
+ function_names.append(function_uri)
305
+ continue
306
+
307
+ function_name = (
308
+ func.metadata.name if hasattr(func, "metadata") else func.name
309
+ )
310
+ if func.kind not in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
311
+ raise mlrun.errors.MLRunInvalidArgumentError(
312
+ f"Input function {function_name} is not a Nuclio function"
313
+ )
314
+ if func.metadata.project != project:
315
+ raise mlrun.errors.MLRunInvalidArgumentError(
316
+ f"input function {function_name} "
317
+ f"does not belong to this project"
318
+ )
319
+ function_uri = mlrun.utils.generate_object_uri(
320
+ project,
321
+ function_name,
322
+ func.metadata.tag,
323
+ func.metadata.hash,
324
+ )
325
+ function_names.append(function_uri)
326
+ return function_names
327
+
328
+
329
+ class APIGateway(ModelObj):
330
+ _dict_fields = [
331
+ "metadata",
332
+ "spec",
333
+ "state",
334
+ ]
335
+
336
+ @min_nuclio_versions("1.13.1")
337
+ def __init__(
338
+ self,
339
+ metadata: APIGatewayMetadata,
340
+ spec: APIGatewaySpec,
341
+ ):
342
+ """
343
+ Initialize the APIGateway instance.
344
+
345
+ :param metadata: (APIGatewayMetadata) The metadata of the API gateway.
346
+ :param spec: (APIGatewaySpec) The spec of the API gateway.
347
+ """
348
+ self.metadata = metadata
349
+ self.spec = spec
152
350
  self.state = ""
153
351
 
352
+ @property
353
+ def metadata(self) -> APIGatewayMetadata:
354
+ return self._metadata
355
+
356
+ @metadata.setter
357
+ def metadata(self, metadata):
358
+ self._metadata = self._verify_dict(metadata, "metadata", APIGatewayMetadata)
359
+
360
+ @property
361
+ def spec(self) -> APIGatewaySpec:
362
+ return self._spec
363
+
364
+ @spec.setter
365
+ def spec(self, spec):
366
+ self._spec = self._verify_dict(spec, "spec", APIGatewaySpec)
367
+
154
368
  def invoke(
155
369
  self,
156
370
  method="POST",
157
- headers: dict = {},
158
- auth: Optional[tuple[str, str]] = None,
371
+ headers: dict = None,
372
+ credentials: Optional[tuple[str, str]] = None,
373
+ path: Optional[str] = None,
159
374
  **kwargs,
160
375
  ):
161
376
  """
@@ -163,7 +378,9 @@ class APIGateway:
163
378
 
164
379
  :param method: (str, optional) The HTTP method for the invocation.
165
380
  :param headers: (dict, optional) The HTTP headers for the invocation.
166
- :param auth: (Optional[tuple[str, str]], optional) The authentication creds for the invocation if required.
381
+ :param credentials: (Optional[tuple[str, str]], optional) The (username,password) for the invocation if required
382
+ can also be set by the environment variable (_, V3IO_ACCESS_KEY) for access key authentication.
383
+ :param path: (str, optional) The sub-path for the invocation.
167
384
  :param kwargs: (dict) Additional keyword arguments.
168
385
 
169
386
  :return: The response from the API gateway invocation.
@@ -180,20 +397,44 @@ class APIGateway:
180
397
  f"API gateway is not ready. " f"Current state: {self.state}"
181
398
  )
182
399
 
400
+ auth = None
401
+
183
402
  if (
184
- self.authentication.authentication_mode
185
- == NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
186
- and not auth
403
+ self.spec.authentication.authentication_mode
404
+ == schemas.APIGatewayAuthenticationMode.basic.value
187
405
  ):
188
- raise mlrun.errors.MLRunInvalidArgumentError(
189
- "API Gateway invocation requires authentication. Please pass credentials"
190
- )
406
+ if not credentials:
407
+ raise mlrun.errors.MLRunInvalidArgumentError(
408
+ "API Gateway invocation requires authentication. Please pass credentials"
409
+ )
410
+ auth = NuclioAuthInfo(
411
+ username=credentials[0], password=credentials[1]
412
+ ).to_requests_auth()
413
+
414
+ if (
415
+ self.spec.authentication.authentication_mode
416
+ == schemas.APIGatewayAuthenticationMode.access_key.value
417
+ ):
418
+ # inject access key from env
419
+ if credentials:
420
+ auth = NuclioAuthInfo(
421
+ username=credentials[0],
422
+ password=credentials[1],
423
+ mode=NuclioAuthKinds.iguazio,
424
+ ).to_requests_auth()
425
+ else:
426
+ auth = NuclioAuthInfo().from_envvar().to_requests_auth()
427
+ if not auth:
428
+ raise mlrun.errors.MLRunInvalidArgumentError(
429
+ "API Gateway invocation requires authentication. Please set V3IO_ACCESS_KEY env var"
430
+ )
431
+ url = urljoin(self.invoke_url, path or "")
191
432
  return requests.request(
192
433
  method=method,
193
- url=self.invoke_url,
194
- headers=headers,
434
+ url=url,
435
+ headers=headers or {},
436
+ auth=auth,
195
437
  **kwargs,
196
- auth=HTTPBasicAuth(*auth) if auth else None,
197
438
  )
198
439
 
199
440
  def wait_for_readiness(self, max_wait_time=90):
@@ -218,24 +459,26 @@ class APIGateway:
218
459
  )
219
460
 
220
461
  def is_ready(self):
221
- if self.state is not mlrun.common.schemas.api_gateway.APIGatewayState.ready:
462
+ if self.state is not schemas.api_gateway.APIGatewayState.ready:
222
463
  # try to sync the state
223
464
  self.sync()
224
- return self.state == mlrun.common.schemas.api_gateway.APIGatewayState.ready
465
+ return self.state == schemas.api_gateway.APIGatewayState.ready
225
466
 
226
467
  def sync(self):
227
468
  """
228
469
  Synchronize the API gateway from the server.
229
470
  """
230
- synced_gateway = mlrun.get_run_db().get_api_gateway(self.name, self.project)
471
+ synced_gateway = mlrun.get_run_db().get_api_gateway(
472
+ self.metadata.name, self.spec.project
473
+ )
231
474
  synced_gateway = self.from_scheme(synced_gateway)
232
475
 
233
- self.host = synced_gateway.host
234
- self.path = synced_gateway.path
235
- self.authentication = synced_gateway.authentication
236
- self.functions = synced_gateway.functions
237
- self.canary = synced_gateway.canary
238
- self.description = synced_gateway.description
476
+ self.spec.host = synced_gateway.spec.host
477
+ self.spec.path = synced_gateway.spec.path
478
+ self.spec.authentication = synced_gateway.spec.authentication
479
+ self.spec.functions = synced_gateway.spec.functions
480
+ self.spec.canary = synced_gateway.spec.canary
481
+ self.spec.description = synced_gateway.spec.description
239
482
  self.state = synced_gateway.state
240
483
 
241
484
  def with_basic_auth(self, username: str, password: str):
@@ -245,18 +488,29 @@ class APIGateway:
245
488
  :param username: (str) The username for basic authentication.
246
489
  :param password: (str) The password for basic authentication.
247
490
  """
248
- self.authentication = BasicAuth(username=username, password=password)
491
+ self.spec.authentication = BasicAuth(username=username, password=password)
492
+
493
+ @min_iguazio_versions("3.5.5")
494
+ def with_access_key_auth(self):
495
+ """
496
+ Set access key authentication for the API gateway.
497
+ """
498
+ self.spec.authentication = AccessKeyAuth()
249
499
 
250
500
  def with_canary(
251
501
  self,
252
502
  functions: Union[
253
- list[str],
254
503
  list[
255
504
  Union[
256
- RemoteRuntime,
257
- ServingRuntime,
505
+ str,
506
+ "mlrun.runtimes.nuclio.function.RemoteRuntime",
507
+ "mlrun.runtimes.nuclio.serving.ServingRuntime",
508
+ "mlrun.runtimes.nuclio.application.ApplicationRuntime",
258
509
  ]
259
510
  ],
511
+ "mlrun.runtimes.nuclio.function.RemoteRuntime",
512
+ "mlrun.runtimes.nuclio.serving.ServingRuntime",
513
+ "mlrun.runtimes.nuclio.application.ApplicationRuntime",
260
514
  ],
261
515
  canary: list[int],
262
516
  ):
@@ -267,7 +521,8 @@ class APIGateway:
267
521
  Can be a list of function names (["my-func1", "my-func2"])
268
522
  or a list of nuclio functions of types
269
523
  :py:class:`~mlrun.runtimes.nuclio.function.RemoteRuntime` OR
270
- :py:class:`~mlrun.runtimes.nuclio.serving.ServingRuntime`
524
+ :py:class:`~mlrun.runtimes.nuclio.serving.ServingRuntime` OR
525
+ :py:class:`~mlrun.runtimes.nuclio.application.ApplicationRuntime`
271
526
  :param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80]
272
527
 
273
528
  """
@@ -276,66 +531,88 @@ class APIGateway:
276
531
  f"Gateway with canary can be created only with two functions, "
277
532
  f"the number of functions passed is {len(functions)}"
278
533
  )
279
- self.functions = self._validate_functions(self.project, functions)
280
- self.canary = self._validate_canary(canary)
534
+ self.spec.validate(
535
+ project=self.spec.project, functions=functions, canary=canary
536
+ )
537
+
538
+ def with_ports(self, ports: list[int]):
539
+ """
540
+ Set ports for the API gateway
541
+
542
+ :param ports: The ports of the API gateway, as a list of integers that correspond to the functions in the
543
+ functions list. for instance: [8050] or [8050, 8081]
544
+ """
545
+ self.spec.validate(
546
+ project=self.spec.project, functions=self.spec.functions, ports=ports
547
+ )
281
548
 
282
549
  @classmethod
283
- def from_scheme(cls, api_gateway: mlrun.common.schemas.APIGateway):
284
- project = api_gateway.metadata.labels.get(PROJECT_NAME_LABEL)
550
+ def from_scheme(cls, api_gateway: schemas.APIGateway):
551
+ project = api_gateway.metadata.labels.get(
552
+ mlrun_constants.MLRunInternalLabels.nuclio_project_name
553
+ )
285
554
  functions, canary = cls._resolve_canary(api_gateway.spec.upstreams)
286
555
  state = (
287
556
  api_gateway.status.state
288
557
  if api_gateway.status
289
- else mlrun.common.schemas.APIGatewayState.none
558
+ else schemas.APIGatewayState.none
290
559
  )
291
- api_gateway = cls(
292
- project=project,
293
- description=api_gateway.spec.description,
294
- name=api_gateway.spec.name,
295
- host=api_gateway.spec.host,
296
- path=api_gateway.spec.path,
297
- authentication=APIGatewayAuthenticator.from_scheme(api_gateway.spec),
298
- functions=functions,
299
- canary=canary,
560
+ new_api_gateway = cls(
561
+ metadata=APIGatewayMetadata(
562
+ name=api_gateway.spec.name,
563
+ ),
564
+ spec=APIGatewaySpec(
565
+ project=project,
566
+ description=api_gateway.spec.description,
567
+ host=api_gateway.spec.host,
568
+ path=api_gateway.spec.path,
569
+ authentication=APIGatewayAuthenticator.from_scheme(api_gateway.spec),
570
+ functions=functions,
571
+ canary=canary,
572
+ ),
300
573
  )
301
- api_gateway.state = state
302
- return api_gateway
574
+ new_api_gateway.state = state
575
+ return new_api_gateway
303
576
 
304
- def to_scheme(self) -> mlrun.common.schemas.APIGateway:
577
+ def to_scheme(self) -> schemas.APIGateway:
305
578
  upstreams = (
306
579
  [
307
- mlrun.common.schemas.APIGatewayUpstream(
308
- nucliofunction={"name": self.functions[0]},
309
- percentage=self.canary[0],
580
+ schemas.APIGatewayUpstream(
581
+ nucliofunction={"name": self.spec.functions[0]},
582
+ percentage=self.spec.canary[0],
310
583
  ),
311
- mlrun.common.schemas.APIGatewayUpstream(
584
+ schemas.APIGatewayUpstream(
312
585
  # do not set percent for the second function,
313
586
  # so we can define which function to display as a primary one in UI
314
- nucliofunction={"name": self.functions[1]},
587
+ nucliofunction={"name": self.spec.functions[1]},
315
588
  ),
316
589
  ]
317
- if self.canary
590
+ if self.spec.canary
318
591
  else [
319
- mlrun.common.schemas.APIGatewayUpstream(
592
+ schemas.APIGatewayUpstream(
320
593
  nucliofunction={"name": function_name},
321
594
  )
322
- for function_name in self.functions
595
+ for function_name in self.spec.functions
323
596
  ]
324
597
  )
325
- api_gateway = mlrun.common.schemas.APIGateway(
326
- metadata=mlrun.common.schemas.APIGatewayMetadata(name=self.name, labels={}),
327
- spec=mlrun.common.schemas.APIGatewaySpec(
328
- name=self.name,
329
- description=self.description,
330
- host=self.host,
331
- path=self.path,
332
- authenticationMode=mlrun.common.schemas.APIGatewayAuthenticationMode.from_str(
333
- self.authentication.authentication_mode
598
+ if self.spec.ports:
599
+ for i, port in enumerate(self.spec.ports):
600
+ upstreams[i].port = port
601
+
602
+ api_gateway = schemas.APIGateway(
603
+ metadata=schemas.APIGatewayMetadata(name=self.metadata.name, labels={}),
604
+ spec=schemas.APIGatewaySpec(
605
+ name=self.metadata.name,
606
+ description=self.spec.description,
607
+ host=self.spec.host,
608
+ path=self.spec.path,
609
+ authenticationMode=schemas.APIGatewayAuthenticationMode.from_str(
610
+ self.spec.authentication.authentication_mode
334
611
  ),
335
612
  upstreams=upstreams,
336
613
  ),
337
614
  )
338
- api_gateway.spec.authentication = self.authentication.to_scheme()
615
+ api_gateway.spec.authentication = self.spec.authentication.to_scheme()
339
616
  return api_gateway
340
617
 
341
618
  @property
@@ -347,103 +624,10 @@ class APIGateway:
347
624
 
348
625
  :return: (str) The invoke URL.
349
626
  """
350
- host = self.host
351
- if not self.host.startswith("http"):
352
- host = f"https://{self.host}"
353
- return urljoin(host, self.path)
354
-
355
- def _validate(
356
- self,
357
- name: str,
358
- project: str,
359
- functions: Union[
360
- list[str],
361
- Union[
362
- list[
363
- Union[
364
- RemoteRuntime,
365
- ServingRuntime,
366
- ]
367
- ],
368
- Union[RemoteRuntime, ServingRuntime],
369
- ],
370
- ],
371
- canary: Optional[list[int]] = None,
372
- ):
373
- if not name:
374
- raise mlrun.errors.MLRunInvalidArgumentError(
375
- "API Gateway name cannot be empty"
376
- )
377
-
378
- self.functions = self._validate_functions(project=project, functions=functions)
379
-
380
- # validating canary
381
- if canary:
382
- self._validate_canary(canary)
383
-
384
- def _validate_canary(self, canary: list[int]):
385
- if len(self.functions) != len(canary):
386
- raise mlrun.errors.MLRunInvalidArgumentError(
387
- "Function and canary lists lengths do not match"
388
- )
389
- for canary_percent in canary:
390
- if canary_percent < 0 or canary_percent > 100:
391
- raise mlrun.errors.MLRunInvalidArgumentError(
392
- "The percentage value must be in the range from 0 to 100"
393
- )
394
- if sum(canary) != 100:
395
- raise mlrun.errors.MLRunInvalidArgumentError(
396
- "The sum of canary function percents should be equal to 100"
397
- )
398
- return canary
399
-
400
- @staticmethod
401
- def _validate_functions(
402
- project: str,
403
- functions: Union[
404
- list[str],
405
- Union[
406
- list[
407
- Union[
408
- RemoteRuntime,
409
- ServingRuntime,
410
- ]
411
- ],
412
- Union[RemoteRuntime, ServingRuntime],
413
- ],
414
- ],
415
- ):
416
- if not isinstance(functions, list):
417
- functions = [functions]
418
-
419
- # validating functions
420
- if not 1 <= len(functions) <= 2:
421
- raise mlrun.errors.MLRunInvalidArgumentError(
422
- f"Gateway can be created from one or two functions, "
423
- f"the number of functions passed is {len(functions)}"
424
- )
425
-
426
- function_names = []
427
- for func in functions:
428
- if isinstance(func, str):
429
- function_names.append(func)
430
- continue
431
-
432
- function_name = (
433
- func.metadata.name if hasattr(func, "metadata") else func.name
434
- )
435
- if func.kind not in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
436
- raise mlrun.errors.MLRunInvalidArgumentError(
437
- f"Input function {function_name} is not a Nuclio function"
438
- )
439
- if func.metadata.project != project:
440
- raise mlrun.errors.MLRunInvalidArgumentError(
441
- f"input function {function_name} "
442
- f"does not belong to this project"
443
- )
444
- nuclio_name = get_fullname(function_name, project, func.metadata.tag)
445
- function_names.append(nuclio_name)
446
- return function_names
627
+ host = self.spec.host
628
+ if not self.spec.host.startswith("http"):
629
+ host = f"https://{self.spec.host}"
630
+ return urljoin(host, self.spec.path)
447
631
 
448
632
  @staticmethod
449
633
  def _generate_basic_auth(username: str, password: str):
@@ -452,7 +636,7 @@ class APIGateway:
452
636
 
453
637
  @staticmethod
454
638
  def _resolve_canary(
455
- upstreams: list[mlrun.common.schemas.APIGatewayUpstream],
639
+ upstreams: list[schemas.APIGatewayUpstream],
456
640
  ) -> tuple[Union[list[str], None], Union[list[int], None]]:
457
641
  if len(upstreams) == 1:
458
642
  return [upstreams[0].nucliofunction.get("name")], None
@@ -475,3 +659,51 @@ class APIGateway:
475
659
  else:
476
660
  # Nuclio only supports 1 or 2 upstream functions
477
661
  return None, None
662
+
663
+ @property
664
+ def name(self):
665
+ return self.metadata.name
666
+
667
+ @name.setter
668
+ def name(self, value):
669
+ self.metadata.name = value
670
+
671
+ @property
672
+ def project(self):
673
+ return self.spec.project
674
+
675
+ @project.setter
676
+ def project(self, value):
677
+ self.spec.project = value
678
+
679
+ @property
680
+ def description(self):
681
+ return self.spec.description
682
+
683
+ @description.setter
684
+ def description(self, value):
685
+ self.spec.description = value
686
+
687
+ @property
688
+ def host(self):
689
+ return self.spec.host
690
+
691
+ @host.setter
692
+ def host(self, value):
693
+ self.spec.host = value
694
+
695
+ @property
696
+ def path(self):
697
+ return self.spec.path
698
+
699
+ @path.setter
700
+ def path(self, value):
701
+ self.spec.path = value
702
+
703
+ @property
704
+ def authentication(self):
705
+ return self.spec.authentication
706
+
707
+ @authentication.setter
708
+ def authentication(self, value):
709
+ self.spec.authentication = value