mlrun 1.10.0rc18__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (167) hide show
  1. mlrun/__init__.py +24 -3
  2. mlrun/__main__.py +0 -4
  3. mlrun/artifacts/dataset.py +2 -2
  4. mlrun/artifacts/document.py +6 -1
  5. mlrun/artifacts/llm_prompt.py +21 -15
  6. mlrun/artifacts/model.py +3 -3
  7. mlrun/artifacts/plots.py +1 -1
  8. mlrun/{model_monitoring/db/tsdb/tdengine → auth}/__init__.py +2 -3
  9. mlrun/auth/nuclio.py +89 -0
  10. mlrun/auth/providers.py +429 -0
  11. mlrun/auth/utils.py +415 -0
  12. mlrun/common/constants.py +14 -0
  13. mlrun/common/model_monitoring/helpers.py +123 -0
  14. mlrun/common/runtimes/constants.py +28 -0
  15. mlrun/common/schemas/__init__.py +14 -3
  16. mlrun/common/schemas/alert.py +2 -2
  17. mlrun/common/schemas/api_gateway.py +3 -0
  18. mlrun/common/schemas/auth.py +12 -10
  19. mlrun/common/schemas/client_spec.py +4 -0
  20. mlrun/common/schemas/constants.py +25 -0
  21. mlrun/common/schemas/frontend_spec.py +1 -8
  22. mlrun/common/schemas/function.py +34 -0
  23. mlrun/common/schemas/hub.py +33 -20
  24. mlrun/common/schemas/model_monitoring/__init__.py +2 -1
  25. mlrun/common/schemas/model_monitoring/constants.py +12 -15
  26. mlrun/common/schemas/model_monitoring/functions.py +13 -4
  27. mlrun/common/schemas/model_monitoring/model_endpoints.py +11 -0
  28. mlrun/common/schemas/pipeline.py +1 -1
  29. mlrun/common/schemas/secret.py +17 -2
  30. mlrun/common/secrets.py +95 -1
  31. mlrun/common/types.py +10 -10
  32. mlrun/config.py +69 -19
  33. mlrun/data_types/infer.py +2 -2
  34. mlrun/datastore/__init__.py +12 -5
  35. mlrun/datastore/azure_blob.py +162 -47
  36. mlrun/datastore/base.py +274 -10
  37. mlrun/datastore/datastore.py +7 -2
  38. mlrun/datastore/datastore_profile.py +84 -22
  39. mlrun/datastore/model_provider/huggingface_provider.py +225 -41
  40. mlrun/datastore/model_provider/mock_model_provider.py +87 -0
  41. mlrun/datastore/model_provider/model_provider.py +206 -74
  42. mlrun/datastore/model_provider/openai_provider.py +226 -66
  43. mlrun/datastore/s3.py +39 -18
  44. mlrun/datastore/sources.py +1 -1
  45. mlrun/datastore/store_resources.py +4 -4
  46. mlrun/datastore/storeytargets.py +17 -12
  47. mlrun/datastore/targets.py +1 -1
  48. mlrun/datastore/utils.py +25 -6
  49. mlrun/datastore/v3io.py +1 -1
  50. mlrun/db/base.py +63 -32
  51. mlrun/db/httpdb.py +373 -153
  52. mlrun/db/nopdb.py +54 -21
  53. mlrun/errors.py +4 -2
  54. mlrun/execution.py +66 -25
  55. mlrun/feature_store/api.py +1 -1
  56. mlrun/feature_store/common.py +1 -1
  57. mlrun/feature_store/feature_vector_utils.py +1 -1
  58. mlrun/feature_store/steps.py +8 -6
  59. mlrun/frameworks/_common/utils.py +3 -3
  60. mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
  61. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +2 -1
  62. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  63. mlrun/frameworks/_ml_common/utils.py +2 -1
  64. mlrun/frameworks/auto_mlrun/auto_mlrun.py +4 -3
  65. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +2 -1
  66. mlrun/frameworks/onnx/dataset.py +2 -1
  67. mlrun/frameworks/onnx/mlrun_interface.py +2 -1
  68. mlrun/frameworks/pytorch/callbacks/logging_callback.py +5 -4
  69. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +2 -1
  70. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +2 -1
  71. mlrun/frameworks/pytorch/utils.py +2 -1
  72. mlrun/frameworks/sklearn/metric.py +2 -1
  73. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +5 -4
  74. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +2 -1
  75. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +2 -1
  76. mlrun/hub/__init__.py +52 -0
  77. mlrun/hub/base.py +142 -0
  78. mlrun/hub/module.py +172 -0
  79. mlrun/hub/step.py +113 -0
  80. mlrun/k8s_utils.py +105 -16
  81. mlrun/launcher/base.py +15 -7
  82. mlrun/launcher/local.py +4 -1
  83. mlrun/model.py +14 -4
  84. mlrun/model_monitoring/__init__.py +0 -1
  85. mlrun/model_monitoring/api.py +65 -28
  86. mlrun/model_monitoring/applications/__init__.py +1 -1
  87. mlrun/model_monitoring/applications/base.py +299 -128
  88. mlrun/model_monitoring/applications/context.py +2 -4
  89. mlrun/model_monitoring/controller.py +132 -58
  90. mlrun/model_monitoring/db/_schedules.py +38 -29
  91. mlrun/model_monitoring/db/_stats.py +6 -16
  92. mlrun/model_monitoring/db/tsdb/__init__.py +9 -7
  93. mlrun/model_monitoring/db/tsdb/base.py +29 -9
  94. mlrun/model_monitoring/db/tsdb/preaggregate.py +234 -0
  95. mlrun/model_monitoring/db/tsdb/stream_graph_steps.py +63 -0
  96. mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_metrics_queries.py +414 -0
  97. mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_predictions_queries.py +376 -0
  98. mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_results_queries.py +590 -0
  99. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_connection.py +434 -0
  100. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_connector.py +541 -0
  101. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_operations.py +808 -0
  102. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_schema.py +502 -0
  103. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_stream.py +163 -0
  104. mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_stream_graph_steps.py +60 -0
  105. mlrun/model_monitoring/db/tsdb/timescaledb/utils/timescaledb_dataframe_processor.py +141 -0
  106. mlrun/model_monitoring/db/tsdb/timescaledb/utils/timescaledb_query_builder.py +585 -0
  107. mlrun/model_monitoring/db/tsdb/timescaledb/writer_graph_steps.py +73 -0
  108. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +20 -9
  109. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +235 -51
  110. mlrun/model_monitoring/features_drift_table.py +2 -1
  111. mlrun/model_monitoring/helpers.py +30 -6
  112. mlrun/model_monitoring/stream_processing.py +34 -28
  113. mlrun/model_monitoring/writer.py +224 -4
  114. mlrun/package/__init__.py +2 -1
  115. mlrun/platforms/__init__.py +0 -43
  116. mlrun/platforms/iguazio.py +8 -4
  117. mlrun/projects/operations.py +17 -11
  118. mlrun/projects/pipelines.py +2 -2
  119. mlrun/projects/project.py +187 -123
  120. mlrun/run.py +95 -21
  121. mlrun/runtimes/__init__.py +2 -186
  122. mlrun/runtimes/base.py +103 -25
  123. mlrun/runtimes/constants.py +225 -0
  124. mlrun/runtimes/daskjob.py +5 -2
  125. mlrun/runtimes/databricks_job/databricks_runtime.py +2 -1
  126. mlrun/runtimes/local.py +5 -2
  127. mlrun/runtimes/mounts.py +20 -2
  128. mlrun/runtimes/nuclio/__init__.py +12 -7
  129. mlrun/runtimes/nuclio/api_gateway.py +36 -6
  130. mlrun/runtimes/nuclio/application/application.py +339 -40
  131. mlrun/runtimes/nuclio/function.py +222 -72
  132. mlrun/runtimes/nuclio/serving.py +132 -42
  133. mlrun/runtimes/pod.py +213 -21
  134. mlrun/runtimes/utils.py +49 -9
  135. mlrun/secrets.py +99 -14
  136. mlrun/serving/__init__.py +2 -0
  137. mlrun/serving/remote.py +84 -11
  138. mlrun/serving/routers.py +26 -44
  139. mlrun/serving/server.py +138 -51
  140. mlrun/serving/serving_wrapper.py +6 -2
  141. mlrun/serving/states.py +997 -283
  142. mlrun/serving/steps.py +62 -0
  143. mlrun/serving/system_steps.py +149 -95
  144. mlrun/serving/v2_serving.py +9 -10
  145. mlrun/track/trackers/mlflow_tracker.py +29 -31
  146. mlrun/utils/helpers.py +292 -94
  147. mlrun/utils/http.py +9 -2
  148. mlrun/utils/notifications/notification/base.py +18 -0
  149. mlrun/utils/notifications/notification/git.py +3 -5
  150. mlrun/utils/notifications/notification/mail.py +39 -16
  151. mlrun/utils/notifications/notification/slack.py +2 -4
  152. mlrun/utils/notifications/notification/webhook.py +2 -5
  153. mlrun/utils/notifications/notification_pusher.py +3 -3
  154. mlrun/utils/version/version.json +2 -2
  155. mlrun/utils/version/version.py +3 -4
  156. {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/METADATA +63 -74
  157. {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/RECORD +161 -143
  158. mlrun/api/schemas/__init__.py +0 -259
  159. mlrun/db/auth_utils.py +0 -152
  160. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +0 -344
  161. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +0 -75
  162. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +0 -281
  163. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +0 -1266
  164. {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/WHEEL +0 -0
  165. {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/entry_points.txt +0 -0
  166. {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/licenses/LICENSE +0 -0
  167. {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/top_level.txt +0 -0
@@ -13,9 +13,9 @@
13
13
  # limitations under the License.
14
14
 
15
15
  from collections import defaultdict
16
- from collections.abc import Iterator
16
+ from collections.abc import Callable, Iterator
17
17
  from datetime import datetime
18
- from typing import Annotated, Any, Callable, Optional, Union
18
+ from typing import Annotated, Any, Optional, Union
19
19
 
20
20
  import pydantic.v1
21
21
 
@@ -27,6 +27,7 @@ class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
27
27
  basic = "basicAuth"
28
28
  none = "none"
29
29
  access_key = "accessKey"
30
+ iguazio = "iguazio"
30
31
 
31
32
  @classmethod
32
33
  def from_str(cls, authentication_mode: str):
@@ -36,6 +37,8 @@ class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
36
37
  return cls.basic
37
38
  elif authentication_mode == "accessKey":
38
39
  return cls.access_key
40
+ elif authentication_mode == "iguazio":
41
+ return cls.iguazio
39
42
  else:
40
43
  raise mlrun.errors.MLRunInvalidArgumentError(
41
44
  f"Authentication mode `{authentication_mode}` is not supported",
@@ -15,8 +15,6 @@
15
15
  import typing
16
16
 
17
17
  import pydantic.v1
18
- from nuclio.auth import AuthInfo as NuclioAuthInfo
19
- from nuclio.auth import AuthKinds as NuclioAuthKinds
20
18
 
21
19
  import mlrun.common.types
22
20
 
@@ -39,8 +37,14 @@ class AuthorizationAction(mlrun.common.types.StrEnum):
39
37
  store = "store"
40
38
 
41
39
 
40
+ class AuthorizationResourceNamespace(mlrun.common.types.StrEnum):
41
+ resources = "resources"
42
+ mgmt = "mgmt"
43
+
44
+
42
45
  class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
43
46
  project = "project"
47
+ project_global = "project-global"
44
48
  log = "log"
45
49
  runtime_resource = "runtime-resource"
46
50
  function = "function"
@@ -55,6 +59,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
55
59
  secret = "secret"
56
60
  run = "run"
57
61
  model_endpoint = "model-endpoint"
62
+ model_monitoring = "model-monitoring"
58
63
  pipeline = "pipeline"
59
64
  hub_source = "hub-source"
60
65
  workflow = "workflow"
@@ -74,6 +79,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
74
79
  return {
75
80
  # project is the resource itself, so no need for both resource_name and project_name
76
81
  AuthorizationResourceTypes.project: "/projects/{project_name}",
82
+ AuthorizationResourceTypes.project_global: "/projects",
77
83
  AuthorizationResourceTypes.project_summaries: "/projects/{project_name}/project-summaries/{resource_name}",
78
84
  AuthorizationResourceTypes.function: "/projects/{project_name}/functions/{resource_name}",
79
85
  AuthorizationResourceTypes.artifact: "/projects/{project_name}/artifacts/{resource_name}",
@@ -96,12 +102,11 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
96
102
  # runtime resource doesn't have an identifier, we don't need any auth granularity behind project level
97
103
  AuthorizationResourceTypes.runtime_resource: "/projects/{project_name}/runtime-resources",
98
104
  AuthorizationResourceTypes.model_endpoint: "/projects/{project_name}/model-endpoints/{resource_name}",
105
+ AuthorizationResourceTypes.model_monitoring: "/projects/{project_name}/model-monitoring/{resource_name}",
99
106
  AuthorizationResourceTypes.pipeline: "/projects/{project_name}/pipelines/{resource_name}",
100
107
  AuthorizationResourceTypes.datastore_profile: "/projects/{project_name}/datastore_profiles",
101
108
  # Hub sources are not project-scoped, and auth is globally on the sources endpoint.
102
- # TODO - this was reverted to /marketplace since MLRun needs to be able to run with old igz versions. Once
103
- # we only have support for igz versions that support /hub (>=3.5.4), change this to "/hub/sources".
104
- AuthorizationResourceTypes.hub_source: "/marketplace/sources",
109
+ AuthorizationResourceTypes.hub_source: "/hub/sources",
105
110
  # workflow define how to run a pipeline and can be considered as the specification of a pipeline.
106
111
  AuthorizationResourceTypes.workflow: "/projects/{project_name}/workflows/{resource_name}",
107
112
  AuthorizationResourceTypes.api_gateway: "/projects/{project_name}/api-gateways/{resource_name}",
@@ -132,15 +137,12 @@ class AuthInfo(pydantic.v1.BaseModel):
132
137
  projects_role: typing.Optional[ProjectsRole] = None
133
138
  planes: list[str] = []
134
139
 
135
- def to_nuclio_auth_info(self):
136
- if self.session != "":
137
- return NuclioAuthInfo(password=self.session, mode=NuclioAuthKinds.iguazio)
138
- return None
139
-
140
140
  def get_member_ids(self) -> list[str]:
141
141
  member_ids = []
142
142
  if self.user_id:
143
143
  member_ids.append(self.user_id)
144
+ if self.username:
145
+ member_ids.append(self.username)
144
146
  if self.user_group_ids:
145
147
  member_ids.extend(self.user_group_ids)
146
148
  return member_ids
@@ -67,3 +67,7 @@ class ClientSpec(pydantic.v1.BaseModel):
67
67
  alerts_mode: typing.Optional[str]
68
68
  system_id: typing.Optional[str]
69
69
  model_endpoint_monitoring_store_prefixes: typing.Optional[dict[str, str]]
70
+ authentication_mode: typing.Optional[str]
71
+ # Iguazio V4 OAuth token provider configuration
72
+ oauth_internal_token_endpoint: typing.Optional[str]
73
+ oauth_external_token_endpoint: typing.Optional[str]
@@ -95,6 +95,21 @@ headers_prefix = "x-mlrun-"
95
95
 
96
96
  class HeaderNames:
97
97
  projects_role = "x-projects-role"
98
+ remote_user = "x-remote-user"
99
+ forwarded_host = "x-forwarded-host"
100
+ data_session_override = "x-data-session-override"
101
+ user_id = "x-user-id"
102
+ user_group_ids = "x-user-group-ids"
103
+ unix_uid = "x-unix-uid"
104
+ v3io_session_key = "x-v3io-session-key"
105
+ v3io_access_key = "x-v3io-access-key"
106
+ v3io_user_id = "x-v3io-user-id"
107
+ v3io_session_planes = "x-v3io-session-planes"
108
+ authorization = "authorization"
109
+ igz_authenticator_kind = "x-igz-authenticator-kind"
110
+ cookies = "cookies"
111
+ cookie = "cookie"
112
+ x_request_id = "x-request-id"
98
113
  patch_mode = f"{headers_prefix}patch-mode"
99
114
  deletion_strategy = f"{headers_prefix}deletion-strategy"
100
115
  secret_store_token = f"{headers_prefix}secret-store-token"
@@ -106,6 +121,16 @@ class HeaderNames:
106
121
  ui_clear_cache = f"{headers_prefix}ui-clear-cache"
107
122
 
108
123
 
124
+ class AuthorizationHeaderPrefixes:
125
+ basic = "Basic "
126
+ bearer = "Bearer "
127
+
128
+
129
+ class CookieNames:
130
+ oauth2_proxy = "_oauth2_proxy"
131
+ iguazio = "session"
132
+
133
+
109
134
  class FeatureStorePartitionByField(mlrun.common.types.StrEnum):
110
135
  name = "name" # Supported for feature-store objects
111
136
 
@@ -31,13 +31,6 @@ class PreemptionNodesFeatureFlag(mlrun.common.types.StrEnum):
31
31
  disabled = "disabled"
32
32
 
33
33
 
34
- class AuthenticationFeatureFlag(mlrun.common.types.StrEnum):
35
- none = "none"
36
- basic = "basic"
37
- bearer = "bearer"
38
- iguazio = "iguazio"
39
-
40
-
41
34
  class NuclioStreamsFeatureFlag(mlrun.common.types.StrEnum):
42
35
  enabled = "enabled"
43
36
  disabled = "disabled"
@@ -45,7 +38,7 @@ class NuclioStreamsFeatureFlag(mlrun.common.types.StrEnum):
45
38
 
46
39
  class FeatureFlags(pydantic.v1.BaseModel):
47
40
  project_membership: ProjectMembershipFeatureFlag
48
- authentication: AuthenticationFeatureFlag
41
+ authentication: mlrun.common.types.AuthenticationMode
49
42
  nuclio_streams: NuclioStreamsFeatureFlag
50
43
  preemption_nodes: PreemptionNodesFeatureFlag
51
44
 
@@ -114,11 +114,21 @@ class StateThresholds(pydantic.v1.BaseModel):
114
114
  default: typing.Optional[dict[str, str]]
115
115
 
116
116
 
117
+ class Backoff(pydantic.v1.BaseModel):
118
+ default_base_delay: typing.Optional[str]
119
+ min_base_delay: typing.Optional[str]
120
+
121
+
122
+ class RetrySpec(pydantic.v1.BaseModel):
123
+ backoff: Backoff
124
+
125
+
117
126
  class FunctionSpec(pydantic.v1.BaseModel):
118
127
  image_pull_secret: typing.Optional[ImagePullSecret]
119
128
  security_context: typing.Optional[SecurityContext]
120
129
  service_account: typing.Optional[ServiceAccount]
121
130
  state_thresholds: typing.Optional[StateThresholds]
131
+ retry: typing.Optional[RetrySpec]
122
132
 
123
133
  class Config:
124
134
  extra = pydantic.v1.Extra.allow
@@ -130,3 +140,27 @@ class Function(pydantic.v1.BaseModel):
130
140
 
131
141
  class Config:
132
142
  extra = pydantic.v1.Extra.allow
143
+
144
+
145
+ class BatchingSpec(pydantic.v1.BaseModel):
146
+ # Set to True to enable batching
147
+ enabled: bool
148
+ # Maximal events to batch together. Default size is 10.
149
+ batch_size: typing.Optional[int]
150
+ # The maximum amount of time to wait before processing the batch. Default timeout is 1s.
151
+ # Once this time passes, the batch is processed even if it hasn’t reached the full batch size.
152
+ timeout: typing.Optional[str]
153
+
154
+ def get_nuclio_batch_config(self):
155
+ if not self.enabled:
156
+ return None
157
+
158
+ config = {"mode": "enable"}
159
+
160
+ if self.batch_size:
161
+ config["batchSize"] = self.batch_size
162
+
163
+ if self.timeout:
164
+ config["timeout"] = self.timeout
165
+
166
+ return config
@@ -12,9 +12,10 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- from datetime import datetime, timezone
15
+ from datetime import UTC, datetime
16
16
  from typing import Optional
17
17
 
18
+ import deepdiff
18
19
  from pydantic.v1 import BaseModel, Extra, Field
19
20
 
20
21
  import mlrun.common.types
@@ -36,9 +37,10 @@ class HubObjectMetadata(BaseModel):
36
37
  extra = Extra.allow
37
38
 
38
39
 
39
- # Currently only functions are supported. Will add more in the future.
40
40
  class HubSourceType(mlrun.common.types.StrEnum):
41
41
  functions = "functions"
42
+ modules = "modules"
43
+ steps = "steps"
42
44
 
43
45
 
44
46
  # Sources-related objects
@@ -46,7 +48,6 @@ class HubSourceSpec(ObjectSpec):
46
48
  path: str # URL to base directory, should include schema (s3://, etc...)
47
49
  channel: str
48
50
  credentials: Optional[dict] = {}
49
- object_type: HubSourceType = Field(HubSourceType.functions, const=True)
50
51
 
51
52
 
52
53
  class HubSource(BaseModel):
@@ -55,18 +56,18 @@ class HubSource(BaseModel):
55
56
  spec: HubSourceSpec
56
57
  status: Optional[ObjectStatus] = ObjectStatus(state="created")
57
58
 
58
- def get_full_uri(self, relative_path):
59
- return f"{self.spec.path}/{self.spec.object_type}/{self.spec.channel}/{relative_path}"
59
+ def get_full_uri(self, relative_path, object_type):
60
+ return f"{self.spec.path}/{object_type}/{self.spec.channel}/{relative_path}"
60
61
 
61
- def get_catalog_uri(self):
62
- return self.get_full_uri(mlrun.mlconf.hub.catalog_filename)
62
+ def get_catalog_uri(self, object_type):
63
+ return self.get_full_uri(mlrun.mlconf.hub.catalog_filename, object_type)
63
64
 
64
65
  @classmethod
65
66
  def generate_default_source(cls):
66
67
  if not mlrun.mlconf.hub.default_source.create:
67
68
  return None
68
69
 
69
- now = datetime.now(timezone.utc)
70
+ now = datetime.now(UTC)
70
71
  hub_metadata = HubObjectMetadata(
71
72
  name=mlrun.mlconf.hub.default_source.name,
72
73
  description=mlrun.mlconf.hub.default_source.description,
@@ -78,11 +79,23 @@ class HubSource(BaseModel):
78
79
  spec=HubSourceSpec(
79
80
  path=mlrun.mlconf.hub.default_source.url,
80
81
  channel=mlrun.mlconf.hub.default_source.channel,
81
- object_type=HubSourceType(mlrun.mlconf.hub.default_source.object_type),
82
82
  ),
83
83
  status=ObjectStatus(state="created"),
84
84
  )
85
85
 
86
+ def diff(self, another_source: "HubSource") -> dict:
87
+ """
88
+ Compare this HubSource with another one.
89
+ Returns a dict of differences (metadata, spec, status).
90
+ """
91
+ exclude_paths = [
92
+ "root['metadata']['updated']",
93
+ "root['metadata']['created']",
94
+ ]
95
+ return deepdiff.DeepDiff(
96
+ self.dict(), another_source.dict(), exclude_paths=exclude_paths
97
+ )
98
+
86
99
 
87
100
  last_source_index = -1
88
101
 
@@ -94,21 +107,16 @@ class IndexedHubSource(BaseModel):
94
107
 
95
108
  # Item-related objects
96
109
  class HubItemMetadata(HubObjectMetadata):
97
- source: HubSourceType = Field(HubSourceType.functions, const=True)
110
+ source: HubSourceType = HubSourceType.functions
98
111
  version: str
99
112
  tag: Optional[str]
100
113
 
101
114
  def get_relative_path(self) -> str:
102
- if self.source == HubSourceType.functions:
103
- # This is needed since the hub deployment script modifies the paths to use _ instead of -.
104
- modified_name = self.name.replace("-", "_")
105
- # Prefer using the tag if exists. Otherwise, use version.
106
- version = self.tag or self.version
107
- return f"{modified_name}/{version}/"
108
- else:
109
- raise mlrun.errors.MLRunInvalidArgumentError(
110
- f"Bad source for hub item - {self.source}"
111
- )
115
+ # This is needed since the hub deployment script modifies the paths to use _ instead of -.
116
+ modified_name = self.name.replace("-", "_")
117
+ # Prefer using the tag if exists. Otherwise, use version.
118
+ version = self.tag or self.version
119
+ return f"{modified_name}/{version}/"
112
120
 
113
121
 
114
122
  class HubItemSpec(ObjectSpec):
@@ -127,3 +135,8 @@ class HubCatalog(BaseModel):
127
135
  kind: ObjectKind = Field(ObjectKind.hub_catalog, const=True)
128
136
  channel: str
129
137
  catalog: list[HubItem]
138
+
139
+
140
+ class HubModuleType(mlrun.common.types.StrEnum):
141
+ generic = "generic"
142
+ monitoring_app = "monitoring_application"
@@ -30,6 +30,7 @@ from .constants import (
30
30
  ModelEndpointMonitoringMetricType,
31
31
  ModelEndpointSchema,
32
32
  ModelMonitoringAppLabel,
33
+ ModelMonitoringInfraLabel,
33
34
  ModelMonitoringMode,
34
35
  MonitoringFunctionNames,
35
36
  PredictionsQueryConstants,
@@ -39,7 +40,7 @@ from .constants import (
39
40
  ResultStatusApp,
40
41
  SpecialApps,
41
42
  StreamProcessingEvent,
42
- TDEngineSuperTables,
43
+ TimescaleDBTables,
43
44
  TSDBTarget,
44
45
  V3IOTSDBTables,
45
46
  VersionedModel,
@@ -34,6 +34,7 @@ class ModelEndpointSchema(MonitoringStrEnum):
34
34
  UID = "uid"
35
35
  PROJECT = "project"
36
36
  ENDPOINT_TYPE = "endpoint_type"
37
+ MODE = "mode"
37
38
  NAME = "name"
38
39
  CREATED = "created"
39
40
  UPDATED = "updated"
@@ -195,6 +196,10 @@ class WriterEventKind(MonitoringStrEnum):
195
196
  RESULT = "result"
196
197
  STATS = "stats"
197
198
 
199
+ @classmethod
200
+ def user_app_outputs(cls):
201
+ return [cls.METRIC, cls.RESULT]
202
+
198
203
 
199
204
  class ControllerEvent(MonitoringStrEnum):
200
205
  KIND = "kind"
@@ -269,7 +274,7 @@ class EventKeyMetrics:
269
274
 
270
275
  class TSDBTarget(MonitoringStrEnum):
271
276
  V3IO_TSDB = "v3io-tsdb"
272
- TDEngine = "tdengine"
277
+ TimescaleDB = "postgresql"
273
278
 
274
279
 
275
280
  class ProjectSecretKeys:
@@ -303,6 +308,7 @@ class FileTargetKind:
303
308
  MONITORING_APPLICATION = "monitoring_application"
304
309
  ERRORS = "errors"
305
310
  STATS = "stats"
311
+ PARQUET_STATS = "parquet_stats"
306
312
  LAST_REQUEST = "last_request"
307
313
 
308
314
 
@@ -326,18 +332,11 @@ class EndpointType(IntEnum):
326
332
  def top_level_list(cls):
327
333
  return [cls.NODE_EP, cls.ROUTER, cls.BATCH_EP]
328
334
 
329
- @classmethod
330
- def real_time_list(cls):
331
- return [cls.NODE_EP, cls.ROUTER, cls.LEAF_EP]
332
-
333
- @classmethod
334
- def batch_list(cls):
335
- return [cls.BATCH_EP]
336
-
337
335
 
338
- class EndpointMode(StrEnum):
339
- REAL_TIME = "real_time"
340
- BATCH = "batch"
336
+ class EndpointMode(IntEnum):
337
+ REAL_TIME = 0
338
+ BATCH = 1
339
+ BATCH_LEGACY = 2 # legacy batch mode, used for endpoints created through the batch inference job
341
340
 
342
341
 
343
342
  class MonitoringFunctionNames(MonitoringStrEnum):
@@ -354,7 +353,7 @@ class V3IOTSDBTables(MonitoringStrEnum):
354
353
  PREDICTIONS = "predictions"
355
354
 
356
355
 
357
- class TDEngineSuperTables(MonitoringStrEnum):
356
+ class TimescaleDBTables(MonitoringStrEnum):
358
357
  APP_RESULTS = "app_results"
359
358
  METRICS = "metrics"
360
359
  PREDICTIONS = "predictions"
@@ -487,8 +486,6 @@ class ModelMonitoringLabels:
487
486
 
488
487
  _RESERVED_FUNCTION_NAMES = MonitoringFunctionNames.list() + [SpecialApps.MLRUN_INFRA]
489
488
 
490
- _RESERVED_EVALUATE_FUNCTION_SUFFIX = "-batch"
491
-
492
489
 
493
490
  class ModelEndpointMonitoringMetricType(StrEnum):
494
491
  RESULT = "result"
@@ -54,12 +54,21 @@ class FunctionSummary(BaseModel):
54
54
 
55
55
  return cls(
56
56
  type=func_type,
57
- name=func_dict["metadata"]["name"],
57
+ name=func_dict["metadata"]["name"]
58
+ if func_type != FunctionsType.APPLICATION
59
+ else func_dict["spec"]
60
+ .get("graph", {})
61
+ .get("steps", {})
62
+ .get("PrepareMonitoringEvent", {})
63
+ .get("class_args", {})
64
+ .get("application_name"),
58
65
  application_class=""
59
66
  if func_type != FunctionsType.APPLICATION
60
- else func_dict["spec"]["graph"]["steps"]["PushToMonitoringWriter"]["after"][
61
- 0
62
- ],
67
+ else func_dict["spec"]
68
+ .get("graph", {})
69
+ .get("steps", {})
70
+ .get("PushToMonitoringWriter", {})
71
+ .get("after", [None])[0],
63
72
  project_name=func_dict["metadata"]["project"],
64
73
  updated_time=func_dict["metadata"].get("updated"),
65
74
  status=func_dict["status"].get("state"),
@@ -28,6 +28,7 @@ from .constants import (
28
28
  FQN_REGEX,
29
29
  MODEL_ENDPOINT_ID_PATTERN,
30
30
  PROJECT_PATTERN,
31
+ EndpointMode,
31
32
  EndpointType,
32
33
  ModelEndpointMonitoringMetricType,
33
34
  ModelMonitoringMode,
@@ -118,6 +119,7 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
118
119
  project: constr(regex=PROJECT_PATTERN)
119
120
  endpoint_type: EndpointType = EndpointType.NODE_EP
120
121
  uid: Optional[constr(regex=MODEL_ENDPOINT_ID_PATTERN)]
122
+ mode: Optional[EndpointMode] = None
121
123
 
122
124
  @classmethod
123
125
  def mutable_fields(cls):
@@ -129,6 +131,15 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
129
131
  return str(v)
130
132
  return v
131
133
 
134
+ @validator("mode", pre=True, always=True)
135
+ def _set_mode_based_on_endpoint_type(cls, v, values): # noqa: N805
136
+ if v is None:
137
+ if values.get("endpoint_type") == EndpointType.BATCH_EP:
138
+ return EndpointMode.BATCH_LEGACY
139
+ else:
140
+ return EndpointMode.REAL_TIME
141
+ return v
142
+
132
143
 
133
144
  class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
134
145
  model_class: Optional[str] = ""
@@ -18,7 +18,7 @@ import pydantic.v1
18
18
 
19
19
 
20
20
  class PipelinesPagination(str):
21
- default_page_size = 20
21
+ default_page_size = 200
22
22
  # https://github.com/kubeflow/pipelines/blob/master/backend/src/apiserver/list/list.go#L363
23
23
  max_page_size = 200
24
24
 
@@ -49,5 +49,20 @@ class SecretKeysData(BaseModel):
49
49
  secret_keys: Optional[list] = []
50
50
 
51
51
 
52
- class UserSecretCreationRequest(SecretsData):
53
- user: str
52
+ class SecretToken(BaseModel):
53
+ name: str
54
+ token: str
55
+
56
+
57
+ class StoreSecretTokensResponse(BaseModel):
58
+ created_tokens: list[str] = []
59
+ updated_tokens: list[str] = []
60
+
61
+
62
+ class SecretTokenInfo(BaseModel):
63
+ name: str
64
+ expiration: int
65
+
66
+
67
+ class ListSecretTokensResponse(BaseModel):
68
+ secret_tokens: list[SecretTokenInfo]
mlrun/common/secrets.py CHANGED
@@ -11,10 +11,32 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
-
14
+ import re
15
+ import typing
15
16
  from abc import ABC, abstractmethod
16
17
 
17
18
  import mlrun.common.schemas
19
+ from mlrun.config import config as mlconf
20
+
21
+ _AUTH_SECRET_NAME_TEMPLATE = re.escape(
22
+ mlconf.secret_stores.kubernetes.auth_secret_name.format(
23
+ hashed_access_key="",
24
+ )
25
+ )
26
+ AUTH_SECRET_PATTERN = re.compile(f"^{_AUTH_SECRET_NAME_TEMPLATE}.*")
27
+
28
+
29
+ def validate_not_forbidden_secret(secret_name: str) -> None:
30
+ """
31
+ Forbid client-supplied references to internal MLRun auth/project secrets.
32
+ No-op when running inside the API server (API enrichments are allowed).
33
+ """
34
+ if not secret_name or mlrun.config.is_running_as_api():
35
+ return
36
+ if AUTH_SECRET_PATTERN.match(secret_name):
37
+ raise mlrun.errors.MLRunInvalidArgumentError(
38
+ f"Forbidden secret '{secret_name}' matches MLRun auth-secret pattern."
39
+ )
18
40
 
19
41
 
20
42
  class SecretProviderInterface(ABC):
@@ -54,6 +76,44 @@ class SecretProviderInterface(ABC):
54
76
  def get_secret_data(self, secret_name, namespace=""):
55
77
  pass
56
78
 
79
+ @abstractmethod
80
+ def store_user_token_secret(
81
+ self,
82
+ username: str,
83
+ token_name: str,
84
+ token: str,
85
+ expiration: int,
86
+ force: bool = False,
87
+ namespace: typing.Optional[str] = None,
88
+ ) -> typing.Optional[mlrun.common.schemas.SecretEventActions]:
89
+ pass
90
+
91
+ @abstractmethod
92
+ def get_user_token_secret_value(
93
+ self,
94
+ username: str,
95
+ token_name: str,
96
+ namespace: typing.Optional[str] = None,
97
+ ) -> str:
98
+ pass
99
+
100
+ @abstractmethod
101
+ def list_user_token_secrets(
102
+ self,
103
+ username: str,
104
+ namespace: typing.Optional[str] = None,
105
+ ) -> list[mlrun.common.schemas.SecretTokenInfo]:
106
+ pass
107
+
108
+ @abstractmethod
109
+ def delete_user_token_secret(
110
+ self,
111
+ username: str,
112
+ token_name: str,
113
+ namespace: typing.Optional[str] = None,
114
+ ) -> None:
115
+ pass
116
+
57
117
 
58
118
  class InMemorySecretProvider(SecretProviderInterface):
59
119
  def __init__(self):
@@ -130,6 +190,40 @@ class InMemorySecretProvider(SecretProviderInterface):
130
190
  def get_secret_data(self, secret_name, namespace=""):
131
191
  return self.secrets_map[secret_name]
132
192
 
193
+ def store_user_token_secret(
194
+ self,
195
+ username: str,
196
+ token_name: str,
197
+ token: str,
198
+ expiration: int,
199
+ force: bool = False,
200
+ namespace: typing.Optional[str] = None,
201
+ ) -> typing.Optional[mlrun.common.schemas.SecretEventActions]:
202
+ raise NotImplementedError()
203
+
204
+ def get_user_token_secret_value(
205
+ self,
206
+ username: str,
207
+ token_name: str,
208
+ namespace: typing.Optional[str] = None,
209
+ ) -> str:
210
+ raise NotImplementedError()
211
+
212
+ def list_user_token_secrets(
213
+ self,
214
+ username: str,
215
+ namespace: typing.Optional[str] = None,
216
+ ) -> list[mlrun.common.schemas.SecretTokenInfo]:
217
+ raise NotImplementedError()
218
+
219
+ def delete_user_token_secret(
220
+ self,
221
+ username: str,
222
+ token_name: str,
223
+ namespace: typing.Optional[str] = None,
224
+ ) -> None:
225
+ raise NotImplementedError()
226
+
133
227
  @staticmethod
134
228
  def _generate_auth_secret_data(username: str, access_key: str):
135
229
  return {
mlrun/common/types.py CHANGED
@@ -14,18 +14,10 @@
14
14
 
15
15
  import enum
16
16
 
17
+ # Alias to Python's built-in StrEnum (Python 3.11+)
18
+ StrEnum = enum.StrEnum
17
19
 
18
- # TODO: From python 3.11 StrEnum is built-in and this will not be needed
19
- class StrEnum(str, enum.Enum):
20
- def __str__(self):
21
- return self.value
22
20
 
23
- def __repr__(self):
24
- return self.value
25
-
26
-
27
- # Partial backport from Python 3.11
28
- # https://docs.python.org/3/library/http.html#http.HTTPMethod
29
21
  class HTTPMethod(StrEnum):
30
22
  GET = "GET"
31
23
  POST = "POST"
@@ -37,3 +29,11 @@ class HTTPMethod(StrEnum):
37
29
  class Operation(StrEnum):
38
30
  ADD = "add"
39
31
  REMOVE = "remove"
32
+
33
+
34
+ class AuthenticationMode(StrEnum):
35
+ NONE = "none"
36
+ BASIC = "basic"
37
+ BEARER = "bearer"
38
+ IGUAZIO = "iguazio"
39
+ IGUAZIO_V4 = "iguazio-v4"