mlrun 1.10.0rc18__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +24 -3
- mlrun/__main__.py +0 -4
- mlrun/artifacts/dataset.py +2 -2
- mlrun/artifacts/document.py +6 -1
- mlrun/artifacts/llm_prompt.py +21 -15
- mlrun/artifacts/model.py +3 -3
- mlrun/artifacts/plots.py +1 -1
- mlrun/{model_monitoring/db/tsdb/tdengine → auth}/__init__.py +2 -3
- mlrun/auth/nuclio.py +89 -0
- mlrun/auth/providers.py +429 -0
- mlrun/auth/utils.py +415 -0
- mlrun/common/constants.py +14 -0
- mlrun/common/model_monitoring/helpers.py +123 -0
- mlrun/common/runtimes/constants.py +28 -0
- mlrun/common/schemas/__init__.py +14 -3
- mlrun/common/schemas/alert.py +2 -2
- mlrun/common/schemas/api_gateway.py +3 -0
- mlrun/common/schemas/auth.py +12 -10
- mlrun/common/schemas/client_spec.py +4 -0
- mlrun/common/schemas/constants.py +25 -0
- mlrun/common/schemas/frontend_spec.py +1 -8
- mlrun/common/schemas/function.py +34 -0
- mlrun/common/schemas/hub.py +33 -20
- mlrun/common/schemas/model_monitoring/__init__.py +2 -1
- mlrun/common/schemas/model_monitoring/constants.py +12 -15
- mlrun/common/schemas/model_monitoring/functions.py +13 -4
- mlrun/common/schemas/model_monitoring/model_endpoints.py +11 -0
- mlrun/common/schemas/pipeline.py +1 -1
- mlrun/common/schemas/secret.py +17 -2
- mlrun/common/secrets.py +95 -1
- mlrun/common/types.py +10 -10
- mlrun/config.py +69 -19
- mlrun/data_types/infer.py +2 -2
- mlrun/datastore/__init__.py +12 -5
- mlrun/datastore/azure_blob.py +162 -47
- mlrun/datastore/base.py +274 -10
- mlrun/datastore/datastore.py +7 -2
- mlrun/datastore/datastore_profile.py +84 -22
- mlrun/datastore/model_provider/huggingface_provider.py +225 -41
- mlrun/datastore/model_provider/mock_model_provider.py +87 -0
- mlrun/datastore/model_provider/model_provider.py +206 -74
- mlrun/datastore/model_provider/openai_provider.py +226 -66
- mlrun/datastore/s3.py +39 -18
- mlrun/datastore/sources.py +1 -1
- mlrun/datastore/store_resources.py +4 -4
- mlrun/datastore/storeytargets.py +17 -12
- mlrun/datastore/targets.py +1 -1
- mlrun/datastore/utils.py +25 -6
- mlrun/datastore/v3io.py +1 -1
- mlrun/db/base.py +63 -32
- mlrun/db/httpdb.py +373 -153
- mlrun/db/nopdb.py +54 -21
- mlrun/errors.py +4 -2
- mlrun/execution.py +66 -25
- mlrun/feature_store/api.py +1 -1
- mlrun/feature_store/common.py +1 -1
- mlrun/feature_store/feature_vector_utils.py +1 -1
- mlrun/feature_store/steps.py +8 -6
- mlrun/frameworks/_common/utils.py +3 -3
- mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +2 -1
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_ml_common/utils.py +2 -1
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +4 -3
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +2 -1
- mlrun/frameworks/onnx/dataset.py +2 -1
- mlrun/frameworks/onnx/mlrun_interface.py +2 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +5 -4
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +2 -1
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +2 -1
- mlrun/frameworks/pytorch/utils.py +2 -1
- mlrun/frameworks/sklearn/metric.py +2 -1
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +5 -4
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +2 -1
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +2 -1
- mlrun/hub/__init__.py +52 -0
- mlrun/hub/base.py +142 -0
- mlrun/hub/module.py +172 -0
- mlrun/hub/step.py +113 -0
- mlrun/k8s_utils.py +105 -16
- mlrun/launcher/base.py +15 -7
- mlrun/launcher/local.py +4 -1
- mlrun/model.py +14 -4
- mlrun/model_monitoring/__init__.py +0 -1
- mlrun/model_monitoring/api.py +65 -28
- mlrun/model_monitoring/applications/__init__.py +1 -1
- mlrun/model_monitoring/applications/base.py +299 -128
- mlrun/model_monitoring/applications/context.py +2 -4
- mlrun/model_monitoring/controller.py +132 -58
- mlrun/model_monitoring/db/_schedules.py +38 -29
- mlrun/model_monitoring/db/_stats.py +6 -16
- mlrun/model_monitoring/db/tsdb/__init__.py +9 -7
- mlrun/model_monitoring/db/tsdb/base.py +29 -9
- mlrun/model_monitoring/db/tsdb/preaggregate.py +234 -0
- mlrun/model_monitoring/db/tsdb/stream_graph_steps.py +63 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_metrics_queries.py +414 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_predictions_queries.py +376 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_results_queries.py +590 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_connection.py +434 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_connector.py +541 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_operations.py +808 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_schema.py +502 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_stream.py +163 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_stream_graph_steps.py +60 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/utils/timescaledb_dataframe_processor.py +141 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/utils/timescaledb_query_builder.py +585 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/writer_graph_steps.py +73 -0
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +20 -9
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +235 -51
- mlrun/model_monitoring/features_drift_table.py +2 -1
- mlrun/model_monitoring/helpers.py +30 -6
- mlrun/model_monitoring/stream_processing.py +34 -28
- mlrun/model_monitoring/writer.py +224 -4
- mlrun/package/__init__.py +2 -1
- mlrun/platforms/__init__.py +0 -43
- mlrun/platforms/iguazio.py +8 -4
- mlrun/projects/operations.py +17 -11
- mlrun/projects/pipelines.py +2 -2
- mlrun/projects/project.py +187 -123
- mlrun/run.py +95 -21
- mlrun/runtimes/__init__.py +2 -186
- mlrun/runtimes/base.py +103 -25
- mlrun/runtimes/constants.py +225 -0
- mlrun/runtimes/daskjob.py +5 -2
- mlrun/runtimes/databricks_job/databricks_runtime.py +2 -1
- mlrun/runtimes/local.py +5 -2
- mlrun/runtimes/mounts.py +20 -2
- mlrun/runtimes/nuclio/__init__.py +12 -7
- mlrun/runtimes/nuclio/api_gateway.py +36 -6
- mlrun/runtimes/nuclio/application/application.py +339 -40
- mlrun/runtimes/nuclio/function.py +222 -72
- mlrun/runtimes/nuclio/serving.py +132 -42
- mlrun/runtimes/pod.py +213 -21
- mlrun/runtimes/utils.py +49 -9
- mlrun/secrets.py +99 -14
- mlrun/serving/__init__.py +2 -0
- mlrun/serving/remote.py +84 -11
- mlrun/serving/routers.py +26 -44
- mlrun/serving/server.py +138 -51
- mlrun/serving/serving_wrapper.py +6 -2
- mlrun/serving/states.py +997 -283
- mlrun/serving/steps.py +62 -0
- mlrun/serving/system_steps.py +149 -95
- mlrun/serving/v2_serving.py +9 -10
- mlrun/track/trackers/mlflow_tracker.py +29 -31
- mlrun/utils/helpers.py +292 -94
- mlrun/utils/http.py +9 -2
- mlrun/utils/notifications/notification/base.py +18 -0
- mlrun/utils/notifications/notification/git.py +3 -5
- mlrun/utils/notifications/notification/mail.py +39 -16
- mlrun/utils/notifications/notification/slack.py +2 -4
- mlrun/utils/notifications/notification/webhook.py +2 -5
- mlrun/utils/notifications/notification_pusher.py +3 -3
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +3 -4
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/METADATA +63 -74
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/RECORD +161 -143
- mlrun/api/schemas/__init__.py +0 -259
- mlrun/db/auth_utils.py +0 -152
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +0 -344
- mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +0 -75
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +0 -281
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +0 -1266
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/WHEEL +0 -0
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/entry_points.txt +0 -0
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/licenses/LICENSE +0 -0
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/top_level.txt +0 -0
mlrun/common/schemas/alert.py
CHANGED
|
@@ -13,9 +13,9 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
from collections import defaultdict
|
|
16
|
-
from collections.abc import Iterator
|
|
16
|
+
from collections.abc import Callable, Iterator
|
|
17
17
|
from datetime import datetime
|
|
18
|
-
from typing import Annotated, Any,
|
|
18
|
+
from typing import Annotated, Any, Optional, Union
|
|
19
19
|
|
|
20
20
|
import pydantic.v1
|
|
21
21
|
|
|
@@ -27,6 +27,7 @@ class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
|
|
|
27
27
|
basic = "basicAuth"
|
|
28
28
|
none = "none"
|
|
29
29
|
access_key = "accessKey"
|
|
30
|
+
iguazio = "iguazio"
|
|
30
31
|
|
|
31
32
|
@classmethod
|
|
32
33
|
def from_str(cls, authentication_mode: str):
|
|
@@ -36,6 +37,8 @@ class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
|
|
|
36
37
|
return cls.basic
|
|
37
38
|
elif authentication_mode == "accessKey":
|
|
38
39
|
return cls.access_key
|
|
40
|
+
elif authentication_mode == "iguazio":
|
|
41
|
+
return cls.iguazio
|
|
39
42
|
else:
|
|
40
43
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
41
44
|
f"Authentication mode `{authentication_mode}` is not supported",
|
mlrun/common/schemas/auth.py
CHANGED
|
@@ -15,8 +15,6 @@
|
|
|
15
15
|
import typing
|
|
16
16
|
|
|
17
17
|
import pydantic.v1
|
|
18
|
-
from nuclio.auth import AuthInfo as NuclioAuthInfo
|
|
19
|
-
from nuclio.auth import AuthKinds as NuclioAuthKinds
|
|
20
18
|
|
|
21
19
|
import mlrun.common.types
|
|
22
20
|
|
|
@@ -39,8 +37,14 @@ class AuthorizationAction(mlrun.common.types.StrEnum):
|
|
|
39
37
|
store = "store"
|
|
40
38
|
|
|
41
39
|
|
|
40
|
+
class AuthorizationResourceNamespace(mlrun.common.types.StrEnum):
|
|
41
|
+
resources = "resources"
|
|
42
|
+
mgmt = "mgmt"
|
|
43
|
+
|
|
44
|
+
|
|
42
45
|
class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
|
|
43
46
|
project = "project"
|
|
47
|
+
project_global = "project-global"
|
|
44
48
|
log = "log"
|
|
45
49
|
runtime_resource = "runtime-resource"
|
|
46
50
|
function = "function"
|
|
@@ -55,6 +59,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
|
|
|
55
59
|
secret = "secret"
|
|
56
60
|
run = "run"
|
|
57
61
|
model_endpoint = "model-endpoint"
|
|
62
|
+
model_monitoring = "model-monitoring"
|
|
58
63
|
pipeline = "pipeline"
|
|
59
64
|
hub_source = "hub-source"
|
|
60
65
|
workflow = "workflow"
|
|
@@ -74,6 +79,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
|
|
|
74
79
|
return {
|
|
75
80
|
# project is the resource itself, so no need for both resource_name and project_name
|
|
76
81
|
AuthorizationResourceTypes.project: "/projects/{project_name}",
|
|
82
|
+
AuthorizationResourceTypes.project_global: "/projects",
|
|
77
83
|
AuthorizationResourceTypes.project_summaries: "/projects/{project_name}/project-summaries/{resource_name}",
|
|
78
84
|
AuthorizationResourceTypes.function: "/projects/{project_name}/functions/{resource_name}",
|
|
79
85
|
AuthorizationResourceTypes.artifact: "/projects/{project_name}/artifacts/{resource_name}",
|
|
@@ -96,12 +102,11 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
|
|
|
96
102
|
# runtime resource doesn't have an identifier, we don't need any auth granularity behind project level
|
|
97
103
|
AuthorizationResourceTypes.runtime_resource: "/projects/{project_name}/runtime-resources",
|
|
98
104
|
AuthorizationResourceTypes.model_endpoint: "/projects/{project_name}/model-endpoints/{resource_name}",
|
|
105
|
+
AuthorizationResourceTypes.model_monitoring: "/projects/{project_name}/model-monitoring/{resource_name}",
|
|
99
106
|
AuthorizationResourceTypes.pipeline: "/projects/{project_name}/pipelines/{resource_name}",
|
|
100
107
|
AuthorizationResourceTypes.datastore_profile: "/projects/{project_name}/datastore_profiles",
|
|
101
108
|
# Hub sources are not project-scoped, and auth is globally on the sources endpoint.
|
|
102
|
-
|
|
103
|
-
# we only have support for igz versions that support /hub (>=3.5.4), change this to "/hub/sources".
|
|
104
|
-
AuthorizationResourceTypes.hub_source: "/marketplace/sources",
|
|
109
|
+
AuthorizationResourceTypes.hub_source: "/hub/sources",
|
|
105
110
|
# workflow define how to run a pipeline and can be considered as the specification of a pipeline.
|
|
106
111
|
AuthorizationResourceTypes.workflow: "/projects/{project_name}/workflows/{resource_name}",
|
|
107
112
|
AuthorizationResourceTypes.api_gateway: "/projects/{project_name}/api-gateways/{resource_name}",
|
|
@@ -132,15 +137,12 @@ class AuthInfo(pydantic.v1.BaseModel):
|
|
|
132
137
|
projects_role: typing.Optional[ProjectsRole] = None
|
|
133
138
|
planes: list[str] = []
|
|
134
139
|
|
|
135
|
-
def to_nuclio_auth_info(self):
|
|
136
|
-
if self.session != "":
|
|
137
|
-
return NuclioAuthInfo(password=self.session, mode=NuclioAuthKinds.iguazio)
|
|
138
|
-
return None
|
|
139
|
-
|
|
140
140
|
def get_member_ids(self) -> list[str]:
|
|
141
141
|
member_ids = []
|
|
142
142
|
if self.user_id:
|
|
143
143
|
member_ids.append(self.user_id)
|
|
144
|
+
if self.username:
|
|
145
|
+
member_ids.append(self.username)
|
|
144
146
|
if self.user_group_ids:
|
|
145
147
|
member_ids.extend(self.user_group_ids)
|
|
146
148
|
return member_ids
|
|
@@ -67,3 +67,7 @@ class ClientSpec(pydantic.v1.BaseModel):
|
|
|
67
67
|
alerts_mode: typing.Optional[str]
|
|
68
68
|
system_id: typing.Optional[str]
|
|
69
69
|
model_endpoint_monitoring_store_prefixes: typing.Optional[dict[str, str]]
|
|
70
|
+
authentication_mode: typing.Optional[str]
|
|
71
|
+
# Iguazio V4 OAuth token provider configuration
|
|
72
|
+
oauth_internal_token_endpoint: typing.Optional[str]
|
|
73
|
+
oauth_external_token_endpoint: typing.Optional[str]
|
|
@@ -95,6 +95,21 @@ headers_prefix = "x-mlrun-"
|
|
|
95
95
|
|
|
96
96
|
class HeaderNames:
|
|
97
97
|
projects_role = "x-projects-role"
|
|
98
|
+
remote_user = "x-remote-user"
|
|
99
|
+
forwarded_host = "x-forwarded-host"
|
|
100
|
+
data_session_override = "x-data-session-override"
|
|
101
|
+
user_id = "x-user-id"
|
|
102
|
+
user_group_ids = "x-user-group-ids"
|
|
103
|
+
unix_uid = "x-unix-uid"
|
|
104
|
+
v3io_session_key = "x-v3io-session-key"
|
|
105
|
+
v3io_access_key = "x-v3io-access-key"
|
|
106
|
+
v3io_user_id = "x-v3io-user-id"
|
|
107
|
+
v3io_session_planes = "x-v3io-session-planes"
|
|
108
|
+
authorization = "authorization"
|
|
109
|
+
igz_authenticator_kind = "x-igz-authenticator-kind"
|
|
110
|
+
cookies = "cookies"
|
|
111
|
+
cookie = "cookie"
|
|
112
|
+
x_request_id = "x-request-id"
|
|
98
113
|
patch_mode = f"{headers_prefix}patch-mode"
|
|
99
114
|
deletion_strategy = f"{headers_prefix}deletion-strategy"
|
|
100
115
|
secret_store_token = f"{headers_prefix}secret-store-token"
|
|
@@ -106,6 +121,16 @@ class HeaderNames:
|
|
|
106
121
|
ui_clear_cache = f"{headers_prefix}ui-clear-cache"
|
|
107
122
|
|
|
108
123
|
|
|
124
|
+
class AuthorizationHeaderPrefixes:
|
|
125
|
+
basic = "Basic "
|
|
126
|
+
bearer = "Bearer "
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class CookieNames:
|
|
130
|
+
oauth2_proxy = "_oauth2_proxy"
|
|
131
|
+
iguazio = "session"
|
|
132
|
+
|
|
133
|
+
|
|
109
134
|
class FeatureStorePartitionByField(mlrun.common.types.StrEnum):
|
|
110
135
|
name = "name" # Supported for feature-store objects
|
|
111
136
|
|
|
@@ -31,13 +31,6 @@ class PreemptionNodesFeatureFlag(mlrun.common.types.StrEnum):
|
|
|
31
31
|
disabled = "disabled"
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
class AuthenticationFeatureFlag(mlrun.common.types.StrEnum):
|
|
35
|
-
none = "none"
|
|
36
|
-
basic = "basic"
|
|
37
|
-
bearer = "bearer"
|
|
38
|
-
iguazio = "iguazio"
|
|
39
|
-
|
|
40
|
-
|
|
41
34
|
class NuclioStreamsFeatureFlag(mlrun.common.types.StrEnum):
|
|
42
35
|
enabled = "enabled"
|
|
43
36
|
disabled = "disabled"
|
|
@@ -45,7 +38,7 @@ class NuclioStreamsFeatureFlag(mlrun.common.types.StrEnum):
|
|
|
45
38
|
|
|
46
39
|
class FeatureFlags(pydantic.v1.BaseModel):
|
|
47
40
|
project_membership: ProjectMembershipFeatureFlag
|
|
48
|
-
authentication:
|
|
41
|
+
authentication: mlrun.common.types.AuthenticationMode
|
|
49
42
|
nuclio_streams: NuclioStreamsFeatureFlag
|
|
50
43
|
preemption_nodes: PreemptionNodesFeatureFlag
|
|
51
44
|
|
mlrun/common/schemas/function.py
CHANGED
|
@@ -114,11 +114,21 @@ class StateThresholds(pydantic.v1.BaseModel):
|
|
|
114
114
|
default: typing.Optional[dict[str, str]]
|
|
115
115
|
|
|
116
116
|
|
|
117
|
+
class Backoff(pydantic.v1.BaseModel):
|
|
118
|
+
default_base_delay: typing.Optional[str]
|
|
119
|
+
min_base_delay: typing.Optional[str]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class RetrySpec(pydantic.v1.BaseModel):
|
|
123
|
+
backoff: Backoff
|
|
124
|
+
|
|
125
|
+
|
|
117
126
|
class FunctionSpec(pydantic.v1.BaseModel):
|
|
118
127
|
image_pull_secret: typing.Optional[ImagePullSecret]
|
|
119
128
|
security_context: typing.Optional[SecurityContext]
|
|
120
129
|
service_account: typing.Optional[ServiceAccount]
|
|
121
130
|
state_thresholds: typing.Optional[StateThresholds]
|
|
131
|
+
retry: typing.Optional[RetrySpec]
|
|
122
132
|
|
|
123
133
|
class Config:
|
|
124
134
|
extra = pydantic.v1.Extra.allow
|
|
@@ -130,3 +140,27 @@ class Function(pydantic.v1.BaseModel):
|
|
|
130
140
|
|
|
131
141
|
class Config:
|
|
132
142
|
extra = pydantic.v1.Extra.allow
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class BatchingSpec(pydantic.v1.BaseModel):
|
|
146
|
+
# Set to True to enable batching
|
|
147
|
+
enabled: bool
|
|
148
|
+
# Maximal events to batch together. Default size is 10.
|
|
149
|
+
batch_size: typing.Optional[int]
|
|
150
|
+
# The maximum amount of time to wait before processing the batch. Default timeout is 1s.
|
|
151
|
+
# Once this time passes, the batch is processed even if it hasn’t reached the full batch size.
|
|
152
|
+
timeout: typing.Optional[str]
|
|
153
|
+
|
|
154
|
+
def get_nuclio_batch_config(self):
|
|
155
|
+
if not self.enabled:
|
|
156
|
+
return None
|
|
157
|
+
|
|
158
|
+
config = {"mode": "enable"}
|
|
159
|
+
|
|
160
|
+
if self.batch_size:
|
|
161
|
+
config["batchSize"] = self.batch_size
|
|
162
|
+
|
|
163
|
+
if self.timeout:
|
|
164
|
+
config["timeout"] = self.timeout
|
|
165
|
+
|
|
166
|
+
return config
|
mlrun/common/schemas/hub.py
CHANGED
|
@@ -12,9 +12,10 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from datetime import
|
|
15
|
+
from datetime import UTC, datetime
|
|
16
16
|
from typing import Optional
|
|
17
17
|
|
|
18
|
+
import deepdiff
|
|
18
19
|
from pydantic.v1 import BaseModel, Extra, Field
|
|
19
20
|
|
|
20
21
|
import mlrun.common.types
|
|
@@ -36,9 +37,10 @@ class HubObjectMetadata(BaseModel):
|
|
|
36
37
|
extra = Extra.allow
|
|
37
38
|
|
|
38
39
|
|
|
39
|
-
# Currently only functions are supported. Will add more in the future.
|
|
40
40
|
class HubSourceType(mlrun.common.types.StrEnum):
|
|
41
41
|
functions = "functions"
|
|
42
|
+
modules = "modules"
|
|
43
|
+
steps = "steps"
|
|
42
44
|
|
|
43
45
|
|
|
44
46
|
# Sources-related objects
|
|
@@ -46,7 +48,6 @@ class HubSourceSpec(ObjectSpec):
|
|
|
46
48
|
path: str # URL to base directory, should include schema (s3://, etc...)
|
|
47
49
|
channel: str
|
|
48
50
|
credentials: Optional[dict] = {}
|
|
49
|
-
object_type: HubSourceType = Field(HubSourceType.functions, const=True)
|
|
50
51
|
|
|
51
52
|
|
|
52
53
|
class HubSource(BaseModel):
|
|
@@ -55,18 +56,18 @@ class HubSource(BaseModel):
|
|
|
55
56
|
spec: HubSourceSpec
|
|
56
57
|
status: Optional[ObjectStatus] = ObjectStatus(state="created")
|
|
57
58
|
|
|
58
|
-
def get_full_uri(self, relative_path):
|
|
59
|
-
return f"{self.spec.path}/{
|
|
59
|
+
def get_full_uri(self, relative_path, object_type):
|
|
60
|
+
return f"{self.spec.path}/{object_type}/{self.spec.channel}/{relative_path}"
|
|
60
61
|
|
|
61
|
-
def get_catalog_uri(self):
|
|
62
|
-
return self.get_full_uri(mlrun.mlconf.hub.catalog_filename)
|
|
62
|
+
def get_catalog_uri(self, object_type):
|
|
63
|
+
return self.get_full_uri(mlrun.mlconf.hub.catalog_filename, object_type)
|
|
63
64
|
|
|
64
65
|
@classmethod
|
|
65
66
|
def generate_default_source(cls):
|
|
66
67
|
if not mlrun.mlconf.hub.default_source.create:
|
|
67
68
|
return None
|
|
68
69
|
|
|
69
|
-
now = datetime.now(
|
|
70
|
+
now = datetime.now(UTC)
|
|
70
71
|
hub_metadata = HubObjectMetadata(
|
|
71
72
|
name=mlrun.mlconf.hub.default_source.name,
|
|
72
73
|
description=mlrun.mlconf.hub.default_source.description,
|
|
@@ -78,11 +79,23 @@ class HubSource(BaseModel):
|
|
|
78
79
|
spec=HubSourceSpec(
|
|
79
80
|
path=mlrun.mlconf.hub.default_source.url,
|
|
80
81
|
channel=mlrun.mlconf.hub.default_source.channel,
|
|
81
|
-
object_type=HubSourceType(mlrun.mlconf.hub.default_source.object_type),
|
|
82
82
|
),
|
|
83
83
|
status=ObjectStatus(state="created"),
|
|
84
84
|
)
|
|
85
85
|
|
|
86
|
+
def diff(self, another_source: "HubSource") -> dict:
|
|
87
|
+
"""
|
|
88
|
+
Compare this HubSource with another one.
|
|
89
|
+
Returns a dict of differences (metadata, spec, status).
|
|
90
|
+
"""
|
|
91
|
+
exclude_paths = [
|
|
92
|
+
"root['metadata']['updated']",
|
|
93
|
+
"root['metadata']['created']",
|
|
94
|
+
]
|
|
95
|
+
return deepdiff.DeepDiff(
|
|
96
|
+
self.dict(), another_source.dict(), exclude_paths=exclude_paths
|
|
97
|
+
)
|
|
98
|
+
|
|
86
99
|
|
|
87
100
|
last_source_index = -1
|
|
88
101
|
|
|
@@ -94,21 +107,16 @@ class IndexedHubSource(BaseModel):
|
|
|
94
107
|
|
|
95
108
|
# Item-related objects
|
|
96
109
|
class HubItemMetadata(HubObjectMetadata):
|
|
97
|
-
source: HubSourceType =
|
|
110
|
+
source: HubSourceType = HubSourceType.functions
|
|
98
111
|
version: str
|
|
99
112
|
tag: Optional[str]
|
|
100
113
|
|
|
101
114
|
def get_relative_path(self) -> str:
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
return f"{modified_name}/{version}/"
|
|
108
|
-
else:
|
|
109
|
-
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
110
|
-
f"Bad source for hub item - {self.source}"
|
|
111
|
-
)
|
|
115
|
+
# This is needed since the hub deployment script modifies the paths to use _ instead of -.
|
|
116
|
+
modified_name = self.name.replace("-", "_")
|
|
117
|
+
# Prefer using the tag if exists. Otherwise, use version.
|
|
118
|
+
version = self.tag or self.version
|
|
119
|
+
return f"{modified_name}/{version}/"
|
|
112
120
|
|
|
113
121
|
|
|
114
122
|
class HubItemSpec(ObjectSpec):
|
|
@@ -127,3 +135,8 @@ class HubCatalog(BaseModel):
|
|
|
127
135
|
kind: ObjectKind = Field(ObjectKind.hub_catalog, const=True)
|
|
128
136
|
channel: str
|
|
129
137
|
catalog: list[HubItem]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class HubModuleType(mlrun.common.types.StrEnum):
|
|
141
|
+
generic = "generic"
|
|
142
|
+
monitoring_app = "monitoring_application"
|
|
@@ -30,6 +30,7 @@ from .constants import (
|
|
|
30
30
|
ModelEndpointMonitoringMetricType,
|
|
31
31
|
ModelEndpointSchema,
|
|
32
32
|
ModelMonitoringAppLabel,
|
|
33
|
+
ModelMonitoringInfraLabel,
|
|
33
34
|
ModelMonitoringMode,
|
|
34
35
|
MonitoringFunctionNames,
|
|
35
36
|
PredictionsQueryConstants,
|
|
@@ -39,7 +40,7 @@ from .constants import (
|
|
|
39
40
|
ResultStatusApp,
|
|
40
41
|
SpecialApps,
|
|
41
42
|
StreamProcessingEvent,
|
|
42
|
-
|
|
43
|
+
TimescaleDBTables,
|
|
43
44
|
TSDBTarget,
|
|
44
45
|
V3IOTSDBTables,
|
|
45
46
|
VersionedModel,
|
|
@@ -34,6 +34,7 @@ class ModelEndpointSchema(MonitoringStrEnum):
|
|
|
34
34
|
UID = "uid"
|
|
35
35
|
PROJECT = "project"
|
|
36
36
|
ENDPOINT_TYPE = "endpoint_type"
|
|
37
|
+
MODE = "mode"
|
|
37
38
|
NAME = "name"
|
|
38
39
|
CREATED = "created"
|
|
39
40
|
UPDATED = "updated"
|
|
@@ -195,6 +196,10 @@ class WriterEventKind(MonitoringStrEnum):
|
|
|
195
196
|
RESULT = "result"
|
|
196
197
|
STATS = "stats"
|
|
197
198
|
|
|
199
|
+
@classmethod
|
|
200
|
+
def user_app_outputs(cls):
|
|
201
|
+
return [cls.METRIC, cls.RESULT]
|
|
202
|
+
|
|
198
203
|
|
|
199
204
|
class ControllerEvent(MonitoringStrEnum):
|
|
200
205
|
KIND = "kind"
|
|
@@ -269,7 +274,7 @@ class EventKeyMetrics:
|
|
|
269
274
|
|
|
270
275
|
class TSDBTarget(MonitoringStrEnum):
|
|
271
276
|
V3IO_TSDB = "v3io-tsdb"
|
|
272
|
-
|
|
277
|
+
TimescaleDB = "postgresql"
|
|
273
278
|
|
|
274
279
|
|
|
275
280
|
class ProjectSecretKeys:
|
|
@@ -303,6 +308,7 @@ class FileTargetKind:
|
|
|
303
308
|
MONITORING_APPLICATION = "monitoring_application"
|
|
304
309
|
ERRORS = "errors"
|
|
305
310
|
STATS = "stats"
|
|
311
|
+
PARQUET_STATS = "parquet_stats"
|
|
306
312
|
LAST_REQUEST = "last_request"
|
|
307
313
|
|
|
308
314
|
|
|
@@ -326,18 +332,11 @@ class EndpointType(IntEnum):
|
|
|
326
332
|
def top_level_list(cls):
|
|
327
333
|
return [cls.NODE_EP, cls.ROUTER, cls.BATCH_EP]
|
|
328
334
|
|
|
329
|
-
@classmethod
|
|
330
|
-
def real_time_list(cls):
|
|
331
|
-
return [cls.NODE_EP, cls.ROUTER, cls.LEAF_EP]
|
|
332
|
-
|
|
333
|
-
@classmethod
|
|
334
|
-
def batch_list(cls):
|
|
335
|
-
return [cls.BATCH_EP]
|
|
336
|
-
|
|
337
335
|
|
|
338
|
-
class EndpointMode(
|
|
339
|
-
REAL_TIME =
|
|
340
|
-
BATCH =
|
|
336
|
+
class EndpointMode(IntEnum):
|
|
337
|
+
REAL_TIME = 0
|
|
338
|
+
BATCH = 1
|
|
339
|
+
BATCH_LEGACY = 2 # legacy batch mode, used for endpoints created through the batch inference job
|
|
341
340
|
|
|
342
341
|
|
|
343
342
|
class MonitoringFunctionNames(MonitoringStrEnum):
|
|
@@ -354,7 +353,7 @@ class V3IOTSDBTables(MonitoringStrEnum):
|
|
|
354
353
|
PREDICTIONS = "predictions"
|
|
355
354
|
|
|
356
355
|
|
|
357
|
-
class
|
|
356
|
+
class TimescaleDBTables(MonitoringStrEnum):
|
|
358
357
|
APP_RESULTS = "app_results"
|
|
359
358
|
METRICS = "metrics"
|
|
360
359
|
PREDICTIONS = "predictions"
|
|
@@ -487,8 +486,6 @@ class ModelMonitoringLabels:
|
|
|
487
486
|
|
|
488
487
|
_RESERVED_FUNCTION_NAMES = MonitoringFunctionNames.list() + [SpecialApps.MLRUN_INFRA]
|
|
489
488
|
|
|
490
|
-
_RESERVED_EVALUATE_FUNCTION_SUFFIX = "-batch"
|
|
491
|
-
|
|
492
489
|
|
|
493
490
|
class ModelEndpointMonitoringMetricType(StrEnum):
|
|
494
491
|
RESULT = "result"
|
|
@@ -54,12 +54,21 @@ class FunctionSummary(BaseModel):
|
|
|
54
54
|
|
|
55
55
|
return cls(
|
|
56
56
|
type=func_type,
|
|
57
|
-
name=func_dict["metadata"]["name"]
|
|
57
|
+
name=func_dict["metadata"]["name"]
|
|
58
|
+
if func_type != FunctionsType.APPLICATION
|
|
59
|
+
else func_dict["spec"]
|
|
60
|
+
.get("graph", {})
|
|
61
|
+
.get("steps", {})
|
|
62
|
+
.get("PrepareMonitoringEvent", {})
|
|
63
|
+
.get("class_args", {})
|
|
64
|
+
.get("application_name"),
|
|
58
65
|
application_class=""
|
|
59
66
|
if func_type != FunctionsType.APPLICATION
|
|
60
|
-
else func_dict["spec"]
|
|
61
|
-
|
|
62
|
-
|
|
67
|
+
else func_dict["spec"]
|
|
68
|
+
.get("graph", {})
|
|
69
|
+
.get("steps", {})
|
|
70
|
+
.get("PushToMonitoringWriter", {})
|
|
71
|
+
.get("after", [None])[0],
|
|
63
72
|
project_name=func_dict["metadata"]["project"],
|
|
64
73
|
updated_time=func_dict["metadata"].get("updated"),
|
|
65
74
|
status=func_dict["status"].get("state"),
|
|
@@ -28,6 +28,7 @@ from .constants import (
|
|
|
28
28
|
FQN_REGEX,
|
|
29
29
|
MODEL_ENDPOINT_ID_PATTERN,
|
|
30
30
|
PROJECT_PATTERN,
|
|
31
|
+
EndpointMode,
|
|
31
32
|
EndpointType,
|
|
32
33
|
ModelEndpointMonitoringMetricType,
|
|
33
34
|
ModelMonitoringMode,
|
|
@@ -118,6 +119,7 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
|
|
|
118
119
|
project: constr(regex=PROJECT_PATTERN)
|
|
119
120
|
endpoint_type: EndpointType = EndpointType.NODE_EP
|
|
120
121
|
uid: Optional[constr(regex=MODEL_ENDPOINT_ID_PATTERN)]
|
|
122
|
+
mode: Optional[EndpointMode] = None
|
|
121
123
|
|
|
122
124
|
@classmethod
|
|
123
125
|
def mutable_fields(cls):
|
|
@@ -129,6 +131,15 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
|
|
|
129
131
|
return str(v)
|
|
130
132
|
return v
|
|
131
133
|
|
|
134
|
+
@validator("mode", pre=True, always=True)
|
|
135
|
+
def _set_mode_based_on_endpoint_type(cls, v, values): # noqa: N805
|
|
136
|
+
if v is None:
|
|
137
|
+
if values.get("endpoint_type") == EndpointType.BATCH_EP:
|
|
138
|
+
return EndpointMode.BATCH_LEGACY
|
|
139
|
+
else:
|
|
140
|
+
return EndpointMode.REAL_TIME
|
|
141
|
+
return v
|
|
142
|
+
|
|
132
143
|
|
|
133
144
|
class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
|
|
134
145
|
model_class: Optional[str] = ""
|
mlrun/common/schemas/pipeline.py
CHANGED
mlrun/common/schemas/secret.py
CHANGED
|
@@ -49,5 +49,20 @@ class SecretKeysData(BaseModel):
|
|
|
49
49
|
secret_keys: Optional[list] = []
|
|
50
50
|
|
|
51
51
|
|
|
52
|
-
class
|
|
53
|
-
|
|
52
|
+
class SecretToken(BaseModel):
|
|
53
|
+
name: str
|
|
54
|
+
token: str
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class StoreSecretTokensResponse(BaseModel):
|
|
58
|
+
created_tokens: list[str] = []
|
|
59
|
+
updated_tokens: list[str] = []
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class SecretTokenInfo(BaseModel):
|
|
63
|
+
name: str
|
|
64
|
+
expiration: int
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class ListSecretTokensResponse(BaseModel):
|
|
68
|
+
secret_tokens: list[SecretTokenInfo]
|
mlrun/common/secrets.py
CHANGED
|
@@ -11,10 +11,32 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
import re
|
|
15
|
+
import typing
|
|
15
16
|
from abc import ABC, abstractmethod
|
|
16
17
|
|
|
17
18
|
import mlrun.common.schemas
|
|
19
|
+
from mlrun.config import config as mlconf
|
|
20
|
+
|
|
21
|
+
_AUTH_SECRET_NAME_TEMPLATE = re.escape(
|
|
22
|
+
mlconf.secret_stores.kubernetes.auth_secret_name.format(
|
|
23
|
+
hashed_access_key="",
|
|
24
|
+
)
|
|
25
|
+
)
|
|
26
|
+
AUTH_SECRET_PATTERN = re.compile(f"^{_AUTH_SECRET_NAME_TEMPLATE}.*")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def validate_not_forbidden_secret(secret_name: str) -> None:
|
|
30
|
+
"""
|
|
31
|
+
Forbid client-supplied references to internal MLRun auth/project secrets.
|
|
32
|
+
No-op when running inside the API server (API enrichments are allowed).
|
|
33
|
+
"""
|
|
34
|
+
if not secret_name or mlrun.config.is_running_as_api():
|
|
35
|
+
return
|
|
36
|
+
if AUTH_SECRET_PATTERN.match(secret_name):
|
|
37
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
38
|
+
f"Forbidden secret '{secret_name}' matches MLRun auth-secret pattern."
|
|
39
|
+
)
|
|
18
40
|
|
|
19
41
|
|
|
20
42
|
class SecretProviderInterface(ABC):
|
|
@@ -54,6 +76,44 @@ class SecretProviderInterface(ABC):
|
|
|
54
76
|
def get_secret_data(self, secret_name, namespace=""):
|
|
55
77
|
pass
|
|
56
78
|
|
|
79
|
+
@abstractmethod
|
|
80
|
+
def store_user_token_secret(
|
|
81
|
+
self,
|
|
82
|
+
username: str,
|
|
83
|
+
token_name: str,
|
|
84
|
+
token: str,
|
|
85
|
+
expiration: int,
|
|
86
|
+
force: bool = False,
|
|
87
|
+
namespace: typing.Optional[str] = None,
|
|
88
|
+
) -> typing.Optional[mlrun.common.schemas.SecretEventActions]:
|
|
89
|
+
pass
|
|
90
|
+
|
|
91
|
+
@abstractmethod
|
|
92
|
+
def get_user_token_secret_value(
|
|
93
|
+
self,
|
|
94
|
+
username: str,
|
|
95
|
+
token_name: str,
|
|
96
|
+
namespace: typing.Optional[str] = None,
|
|
97
|
+
) -> str:
|
|
98
|
+
pass
|
|
99
|
+
|
|
100
|
+
@abstractmethod
|
|
101
|
+
def list_user_token_secrets(
|
|
102
|
+
self,
|
|
103
|
+
username: str,
|
|
104
|
+
namespace: typing.Optional[str] = None,
|
|
105
|
+
) -> list[mlrun.common.schemas.SecretTokenInfo]:
|
|
106
|
+
pass
|
|
107
|
+
|
|
108
|
+
@abstractmethod
|
|
109
|
+
def delete_user_token_secret(
|
|
110
|
+
self,
|
|
111
|
+
username: str,
|
|
112
|
+
token_name: str,
|
|
113
|
+
namespace: typing.Optional[str] = None,
|
|
114
|
+
) -> None:
|
|
115
|
+
pass
|
|
116
|
+
|
|
57
117
|
|
|
58
118
|
class InMemorySecretProvider(SecretProviderInterface):
|
|
59
119
|
def __init__(self):
|
|
@@ -130,6 +190,40 @@ class InMemorySecretProvider(SecretProviderInterface):
|
|
|
130
190
|
def get_secret_data(self, secret_name, namespace=""):
|
|
131
191
|
return self.secrets_map[secret_name]
|
|
132
192
|
|
|
193
|
+
def store_user_token_secret(
|
|
194
|
+
self,
|
|
195
|
+
username: str,
|
|
196
|
+
token_name: str,
|
|
197
|
+
token: str,
|
|
198
|
+
expiration: int,
|
|
199
|
+
force: bool = False,
|
|
200
|
+
namespace: typing.Optional[str] = None,
|
|
201
|
+
) -> typing.Optional[mlrun.common.schemas.SecretEventActions]:
|
|
202
|
+
raise NotImplementedError()
|
|
203
|
+
|
|
204
|
+
def get_user_token_secret_value(
|
|
205
|
+
self,
|
|
206
|
+
username: str,
|
|
207
|
+
token_name: str,
|
|
208
|
+
namespace: typing.Optional[str] = None,
|
|
209
|
+
) -> str:
|
|
210
|
+
raise NotImplementedError()
|
|
211
|
+
|
|
212
|
+
def list_user_token_secrets(
|
|
213
|
+
self,
|
|
214
|
+
username: str,
|
|
215
|
+
namespace: typing.Optional[str] = None,
|
|
216
|
+
) -> list[mlrun.common.schemas.SecretTokenInfo]:
|
|
217
|
+
raise NotImplementedError()
|
|
218
|
+
|
|
219
|
+
def delete_user_token_secret(
|
|
220
|
+
self,
|
|
221
|
+
username: str,
|
|
222
|
+
token_name: str,
|
|
223
|
+
namespace: typing.Optional[str] = None,
|
|
224
|
+
) -> None:
|
|
225
|
+
raise NotImplementedError()
|
|
226
|
+
|
|
133
227
|
@staticmethod
|
|
134
228
|
def _generate_auth_secret_data(username: str, access_key: str):
|
|
135
229
|
return {
|
mlrun/common/types.py
CHANGED
|
@@ -14,18 +14,10 @@
|
|
|
14
14
|
|
|
15
15
|
import enum
|
|
16
16
|
|
|
17
|
+
# Alias to Python's built-in StrEnum (Python 3.11+)
|
|
18
|
+
StrEnum = enum.StrEnum
|
|
17
19
|
|
|
18
|
-
# TODO: From python 3.11 StrEnum is built-in and this will not be needed
|
|
19
|
-
class StrEnum(str, enum.Enum):
|
|
20
|
-
def __str__(self):
|
|
21
|
-
return self.value
|
|
22
20
|
|
|
23
|
-
def __repr__(self):
|
|
24
|
-
return self.value
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
# Partial backport from Python 3.11
|
|
28
|
-
# https://docs.python.org/3/library/http.html#http.HTTPMethod
|
|
29
21
|
class HTTPMethod(StrEnum):
|
|
30
22
|
GET = "GET"
|
|
31
23
|
POST = "POST"
|
|
@@ -37,3 +29,11 @@ class HTTPMethod(StrEnum):
|
|
|
37
29
|
class Operation(StrEnum):
|
|
38
30
|
ADD = "add"
|
|
39
31
|
REMOVE = "remove"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class AuthenticationMode(StrEnum):
|
|
35
|
+
NONE = "none"
|
|
36
|
+
BASIC = "basic"
|
|
37
|
+
BEARER = "bearer"
|
|
38
|
+
IGUAZIO = "iguazio"
|
|
39
|
+
IGUAZIO_V4 = "iguazio-v4"
|