mlrun 1.4.0rc25__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (184) hide show
  1. mlrun/__init__.py +2 -35
  2. mlrun/__main__.py +3 -41
  3. mlrun/api/api/api.py +6 -0
  4. mlrun/api/api/endpoints/feature_store.py +0 -4
  5. mlrun/api/api/endpoints/files.py +14 -2
  6. mlrun/api/api/endpoints/frontend_spec.py +2 -1
  7. mlrun/api/api/endpoints/functions.py +95 -59
  8. mlrun/api/api/endpoints/grafana_proxy.py +9 -9
  9. mlrun/api/api/endpoints/logs.py +17 -3
  10. mlrun/api/api/endpoints/model_endpoints.py +3 -2
  11. mlrun/api/api/endpoints/pipelines.py +1 -5
  12. mlrun/api/api/endpoints/projects.py +88 -0
  13. mlrun/api/api/endpoints/runs.py +48 -6
  14. mlrun/api/api/endpoints/submit.py +2 -1
  15. mlrun/api/api/endpoints/workflows.py +355 -0
  16. mlrun/api/api/utils.py +3 -4
  17. mlrun/api/crud/__init__.py +1 -0
  18. mlrun/api/crud/client_spec.py +6 -2
  19. mlrun/api/crud/feature_store.py +5 -0
  20. mlrun/api/crud/model_monitoring/__init__.py +1 -0
  21. mlrun/api/crud/model_monitoring/deployment.py +497 -0
  22. mlrun/api/crud/model_monitoring/grafana.py +96 -42
  23. mlrun/api/crud/model_monitoring/helpers.py +159 -0
  24. mlrun/api/crud/model_monitoring/model_endpoints.py +202 -476
  25. mlrun/api/crud/notifications.py +9 -4
  26. mlrun/api/crud/pipelines.py +6 -11
  27. mlrun/api/crud/projects.py +2 -2
  28. mlrun/api/crud/runtime_resources.py +4 -3
  29. mlrun/api/crud/runtimes/nuclio/helpers.py +5 -1
  30. mlrun/api/crud/secrets.py +21 -0
  31. mlrun/api/crud/workflows.py +352 -0
  32. mlrun/api/db/base.py +16 -1
  33. mlrun/api/db/init_db.py +2 -4
  34. mlrun/api/db/session.py +1 -1
  35. mlrun/api/db/sqldb/db.py +129 -31
  36. mlrun/api/db/sqldb/models/models_mysql.py +15 -1
  37. mlrun/api/db/sqldb/models/models_sqlite.py +16 -2
  38. mlrun/api/launcher.py +38 -6
  39. mlrun/api/main.py +3 -2
  40. mlrun/api/rundb/__init__.py +13 -0
  41. mlrun/{db → api/rundb}/sqldb.py +36 -84
  42. mlrun/api/runtime_handlers/__init__.py +56 -0
  43. mlrun/api/runtime_handlers/base.py +1247 -0
  44. mlrun/api/runtime_handlers/daskjob.py +209 -0
  45. mlrun/api/runtime_handlers/kubejob.py +37 -0
  46. mlrun/api/runtime_handlers/mpijob.py +147 -0
  47. mlrun/api/runtime_handlers/remotesparkjob.py +29 -0
  48. mlrun/api/runtime_handlers/sparkjob.py +148 -0
  49. mlrun/api/schemas/__init__.py +17 -6
  50. mlrun/api/utils/builder.py +1 -4
  51. mlrun/api/utils/clients/chief.py +14 -0
  52. mlrun/api/utils/clients/iguazio.py +33 -33
  53. mlrun/api/utils/clients/nuclio.py +2 -2
  54. mlrun/api/utils/periodic.py +9 -2
  55. mlrun/api/utils/projects/follower.py +14 -7
  56. mlrun/api/utils/projects/leader.py +2 -1
  57. mlrun/api/utils/projects/remotes/nop_follower.py +2 -2
  58. mlrun/api/utils/projects/remotes/nop_leader.py +2 -2
  59. mlrun/api/utils/runtimes/__init__.py +14 -0
  60. mlrun/api/utils/runtimes/nuclio.py +43 -0
  61. mlrun/api/utils/scheduler.py +98 -15
  62. mlrun/api/utils/singletons/db.py +5 -1
  63. mlrun/api/utils/singletons/project_member.py +4 -1
  64. mlrun/api/utils/singletons/scheduler.py +1 -1
  65. mlrun/artifacts/base.py +6 -6
  66. mlrun/artifacts/dataset.py +4 -4
  67. mlrun/artifacts/manager.py +2 -3
  68. mlrun/artifacts/model.py +2 -2
  69. mlrun/artifacts/plots.py +8 -8
  70. mlrun/common/db/__init__.py +14 -0
  71. mlrun/common/helpers.py +37 -0
  72. mlrun/{mlutils → common/model_monitoring}/__init__.py +3 -2
  73. mlrun/common/model_monitoring/helpers.py +69 -0
  74. mlrun/common/schemas/__init__.py +13 -1
  75. mlrun/common/schemas/auth.py +4 -1
  76. mlrun/common/schemas/client_spec.py +1 -1
  77. mlrun/common/schemas/function.py +17 -0
  78. mlrun/common/schemas/model_monitoring/__init__.py +48 -0
  79. mlrun/common/{model_monitoring.py → schemas/model_monitoring/constants.py} +11 -23
  80. mlrun/common/schemas/model_monitoring/grafana.py +55 -0
  81. mlrun/common/schemas/{model_endpoints.py → model_monitoring/model_endpoints.py} +32 -65
  82. mlrun/common/schemas/notification.py +1 -0
  83. mlrun/common/schemas/object.py +4 -0
  84. mlrun/common/schemas/project.py +1 -0
  85. mlrun/common/schemas/regex.py +1 -1
  86. mlrun/common/schemas/runs.py +1 -8
  87. mlrun/common/schemas/schedule.py +1 -8
  88. mlrun/common/schemas/workflow.py +54 -0
  89. mlrun/config.py +45 -42
  90. mlrun/datastore/__init__.py +21 -0
  91. mlrun/datastore/base.py +1 -1
  92. mlrun/datastore/datastore.py +9 -0
  93. mlrun/datastore/dbfs_store.py +168 -0
  94. mlrun/datastore/helpers.py +18 -0
  95. mlrun/datastore/sources.py +1 -0
  96. mlrun/datastore/store_resources.py +2 -5
  97. mlrun/datastore/v3io.py +1 -2
  98. mlrun/db/__init__.py +4 -68
  99. mlrun/db/base.py +12 -0
  100. mlrun/db/factory.py +65 -0
  101. mlrun/db/httpdb.py +175 -20
  102. mlrun/db/nopdb.py +4 -2
  103. mlrun/execution.py +4 -2
  104. mlrun/feature_store/__init__.py +1 -0
  105. mlrun/feature_store/api.py +1 -2
  106. mlrun/feature_store/common.py +2 -1
  107. mlrun/feature_store/feature_set.py +1 -11
  108. mlrun/feature_store/feature_vector.py +340 -2
  109. mlrun/feature_store/ingestion.py +5 -10
  110. mlrun/feature_store/retrieval/base.py +118 -104
  111. mlrun/feature_store/retrieval/dask_merger.py +17 -10
  112. mlrun/feature_store/retrieval/job.py +4 -1
  113. mlrun/feature_store/retrieval/local_merger.py +18 -18
  114. mlrun/feature_store/retrieval/spark_merger.py +21 -14
  115. mlrun/feature_store/retrieval/storey_merger.py +22 -16
  116. mlrun/kfpops.py +3 -9
  117. mlrun/launcher/base.py +57 -53
  118. mlrun/launcher/client.py +5 -4
  119. mlrun/launcher/factory.py +24 -13
  120. mlrun/launcher/local.py +6 -6
  121. mlrun/launcher/remote.py +4 -4
  122. mlrun/lists.py +0 -11
  123. mlrun/model.py +11 -17
  124. mlrun/model_monitoring/__init__.py +2 -22
  125. mlrun/model_monitoring/features_drift_table.py +1 -1
  126. mlrun/model_monitoring/helpers.py +22 -210
  127. mlrun/model_monitoring/model_endpoint.py +1 -1
  128. mlrun/model_monitoring/model_monitoring_batch.py +127 -50
  129. mlrun/model_monitoring/prometheus.py +219 -0
  130. mlrun/model_monitoring/stores/__init__.py +16 -11
  131. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +95 -23
  132. mlrun/model_monitoring/stores/models/mysql.py +47 -29
  133. mlrun/model_monitoring/stores/models/sqlite.py +47 -29
  134. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +31 -19
  135. mlrun/model_monitoring/{stream_processing_fs.py → stream_processing.py} +206 -64
  136. mlrun/model_monitoring/tracking_policy.py +104 -0
  137. mlrun/package/packager.py +6 -8
  138. mlrun/package/packagers/default_packager.py +121 -10
  139. mlrun/package/packagers/numpy_packagers.py +1 -1
  140. mlrun/platforms/__init__.py +0 -2
  141. mlrun/platforms/iguazio.py +0 -56
  142. mlrun/projects/pipelines.py +53 -159
  143. mlrun/projects/project.py +10 -37
  144. mlrun/render.py +1 -1
  145. mlrun/run.py +8 -124
  146. mlrun/runtimes/__init__.py +6 -42
  147. mlrun/runtimes/base.py +29 -1249
  148. mlrun/runtimes/daskjob.py +2 -198
  149. mlrun/runtimes/funcdoc.py +0 -9
  150. mlrun/runtimes/function.py +25 -29
  151. mlrun/runtimes/kubejob.py +5 -29
  152. mlrun/runtimes/local.py +1 -1
  153. mlrun/runtimes/mpijob/__init__.py +2 -2
  154. mlrun/runtimes/mpijob/abstract.py +10 -1
  155. mlrun/runtimes/mpijob/v1.py +0 -76
  156. mlrun/runtimes/mpijob/v1alpha1.py +1 -74
  157. mlrun/runtimes/nuclio.py +3 -2
  158. mlrun/runtimes/pod.py +28 -18
  159. mlrun/runtimes/remotesparkjob.py +1 -15
  160. mlrun/runtimes/serving.py +14 -6
  161. mlrun/runtimes/sparkjob/__init__.py +0 -1
  162. mlrun/runtimes/sparkjob/abstract.py +4 -131
  163. mlrun/runtimes/utils.py +0 -26
  164. mlrun/serving/routers.py +7 -7
  165. mlrun/serving/server.py +11 -8
  166. mlrun/serving/states.py +7 -1
  167. mlrun/serving/v2_serving.py +6 -6
  168. mlrun/utils/helpers.py +23 -42
  169. mlrun/utils/notifications/notification/__init__.py +4 -0
  170. mlrun/utils/notifications/notification/webhook.py +61 -0
  171. mlrun/utils/notifications/notification_pusher.py +5 -25
  172. mlrun/utils/regex.py +7 -2
  173. mlrun/utils/version/version.json +2 -2
  174. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/METADATA +26 -25
  175. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/RECORD +180 -158
  176. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/WHEEL +1 -1
  177. mlrun/mlutils/data.py +0 -160
  178. mlrun/mlutils/models.py +0 -78
  179. mlrun/mlutils/plots.py +0 -902
  180. mlrun/utils/model_monitoring.py +0 -249
  181. /mlrun/{api/db/sqldb/session.py → common/db/sql_session.py} +0 -0
  182. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/LICENSE +0 -0
  183. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/entry_points.txt +0 -0
  184. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,37 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ import typing
16
+
17
+
18
+ def parse_versioned_object_uri(
19
+ uri: str, default_project: str = ""
20
+ ) -> typing.Tuple[str, str, str, str]:
21
+ project = default_project
22
+ tag = ""
23
+ hash_key = ""
24
+ if "/" in uri:
25
+ loc = uri.find("/")
26
+ project = uri[:loc]
27
+ uri = uri[loc + 1 :]
28
+ if ":" in uri:
29
+ loc = uri.find(":")
30
+ tag = uri[loc + 1 :]
31
+ uri = uri[:loc]
32
+ if "@" in uri:
33
+ loc = uri.find("@")
34
+ hash_key = uri[loc + 1 :]
35
+ uri = uri[:loc]
36
+
37
+ return project, uri, tag, hash_key
@@ -12,6 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
+
15
16
  # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
- # for backwards compatibility
17
- from ..utils.helpers import create_class, create_function
17
+
18
+ from .helpers import create_model_endpoint_uid
@@ -0,0 +1,69 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ import mlrun.common
17
+ from mlrun.common.schemas.model_monitoring import (
18
+ EndpointUID,
19
+ FunctionURI,
20
+ VersionedModel,
21
+ )
22
+
23
+
24
+ def create_model_endpoint_uid(function_uri: str, versioned_model: str):
25
+ function_uri = FunctionURI.from_string(function_uri)
26
+ versioned_model = VersionedModel.from_string(versioned_model)
27
+
28
+ if (
29
+ not function_uri.project
30
+ or not function_uri.function
31
+ or not versioned_model.model
32
+ ):
33
+ raise ValueError("Both function_uri and versioned_model have to be initialized")
34
+
35
+ uid = EndpointUID(
36
+ function_uri.project,
37
+ function_uri.function,
38
+ function_uri.tag,
39
+ function_uri.hash_key,
40
+ versioned_model.model,
41
+ versioned_model.version,
42
+ )
43
+
44
+ return uid
45
+
46
+
47
+ def parse_model_endpoint_project_prefix(path: str, project_name: str):
48
+ return path.split(project_name, 1)[0] + project_name
49
+
50
+
51
+ def parse_model_endpoint_store_prefix(store_prefix: str):
52
+ endpoint, parsed_url = mlrun.platforms.iguazio.parse_path(store_prefix)
53
+ container, path = parsed_url.split("/", 1)
54
+ return endpoint, container, path
55
+
56
+
57
+ def parse_monitoring_stream_path(stream_uri: str, project: str):
58
+ if stream_uri.startswith("kafka://"):
59
+ if "?topic" in stream_uri:
60
+ raise mlrun.errors.MLRunInvalidArgumentError(
61
+ "Custom kafka topic is not allowed"
62
+ )
63
+ # Add topic to stream kafka uri
64
+ stream_uri += f"?topic=monitoring_stream_{project}"
65
+
66
+ elif stream_uri.startswith("v3io://") and mlrun.mlconf.is_ce_mode():
67
+ # V3IO is not supported in CE mode, generating a default http stream path
68
+ stream_uri = mlrun.mlconf.model_endpoint_monitoring.default_http_sink
69
+ return stream_uri
@@ -97,7 +97,11 @@ from .hub import (
97
97
  )
98
98
  from .k8s import NodeSelectorOperator, Resources, ResourceSpec
99
99
  from .memory_reports import MostCommonObjectTypesReport, ObjectTypeReport
100
- from .model_endpoints import (
100
+ from .model_monitoring import (
101
+ EndpointType,
102
+ EndpointUID,
103
+ EventFieldType,
104
+ EventKeyMetrics,
101
105
  Features,
102
106
  FeatureValues,
103
107
  GrafanaColumn,
@@ -111,7 +115,9 @@ from .model_endpoints import (
111
115
  ModelEndpointMetadata,
112
116
  ModelEndpointSpec,
113
117
  ModelEndpointStatus,
118
+ ModelMonitoringMode,
114
119
  ModelMonitoringStoreKinds,
120
+ TimeSeriesTarget,
115
121
  )
116
122
  from .notification import (
117
123
  Notification,
@@ -165,3 +171,9 @@ from .secret import (
165
171
  UserSecretCreationRequest,
166
172
  )
167
173
  from .tag import Tag, TagObjects
174
+ from .workflow import (
175
+ GetWorkflowResponse,
176
+ WorkflowRequest,
177
+ WorkflowResponse,
178
+ WorkflowSpec,
179
+ )
@@ -57,6 +57,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
57
57
  model_endpoint = "model-endpoint"
58
58
  pipeline = "pipeline"
59
59
  hub_source = "hub-source"
60
+ workflow = "workflow"
60
61
 
61
62
  def to_resource_string(
62
63
  self,
@@ -87,8 +88,10 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
87
88
  AuthorizationResourceTypes.pipeline: "/projects/{project_name}/pipelines/{resource_name}",
88
89
  # Hub sources are not project-scoped, and auth is globally on the sources endpoint.
89
90
  # TODO - this was reverted to /marketplace since MLRun needs to be able to run with old igz versions. Once
90
- # we only have support for igz versions that support /hub (>=3.5.4), change this to "/hub/sources".
91
+ # we only have support for igz versions that support /hub (>=3.5.4), change this to "/hub/sources".
91
92
  AuthorizationResourceTypes.hub_source: "/marketplace/sources",
93
+ # workflow define how to run a pipeline and can be considered as the specification of a pipeline.
94
+ AuthorizationResourceTypes.workflow: "/projects/{project_name}/workflows/{resource_name}",
92
95
  }[self].format(project_name=project_name, resource_name=resource_name)
93
96
 
94
97
 
@@ -57,7 +57,7 @@ class ClientSpec(pydantic.BaseModel):
57
57
  redis_url: typing.Optional[str]
58
58
  redis_type: typing.Optional[str]
59
59
  sql_url: typing.Optional[str]
60
-
60
+ model_endpoint_monitoring_store_type: typing.Optional[str]
61
61
  # ce_mode is deprecated, we will use the full ce config instead and ce_mode will be removed in 1.6.0
62
62
  ce_mode: typing.Optional[str]
63
63
  ce: typing.Optional[dict]
@@ -45,6 +45,23 @@ class FunctionState:
45
45
  # same goes for the build which is not coming from the pod, but is used and we can't just omit it for BC reasons
46
46
  build = "build"
47
47
 
48
+ @classmethod
49
+ def get_function_state_from_pod_state(cls, pod_state: str):
50
+ if pod_state == "succeeded":
51
+ return cls.ready
52
+ if pod_state in ["failed", "error"]:
53
+ return cls.error
54
+ if pod_state in ["running", "pending"]:
55
+ return getattr(cls, pod_state)
56
+ return cls.unknown
57
+
58
+ @classmethod
59
+ def terminal_states(cls):
60
+ return [
61
+ cls.ready,
62
+ cls.error,
63
+ ]
64
+
48
65
 
49
66
  class PreemptionModes(mlrun.common.types.StrEnum):
50
67
  # makes function pods be able to run on preemptible nodes
@@ -0,0 +1,48 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
+
17
+ from .constants import (
18
+ EndpointType,
19
+ EndpointUID,
20
+ EventFieldType,
21
+ EventKeyMetrics,
22
+ EventLiveStats,
23
+ FileTargetKind,
24
+ FunctionURI,
25
+ ModelMonitoringMode,
26
+ ModelMonitoringStoreKinds,
27
+ ProjectSecretKeys,
28
+ PrometheusMetric,
29
+ TimeSeriesTarget,
30
+ VersionedModel,
31
+ )
32
+ from .grafana import (
33
+ GrafanaColumn,
34
+ GrafanaDataPoint,
35
+ GrafanaNumberColumn,
36
+ GrafanaStringColumn,
37
+ GrafanaTable,
38
+ GrafanaTimeSeriesTarget,
39
+ )
40
+ from .model_endpoints import (
41
+ Features,
42
+ FeatureValues,
43
+ ModelEndpoint,
44
+ ModelEndpointList,
45
+ ModelEndpointMetadata,
46
+ ModelEndpointSpec,
47
+ ModelEndpointStatus,
48
+ )
@@ -12,12 +12,12 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
-
16
15
  import enum
17
16
  import hashlib
18
17
  from dataclasses import dataclass
19
18
  from typing import Optional
20
19
 
20
+ import mlrun.common.helpers
21
21
  import mlrun.utils
22
22
 
23
23
 
@@ -49,6 +49,7 @@ class EventFieldType:
49
49
  ENTITIES = "entities"
50
50
  FIRST_REQUEST = "first_request"
51
51
  LAST_REQUEST = "last_request"
52
+ METRIC = "metric"
52
53
  METRICS = "metrics"
53
54
  TIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
54
55
  BATCH_INTERVALS_DICT = "batch_intervals_dict"
@@ -71,6 +72,7 @@ class EventFieldType:
71
72
  MONITOR_CONFIGURATION = "monitor_configuration"
72
73
  FEATURE_SET_URI = "monitoring_feature_set_uri"
73
74
  ALGORITHM = "algorithm"
75
+ VALUE = "value"
74
76
 
75
77
 
76
78
  class EventLiveStats:
@@ -129,27 +131,13 @@ class EndpointType(enum.IntEnum):
129
131
  LEAF_EP = 3 # end point that is a child of a router
130
132
 
131
133
 
132
- def create_model_endpoint_uid(function_uri: str, versioned_model: str):
133
- function_uri = FunctionURI.from_string(function_uri)
134
- versioned_model = VersionedModel.from_string(versioned_model)
135
-
136
- if (
137
- not function_uri.project
138
- or not function_uri.function
139
- or not versioned_model.model
140
- ):
141
- raise ValueError("Both function_uri and versioned_model have to be initialized")
142
-
143
- uid = EndpointUID(
144
- function_uri.project,
145
- function_uri.function,
146
- function_uri.tag,
147
- function_uri.hash_key,
148
- versioned_model.model,
149
- versioned_model.version,
150
- )
151
-
152
- return uid
134
+ class PrometheusMetric:
135
+ PREDICTIONS_TOTAL = "predictions_total"
136
+ MODEL_LATENCY_SECONDS = "model_latency_seconds"
137
+ INCOME_FEATURES = "income_features"
138
+ ERRORS_TOTAL = "errors_total"
139
+ DRIFT_METRICS = "drift_metrics"
140
+ DRIFT_STATUS = "drift_status"
153
141
 
154
142
 
155
143
  @dataclass
@@ -161,7 +149,7 @@ class FunctionURI:
161
149
 
162
150
  @classmethod
163
151
  def from_string(cls, function_uri):
164
- project, uri, tag, hash_key = mlrun.utils.parse_versioned_object_uri(
152
+ project, uri, tag, hash_key = mlrun.common.helpers.parse_versioned_object_uri(
165
153
  function_uri
166
154
  )
167
155
  return cls(
@@ -0,0 +1,55 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ from typing import List, Optional, Tuple, Union
17
+
18
+ from pydantic import BaseModel
19
+
20
+
21
+ class GrafanaColumn(BaseModel):
22
+ text: str
23
+ type: str
24
+
25
+
26
+ class GrafanaNumberColumn(GrafanaColumn):
27
+ text: str
28
+ type: str = "number"
29
+
30
+
31
+ class GrafanaStringColumn(GrafanaColumn):
32
+ text: str
33
+ type: str = "string"
34
+
35
+
36
+ class GrafanaTable(BaseModel):
37
+ columns: List[GrafanaColumn]
38
+ rows: List[List[Optional[Union[float, int, str]]]] = []
39
+ type: str = "table"
40
+
41
+ def add_row(self, *args):
42
+ self.rows.append(list(args))
43
+
44
+
45
+ class GrafanaDataPoint(BaseModel):
46
+ value: float
47
+ timestamp: int # Unix timestamp in milliseconds
48
+
49
+
50
+ class GrafanaTimeSeriesTarget(BaseModel):
51
+ target: str
52
+ datapoints: List[Tuple[float, int]] = []
53
+
54
+ def add_data_point(self, data_point: GrafanaDataPoint):
55
+ self.datapoints.append((data_point.value, data_point.timestamp))
@@ -16,13 +16,21 @@
16
16
  import enum
17
17
  import json
18
18
  import typing
19
- from typing import Any, Dict, List, Optional, Tuple, Union
19
+ from typing import Any, Dict, List, Optional
20
20
 
21
21
  from pydantic import BaseModel, Field
22
22
  from pydantic.main import Extra
23
23
 
24
24
  import mlrun.common.model_monitoring
25
- from mlrun.common.schemas.object import ObjectKind, ObjectSpec, ObjectStatus
25
+
26
+ from ..object import ObjectKind, ObjectSpec, ObjectStatus
27
+ from .constants import (
28
+ EndpointType,
29
+ EventFieldType,
30
+ EventKeyMetrics,
31
+ EventLiveStats,
32
+ ModelMonitoringMode,
33
+ )
26
34
 
27
35
 
28
36
  class ModelMonitoringStoreKinds:
@@ -49,7 +57,7 @@ class ModelEndpointMetadata(BaseModel):
49
57
  """
50
58
  new_object = cls()
51
59
  if json_parse_values is None:
52
- json_parse_values = [mlrun.common.model_monitoring.EventFieldType.LABELS]
60
+ json_parse_values = [EventFieldType.LABELS]
53
61
 
54
62
  return _mapping_attributes(
55
63
  base_model=new_object,
@@ -69,9 +77,7 @@ class ModelEndpointSpec(ObjectSpec):
69
77
  algorithm: Optional[str] = ""
70
78
  monitor_configuration: Optional[dict] = {}
71
79
  active: Optional[bool] = True
72
- monitoring_mode: Optional[
73
- mlrun.common.model_monitoring.ModelMonitoringMode
74
- ] = mlrun.common.model_monitoring.ModelMonitoringMode.disabled.value
80
+ monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled.value
75
81
 
76
82
  @classmethod
77
83
  def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: typing.List = None):
@@ -84,9 +90,9 @@ class ModelEndpointSpec(ObjectSpec):
84
90
  new_object = cls()
85
91
  if json_parse_values is None:
86
92
  json_parse_values = [
87
- mlrun.common.model_monitoring.EventFieldType.FEATURE_NAMES,
88
- mlrun.common.model_monitoring.EventFieldType.LABEL_NAMES,
89
- mlrun.common.model_monitoring.EventFieldType.MONITOR_CONFIGURATION,
93
+ EventFieldType.FEATURE_NAMES,
94
+ EventFieldType.LABEL_NAMES,
95
+ EventFieldType.MONITOR_CONFIGURATION,
90
96
  ]
91
97
  return _mapping_attributes(
92
98
  base_model=new_object,
@@ -149,17 +155,15 @@ class ModelEndpointStatus(ObjectStatus):
149
155
  drift_status: Optional[str] = ""
150
156
  drift_measures: Optional[dict] = {}
151
157
  metrics: Optional[Dict[str, Dict[str, Any]]] = {
152
- mlrun.common.model_monitoring.EventKeyMetrics.GENERIC: {
153
- mlrun.common.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
154
- mlrun.common.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
158
+ EventKeyMetrics.GENERIC: {
159
+ EventLiveStats.LATENCY_AVG_1H: 0,
160
+ EventLiveStats.PREDICTIONS_PER_SECOND: 0,
155
161
  }
156
162
  }
157
163
  features: Optional[List[Features]] = []
158
164
  children: Optional[List[str]] = []
159
165
  children_uids: Optional[List[str]] = []
160
- endpoint_type: Optional[
161
- mlrun.common.model_monitoring.EndpointType
162
- ] = mlrun.common.model_monitoring.EndpointType.NODE_EP.value
166
+ endpoint_type: Optional[EndpointType] = EndpointType.NODE_EP.value
163
167
  monitoring_feature_set_uri: Optional[str] = ""
164
168
  state: Optional[str] = ""
165
169
 
@@ -177,13 +181,13 @@ class ModelEndpointStatus(ObjectStatus):
177
181
  new_object = cls()
178
182
  if json_parse_values is None:
179
183
  json_parse_values = [
180
- mlrun.common.model_monitoring.EventFieldType.FEATURE_STATS,
181
- mlrun.common.model_monitoring.EventFieldType.CURRENT_STATS,
182
- mlrun.common.model_monitoring.EventFieldType.DRIFT_MEASURES,
183
- mlrun.common.model_monitoring.EventFieldType.METRICS,
184
- mlrun.common.model_monitoring.EventFieldType.CHILDREN,
185
- mlrun.common.model_monitoring.EventFieldType.CHILDREN_UIDS,
186
- mlrun.common.model_monitoring.EventFieldType.ENDPOINT_TYPE,
184
+ EventFieldType.FEATURE_STATS,
185
+ EventFieldType.CURRENT_STATS,
186
+ EventFieldType.DRIFT_MEASURES,
187
+ EventFieldType.METRICS,
188
+ EventFieldType.CHILDREN,
189
+ EventFieldType.CHILDREN_UIDS,
190
+ EventFieldType.ENDPOINT_TYPE,
187
191
  ]
188
192
  return _mapping_attributes(
189
193
  base_model=new_object,
@@ -236,17 +240,17 @@ class ModelEndpoint(BaseModel):
236
240
  else:
237
241
  flatten_dict[key] = current_value
238
242
 
239
- if mlrun.common.model_monitoring.EventFieldType.METRICS not in flatten_dict:
243
+ if EventFieldType.METRICS not in flatten_dict:
240
244
  # Initialize metrics dictionary
241
- flatten_dict[mlrun.common.model_monitoring.EventFieldType.METRICS] = {
242
- mlrun.common.model_monitoring.EventKeyMetrics.GENERIC: {
243
- mlrun.common.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
244
- mlrun.common.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
245
+ flatten_dict[EventFieldType.METRICS] = {
246
+ EventKeyMetrics.GENERIC: {
247
+ EventLiveStats.LATENCY_AVG_1H: 0,
248
+ EventLiveStats.PREDICTIONS_PER_SECOND: 0,
245
249
  }
246
250
  }
247
251
 
248
252
  # Remove the features from the dictionary as this field will be filled only within the feature analysis process
249
- flatten_dict.pop(mlrun.common.model_monitoring.EventFieldType.FEATURES, None)
253
+ flatten_dict.pop(EventFieldType.FEATURES, None)
250
254
  return flatten_dict
251
255
 
252
256
  @classmethod
@@ -268,43 +272,6 @@ class ModelEndpointList(BaseModel):
268
272
  endpoints: List[ModelEndpoint] = []
269
273
 
270
274
 
271
- class GrafanaColumn(BaseModel):
272
- text: str
273
- type: str
274
-
275
-
276
- class GrafanaNumberColumn(GrafanaColumn):
277
- text: str
278
- type: str = "number"
279
-
280
-
281
- class GrafanaStringColumn(GrafanaColumn):
282
- text: str
283
- type: str = "string"
284
-
285
-
286
- class GrafanaTable(BaseModel):
287
- columns: List[GrafanaColumn]
288
- rows: List[List[Optional[Union[float, int, str]]]] = []
289
- type: str = "table"
290
-
291
- def add_row(self, *args):
292
- self.rows.append(list(args))
293
-
294
-
295
- class GrafanaDataPoint(BaseModel):
296
- value: float
297
- timestamp: int # Unix timestamp in milliseconds
298
-
299
-
300
- class GrafanaTimeSeriesTarget(BaseModel):
301
- target: str
302
- datapoints: List[Tuple[float, int]] = []
303
-
304
- def add_data_point(self, data_point: GrafanaDataPoint):
305
- self.datapoints.append((data_point.value, data_point.timestamp))
306
-
307
-
308
275
  def _mapping_attributes(
309
276
  base_model: BaseModel,
310
277
  flattened_dictionary: dict,
@@ -25,6 +25,7 @@ class NotificationKind(mlrun.common.types.StrEnum):
25
25
  git = "git"
26
26
  ipython = "ipython"
27
27
  slack = "slack"
28
+ webhook = "webhook"
28
29
 
29
30
 
30
31
  class NotificationSeverity(mlrun.common.types.StrEnum):
@@ -78,3 +78,7 @@ class ObjectKind(mlrun.common.types.StrEnum):
78
78
  hub_source = "HubSource"
79
79
  hub_item = "HubItem"
80
80
  hub_catalog = "HubCatalog"
81
+
82
+
83
+ class ObjectStatusState(mlrun.common.types.StrEnum):
84
+ CREATED = "created"
@@ -84,6 +84,7 @@ class ProjectSpec(pydantic.BaseModel):
84
84
  origin_url: typing.Optional[str] = None
85
85
  desired_state: typing.Optional[ProjectDesiredState] = ProjectDesiredState.online
86
86
  custom_packagers: typing.Optional[typing.List[typing.Tuple[str, bool]]] = None
87
+ default_image: typing.Optional[str] = None
87
88
 
88
89
  class Config:
89
90
  extra = pydantic.Extra.allow
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -14,17 +14,10 @@
14
14
 
15
15
  import typing
16
16
 
17
- # TODO: When we remove support for python 3.7, we can use Literal from the typing package.
18
- # Remove the following try/except block with import from typing_extensions.
19
- try:
20
- from typing import Literal
21
- except ImportError:
22
- from typing_extensions import Literal
23
-
24
17
  import pydantic
25
18
 
26
19
 
27
20
  class RunIdentifier(pydantic.BaseModel):
28
- kind: Literal["run"] = "run"
21
+ kind: typing.Literal["run"] = "run"
29
22
  uid: typing.Optional[str]
30
23
  iter: typing.Optional[int]
@@ -13,14 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  from datetime import datetime
16
- from typing import Any, List, Optional, Union
17
-
18
- # TODO: When we remove support for python 3.7, we can use Literal from the typing package.
19
- # Remove the following try/except block with import from typing_extensions.
20
- try:
21
- from typing import Literal
22
- except ImportError:
23
- from typing_extensions import Literal
16
+ from typing import Any, List, Literal, Optional, Union
24
17
 
25
18
  from pydantic import BaseModel
26
19
 
@@ -0,0 +1,54 @@
1
+ # Copyright 2018 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ import typing
16
+
17
+ import pydantic
18
+
19
+ from .schedule import ScheduleCronTrigger
20
+
21
+
22
+ class WorkflowSpec(pydantic.BaseModel):
23
+ name: str
24
+ engine: typing.Optional[str] = None
25
+ code: typing.Optional[str] = None
26
+ path: typing.Optional[str] = None
27
+ args: typing.Optional[dict] = None
28
+ handler: typing.Optional[str] = None
29
+ ttl: typing.Optional[int] = None
30
+ args_schema: typing.Optional[list] = None
31
+ schedule: typing.Union[str, ScheduleCronTrigger] = None
32
+ run_local: typing.Optional[bool] = None
33
+ image: typing.Optional[str] = None
34
+
35
+
36
+ class WorkflowRequest(pydantic.BaseModel):
37
+ spec: typing.Optional[WorkflowSpec] = None
38
+ arguments: typing.Optional[typing.Dict] = None
39
+ artifact_path: typing.Optional[str] = None
40
+ source: typing.Optional[str] = None
41
+ run_name: typing.Optional[str] = None
42
+ namespace: typing.Optional[str] = None
43
+
44
+
45
+ class WorkflowResponse(pydantic.BaseModel):
46
+ project: str = None
47
+ name: str = None
48
+ status: str = None
49
+ run_id: typing.Optional[str] = None
50
+ schedule: typing.Union[str, ScheduleCronTrigger] = None
51
+
52
+
53
+ class GetWorkflowResponse(pydantic.BaseModel):
54
+ workflow_id: str = None