mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (275) hide show
  1. mlrun/__init__.py +26 -22
  2. mlrun/__main__.py +15 -16
  3. mlrun/alerts/alert.py +150 -15
  4. mlrun/api/schemas/__init__.py +1 -9
  5. mlrun/artifacts/__init__.py +2 -3
  6. mlrun/artifacts/base.py +62 -19
  7. mlrun/artifacts/dataset.py +17 -17
  8. mlrun/artifacts/document.py +454 -0
  9. mlrun/artifacts/manager.py +28 -18
  10. mlrun/artifacts/model.py +91 -59
  11. mlrun/artifacts/plots.py +2 -2
  12. mlrun/common/constants.py +8 -0
  13. mlrun/common/formatters/__init__.py +1 -0
  14. mlrun/common/formatters/artifact.py +1 -1
  15. mlrun/common/formatters/feature_set.py +2 -0
  16. mlrun/common/formatters/function.py +1 -0
  17. mlrun/{model_monitoring/db/stores/v3io_kv/__init__.py → common/formatters/model_endpoint.py} +17 -0
  18. mlrun/common/formatters/pipeline.py +1 -2
  19. mlrun/common/formatters/project.py +9 -0
  20. mlrun/common/model_monitoring/__init__.py +0 -5
  21. mlrun/common/model_monitoring/helpers.py +12 -62
  22. mlrun/common/runtimes/constants.py +25 -4
  23. mlrun/common/schemas/__init__.py +9 -5
  24. mlrun/common/schemas/alert.py +114 -19
  25. mlrun/common/schemas/api_gateway.py +3 -3
  26. mlrun/common/schemas/artifact.py +22 -9
  27. mlrun/common/schemas/auth.py +8 -4
  28. mlrun/common/schemas/background_task.py +7 -7
  29. mlrun/common/schemas/client_spec.py +4 -4
  30. mlrun/common/schemas/clusterization_spec.py +2 -2
  31. mlrun/common/schemas/common.py +53 -3
  32. mlrun/common/schemas/constants.py +15 -0
  33. mlrun/common/schemas/datastore_profile.py +1 -1
  34. mlrun/common/schemas/feature_store.py +9 -9
  35. mlrun/common/schemas/frontend_spec.py +4 -4
  36. mlrun/common/schemas/function.py +10 -10
  37. mlrun/common/schemas/hub.py +1 -1
  38. mlrun/common/schemas/k8s.py +3 -3
  39. mlrun/common/schemas/memory_reports.py +3 -3
  40. mlrun/common/schemas/model_monitoring/__init__.py +4 -8
  41. mlrun/common/schemas/model_monitoring/constants.py +127 -46
  42. mlrun/common/schemas/model_monitoring/grafana.py +18 -12
  43. mlrun/common/schemas/model_monitoring/model_endpoints.py +154 -160
  44. mlrun/common/schemas/notification.py +24 -3
  45. mlrun/common/schemas/object.py +1 -1
  46. mlrun/common/schemas/pagination.py +4 -4
  47. mlrun/common/schemas/partition.py +142 -0
  48. mlrun/common/schemas/pipeline.py +3 -3
  49. mlrun/common/schemas/project.py +26 -18
  50. mlrun/common/schemas/runs.py +3 -3
  51. mlrun/common/schemas/runtime_resource.py +5 -5
  52. mlrun/common/schemas/schedule.py +1 -1
  53. mlrun/common/schemas/secret.py +1 -1
  54. mlrun/{model_monitoring/db/stores/sqldb/__init__.py → common/schemas/serving.py} +10 -1
  55. mlrun/common/schemas/tag.py +3 -3
  56. mlrun/common/schemas/workflow.py +6 -5
  57. mlrun/common/types.py +1 -0
  58. mlrun/config.py +157 -89
  59. mlrun/data_types/__init__.py +5 -3
  60. mlrun/data_types/infer.py +13 -3
  61. mlrun/data_types/spark.py +2 -1
  62. mlrun/datastore/__init__.py +59 -18
  63. mlrun/datastore/alibaba_oss.py +4 -1
  64. mlrun/datastore/azure_blob.py +4 -1
  65. mlrun/datastore/base.py +19 -24
  66. mlrun/datastore/datastore.py +10 -4
  67. mlrun/datastore/datastore_profile.py +178 -45
  68. mlrun/datastore/dbfs_store.py +4 -1
  69. mlrun/datastore/filestore.py +4 -1
  70. mlrun/datastore/google_cloud_storage.py +4 -1
  71. mlrun/datastore/hdfs.py +4 -1
  72. mlrun/datastore/inmem.py +4 -1
  73. mlrun/datastore/redis.py +4 -1
  74. mlrun/datastore/s3.py +14 -3
  75. mlrun/datastore/sources.py +89 -92
  76. mlrun/datastore/store_resources.py +7 -4
  77. mlrun/datastore/storeytargets.py +51 -16
  78. mlrun/datastore/targets.py +38 -31
  79. mlrun/datastore/utils.py +87 -4
  80. mlrun/datastore/v3io.py +4 -1
  81. mlrun/datastore/vectorstore.py +291 -0
  82. mlrun/datastore/wasbfs/fs.py +13 -12
  83. mlrun/db/base.py +286 -100
  84. mlrun/db/httpdb.py +1562 -490
  85. mlrun/db/nopdb.py +250 -83
  86. mlrun/errors.py +6 -2
  87. mlrun/execution.py +194 -50
  88. mlrun/feature_store/__init__.py +2 -10
  89. mlrun/feature_store/api.py +20 -458
  90. mlrun/feature_store/common.py +9 -9
  91. mlrun/feature_store/feature_set.py +20 -18
  92. mlrun/feature_store/feature_vector.py +105 -479
  93. mlrun/feature_store/feature_vector_utils.py +466 -0
  94. mlrun/feature_store/retrieval/base.py +15 -11
  95. mlrun/feature_store/retrieval/job.py +2 -1
  96. mlrun/feature_store/retrieval/storey_merger.py +1 -1
  97. mlrun/feature_store/steps.py +3 -3
  98. mlrun/features.py +30 -13
  99. mlrun/frameworks/__init__.py +1 -2
  100. mlrun/frameworks/_common/__init__.py +1 -2
  101. mlrun/frameworks/_common/artifacts_library.py +2 -2
  102. mlrun/frameworks/_common/mlrun_interface.py +10 -6
  103. mlrun/frameworks/_common/model_handler.py +31 -31
  104. mlrun/frameworks/_common/producer.py +3 -1
  105. mlrun/frameworks/_dl_common/__init__.py +1 -2
  106. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
  107. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
  108. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
  109. mlrun/frameworks/_ml_common/__init__.py +1 -2
  110. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
  111. mlrun/frameworks/_ml_common/model_handler.py +21 -21
  112. mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
  113. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
  114. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  115. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  116. mlrun/frameworks/auto_mlrun/__init__.py +1 -2
  117. mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
  118. mlrun/frameworks/huggingface/__init__.py +1 -2
  119. mlrun/frameworks/huggingface/model_server.py +9 -9
  120. mlrun/frameworks/lgbm/__init__.py +47 -44
  121. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
  122. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
  123. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
  124. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
  125. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
  126. mlrun/frameworks/lgbm/model_handler.py +15 -11
  127. mlrun/frameworks/lgbm/model_server.py +11 -7
  128. mlrun/frameworks/lgbm/utils.py +2 -2
  129. mlrun/frameworks/onnx/__init__.py +1 -2
  130. mlrun/frameworks/onnx/dataset.py +3 -3
  131. mlrun/frameworks/onnx/mlrun_interface.py +2 -2
  132. mlrun/frameworks/onnx/model_handler.py +7 -5
  133. mlrun/frameworks/onnx/model_server.py +8 -6
  134. mlrun/frameworks/parallel_coordinates.py +11 -11
  135. mlrun/frameworks/pytorch/__init__.py +22 -23
  136. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
  137. mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
  138. mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
  139. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
  140. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
  141. mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
  142. mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
  143. mlrun/frameworks/pytorch/model_handler.py +21 -17
  144. mlrun/frameworks/pytorch/model_server.py +13 -9
  145. mlrun/frameworks/sklearn/__init__.py +19 -18
  146. mlrun/frameworks/sklearn/estimator.py +2 -2
  147. mlrun/frameworks/sklearn/metric.py +3 -3
  148. mlrun/frameworks/sklearn/metrics_library.py +8 -6
  149. mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
  150. mlrun/frameworks/sklearn/model_handler.py +4 -3
  151. mlrun/frameworks/tf_keras/__init__.py +11 -12
  152. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
  153. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
  154. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
  155. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
  156. mlrun/frameworks/tf_keras/model_handler.py +17 -13
  157. mlrun/frameworks/tf_keras/model_server.py +12 -8
  158. mlrun/frameworks/xgboost/__init__.py +19 -18
  159. mlrun/frameworks/xgboost/model_handler.py +13 -9
  160. mlrun/k8s_utils.py +2 -5
  161. mlrun/launcher/base.py +3 -4
  162. mlrun/launcher/client.py +2 -2
  163. mlrun/launcher/local.py +6 -2
  164. mlrun/launcher/remote.py +1 -1
  165. mlrun/lists.py +8 -4
  166. mlrun/model.py +132 -46
  167. mlrun/model_monitoring/__init__.py +3 -5
  168. mlrun/model_monitoring/api.py +113 -98
  169. mlrun/model_monitoring/applications/__init__.py +0 -5
  170. mlrun/model_monitoring/applications/_application_steps.py +81 -50
  171. mlrun/model_monitoring/applications/base.py +467 -14
  172. mlrun/model_monitoring/applications/context.py +212 -134
  173. mlrun/model_monitoring/{db/stores/base → applications/evidently}/__init__.py +6 -2
  174. mlrun/model_monitoring/applications/evidently/base.py +146 -0
  175. mlrun/model_monitoring/applications/histogram_data_drift.py +89 -56
  176. mlrun/model_monitoring/applications/results.py +67 -15
  177. mlrun/model_monitoring/controller.py +701 -315
  178. mlrun/model_monitoring/db/__init__.py +0 -2
  179. mlrun/model_monitoring/db/_schedules.py +242 -0
  180. mlrun/model_monitoring/db/_stats.py +189 -0
  181. mlrun/model_monitoring/db/tsdb/__init__.py +33 -22
  182. mlrun/model_monitoring/db/tsdb/base.py +243 -49
  183. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +76 -36
  184. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
  185. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +213 -0
  186. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +534 -88
  187. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
  188. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +436 -106
  189. mlrun/model_monitoring/helpers.py +356 -114
  190. mlrun/model_monitoring/stream_processing.py +190 -345
  191. mlrun/model_monitoring/tracking_policy.py +11 -4
  192. mlrun/model_monitoring/writer.py +49 -90
  193. mlrun/package/__init__.py +3 -6
  194. mlrun/package/context_handler.py +2 -2
  195. mlrun/package/packager.py +12 -9
  196. mlrun/package/packagers/__init__.py +0 -2
  197. mlrun/package/packagers/default_packager.py +14 -11
  198. mlrun/package/packagers/numpy_packagers.py +16 -7
  199. mlrun/package/packagers/pandas_packagers.py +18 -18
  200. mlrun/package/packagers/python_standard_library_packagers.py +25 -11
  201. mlrun/package/packagers_manager.py +35 -32
  202. mlrun/package/utils/__init__.py +0 -3
  203. mlrun/package/utils/_pickler.py +6 -6
  204. mlrun/platforms/__init__.py +47 -16
  205. mlrun/platforms/iguazio.py +4 -1
  206. mlrun/projects/operations.py +30 -30
  207. mlrun/projects/pipelines.py +116 -47
  208. mlrun/projects/project.py +1292 -329
  209. mlrun/render.py +5 -9
  210. mlrun/run.py +57 -14
  211. mlrun/runtimes/__init__.py +1 -3
  212. mlrun/runtimes/base.py +30 -22
  213. mlrun/runtimes/daskjob.py +9 -9
  214. mlrun/runtimes/databricks_job/databricks_runtime.py +6 -5
  215. mlrun/runtimes/function_reference.py +5 -2
  216. mlrun/runtimes/generators.py +3 -2
  217. mlrun/runtimes/kubejob.py +6 -7
  218. mlrun/runtimes/mounts.py +574 -0
  219. mlrun/runtimes/mpijob/__init__.py +0 -2
  220. mlrun/runtimes/mpijob/abstract.py +7 -6
  221. mlrun/runtimes/nuclio/api_gateway.py +7 -7
  222. mlrun/runtimes/nuclio/application/application.py +11 -13
  223. mlrun/runtimes/nuclio/application/reverse_proxy.go +66 -64
  224. mlrun/runtimes/nuclio/function.py +127 -70
  225. mlrun/runtimes/nuclio/serving.py +105 -37
  226. mlrun/runtimes/pod.py +159 -54
  227. mlrun/runtimes/remotesparkjob.py +3 -2
  228. mlrun/runtimes/sparkjob/__init__.py +0 -2
  229. mlrun/runtimes/sparkjob/spark3job.py +22 -12
  230. mlrun/runtimes/utils.py +7 -6
  231. mlrun/secrets.py +2 -2
  232. mlrun/serving/__init__.py +8 -0
  233. mlrun/serving/merger.py +7 -5
  234. mlrun/serving/remote.py +35 -22
  235. mlrun/serving/routers.py +186 -240
  236. mlrun/serving/server.py +41 -10
  237. mlrun/serving/states.py +432 -118
  238. mlrun/serving/utils.py +13 -2
  239. mlrun/serving/v1_serving.py +3 -2
  240. mlrun/serving/v2_serving.py +161 -203
  241. mlrun/track/__init__.py +1 -1
  242. mlrun/track/tracker.py +2 -2
  243. mlrun/track/trackers/mlflow_tracker.py +6 -5
  244. mlrun/utils/async_http.py +35 -22
  245. mlrun/utils/clones.py +7 -4
  246. mlrun/utils/helpers.py +511 -58
  247. mlrun/utils/logger.py +119 -13
  248. mlrun/utils/notifications/notification/__init__.py +22 -19
  249. mlrun/utils/notifications/notification/base.py +39 -15
  250. mlrun/utils/notifications/notification/console.py +6 -6
  251. mlrun/utils/notifications/notification/git.py +11 -11
  252. mlrun/utils/notifications/notification/ipython.py +10 -9
  253. mlrun/utils/notifications/notification/mail.py +176 -0
  254. mlrun/utils/notifications/notification/slack.py +16 -8
  255. mlrun/utils/notifications/notification/webhook.py +24 -8
  256. mlrun/utils/notifications/notification_pusher.py +191 -200
  257. mlrun/utils/regex.py +12 -2
  258. mlrun/utils/version/version.json +2 -2
  259. {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/METADATA +81 -54
  260. mlrun-1.8.0.dist-info/RECORD +351 -0
  261. {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/WHEEL +1 -1
  262. mlrun/model_monitoring/applications/evidently_base.py +0 -137
  263. mlrun/model_monitoring/db/stores/__init__.py +0 -136
  264. mlrun/model_monitoring/db/stores/base/store.py +0 -213
  265. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -71
  266. mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -190
  267. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +0 -103
  268. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -40
  269. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +0 -659
  270. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +0 -726
  271. mlrun/model_monitoring/model_endpoint.py +0 -118
  272. mlrun-1.7.2rc3.dist-info/RECORD +0 -351
  273. {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/entry_points.txt +0 -0
  274. {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info/licenses}/LICENSE +0 -0
  275. {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/top_level.txt +0 -0
@@ -11,27 +11,22 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
-
15
- import enum
14
+ import abc
16
15
  import json
17
16
  from datetime import datetime
18
17
  from typing import Any, NamedTuple, Optional, TypeVar
19
18
 
20
- from pydantic import BaseModel, Extra, Field, constr, validator
19
+ from pydantic.v1 import BaseModel, Field, constr
21
20
 
22
21
  # TODO: remove the unused import below after `mlrun.datastore` and `mlrun.utils` usage is removed.
23
22
  # At the moment `make lint` fails if this is removed.
24
- import mlrun.common.model_monitoring
25
-
26
- from ..object import ObjectKind, ObjectSpec, ObjectStatus
23
+ from ..object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
24
+ from . import ModelEndpointSchema
27
25
  from .constants import (
28
26
  FQN_REGEX,
29
27
  MODEL_ENDPOINT_ID_PATTERN,
30
28
  PROJECT_PATTERN,
31
29
  EndpointType,
32
- EventFieldType,
33
- EventKeyMetrics,
34
- EventLiveStats,
35
30
  ModelEndpointMonitoringMetricType,
36
31
  ModelMonitoringMode,
37
32
  ResultKindApp,
@@ -41,83 +36,6 @@ from .constants import (
41
36
  Model = TypeVar("Model", bound=BaseModel)
42
37
 
43
38
 
44
- class ModelMonitoringStoreKinds:
45
- # TODO: do changes in examples & demos In 1.5.0 remove
46
- ENDPOINTS = "endpoints"
47
- EVENTS = "events"
48
-
49
-
50
- class ModelEndpointMetadata(BaseModel):
51
- project: constr(regex=PROJECT_PATTERN)
52
- uid: constr(regex=MODEL_ENDPOINT_ID_PATTERN)
53
- labels: Optional[dict] = {}
54
-
55
- class Config:
56
- extra = Extra.allow
57
-
58
- @classmethod
59
- def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
60
- """Create a `ModelEndpointMetadata` object from an endpoint dictionary
61
-
62
- :param endpoint_dict: Model endpoint dictionary.
63
- :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
64
- dictionary using json.loads().
65
- """
66
- if json_parse_values is None:
67
- json_parse_values = [EventFieldType.LABELS]
68
-
69
- return _mapping_attributes(
70
- model_class=cls,
71
- flattened_dictionary=endpoint_dict,
72
- json_parse_values=json_parse_values,
73
- )
74
-
75
-
76
- class ModelEndpointSpec(ObjectSpec):
77
- function_uri: Optional[str] = "" # <project_name>/<function_name>:<tag>
78
- model: Optional[str] = "" # <model_name>:<version>
79
- model_class: Optional[str] = ""
80
- model_uri: Optional[str] = ""
81
- feature_names: Optional[list[str]] = []
82
- label_names: Optional[list[str]] = []
83
- stream_path: Optional[str] = ""
84
- algorithm: Optional[str] = ""
85
- monitor_configuration: Optional[dict] = {}
86
- active: Optional[bool] = True
87
- monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled.value
88
-
89
- @classmethod
90
- def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
91
- """Create a `ModelEndpointSpec` object from an endpoint dictionary
92
-
93
- :param endpoint_dict: Model endpoint dictionary.
94
- :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
95
- dictionary using json.loads().
96
- """
97
- if json_parse_values is None:
98
- json_parse_values = [
99
- EventFieldType.FEATURE_NAMES,
100
- EventFieldType.LABEL_NAMES,
101
- EventFieldType.MONITOR_CONFIGURATION,
102
- ]
103
- return _mapping_attributes(
104
- model_class=cls,
105
- flattened_dictionary=endpoint_dict,
106
- json_parse_values=json_parse_values,
107
- )
108
-
109
- @validator("model_uri")
110
- @classmethod
111
- def validate_model_uri(cls, model_uri):
112
- """Validate that the model uri includes the required prefix"""
113
- prefix, uri = mlrun.datastore.parse_store_uri(model_uri)
114
- if prefix and prefix != mlrun.utils.helpers.StorePrefix.Model:
115
- return mlrun.datastore.get_store_uri(
116
- mlrun.utils.helpers.StorePrefix.Model, uri
117
- )
118
- return model_uri
119
-
120
-
121
39
  class Histogram(BaseModel):
122
40
  buckets: list[float]
123
41
  counts: list[int]
@@ -163,65 +81,117 @@ class Features(BaseModel):
163
81
  )
164
82
 
165
83
 
166
- class ModelEndpointStatus(ObjectStatus):
167
- feature_stats: Optional[dict] = {}
168
- current_stats: Optional[dict] = {}
169
- first_request: Optional[str] = ""
170
- last_request: Optional[str] = ""
171
- error_count: Optional[int] = 0
172
- drift_status: Optional[str] = ""
173
- drift_measures: Optional[dict] = {}
174
- metrics: Optional[dict[str, dict[str, Any]]] = {
175
- EventKeyMetrics.GENERIC: {
176
- EventLiveStats.LATENCY_AVG_1H: 0,
177
- EventLiveStats.PREDICTIONS_PER_SECOND: 0,
178
- }
179
- }
180
- features: Optional[list[Features]] = []
181
- children: Optional[list[str]] = []
182
- children_uids: Optional[list[str]] = []
183
- endpoint_type: Optional[EndpointType] = EndpointType.NODE_EP
184
- monitoring_feature_set_uri: Optional[str] = ""
185
- state: Optional[str] = ""
186
-
187
- class Config:
188
- extra = Extra.allow
84
+ class ModelEndpointParser(abc.ABC, BaseModel):
85
+ @classmethod
86
+ def json_parse_values(cls) -> list[str]:
87
+ return []
189
88
 
190
89
  @classmethod
191
- def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
192
- """Create a `ModelEndpointStatus` object from an endpoint dictionary
90
+ def from_flat_dict(
91
+ cls,
92
+ endpoint_dict: dict,
93
+ json_parse_values: Optional[list] = None,
94
+ validate: bool = True,
95
+ ) -> "ModelEndpointParser":
96
+ """Create a `ModelEndpointParser` object from an endpoint dictionary
193
97
 
194
98
  :param endpoint_dict: Model endpoint dictionary.
195
99
  :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
196
100
  dictionary using json.loads().
101
+ :param validate: Whether to validate the flattened dictionary.
102
+ Skip validation to optimize performance when it is safe to do so.
197
103
  """
198
104
  if json_parse_values is None:
199
- json_parse_values = [
200
- EventFieldType.FEATURE_STATS,
201
- EventFieldType.CURRENT_STATS,
202
- EventFieldType.DRIFT_MEASURES,
203
- EventFieldType.METRICS,
204
- EventFieldType.CHILDREN,
205
- EventFieldType.CHILDREN_UIDS,
206
- EventFieldType.ENDPOINT_TYPE,
207
- ]
105
+ json_parse_values = cls.json_parse_values()
106
+
208
107
  return _mapping_attributes(
209
108
  model_class=cls,
210
109
  flattened_dictionary=endpoint_dict,
211
110
  json_parse_values=json_parse_values,
111
+ validate=validate,
212
112
  )
213
113
 
214
114
 
115
+ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
116
+ project: constr(regex=PROJECT_PATTERN)
117
+ endpoint_type: EndpointType = EndpointType.NODE_EP
118
+ uid: Optional[constr(regex=MODEL_ENDPOINT_ID_PATTERN)]
119
+
120
+ @classmethod
121
+ def mutable_fields(cls):
122
+ return ["labels"]
123
+
124
+
125
+ class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
126
+ model_class: Optional[str] = ""
127
+ function_name: Optional[str] = ""
128
+ function_tag: Optional[str] = ""
129
+ model_path: Optional[str] = ""
130
+ model_name: Optional[str] = ""
131
+ model_tags: Optional[list[str]] = []
132
+ _model_id: Optional[int] = ""
133
+ feature_names: Optional[list[str]] = []
134
+ label_names: Optional[list[str]] = []
135
+ feature_stats: Optional[dict] = {}
136
+ function_uri: Optional[str] = "" # <project_name>/<function_hash>
137
+ model_uri: Optional[str] = ""
138
+ children: Optional[list[str]] = []
139
+ children_uids: Optional[list[str]] = []
140
+ monitoring_feature_set_uri: Optional[str] = ""
141
+
142
+ @classmethod
143
+ def mutable_fields(cls):
144
+ return [
145
+ "model_path",
146
+ "model_class",
147
+ "feature_names",
148
+ "label_names",
149
+ "children",
150
+ "children_uids",
151
+ ]
152
+
153
+
154
+ class ModelEndpointStatus(ObjectStatus, ModelEndpointParser):
155
+ state: Optional[str] = "unknown" # will be updated according to the function state
156
+ first_request: Optional[datetime] = None
157
+ monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled
158
+ sampling_percentage: Optional[float] = 100
159
+
160
+ # operative
161
+ last_request: Optional[datetime] = None
162
+ result_status: Optional[int] = -1
163
+ avg_latency: Optional[float] = None
164
+ error_count: Optional[int] = 0
165
+ current_stats: Optional[dict] = {}
166
+ current_stats_timestamp: Optional[datetime] = None
167
+ drift_measures: Optional[dict] = {}
168
+ drift_measures_timestamp: Optional[datetime] = None
169
+
170
+ @classmethod
171
+ def mutable_fields(cls):
172
+ return [
173
+ "monitoring_mode",
174
+ "first_request",
175
+ "last_request",
176
+ "sampling_percentage",
177
+ ]
178
+
179
+
215
180
  class ModelEndpoint(BaseModel):
216
181
  kind: ObjectKind = Field(ObjectKind.model_endpoint, const=True)
217
182
  metadata: ModelEndpointMetadata
218
- spec: ModelEndpointSpec = ModelEndpointSpec()
219
- status: ModelEndpointStatus = ModelEndpointStatus()
183
+ spec: ModelEndpointSpec
184
+ status: ModelEndpointStatus
220
185
 
221
- class Config:
222
- extra = Extra.allow
186
+ @classmethod
187
+ def mutable_fields(cls):
188
+ return (
189
+ ModelEndpointMetadata.mutable_fields()
190
+ + ModelEndpointSpec.mutable_fields()
191
+ + ModelEndpointStatus.mutable_fields()
192
+ )
223
193
 
224
- def flat_dict(self):
194
+ def flat_dict(self) -> dict[str, Any]:
225
195
  """Generate a flattened `ModelEndpoint` dictionary. The flattened dictionary result is important for storing
226
196
  the model endpoint object in the database.
227
197
 
@@ -229,54 +199,60 @@ class ModelEndpoint(BaseModel):
229
199
  """
230
200
  # Convert the ModelEndpoint object into a dictionary using BaseModel dict() function
231
201
  # In addition, remove the BaseModel kind as it is not required by the DB schema
232
- model_endpoint_dictionary = self.dict(exclude={"kind"})
233
202
 
203
+ model_endpoint_dictionary = self.dict(exclude={"kind"})
204
+ exclude = {
205
+ "tag",
206
+ ModelEndpointSchema.FEATURE_STATS,
207
+ ModelEndpointSchema.CURRENT_STATS,
208
+ ModelEndpointSchema.DRIFT_MEASURES,
209
+ ModelEndpointSchema.FUNCTION_URI,
210
+ }
234
211
  # Initialize a flattened dictionary that will be filled with the model endpoint dictionary attributes
235
212
  flatten_dict = {}
236
213
  for k_object in model_endpoint_dictionary:
237
214
  for key in model_endpoint_dictionary[k_object]:
238
- # Extract the value of the current field
239
- current_value = model_endpoint_dictionary[k_object][key]
240
-
241
- # If the value is not from type str or bool (e.g. dict), convert it into a JSON string
242
- # for matching the database required format
243
- if not isinstance(current_value, (str, bool, int)) or isinstance(
244
- current_value, enum.IntEnum
245
- ):
246
- flatten_dict[key] = json.dumps(current_value)
247
- else:
248
- flatten_dict[key] = current_value
249
-
250
- if EventFieldType.METRICS not in flatten_dict:
251
- # Initialize metrics dictionary
252
- flatten_dict[EventFieldType.METRICS] = {
253
- EventKeyMetrics.GENERIC: {
254
- EventLiveStats.LATENCY_AVG_1H: 0,
255
- EventLiveStats.PREDICTIONS_PER_SECOND: 0,
256
- }
257
- }
258
-
259
- # Remove the features from the dictionary as this field will be filled only within the feature analysis process
260
- flatten_dict.pop(EventFieldType.FEATURES, None)
215
+ if key not in exclude:
216
+ # Extract the value of the current field
217
+ flatten_dict[key] = model_endpoint_dictionary[k_object][key]
218
+
261
219
  return flatten_dict
262
220
 
263
221
  @classmethod
264
- def from_flat_dict(cls, endpoint_dict: dict) -> "ModelEndpoint":
222
+ def from_flat_dict(
223
+ cls, endpoint_dict: dict, validate: bool = True
224
+ ) -> "ModelEndpoint":
265
225
  """Create a `ModelEndpoint` object from an endpoint flattened dictionary. Because the provided dictionary
266
226
  is flattened, we pass it as is to the subclasses without splitting the keys into spec, metadata, and status.
267
227
 
268
228
  :param endpoint_dict: Model endpoint dictionary.
229
+ :param validate: Whether to validate the flattened dictionary.
230
+ Skip validation to optimize performance when it is safe to do so.
269
231
  """
270
232
 
271
233
  return cls(
272
- metadata=ModelEndpointMetadata.from_flat_dict(endpoint_dict=endpoint_dict),
273
- spec=ModelEndpointSpec.from_flat_dict(endpoint_dict=endpoint_dict),
274
- status=ModelEndpointStatus.from_flat_dict(endpoint_dict=endpoint_dict),
234
+ metadata=ModelEndpointMetadata.from_flat_dict(
235
+ endpoint_dict=endpoint_dict, validate=validate
236
+ ),
237
+ spec=ModelEndpointSpec.from_flat_dict(
238
+ endpoint_dict=endpoint_dict, validate=validate
239
+ ),
240
+ status=ModelEndpointStatus.from_flat_dict(
241
+ endpoint_dict=endpoint_dict, validate=validate
242
+ ),
243
+ )
244
+
245
+ def get(self, field, default=None):
246
+ return (
247
+ getattr(self.metadata, field, None)
248
+ or getattr(self.spec, field, None)
249
+ or getattr(self.status, field, None)
250
+ or default
275
251
  )
276
252
 
277
253
 
278
254
  class ModelEndpointList(BaseModel):
279
- endpoints: list[ModelEndpoint] = []
255
+ endpoints: list[ModelEndpoint]
280
256
 
281
257
 
282
258
  class ModelEndpointMonitoringMetric(BaseModel):
@@ -284,10 +260,17 @@ class ModelEndpointMonitoringMetric(BaseModel):
284
260
  app: str
285
261
  type: ModelEndpointMonitoringMetricType
286
262
  name: str
287
- full_name: str
263
+ full_name: Optional[str] = None
264
+ kind: Optional[ResultKindApp] = None
265
+
266
+ def __init__(self, **kwargs):
267
+ super().__init__(**kwargs)
268
+ self.full_name = compose_full_name(
269
+ project=self.project, app=self.app, name=self.name, type=self.type
270
+ )
288
271
 
289
272
 
290
- def _compose_full_name(
273
+ def compose_full_name(
291
274
  *,
292
275
  project: str,
293
276
  app: str,
@@ -315,6 +298,7 @@ class _ResultPoint(NamedTuple):
315
298
  timestamp: datetime
316
299
  value: float
317
300
  status: ResultStatusApp
301
+ extra_data: Optional[str] = ""
318
302
 
319
303
 
320
304
  class _ModelEndpointMonitoringMetricValuesBase(BaseModel):
@@ -343,7 +327,10 @@ class ModelEndpointMonitoringMetricNoData(_ModelEndpointMonitoringMetricValuesBa
343
327
 
344
328
 
345
329
  def _mapping_attributes(
346
- model_class: type[Model], flattened_dictionary: dict, json_parse_values: list
330
+ model_class: type[Model],
331
+ flattened_dictionary: dict,
332
+ json_parse_values: list,
333
+ validate: bool = True,
347
334
  ) -> Model:
348
335
  """Generate a `BaseModel` object with the provided dictionary attributes.
349
336
 
@@ -351,8 +338,10 @@ def _mapping_attributes(
351
338
  :param flattened_dictionary: Flattened dictionary that contains the model endpoint attributes.
352
339
  :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
353
340
  dictionary using json.loads().
341
+ :param validate: Whether to validate the flattened dictionary.
342
+ Skip validation to optimize performance when it is safe to do so.
354
343
  """
355
- # Get the fields of the provided base model object. These fields will be used to filter to relevent keys
344
+ # Get the fields of the provided base model object. These fields will be used to filter to relevant keys
356
345
  # from the flattened dictionary.
357
346
  wanted_keys = model_class.__fields__.keys()
358
347
 
@@ -365,10 +354,15 @@ def _mapping_attributes(
365
354
  dict_to_parse[field_key] = _json_loads_if_not_none(
366
355
  flattened_dictionary[field_key]
367
356
  )
368
- else:
357
+ elif flattened_dictionary[field_key] != "null":
369
358
  dict_to_parse[field_key] = flattened_dictionary[field_key]
359
+ else:
360
+ dict_to_parse[field_key] = None
361
+
362
+ if validate:
363
+ return model_class.parse_obj(dict_to_parse)
370
364
 
371
- return model_class.parse_obj(dict_to_parse)
365
+ return model_class.construct(**dict_to_parse)
372
366
 
373
367
 
374
368
  def _json_loads_if_not_none(field: Any) -> Any:
@@ -15,8 +15,9 @@
15
15
  import datetime
16
16
  import enum
17
17
  import typing
18
+ from typing import Optional
18
19
 
19
- import pydantic
20
+ import pydantic.v1
20
21
 
21
22
  import mlrun.common.types
22
23
 
@@ -45,6 +46,13 @@ class NotificationKind(mlrun.common.types.StrEnum):
45
46
  slack: str = "slack"
46
47
  """**webhook** - The slack webhook to which to send the notification."""
47
48
 
49
+ mail: str = "mail"
50
+ """
51
+ **email_addresses** - The target mails\n
52
+ **subject** - The subject of the mail\n
53
+ **body** - The body of the mail\n
54
+ """
55
+
48
56
  webhook: str = "webhook"
49
57
  """
50
58
  **url** - The webhook url to which to send the notification.\n
@@ -86,7 +94,7 @@ class NotificationLimits(enum.Enum):
86
94
  ) # 900KB (k8s secret size limit is 1MB minus buffer for metadata)
87
95
 
88
96
 
89
- class Notification(pydantic.BaseModel):
97
+ class Notification(pydantic.v1.BaseModel):
90
98
  """
91
99
  Notification object schema
92
100
 
@@ -120,5 +128,18 @@ class Notification(pydantic.BaseModel):
120
128
  reason: typing.Optional[str] = None
121
129
 
122
130
 
123
- class SetNotificationRequest(pydantic.BaseModel):
131
+ class SetNotificationRequest(pydantic.v1.BaseModel):
124
132
  notifications: list[Notification] = None
133
+
134
+
135
+ class NotificationSummary(pydantic.v1.BaseModel):
136
+ failed: int = 0
137
+ succeeded: int = 0
138
+
139
+
140
+ class NotificationState(pydantic.v1.BaseModel):
141
+ kind: str
142
+ err: Optional[
143
+ str
144
+ ] # empty error means that the notifications were sent successfully
145
+ summary: NotificationSummary
@@ -15,7 +15,7 @@
15
15
  from datetime import datetime
16
16
  from typing import Optional
17
17
 
18
- from pydantic import BaseModel, Extra
18
+ from pydantic.v1 import BaseModel, Extra
19
19
 
20
20
  import mlrun.common.types
21
21
 
@@ -14,13 +14,13 @@
14
14
 
15
15
  import typing
16
16
 
17
- import pydantic
17
+ import pydantic.v1
18
18
 
19
19
 
20
- class PaginationInfo(pydantic.BaseModel):
20
+ class PaginationInfo(pydantic.v1.BaseModel):
21
21
  class Config:
22
22
  allow_population_by_field_name = True
23
23
 
24
24
  page: typing.Optional[int]
25
- page_size: typing.Optional[int] = pydantic.Field(alias="page-size")
26
- page_token: typing.Optional[str] = pydantic.Field(alias="page-token")
25
+ page_size: typing.Optional[int] = pydantic.v1.Field(alias="page-size")
26
+ page_token: typing.Optional[str] = pydantic.v1.Field(alias="page-token")
@@ -0,0 +1,142 @@
1
+ # Copyright 2024 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ from datetime import datetime, timedelta
16
+
17
+ from mlrun.common.types import StrEnum
18
+
19
+
20
+ class PartitionInterval(StrEnum):
21
+ DAY = "DAY"
22
+ MONTH = "MONTH"
23
+ YEARWEEK = "YEARWEEK"
24
+
25
+ @classmethod
26
+ def is_valid(cls, value: str) -> bool:
27
+ return value in cls._value2member_map_
28
+
29
+ @classmethod
30
+ def valid_intervals(cls) -> list:
31
+ return list(cls._value2member_map_.keys())
32
+
33
+ def as_duration(self) -> timedelta:
34
+ """
35
+ Convert the partition interval to a duration-like timedelta.
36
+
37
+ Returns:
38
+ timedelta: A duration representing the partition interval.
39
+ """
40
+ if self == PartitionInterval.DAY:
41
+ return timedelta(days=1)
42
+ elif self == PartitionInterval.MONTH:
43
+ # Approximate a month as 30 days
44
+ return timedelta(days=30)
45
+ elif self == PartitionInterval.YEARWEEK:
46
+ return timedelta(weeks=1)
47
+
48
+ @classmethod
49
+ def from_expression(cls, partition_expression: str):
50
+ """
51
+ Returns the corresponding PartitionInterval for a given partition expression,
52
+ or None if the function is not mapped.
53
+
54
+ :param partition_expression: The partition expression to map to an interval.
55
+ :return: PartitionInterval corresponding to the expression, or `month` if no match is found.
56
+ """
57
+
58
+ # Match the provided function string to the correct interval
59
+ partition_expression = partition_expression.upper()
60
+ if "YEARWEEK" in partition_expression:
61
+ return cls.YEARWEEK
62
+ elif "DAYOFMONTH" in partition_expression:
63
+ return cls.DAY
64
+ else:
65
+ return cls.MONTH
66
+
67
+ def get_partition_info(
68
+ self,
69
+ start_datetime: datetime,
70
+ partition_number: int = 1,
71
+ ) -> list[tuple[str, str]]:
72
+ """
73
+ Generates partition details for a specified number of partitions starting from a given datetime.
74
+
75
+ :param start_datetime: The starting datetime used for generating partition details.
76
+ :param partition_number: The number of partitions to generate details for.
77
+
78
+ :return: A list of tuples:
79
+ - partition_name: The name for the partition.
80
+ - partition_value: The "LESS THAN" value for the next partition boundary.
81
+ """
82
+ partitioning_information_list = []
83
+ current_datetime = start_datetime
84
+
85
+ for _ in range(partition_number):
86
+ partition_name = self.get_partition_name(current_datetime)
87
+ partition_boundary_date = self.get_next_partition_time(current_datetime)
88
+ partition_value = self.get_partition_name(partition_boundary_date)
89
+ partitioning_information_list.append((partition_name, partition_value))
90
+
91
+ # Move to the next interval
92
+ current_datetime = partition_boundary_date
93
+
94
+ return partitioning_information_list
95
+
96
+ def get_next_partition_time(self, current_datetime: datetime) -> datetime:
97
+ """
98
+ Calculates the next partition boundary time based on the specified partition interval.
99
+ :param current_datetime: The current datetime from which the next interval is calculated.
100
+
101
+ :return: A datetime object representing the start of the next partition interval.
102
+ - If the interval is DAY, it advances by one day.
103
+ - If the interval is MONTH, it advances to the first day of the next month.
104
+ - If the interval is YEARWEEK, it advances by one week.
105
+ """
106
+ if self == PartitionInterval.DAY:
107
+ return current_datetime + timedelta(days=1)
108
+ elif self == PartitionInterval.MONTH:
109
+ return (current_datetime.replace(day=1) + timedelta(days=32)).replace(day=1)
110
+ elif self == PartitionInterval.YEARWEEK:
111
+ return current_datetime + timedelta(weeks=1)
112
+
113
+ def get_partition_name(self, current_datetime: datetime) -> str:
114
+ if self == PartitionInterval.DAY:
115
+ return current_datetime.strftime("%Y%m%d")
116
+ elif self == PartitionInterval.MONTH:
117
+ return current_datetime.strftime("%Y%m")
118
+ elif self == PartitionInterval.YEARWEEK:
119
+ year, week, _ = current_datetime.isocalendar()
120
+ return f"{year}{week:02d}"
121
+
122
+ def get_partition_expression(self, column_name: str):
123
+ if self == PartitionInterval.YEARWEEK:
124
+ return f"YEARWEEK({column_name}, 1)"
125
+ elif self == PartitionInterval.DAY:
126
+ # generates value in format %Y%m%d in mysql
127
+ # mysql query example: `select YEAR(NOW())*10000 + MONTH(NOW())*100 + DAY(NOW());`
128
+ return f"YEAR({column_name}) * 10000 + MONTH({column_name}) * 100 + DAY({column_name})"
129
+ elif self == PartitionInterval.MONTH:
130
+ # generates value in format %Y%m in mysql
131
+ # mysql query example: `select YEAR(NOW())*100 + MONTH(NOW());`
132
+ return f"YEAR({column_name}) * 100 + MONTH({column_name})"
133
+
134
+ def get_number_of_partitions(self, days: int) -> int:
135
+ # Calculate the number partitions based on given number of days
136
+ if self == PartitionInterval.DAY:
137
+ return days
138
+ elif self == PartitionInterval.MONTH:
139
+ # Average number days in a month is 30.44
140
+ return int(days / 30.44)
141
+ elif self == PartitionInterval.YEARWEEK:
142
+ return int(days / 7)
@@ -14,7 +14,7 @@
14
14
  #
15
15
  import typing
16
16
 
17
- import pydantic
17
+ import pydantic.v1
18
18
  from deprecated import deprecated
19
19
 
20
20
  import mlrun.common.types
@@ -22,7 +22,7 @@ import mlrun.common.types
22
22
 
23
23
  @deprecated(
24
24
  version="1.7.0",
25
- reason="mlrun.common.schemas.PipelinesFormat is deprecated and will be removed in 1.9.0. "
25
+ reason="mlrun.common.schemas.PipelinesFormat is deprecated and will be removed in 1.10.0. "
26
26
  "Use mlrun.common.formatters.PipelineFormat instead.",
27
27
  category=FutureWarning,
28
28
  )
@@ -39,7 +39,7 @@ class PipelinesPagination(str):
39
39
  max_page_size = 200
40
40
 
41
41
 
42
- class PipelinesOutput(pydantic.BaseModel):
42
+ class PipelinesOutput(pydantic.v1.BaseModel):
43
43
  # use the format query param to control what is returned
44
44
  runs: list[typing.Union[dict, str]]
45
45
  total_size: int