mlrun 1.8.0rc4__py3-none-any.whl → 1.8.0rc5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +4 -3
- mlrun/alerts/alert.py +129 -2
- mlrun/common/schemas/alert.py +3 -0
- mlrun/common/schemas/artifact.py +4 -0
- mlrun/db/factory.py +0 -3
- mlrun/db/httpdb.py +6 -2
- mlrun/execution.py +32 -25
- mlrun/model.py +4 -0
- mlrun/model_monitoring/applications/_application_steps.py +1 -1
- mlrun/model_monitoring/applications/base.py +65 -6
- mlrun/model_monitoring/applications/context.py +49 -16
- mlrun/model_monitoring/applications/evidently_base.py +0 -1
- mlrun/model_monitoring/applications/histogram_data_drift.py +2 -6
- mlrun/projects/project.py +24 -19
- mlrun/utils/notifications/notification/mail.py +20 -4
- mlrun/utils/notifications/notification_pusher.py +34 -13
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.8.0rc4.dist-info → mlrun-1.8.0rc5.dist-info}/METADATA +5 -4
- {mlrun-1.8.0rc4.dist-info → mlrun-1.8.0rc5.dist-info}/RECORD +23 -23
- {mlrun-1.8.0rc4.dist-info → mlrun-1.8.0rc5.dist-info}/LICENSE +0 -0
- {mlrun-1.8.0rc4.dist-info → mlrun-1.8.0rc5.dist-info}/WHEEL +0 -0
- {mlrun-1.8.0rc4.dist-info → mlrun-1.8.0rc5.dist-info}/entry_points.txt +0 -0
- {mlrun-1.8.0rc4.dist-info → mlrun-1.8.0rc5.dist-info}/top_level.txt +0 -0
mlrun/__init__.py
CHANGED
|
@@ -39,6 +39,7 @@ from .execution import MLClientCtx
|
|
|
39
39
|
from .model import RunObject, RunTemplate, new_task
|
|
40
40
|
from .package import ArtifactType, DefaultPackager, Packager, handler
|
|
41
41
|
from .projects import (
|
|
42
|
+
MlrunProject,
|
|
42
43
|
ProjectMetadata,
|
|
43
44
|
build_function,
|
|
44
45
|
deploy_function,
|
|
@@ -162,10 +163,10 @@ def set_environment(
|
|
|
162
163
|
return mlconf.default_project, mlconf.artifact_path
|
|
163
164
|
|
|
164
165
|
|
|
165
|
-
def get_current_project(silent=False):
|
|
166
|
+
def get_current_project(silent: bool = False) -> Optional[MlrunProject]:
|
|
166
167
|
if not pipeline_context.project and not silent:
|
|
167
168
|
raise MLRunInvalidArgumentError(
|
|
168
|
-
"current project is
|
|
169
|
+
"No current project is initialized. Use new, get or load project functions first."
|
|
169
170
|
)
|
|
170
171
|
return pipeline_context.project
|
|
171
172
|
|
|
@@ -182,7 +183,7 @@ def get_sample_path(subpath=""):
|
|
|
182
183
|
return samples_path
|
|
183
184
|
|
|
184
185
|
|
|
185
|
-
def set_env_from_file(env_file: str, return_dict: bool = False):
|
|
186
|
+
def set_env_from_file(env_file: str, return_dict: bool = False) -> Optional[dict]:
|
|
186
187
|
"""Read and set and/or return environment variables from a file
|
|
187
188
|
the env file should have lines in the form KEY=VALUE, comment line start with "#"
|
|
188
189
|
|
mlrun/alerts/alert.py
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
from datetime import datetime
|
|
15
15
|
from typing import Optional, Union
|
|
16
16
|
|
|
17
17
|
import mlrun
|
|
@@ -30,6 +30,7 @@ class AlertConfig(ModelObj):
|
|
|
30
30
|
"state",
|
|
31
31
|
"count",
|
|
32
32
|
"created",
|
|
33
|
+
"updated",
|
|
33
34
|
]
|
|
34
35
|
_fields_to_serialize = ModelObj._fields_to_serialize + [
|
|
35
36
|
"entities",
|
|
@@ -55,6 +56,7 @@ class AlertConfig(ModelObj):
|
|
|
55
56
|
state: alert_objects.AlertActiveState = None,
|
|
56
57
|
created: Optional[str] = None,
|
|
57
58
|
count: Optional[int] = None,
|
|
59
|
+
updated: Optional[str] = None,
|
|
58
60
|
):
|
|
59
61
|
"""Alert config object
|
|
60
62
|
|
|
@@ -118,6 +120,7 @@ class AlertConfig(ModelObj):
|
|
|
118
120
|
:param state: State of the alert, may be active/inactive (user should not supply it)
|
|
119
121
|
:param created: When the alert is created (user should not supply it)
|
|
120
122
|
:param count: Internal counter of the alert (user should not supply it)
|
|
123
|
+
:param updated: The last update time of the alert (user should not supply it)
|
|
121
124
|
"""
|
|
122
125
|
self.project = project
|
|
123
126
|
self.name = name
|
|
@@ -131,12 +134,39 @@ class AlertConfig(ModelObj):
|
|
|
131
134
|
self.entities = entities
|
|
132
135
|
self.id = id
|
|
133
136
|
self.state = state
|
|
134
|
-
self.
|
|
137
|
+
self._created = created
|
|
135
138
|
self.count = count
|
|
139
|
+
self._updated = updated
|
|
136
140
|
|
|
137
141
|
if template:
|
|
138
142
|
self._apply_template(template)
|
|
139
143
|
|
|
144
|
+
@property
|
|
145
|
+
def created(self) -> datetime:
|
|
146
|
+
"""
|
|
147
|
+
Get the `created` field as a datetime object.
|
|
148
|
+
"""
|
|
149
|
+
if isinstance(self._created, str):
|
|
150
|
+
return datetime.fromisoformat(self._created)
|
|
151
|
+
return self._created
|
|
152
|
+
|
|
153
|
+
@created.setter
|
|
154
|
+
def created(self, created):
|
|
155
|
+
self._created = created
|
|
156
|
+
|
|
157
|
+
@property
|
|
158
|
+
def updated(self) -> datetime:
|
|
159
|
+
"""
|
|
160
|
+
Get the `updated` field as a datetime object.
|
|
161
|
+
"""
|
|
162
|
+
if isinstance(self._updated, str):
|
|
163
|
+
return datetime.fromisoformat(self._updated)
|
|
164
|
+
return self._updated
|
|
165
|
+
|
|
166
|
+
@updated.setter
|
|
167
|
+
def updated(self, updated):
|
|
168
|
+
self._updated = updated
|
|
169
|
+
|
|
140
170
|
def validate_required_fields(self):
|
|
141
171
|
if not self.name:
|
|
142
172
|
raise mlrun.errors.MLRunInvalidArgumentError("Alert name must be provided")
|
|
@@ -253,3 +283,100 @@ class AlertConfig(ModelObj):
|
|
|
253
283
|
self.criteria = self.criteria or template.criteria
|
|
254
284
|
self.trigger = self.trigger or template.trigger
|
|
255
285
|
self.reset_policy = self.reset_policy or template.reset_policy
|
|
286
|
+
|
|
287
|
+
def list_activations(
|
|
288
|
+
self,
|
|
289
|
+
since: Optional[datetime] = None,
|
|
290
|
+
until: Optional[datetime] = None,
|
|
291
|
+
from_last_update: bool = False,
|
|
292
|
+
) -> list[mlrun.common.schemas.alert.AlertActivation]:
|
|
293
|
+
"""
|
|
294
|
+
Retrieve a list of all alert activations.
|
|
295
|
+
|
|
296
|
+
:param since: Filters for alert activations occurring after this timestamp.
|
|
297
|
+
:param until: Filters for alert activations occurring before this timestamp.
|
|
298
|
+
:param from_last_update: If set to True, retrieves alert activations since the alert's last update time.
|
|
299
|
+
if both since and from_last_update=True are provided, from_last_update takes precedence
|
|
300
|
+
and the since value will be overridden by the alert's last update timestamp.
|
|
301
|
+
|
|
302
|
+
:returns: A list of alert activations matching the provided filters.
|
|
303
|
+
"""
|
|
304
|
+
db = mlrun.get_run_db()
|
|
305
|
+
if from_last_update and self._updated:
|
|
306
|
+
since = self.updated
|
|
307
|
+
|
|
308
|
+
return db.list_alert_activations(
|
|
309
|
+
project=self.project,
|
|
310
|
+
name=self.name,
|
|
311
|
+
since=since,
|
|
312
|
+
until=until,
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
def paginated_list_activations(
|
|
316
|
+
self,
|
|
317
|
+
*args,
|
|
318
|
+
page: Optional[int] = None,
|
|
319
|
+
page_size: Optional[int] = None,
|
|
320
|
+
page_token: Optional[str] = None,
|
|
321
|
+
from_last_update: bool = False,
|
|
322
|
+
**kwargs,
|
|
323
|
+
) -> tuple[mlrun.common.schemas.alert.AlertActivation, Optional[str]]:
|
|
324
|
+
"""
|
|
325
|
+
List alerts activations with support for pagination and various filtering options.
|
|
326
|
+
|
|
327
|
+
This method retrieves a paginated list of alert activations based on the specified filter parameters.
|
|
328
|
+
Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
|
|
329
|
+
will return a list of alert activations that match the filtering criteria provided.
|
|
330
|
+
|
|
331
|
+
For detailed information about the parameters, refer to the list_activations method:
|
|
332
|
+
See :py:func:`~list_activations` for more details.
|
|
333
|
+
|
|
334
|
+
Examples::
|
|
335
|
+
|
|
336
|
+
# Fetch first page of alert activations with page size of 5
|
|
337
|
+
alert_activations, token = alert_config.paginated_list_activations(page_size=5)
|
|
338
|
+
# Fetch next page using the pagination token from the previous response
|
|
339
|
+
alert_activations, token = alert_config.paginated_list_activations(
|
|
340
|
+
page_token=token
|
|
341
|
+
)
|
|
342
|
+
# Fetch alert activations for a specific page (e.g., page 3)
|
|
343
|
+
alert_activations, token = alert_config.paginated_list_activations(
|
|
344
|
+
page=3, page_size=5
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
# Automatically iterate over all pages without explicitly specifying the page number
|
|
348
|
+
alert_activations = []
|
|
349
|
+
token = None
|
|
350
|
+
while True:
|
|
351
|
+
page_alert_activations, token = alert_config.paginated_list_activations(
|
|
352
|
+
page_token=token, page_size=5
|
|
353
|
+
)
|
|
354
|
+
alert_activations.extend(page_alert_activations)
|
|
355
|
+
|
|
356
|
+
# If token is None and page_alert_activations is empty, we've reached the end (no more activations).
|
|
357
|
+
# If token is None and page_alert_activations is not empty, we've fetched the last page of activations.
|
|
358
|
+
if not token:
|
|
359
|
+
break
|
|
360
|
+
print(f"Total alert activations retrieved: {len(alert_activations)}")
|
|
361
|
+
|
|
362
|
+
:param page: The page number to retrieve. If not provided, the next page will be retrieved.
|
|
363
|
+
:param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
|
|
364
|
+
:param page_token: A pagination token used to retrieve the next page of results. Should not be provided
|
|
365
|
+
for the first request.
|
|
366
|
+
:param from_last_update: If set to True, retrieves alert activations since the alert's last update time.
|
|
367
|
+
|
|
368
|
+
:returns: A tuple containing the list of alert activations and an optional `page_token` for pagination.
|
|
369
|
+
"""
|
|
370
|
+
if from_last_update and self._updated:
|
|
371
|
+
kwargs["since"] = self.updated
|
|
372
|
+
|
|
373
|
+
db = mlrun.get_run_db()
|
|
374
|
+
return db.paginated_list_alert_activations(
|
|
375
|
+
*args,
|
|
376
|
+
project=self.project,
|
|
377
|
+
name=self.name,
|
|
378
|
+
page=page,
|
|
379
|
+
page_size=page_size,
|
|
380
|
+
page_token=page_token,
|
|
381
|
+
**kwargs,
|
|
382
|
+
)
|
mlrun/common/schemas/alert.py
CHANGED
|
@@ -156,6 +156,7 @@ class AlertConfig(pydantic.v1.BaseModel):
|
|
|
156
156
|
notifications: pydantic.v1.conlist(AlertNotification, min_items=1)
|
|
157
157
|
state: AlertActiveState = AlertActiveState.INACTIVE
|
|
158
158
|
count: Optional[int] = 0
|
|
159
|
+
updated: datetime = None
|
|
159
160
|
|
|
160
161
|
def get_raw_notifications(self) -> list[notification_objects.Notification]:
|
|
161
162
|
return [
|
|
@@ -203,6 +204,7 @@ class AlertTemplate(
|
|
|
203
204
|
|
|
204
205
|
|
|
205
206
|
class AlertActivation(pydantic.v1.BaseModel):
|
|
207
|
+
id: int
|
|
206
208
|
name: str
|
|
207
209
|
project: str
|
|
208
210
|
severity: AlertSeverity
|
|
@@ -213,3 +215,4 @@ class AlertActivation(pydantic.v1.BaseModel):
|
|
|
213
215
|
event_kind: EventKind
|
|
214
216
|
number_of_events: int
|
|
215
217
|
notifications: list[notification_objects.NotificationState]
|
|
218
|
+
reset_time: Optional[datetime] = None
|
mlrun/common/schemas/artifact.py
CHANGED
|
@@ -25,6 +25,7 @@ from .object import ObjectStatus
|
|
|
25
25
|
class ArtifactCategories(mlrun.common.types.StrEnum):
|
|
26
26
|
model = "model"
|
|
27
27
|
dataset = "dataset"
|
|
28
|
+
document = "document"
|
|
28
29
|
other = "other"
|
|
29
30
|
|
|
30
31
|
# we define the link as a category to prevent import cycles, but it's not a real category
|
|
@@ -38,11 +39,14 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
|
|
|
38
39
|
return [ArtifactCategories.model.value, link_kind], False
|
|
39
40
|
if self.value == ArtifactCategories.dataset.value:
|
|
40
41
|
return [ArtifactCategories.dataset.value, link_kind], False
|
|
42
|
+
if self.value == ArtifactCategories.document.value:
|
|
43
|
+
return [ArtifactCategories.document.value, link_kind], False
|
|
41
44
|
if self.value == ArtifactCategories.other.value:
|
|
42
45
|
return (
|
|
43
46
|
[
|
|
44
47
|
ArtifactCategories.model.value,
|
|
45
48
|
ArtifactCategories.dataset.value,
|
|
49
|
+
ArtifactCategories.document.value,
|
|
46
50
|
],
|
|
47
51
|
True,
|
|
48
52
|
)
|
mlrun/db/factory.py
CHANGED
|
@@ -54,9 +54,6 @@ class RunDBFactory(
|
|
|
54
54
|
self._run_db = self._rundb_container.nop(url)
|
|
55
55
|
|
|
56
56
|
else:
|
|
57
|
-
# TODO: this practically makes the SQLRunDB a singleton, which mean that its session is shared, needs
|
|
58
|
-
# to be refreshed frequently and cannot be used concurrently.
|
|
59
|
-
# The SQLRunDB should always get its session from the FastAPI dependency injection.
|
|
60
57
|
self._run_db = self._rundb_container.run_db(url)
|
|
61
58
|
|
|
62
59
|
self._run_db.connect(secrets=secrets)
|
mlrun/db/httpdb.py
CHANGED
|
@@ -5074,7 +5074,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
5074
5074
|
until: Optional[datetime] = None,
|
|
5075
5075
|
entity: Optional[str] = None,
|
|
5076
5076
|
severity: Optional[
|
|
5077
|
-
|
|
5077
|
+
Union[
|
|
5078
|
+
mlrun.common.schemas.alert.AlertSeverity,
|
|
5079
|
+
str,
|
|
5080
|
+
list[Union[mlrun.common.schemas.alert.AlertSeverity, str]],
|
|
5081
|
+
]
|
|
5078
5082
|
] = None,
|
|
5079
5083
|
entity_kind: Optional[
|
|
5080
5084
|
Union[mlrun.common.schemas.alert.EventEntityKind, str]
|
|
@@ -5091,7 +5095,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
5091
5095
|
"since": datetime_to_iso(since),
|
|
5092
5096
|
"until": datetime_to_iso(until),
|
|
5093
5097
|
"entity": entity,
|
|
5094
|
-
"severity": severity,
|
|
5098
|
+
"severity": mlrun.utils.helpers.as_list(severity) if severity else None,
|
|
5095
5099
|
"entity-kind": entity_kind,
|
|
5096
5100
|
"event-kind": event_kind,
|
|
5097
5101
|
"page": page,
|
mlrun/execution.py
CHANGED
|
@@ -17,7 +17,7 @@ import os
|
|
|
17
17
|
import uuid
|
|
18
18
|
import warnings
|
|
19
19
|
from copy import deepcopy
|
|
20
|
-
from typing import Optional, Union
|
|
20
|
+
from typing import Optional, Union, cast
|
|
21
21
|
|
|
22
22
|
import numpy as np
|
|
23
23
|
import yaml
|
|
@@ -42,6 +42,7 @@ from .features import Feature
|
|
|
42
42
|
from .model import HyperParamOptions
|
|
43
43
|
from .secrets import SecretsStore
|
|
44
44
|
from .utils import (
|
|
45
|
+
Logger,
|
|
45
46
|
RunKeys,
|
|
46
47
|
dict_to_json,
|
|
47
48
|
dict_to_yaml,
|
|
@@ -158,7 +159,7 @@ class MLClientCtx:
|
|
|
158
159
|
return self._project
|
|
159
160
|
|
|
160
161
|
@property
|
|
161
|
-
def logger(self):
|
|
162
|
+
def logger(self) -> Logger:
|
|
162
163
|
"""Built-in logger interface
|
|
163
164
|
|
|
164
165
|
Example::
|
|
@@ -628,7 +629,7 @@ class MLClientCtx:
|
|
|
628
629
|
format=None,
|
|
629
630
|
db_key=None,
|
|
630
631
|
**kwargs,
|
|
631
|
-
):
|
|
632
|
+
) -> Artifact:
|
|
632
633
|
"""Log an output artifact and optionally upload it to datastore
|
|
633
634
|
|
|
634
635
|
Example::
|
|
@@ -698,7 +699,7 @@ class MLClientCtx:
|
|
|
698
699
|
extra_data=None,
|
|
699
700
|
label_column: Optional[str] = None,
|
|
700
701
|
**kwargs,
|
|
701
|
-
):
|
|
702
|
+
) -> DatasetArtifact:
|
|
702
703
|
"""Log a dataset artifact and optionally upload it to datastore
|
|
703
704
|
|
|
704
705
|
If the dataset exists with the same key and tag, it will be overwritten.
|
|
@@ -736,7 +737,7 @@ class MLClientCtx:
|
|
|
736
737
|
:param db_key: The key to use in the artifact DB table, by default its run name + '_' + key
|
|
737
738
|
db_key=False will not register it in the artifacts table
|
|
738
739
|
|
|
739
|
-
:returns:
|
|
740
|
+
:returns: Dataset artifact object
|
|
740
741
|
"""
|
|
741
742
|
ds = DatasetArtifact(
|
|
742
743
|
key,
|
|
@@ -749,16 +750,19 @@ class MLClientCtx:
|
|
|
749
750
|
**kwargs,
|
|
750
751
|
)
|
|
751
752
|
|
|
752
|
-
item =
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
753
|
+
item = cast(
|
|
754
|
+
DatasetArtifact,
|
|
755
|
+
self._artifacts_manager.log_artifact(
|
|
756
|
+
self,
|
|
757
|
+
ds,
|
|
758
|
+
local_path=local_path,
|
|
759
|
+
artifact_path=extend_artifact_path(artifact_path, self.artifact_path),
|
|
760
|
+
target_path=target_path,
|
|
761
|
+
tag=tag,
|
|
762
|
+
upload=upload,
|
|
763
|
+
db_key=db_key,
|
|
764
|
+
labels=labels,
|
|
765
|
+
),
|
|
762
766
|
)
|
|
763
767
|
self._update_run()
|
|
764
768
|
return item
|
|
@@ -786,7 +790,7 @@ class MLClientCtx:
|
|
|
786
790
|
extra_data=None,
|
|
787
791
|
db_key=None,
|
|
788
792
|
**kwargs,
|
|
789
|
-
):
|
|
793
|
+
) -> ModelArtifact:
|
|
790
794
|
"""Log a model artifact and optionally upload it to datastore
|
|
791
795
|
|
|
792
796
|
Example::
|
|
@@ -828,7 +832,7 @@ class MLClientCtx:
|
|
|
828
832
|
:param db_key: The key to use in the artifact DB table, by default its run name + '_' + key
|
|
829
833
|
db_key=False will not register it in the artifacts table
|
|
830
834
|
|
|
831
|
-
:returns:
|
|
835
|
+
:returns: Model artifact object
|
|
832
836
|
"""
|
|
833
837
|
|
|
834
838
|
if training_set is not None and inputs:
|
|
@@ -855,14 +859,17 @@ class MLClientCtx:
|
|
|
855
859
|
if training_set is not None:
|
|
856
860
|
model.infer_from_df(training_set, label_column)
|
|
857
861
|
|
|
858
|
-
item =
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
862
|
+
item = cast(
|
|
863
|
+
ModelArtifact,
|
|
864
|
+
self._artifacts_manager.log_artifact(
|
|
865
|
+
self,
|
|
866
|
+
model,
|
|
867
|
+
artifact_path=extend_artifact_path(artifact_path, self.artifact_path),
|
|
868
|
+
tag=tag,
|
|
869
|
+
upload=upload,
|
|
870
|
+
db_key=db_key,
|
|
871
|
+
labels=labels,
|
|
872
|
+
),
|
|
866
873
|
)
|
|
867
874
|
self._update_run()
|
|
868
875
|
return item
|
mlrun/model.py
CHANGED
|
@@ -117,6 +117,8 @@ class ModelObj:
|
|
|
117
117
|
# If one of the attributes is a third party object that has to_dict method (such as k8s objects), then
|
|
118
118
|
# add it to the object's _fields_to_serialize attribute and handle it in the _serialize_field method.
|
|
119
119
|
if hasattr(field_value, "to_dict"):
|
|
120
|
+
# TODO: Allow passing fields to exclude from the parent object to the child object
|
|
121
|
+
# e.g.: run.to_dict(exclude=["status.artifacts"])
|
|
120
122
|
field_value = field_value.to_dict(strip=strip)
|
|
121
123
|
if self._is_valid_field_value_for_serialization(
|
|
122
124
|
field_name, field_value, strip
|
|
@@ -1267,6 +1269,8 @@ class RunSpec(ModelObj):
|
|
|
1267
1269
|
class RunStatus(ModelObj):
|
|
1268
1270
|
"""Run status"""
|
|
1269
1271
|
|
|
1272
|
+
_default_fields_to_strip = ModelObj._default_fields_to_strip + ["artifacts"]
|
|
1273
|
+
|
|
1270
1274
|
def __init__(
|
|
1271
1275
|
self,
|
|
1272
1276
|
state=None,
|
|
@@ -135,10 +135,10 @@ class _PrepareMonitoringEvent(StepToDict):
|
|
|
135
135
|
:return: Application context.
|
|
136
136
|
"""
|
|
137
137
|
application_context = MonitoringApplicationContext(
|
|
138
|
-
graph_context=self.graph_context,
|
|
139
138
|
application_name=self.application_name,
|
|
140
139
|
event=event,
|
|
141
140
|
model_endpoint_dict=self.model_endpoints,
|
|
141
|
+
graph_context=self.graph_context,
|
|
142
142
|
)
|
|
143
143
|
|
|
144
144
|
self.model_endpoints.setdefault(
|
|
@@ -13,8 +13,9 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
from abc import ABC, abstractmethod
|
|
16
|
-
from typing import Any, Union
|
|
16
|
+
from typing import Any, Optional, Union, cast
|
|
17
17
|
|
|
18
|
+
import mlrun
|
|
18
19
|
import mlrun.model_monitoring.applications.context as mm_context
|
|
19
20
|
import mlrun.model_monitoring.applications.results as mm_results
|
|
20
21
|
from mlrun.serving.utils import MonitoringApplicationToDict
|
|
@@ -22,12 +23,12 @@ from mlrun.serving.utils import MonitoringApplicationToDict
|
|
|
22
23
|
|
|
23
24
|
class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
|
|
24
25
|
"""
|
|
25
|
-
|
|
26
|
+
The base class for a model monitoring application.
|
|
26
27
|
Inherit from this class to create a custom model monitoring application.
|
|
27
28
|
|
|
28
|
-
example
|
|
29
|
+
For example, :code:`MyApp` below is a simplistic custom application::
|
|
29
30
|
|
|
30
|
-
class MyApp(
|
|
31
|
+
class MyApp(ModelMonitoringApplicationBase):
|
|
31
32
|
def do_tracking(
|
|
32
33
|
self,
|
|
33
34
|
monitoring_context: mm_context.MonitoringApplicationContext,
|
|
@@ -43,8 +44,6 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
|
|
|
43
44
|
kind=mm_constant.ResultKindApp.data_drift,
|
|
44
45
|
status=mm_constant.ResultStatusApp.detected,
|
|
45
46
|
)
|
|
46
|
-
|
|
47
|
-
|
|
48
47
|
"""
|
|
49
48
|
|
|
50
49
|
kind = "monitoring_application"
|
|
@@ -62,6 +61,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
|
|
|
62
61
|
]:
|
|
63
62
|
"""
|
|
64
63
|
Process the monitoring event and return application results & metrics.
|
|
64
|
+
Note: this method is internal and should not be called directly or overridden.
|
|
65
65
|
|
|
66
66
|
:param monitoring_context: (MonitoringApplicationContext) The monitoring application context.
|
|
67
67
|
:returns: A tuple of:
|
|
@@ -80,6 +80,65 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
|
|
|
80
80
|
results = results if isinstance(results, list) else [results]
|
|
81
81
|
return results, monitoring_context
|
|
82
82
|
|
|
83
|
+
def _handler(self, context: "mlrun.MLClientCtx"):
|
|
84
|
+
"""
|
|
85
|
+
A custom handler that wraps the application's logic implemented in
|
|
86
|
+
:py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
|
|
87
|
+
for an MLRun job.
|
|
88
|
+
This method should not be called directly.
|
|
89
|
+
"""
|
|
90
|
+
monitoring_context = mm_context.MonitoringApplicationContext(
|
|
91
|
+
event={},
|
|
92
|
+
application_name=self.__class__.__name__,
|
|
93
|
+
logger=context.logger,
|
|
94
|
+
artifacts_logger=context,
|
|
95
|
+
)
|
|
96
|
+
result = self.do_tracking(monitoring_context)
|
|
97
|
+
return result
|
|
98
|
+
|
|
99
|
+
@classmethod
|
|
100
|
+
def evaluate(
|
|
101
|
+
cls,
|
|
102
|
+
func_path: Optional[str] = None,
|
|
103
|
+
func_name: Optional[str] = None,
|
|
104
|
+
tag: Optional[str] = None,
|
|
105
|
+
run_local: bool = True,
|
|
106
|
+
) -> "mlrun.RunObject":
|
|
107
|
+
"""
|
|
108
|
+
Call this function to run the application's
|
|
109
|
+
:py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
|
|
110
|
+
model monitoring logic as a :py:class:`~mlrun.runtimes.KubejobRuntime`, which is an MLRun function.
|
|
111
|
+
|
|
112
|
+
:param func_path: The path to the function. If not passed, the current notebook is used.
|
|
113
|
+
:param func_name: The name of the function. If not passed, the class name is used.
|
|
114
|
+
:param tag: An optional tag for the function.
|
|
115
|
+
:param run_local: Whether to run the function locally or remotely.
|
|
116
|
+
|
|
117
|
+
:returns: The output of the
|
|
118
|
+
:py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
|
|
119
|
+
method wrapped in a :py:class:`~mlrun.model.RunObject`.
|
|
120
|
+
"""
|
|
121
|
+
if not run_local:
|
|
122
|
+
raise NotImplementedError # ML-8360
|
|
123
|
+
|
|
124
|
+
project = cast("mlrun.MlrunProject", mlrun.get_current_project())
|
|
125
|
+
class_name = cls.__name__
|
|
126
|
+
name = func_name if func_name is not None else class_name
|
|
127
|
+
handler = f"{class_name}::{cls._handler.__name__}"
|
|
128
|
+
|
|
129
|
+
job = cast(
|
|
130
|
+
mlrun.runtimes.KubejobRuntime,
|
|
131
|
+
project.set_function(
|
|
132
|
+
func=func_path,
|
|
133
|
+
name=name,
|
|
134
|
+
kind=mlrun.runtimes.KubejobRuntime.kind,
|
|
135
|
+
handler=handler,
|
|
136
|
+
tag=tag,
|
|
137
|
+
),
|
|
138
|
+
)
|
|
139
|
+
run_result = job.run(local=run_local)
|
|
140
|
+
return run_result
|
|
141
|
+
|
|
83
142
|
@abstractmethod
|
|
84
143
|
def do_tracking(
|
|
85
144
|
self,
|
|
@@ -14,13 +14,15 @@
|
|
|
14
14
|
|
|
15
15
|
import json
|
|
16
16
|
import socket
|
|
17
|
-
from typing import Any, Optional, cast
|
|
17
|
+
from typing import Any, Optional, Protocol, cast
|
|
18
18
|
|
|
19
|
+
import nuclio.request
|
|
19
20
|
import numpy as np
|
|
20
21
|
import pandas as pd
|
|
21
22
|
|
|
22
23
|
import mlrun.common.constants as mlrun_constants
|
|
23
24
|
import mlrun.common.schemas.model_monitoring.constants as mm_constants
|
|
25
|
+
import mlrun.errors
|
|
24
26
|
import mlrun.feature_store as fstore
|
|
25
27
|
import mlrun.features
|
|
26
28
|
import mlrun.serving
|
|
@@ -34,6 +36,16 @@ from mlrun.model_monitoring.helpers import (
|
|
|
34
36
|
from mlrun.model_monitoring.model_endpoint import ModelEndpoint
|
|
35
37
|
|
|
36
38
|
|
|
39
|
+
class _ArtifactsLogger(Protocol):
|
|
40
|
+
"""
|
|
41
|
+
Classes that implement this protocol are :code:`MlrunProject` and :code:`MLClientCtx`.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def log_artifact(self, *args, **kwargs) -> Artifact: ...
|
|
45
|
+
def log_dataset(self, *args, **kwargs) -> DatasetArtifact: ...
|
|
46
|
+
def log_model(self, *args, **kwargs) -> ModelArtifact: ...
|
|
47
|
+
|
|
48
|
+
|
|
37
49
|
class MonitoringApplicationContext:
|
|
38
50
|
"""
|
|
39
51
|
The monitoring context holds all the relevant information for the monitoring application,
|
|
@@ -60,36 +72,57 @@ class MonitoringApplicationContext:
|
|
|
60
72
|
and a list of extra data items.
|
|
61
73
|
"""
|
|
62
74
|
|
|
75
|
+
_logger_name = "monitoring-application"
|
|
76
|
+
|
|
63
77
|
def __init__(
|
|
64
78
|
self,
|
|
65
79
|
*,
|
|
66
|
-
graph_context: mlrun.serving.GraphContext,
|
|
67
80
|
application_name: str,
|
|
68
81
|
event: dict[str, Any],
|
|
69
|
-
model_endpoint_dict: dict[str, ModelEndpoint],
|
|
82
|
+
model_endpoint_dict: Optional[dict[str, ModelEndpoint]] = None,
|
|
83
|
+
logger: Optional[mlrun.utils.Logger] = None,
|
|
84
|
+
graph_context: Optional[mlrun.serving.GraphContext] = None,
|
|
85
|
+
artifacts_logger: Optional[_ArtifactsLogger] = None,
|
|
70
86
|
) -> None:
|
|
71
87
|
"""
|
|
72
|
-
Initialize a
|
|
88
|
+
Initialize a :code:`MonitoringApplicationContext` object.
|
|
73
89
|
Note: this object should not be instantiated manually.
|
|
74
90
|
|
|
75
91
|
:param application_name: The application name.
|
|
76
92
|
:param event: The instance data dictionary.
|
|
77
|
-
:param model_endpoint_dict:
|
|
93
|
+
:param model_endpoint_dict: Optional - dictionary of model endpoints.
|
|
94
|
+
:param logger: Optional - MLRun logger instance.
|
|
95
|
+
:param graph_context: Optional - GraphContext instance.
|
|
96
|
+
:param artifacts_logger: Optional - an object that can log artifacts,
|
|
97
|
+
typically :py:class:`~mlrun.projects.MlrunProject` or
|
|
98
|
+
:py:class:`~mlrun.execution.MLClientCtx`.
|
|
78
99
|
"""
|
|
79
100
|
self.application_name = application_name
|
|
80
101
|
|
|
81
|
-
|
|
82
|
-
|
|
102
|
+
if graph_context:
|
|
103
|
+
self.project_name = graph_context.project
|
|
104
|
+
self.project = mlrun.load_project(url=self.project_name)
|
|
105
|
+
else:
|
|
106
|
+
self.project = cast("mlrun.MlrunProject", mlrun.get_current_project())
|
|
107
|
+
self.project_name = self.project.name
|
|
108
|
+
|
|
109
|
+
self._artifacts_logger: _ArtifactsLogger = artifacts_logger or self.project
|
|
83
110
|
|
|
84
111
|
# MLRun Logger
|
|
85
|
-
self.logger = mlrun.utils.create_logger(
|
|
112
|
+
self.logger = logger or mlrun.utils.create_logger(
|
|
86
113
|
level=mlrun.mlconf.log_level,
|
|
87
114
|
formatter_kind=mlrun.mlconf.log_formatter,
|
|
88
|
-
name=
|
|
115
|
+
name=self._logger_name,
|
|
89
116
|
)
|
|
90
117
|
# Nuclio logger - `nuclio.request.Logger`.
|
|
91
|
-
# Note: this logger
|
|
92
|
-
self.nuclio_logger =
|
|
118
|
+
# Note: this logger accepts keyword arguments only in its `_with` methods, e.g. `info_with`.
|
|
119
|
+
self.nuclio_logger = (
|
|
120
|
+
graph_context.logger
|
|
121
|
+
if graph_context
|
|
122
|
+
else nuclio.request.Logger(
|
|
123
|
+
level=mlrun.mlconf.log_level, name=self._logger_name
|
|
124
|
+
)
|
|
125
|
+
)
|
|
93
126
|
|
|
94
127
|
# event data
|
|
95
128
|
self.start_infer_time = pd.Timestamp(
|
|
@@ -113,8 +146,8 @@ class MonitoringApplicationContext:
|
|
|
113
146
|
|
|
114
147
|
# Persistent data - fetched when needed
|
|
115
148
|
self._sample_df: Optional[pd.DataFrame] = None
|
|
116
|
-
self._model_endpoint: Optional[ModelEndpoint] =
|
|
117
|
-
self.endpoint_id
|
|
149
|
+
self._model_endpoint: Optional[ModelEndpoint] = (
|
|
150
|
+
model_endpoint_dict.get(self.endpoint_id) if model_endpoint_dict else None
|
|
118
151
|
)
|
|
119
152
|
|
|
120
153
|
def _get_default_labels(self) -> dict[str, str]:
|
|
@@ -237,7 +270,7 @@ class MonitoringApplicationContext:
|
|
|
237
270
|
See :func:`~mlrun.projects.MlrunProject.log_artifact` for the documentation.
|
|
238
271
|
"""
|
|
239
272
|
labels = self._add_default_labels(labels)
|
|
240
|
-
return self.
|
|
273
|
+
return self._artifacts_logger.log_artifact(
|
|
241
274
|
item,
|
|
242
275
|
body=body,
|
|
243
276
|
tag=tag,
|
|
@@ -272,7 +305,7 @@ class MonitoringApplicationContext:
|
|
|
272
305
|
See :func:`~mlrun.projects.MlrunProject.log_dataset` for the documentation.
|
|
273
306
|
"""
|
|
274
307
|
labels = self._add_default_labels(labels)
|
|
275
|
-
return self.
|
|
308
|
+
return self._artifacts_logger.log_dataset(
|
|
276
309
|
key,
|
|
277
310
|
df,
|
|
278
311
|
tag=tag,
|
|
@@ -317,7 +350,7 @@ class MonitoringApplicationContext:
|
|
|
317
350
|
See :func:`~mlrun.projects.MlrunProject.log_model` for the documentation.
|
|
318
351
|
"""
|
|
319
352
|
labels = self._add_default_labels(labels)
|
|
320
|
-
return self.
|
|
353
|
+
return self._artifacts_logger.log_model(
|
|
321
354
|
key,
|
|
322
355
|
body=body,
|
|
323
356
|
framework=framework,
|
|
@@ -76,7 +76,6 @@ class EvidentlyModelMonitoringApplicationBase(
|
|
|
76
76
|
|
|
77
77
|
:param evidently_workspace_path: (str) The path to the Evidently workspace.
|
|
78
78
|
:param evidently_project_id: (str) The ID of the Evidently project.
|
|
79
|
-
|
|
80
79
|
"""
|
|
81
80
|
|
|
82
81
|
# TODO : more then one project (mep -> project)
|
|
@@ -113,7 +113,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
|
|
|
113
113
|
|
|
114
114
|
project.enable_model_monitoring()
|
|
115
115
|
|
|
116
|
-
To avoid it, pass
|
|
116
|
+
To avoid it, pass :code:`deploy_histogram_data_drift_app=False`.
|
|
117
117
|
"""
|
|
118
118
|
|
|
119
119
|
NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
|
|
@@ -331,8 +331,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
|
|
|
331
331
|
)
|
|
332
332
|
|
|
333
333
|
def do_tracking(
|
|
334
|
-
self,
|
|
335
|
-
monitoring_context: mm_context.MonitoringApplicationContext,
|
|
334
|
+
self, monitoring_context: mm_context.MonitoringApplicationContext
|
|
336
335
|
) -> list[
|
|
337
336
|
Union[
|
|
338
337
|
mm_results.ModelMonitoringApplicationResult,
|
|
@@ -342,9 +341,6 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
|
|
|
342
341
|
]:
|
|
343
342
|
"""
|
|
344
343
|
Calculate and return the data drift metrics, averaged over the features.
|
|
345
|
-
|
|
346
|
-
Refer to `ModelMonitoringApplicationBaseV2` for the meaning of the
|
|
347
|
-
function arguments.
|
|
348
344
|
"""
|
|
349
345
|
monitoring_context.logger.debug("Starting to run the application")
|
|
350
346
|
if not monitoring_context.feature_stats:
|
mlrun/projects/project.py
CHANGED
|
@@ -28,7 +28,7 @@ import warnings
|
|
|
28
28
|
import zipfile
|
|
29
29
|
from copy import deepcopy
|
|
30
30
|
from os import environ, makedirs, path
|
|
31
|
-
from typing import Callable, Optional, Union
|
|
31
|
+
from typing import Callable, Optional, Union, cast
|
|
32
32
|
|
|
33
33
|
import dotenv
|
|
34
34
|
import git
|
|
@@ -1732,7 +1732,7 @@ class MlrunProject(ModelObj):
|
|
|
1732
1732
|
:param upload: upload to datastore (default is True)
|
|
1733
1733
|
:param labels: a set of key/value labels to tag the artifact with
|
|
1734
1734
|
|
|
1735
|
-
:returns: artifact object
|
|
1735
|
+
:returns: dataset artifact object
|
|
1736
1736
|
"""
|
|
1737
1737
|
ds = DatasetArtifact(
|
|
1738
1738
|
key,
|
|
@@ -1745,14 +1745,17 @@ class MlrunProject(ModelObj):
|
|
|
1745
1745
|
**kwargs,
|
|
1746
1746
|
)
|
|
1747
1747
|
|
|
1748
|
-
item =
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1748
|
+
item = cast(
|
|
1749
|
+
DatasetArtifact,
|
|
1750
|
+
self.log_artifact(
|
|
1751
|
+
ds,
|
|
1752
|
+
local_path=local_path,
|
|
1753
|
+
artifact_path=artifact_path,
|
|
1754
|
+
target_path=target_path,
|
|
1755
|
+
tag=tag,
|
|
1756
|
+
upload=upload,
|
|
1757
|
+
labels=labels,
|
|
1758
|
+
),
|
|
1756
1759
|
)
|
|
1757
1760
|
return item
|
|
1758
1761
|
|
|
@@ -1820,7 +1823,7 @@ class MlrunProject(ModelObj):
|
|
|
1820
1823
|
:param extra_data: key/value list of extra files/charts to link with this dataset
|
|
1821
1824
|
value can be absolute path | relative path (to model dir) | bytes | artifact object
|
|
1822
1825
|
|
|
1823
|
-
:returns: artifact object
|
|
1826
|
+
:returns: model artifact object
|
|
1824
1827
|
"""
|
|
1825
1828
|
|
|
1826
1829
|
if training_set is not None and inputs:
|
|
@@ -1847,12 +1850,15 @@ class MlrunProject(ModelObj):
|
|
|
1847
1850
|
if training_set is not None:
|
|
1848
1851
|
model.infer_from_df(training_set, label_column)
|
|
1849
1852
|
|
|
1850
|
-
item =
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1853
|
+
item = cast(
|
|
1854
|
+
ModelArtifact,
|
|
1855
|
+
self.log_artifact(
|
|
1856
|
+
model,
|
|
1857
|
+
artifact_path=artifact_path,
|
|
1858
|
+
tag=tag,
|
|
1859
|
+
upload=upload,
|
|
1860
|
+
labels=labels,
|
|
1861
|
+
),
|
|
1856
1862
|
)
|
|
1857
1863
|
return item
|
|
1858
1864
|
|
|
@@ -2408,7 +2414,7 @@ class MlrunProject(ModelObj):
|
|
|
2408
2414
|
|
|
2409
2415
|
def set_function(
|
|
2410
2416
|
self,
|
|
2411
|
-
func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,
|
|
2417
|
+
func: typing.Union[str, mlrun.runtimes.BaseRuntime, None] = None,
|
|
2412
2418
|
name: str = "",
|
|
2413
2419
|
kind: str = "job",
|
|
2414
2420
|
image: Optional[str] = None,
|
|
@@ -4840,7 +4846,6 @@ class MlrunProject(ModelObj):
|
|
|
4840
4846
|
page=page,
|
|
4841
4847
|
page_size=page_size,
|
|
4842
4848
|
page_token=page_token,
|
|
4843
|
-
return_all=False,
|
|
4844
4849
|
**kwargs,
|
|
4845
4850
|
)
|
|
4846
4851
|
|
|
@@ -69,6 +69,7 @@ class MailNotification(base.NotificationBase):
|
|
|
69
69
|
alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
|
|
70
70
|
event_data: typing.Optional[mlrun.common.schemas.Event] = None,
|
|
71
71
|
):
|
|
72
|
+
message = self.params.get("message", message)
|
|
72
73
|
self.params.setdefault("subject", f"[{severity}] {message}")
|
|
73
74
|
self.params.setdefault("body", message)
|
|
74
75
|
await self._send_email(**self.params)
|
|
@@ -84,13 +85,28 @@ class MailNotification(base.NotificationBase):
|
|
|
84
85
|
params.setdefault("server_port", DEFAULT_SMTP_PORT)
|
|
85
86
|
|
|
86
87
|
default_mail_address = params.pop("default_email_addresses", "")
|
|
87
|
-
email_addresses =
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
params["email_addresses"] = email_addresses
|
|
88
|
+
params["email_addresses"] = cls._merge_mail_addresses(
|
|
89
|
+
default_mail_address, params.get("email_addresses", "")
|
|
90
|
+
)
|
|
91
91
|
|
|
92
92
|
return params
|
|
93
93
|
|
|
94
|
+
@classmethod
|
|
95
|
+
def _merge_mail_addresses(
|
|
96
|
+
cls,
|
|
97
|
+
default_mail_address: typing.Union[str, list],
|
|
98
|
+
email_addresses: typing.Union[str, list],
|
|
99
|
+
) -> str:
|
|
100
|
+
if isinstance(default_mail_address, str):
|
|
101
|
+
default_mail_address = (
|
|
102
|
+
default_mail_address.split(",") if default_mail_address else []
|
|
103
|
+
)
|
|
104
|
+
if isinstance(email_addresses, str):
|
|
105
|
+
email_addresses = email_addresses.split(",") if email_addresses else []
|
|
106
|
+
email_addresses.extend(default_mail_address)
|
|
107
|
+
email_addresses_str = ",".join(email_addresses)
|
|
108
|
+
return email_addresses_str
|
|
109
|
+
|
|
94
110
|
@classmethod
|
|
95
111
|
def _validate_emails(cls, params):
|
|
96
112
|
cls._validate_email_address(params["sender_address"])
|
|
@@ -118,21 +118,37 @@ class NotificationPusher(_NotificationPusherBase):
|
|
|
118
118
|
] = []
|
|
119
119
|
|
|
120
120
|
for run in self._runs:
|
|
121
|
-
|
|
122
|
-
|
|
121
|
+
try:
|
|
122
|
+
self._process_run(run)
|
|
123
|
+
except Exception as exc:
|
|
124
|
+
logger.warning(
|
|
125
|
+
"Failed to process run",
|
|
126
|
+
run_uid=run.metadata.uid,
|
|
127
|
+
error=mlrun.errors.err_to_str(exc),
|
|
128
|
+
)
|
|
123
129
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
notification.name
|
|
128
|
-
).get("status", mlrun.common.schemas.NotificationStatus.PENDING)
|
|
129
|
-
except (AttributeError, KeyError):
|
|
130
|
-
notification.status = (
|
|
131
|
-
mlrun.common.schemas.NotificationStatus.PENDING
|
|
132
|
-
)
|
|
130
|
+
def _process_run(self, run):
|
|
131
|
+
if isinstance(run, dict):
|
|
132
|
+
run = mlrun.model.RunObject.from_dict(run)
|
|
133
133
|
|
|
134
|
-
|
|
135
|
-
|
|
134
|
+
for notification in run.spec.notifications:
|
|
135
|
+
try:
|
|
136
|
+
self._process_notification(notification, run)
|
|
137
|
+
except Exception as exc:
|
|
138
|
+
logger.warning(
|
|
139
|
+
"Failed to process notification",
|
|
140
|
+
run_uid=run.metadata.uid,
|
|
141
|
+
notification=notification,
|
|
142
|
+
error=mlrun.errors.err_to_str(exc),
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
def _process_notification(self, notification, run):
|
|
146
|
+
notification.status = run.status.notifications.get(notification.name, {}).get(
|
|
147
|
+
"status",
|
|
148
|
+
mlrun.common.schemas.NotificationStatus.PENDING,
|
|
149
|
+
)
|
|
150
|
+
if self._should_notify(run, notification):
|
|
151
|
+
self._load_notification(run, notification)
|
|
136
152
|
|
|
137
153
|
def push(self):
|
|
138
154
|
"""
|
|
@@ -176,6 +192,11 @@ class NotificationPusher(_NotificationPusherBase):
|
|
|
176
192
|
logger.warning(
|
|
177
193
|
"Failed to push notification async",
|
|
178
194
|
error=mlrun.errors.err_to_str(result),
|
|
195
|
+
traceback=traceback.format_exception(
|
|
196
|
+
etype=type(result),
|
|
197
|
+
value=result,
|
|
198
|
+
tb=result.__traceback__,
|
|
199
|
+
),
|
|
179
200
|
)
|
|
180
201
|
|
|
181
202
|
logger.debug(
|
mlrun/utils/version/version.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mlrun
|
|
3
|
-
Version: 1.8.
|
|
3
|
+
Version: 1.8.0rc5
|
|
4
4
|
Summary: Tracking and config of machine learning runs
|
|
5
5
|
Home-page: https://github.com/mlrun/mlrun
|
|
6
6
|
Author: Yaron Haviv
|
|
@@ -32,13 +32,14 @@ Requires-Dist: nuclio-jupyter~=0.11.1
|
|
|
32
32
|
Requires-Dist: numpy<1.27.0,>=1.26.4
|
|
33
33
|
Requires-Dist: pandas<2.2,>=1.2
|
|
34
34
|
Requires-Dist: pyarrow<18,>=10.0
|
|
35
|
-
Requires-Dist: pyyaml<7,>=
|
|
35
|
+
Requires-Dist: pyyaml<7,>=6.0.2
|
|
36
36
|
Requires-Dist: requests~=2.32
|
|
37
37
|
Requires-Dist: tabulate~=0.8.6
|
|
38
38
|
Requires-Dist: v3io~=0.6.9
|
|
39
39
|
Requires-Dist: pydantic>=1.10.15
|
|
40
40
|
Requires-Dist: mergedeep~=1.3
|
|
41
|
-
Requires-Dist: v3io-frames~=0.10.14
|
|
41
|
+
Requires-Dist: v3io-frames~=0.10.14; python_version < "3.11"
|
|
42
|
+
Requires-Dist: v3io-frames!=0.11.*,!=0.12.*,>=0.10.14; python_version >= "3.11"
|
|
42
43
|
Requires-Dist: semver~=3.0
|
|
43
44
|
Requires-Dist: dependency-injector~=4.41
|
|
44
45
|
Requires-Dist: fsspec<2024.7,>=2023.9.2
|
|
@@ -51,7 +52,7 @@ Requires-Dist: deprecated~=1.2
|
|
|
51
52
|
Requires-Dist: jinja2>=3.1.3,~=3.1
|
|
52
53
|
Requires-Dist: orjson<4,>=3.9.15
|
|
53
54
|
Requires-Dist: mlrun-pipelines-kfp-common~=0.2.3
|
|
54
|
-
Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.2.2
|
|
55
|
+
Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.2.2; python_version < "3.11"
|
|
55
56
|
Requires-Dist: docstring_parser~=0.16
|
|
56
57
|
Requires-Dist: aiosmtplib~=3.0
|
|
57
58
|
Provides-Extra: s3
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
mlrun/__init__.py,sha256=
|
|
1
|
+
mlrun/__init__.py,sha256=V11600nBHBHT84iZAFzZ7p1kbhEWaX99wzBIivl4ozo,7453
|
|
2
2
|
mlrun/__main__.py,sha256=o65gXHhmFA9GV_n2mqmAO80nW3MAwo_s7j80IKgCzRE,45949
|
|
3
3
|
mlrun/config.py,sha256=ejQ-goHH1vBiLqZ6WxV9191Wsw8k5NEBHzyRNvOfjzg,71029
|
|
4
4
|
mlrun/errors.py,sha256=5raKb1PXQpTcIvWQ4sr1qn2IS7P_GT_FydBJ0dXkVuc,8097
|
|
5
|
-
mlrun/execution.py,sha256=
|
|
5
|
+
mlrun/execution.py,sha256=A8YBa3LCFFEeG0v-NUvQlgu85KnMcVESGi-zxayS99k,46892
|
|
6
6
|
mlrun/features.py,sha256=ReBaNGsBYXqcbgI012n-SO_j6oHIbk_Vpv0CGPXbUmo,15842
|
|
7
7
|
mlrun/k8s_utils.py,sha256=mRQMs6NzPq36vx1n5_2BfFapXysc8wv3NcrZ77_2ANA,8949
|
|
8
8
|
mlrun/lists.py,sha256=1hFv3Iyu5DVX1kdBGJmwUoY0CqrzauhKdSq9g3piHb4,8442
|
|
9
|
-
mlrun/model.py,sha256=
|
|
9
|
+
mlrun/model.py,sha256=yn-WrQpOX1NodvqeA9m5G9gyU2szM4QVarAEempRe-k,85256
|
|
10
10
|
mlrun/render.py,sha256=940H9fBBFeghH4dlifbURvtjlvw4GlWdAXezN6ky4rI,13275
|
|
11
11
|
mlrun/run.py,sha256=OJRX96ijJL8DYRLgOAtcx1ZNlTIpgXXImKmbsHoFuBU,43743
|
|
12
12
|
mlrun/secrets.py,sha256=ibtCK79u7JVBZF6F0SP1-xXXF5MyrLEUs_TCWiJAnlc,7798
|
|
13
13
|
mlrun/alerts/__init__.py,sha256=0gtG1BG0DXxFrXegIkjbM1XEN4sP9ODo0ucXrNld1hU,601
|
|
14
|
-
mlrun/alerts/alert.py,sha256=
|
|
14
|
+
mlrun/alerts/alert.py,sha256=mTROlDXzQw5gyWBFaUnykai3wpvjmgRmo28p0ytbzIU,15930
|
|
15
15
|
mlrun/api/schemas/__init__.py,sha256=fEWH4I8hr5AdRJ7yoW44RlFB6NHkYDxyomP5J6ct1z4,14248
|
|
16
16
|
mlrun/artifacts/__init__.py,sha256=_-yRrD27eVA5iuU28UmDZfdkhwadzQy_Q03kAuueSHg,1141
|
|
17
17
|
mlrun/artifacts/base.py,sha256=W2iG9HOG3U0hTrJyI_zEm0rthgopzXTRu1oMsKruzQg,29694
|
|
@@ -40,9 +40,9 @@ mlrun/common/model_monitoring/__init__.py,sha256=ePaXfZQaAH_okyfJKK0-DllguTVHRYM
|
|
|
40
40
|
mlrun/common/model_monitoring/helpers.py,sha256=TA24rF5qlLouEWGMDhXp7zXncIMgwykmVYq8PPakMFw,4422
|
|
41
41
|
mlrun/common/runtimes/constants.py,sha256=Mok3m9Rv182TTMp7uYNfWalm9Xcz86yva-4fTxfMOPI,10988
|
|
42
42
|
mlrun/common/schemas/__init__.py,sha256=cjBoE1NMiN8L36nd1iEkb2n4ogG-bbiRo5VdJMg8jXY,5381
|
|
43
|
-
mlrun/common/schemas/alert.py,sha256
|
|
43
|
+
mlrun/common/schemas/alert.py,sha256=-Mz1QhyqGU6dyEBEIKZmKvR4v3qmGMp6-bWbDC9joy0,7258
|
|
44
44
|
mlrun/common/schemas/api_gateway.py,sha256=3a0QxECLmoDkD5IiOKtXJL-uiWB26Hg55WMA3nULYuI,7127
|
|
45
|
-
mlrun/common/schemas/artifact.py,sha256=
|
|
45
|
+
mlrun/common/schemas/artifact.py,sha256=i29BeZ4MoOLMST3WlApX1Nli0vy-M7Zj_oTm8jySlw4,3805
|
|
46
46
|
mlrun/common/schemas/auth.py,sha256=AGbBNvQq_vcvhX_NLqbT-QPHL4BAJMB3xwBXW7cFvpo,6761
|
|
47
47
|
mlrun/common/schemas/background_task.py,sha256=ofWRAQGGEkXEu79Dbw7tT_5GPolR09Lc3Ebg2r0fT24,1728
|
|
48
48
|
mlrun/common/schemas/client_spec.py,sha256=40zyebkqHSvMHUQzZbbifgTofF5zn7LOZLZdp36TPlo,2898
|
|
@@ -109,8 +109,8 @@ mlrun/datastore/wasbfs/fs.py,sha256=ge8NK__5vTcFT-krI155_8RDUywQw4SIRX6BWATXy9Q,
|
|
|
109
109
|
mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
|
|
110
110
|
mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
|
|
111
111
|
mlrun/db/base.py,sha256=bui1BYkQDnugs6gdQEzL0YASEFqGVhqCxwV0B58OBds,28651
|
|
112
|
-
mlrun/db/factory.py,sha256=
|
|
113
|
-
mlrun/db/httpdb.py,sha256=
|
|
112
|
+
mlrun/db/factory.py,sha256=yP2vVmveUE7LYTCHbS6lQIxP9rW--zdISWuPd_I3d_4,2111
|
|
113
|
+
mlrun/db/httpdb.py,sha256=igJQB11NK7E55fc5IsI7F7zgksgEbYfz6gSWbjr0gqU,221919
|
|
114
114
|
mlrun/db/nopdb.py,sha256=SJv8TfggnpbYJSv46UHsPT7_dGEarbBiNa5iG6Pwt90,25470
|
|
115
115
|
mlrun/feature_store/__init__.py,sha256=AVnY2AFUNc2dKxLLUMx2K3Wo1eGviv0brDcYlDnmtf4,1506
|
|
116
116
|
mlrun/feature_store/api.py,sha256=mrvFNwnX7GviWBVklruXbqLmXKSMGt0P6cJtmy9Z8Nw,50394
|
|
@@ -226,11 +226,11 @@ mlrun/model_monitoring/stream_processing.py,sha256=XulKYz0xvnFrKdN9wcQT0WCHRbS3H
|
|
|
226
226
|
mlrun/model_monitoring/tracking_policy.py,sha256=PBIGrUYWrwcE5gwXupBIVzOb0QRRwPJsgQm_yLGQxB4,5595
|
|
227
227
|
mlrun/model_monitoring/writer.py,sha256=kiaXcqshUQ48YuxGps5dHEasDSIEQSh6k9Z9vd-MttE,10919
|
|
228
228
|
mlrun/model_monitoring/applications/__init__.py,sha256=QYvzgCutFdAkzqKPD3mvkX_3c1X4tzd-kW8ojUOE9ic,889
|
|
229
|
-
mlrun/model_monitoring/applications/_application_steps.py,sha256=
|
|
230
|
-
mlrun/model_monitoring/applications/base.py,sha256=
|
|
231
|
-
mlrun/model_monitoring/applications/context.py,sha256=
|
|
232
|
-
mlrun/model_monitoring/applications/evidently_base.py,sha256=
|
|
233
|
-
mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=
|
|
229
|
+
mlrun/model_monitoring/applications/_application_steps.py,sha256=oF3Mk8kDDI9tnqgQgKSgdoTCI8oS0_3Xsc3ifSArLjA,7050
|
|
230
|
+
mlrun/model_monitoring/applications/base.py,sha256=ObHX1zSBjoEoaS1HE77Q05Gcpgvd_uza6ttRt16NgPU,6902
|
|
231
|
+
mlrun/model_monitoring/applications/context.py,sha256=QRrv_PEabzAMD2lZSQTAzG3jxmDwJtdXwnRrqUlJhBs,13970
|
|
232
|
+
mlrun/model_monitoring/applications/evidently_base.py,sha256=hRjXuXf6xf8sbjGt9yYfGDUGnvS5rV3W7tkJroF3QJA,5098
|
|
233
|
+
mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=DW_-DIBTy6sP-rKbt4BY-MrwswYsuQmZpllJtHrsNso,14454
|
|
234
234
|
mlrun/model_monitoring/applications/results.py,sha256=oh1z9oacWWP4azVXm_Fx7j8uXSfdkB9T4mtGwyPBveE,5748
|
|
235
235
|
mlrun/model_monitoring/db/__init__.py,sha256=6Ic-X3Fh9XLPYMytmevGNSs-Hii1rAjLLoFTSPwTguw,736
|
|
236
236
|
mlrun/model_monitoring/db/_schedules.py,sha256=NTO1rbSyhW1JidpBDSN39ZBD0ctp5pbJFYQwxKRIRrs,5821
|
|
@@ -280,7 +280,7 @@ mlrun/platforms/iguazio.py,sha256=6VBTq8eQ3mzT96tzjYhAtcMQ2VjF4x8LpIPW5DAcX2Q,13
|
|
|
280
280
|
mlrun/projects/__init__.py,sha256=0Krf0WIKfnZa71WthYOg0SoaTodGg3sV_hK3f_OlTPI,1220
|
|
281
281
|
mlrun/projects/operations.py,sha256=VXUlMrouFTls-I-bMhdN5pPfQ34TR7bFQ-NUSWNvl84,20029
|
|
282
282
|
mlrun/projects/pipelines.py,sha256=DWAWkFXLDoPIaKySgdn4XVqkIMfxCcCNUdw7XwAsfno,45924
|
|
283
|
-
mlrun/projects/project.py,sha256=
|
|
283
|
+
mlrun/projects/project.py,sha256=v7gRacQi9nDlIB9mW88xV_DX52sGDiFgA0bzRVbXsEM,219036
|
|
284
284
|
mlrun/runtimes/__init__.py,sha256=J9Sy2HiyMlztNv6VUurMzF5H2XzttNil8nRsWDsqLyg,8923
|
|
285
285
|
mlrun/runtimes/base.py,sha256=Yt2l7srrXjK783cunBEKH0yQxQZRH8lkedXNOXuLbbo,37841
|
|
286
286
|
mlrun/runtimes/daskjob.py,sha256=JwuGvOiPsxEDHHMMUS4Oie4hLlYYIZwihAl6DjroTY0,19521
|
|
@@ -340,21 +340,21 @@ mlrun/utils/singleton.py,sha256=p1Y-X0mPSs_At092GS-pZCA8CTR62HOqPU07_ZH6-To,869
|
|
|
340
340
|
mlrun/utils/v3io_clients.py,sha256=0aCFiQFBmgdSeLzJr_nEP6SG-zyieSgH8RdtcUq4dc0,1294
|
|
341
341
|
mlrun/utils/vault.py,sha256=xUiKL17dCXjwQJ33YRzQj0oadUXATlFWPzKKYAESoQk,10447
|
|
342
342
|
mlrun/utils/notifications/__init__.py,sha256=eUzQDBxSQmMZASRY-YAnYS6tL5801P0wEjycp3Dvoe0,990
|
|
343
|
-
mlrun/utils/notifications/notification_pusher.py,sha256=
|
|
343
|
+
mlrun/utils/notifications/notification_pusher.py,sha256=j6wzGhf7r03tYKlTjIDyHTDIVJnWC4q945j-lrkLGsY,28181
|
|
344
344
|
mlrun/utils/notifications/notification/__init__.py,sha256=9Rfy6Jm8n0LaEDO1VAQb6kIbr7_uVuQhK1pS_abELIY,2581
|
|
345
345
|
mlrun/utils/notifications/notification/base.py,sha256=r-5eAGm2xz5VZGbV6NfeRV-heF7WzOYi-qWwDK2nR7U,5297
|
|
346
346
|
mlrun/utils/notifications/notification/console.py,sha256=ICbIhOf9fEBJky_3j9TFiKAewDGyDHJr9l4VeT7G2sc,2745
|
|
347
347
|
mlrun/utils/notifications/notification/git.py,sha256=t2lqRrPRBO4awf_uhxJreH9CpcbYSH8T3CvHtwspHkE,6306
|
|
348
348
|
mlrun/utils/notifications/notification/ipython.py,sha256=9uZvI1uOLFaNuAsfJPXmL3l6dOzFoWdBK5GYNYFAfks,2282
|
|
349
|
-
mlrun/utils/notifications/notification/mail.py,sha256=
|
|
349
|
+
mlrun/utils/notifications/notification/mail.py,sha256=9SqyRsnkSZzVSV8m7f0e9id94TMpUpNisc1MxFq7pP8,5632
|
|
350
350
|
mlrun/utils/notifications/notification/slack.py,sha256=NKV4RFiY3gLsS8uPppgniPLyag8zJ9O1VhixoXkM7kw,7108
|
|
351
351
|
mlrun/utils/notifications/notification/webhook.py,sha256=lSGKCQMa-TstKbMpZnU5uQkW14tzIaqjBHDXUNh9dlU,4848
|
|
352
352
|
mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
|
|
353
|
-
mlrun/utils/version/version.json,sha256=
|
|
353
|
+
mlrun/utils/version/version.json,sha256=DbPlUivo2nYIYvL3oYIyra0hfd-SaXYrZRucx5kI0co,88
|
|
354
354
|
mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
|
|
355
|
-
mlrun-1.8.
|
|
356
|
-
mlrun-1.8.
|
|
357
|
-
mlrun-1.8.
|
|
358
|
-
mlrun-1.8.
|
|
359
|
-
mlrun-1.8.
|
|
360
|
-
mlrun-1.8.
|
|
355
|
+
mlrun-1.8.0rc5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
356
|
+
mlrun-1.8.0rc5.dist-info/METADATA,sha256=OYH1N1RxV6yv_ppwYsuqEG52LqxpebrTY1QzsI_QpSc,24456
|
|
357
|
+
mlrun-1.8.0rc5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
|
358
|
+
mlrun-1.8.0rc5.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
|
|
359
|
+
mlrun-1.8.0rc5.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
|
|
360
|
+
mlrun-1.8.0rc5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|