mlrun 1.7.0rc32__py3-none-any.whl → 1.7.0rc34__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/common/schemas/__init__.py +1 -0
- mlrun/common/schemas/common.py +3 -0
- mlrun/common/schemas/function.py +7 -0
- mlrun/common/schemas/project.py +35 -3
- mlrun/config.py +9 -1
- mlrun/datastore/base.py +5 -1
- mlrun/db/base.py +8 -3
- mlrun/db/httpdb.py +10 -8
- mlrun/db/nopdb.py +3 -1
- mlrun/execution.py +1 -3
- mlrun/model.py +142 -22
- mlrun/model_monitoring/applications/context.py +13 -15
- mlrun/model_monitoring/controller.py +1 -1
- mlrun/model_monitoring/db/stores/base/store.py +2 -0
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +9 -23
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +5 -20
- mlrun/model_monitoring/stream_processing.py +6 -0
- mlrun/projects/project.py +8 -2
- mlrun/run.py +22 -9
- mlrun/runtimes/nuclio/api_gateway.py +37 -7
- mlrun/runtimes/nuclio/application/application.py +50 -9
- mlrun/runtimes/nuclio/function.py +3 -0
- mlrun/runtimes/nuclio/serving.py +5 -5
- mlrun/serving/server.py +12 -7
- mlrun/serving/states.py +13 -1
- mlrun/utils/db.py +3 -0
- mlrun/utils/helpers.py +3 -5
- mlrun/utils/notifications/notification/webhook.py +8 -1
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc32.dist-info → mlrun-1.7.0rc34.dist-info}/METADATA +3 -3
- {mlrun-1.7.0rc32.dist-info → mlrun-1.7.0rc34.dist-info}/RECORD +35 -35
- {mlrun-1.7.0rc32.dist-info → mlrun-1.7.0rc34.dist-info}/WHEEL +1 -1
- {mlrun-1.7.0rc32.dist-info → mlrun-1.7.0rc34.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc32.dist-info → mlrun-1.7.0rc34.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc32.dist-info → mlrun-1.7.0rc34.dist-info}/top_level.txt +0 -0
mlrun/common/schemas/__init__.py
CHANGED
mlrun/common/schemas/common.py
CHANGED
mlrun/common/schemas/function.py
CHANGED
|
@@ -119,6 +119,13 @@ class FunctionSpec(pydantic.BaseModel):
|
|
|
119
119
|
service_account: typing.Optional[ServiceAccount]
|
|
120
120
|
state_thresholds: typing.Optional[StateThresholds]
|
|
121
121
|
|
|
122
|
+
class Config:
|
|
123
|
+
extra = pydantic.Extra.allow
|
|
124
|
+
|
|
122
125
|
|
|
123
126
|
class Function(pydantic.BaseModel):
|
|
124
127
|
spec: typing.Optional[FunctionSpec]
|
|
128
|
+
application: typing.Optional[dict[str, typing.Any]]
|
|
129
|
+
|
|
130
|
+
class Config:
|
|
131
|
+
extra = pydantic.Extra.allow
|
mlrun/common/schemas/project.py
CHANGED
|
@@ -100,6 +100,29 @@ class ProjectSpec(pydantic.BaseModel):
|
|
|
100
100
|
extra = pydantic.Extra.allow
|
|
101
101
|
|
|
102
102
|
|
|
103
|
+
class ProjectSpecOut(pydantic.BaseModel):
|
|
104
|
+
description: typing.Optional[str] = None
|
|
105
|
+
owner: typing.Optional[str] = None
|
|
106
|
+
goals: typing.Optional[str] = None
|
|
107
|
+
params: typing.Optional[dict] = {}
|
|
108
|
+
functions: typing.Optional[list] = []
|
|
109
|
+
workflows: typing.Optional[list] = []
|
|
110
|
+
artifacts: typing.Optional[list] = []
|
|
111
|
+
artifact_path: typing.Optional[str] = None
|
|
112
|
+
conda: typing.Optional[str] = None
|
|
113
|
+
source: typing.Optional[str] = None
|
|
114
|
+
subpath: typing.Optional[str] = None
|
|
115
|
+
origin_url: typing.Optional[str] = None
|
|
116
|
+
desired_state: typing.Optional[ProjectDesiredState] = ProjectDesiredState.online
|
|
117
|
+
custom_packagers: typing.Optional[list[tuple[str, bool]]] = None
|
|
118
|
+
default_image: typing.Optional[str] = None
|
|
119
|
+
build: typing.Any = None
|
|
120
|
+
default_function_node_selector: typing.Optional[dict] = {}
|
|
121
|
+
|
|
122
|
+
class Config:
|
|
123
|
+
extra = pydantic.Extra.allow
|
|
124
|
+
|
|
125
|
+
|
|
103
126
|
class Project(pydantic.BaseModel):
|
|
104
127
|
kind: ObjectKind = pydantic.Field(ObjectKind.project, const=True)
|
|
105
128
|
metadata: ProjectMetadata
|
|
@@ -107,6 +130,15 @@ class Project(pydantic.BaseModel):
|
|
|
107
130
|
status: ObjectStatus = ObjectStatus()
|
|
108
131
|
|
|
109
132
|
|
|
133
|
+
# The reason we have a different schema for the response model is that we don't want to validate project.spec.build in
|
|
134
|
+
# the response as the validation was added late and there may be corrupted values in the DB.
|
|
135
|
+
class ProjectOut(pydantic.BaseModel):
|
|
136
|
+
kind: ObjectKind = pydantic.Field(ObjectKind.project, const=True)
|
|
137
|
+
metadata: ProjectMetadata
|
|
138
|
+
spec: ProjectSpecOut = ProjectSpecOut()
|
|
139
|
+
status: ObjectStatus = ObjectStatus()
|
|
140
|
+
|
|
141
|
+
|
|
110
142
|
class ProjectOwner(pydantic.BaseModel):
|
|
111
143
|
username: str
|
|
112
144
|
access_key: str
|
|
@@ -134,16 +166,16 @@ class IguazioProject(pydantic.BaseModel):
|
|
|
134
166
|
|
|
135
167
|
|
|
136
168
|
# The format query param controls the project type used:
|
|
137
|
-
# full -
|
|
169
|
+
# full - ProjectOut
|
|
138
170
|
# name_only - str
|
|
139
171
|
# summary - ProjectSummary
|
|
140
172
|
# leader - currently only IguazioProject supported
|
|
141
173
|
# The way pydantic handles typing.Union is that it takes the object and tries to coerce it to be the types of the
|
|
142
|
-
# union by the definition order. Therefore we can't currently add generic dict for all leader formats, but we need
|
|
174
|
+
# union by the definition order. Therefore, we can't currently add generic dict for all leader formats, but we need
|
|
143
175
|
# to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
|
|
144
176
|
# https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
|
|
145
177
|
ProjectOutput = typing.TypeVar(
|
|
146
|
-
"ProjectOutput",
|
|
178
|
+
"ProjectOutput", ProjectOut, str, ProjectSummary, IguazioProject
|
|
147
179
|
)
|
|
148
180
|
|
|
149
181
|
|
mlrun/config.py
CHANGED
|
@@ -252,7 +252,7 @@ default_config = {
|
|
|
252
252
|
},
|
|
253
253
|
"application": {
|
|
254
254
|
"default_sidecar_internal_port": 8050,
|
|
255
|
-
"default_authentication_mode":
|
|
255
|
+
"default_authentication_mode": mlrun.common.schemas.APIGatewayAuthenticationMode.none,
|
|
256
256
|
},
|
|
257
257
|
},
|
|
258
258
|
# TODO: function defaults should be moved to the function spec config above
|
|
@@ -1055,6 +1055,14 @@ class Config:
|
|
|
1055
1055
|
resource_requirement.pop(gpu)
|
|
1056
1056
|
return resource_requirement
|
|
1057
1057
|
|
|
1058
|
+
def force_api_gateway_ssl_redirect(self):
|
|
1059
|
+
"""
|
|
1060
|
+
Get the default value for the ssl_redirect configuration.
|
|
1061
|
+
In Iguazio we always want to redirect to HTTPS, in other cases we don't.
|
|
1062
|
+
:return: True if we should redirect to HTTPS, False otherwise.
|
|
1063
|
+
"""
|
|
1064
|
+
return self.is_running_on_iguazio()
|
|
1065
|
+
|
|
1058
1066
|
def to_dict(self):
|
|
1059
1067
|
return copy.deepcopy(self._cfg)
|
|
1060
1068
|
|
mlrun/datastore/base.py
CHANGED
|
@@ -215,7 +215,11 @@ class DataStore:
|
|
|
215
215
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
216
216
|
"When providing start_time or end_time, must provide time_column"
|
|
217
217
|
)
|
|
218
|
-
if
|
|
218
|
+
if (
|
|
219
|
+
start_time
|
|
220
|
+
and end_time
|
|
221
|
+
and start_time.utcoffset() != end_time.utcoffset()
|
|
222
|
+
):
|
|
219
223
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
220
224
|
"start_time and end_time must have the same time zone"
|
|
221
225
|
)
|
mlrun/db/base.py
CHANGED
|
@@ -175,7 +175,9 @@ class RunDBInterface(ABC):
|
|
|
175
175
|
pass
|
|
176
176
|
|
|
177
177
|
@abstractmethod
|
|
178
|
-
def list_functions(
|
|
178
|
+
def list_functions(
|
|
179
|
+
self, name=None, project="", tag="", labels=None, since=None, until=None
|
|
180
|
+
):
|
|
179
181
|
pass
|
|
180
182
|
|
|
181
183
|
@abstractmethod
|
|
@@ -688,8 +690,11 @@ class RunDBInterface(ABC):
|
|
|
688
690
|
@abstractmethod
|
|
689
691
|
def store_api_gateway(
|
|
690
692
|
self,
|
|
691
|
-
api_gateway:
|
|
692
|
-
|
|
693
|
+
api_gateway: Union[
|
|
694
|
+
mlrun.common.schemas.APIGateway,
|
|
695
|
+
"mlrun.runtimes.nuclio.api_gateway.APIGateway",
|
|
696
|
+
],
|
|
697
|
+
project: Optional[str] = None,
|
|
693
698
|
):
|
|
694
699
|
pass
|
|
695
700
|
|
mlrun/db/httpdb.py
CHANGED
|
@@ -1015,7 +1015,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1015
1015
|
"format": format_,
|
|
1016
1016
|
"tag": tag,
|
|
1017
1017
|
"tree": tree,
|
|
1018
|
-
"
|
|
1018
|
+
"object_uid": uid,
|
|
1019
1019
|
}
|
|
1020
1020
|
if iter is not None:
|
|
1021
1021
|
params["iter"] = str(iter)
|
|
@@ -1051,7 +1051,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1051
1051
|
"key": key,
|
|
1052
1052
|
"tag": tag,
|
|
1053
1053
|
"tree": tree,
|
|
1054
|
-
"
|
|
1054
|
+
"object_uid": uid,
|
|
1055
1055
|
"iter": iter,
|
|
1056
1056
|
"deletion_strategy": deletion_strategy,
|
|
1057
1057
|
}
|
|
@@ -1071,8 +1071,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1071
1071
|
project=None,
|
|
1072
1072
|
tag=None,
|
|
1073
1073
|
labels: Optional[Union[dict[str, str], list[str]]] = None,
|
|
1074
|
-
since=None,
|
|
1075
|
-
until=None,
|
|
1074
|
+
since: Optional[datetime] = None,
|
|
1075
|
+
until: Optional[datetime] = None,
|
|
1076
1076
|
iter: int = None,
|
|
1077
1077
|
best_iteration: bool = False,
|
|
1078
1078
|
kind: str = None,
|
|
@@ -1102,8 +1102,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1102
1102
|
:param tag: Return artifacts assigned this tag.
|
|
1103
1103
|
:param labels: Return artifacts that have these labels. Labels can either be a dictionary {"label": "value"} or
|
|
1104
1104
|
a list of "label=value" (match label key and value) or "label" (match just label key) strings.
|
|
1105
|
-
:param since:
|
|
1106
|
-
:param until:
|
|
1105
|
+
:param since: Return artifacts updated after this date (as datetime object).
|
|
1106
|
+
:param until: Return artifacts updated before this date (as datetime object).
|
|
1107
1107
|
:param iter: Return artifacts from a specific iteration (where ``iter=0`` means the root iteration). If
|
|
1108
1108
|
``None`` (default) return artifacts from all iterations.
|
|
1109
1109
|
:param best_iteration: Returns the artifact which belongs to the best iteration of a given run, in the case of
|
|
@@ -1137,6 +1137,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1137
1137
|
"format": format_,
|
|
1138
1138
|
"producer_uri": producer_uri,
|
|
1139
1139
|
"limit": limit,
|
|
1140
|
+
"since": datetime_to_iso(since),
|
|
1141
|
+
"until": datetime_to_iso(until),
|
|
1140
1142
|
}
|
|
1141
1143
|
error = "list artifacts"
|
|
1142
1144
|
endpoint_path = f"projects/{project}/artifacts"
|
|
@@ -1260,7 +1262,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1260
1262
|
|
|
1261
1263
|
:param name: Return only functions with a specific name.
|
|
1262
1264
|
:param project: Return functions belonging to this project. If not specified, the default project is used.
|
|
1263
|
-
:param tag: Return function versions with specific tags.
|
|
1265
|
+
:param tag: Return function versions with specific tags. To return only tagged functions, set tag to ``"*"``.
|
|
1264
1266
|
:param labels: Return functions that have specific labels assigned to them.
|
|
1265
1267
|
:param since: Return functions updated after this date (as datetime object).
|
|
1266
1268
|
:param until: Return functions updated before this date (as datetime object).
|
|
@@ -1684,7 +1686,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1684
1686
|
last_log_timestamp = float(
|
|
1685
1687
|
resp.headers.get("x-mlrun-last-timestamp", "0.0")
|
|
1686
1688
|
)
|
|
1687
|
-
if func.kind in mlrun.runtimes.RuntimeKinds.
|
|
1689
|
+
if func.kind in mlrun.runtimes.RuntimeKinds.pure_nuclio_deployed_runtimes():
|
|
1688
1690
|
mlrun.runtimes.nuclio.function.enrich_nuclio_function_from_headers(
|
|
1689
1691
|
func, resp.headers
|
|
1690
1692
|
)
|
mlrun/db/nopdb.py
CHANGED
|
@@ -178,7 +178,9 @@ class NopDB(RunDBInterface):
|
|
|
178
178
|
def delete_function(self, name: str, project: str = ""):
|
|
179
179
|
pass
|
|
180
180
|
|
|
181
|
-
def list_functions(
|
|
181
|
+
def list_functions(
|
|
182
|
+
self, name=None, project="", tag="", labels=None, since=None, until=None
|
|
183
|
+
):
|
|
182
184
|
pass
|
|
183
185
|
|
|
184
186
|
def tag_objects(
|
mlrun/execution.py
CHANGED
|
@@ -78,7 +78,6 @@ class MLClientCtx:
|
|
|
78
78
|
self._tmpfile = tmp
|
|
79
79
|
self._logger = log_stream or logger
|
|
80
80
|
self._log_level = "info"
|
|
81
|
-
self._matrics_db = None
|
|
82
81
|
self._autocommit = autocommit
|
|
83
82
|
self._notifications = []
|
|
84
83
|
self._state_thresholds = {}
|
|
@@ -103,8 +102,7 @@ class MLClientCtx:
|
|
|
103
102
|
self._error = None
|
|
104
103
|
self._commit = ""
|
|
105
104
|
self._host = None
|
|
106
|
-
self._start_time = now_date()
|
|
107
|
-
self._last_update = now_date()
|
|
105
|
+
self._start_time = self._last_update = now_date()
|
|
108
106
|
self._iteration_results = None
|
|
109
107
|
self._children = []
|
|
110
108
|
self._parent = None
|
mlrun/model.py
CHANGED
|
@@ -1490,14 +1490,37 @@ class RunObject(RunTemplate):
|
|
|
1490
1490
|
)
|
|
1491
1491
|
return ""
|
|
1492
1492
|
|
|
1493
|
-
def output(self, key):
|
|
1494
|
-
"""
|
|
1493
|
+
def output(self, key: str):
|
|
1494
|
+
"""
|
|
1495
|
+
Return the value of a specific result or artifact by key.
|
|
1496
|
+
|
|
1497
|
+
This method waits for the outputs to complete and retrieves the value corresponding to the provided key.
|
|
1498
|
+
If the key exists in the results, it returns the corresponding result value.
|
|
1499
|
+
If not found in results, it attempts to fetch the artifact by key (cached in the run status).
|
|
1500
|
+
If the artifact is not found, it tries to fetch the artifact URI by key.
|
|
1501
|
+
If no artifact or result is found for the key, returns None.
|
|
1502
|
+
|
|
1503
|
+
:param key: The key of the result or artifact to retrieve.
|
|
1504
|
+
:return: The value of the result or the artifact URI corresponding to the key, or None if not found.
|
|
1505
|
+
"""
|
|
1495
1506
|
self._outputs_wait_for_completion()
|
|
1507
|
+
|
|
1508
|
+
# Check if the key exists in results and return the result value
|
|
1496
1509
|
if self.status.results and key in self.status.results:
|
|
1497
|
-
return self.status.results
|
|
1510
|
+
return self.status.results[key]
|
|
1511
|
+
|
|
1512
|
+
# Artifacts are usually cached in the run object under `status.artifacts`. However, the artifacts are not
|
|
1513
|
+
# stored in the DB as part of the run. The server may enrich the run with the artifacts or provide
|
|
1514
|
+
# `status.artifact_uris` instead. See mlrun.common.formatters.run.RunFormat.
|
|
1515
|
+
# When running locally - `status.artifact_uri` does not exist in the run.
|
|
1516
|
+
# When listing runs - `status.artifacts` does not exist in the run.
|
|
1498
1517
|
artifact = self._artifact(key)
|
|
1499
1518
|
if artifact:
|
|
1500
1519
|
return get_artifact_target(artifact, self.metadata.project)
|
|
1520
|
+
|
|
1521
|
+
if self.status.artifact_uris and key in self.status.artifact_uris:
|
|
1522
|
+
return self.status.artifact_uris[key]
|
|
1523
|
+
|
|
1501
1524
|
return None
|
|
1502
1525
|
|
|
1503
1526
|
@property
|
|
@@ -1510,26 +1533,50 @@ class RunObject(RunTemplate):
|
|
|
1510
1533
|
|
|
1511
1534
|
@property
|
|
1512
1535
|
def outputs(self):
|
|
1513
|
-
"""
|
|
1514
|
-
outputs
|
|
1536
|
+
"""
|
|
1537
|
+
Return a dictionary of outputs, including result values and artifact URIs.
|
|
1538
|
+
|
|
1539
|
+
This method waits for the outputs to complete and combines result values
|
|
1540
|
+
and artifact URIs into a single dictionary. If there are multiple artifacts
|
|
1541
|
+
for the same key, only include the artifact that does not have the "latest" tag.
|
|
1542
|
+
If there is no other tag, include the "latest" tag as a fallback.
|
|
1543
|
+
|
|
1544
|
+
:return: Dictionary containing result values and artifact URIs.
|
|
1545
|
+
"""
|
|
1515
1546
|
self._outputs_wait_for_completion()
|
|
1547
|
+
outputs = {}
|
|
1548
|
+
|
|
1549
|
+
# Add results if available
|
|
1516
1550
|
if self.status.results:
|
|
1517
|
-
outputs
|
|
1551
|
+
outputs.update(self.status.results)
|
|
1552
|
+
|
|
1553
|
+
# Artifacts are usually cached in the run object under `status.artifacts`. However, the artifacts are not
|
|
1554
|
+
# stored in the DB as part of the run. The server may enrich the run with the artifacts or provide
|
|
1555
|
+
# `status.artifact_uris` instead. See mlrun.common.formatters.run.RunFormat.
|
|
1556
|
+
# When running locally - `status.artifact_uri` does not exist in the run.
|
|
1557
|
+
# When listing runs - `status.artifacts` does not exist in the run.
|
|
1518
1558
|
if self.status.artifacts:
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1559
|
+
outputs.update(self._process_artifacts(self.status.artifacts))
|
|
1560
|
+
elif self.status.artifact_uris:
|
|
1561
|
+
outputs.update(self.status.artifact_uris)
|
|
1562
|
+
|
|
1522
1563
|
return outputs
|
|
1523
1564
|
|
|
1524
|
-
def artifact(self, key) -> "mlrun.DataItem":
|
|
1525
|
-
"""
|
|
1565
|
+
def artifact(self, key: str) -> "mlrun.DataItem":
|
|
1566
|
+
"""Return artifact DataItem by key.
|
|
1567
|
+
|
|
1568
|
+
This method waits for the outputs to complete, searches for the artifact matching the given key,
|
|
1569
|
+
and returns a DataItem if the artifact is found.
|
|
1570
|
+
|
|
1571
|
+
:param key: The key of the artifact to find.
|
|
1572
|
+
:return: A DataItem corresponding to the artifact with the given key, or None if no such artifact is found.
|
|
1573
|
+
"""
|
|
1526
1574
|
self._outputs_wait_for_completion()
|
|
1527
1575
|
artifact = self._artifact(key)
|
|
1528
|
-
if artifact:
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
return None
|
|
1576
|
+
if not artifact:
|
|
1577
|
+
return None
|
|
1578
|
+
uri = get_artifact_target(artifact, self.metadata.project)
|
|
1579
|
+
return mlrun.get_dataitem(uri) if uri else None
|
|
1533
1580
|
|
|
1534
1581
|
def _outputs_wait_for_completion(
|
|
1535
1582
|
self,
|
|
@@ -1547,12 +1594,85 @@ class RunObject(RunTemplate):
|
|
|
1547
1594
|
)
|
|
1548
1595
|
|
|
1549
1596
|
def _artifact(self, key):
|
|
1550
|
-
"""
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1597
|
+
"""
|
|
1598
|
+
Return the last artifact DataItem that matches the given key.
|
|
1599
|
+
|
|
1600
|
+
If multiple artifacts with the same key exist, return the last one in the list.
|
|
1601
|
+
If there are artifacts with different tags, the method will return the one with a tag other than 'latest'
|
|
1602
|
+
if available.
|
|
1603
|
+
If no artifact with the given key is found, return None.
|
|
1604
|
+
|
|
1605
|
+
:param key: The key of the artifact to retrieve.
|
|
1606
|
+
:return: The last artifact DataItem with the given key, or None if no such artifact is found.
|
|
1607
|
+
"""
|
|
1608
|
+
if not self.status.artifacts:
|
|
1609
|
+
return None
|
|
1610
|
+
|
|
1611
|
+
# Collect artifacts that match the key
|
|
1612
|
+
matching_artifacts = [
|
|
1613
|
+
artifact
|
|
1614
|
+
for artifact in self.status.artifacts
|
|
1615
|
+
if artifact["metadata"].get("key") == key
|
|
1616
|
+
]
|
|
1617
|
+
|
|
1618
|
+
if not matching_artifacts:
|
|
1619
|
+
return None
|
|
1620
|
+
|
|
1621
|
+
# Sort matching artifacts by creation date in ascending order.
|
|
1622
|
+
# The last element in the list will be the one created most recently.
|
|
1623
|
+
# In case the `created` field does not exist in the artifact, that artifact will appear first in the sorted list
|
|
1624
|
+
matching_artifacts.sort(
|
|
1625
|
+
key=lambda artifact: artifact["metadata"].get("created", datetime.min)
|
|
1626
|
+
)
|
|
1627
|
+
|
|
1628
|
+
# Filter out artifacts with 'latest' tag
|
|
1629
|
+
non_latest_artifacts = [
|
|
1630
|
+
artifact
|
|
1631
|
+
for artifact in matching_artifacts
|
|
1632
|
+
if artifact["metadata"].get("tag") != "latest"
|
|
1633
|
+
]
|
|
1634
|
+
|
|
1635
|
+
# Return the last non-'latest' artifact if available, otherwise return the last artifact
|
|
1636
|
+
# In the case of only one tag, `status.artifacts` includes [v1, latest]. In that case, we want to return v1.
|
|
1637
|
+
# In the case of multiple tags, `status.artifacts` includes [v1, latest, v2, v3].
|
|
1638
|
+
# In that case, we need to return the last one (v3).
|
|
1639
|
+
return (non_latest_artifacts or matching_artifacts)[-1]
|
|
1640
|
+
|
|
1641
|
+
def _process_artifacts(self, artifacts):
|
|
1642
|
+
artifacts_by_key = {}
|
|
1643
|
+
|
|
1644
|
+
# Organize artifacts by key
|
|
1645
|
+
for artifact in artifacts:
|
|
1646
|
+
key = artifact["metadata"]["key"]
|
|
1647
|
+
if key not in artifacts_by_key:
|
|
1648
|
+
artifacts_by_key[key] = []
|
|
1649
|
+
artifacts_by_key[key].append(artifact)
|
|
1650
|
+
|
|
1651
|
+
outputs = {}
|
|
1652
|
+
for key, artifacts in artifacts_by_key.items():
|
|
1653
|
+
# Sort matching artifacts by creation date in ascending order.
|
|
1654
|
+
# The last element in the list will be the one created most recently.
|
|
1655
|
+
# In case the `created` field does not exist in the artifactthat artifact will appear
|
|
1656
|
+
# first in the sorted list
|
|
1657
|
+
artifacts.sort(
|
|
1658
|
+
key=lambda artifact: artifact["metadata"].get("created", datetime.min)
|
|
1659
|
+
)
|
|
1660
|
+
|
|
1661
|
+
# Filter out artifacts with 'latest' tag
|
|
1662
|
+
non_latest_artifacts = [
|
|
1663
|
+
artifact
|
|
1664
|
+
for artifact in artifacts
|
|
1665
|
+
if artifact["metadata"].get("tag") != "latest"
|
|
1666
|
+
]
|
|
1667
|
+
|
|
1668
|
+
# Save the last non-'latest' artifact if available, otherwise save the last artifact
|
|
1669
|
+
# In the case of only one tag, `artifacts` includes [v1, latest], in that case, we want to save v1.
|
|
1670
|
+
# In the case of multiple tags, `artifacts` includes [v1, latest, v2, v3].
|
|
1671
|
+
# In that case, we need to save the last one (v3).
|
|
1672
|
+
artifact_to_save = (non_latest_artifacts or artifacts)[-1]
|
|
1673
|
+
outputs[key] = get_artifact_target(artifact_to_save, self.metadata.project)
|
|
1674
|
+
|
|
1675
|
+
return outputs
|
|
1556
1676
|
|
|
1557
1677
|
def uid(self):
|
|
1558
1678
|
"""run unique id"""
|
|
@@ -56,7 +56,7 @@ class MonitoringApplicationContext(MLClientCtx):
|
|
|
56
56
|
def __init__(self, **kwargs):
|
|
57
57
|
super().__init__(**kwargs)
|
|
58
58
|
|
|
59
|
-
def
|
|
59
|
+
def _enrich_data(self):
|
|
60
60
|
self.application_name: typing.Optional[str] = None
|
|
61
61
|
self.start_infer_time: typing.Optional[pd.Timestamp] = None
|
|
62
62
|
self.end_infer_time: typing.Optional[pd.Timestamp] = None
|
|
@@ -87,39 +87,37 @@ class MonitoringApplicationContext(MLClientCtx):
|
|
|
87
87
|
"""
|
|
88
88
|
|
|
89
89
|
if not context:
|
|
90
|
-
|
|
90
|
+
ctx = (
|
|
91
91
|
super().from_dict(
|
|
92
92
|
attrs=attrs.get(mm_constants.ApplicationEvent.MLRUN_CONTEXT, {}),
|
|
93
93
|
**kwargs,
|
|
94
94
|
),
|
|
95
95
|
)
|
|
96
96
|
else:
|
|
97
|
-
|
|
98
|
-
|
|
97
|
+
ctx = context
|
|
98
|
+
cls._enrich_data(ctx)
|
|
99
99
|
|
|
100
|
-
|
|
100
|
+
ctx.start_infer_time = pd.Timestamp(
|
|
101
101
|
attrs.get(mm_constants.ApplicationEvent.START_INFER_TIME)
|
|
102
102
|
)
|
|
103
|
-
|
|
103
|
+
ctx.end_infer_time = pd.Timestamp(
|
|
104
104
|
attrs.get(mm_constants.ApplicationEvent.END_INFER_TIME)
|
|
105
105
|
)
|
|
106
|
-
|
|
106
|
+
ctx.latest_request = pd.Timestamp(
|
|
107
107
|
attrs.get(mm_constants.ApplicationEvent.LAST_REQUEST)
|
|
108
108
|
)
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
)
|
|
112
|
-
self._feature_stats = json.loads(
|
|
109
|
+
ctx.application_name = attrs.get(mm_constants.ApplicationEvent.APPLICATION_NAME)
|
|
110
|
+
ctx._feature_stats = json.loads(
|
|
113
111
|
attrs.get(mm_constants.ApplicationEvent.FEATURE_STATS, "{}")
|
|
114
112
|
)
|
|
115
|
-
|
|
113
|
+
ctx._sample_df_stats = json.loads(
|
|
116
114
|
attrs.get(mm_constants.ApplicationEvent.CURRENT_STATS, "{}")
|
|
117
115
|
)
|
|
118
116
|
|
|
119
|
-
|
|
120
|
-
|
|
117
|
+
ctx.endpoint_id = attrs.get(mm_constants.ApplicationEvent.ENDPOINT_ID)
|
|
118
|
+
ctx._model_endpoint = model_endpoint_dict.get(ctx.endpoint_id)
|
|
121
119
|
|
|
122
|
-
return
|
|
120
|
+
return ctx
|
|
123
121
|
|
|
124
122
|
@property
|
|
125
123
|
def sample_df(self) -> pd.DataFrame:
|
|
@@ -328,7 +328,7 @@ class MonitoringApplicationController:
|
|
|
328
328
|
logger.info("Start running monitoring controller")
|
|
329
329
|
try:
|
|
330
330
|
applications_names = []
|
|
331
|
-
endpoints = self.db.list_model_endpoints()
|
|
331
|
+
endpoints = self.db.list_model_endpoints(include_stats=True)
|
|
332
332
|
if not endpoints:
|
|
333
333
|
logger.info("No model endpoints found", project=self.project)
|
|
334
334
|
return
|
|
@@ -94,6 +94,7 @@ class StoreBase(ABC):
|
|
|
94
94
|
labels: list[str] = None,
|
|
95
95
|
top_level: bool = None,
|
|
96
96
|
uids: list = None,
|
|
97
|
+
include_stats: bool = None,
|
|
97
98
|
) -> list[dict[str, typing.Any]]:
|
|
98
99
|
"""
|
|
99
100
|
Returns a list of model endpoint dictionaries, supports filtering by model, function, labels or top level.
|
|
@@ -107,6 +108,7 @@ class StoreBase(ABC):
|
|
|
107
108
|
key (i.e. "key").
|
|
108
109
|
:param top_level: If True will return only routers and endpoint that are NOT children of any router.
|
|
109
110
|
:param uids: List of model endpoint unique ids to include in the result.
|
|
111
|
+
:param include_stats: If True, will include model endpoint statistics in the result.
|
|
110
112
|
|
|
111
113
|
:return: A list of model endpoint dictionaries.
|
|
112
114
|
"""
|
|
@@ -266,22 +266,8 @@ class SQLStoreBase(StoreBase):
|
|
|
266
266
|
labels: list[str] = None,
|
|
267
267
|
top_level: bool = None,
|
|
268
268
|
uids: list = None,
|
|
269
|
+
include_stats: bool = None,
|
|
269
270
|
) -> list[dict[str, typing.Any]]:
|
|
270
|
-
"""
|
|
271
|
-
Returns a list of model endpoint dictionaries, supports filtering by model, function, labels or top level.
|
|
272
|
-
By default, when no filters are applied, all available model endpoints for the given project will
|
|
273
|
-
be listed.
|
|
274
|
-
|
|
275
|
-
:param model: The name of the model to filter by.
|
|
276
|
-
:param function: The name of the function to filter by.
|
|
277
|
-
:param labels: A list of labels to filter by. Label filters work by either filtering a specific value
|
|
278
|
-
of a label (i.e. list("key=value")) or by looking for the existence of a given
|
|
279
|
-
key (i.e. "key").
|
|
280
|
-
:param top_level: If True will return only routers and endpoint that are NOT children of any router.
|
|
281
|
-
:param uids: List of model endpoint unique ids to include in the result.
|
|
282
|
-
|
|
283
|
-
:return: A list of model endpoint dictionaries.
|
|
284
|
-
"""
|
|
285
271
|
# Generate an empty model endpoints that will be filled afterwards with model endpoint dictionaries
|
|
286
272
|
endpoint_list = []
|
|
287
273
|
|
|
@@ -291,14 +277,8 @@ class SQLStoreBase(StoreBase):
|
|
|
291
277
|
# Get the model endpoints records using sqlalchemy ORM
|
|
292
278
|
with create_session(dsn=self._sql_connection_string) as session:
|
|
293
279
|
# Generate the list query
|
|
294
|
-
query = (
|
|
295
|
-
|
|
296
|
-
.options(
|
|
297
|
-
# Exclude these fields when listing model endpoints to avoid returning too much data (ML-6594)
|
|
298
|
-
sqlalchemy.orm.defer(mm_schemas.EventFieldType.FEATURE_STATS),
|
|
299
|
-
sqlalchemy.orm.defer(mm_schemas.EventFieldType.CURRENT_STATS),
|
|
300
|
-
)
|
|
301
|
-
.filter_by(project=self.project)
|
|
280
|
+
query = session.query(self.model_endpoints_table).filter_by(
|
|
281
|
+
project=self.project
|
|
302
282
|
)
|
|
303
283
|
|
|
304
284
|
# Apply filters
|
|
@@ -347,6 +327,12 @@ class SQLStoreBase(StoreBase):
|
|
|
347
327
|
):
|
|
348
328
|
continue
|
|
349
329
|
|
|
330
|
+
if not include_stats:
|
|
331
|
+
# Exclude these fields when listing model endpoints to avoid returning too much data (ML-6594)
|
|
332
|
+
# TODO: Remove stats from table schema (ML-7196)
|
|
333
|
+
endpoint_dict.pop(mm_schemas.EventFieldType.FEATURE_STATS)
|
|
334
|
+
endpoint_dict.pop(mm_schemas.EventFieldType.CURRENT_STATS)
|
|
335
|
+
|
|
350
336
|
endpoint_list.append(endpoint_dict)
|
|
351
337
|
|
|
352
338
|
return endpoint_list
|
|
@@ -226,24 +226,8 @@ class KVStoreBase(StoreBase):
|
|
|
226
226
|
labels: list[str] = None,
|
|
227
227
|
top_level: bool = None,
|
|
228
228
|
uids: list = None,
|
|
229
|
+
include_stats: bool = None,
|
|
229
230
|
) -> list[dict[str, typing.Any]]:
|
|
230
|
-
"""
|
|
231
|
-
Returns a list of model endpoint dictionaries, supports filtering by model, function, labels or top level.
|
|
232
|
-
By default, when no filters are applied, all available model endpoints for the given project will
|
|
233
|
-
be listed.
|
|
234
|
-
|
|
235
|
-
:param model: The name of the model to filter by.
|
|
236
|
-
:param function: The name of the function to filter by.
|
|
237
|
-
:param labels: A list of labels to filter by. Label filters work by either filtering a specific value
|
|
238
|
-
of a label (i.e. list("key=value")) or by looking for the existence of a given
|
|
239
|
-
key (i.e. "key").
|
|
240
|
-
:param top_level: If True will return only routers and endpoint that are NOT children of any router.
|
|
241
|
-
:param uids: List of model endpoint unique ids to include in the result.
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
:return: A list of model endpoint dictionaries.
|
|
245
|
-
"""
|
|
246
|
-
|
|
247
231
|
# # Initialize an empty model endpoints list
|
|
248
232
|
endpoint_list = []
|
|
249
233
|
|
|
@@ -283,9 +267,10 @@ class KVStoreBase(StoreBase):
|
|
|
283
267
|
endpoint_dict = self.get_model_endpoint(
|
|
284
268
|
endpoint_id=endpoint_id,
|
|
285
269
|
)
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
270
|
+
if not include_stats:
|
|
271
|
+
# Exclude these fields when listing model endpoints to avoid returning too much data (ML-6594)
|
|
272
|
+
endpoint_dict.pop(mm_schemas.EventFieldType.FEATURE_STATS)
|
|
273
|
+
endpoint_dict.pop(mm_schemas.EventFieldType.CURRENT_STATS)
|
|
289
274
|
|
|
290
275
|
if labels and not self._validate_labels(
|
|
291
276
|
endpoint_dict=endpoint_dict, labels=labels
|
|
@@ -773,6 +773,12 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
|
|
|
773
773
|
|
|
774
774
|
feature_values = event[EventFieldType.FEATURES]
|
|
775
775
|
label_values = event[EventFieldType.PREDICTION]
|
|
776
|
+
|
|
777
|
+
for index in range(len(feature_values)):
|
|
778
|
+
feature_value = feature_values[index]
|
|
779
|
+
if isinstance(feature_value, int):
|
|
780
|
+
feature_values[index] = float(feature_value)
|
|
781
|
+
|
|
776
782
|
# Get feature names and label columns
|
|
777
783
|
if endpoint_id not in self.feature_names:
|
|
778
784
|
endpoint_record = mlrun.model_monitoring.helpers.get_endpoint_record(
|