mlrun 1.7.0rc20__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +10 -8
- mlrun/alerts/alert.py +55 -18
- mlrun/api/schemas/__init__.py +3 -3
- mlrun/artifacts/manager.py +26 -0
- mlrun/common/constants.py +3 -2
- mlrun/common/formatters/__init__.py +1 -0
- mlrun/common/formatters/artifact.py +26 -3
- mlrun/common/formatters/base.py +44 -9
- mlrun/common/formatters/function.py +12 -7
- mlrun/common/formatters/run.py +26 -0
- mlrun/common/helpers.py +11 -0
- mlrun/common/schemas/__init__.py +4 -0
- mlrun/common/schemas/alert.py +5 -9
- mlrun/common/schemas/api_gateway.py +64 -16
- mlrun/common/schemas/artifact.py +11 -0
- mlrun/common/schemas/constants.py +3 -0
- mlrun/common/schemas/feature_store.py +58 -28
- mlrun/common/schemas/model_monitoring/constants.py +21 -12
- mlrun/common/schemas/model_monitoring/model_endpoints.py +0 -12
- mlrun/common/schemas/pipeline.py +16 -0
- mlrun/common/schemas/project.py +17 -0
- mlrun/common/schemas/runs.py +17 -0
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/types.py +6 -0
- mlrun/config.py +17 -25
- mlrun/datastore/azure_blob.py +2 -1
- mlrun/datastore/datastore.py +3 -3
- mlrun/datastore/google_cloud_storage.py +6 -2
- mlrun/datastore/snowflake_utils.py +3 -1
- mlrun/datastore/sources.py +26 -11
- mlrun/datastore/store_resources.py +2 -0
- mlrun/datastore/targets.py +68 -16
- mlrun/db/base.py +83 -2
- mlrun/db/httpdb.py +280 -63
- mlrun/db/nopdb.py +60 -3
- mlrun/errors.py +5 -3
- mlrun/execution.py +28 -13
- mlrun/feature_store/feature_vector.py +8 -0
- mlrun/feature_store/retrieval/spark_merger.py +13 -2
- mlrun/launcher/local.py +4 -0
- mlrun/launcher/remote.py +1 -0
- mlrun/model.py +32 -3
- mlrun/model_monitoring/api.py +7 -52
- mlrun/model_monitoring/applications/base.py +5 -7
- mlrun/model_monitoring/applications/histogram_data_drift.py +1 -1
- mlrun/model_monitoring/db/stores/__init__.py +37 -24
- mlrun/model_monitoring/db/stores/base/store.py +40 -1
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +42 -87
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +27 -35
- mlrun/model_monitoring/db/tsdb/__init__.py +15 -15
- mlrun/model_monitoring/db/tsdb/base.py +1 -14
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +22 -18
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +86 -56
- mlrun/model_monitoring/helpers.py +34 -9
- mlrun/model_monitoring/stream_processing.py +12 -11
- mlrun/model_monitoring/writer.py +11 -11
- mlrun/projects/operations.py +5 -0
- mlrun/projects/pipelines.py +35 -21
- mlrun/projects/project.py +216 -107
- mlrun/render.py +10 -5
- mlrun/run.py +15 -5
- mlrun/runtimes/__init__.py +2 -0
- mlrun/runtimes/base.py +17 -4
- mlrun/runtimes/daskjob.py +8 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +1 -0
- mlrun/runtimes/local.py +23 -4
- mlrun/runtimes/nuclio/application/application.py +0 -2
- mlrun/runtimes/nuclio/function.py +31 -2
- mlrun/runtimes/nuclio/serving.py +9 -6
- mlrun/runtimes/pod.py +5 -29
- mlrun/runtimes/remotesparkjob.py +8 -2
- mlrun/serving/__init__.py +8 -1
- mlrun/serving/routers.py +75 -59
- mlrun/serving/server.py +11 -0
- mlrun/serving/states.py +80 -8
- mlrun/serving/utils.py +19 -11
- mlrun/serving/v2_serving.py +66 -39
- mlrun/utils/helpers.py +91 -11
- mlrun/utils/logger.py +36 -2
- mlrun/utils/notifications/notification/base.py +43 -7
- mlrun/utils/notifications/notification/git.py +21 -0
- mlrun/utils/notifications/notification/slack.py +9 -14
- mlrun/utils/notifications/notification/webhook.py +41 -1
- mlrun/utils/notifications/notification_pusher.py +3 -9
- mlrun/utils/regex.py +9 -0
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/METADATA +16 -9
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/RECORD +92 -91
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/WHEEL +1 -1
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/top_level.txt +0 -0
|
@@ -31,17 +31,12 @@ class ObjectStoreFactory(enum.Enum):
|
|
|
31
31
|
def to_object_store(
|
|
32
32
|
self,
|
|
33
33
|
project: str,
|
|
34
|
-
|
|
35
|
-
secret_provider: typing.Callable = None,
|
|
34
|
+
**kwargs,
|
|
36
35
|
) -> StoreBase:
|
|
37
36
|
"""
|
|
38
37
|
Return a StoreBase object based on the provided enum value.
|
|
39
38
|
|
|
40
39
|
:param project: The name of the project.
|
|
41
|
-
:param access_key: Access key with permission to the DB table. Note that if access key is None
|
|
42
|
-
and the endpoint target is from type KV then the access key will be
|
|
43
|
-
retrieved from the environment variable.
|
|
44
|
-
:param secret_provider: An optional secret provider to get the connection string secret.
|
|
45
40
|
|
|
46
41
|
:return: `StoreBase` object.
|
|
47
42
|
|
|
@@ -50,10 +45,7 @@ class ObjectStoreFactory(enum.Enum):
|
|
|
50
45
|
if self == self.v3io_nosql:
|
|
51
46
|
from mlrun.model_monitoring.db.stores.v3io_kv.kv_store import KVStoreBase
|
|
52
47
|
|
|
53
|
-
|
|
54
|
-
access_key = access_key or mlrun.mlconf.get_v3io_access_key()
|
|
55
|
-
|
|
56
|
-
return KVStoreBase(project=project, access_key=access_key)
|
|
48
|
+
return KVStoreBase(project=project)
|
|
57
49
|
|
|
58
50
|
# Assuming SQL store target if store type is not KV.
|
|
59
51
|
# Update these lines once there are more than two store target types.
|
|
@@ -62,7 +54,7 @@ class ObjectStoreFactory(enum.Enum):
|
|
|
62
54
|
|
|
63
55
|
return SQLStoreBase(
|
|
64
56
|
project=project,
|
|
65
|
-
|
|
57
|
+
**kwargs,
|
|
66
58
|
)
|
|
67
59
|
|
|
68
60
|
@classmethod
|
|
@@ -71,7 +63,7 @@ class ObjectStoreFactory(enum.Enum):
|
|
|
71
63
|
:param value: Provided enum (invalid) value.
|
|
72
64
|
"""
|
|
73
65
|
valid_values = list(cls.__members__.keys())
|
|
74
|
-
raise mlrun.errors.
|
|
66
|
+
raise mlrun.errors.MLRunInvalidMMStoreType(
|
|
75
67
|
f"{value} is not a valid endpoint store, please choose a valid value: %{valid_values}."
|
|
76
68
|
)
|
|
77
69
|
|
|
@@ -79,7 +71,7 @@ class ObjectStoreFactory(enum.Enum):
|
|
|
79
71
|
def get_model_endpoint_store(
|
|
80
72
|
project: str,
|
|
81
73
|
access_key: str = None,
|
|
82
|
-
secret_provider: typing.Callable = None,
|
|
74
|
+
secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
|
|
83
75
|
) -> StoreBase:
|
|
84
76
|
# Leaving here for backwards compatibility
|
|
85
77
|
warnings.warn(
|
|
@@ -95,24 +87,45 @@ def get_model_endpoint_store(
|
|
|
95
87
|
|
|
96
88
|
def get_store_object(
|
|
97
89
|
project: str,
|
|
98
|
-
|
|
99
|
-
|
|
90
|
+
secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
|
|
91
|
+
store_connection_string: typing.Optional[str] = None,
|
|
92
|
+
**kwargs,
|
|
100
93
|
) -> StoreBase:
|
|
101
94
|
"""
|
|
102
|
-
|
|
95
|
+
Generate a store object. If a connection string is provided, the store type will be updated according to the
|
|
96
|
+
connection string. Currently, the supported store types are SQL and v3io-nosql.
|
|
103
97
|
|
|
104
|
-
:param project:
|
|
105
|
-
:param
|
|
106
|
-
:param
|
|
98
|
+
:param project: The name of the project.
|
|
99
|
+
:param secret_provider: An optional secret provider to get the connection string secret.
|
|
100
|
+
:param store_connection_string: Optional explicit connection string of the store.
|
|
107
101
|
|
|
108
|
-
:return: `StoreBase` object. Using this object, the user can apply different operations
|
|
109
|
-
|
|
102
|
+
:return: `StoreBase` object. Using this object, the user can apply different operations such as write, update, get
|
|
103
|
+
and delete a model endpoint record.
|
|
110
104
|
"""
|
|
111
105
|
|
|
106
|
+
store_connection_string = (
|
|
107
|
+
store_connection_string
|
|
108
|
+
or mlrun.model_monitoring.helpers.get_connection_string(
|
|
109
|
+
secret_provider=secret_provider
|
|
110
|
+
)
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
if store_connection_string and (
|
|
114
|
+
store_connection_string.startswith("mysql")
|
|
115
|
+
or store_connection_string.startswith("sqlite")
|
|
116
|
+
):
|
|
117
|
+
store_type = mlrun.common.schemas.model_monitoring.ModelEndpointTarget.SQL
|
|
118
|
+
kwargs["store_connection_string"] = store_connection_string
|
|
119
|
+
elif store_connection_string and store_connection_string == "v3io":
|
|
120
|
+
store_type = (
|
|
121
|
+
mlrun.common.schemas.model_monitoring.ModelEndpointTarget.V3IO_NOSQL
|
|
122
|
+
)
|
|
123
|
+
else:
|
|
124
|
+
store_type = None
|
|
112
125
|
# Get store type value from ObjectStoreFactory enum class
|
|
113
|
-
|
|
126
|
+
store_type_fact = ObjectStoreFactory(store_type)
|
|
114
127
|
|
|
115
128
|
# Convert into store target object
|
|
116
|
-
return
|
|
117
|
-
project=project,
|
|
129
|
+
return store_type_fact.to_object_store(
|
|
130
|
+
project=project, secret_provider=secret_provider, **kwargs
|
|
118
131
|
)
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
import json
|
|
15
15
|
import typing
|
|
16
16
|
from abc import ABC, abstractmethod
|
|
17
17
|
|
|
@@ -19,6 +19,7 @@ import mlrun.common.schemas.model_monitoring as mm_schemas
|
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
class StoreBase(ABC):
|
|
22
|
+
type: typing.ClassVar[str]
|
|
22
23
|
"""
|
|
23
24
|
An abstract class to handle the store object in the DB target.
|
|
24
25
|
"""
|
|
@@ -169,3 +170,41 @@ class StoreBase(ABC):
|
|
|
169
170
|
|
|
170
171
|
:return: A list of the available metrics.
|
|
171
172
|
"""
|
|
173
|
+
|
|
174
|
+
@staticmethod
|
|
175
|
+
def _validate_labels(
|
|
176
|
+
endpoint_dict: dict,
|
|
177
|
+
labels: list,
|
|
178
|
+
) -> bool:
|
|
179
|
+
"""Validate that the model endpoint dictionary has the provided labels. There are 2 possible cases:
|
|
180
|
+
1 - Labels were provided as a list of key-values pairs (e.g. ['label_1=value_1', 'label_2=value_2']): Validate
|
|
181
|
+
that each pair exist in the endpoint dictionary.
|
|
182
|
+
2 - Labels were provided as a list of key labels (e.g. ['label_1', 'label_2']): Validate that each key exist in
|
|
183
|
+
the endpoint labels dictionary.
|
|
184
|
+
|
|
185
|
+
:param endpoint_dict: Dictionary of the model endpoint records.
|
|
186
|
+
:param labels: List of dictionary of required labels.
|
|
187
|
+
|
|
188
|
+
:return: True if the labels exist in the endpoint labels dictionary, otherwise False.
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
# Convert endpoint labels into dictionary
|
|
192
|
+
endpoint_labels = json.loads(
|
|
193
|
+
endpoint_dict.get(mm_schemas.EventFieldType.LABELS)
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
for label in labels:
|
|
197
|
+
# Case 1 - label is a key=value pair
|
|
198
|
+
if "=" in label:
|
|
199
|
+
lbl, value = list(map(lambda x: x.strip(), label.split("=")))
|
|
200
|
+
if lbl not in endpoint_labels or str(endpoint_labels[lbl]) != value:
|
|
201
|
+
return False
|
|
202
|
+
# Case 2 - label is just a key
|
|
203
|
+
else:
|
|
204
|
+
if label not in endpoint_labels:
|
|
205
|
+
return False
|
|
206
|
+
|
|
207
|
+
return True
|
|
208
|
+
|
|
209
|
+
def create_tables(self):
|
|
210
|
+
pass
|
|
@@ -13,7 +13,6 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import datetime
|
|
16
|
-
import json
|
|
17
16
|
import typing
|
|
18
17
|
import uuid
|
|
19
18
|
|
|
@@ -21,18 +20,20 @@ import pandas as pd
|
|
|
21
20
|
import sqlalchemy
|
|
22
21
|
import sqlalchemy.exc
|
|
23
22
|
import sqlalchemy.orm
|
|
23
|
+
from sqlalchemy.engine import make_url
|
|
24
24
|
from sqlalchemy.sql.elements import BinaryExpression
|
|
25
25
|
|
|
26
26
|
import mlrun.common.model_monitoring.helpers
|
|
27
27
|
import mlrun.common.schemas.model_monitoring as mm_schemas
|
|
28
|
-
import mlrun.model_monitoring.db
|
|
29
28
|
import mlrun.model_monitoring.db.stores.sqldb.models
|
|
30
29
|
import mlrun.model_monitoring.helpers
|
|
31
30
|
from mlrun.common.db.sql_session import create_session, get_engine
|
|
31
|
+
from mlrun.model_monitoring.db import StoreBase
|
|
32
32
|
from mlrun.utils import datetime_now, logger
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
class SQLStoreBase(
|
|
35
|
+
class SQLStoreBase(StoreBase):
|
|
36
|
+
type: typing.ClassVar[str] = mm_schemas.ModelEndpointTarget.SQL
|
|
36
37
|
"""
|
|
37
38
|
Handles the DB operations when the DB target is from type SQL. For the SQL operations, we use SQLAlchemy, a Python
|
|
38
39
|
SQL toolkit that handles the communication with the database. When using SQL for storing the model monitoring
|
|
@@ -44,24 +45,27 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
44
45
|
def __init__(
|
|
45
46
|
self,
|
|
46
47
|
project: str,
|
|
47
|
-
|
|
48
|
+
**kwargs,
|
|
48
49
|
):
|
|
49
50
|
"""
|
|
50
51
|
Initialize SQL store target object.
|
|
51
52
|
|
|
52
53
|
:param project: The name of the project.
|
|
53
|
-
:param secret_provider: An optional secret provider to get the connection string secret.
|
|
54
54
|
"""
|
|
55
55
|
|
|
56
56
|
super().__init__(project=project)
|
|
57
57
|
|
|
58
|
-
|
|
59
|
-
mlrun.
|
|
60
|
-
|
|
58
|
+
if "store_connection_string" not in kwargs:
|
|
59
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
60
|
+
"connection_string is a required parameter for SQLStoreBase."
|
|
61
61
|
)
|
|
62
|
-
)
|
|
63
62
|
|
|
63
|
+
self._sql_connection_string = kwargs.get("store_connection_string")
|
|
64
64
|
self._engine = get_engine(dsn=self._sql_connection_string)
|
|
65
|
+
self._init_tables()
|
|
66
|
+
|
|
67
|
+
def create_tables(self):
|
|
68
|
+
self._create_tables_if_not_exist()
|
|
65
69
|
|
|
66
70
|
def _init_tables(self):
|
|
67
71
|
self._init_model_endpoints_table()
|
|
@@ -70,13 +74,13 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
70
74
|
self._init_monitoring_schedules_table()
|
|
71
75
|
|
|
72
76
|
def _init_model_endpoints_table(self):
|
|
73
|
-
self.
|
|
77
|
+
self.model_endpoints_table = (
|
|
74
78
|
mlrun.model_monitoring.db.stores.sqldb.models._get_model_endpoints_table(
|
|
75
79
|
connection_string=self._sql_connection_string
|
|
76
80
|
)
|
|
77
81
|
)
|
|
78
82
|
self._tables[mm_schemas.EventFieldType.MODEL_ENDPOINTS] = (
|
|
79
|
-
self.
|
|
83
|
+
self.model_endpoints_table
|
|
80
84
|
)
|
|
81
85
|
|
|
82
86
|
def _init_application_results_table(self):
|
|
@@ -150,22 +154,17 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
150
154
|
:param criteria: A list of binary expressions that filter the query.
|
|
151
155
|
"""
|
|
152
156
|
with create_session(dsn=self._sql_connection_string) as session:
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
)
|
|
165
|
-
except sqlalchemy.exc.ProgrammingError:
|
|
166
|
-
# Probably table doesn't exist, try to create tables
|
|
167
|
-
self._create_tables_if_not_exist()
|
|
168
|
-
return
|
|
157
|
+
logger.debug(
|
|
158
|
+
"Querying the DB",
|
|
159
|
+
table=table.__name__,
|
|
160
|
+
criteria=[str(criterion) for criterion in criteria],
|
|
161
|
+
)
|
|
162
|
+
# Generate the get query
|
|
163
|
+
return (
|
|
164
|
+
session.query(table) # pyright: ignore[reportOptionalCall]
|
|
165
|
+
.filter(*criteria)
|
|
166
|
+
.one_or_none()
|
|
167
|
+
)
|
|
169
168
|
|
|
170
169
|
def _delete(
|
|
171
170
|
self,
|
|
@@ -213,14 +212,13 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
213
212
|
of the attributes dictionary should exist in the SQL table.
|
|
214
213
|
|
|
215
214
|
"""
|
|
216
|
-
self._init_model_endpoints_table()
|
|
217
215
|
|
|
218
216
|
attributes.pop(mm_schemas.EventFieldType.ENDPOINT_ID, None)
|
|
219
217
|
|
|
220
218
|
self._update(
|
|
221
219
|
attributes=attributes,
|
|
222
|
-
table=self.
|
|
223
|
-
criteria=[self.
|
|
220
|
+
table=self.model_endpoints_table,
|
|
221
|
+
criteria=[self.model_endpoints_table.uid == endpoint_id],
|
|
224
222
|
)
|
|
225
223
|
|
|
226
224
|
def delete_model_endpoint(self, endpoint_id: str) -> None:
|
|
@@ -229,11 +227,10 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
229
227
|
|
|
230
228
|
:param endpoint_id: The unique id of the model endpoint.
|
|
231
229
|
"""
|
|
232
|
-
self._init_model_endpoints_table()
|
|
233
230
|
# Delete the model endpoint record using sqlalchemy ORM
|
|
234
231
|
self._delete(
|
|
235
|
-
table=self.
|
|
236
|
-
criteria=[self.
|
|
232
|
+
table=self.model_endpoints_table,
|
|
233
|
+
criteria=[self.model_endpoints_table.uid == endpoint_id],
|
|
237
234
|
)
|
|
238
235
|
|
|
239
236
|
def get_model_endpoint(
|
|
@@ -249,12 +246,11 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
249
246
|
|
|
250
247
|
:raise MLRunNotFoundError: If the model endpoints table was not found or the model endpoint id was not found.
|
|
251
248
|
"""
|
|
252
|
-
self._init_model_endpoints_table()
|
|
253
249
|
|
|
254
250
|
# Get the model endpoint record
|
|
255
251
|
endpoint_record = self._get(
|
|
256
|
-
table=self.
|
|
257
|
-
criteria=[self.
|
|
252
|
+
table=self.model_endpoints_table,
|
|
253
|
+
criteria=[self.model_endpoints_table.uid == endpoint_id],
|
|
258
254
|
)
|
|
259
255
|
|
|
260
256
|
if not endpoint_record:
|
|
@@ -286,23 +282,22 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
286
282
|
|
|
287
283
|
:return: A list of model endpoint dictionaries.
|
|
288
284
|
"""
|
|
289
|
-
self._init_model_endpoints_table()
|
|
290
285
|
# Generate an empty model endpoints that will be filled afterwards with model endpoint dictionaries
|
|
291
286
|
endpoint_list = []
|
|
292
287
|
|
|
293
288
|
model_endpoints_table = (
|
|
294
|
-
self.
|
|
289
|
+
self.model_endpoints_table.__table__ # pyright: ignore[reportAttributeAccessIssue]
|
|
295
290
|
)
|
|
296
|
-
|
|
297
291
|
# Get the model endpoints records using sqlalchemy ORM
|
|
298
292
|
with create_session(dsn=self._sql_connection_string) as session:
|
|
299
293
|
# Generate the list query
|
|
300
|
-
query = session.query(self.
|
|
294
|
+
query = session.query(self.model_endpoints_table).filter_by(
|
|
301
295
|
project=self.project
|
|
302
296
|
)
|
|
303
297
|
|
|
304
298
|
# Apply filters
|
|
305
299
|
if model:
|
|
300
|
+
model = model if ":" in model else f"{model}:latest"
|
|
306
301
|
query = self._filter_values(
|
|
307
302
|
query=query,
|
|
308
303
|
model_endpoints_table=model_endpoints_table,
|
|
@@ -310,11 +305,12 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
310
305
|
filtered_values=[model],
|
|
311
306
|
)
|
|
312
307
|
if function:
|
|
308
|
+
function_uri = f"{self.project}/{function}"
|
|
313
309
|
query = self._filter_values(
|
|
314
310
|
query=query,
|
|
315
311
|
model_endpoints_table=model_endpoints_table,
|
|
316
|
-
key_filter=mm_schemas.EventFieldType.
|
|
317
|
-
filtered_values=[
|
|
312
|
+
key_filter=mm_schemas.EventFieldType.FUNCTION_URI,
|
|
313
|
+
filtered_values=[function_uri],
|
|
318
314
|
)
|
|
319
315
|
if uids:
|
|
320
316
|
query = self._filter_values(
|
|
@@ -364,11 +360,9 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
364
360
|
"""
|
|
365
361
|
|
|
366
362
|
if kind == mm_schemas.WriterEventKind.METRIC:
|
|
367
|
-
self._init_application_metrics_table()
|
|
368
363
|
table = self.application_metrics_table
|
|
369
364
|
table_name = mm_schemas.FileTargetKind.APP_METRICS
|
|
370
365
|
elif kind == mm_schemas.WriterEventKind.RESULT:
|
|
371
|
-
self._init_application_results_table()
|
|
372
366
|
table = self.application_results_table
|
|
373
367
|
table_name = mm_schemas.FileTargetKind.APP_RESULTS
|
|
374
368
|
else:
|
|
@@ -442,7 +436,6 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
442
436
|
:return: Timestamp as a Unix time.
|
|
443
437
|
:raise: MLRunNotFoundError if last analyzed value is not found.
|
|
444
438
|
"""
|
|
445
|
-
self._init_monitoring_schedules_table()
|
|
446
439
|
monitoring_schedule_record = self._get(
|
|
447
440
|
table=self.MonitoringSchedulesTable,
|
|
448
441
|
criteria=self._get_filter_criteria(
|
|
@@ -469,8 +462,6 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
469
462
|
:param last_analyzed: Timestamp as a Unix time that represents the last analyzed time of a certain
|
|
470
463
|
application and model endpoint.
|
|
471
464
|
"""
|
|
472
|
-
self._init_monitoring_schedules_table()
|
|
473
|
-
|
|
474
465
|
criteria = self._get_filter_criteria(
|
|
475
466
|
table=self.MonitoringSchedulesTable,
|
|
476
467
|
endpoint_id=endpoint_id,
|
|
@@ -500,7 +491,6 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
500
491
|
def _delete_last_analyzed(
|
|
501
492
|
self, endpoint_id: str, application_name: typing.Optional[str] = None
|
|
502
493
|
) -> None:
|
|
503
|
-
self._init_monitoring_schedules_table()
|
|
504
494
|
criteria = self._get_filter_criteria(
|
|
505
495
|
table=self.MonitoringSchedulesTable,
|
|
506
496
|
endpoint_id=endpoint_id,
|
|
@@ -512,7 +502,6 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
512
502
|
def _delete_application_result(
|
|
513
503
|
self, endpoint_id: str, application_name: typing.Optional[str] = None
|
|
514
504
|
) -> None:
|
|
515
|
-
self._init_application_results_table()
|
|
516
505
|
criteria = self._get_filter_criteria(
|
|
517
506
|
table=self.application_results_table,
|
|
518
507
|
endpoint_id=endpoint_id,
|
|
@@ -524,7 +513,6 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
524
513
|
def _delete_application_metrics(
|
|
525
514
|
self, endpoint_id: str, application_name: typing.Optional[str] = None
|
|
526
515
|
) -> None:
|
|
527
|
-
self._init_application_metrics_table()
|
|
528
516
|
criteria = self._get_filter_criteria(
|
|
529
517
|
table=self.application_metrics_table,
|
|
530
518
|
endpoint_id=endpoint_id,
|
|
@@ -538,8 +526,12 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
538
526
|
|
|
539
527
|
for table in self._tables:
|
|
540
528
|
# Create table if not exist. The `metadata` contains the `ModelEndpointsTable`
|
|
529
|
+
db_name = make_url(self._sql_connection_string).database
|
|
541
530
|
if not self._engine.has_table(table):
|
|
531
|
+
logger.info(f"Creating table {table} on {db_name} db.")
|
|
542
532
|
self._tables[table].metadata.create_all(bind=self._engine)
|
|
533
|
+
else:
|
|
534
|
+
logger.info(f"Table {table} already exists on {db_name} db.")
|
|
543
535
|
|
|
544
536
|
@staticmethod
|
|
545
537
|
def _filter_values(
|
|
@@ -581,41 +573,6 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
581
573
|
# Apply AND operator on the SQL query object with the filters tuple
|
|
582
574
|
return query.filter(sqlalchemy.and_(*filter_query))
|
|
583
575
|
|
|
584
|
-
@staticmethod
|
|
585
|
-
def _validate_labels(
|
|
586
|
-
endpoint_dict: dict,
|
|
587
|
-
labels: list,
|
|
588
|
-
) -> bool:
|
|
589
|
-
"""Validate that the model endpoint dictionary has the provided labels. There are 2 possible cases:
|
|
590
|
-
1 - Labels were provided as a list of key-values pairs (e.g. ['label_1=value_1', 'label_2=value_2']): Validate
|
|
591
|
-
that each pair exist in the endpoint dictionary.
|
|
592
|
-
2 - Labels were provided as a list of key labels (e.g. ['label_1', 'label_2']): Validate that each key exist in
|
|
593
|
-
the endpoint labels dictionary.
|
|
594
|
-
|
|
595
|
-
:param endpoint_dict: Dictionary of the model endpoint records.
|
|
596
|
-
:param labels: List of dictionary of required labels.
|
|
597
|
-
|
|
598
|
-
:return: True if the labels exist in the endpoint labels dictionary, otherwise False.
|
|
599
|
-
"""
|
|
600
|
-
|
|
601
|
-
# Convert endpoint labels into dictionary
|
|
602
|
-
endpoint_labels = json.loads(
|
|
603
|
-
endpoint_dict.get(mm_schemas.EventFieldType.LABELS)
|
|
604
|
-
)
|
|
605
|
-
|
|
606
|
-
for label in labels:
|
|
607
|
-
# Case 1 - label is a key=value pair
|
|
608
|
-
if "=" in label:
|
|
609
|
-
lbl, value = list(map(lambda x: x.strip(), label.split("=")))
|
|
610
|
-
if lbl not in endpoint_labels or str(endpoint_labels[lbl]) != value:
|
|
611
|
-
return False
|
|
612
|
-
# Case 2 - label is just a key
|
|
613
|
-
else:
|
|
614
|
-
if label not in endpoint_labels:
|
|
615
|
-
return False
|
|
616
|
-
|
|
617
|
-
return True
|
|
618
|
-
|
|
619
576
|
def delete_model_endpoints_resources(self) -> None:
|
|
620
577
|
"""
|
|
621
578
|
Delete all the model monitoring resources of the project in the SQL tables.
|
|
@@ -650,11 +607,9 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
650
607
|
type=type,
|
|
651
608
|
)
|
|
652
609
|
if type == mm_schemas.ModelEndpointMonitoringMetricType.METRIC:
|
|
653
|
-
self._init_application_metrics_table()
|
|
654
610
|
table = self.application_metrics_table
|
|
655
611
|
name_col = mm_schemas.MetricData.METRIC_NAME
|
|
656
612
|
else:
|
|
657
|
-
self._init_application_results_table()
|
|
658
613
|
table = self.application_results_table
|
|
659
614
|
name_col = mm_schemas.ResultData.RESULT_NAME
|
|
660
615
|
|
|
@@ -13,7 +13,6 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import json
|
|
16
|
-
import os
|
|
17
16
|
import typing
|
|
18
17
|
from dataclasses import dataclass
|
|
19
18
|
from http import HTTPStatus
|
|
@@ -24,8 +23,8 @@ import v3io.dataplane.response
|
|
|
24
23
|
|
|
25
24
|
import mlrun.common.model_monitoring.helpers
|
|
26
25
|
import mlrun.common.schemas.model_monitoring as mm_schemas
|
|
27
|
-
import mlrun.model_monitoring.db
|
|
28
26
|
import mlrun.utils.v3io_clients
|
|
27
|
+
from mlrun.model_monitoring.db import StoreBase
|
|
29
28
|
from mlrun.utils import logger
|
|
30
29
|
|
|
31
30
|
# Fields to encode before storing in the KV table or to decode after retrieving
|
|
@@ -89,18 +88,21 @@ _KIND_TO_SCHEMA_PARAMS: dict[mm_schemas.WriterEventKind, SchemaParams] = {
|
|
|
89
88
|
_EXCLUDE_SCHEMA_FILTER_EXPRESSION = '__name!=".#schema"'
|
|
90
89
|
|
|
91
90
|
|
|
92
|
-
class KVStoreBase(
|
|
91
|
+
class KVStoreBase(StoreBase):
|
|
92
|
+
type: typing.ClassVar[str] = "v3io-nosql"
|
|
93
93
|
"""
|
|
94
94
|
Handles the DB operations when the DB target is from type KV. For the KV operations, we use an instance of V3IO
|
|
95
95
|
client and usually the KV table can be found under v3io:///users/pipelines/project-name/model-endpoints/endpoints/.
|
|
96
96
|
"""
|
|
97
97
|
|
|
98
|
-
def __init__(
|
|
98
|
+
def __init__(
|
|
99
|
+
self,
|
|
100
|
+
project: str,
|
|
101
|
+
) -> None:
|
|
99
102
|
super().__init__(project=project)
|
|
100
103
|
# Initialize a V3IO client instance
|
|
101
|
-
self.access_key = access_key or os.environ.get("V3IO_ACCESS_KEY")
|
|
102
104
|
self.client = mlrun.utils.v3io_clients.get_v3io_client(
|
|
103
|
-
endpoint=mlrun.mlconf.v3io_api,
|
|
105
|
+
endpoint=mlrun.mlconf.v3io_api,
|
|
104
106
|
)
|
|
105
107
|
# Get the KV table path and container
|
|
106
108
|
self.path, self.container = self._get_path_and_container()
|
|
@@ -186,7 +188,6 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
186
188
|
table_path=self.path,
|
|
187
189
|
key=endpoint_id,
|
|
188
190
|
raise_for_status=v3io.dataplane.RaiseForStatus.never,
|
|
189
|
-
access_key=self.access_key,
|
|
190
191
|
)
|
|
191
192
|
endpoint = endpoint.output.item
|
|
192
193
|
|
|
@@ -255,7 +256,6 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
255
256
|
self.project,
|
|
256
257
|
function,
|
|
257
258
|
model,
|
|
258
|
-
labels,
|
|
259
259
|
top_level,
|
|
260
260
|
),
|
|
261
261
|
raise_for_status=v3io.dataplane.RaiseForStatus.never,
|
|
@@ -268,7 +268,6 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
268
268
|
exc=mlrun.errors.err_to_str(exc),
|
|
269
269
|
)
|
|
270
270
|
return endpoint_list
|
|
271
|
-
|
|
272
271
|
# Create a list of model endpoints unique ids
|
|
273
272
|
if uids is None:
|
|
274
273
|
uids = []
|
|
@@ -281,10 +280,16 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
281
280
|
|
|
282
281
|
# Add each relevant model endpoint to the model endpoints list
|
|
283
282
|
for endpoint_id in uids:
|
|
284
|
-
|
|
283
|
+
endpoint_dict = self.get_model_endpoint(
|
|
285
284
|
endpoint_id=endpoint_id,
|
|
286
285
|
)
|
|
287
|
-
|
|
286
|
+
|
|
287
|
+
if labels and not self._validate_labels(
|
|
288
|
+
endpoint_dict=endpoint_dict, labels=labels
|
|
289
|
+
):
|
|
290
|
+
continue
|
|
291
|
+
|
|
292
|
+
endpoint_list.append(endpoint_dict)
|
|
288
293
|
|
|
289
294
|
return endpoint_list
|
|
290
295
|
|
|
@@ -499,7 +504,6 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
499
504
|
|
|
500
505
|
def _get_frames_client(self):
|
|
501
506
|
return mlrun.utils.v3io_clients.get_frames_client(
|
|
502
|
-
token=self.access_key,
|
|
503
507
|
address=mlrun.mlconf.v3io_framesd,
|
|
504
508
|
container=self.container,
|
|
505
509
|
)
|
|
@@ -509,20 +513,16 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
509
513
|
project: str,
|
|
510
514
|
function: str = None,
|
|
511
515
|
model: str = None,
|
|
512
|
-
labels: list[str] = None,
|
|
513
516
|
top_level: bool = False,
|
|
514
517
|
) -> str:
|
|
515
518
|
"""
|
|
516
519
|
Convert the provided filters into a valid filter expression. The expected filter expression includes different
|
|
517
520
|
conditions, divided by ' AND '.
|
|
518
521
|
|
|
519
|
-
:param project:
|
|
520
|
-
:param model:
|
|
521
|
-
:param function:
|
|
522
|
-
:param
|
|
523
|
-
a label (i.e. list("key=value")) or by looking for the existence of a given
|
|
524
|
-
key (i.e. "key").
|
|
525
|
-
:param top_level: If True will return only routers and endpoint that are NOT children of any router.
|
|
522
|
+
:param project: The name of the project.
|
|
523
|
+
:param model: The name of the model to filter by.
|
|
524
|
+
:param function: The name of the function to filter by.
|
|
525
|
+
:param top_level: If True will return only routers and endpoint that are NOT children of any router.
|
|
526
526
|
|
|
527
527
|
:return: A valid filter expression as a string.
|
|
528
528
|
|
|
@@ -533,25 +533,17 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
|
|
|
533
533
|
raise mlrun.errors.MLRunInvalidArgumentError("project can't be empty")
|
|
534
534
|
|
|
535
535
|
# Add project filter
|
|
536
|
-
filter_expression = [f"
|
|
536
|
+
filter_expression = [f"{mm_schemas.EventFieldType.PROJECT}=='{project}'"]
|
|
537
537
|
|
|
538
538
|
# Add function and model filters
|
|
539
539
|
if function:
|
|
540
|
-
|
|
540
|
+
function_uri = f"{project}/{function}" if function else None
|
|
541
|
+
filter_expression.append(
|
|
542
|
+
f"{mm_schemas.EventFieldType.FUNCTION_URI}=='{function_uri}'"
|
|
543
|
+
)
|
|
541
544
|
if model:
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
# Add labels filters
|
|
545
|
-
if labels:
|
|
546
|
-
for label in labels:
|
|
547
|
-
if not label.startswith("_"):
|
|
548
|
-
label = f"_{label}"
|
|
549
|
-
|
|
550
|
-
if "=" in label:
|
|
551
|
-
lbl, value = list(map(lambda x: x.strip(), label.split("=")))
|
|
552
|
-
filter_expression.append(f"{lbl}=='{value}'")
|
|
553
|
-
else:
|
|
554
|
-
filter_expression.append(f"exists({label})")
|
|
545
|
+
model = model if ":" in model else f"{model}:latest"
|
|
546
|
+
filter_expression.append(f"{mm_schemas.EventFieldType.MODEL}=='{model}'")
|
|
555
547
|
|
|
556
548
|
# Apply top_level filter (remove endpoints that considered a child of a router)
|
|
557
549
|
if top_level:
|