mlrun 1.7.0rc26__py3-none-any.whl → 1.7.0rc29__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +7 -7
- mlrun/alerts/alert.py +13 -1
- mlrun/artifacts/manager.py +5 -0
- mlrun/common/constants.py +2 -2
- mlrun/common/formatters/base.py +9 -9
- mlrun/common/schemas/alert.py +4 -8
- mlrun/common/schemas/api_gateway.py +7 -0
- mlrun/common/schemas/constants.py +3 -0
- mlrun/common/schemas/model_monitoring/__init__.py +1 -0
- mlrun/common/schemas/model_monitoring/constants.py +27 -12
- mlrun/common/schemas/model_monitoring/model_endpoints.py +0 -12
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/config.py +16 -9
- mlrun/datastore/azure_blob.py +2 -1
- mlrun/datastore/base.py +1 -5
- mlrun/datastore/datastore.py +3 -3
- mlrun/datastore/inmem.py +1 -1
- mlrun/datastore/snowflake_utils.py +3 -1
- mlrun/datastore/sources.py +26 -11
- mlrun/datastore/store_resources.py +2 -0
- mlrun/datastore/targets.py +60 -25
- mlrun/db/base.py +10 -0
- mlrun/db/httpdb.py +41 -30
- mlrun/db/nopdb.py +10 -1
- mlrun/errors.py +4 -0
- mlrun/execution.py +18 -10
- mlrun/feature_store/retrieval/spark_merger.py +2 -1
- mlrun/launcher/local.py +2 -2
- mlrun/model.py +30 -0
- mlrun/model_monitoring/api.py +6 -52
- mlrun/model_monitoring/applications/histogram_data_drift.py +4 -1
- mlrun/model_monitoring/db/stores/__init__.py +21 -9
- mlrun/model_monitoring/db/stores/base/store.py +39 -1
- mlrun/model_monitoring/db/stores/sqldb/models/base.py +9 -7
- mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +4 -2
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +34 -79
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +19 -27
- mlrun/model_monitoring/db/tsdb/__init__.py +19 -14
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +4 -2
- mlrun/model_monitoring/helpers.py +9 -5
- mlrun/model_monitoring/writer.py +1 -5
- mlrun/projects/operations.py +1 -0
- mlrun/projects/project.py +71 -75
- mlrun/render.py +10 -5
- mlrun/run.py +2 -2
- mlrun/runtimes/daskjob.py +7 -1
- mlrun/runtimes/local.py +24 -7
- mlrun/runtimes/nuclio/function.py +20 -0
- mlrun/runtimes/pod.py +5 -29
- mlrun/serving/routers.py +75 -59
- mlrun/serving/server.py +1 -0
- mlrun/serving/v2_serving.py +8 -1
- mlrun/utils/helpers.py +46 -2
- mlrun/utils/logger.py +36 -2
- mlrun/utils/notifications/notification/base.py +4 -0
- mlrun/utils/notifications/notification/git.py +21 -0
- mlrun/utils/notifications/notification/slack.py +8 -0
- mlrun/utils/notifications/notification/webhook.py +41 -1
- mlrun/utils/notifications/notification_pusher.py +2 -2
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc26.dist-info → mlrun-1.7.0rc29.dist-info}/METADATA +9 -4
- {mlrun-1.7.0rc26.dist-info → mlrun-1.7.0rc29.dist-info}/RECORD +66 -66
- {mlrun-1.7.0rc26.dist-info → mlrun-1.7.0rc29.dist-info}/WHEEL +1 -1
- {mlrun-1.7.0rc26.dist-info → mlrun-1.7.0rc29.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc26.dist-info → mlrun-1.7.0rc29.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc26.dist-info → mlrun-1.7.0rc29.dist-info}/top_level.txt +0 -0
mlrun/datastore/targets.py
CHANGED
|
@@ -29,7 +29,10 @@ from mergedeep import merge
|
|
|
29
29
|
import mlrun
|
|
30
30
|
import mlrun.utils.helpers
|
|
31
31
|
from mlrun.config import config
|
|
32
|
-
from mlrun.datastore.snowflake_utils import
|
|
32
|
+
from mlrun.datastore.snowflake_utils import (
|
|
33
|
+
get_snowflake_password,
|
|
34
|
+
get_snowflake_spark_options,
|
|
35
|
+
)
|
|
33
36
|
from mlrun.datastore.utils import transform_list_filters_to_tuple
|
|
34
37
|
from mlrun.model import DataSource, DataTarget, DataTargetBase, TargetPathObject
|
|
35
38
|
from mlrun.utils import logger, now_date
|
|
@@ -546,9 +549,7 @@ class BaseStoreTarget(DataTargetBase):
|
|
|
546
549
|
os.makedirs(dir, exist_ok=True)
|
|
547
550
|
target_df = df
|
|
548
551
|
partition_cols = None # single parquet file
|
|
549
|
-
if not
|
|
550
|
-
".pq"
|
|
551
|
-
): # directory
|
|
552
|
+
if not mlrun.utils.helpers.is_parquet_file(target_path): # directory
|
|
552
553
|
partition_cols = []
|
|
553
554
|
if timestamp_key and (
|
|
554
555
|
self.partitioned or self.time_partitioning_granularity
|
|
@@ -775,6 +776,10 @@ class BaseStoreTarget(DataTargetBase):
|
|
|
775
776
|
def get_dask_options(self):
|
|
776
777
|
raise NotImplementedError()
|
|
777
778
|
|
|
779
|
+
@property
|
|
780
|
+
def source_spark_attributes(self) -> dict:
|
|
781
|
+
return {}
|
|
782
|
+
|
|
778
783
|
|
|
779
784
|
class ParquetTarget(BaseStoreTarget):
|
|
780
785
|
"""Parquet target storage driver, used to materialize feature set/vector data into parquet files.
|
|
@@ -911,10 +916,8 @@ class ParquetTarget(BaseStoreTarget):
|
|
|
911
916
|
if time_unit == time_partitioning_granularity:
|
|
912
917
|
break
|
|
913
918
|
|
|
914
|
-
if (
|
|
915
|
-
|
|
916
|
-
and not self.get_target_path().endswith(".parquet")
|
|
917
|
-
and not self.get_target_path().endswith(".pq")
|
|
919
|
+
if not self.partitioned and not mlrun.utils.helpers.is_parquet_file(
|
|
920
|
+
self.get_target_path()
|
|
918
921
|
):
|
|
919
922
|
partition_cols = []
|
|
920
923
|
|
|
@@ -1033,9 +1036,7 @@ class ParquetTarget(BaseStoreTarget):
|
|
|
1033
1036
|
return result
|
|
1034
1037
|
|
|
1035
1038
|
def is_single_file(self):
|
|
1036
|
-
|
|
1037
|
-
return self.path.endswith(".parquet") or self.path.endswith(".pq")
|
|
1038
|
-
return False
|
|
1039
|
+
return mlrun.utils.helpers.is_parquet_file(self.path)
|
|
1039
1040
|
|
|
1040
1041
|
def prepare_spark_df(self, df, key_columns, timestamp_key=None, spark_options=None):
|
|
1041
1042
|
# If partitioning by time, add the necessary columns
|
|
@@ -1208,19 +1209,20 @@ class SnowflakeTarget(BaseStoreTarget):
|
|
|
1208
1209
|
warehouse: str = None,
|
|
1209
1210
|
table_name: str = None,
|
|
1210
1211
|
):
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
"
|
|
1214
|
-
|
|
1215
|
-
"
|
|
1216
|
-
|
|
1217
|
-
"
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1212
|
+
attributes = attributes or {}
|
|
1213
|
+
if url:
|
|
1214
|
+
attributes["url"] = url
|
|
1215
|
+
if user:
|
|
1216
|
+
attributes["user"] = user
|
|
1217
|
+
if database:
|
|
1218
|
+
attributes["database"] = database
|
|
1219
|
+
if db_schema:
|
|
1220
|
+
attributes["db_schema"] = db_schema
|
|
1221
|
+
if warehouse:
|
|
1222
|
+
attributes["warehouse"] = warehouse
|
|
1223
|
+
if table_name:
|
|
1224
|
+
attributes["table"] = table_name
|
|
1225
|
+
|
|
1224
1226
|
super().__init__(
|
|
1225
1227
|
name,
|
|
1226
1228
|
path,
|
|
@@ -1244,7 +1246,31 @@ class SnowflakeTarget(BaseStoreTarget):
|
|
|
1244
1246
|
return spark_options
|
|
1245
1247
|
|
|
1246
1248
|
def purge(self):
|
|
1247
|
-
|
|
1249
|
+
import snowflake.connector
|
|
1250
|
+
|
|
1251
|
+
missing = [
|
|
1252
|
+
key
|
|
1253
|
+
for key in ["database", "db_schema", "table", "url", "user", "warehouse"]
|
|
1254
|
+
if self.attributes.get(key) is None
|
|
1255
|
+
]
|
|
1256
|
+
if missing:
|
|
1257
|
+
raise mlrun.errors.MLRunRuntimeError(
|
|
1258
|
+
f"Can't purge Snowflake target, "
|
|
1259
|
+
f"some attributes are missing: {', '.join(missing)}"
|
|
1260
|
+
)
|
|
1261
|
+
account = self.attributes["url"].replace(".snowflakecomputing.com", "")
|
|
1262
|
+
|
|
1263
|
+
with snowflake.connector.connect(
|
|
1264
|
+
account=account,
|
|
1265
|
+
user=self.attributes["user"],
|
|
1266
|
+
password=get_snowflake_password(),
|
|
1267
|
+
warehouse=self.attributes["warehouse"],
|
|
1268
|
+
) as snowflake_connector:
|
|
1269
|
+
drop_statement = (
|
|
1270
|
+
f"DROP TABLE IF EXISTS {self.attributes['database']}.{self.attributes['db_schema']}"
|
|
1271
|
+
f".{self.attributes['table']}"
|
|
1272
|
+
)
|
|
1273
|
+
snowflake_connector.execute_string(drop_statement)
|
|
1248
1274
|
|
|
1249
1275
|
def as_df(
|
|
1250
1276
|
self,
|
|
@@ -1259,6 +1285,15 @@ class SnowflakeTarget(BaseStoreTarget):
|
|
|
1259
1285
|
):
|
|
1260
1286
|
raise NotImplementedError()
|
|
1261
1287
|
|
|
1288
|
+
@property
|
|
1289
|
+
def source_spark_attributes(self) -> dict:
|
|
1290
|
+
keys = ["url", "user", "database", "db_schema", "warehouse"]
|
|
1291
|
+
attributes = self.attributes or {}
|
|
1292
|
+
snowflake_dict = {key: attributes.get(key) for key in keys}
|
|
1293
|
+
table = attributes.get("table")
|
|
1294
|
+
snowflake_dict["query"] = f"SELECT * from {table}" if table else None
|
|
1295
|
+
return snowflake_dict
|
|
1296
|
+
|
|
1262
1297
|
|
|
1263
1298
|
class NoSqlBaseTarget(BaseStoreTarget):
|
|
1264
1299
|
is_table = True
|
mlrun/db/base.py
CHANGED
|
@@ -154,6 +154,7 @@ class RunDBInterface(ABC):
|
|
|
154
154
|
mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
|
|
155
155
|
),
|
|
156
156
|
secrets: dict = None,
|
|
157
|
+
iter=None,
|
|
157
158
|
):
|
|
158
159
|
pass
|
|
159
160
|
|
|
@@ -891,6 +892,7 @@ class RunDBInterface(ABC):
|
|
|
891
892
|
image: str = "mlrun/mlrun",
|
|
892
893
|
deploy_histogram_data_drift_app: bool = True,
|
|
893
894
|
rebuild_images: bool = False,
|
|
895
|
+
fetch_credentials_from_sys_config: bool = False,
|
|
894
896
|
) -> None:
|
|
895
897
|
pass
|
|
896
898
|
|
|
@@ -917,3 +919,11 @@ class RunDBInterface(ABC):
|
|
|
917
919
|
self, project: str, image: str = "mlrun/mlrun"
|
|
918
920
|
) -> None:
|
|
919
921
|
pass
|
|
922
|
+
|
|
923
|
+
@abstractmethod
|
|
924
|
+
def set_model_monitoring_credentials(
|
|
925
|
+
self,
|
|
926
|
+
project: str,
|
|
927
|
+
credentials: dict[str, str],
|
|
928
|
+
) -> None:
|
|
929
|
+
pass
|
mlrun/db/httpdb.py
CHANGED
|
@@ -38,6 +38,7 @@ import mlrun.model_monitoring.model_endpoint
|
|
|
38
38
|
import mlrun.platforms
|
|
39
39
|
import mlrun.projects
|
|
40
40
|
import mlrun.runtimes.nuclio.api_gateway
|
|
41
|
+
import mlrun.runtimes.nuclio.function
|
|
41
42
|
import mlrun.utils
|
|
42
43
|
from mlrun.alerts.alert import AlertConfig
|
|
43
44
|
from mlrun.db.auth_utils import OAuthClientIDTokenProvider, StaticTokenProvider
|
|
@@ -536,6 +537,10 @@ class HTTPRunDB(RunDBInterface):
|
|
|
536
537
|
server_cfg.get("model_monitoring_tsdb_connection")
|
|
537
538
|
or config.model_endpoint_monitoring.tsdb_connection
|
|
538
539
|
)
|
|
540
|
+
config.model_endpoint_monitoring.stream_connection = (
|
|
541
|
+
server_cfg.get("stream_connection")
|
|
542
|
+
or config.model_endpoint_monitoring.stream_connection
|
|
543
|
+
)
|
|
539
544
|
config.packagers = server_cfg.get("packagers") or config.packagers
|
|
540
545
|
server_data_prefixes = server_cfg.get("feature_store_data_prefixes") or {}
|
|
541
546
|
for prefix in ["default", "nosql", "redisnosql"]:
|
|
@@ -870,7 +875,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
870
875
|
):
|
|
871
876
|
# default to last week on no filter
|
|
872
877
|
start_time_from = datetime.now() - timedelta(days=7)
|
|
873
|
-
partition_by = mlrun.common.schemas.RunPartitionByField.
|
|
878
|
+
partition_by = mlrun.common.schemas.RunPartitionByField.project_and_name
|
|
874
879
|
partition_sort_by = mlrun.common.schemas.SortField.updated
|
|
875
880
|
|
|
876
881
|
params = {
|
|
@@ -1028,6 +1033,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1028
1033
|
mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
|
|
1029
1034
|
),
|
|
1030
1035
|
secrets: dict = None,
|
|
1036
|
+
iter=None,
|
|
1031
1037
|
):
|
|
1032
1038
|
"""Delete an artifact.
|
|
1033
1039
|
|
|
@@ -1046,6 +1052,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1046
1052
|
"tag": tag,
|
|
1047
1053
|
"tree": tree,
|
|
1048
1054
|
"uid": uid,
|
|
1055
|
+
"iter": iter,
|
|
1049
1056
|
"deletion_strategy": deletion_strategy,
|
|
1050
1057
|
}
|
|
1051
1058
|
error = f"del artifact {project}/{key}"
|
|
@@ -1610,20 +1617,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1610
1617
|
raise RunDBError("bad function build response")
|
|
1611
1618
|
|
|
1612
1619
|
if resp.headers:
|
|
1613
|
-
func.status.state = resp.headers.get("x-mlrun-function-status", "")
|
|
1614
1620
|
last_log_timestamp = float(
|
|
1615
1621
|
resp.headers.get("x-mlrun-last-timestamp", "0.0")
|
|
1616
1622
|
)
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
func.status.internal_invocation_urls = resp.headers.get(
|
|
1620
|
-
"x-mlrun-internal-invocation-urls", ""
|
|
1621
|
-
).split(",")
|
|
1622
|
-
func.status.external_invocation_urls = resp.headers.get(
|
|
1623
|
-
"x-mlrun-external-invocation-urls", ""
|
|
1624
|
-
).split(",")
|
|
1625
|
-
func.status.container_image = resp.headers.get(
|
|
1626
|
-
"x-mlrun-container-image", ""
|
|
1623
|
+
mlrun.runtimes.nuclio.function.enrich_nuclio_function_from_headers(
|
|
1624
|
+
func, resp.headers
|
|
1627
1625
|
)
|
|
1628
1626
|
|
|
1629
1627
|
text = ""
|
|
@@ -1681,16 +1679,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1681
1679
|
resp.headers.get("x-mlrun-last-timestamp", "0.0")
|
|
1682
1680
|
)
|
|
1683
1681
|
if func.kind in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
func.status.internal_invocation_urls = resp.headers.get(
|
|
1687
|
-
"x-mlrun-internal-invocation-urls", ""
|
|
1688
|
-
).split(",")
|
|
1689
|
-
func.status.external_invocation_urls = resp.headers.get(
|
|
1690
|
-
"x-mlrun-external-invocation-urls", ""
|
|
1691
|
-
).split(",")
|
|
1692
|
-
func.status.container_image = resp.headers.get(
|
|
1693
|
-
"x-mlrun-container-image", ""
|
|
1682
|
+
mlrun.runtimes.nuclio.function.enrich_nuclio_function_from_headers(
|
|
1683
|
+
func, resp.headers
|
|
1694
1684
|
)
|
|
1695
1685
|
|
|
1696
1686
|
builder_pod = resp.headers.get("builder_pod", "")
|
|
@@ -3397,6 +3387,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3397
3387
|
image: str = "mlrun/mlrun",
|
|
3398
3388
|
deploy_histogram_data_drift_app: bool = True,
|
|
3399
3389
|
rebuild_images: bool = False,
|
|
3390
|
+
fetch_credentials_from_sys_config: bool = False,
|
|
3400
3391
|
) -> None:
|
|
3401
3392
|
"""
|
|
3402
3393
|
Deploy model monitoring application controller, writer and stream functions.
|
|
@@ -3406,14 +3397,16 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3406
3397
|
The stream function goal is to monitor the log of the data stream. It is triggered when a new log entry
|
|
3407
3398
|
is detected. It processes the new events into statistics that are then written to statistics databases.
|
|
3408
3399
|
|
|
3409
|
-
:param project:
|
|
3410
|
-
:param base_period:
|
|
3411
|
-
|
|
3412
|
-
:param image:
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
:param deploy_histogram_data_drift_app:
|
|
3416
|
-
:param rebuild_images:
|
|
3400
|
+
:param project: Project name.
|
|
3401
|
+
:param base_period: The time period in minutes in which the model monitoring controller
|
|
3402
|
+
function triggers. By default, the base period is 10 minutes.
|
|
3403
|
+
:param image: The image of the model monitoring controller, writer & monitoring
|
|
3404
|
+
stream functions, which are real time nuclio functions.
|
|
3405
|
+
By default, the image is mlrun/mlrun.
|
|
3406
|
+
:param deploy_histogram_data_drift_app: If true, deploy the default histogram-based data drift application.
|
|
3407
|
+
:param rebuild_images: If true, force rebuild of model monitoring infrastructure images.
|
|
3408
|
+
:param fetch_credentials_from_sys_config: If true, fetch the credentials from the system configuration.
|
|
3409
|
+
|
|
3417
3410
|
"""
|
|
3418
3411
|
self.api_call(
|
|
3419
3412
|
method=mlrun.common.types.HTTPMethod.POST,
|
|
@@ -3423,6 +3416,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3423
3416
|
"image": image,
|
|
3424
3417
|
"deploy_histogram_data_drift_app": deploy_histogram_data_drift_app,
|
|
3425
3418
|
"rebuild_images": rebuild_images,
|
|
3419
|
+
"fetch_credentials_from_sys_config": fetch_credentials_from_sys_config,
|
|
3426
3420
|
},
|
|
3427
3421
|
)
|
|
3428
3422
|
|
|
@@ -3548,6 +3542,23 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3548
3542
|
params={"image": image},
|
|
3549
3543
|
)
|
|
3550
3544
|
|
|
3545
|
+
def set_model_monitoring_credentials(
|
|
3546
|
+
self,
|
|
3547
|
+
project: str,
|
|
3548
|
+
credentials: dict[str, str],
|
|
3549
|
+
) -> None:
|
|
3550
|
+
"""
|
|
3551
|
+
Set the credentials for the model monitoring application.
|
|
3552
|
+
|
|
3553
|
+
:param project: Project name.
|
|
3554
|
+
:param credentials: Credentials to set.
|
|
3555
|
+
"""
|
|
3556
|
+
self.api_call(
|
|
3557
|
+
method=mlrun.common.types.HTTPMethod.POST,
|
|
3558
|
+
path=f"projects/{project}/model-monitoring/set-model-monitoring-credentials",
|
|
3559
|
+
params={**credentials},
|
|
3560
|
+
)
|
|
3561
|
+
|
|
3551
3562
|
def create_hub_source(
|
|
3552
3563
|
self, source: Union[dict, mlrun.common.schemas.IndexedHubSource]
|
|
3553
3564
|
):
|
mlrun/db/nopdb.py
CHANGED
|
@@ -162,6 +162,7 @@ class NopDB(RunDBInterface):
|
|
|
162
162
|
mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
|
|
163
163
|
),
|
|
164
164
|
secrets: dict = None,
|
|
165
|
+
iter=None,
|
|
165
166
|
):
|
|
166
167
|
pass
|
|
167
168
|
|
|
@@ -708,6 +709,7 @@ class NopDB(RunDBInterface):
|
|
|
708
709
|
image: str = "mlrun/mlrun",
|
|
709
710
|
deploy_histogram_data_drift_app: bool = True,
|
|
710
711
|
rebuild_images: bool = False,
|
|
712
|
+
fetch_credentials_from_sys_config: bool = False,
|
|
711
713
|
) -> None:
|
|
712
714
|
pass
|
|
713
715
|
|
|
@@ -730,7 +732,14 @@ class NopDB(RunDBInterface):
|
|
|
730
732
|
def deploy_histogram_data_drift_app(
|
|
731
733
|
self, project: str, image: str = "mlrun/mlrun"
|
|
732
734
|
) -> None:
|
|
733
|
-
|
|
735
|
+
pass
|
|
736
|
+
|
|
737
|
+
def set_model_monitoring_credentials(
|
|
738
|
+
self,
|
|
739
|
+
project: str,
|
|
740
|
+
credentials: dict[str, str],
|
|
741
|
+
) -> None:
|
|
742
|
+
pass
|
|
734
743
|
|
|
735
744
|
def generate_event(
|
|
736
745
|
self, name: str, event_data: Union[dict, mlrun.common.schemas.Event], project=""
|
mlrun/errors.py
CHANGED
|
@@ -205,6 +205,10 @@ class MLRunTimeoutError(MLRunHTTPStatusError, TimeoutError):
|
|
|
205
205
|
error_status_code = HTTPStatus.GATEWAY_TIMEOUT.value
|
|
206
206
|
|
|
207
207
|
|
|
208
|
+
class MLRunInvalidMMStoreType(MLRunHTTPStatusError, ValueError):
|
|
209
|
+
error_status_code = HTTPStatus.BAD_REQUEST.value
|
|
210
|
+
|
|
211
|
+
|
|
208
212
|
class MLRunRetryExhaustedError(Exception):
|
|
209
213
|
pass
|
|
210
214
|
|
mlrun/execution.py
CHANGED
|
@@ -34,13 +34,13 @@ from .features import Feature
|
|
|
34
34
|
from .model import HyperParamOptions
|
|
35
35
|
from .secrets import SecretsStore
|
|
36
36
|
from .utils import (
|
|
37
|
+
RunKeys,
|
|
37
38
|
dict_to_json,
|
|
38
39
|
dict_to_yaml,
|
|
39
40
|
get_in,
|
|
40
41
|
is_relative_path,
|
|
41
42
|
logger,
|
|
42
43
|
now_date,
|
|
43
|
-
run_keys,
|
|
44
44
|
to_date_str,
|
|
45
45
|
update_in,
|
|
46
46
|
)
|
|
@@ -85,6 +85,7 @@ class MLClientCtx:
|
|
|
85
85
|
|
|
86
86
|
self._labels = {}
|
|
87
87
|
self._annotations = {}
|
|
88
|
+
self._node_selector = {}
|
|
88
89
|
|
|
89
90
|
self._function = ""
|
|
90
91
|
self._parameters = {}
|
|
@@ -207,6 +208,11 @@ class MLClientCtx:
|
|
|
207
208
|
"""Dictionary with labels (read-only)"""
|
|
208
209
|
return deepcopy(self._labels)
|
|
209
210
|
|
|
211
|
+
@property
|
|
212
|
+
def node_selector(self):
|
|
213
|
+
"""Dictionary with node selectors (read-only)"""
|
|
214
|
+
return deepcopy(self._node_selector)
|
|
215
|
+
|
|
210
216
|
@property
|
|
211
217
|
def annotations(self):
|
|
212
218
|
"""Dictionary with annotations (read-only)"""
|
|
@@ -365,7 +371,7 @@ class MLClientCtx:
|
|
|
365
371
|
self._labels = meta.get("labels", self._labels)
|
|
366
372
|
spec = attrs.get("spec")
|
|
367
373
|
if spec:
|
|
368
|
-
self._secrets_manager = SecretsStore.from_list(spec.get(
|
|
374
|
+
self._secrets_manager = SecretsStore.from_list(spec.get(RunKeys.secrets))
|
|
369
375
|
self._log_level = spec.get("log_level", self._log_level)
|
|
370
376
|
self._function = spec.get("function", self._function)
|
|
371
377
|
self._parameters = spec.get("parameters", self._parameters)
|
|
@@ -383,13 +389,14 @@ class MLClientCtx:
|
|
|
383
389
|
self._allow_empty_resources = spec.get(
|
|
384
390
|
"allow_empty_resources", self._allow_empty_resources
|
|
385
391
|
)
|
|
386
|
-
self.artifact_path = spec.get(
|
|
387
|
-
self._in_path = spec.get(
|
|
388
|
-
inputs = spec.get(
|
|
392
|
+
self.artifact_path = spec.get(RunKeys.output_path, self.artifact_path)
|
|
393
|
+
self._in_path = spec.get(RunKeys.input_path, self._in_path)
|
|
394
|
+
inputs = spec.get(RunKeys.inputs)
|
|
389
395
|
self._notifications = spec.get("notifications", self._notifications)
|
|
390
396
|
self._state_thresholds = spec.get(
|
|
391
397
|
"state_thresholds", self._state_thresholds
|
|
392
398
|
)
|
|
399
|
+
self._node_selector = spec.get("node_selector", self._node_selector)
|
|
393
400
|
self._reset_on_run = spec.get("reset_on_run", self._reset_on_run)
|
|
394
401
|
|
|
395
402
|
self._init_dbs(rundb)
|
|
@@ -567,7 +574,7 @@ class MLClientCtx:
|
|
|
567
574
|
self._results["best_iteration"] = best
|
|
568
575
|
for k, v in get_in(task, ["status", "results"], {}).items():
|
|
569
576
|
self._results[k] = v
|
|
570
|
-
for artifact in get_in(task, ["status",
|
|
577
|
+
for artifact in get_in(task, ["status", RunKeys.artifacts], []):
|
|
571
578
|
self._artifacts_manager.artifacts[artifact["metadata"]["key"]] = (
|
|
572
579
|
artifact
|
|
573
580
|
)
|
|
@@ -939,10 +946,11 @@ class MLClientCtx:
|
|
|
939
946
|
"parameters": self._parameters,
|
|
940
947
|
"handler": self._handler,
|
|
941
948
|
"outputs": self._outputs,
|
|
942
|
-
|
|
943
|
-
|
|
949
|
+
RunKeys.output_path: self.artifact_path,
|
|
950
|
+
RunKeys.inputs: self._inputs,
|
|
944
951
|
"notifications": self._notifications,
|
|
945
952
|
"state_thresholds": self._state_thresholds,
|
|
953
|
+
"node_selector": self._node_selector,
|
|
946
954
|
},
|
|
947
955
|
"status": {
|
|
948
956
|
"results": self._results,
|
|
@@ -964,7 +972,7 @@ class MLClientCtx:
|
|
|
964
972
|
set_if_not_none(struct["status"], "commit", self._commit)
|
|
965
973
|
set_if_not_none(struct["status"], "iterations", self._iteration_results)
|
|
966
974
|
|
|
967
|
-
struct["status"][
|
|
975
|
+
struct["status"][RunKeys.artifacts] = self._artifacts_manager.artifact_list()
|
|
968
976
|
self._data_stores.to_dict(struct["spec"])
|
|
969
977
|
return struct
|
|
970
978
|
|
|
@@ -1058,7 +1066,7 @@ class MLClientCtx:
|
|
|
1058
1066
|
set_if_not_none(struct, "status.commit", self._commit)
|
|
1059
1067
|
set_if_not_none(struct, "status.iterations", self._iteration_results)
|
|
1060
1068
|
|
|
1061
|
-
struct[f"status.{
|
|
1069
|
+
struct[f"status.{RunKeys.artifacts}"] = self._artifacts_manager.artifact_list()
|
|
1062
1070
|
return struct
|
|
1063
1071
|
|
|
1064
1072
|
def _init_dbs(self, rundb):
|
|
@@ -45,6 +45,7 @@ def spark_df_to_pandas(spark_df):
|
|
|
45
45
|
),
|
|
46
46
|
)
|
|
47
47
|
type_conversion_dict[field.name] = "datetime64[ns]"
|
|
48
|
+
|
|
48
49
|
df = PandasConversionMixin.toPandas(spark_df)
|
|
49
50
|
if type_conversion_dict:
|
|
50
51
|
df = df.astype(type_conversion_dict)
|
|
@@ -252,7 +253,7 @@ class SparkFeatureMerger(BaseMerger):
|
|
|
252
253
|
)
|
|
253
254
|
source_kind = target.kind
|
|
254
255
|
source_path = target.get_target_path()
|
|
255
|
-
|
|
256
|
+
source_kwargs = target.source_spark_attributes
|
|
256
257
|
# handling case where there are multiple feature sets and user creates vector where
|
|
257
258
|
# entity_timestamp_column is from a specific feature set (can't be entity timestamp)
|
|
258
259
|
source_driver = mlrun.datastore.sources.source_kind_to_driver[source_kind]
|
mlrun/launcher/local.py
CHANGED
|
@@ -72,9 +72,9 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
|
|
|
72
72
|
reset_on_run: Optional[bool] = None,
|
|
73
73
|
) -> "mlrun.run.RunObject":
|
|
74
74
|
# do not allow local function to be scheduled
|
|
75
|
-
if
|
|
75
|
+
if schedule is not None:
|
|
76
76
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
77
|
-
"
|
|
77
|
+
f"Unexpected {schedule=} parameter for local function execution"
|
|
78
78
|
)
|
|
79
79
|
|
|
80
80
|
self.enrich_runtime(runtime, project)
|
mlrun/model.py
CHANGED
|
@@ -732,6 +732,34 @@ class Notification(ModelObj):
|
|
|
732
732
|
"Notification params size exceeds max size of 1 MB"
|
|
733
733
|
)
|
|
734
734
|
|
|
735
|
+
def validate_notification_params(self):
|
|
736
|
+
notification_class = mlrun.utils.notifications.NotificationTypes(
|
|
737
|
+
self.kind
|
|
738
|
+
).get_notification()
|
|
739
|
+
|
|
740
|
+
secret_params = self.secret_params or {}
|
|
741
|
+
params = self.params or {}
|
|
742
|
+
|
|
743
|
+
# if the secret_params are already masked - no need to validate
|
|
744
|
+
params_secret = secret_params.get("secret", "")
|
|
745
|
+
if params_secret:
|
|
746
|
+
if len(secret_params) > 1:
|
|
747
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
748
|
+
"When the 'secret' key is present, 'secret_params' should not contain any other keys."
|
|
749
|
+
)
|
|
750
|
+
return
|
|
751
|
+
|
|
752
|
+
if not secret_params and not params:
|
|
753
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
754
|
+
"Both 'secret_params' and 'params' are empty, at least one must be defined."
|
|
755
|
+
)
|
|
756
|
+
if secret_params and params and secret_params != params:
|
|
757
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
758
|
+
"Both 'secret_params' and 'params' are defined but they contain different values"
|
|
759
|
+
)
|
|
760
|
+
|
|
761
|
+
notification_class.validate_params(secret_params or params)
|
|
762
|
+
|
|
735
763
|
@staticmethod
|
|
736
764
|
def validate_notification_uniqueness(notifications: list["Notification"]):
|
|
737
765
|
"""Validate that all notifications in the list are unique by name"""
|
|
@@ -873,6 +901,7 @@ class RunSpec(ModelObj):
|
|
|
873
901
|
notifications=None,
|
|
874
902
|
state_thresholds=None,
|
|
875
903
|
reset_on_run=None,
|
|
904
|
+
node_selector=None,
|
|
876
905
|
):
|
|
877
906
|
# A dictionary of parsing configurations that will be read from the inputs the user set. The keys are the inputs
|
|
878
907
|
# keys (parameter names) and the values are the type hint given in the input keys after the colon.
|
|
@@ -910,6 +939,7 @@ class RunSpec(ModelObj):
|
|
|
910
939
|
self._notifications = notifications or []
|
|
911
940
|
self.state_thresholds = state_thresholds or {}
|
|
912
941
|
self.reset_on_run = reset_on_run
|
|
942
|
+
self.node_selector = node_selector or {}
|
|
913
943
|
|
|
914
944
|
def _serialize_field(
|
|
915
945
|
self, struct: dict, field_name: str = None, strip: bool = False
|
mlrun/model_monitoring/api.py
CHANGED
|
@@ -47,8 +47,8 @@ def get_or_create_model_endpoint(
|
|
|
47
47
|
function_name: str = "",
|
|
48
48
|
context: mlrun.MLClientCtx = None,
|
|
49
49
|
sample_set_statistics: dict[str, typing.Any] = None,
|
|
50
|
-
drift_threshold: float = None,
|
|
51
|
-
possible_drift_threshold: float = None,
|
|
50
|
+
drift_threshold: typing.Optional[float] = None,
|
|
51
|
+
possible_drift_threshold: typing.Optional[float] = None,
|
|
52
52
|
monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.disabled,
|
|
53
53
|
db_session=None,
|
|
54
54
|
) -> ModelEndpoint:
|
|
@@ -69,14 +69,14 @@ def get_or_create_model_endpoint(
|
|
|
69
69
|
full function hash.
|
|
70
70
|
:param sample_set_statistics: Dictionary of sample set statistics that will be used as a reference data for
|
|
71
71
|
the new model endpoint (applicable only to new endpoint_id).
|
|
72
|
-
:param drift_threshold: The threshold of which to mark drifts (applicable only to new
|
|
73
|
-
|
|
72
|
+
:param drift_threshold: (deprecated) The threshold of which to mark drifts (applicable only to new
|
|
73
|
+
endpoint_id).
|
|
74
|
+
:param possible_drift_threshold: (deprecated) The threshold of which to mark possible drifts (applicable only to new
|
|
74
75
|
endpoint_id).
|
|
75
76
|
:param monitoring_mode: If enabled, apply model monitoring features on the provided endpoint id
|
|
76
77
|
(applicable only to new endpoint_id).
|
|
77
78
|
:param db_session: A runtime session that manages the current dialog with the database.
|
|
78
79
|
|
|
79
|
-
|
|
80
80
|
:return: A ModelEndpoint object
|
|
81
81
|
"""
|
|
82
82
|
|
|
@@ -98,8 +98,6 @@ def get_or_create_model_endpoint(
|
|
|
98
98
|
model_endpoint=model_endpoint,
|
|
99
99
|
model_path=model_path,
|
|
100
100
|
sample_set_statistics=sample_set_statistics,
|
|
101
|
-
drift_threshold=drift_threshold,
|
|
102
|
-
possible_drift_threshold=possible_drift_threshold,
|
|
103
101
|
)
|
|
104
102
|
|
|
105
103
|
except mlrun.errors.MLRunNotFoundError:
|
|
@@ -113,8 +111,6 @@ def get_or_create_model_endpoint(
|
|
|
113
111
|
function_name=function_name,
|
|
114
112
|
context=context,
|
|
115
113
|
sample_set_statistics=sample_set_statistics,
|
|
116
|
-
drift_threshold=drift_threshold,
|
|
117
|
-
possible_drift_threshold=possible_drift_threshold,
|
|
118
114
|
monitoring_mode=monitoring_mode,
|
|
119
115
|
)
|
|
120
116
|
return model_endpoint
|
|
@@ -241,9 +237,7 @@ def _model_endpoint_validations(
|
|
|
241
237
|
model_endpoint: ModelEndpoint,
|
|
242
238
|
model_path: str = "",
|
|
243
239
|
sample_set_statistics: dict[str, typing.Any] = None,
|
|
244
|
-
|
|
245
|
-
possible_drift_threshold: float = None,
|
|
246
|
-
):
|
|
240
|
+
) -> None:
|
|
247
241
|
"""
|
|
248
242
|
Validate that provided model endpoint configurations match the stored fields of the provided `ModelEndpoint`
|
|
249
243
|
object. Usually, this method is called by `get_or_create_model_endpoint()` in cases that the model endpoint
|
|
@@ -257,11 +251,6 @@ def _model_endpoint_validations(
|
|
|
257
251
|
is forbidden to provide a different reference data to that model endpoint.
|
|
258
252
|
In case of discrepancy between the provided `sample_set_statistics` and the
|
|
259
253
|
`model_endpoints.spec.feature_stats`, a warning will be presented to the user.
|
|
260
|
-
:param drift_threshold: The threshold of which to mark drifts. Should be similar to the drift threshold
|
|
261
|
-
that has already assigned to the current model endpoint.
|
|
262
|
-
:param possible_drift_threshold: The threshold of which to mark possible drifts. Should be similar to the possible
|
|
263
|
-
drift threshold that has already assigned to the current model endpoint.
|
|
264
|
-
|
|
265
254
|
"""
|
|
266
255
|
# Model path
|
|
267
256
|
if model_path and model_endpoint.spec.model_uri != model_path:
|
|
@@ -280,28 +269,6 @@ def _model_endpoint_validations(
|
|
|
280
269
|
"Provided sample set statistics is different from the registered statistics. "
|
|
281
270
|
"If new sample set statistics is to be used, new model endpoint should be created"
|
|
282
271
|
)
|
|
283
|
-
# drift and possible drift thresholds
|
|
284
|
-
if drift_threshold:
|
|
285
|
-
current_drift_threshold = model_endpoint.spec.monitor_configuration.get(
|
|
286
|
-
mm_constants.EventFieldType.DRIFT_DETECTED_THRESHOLD,
|
|
287
|
-
mlrun.mlconf.model_endpoint_monitoring.drift_thresholds.default.drift_detected,
|
|
288
|
-
)
|
|
289
|
-
if current_drift_threshold != drift_threshold:
|
|
290
|
-
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
291
|
-
f"Cannot change existing drift threshold. Expected {current_drift_threshold}, got {drift_threshold} "
|
|
292
|
-
f"Please update drift threshold or generate a new model endpoint record"
|
|
293
|
-
)
|
|
294
|
-
|
|
295
|
-
if possible_drift_threshold:
|
|
296
|
-
current_possible_drift_threshold = model_endpoint.spec.monitor_configuration.get(
|
|
297
|
-
mm_constants.EventFieldType.POSSIBLE_DRIFT_THRESHOLD,
|
|
298
|
-
mlrun.mlconf.model_endpoint_monitoring.drift_thresholds.default.possible_drift,
|
|
299
|
-
)
|
|
300
|
-
if current_possible_drift_threshold != possible_drift_threshold:
|
|
301
|
-
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
302
|
-
f"Cannot change existing possible drift threshold. Expected {current_possible_drift_threshold}, "
|
|
303
|
-
f"got {possible_drift_threshold}. Please update drift threshold or generate a new model endpoint record"
|
|
304
|
-
)
|
|
305
272
|
|
|
306
273
|
|
|
307
274
|
def write_monitoring_df(
|
|
@@ -354,8 +321,6 @@ def _generate_model_endpoint(
|
|
|
354
321
|
function_name: str,
|
|
355
322
|
context: mlrun.MLClientCtx,
|
|
356
323
|
sample_set_statistics: dict[str, typing.Any],
|
|
357
|
-
drift_threshold: float,
|
|
358
|
-
possible_drift_threshold: float,
|
|
359
324
|
monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.disabled,
|
|
360
325
|
) -> ModelEndpoint:
|
|
361
326
|
"""
|
|
@@ -374,8 +339,6 @@ def _generate_model_endpoint(
|
|
|
374
339
|
:param sample_set_statistics: Dictionary of sample set statistics that will be used as a reference data for
|
|
375
340
|
the current model endpoint. Will be stored under
|
|
376
341
|
`model_endpoint.status.feature_stats`.
|
|
377
|
-
:param drift_threshold: The threshold of which to mark drifts.
|
|
378
|
-
:param possible_drift_threshold: The threshold of which to mark possible drifts.
|
|
379
342
|
|
|
380
343
|
:return `mlrun.model_monitoring.model_endpoint.ModelEndpoint` object.
|
|
381
344
|
"""
|
|
@@ -393,15 +356,6 @@ def _generate_model_endpoint(
|
|
|
393
356
|
model_endpoint.spec.model_uri = model_path
|
|
394
357
|
model_endpoint.spec.model = model_endpoint_name
|
|
395
358
|
model_endpoint.spec.model_class = "drift-analysis"
|
|
396
|
-
if drift_threshold:
|
|
397
|
-
model_endpoint.spec.monitor_configuration[
|
|
398
|
-
mm_constants.EventFieldType.DRIFT_DETECTED_THRESHOLD
|
|
399
|
-
] = drift_threshold
|
|
400
|
-
if possible_drift_threshold:
|
|
401
|
-
model_endpoint.spec.monitor_configuration[
|
|
402
|
-
mm_constants.EventFieldType.POSSIBLE_DRIFT_THRESHOLD
|
|
403
|
-
] = possible_drift_threshold
|
|
404
|
-
|
|
405
359
|
model_endpoint.spec.monitoring_mode = monitoring_mode
|
|
406
360
|
model_endpoint.status.first_request = model_endpoint.status.last_request = (
|
|
407
361
|
datetime_now().isoformat()
|