mlrun 1.7.0rc25__py3-none-any.whl → 1.7.0rc29__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +7 -7
- mlrun/alerts/alert.py +13 -1
- mlrun/artifacts/manager.py +5 -0
- mlrun/common/constants.py +2 -2
- mlrun/common/formatters/base.py +9 -9
- mlrun/common/schemas/alert.py +4 -8
- mlrun/common/schemas/api_gateway.py +7 -0
- mlrun/common/schemas/constants.py +3 -0
- mlrun/common/schemas/model_monitoring/__init__.py +1 -0
- mlrun/common/schemas/model_monitoring/constants.py +27 -12
- mlrun/common/schemas/model_monitoring/model_endpoints.py +0 -12
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/config.py +16 -9
- mlrun/datastore/azure_blob.py +2 -1
- mlrun/datastore/base.py +1 -5
- mlrun/datastore/datastore.py +3 -3
- mlrun/datastore/google_cloud_storage.py +6 -2
- mlrun/datastore/inmem.py +1 -1
- mlrun/datastore/snowflake_utils.py +3 -1
- mlrun/datastore/sources.py +26 -11
- mlrun/datastore/store_resources.py +2 -0
- mlrun/datastore/targets.py +60 -25
- mlrun/db/base.py +11 -0
- mlrun/db/httpdb.py +47 -33
- mlrun/db/nopdb.py +11 -1
- mlrun/errors.py +4 -0
- mlrun/execution.py +18 -10
- mlrun/feature_store/retrieval/spark_merger.py +2 -1
- mlrun/launcher/local.py +2 -2
- mlrun/model.py +30 -0
- mlrun/model_monitoring/api.py +6 -52
- mlrun/model_monitoring/applications/histogram_data_drift.py +4 -1
- mlrun/model_monitoring/db/stores/__init__.py +21 -9
- mlrun/model_monitoring/db/stores/base/store.py +39 -1
- mlrun/model_monitoring/db/stores/sqldb/models/base.py +9 -7
- mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +4 -2
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +34 -79
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +19 -27
- mlrun/model_monitoring/db/tsdb/__init__.py +19 -14
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +4 -2
- mlrun/model_monitoring/helpers.py +9 -5
- mlrun/model_monitoring/writer.py +1 -5
- mlrun/projects/operations.py +1 -0
- mlrun/projects/project.py +76 -76
- mlrun/render.py +10 -5
- mlrun/run.py +2 -2
- mlrun/runtimes/daskjob.py +7 -1
- mlrun/runtimes/local.py +24 -7
- mlrun/runtimes/nuclio/function.py +20 -0
- mlrun/runtimes/pod.py +5 -29
- mlrun/serving/routers.py +75 -59
- mlrun/serving/server.py +1 -0
- mlrun/serving/v2_serving.py +8 -1
- mlrun/utils/helpers.py +46 -2
- mlrun/utils/logger.py +36 -2
- mlrun/utils/notifications/notification/base.py +4 -0
- mlrun/utils/notifications/notification/git.py +21 -0
- mlrun/utils/notifications/notification/slack.py +8 -0
- mlrun/utils/notifications/notification/webhook.py +41 -1
- mlrun/utils/notifications/notification_pusher.py +2 -2
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc25.dist-info → mlrun-1.7.0rc29.dist-info}/METADATA +11 -6
- {mlrun-1.7.0rc25.dist-info → mlrun-1.7.0rc29.dist-info}/RECORD +67 -67
- {mlrun-1.7.0rc25.dist-info → mlrun-1.7.0rc29.dist-info}/WHEEL +1 -1
- {mlrun-1.7.0rc25.dist-info → mlrun-1.7.0rc29.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc25.dist-info → mlrun-1.7.0rc29.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc25.dist-info → mlrun-1.7.0rc29.dist-info}/top_level.txt +0 -0
mlrun/runtimes/pod.py
CHANGED
|
@@ -532,7 +532,9 @@ class KubeResourceSpec(FunctionSpec):
|
|
|
532
532
|
return
|
|
533
533
|
|
|
534
534
|
# merge node selectors - precedence to existing node selector
|
|
535
|
-
self.node_selector =
|
|
535
|
+
self.node_selector = mlrun.utils.helpers.merge_with_precedence(
|
|
536
|
+
node_selector, self.node_selector
|
|
537
|
+
)
|
|
536
538
|
|
|
537
539
|
def _merge_tolerations(
|
|
538
540
|
self,
|
|
@@ -1038,32 +1040,6 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
|
|
|
1038
1040
|
return True
|
|
1039
1041
|
return False
|
|
1040
1042
|
|
|
1041
|
-
def enrich_runtime_spec(
|
|
1042
|
-
self,
|
|
1043
|
-
project_node_selector: dict[str, str],
|
|
1044
|
-
):
|
|
1045
|
-
"""
|
|
1046
|
-
Enriches the runtime spec with the project-level node selector.
|
|
1047
|
-
|
|
1048
|
-
This method merges the project-level node selector with the existing function node_selector.
|
|
1049
|
-
The merge logic used here combines the two dictionaries, giving precedence to
|
|
1050
|
-
the keys in the runtime node_selector. If there are conflicting keys between the
|
|
1051
|
-
two dictionaries, the values from self.spec.node_selector will overwrite the
|
|
1052
|
-
values from project_node_selector.
|
|
1053
|
-
|
|
1054
|
-
Example:
|
|
1055
|
-
Suppose self.spec.node_selector = {"type": "gpu", "zone": "us-east-1"}
|
|
1056
|
-
and project_node_selector = {"type": "cpu", "environment": "production"}.
|
|
1057
|
-
After the merge, the resulting node_selector will be:
|
|
1058
|
-
{"type": "gpu", "zone": "us-east-1", "environment": "production"}
|
|
1059
|
-
|
|
1060
|
-
Note:
|
|
1061
|
-
- The merge uses the ** operator, also known as the "unpacking" operator in Python,
|
|
1062
|
-
combining key-value pairs from each dictionary. Later dictionaries take precedence
|
|
1063
|
-
when there are conflicting keys.
|
|
1064
|
-
"""
|
|
1065
|
-
self.spec.node_selector = {**project_node_selector, **self.spec.node_selector}
|
|
1066
|
-
|
|
1067
1043
|
def _set_env(self, name, value=None, value_from=None):
|
|
1068
1044
|
new_var = k8s_client.V1EnvVar(name=name, value=value, value_from=value_from)
|
|
1069
1045
|
|
|
@@ -1542,7 +1518,7 @@ def get_sanitized_attribute(spec, attribute_name: str):
|
|
|
1542
1518
|
|
|
1543
1519
|
# check if attribute of type dict, and then check if type is sanitized
|
|
1544
1520
|
if isinstance(attribute, dict):
|
|
1545
|
-
if attribute_config["not_sanitized_class"]
|
|
1521
|
+
if not isinstance(attribute_config["not_sanitized_class"], dict):
|
|
1546
1522
|
raise mlrun.errors.MLRunInvalidArgumentTypeError(
|
|
1547
1523
|
f"expected to be of type {attribute_config.get('not_sanitized_class')} but got dict"
|
|
1548
1524
|
)
|
|
@@ -1552,7 +1528,7 @@ def get_sanitized_attribute(spec, attribute_name: str):
|
|
|
1552
1528
|
elif isinstance(attribute, list) and not isinstance(
|
|
1553
1529
|
attribute[0], attribute_config["sub_attribute_type"]
|
|
1554
1530
|
):
|
|
1555
|
-
if attribute_config["not_sanitized_class"]
|
|
1531
|
+
if not isinstance(attribute_config["not_sanitized_class"], list):
|
|
1556
1532
|
raise mlrun.errors.MLRunInvalidArgumentTypeError(
|
|
1557
1533
|
f"expected to be of type {attribute_config.get('not_sanitized_class')} but got list"
|
|
1558
1534
|
)
|
mlrun/serving/routers.py
CHANGED
|
@@ -1030,74 +1030,90 @@ def _init_endpoint_record(
|
|
|
1030
1030
|
function_uri=graph_server.function_uri, versioned_model=versioned_model_name
|
|
1031
1031
|
).uid
|
|
1032
1032
|
|
|
1033
|
-
# If model endpoint object was found in DB, skip the creation process.
|
|
1034
1033
|
try:
|
|
1035
|
-
mlrun.get_run_db().get_model_endpoint(
|
|
1036
|
-
|
|
1034
|
+
model_ep = mlrun.get_run_db().get_model_endpoint(
|
|
1035
|
+
project=project, endpoint_id=endpoint_uid
|
|
1036
|
+
)
|
|
1037
1037
|
except mlrun.errors.MLRunNotFoundError:
|
|
1038
|
-
|
|
1038
|
+
model_ep = None
|
|
1039
|
+
except mlrun.errors.MLRunBadRequestError as err:
|
|
1040
|
+
logger.debug(
|
|
1041
|
+
f"Cant reach to model endpoints store, due to : {err}",
|
|
1042
|
+
)
|
|
1043
|
+
return
|
|
1039
1044
|
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
),
|
|
1058
|
-
active=True,
|
|
1059
|
-
monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
|
|
1060
|
-
if voting_ensemble.context.server.track_models
|
|
1061
|
-
else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled,
|
|
1062
|
-
),
|
|
1063
|
-
status=mlrun.common.schemas.ModelEndpointStatus(
|
|
1064
|
-
children=list(voting_ensemble.routes.keys()),
|
|
1065
|
-
endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.ROUTER,
|
|
1066
|
-
children_uids=children_uids,
|
|
1045
|
+
if voting_ensemble.context.server.track_models and not model_ep:
|
|
1046
|
+
logger.info("Creating a new model endpoint record", endpoint_id=endpoint_uid)
|
|
1047
|
+
# Get the children model endpoints ids
|
|
1048
|
+
children_uids = []
|
|
1049
|
+
for _, c in voting_ensemble.routes.items():
|
|
1050
|
+
if hasattr(c, "endpoint_uid"):
|
|
1051
|
+
children_uids.append(c.endpoint_uid)
|
|
1052
|
+
model_endpoint = mlrun.common.schemas.ModelEndpoint(
|
|
1053
|
+
metadata=mlrun.common.schemas.ModelEndpointMetadata(
|
|
1054
|
+
project=project, uid=endpoint_uid
|
|
1055
|
+
),
|
|
1056
|
+
spec=mlrun.common.schemas.ModelEndpointSpec(
|
|
1057
|
+
function_uri=graph_server.function_uri,
|
|
1058
|
+
model=versioned_model_name,
|
|
1059
|
+
model_class=voting_ensemble.__class__.__name__,
|
|
1060
|
+
stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
|
|
1061
|
+
project=project, kind="stream"
|
|
1067
1062
|
),
|
|
1068
|
-
|
|
1063
|
+
active=True,
|
|
1064
|
+
monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled,
|
|
1065
|
+
),
|
|
1066
|
+
status=mlrun.common.schemas.ModelEndpointStatus(
|
|
1067
|
+
children=list(voting_ensemble.routes.keys()),
|
|
1068
|
+
endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.ROUTER,
|
|
1069
|
+
children_uids=children_uids,
|
|
1070
|
+
),
|
|
1071
|
+
)
|
|
1069
1072
|
|
|
1070
|
-
|
|
1073
|
+
db = mlrun.get_run_db()
|
|
1074
|
+
|
|
1075
|
+
db.create_model_endpoint(
|
|
1076
|
+
project=project,
|
|
1077
|
+
endpoint_id=model_endpoint.metadata.uid,
|
|
1078
|
+
model_endpoint=model_endpoint.dict(),
|
|
1079
|
+
)
|
|
1071
1080
|
|
|
1081
|
+
# Update model endpoint children type
|
|
1082
|
+
for model_endpoint in children_uids:
|
|
1083
|
+
current_endpoint = db.get_model_endpoint(
|
|
1084
|
+
project=project, endpoint_id=model_endpoint
|
|
1085
|
+
)
|
|
1086
|
+
current_endpoint.status.endpoint_type = (
|
|
1087
|
+
mlrun.common.schemas.model_monitoring.EndpointType.LEAF_EP
|
|
1088
|
+
)
|
|
1072
1089
|
db.create_model_endpoint(
|
|
1073
1090
|
project=project,
|
|
1074
|
-
endpoint_id=model_endpoint
|
|
1075
|
-
model_endpoint=
|
|
1076
|
-
)
|
|
1077
|
-
|
|
1078
|
-
# Update model endpoint children type
|
|
1079
|
-
for model_endpoint in children_uids:
|
|
1080
|
-
current_endpoint = db.get_model_endpoint(
|
|
1081
|
-
project=project, endpoint_id=model_endpoint
|
|
1082
|
-
)
|
|
1083
|
-
current_endpoint.status.endpoint_type = (
|
|
1084
|
-
mlrun.common.schemas.model_monitoring.EndpointType.LEAF_EP
|
|
1085
|
-
)
|
|
1086
|
-
db.create_model_endpoint(
|
|
1087
|
-
project=project,
|
|
1088
|
-
endpoint_id=model_endpoint,
|
|
1089
|
-
model_endpoint=current_endpoint,
|
|
1090
|
-
)
|
|
1091
|
-
|
|
1092
|
-
except Exception as exc:
|
|
1093
|
-
logger.warning(
|
|
1094
|
-
"Failed creating model endpoint record",
|
|
1095
|
-
exc=err_to_str(exc),
|
|
1096
|
-
traceback=traceback.format_exc(),
|
|
1091
|
+
endpoint_id=model_endpoint,
|
|
1092
|
+
model_endpoint=current_endpoint,
|
|
1097
1093
|
)
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1094
|
+
elif (
|
|
1095
|
+
model_ep
|
|
1096
|
+
and (
|
|
1097
|
+
model_ep.spec.monitoring_mode
|
|
1098
|
+
== mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
|
|
1099
|
+
)
|
|
1100
|
+
!= voting_ensemble.context.server.track_models
|
|
1101
|
+
):
|
|
1102
|
+
monitoring_mode = (
|
|
1103
|
+
mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
|
|
1104
|
+
if voting_ensemble.context.server.track_models
|
|
1105
|
+
else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled
|
|
1106
|
+
)
|
|
1107
|
+
db = mlrun.get_run_db()
|
|
1108
|
+
db.patch_model_endpoint(
|
|
1109
|
+
project=project,
|
|
1110
|
+
endpoint_id=endpoint_uid,
|
|
1111
|
+
attributes={"monitoring_mode": monitoring_mode},
|
|
1112
|
+
)
|
|
1113
|
+
logger.debug(
|
|
1114
|
+
f"Updating model endpoint monitoring_mode to {monitoring_mode}",
|
|
1115
|
+
endpoint_id=endpoint_uid,
|
|
1116
|
+
)
|
|
1101
1117
|
|
|
1102
1118
|
return endpoint_uid
|
|
1103
1119
|
|
mlrun/serving/server.py
CHANGED
mlrun/serving/v2_serving.py
CHANGED
|
@@ -531,7 +531,9 @@ def _init_endpoint_record(
|
|
|
531
531
|
if model.model_path and model.model_path.startswith("store://"):
|
|
532
532
|
# Enrich the model server with the model artifact metadata
|
|
533
533
|
model.get_model()
|
|
534
|
-
|
|
534
|
+
if not model.version:
|
|
535
|
+
# Enrich the model version with the model artifact tag
|
|
536
|
+
model.version = model.model_spec.tag
|
|
535
537
|
model.labels = model.model_spec.labels
|
|
536
538
|
versioned_model_name = f"{model.name}:{model.version}"
|
|
537
539
|
else:
|
|
@@ -548,6 +550,11 @@ def _init_endpoint_record(
|
|
|
548
550
|
)
|
|
549
551
|
except mlrun.errors.MLRunNotFoundError:
|
|
550
552
|
model_ep = None
|
|
553
|
+
except mlrun.errors.MLRunBadRequestError as err:
|
|
554
|
+
logger.debug(
|
|
555
|
+
f"Cant reach to model endpoints store, due to : {err}",
|
|
556
|
+
)
|
|
557
|
+
return
|
|
551
558
|
|
|
552
559
|
if model.context.server.track_models and not model_ep:
|
|
553
560
|
logger.debug("Creating a new model endpoint record", endpoint_id=uid)
|
mlrun/utils/helpers.py
CHANGED
|
@@ -109,10 +109,13 @@ def get_artifact_target(item: dict, project=None):
|
|
|
109
109
|
db_key = item["spec"].get("db_key")
|
|
110
110
|
project_str = project or item["metadata"].get("project")
|
|
111
111
|
tree = item["metadata"].get("tree")
|
|
112
|
+
tag = item["metadata"].get("tag")
|
|
112
113
|
|
|
113
114
|
kind = item.get("kind")
|
|
114
115
|
if kind in ["dataset", "model", "artifact"] and db_key:
|
|
115
116
|
target = f"{DB_SCHEMA}://{StorePrefix.Artifact}/{project_str}/{db_key}"
|
|
117
|
+
if tag:
|
|
118
|
+
target = f"{target}:{tag}"
|
|
116
119
|
if tree:
|
|
117
120
|
target = f"{target}@{tree}"
|
|
118
121
|
return target
|
|
@@ -149,7 +152,7 @@ if is_ipython and config.nest_asyncio_enabled in ["1", "True"]:
|
|
|
149
152
|
nest_asyncio.apply()
|
|
150
153
|
|
|
151
154
|
|
|
152
|
-
class
|
|
155
|
+
class RunKeys:
|
|
153
156
|
input_path = "input_path"
|
|
154
157
|
output_path = "output_path"
|
|
155
158
|
inputs = "inputs"
|
|
@@ -160,6 +163,10 @@ class run_keys:
|
|
|
160
163
|
secrets = "secret_sources"
|
|
161
164
|
|
|
162
165
|
|
|
166
|
+
# for Backward compatibility
|
|
167
|
+
run_keys = RunKeys
|
|
168
|
+
|
|
169
|
+
|
|
163
170
|
def verify_field_regex(
|
|
164
171
|
field_name,
|
|
165
172
|
field_value,
|
|
@@ -812,7 +819,6 @@ def enrich_image_url(
|
|
|
812
819
|
tag += resolve_image_tag_suffix(
|
|
813
820
|
mlrun_version=mlrun_version, python_version=client_python_version
|
|
814
821
|
)
|
|
815
|
-
registry = config.images_registry
|
|
816
822
|
|
|
817
823
|
# it's an mlrun image if the repository is mlrun
|
|
818
824
|
is_mlrun_image = image_url.startswith("mlrun/") or "/mlrun/" in image_url
|
|
@@ -820,6 +826,10 @@ def enrich_image_url(
|
|
|
820
826
|
if is_mlrun_image and tag and ":" not in image_url:
|
|
821
827
|
image_url = f"{image_url}:{tag}"
|
|
822
828
|
|
|
829
|
+
registry = (
|
|
830
|
+
config.images_registry if is_mlrun_image else config.vendor_images_registry
|
|
831
|
+
)
|
|
832
|
+
|
|
823
833
|
enrich_registry = False
|
|
824
834
|
# enrich registry only if images_to_enrich_registry provided
|
|
825
835
|
# example: "^mlrun/*" means enrich only if the image repository is mlrun and registry is not specified (in which
|
|
@@ -1259,6 +1269,10 @@ def _fill_project_path_template(artifact_path, project):
|
|
|
1259
1269
|
return artifact_path
|
|
1260
1270
|
|
|
1261
1271
|
|
|
1272
|
+
def to_non_empty_values_dict(input_dict: dict) -> dict:
|
|
1273
|
+
return {key: value for key, value in input_dict.items() if value}
|
|
1274
|
+
|
|
1275
|
+
|
|
1262
1276
|
def str_to_timestamp(time_str: str, now_time: Timestamp = None):
|
|
1263
1277
|
"""convert fixed/relative time string to Pandas Timestamp
|
|
1264
1278
|
|
|
@@ -1606,6 +1620,30 @@ def additional_filters_warning(additional_filters, class_name):
|
|
|
1606
1620
|
)
|
|
1607
1621
|
|
|
1608
1622
|
|
|
1623
|
+
def merge_with_precedence(first_dict: dict, second_dict: dict) -> dict:
|
|
1624
|
+
"""
|
|
1625
|
+
Merge two dictionaries with precedence given to keys from the second dictionary.
|
|
1626
|
+
|
|
1627
|
+
This function merges two dictionaries, `first_dict` and `second_dict`, where keys from `second_dict`
|
|
1628
|
+
take precedence in case of conflicts. If both dictionaries contain the same key,
|
|
1629
|
+
the value from `second_dict` will overwrite the value from `first_dict`.
|
|
1630
|
+
|
|
1631
|
+
Example:
|
|
1632
|
+
>>> first_dict = {"key1": "value1", "key2": "value2"}
|
|
1633
|
+
>>> second_dict = {"key2": "new_value2", "key3": "value3"}
|
|
1634
|
+
>>> merge_with_precedence(first_dict, second_dict)
|
|
1635
|
+
{'key1': 'value1', 'key2': 'new_value2', 'key3': 'value3'}
|
|
1636
|
+
|
|
1637
|
+
Note:
|
|
1638
|
+
- The merge operation uses the ** operator in Python, which combines key-value pairs
|
|
1639
|
+
from each dictionary. Later dictionaries take precedence when there are conflicting keys.
|
|
1640
|
+
"""
|
|
1641
|
+
return {
|
|
1642
|
+
**(first_dict or {}),
|
|
1643
|
+
**(second_dict or {}),
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
|
|
1609
1647
|
def validate_component_version_compatibility(
|
|
1610
1648
|
component_name: typing.Literal["iguazio", "nuclio"], *min_versions: str
|
|
1611
1649
|
):
|
|
@@ -1663,6 +1701,12 @@ def format_alert_summary(
|
|
|
1663
1701
|
return result
|
|
1664
1702
|
|
|
1665
1703
|
|
|
1704
|
+
def is_parquet_file(file_path, format_=None):
|
|
1705
|
+
return (file_path and file_path.endswith((".parquet", ".pq"))) or (
|
|
1706
|
+
format_ == "parquet"
|
|
1707
|
+
)
|
|
1708
|
+
|
|
1709
|
+
|
|
1666
1710
|
def _reload(module, max_recursion_depth):
|
|
1667
1711
|
"""Recursively reload modules."""
|
|
1668
1712
|
if max_recursion_depth <= 0:
|
mlrun/utils/logger.py
CHANGED
|
@@ -13,8 +13,10 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import logging
|
|
16
|
+
import os
|
|
16
17
|
import typing
|
|
17
18
|
from enum import Enum
|
|
19
|
+
from functools import cached_property
|
|
18
20
|
from sys import stdout
|
|
19
21
|
from traceback import format_exception
|
|
20
22
|
from typing import IO, Optional, Union
|
|
@@ -92,6 +94,16 @@ class HumanReadableFormatter(_BaseFormatter):
|
|
|
92
94
|
|
|
93
95
|
|
|
94
96
|
class HumanReadableExtendedFormatter(HumanReadableFormatter):
|
|
97
|
+
_colors = {
|
|
98
|
+
logging.NOTSET: "",
|
|
99
|
+
logging.DEBUG: "\x1b[34m",
|
|
100
|
+
logging.INFO: "\x1b[36m",
|
|
101
|
+
logging.WARNING: "\x1b[33m",
|
|
102
|
+
logging.ERROR: "\x1b[0;31m",
|
|
103
|
+
logging.CRITICAL: "\x1b[1;31m",
|
|
104
|
+
}
|
|
105
|
+
_color_reset = "\x1b[0m"
|
|
106
|
+
|
|
95
107
|
def format(self, record) -> str:
|
|
96
108
|
more = ""
|
|
97
109
|
record_with = self._record_with(record)
|
|
@@ -113,12 +125,34 @@ class HumanReadableExtendedFormatter(HumanReadableFormatter):
|
|
|
113
125
|
[f"{key}: {_format_value(val)}" for key, val in record_with.items()]
|
|
114
126
|
)
|
|
115
127
|
return (
|
|
116
|
-
"> "
|
|
128
|
+
f"{self._get_message_color(record.levelno)}> "
|
|
117
129
|
f"{self.formatTime(record, self.datefmt)} "
|
|
118
130
|
f"[{record.name}:{record.levelname.lower()}] "
|
|
119
|
-
f"{record.getMessage()}{more}"
|
|
131
|
+
f"{record.getMessage()}{more}{self._get_color_reset()}"
|
|
120
132
|
)
|
|
121
133
|
|
|
134
|
+
def _get_color_reset(self):
|
|
135
|
+
if not self._have_color_support:
|
|
136
|
+
return ""
|
|
137
|
+
|
|
138
|
+
return self._color_reset
|
|
139
|
+
|
|
140
|
+
def _get_message_color(self, levelno):
|
|
141
|
+
if not self._have_color_support:
|
|
142
|
+
return ""
|
|
143
|
+
|
|
144
|
+
return self._colors[levelno]
|
|
145
|
+
|
|
146
|
+
@cached_property
|
|
147
|
+
def _have_color_support(self):
|
|
148
|
+
if os.environ.get("PYCHARM_HOSTED"):
|
|
149
|
+
return True
|
|
150
|
+
if os.environ.get("NO_COLOR"):
|
|
151
|
+
return False
|
|
152
|
+
if os.environ.get("CLICOLOR_FORCE"):
|
|
153
|
+
return True
|
|
154
|
+
return stdout.isatty()
|
|
155
|
+
|
|
122
156
|
|
|
123
157
|
class Logger:
|
|
124
158
|
def __init__(
|
|
@@ -30,6 +30,27 @@ class GitNotification(NotificationBase):
|
|
|
30
30
|
API/Client notification for setting a rich run statuses git issue comment (github/gitlab)
|
|
31
31
|
"""
|
|
32
32
|
|
|
33
|
+
@classmethod
|
|
34
|
+
def validate_params(cls, params):
|
|
35
|
+
git_repo = params.get("repo", None)
|
|
36
|
+
git_issue = params.get("issue", None)
|
|
37
|
+
git_merge_request = params.get("merge_request", None)
|
|
38
|
+
token = (
|
|
39
|
+
params.get("token", None)
|
|
40
|
+
or params.get("GIT_TOKEN", None)
|
|
41
|
+
or params.get("GITHUB_TOKEN", None)
|
|
42
|
+
)
|
|
43
|
+
if not git_repo:
|
|
44
|
+
raise ValueError("Parameter 'repo' is required for GitNotification")
|
|
45
|
+
|
|
46
|
+
if not token:
|
|
47
|
+
raise ValueError("Parameter 'token' is required for GitNotification")
|
|
48
|
+
|
|
49
|
+
if not git_issue and not git_merge_request:
|
|
50
|
+
raise ValueError(
|
|
51
|
+
"At least one of 'issue' or 'merge_request' is required for GitNotification"
|
|
52
|
+
)
|
|
53
|
+
|
|
33
54
|
async def push(
|
|
34
55
|
self,
|
|
35
56
|
message: str,
|
|
@@ -35,6 +35,14 @@ class SlackNotification(NotificationBase):
|
|
|
35
35
|
"skipped": ":zzz:",
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
@classmethod
|
|
39
|
+
def validate_params(cls, params):
|
|
40
|
+
webhook = params.get("webhook", None) or mlrun.get_secret_or_env(
|
|
41
|
+
"SLACK_WEBHOOK"
|
|
42
|
+
)
|
|
43
|
+
if not webhook:
|
|
44
|
+
raise ValueError("Parameter 'webhook' is required for SlackNotification")
|
|
45
|
+
|
|
38
46
|
async def push(
|
|
39
47
|
self,
|
|
40
48
|
message: str,
|
|
@@ -28,6 +28,12 @@ class WebhookNotification(NotificationBase):
|
|
|
28
28
|
API/Client notification for sending run statuses in a http request
|
|
29
29
|
"""
|
|
30
30
|
|
|
31
|
+
@classmethod
|
|
32
|
+
def validate_params(cls, params):
|
|
33
|
+
url = params.get("url", None)
|
|
34
|
+
if not url:
|
|
35
|
+
raise ValueError("Parameter 'url' is required for WebhookNotification")
|
|
36
|
+
|
|
31
37
|
async def push(
|
|
32
38
|
self,
|
|
33
39
|
message: str,
|
|
@@ -63,7 +69,7 @@ class WebhookNotification(NotificationBase):
|
|
|
63
69
|
request_body["custom_html"] = custom_html
|
|
64
70
|
|
|
65
71
|
if override_body:
|
|
66
|
-
request_body = override_body
|
|
72
|
+
request_body = self._serialize_runs_in_request_body(override_body, runs)
|
|
67
73
|
|
|
68
74
|
# Specify the `verify_ssl` parameter value only for HTTPS urls.
|
|
69
75
|
# The `ClientSession` allows using `ssl=None` for the default SSL check,
|
|
@@ -77,3 +83,37 @@ class WebhookNotification(NotificationBase):
|
|
|
77
83
|
url, headers=headers, json=request_body, ssl=verify_ssl
|
|
78
84
|
)
|
|
79
85
|
response.raise_for_status()
|
|
86
|
+
|
|
87
|
+
@staticmethod
|
|
88
|
+
def _serialize_runs_in_request_body(override_body, runs):
|
|
89
|
+
str_parsed_runs = ""
|
|
90
|
+
runs = runs or []
|
|
91
|
+
|
|
92
|
+
def parse_runs():
|
|
93
|
+
parsed_runs = []
|
|
94
|
+
for run in runs:
|
|
95
|
+
if hasattr(run, "to_dict"):
|
|
96
|
+
run = run.to_dict()
|
|
97
|
+
if isinstance(run, dict):
|
|
98
|
+
parsed_run = {
|
|
99
|
+
"project": run["metadata"]["project"],
|
|
100
|
+
"name": run["metadata"]["name"],
|
|
101
|
+
"host": run["metadata"]["labels"]["host"],
|
|
102
|
+
"status": {"state": run["status"]["state"]},
|
|
103
|
+
}
|
|
104
|
+
if run["status"].get("error", None):
|
|
105
|
+
parsed_run["status"]["error"] = run["status"]["error"]
|
|
106
|
+
elif run["status"].get("results", None):
|
|
107
|
+
parsed_run["status"]["results"] = run["status"]["results"]
|
|
108
|
+
parsed_runs.append(parsed_run)
|
|
109
|
+
return str(parsed_runs)
|
|
110
|
+
|
|
111
|
+
if isinstance(override_body, dict):
|
|
112
|
+
for key, value in override_body.items():
|
|
113
|
+
if "{{ runs }}" or "{{runs}}" in value:
|
|
114
|
+
if not str_parsed_runs:
|
|
115
|
+
str_parsed_runs = parse_runs()
|
|
116
|
+
override_body[key] = value.replace(
|
|
117
|
+
"{{ runs }}", str_parsed_runs
|
|
118
|
+
).replace("{{runs}}", str_parsed_runs)
|
|
119
|
+
return override_body
|
|
@@ -397,7 +397,7 @@ class NotificationPusher(_NotificationPusherBase):
|
|
|
397
397
|
try:
|
|
398
398
|
_run = db.list_runs(
|
|
399
399
|
project=run.metadata.project,
|
|
400
|
-
labels=f"mlrun_constants.MLRunInternalLabels.runner_pod={_step.node_name}",
|
|
400
|
+
labels=f"{mlrun_constants.MLRunInternalLabels.runner_pod}={_step.node_name}",
|
|
401
401
|
)[0]
|
|
402
402
|
except IndexError:
|
|
403
403
|
_run = {
|
|
@@ -484,7 +484,7 @@ class NotificationPusher(_NotificationPusherBase):
|
|
|
484
484
|
def _get_workflow_manifest(
|
|
485
485
|
workflow_id: str,
|
|
486
486
|
) -> typing.Optional[mlrun_pipelines.models.PipelineManifest]:
|
|
487
|
-
kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf)
|
|
487
|
+
kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf.kfp_url)
|
|
488
488
|
|
|
489
489
|
# arbitrary timeout of 5 seconds, the workflow should be done by now
|
|
490
490
|
kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)
|
mlrun/utils/version/version.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mlrun
|
|
3
|
-
Version: 1.7.
|
|
3
|
+
Version: 1.7.0rc29
|
|
4
4
|
Summary: Tracking and config of machine learning runs
|
|
5
5
|
Home-page: https://github.com/mlrun/mlrun
|
|
6
6
|
Author: Yaron Haviv
|
|
@@ -28,17 +28,17 @@ Requires-Dist: aiohttp-retry ~=2.8
|
|
|
28
28
|
Requires-Dist: click ~=8.1
|
|
29
29
|
Requires-Dist: nest-asyncio ~=1.0
|
|
30
30
|
Requires-Dist: ipython ~=8.10
|
|
31
|
-
Requires-Dist: nuclio-jupyter ~=0.
|
|
31
|
+
Requires-Dist: nuclio-jupyter ~=0.10.0
|
|
32
32
|
Requires-Dist: numpy <1.27.0,>=1.16.5
|
|
33
33
|
Requires-Dist: pandas <2.2,>=1.2
|
|
34
34
|
Requires-Dist: pyarrow <15,>=10.0
|
|
35
|
-
Requires-Dist: pyyaml
|
|
35
|
+
Requires-Dist: pyyaml <7,>=5.4.1
|
|
36
36
|
Requires-Dist: requests ~=2.31
|
|
37
37
|
Requires-Dist: tabulate ~=0.8.6
|
|
38
38
|
Requires-Dist: v3io ~=0.6.4
|
|
39
39
|
Requires-Dist: pydantic <1.10.15,>=1.10.8
|
|
40
40
|
Requires-Dist: mergedeep ~=1.3
|
|
41
|
-
Requires-Dist: v3io-frames ~=0.10.
|
|
41
|
+
Requires-Dist: v3io-frames ~=0.10.14
|
|
42
42
|
Requires-Dist: semver ~=3.0
|
|
43
43
|
Requires-Dist: dependency-injector ~=4.41
|
|
44
44
|
Requires-Dist: fsspec <2024.4,>=2023.9.2
|
|
@@ -50,8 +50,8 @@ Requires-Dist: setuptools ~=69.1
|
|
|
50
50
|
Requires-Dist: deprecated ~=1.2
|
|
51
51
|
Requires-Dist: jinja2 >=3.1.3,~=3.1
|
|
52
52
|
Requires-Dist: orjson <4,>=3.9.15
|
|
53
|
-
Requires-Dist: mlrun-pipelines-kfp-common
|
|
54
|
-
Requires-Dist: mlrun-pipelines-kfp-v1-8
|
|
53
|
+
Requires-Dist: mlrun-pipelines-kfp-common ~=0.1.2
|
|
54
|
+
Requires-Dist: mlrun-pipelines-kfp-v1-8 ~=0.1.2
|
|
55
55
|
Provides-Extra: alibaba-oss
|
|
56
56
|
Requires-Dist: ossfs ==2023.12.0 ; extra == 'alibaba-oss'
|
|
57
57
|
Requires-Dist: oss2 ==2.18.1 ; extra == 'alibaba-oss'
|
|
@@ -81,6 +81,7 @@ Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'all'
|
|
|
81
81
|
Requires-Dist: pyopenssl >=23 ; extra == 'all'
|
|
82
82
|
Requires-Dist: redis ~=4.3 ; extra == 'all'
|
|
83
83
|
Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'all'
|
|
84
|
+
Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'all'
|
|
84
85
|
Requires-Dist: sqlalchemy ~=1.4 ; extra == 'all'
|
|
85
86
|
Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'all'
|
|
86
87
|
Provides-Extra: api
|
|
@@ -130,6 +131,7 @@ Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'complete'
|
|
|
130
131
|
Requires-Dist: pyopenssl >=23 ; extra == 'complete'
|
|
131
132
|
Requires-Dist: redis ~=4.3 ; extra == 'complete'
|
|
132
133
|
Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'complete'
|
|
134
|
+
Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete'
|
|
133
135
|
Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete'
|
|
134
136
|
Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete'
|
|
135
137
|
Provides-Extra: complete-api
|
|
@@ -164,6 +166,7 @@ Requires-Dist: pymysql ~=1.0 ; extra == 'complete-api'
|
|
|
164
166
|
Requires-Dist: pyopenssl >=23 ; extra == 'complete-api'
|
|
165
167
|
Requires-Dist: redis ~=4.3 ; extra == 'complete-api'
|
|
166
168
|
Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'complete-api'
|
|
169
|
+
Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete-api'
|
|
167
170
|
Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete-api'
|
|
168
171
|
Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete-api'
|
|
169
172
|
Requires-Dist: timelength ~=1.1 ; extra == 'complete-api'
|
|
@@ -196,6 +199,8 @@ Provides-Extra: s3
|
|
|
196
199
|
Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 's3'
|
|
197
200
|
Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 's3'
|
|
198
201
|
Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 's3'
|
|
202
|
+
Provides-Extra: snowflake
|
|
203
|
+
Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'snowflake'
|
|
199
204
|
Provides-Extra: sqlalchemy
|
|
200
205
|
Requires-Dist: sqlalchemy ~=1.4 ; extra == 'sqlalchemy'
|
|
201
206
|
Provides-Extra: tdengine
|