mlrun 1.7.0rc20__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +10 -8
- mlrun/alerts/alert.py +55 -18
- mlrun/api/schemas/__init__.py +3 -3
- mlrun/artifacts/manager.py +26 -0
- mlrun/common/constants.py +3 -2
- mlrun/common/formatters/__init__.py +1 -0
- mlrun/common/formatters/artifact.py +26 -3
- mlrun/common/formatters/base.py +44 -9
- mlrun/common/formatters/function.py +12 -7
- mlrun/common/formatters/run.py +26 -0
- mlrun/common/helpers.py +11 -0
- mlrun/common/schemas/__init__.py +4 -0
- mlrun/common/schemas/alert.py +5 -9
- mlrun/common/schemas/api_gateway.py +64 -16
- mlrun/common/schemas/artifact.py +11 -0
- mlrun/common/schemas/constants.py +3 -0
- mlrun/common/schemas/feature_store.py +58 -28
- mlrun/common/schemas/model_monitoring/constants.py +21 -12
- mlrun/common/schemas/model_monitoring/model_endpoints.py +0 -12
- mlrun/common/schemas/pipeline.py +16 -0
- mlrun/common/schemas/project.py +17 -0
- mlrun/common/schemas/runs.py +17 -0
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/types.py +6 -0
- mlrun/config.py +17 -25
- mlrun/datastore/azure_blob.py +2 -1
- mlrun/datastore/datastore.py +3 -3
- mlrun/datastore/google_cloud_storage.py +6 -2
- mlrun/datastore/snowflake_utils.py +3 -1
- mlrun/datastore/sources.py +26 -11
- mlrun/datastore/store_resources.py +2 -0
- mlrun/datastore/targets.py +68 -16
- mlrun/db/base.py +83 -2
- mlrun/db/httpdb.py +280 -63
- mlrun/db/nopdb.py +60 -3
- mlrun/errors.py +5 -3
- mlrun/execution.py +28 -13
- mlrun/feature_store/feature_vector.py +8 -0
- mlrun/feature_store/retrieval/spark_merger.py +13 -2
- mlrun/launcher/local.py +4 -0
- mlrun/launcher/remote.py +1 -0
- mlrun/model.py +32 -3
- mlrun/model_monitoring/api.py +7 -52
- mlrun/model_monitoring/applications/base.py +5 -7
- mlrun/model_monitoring/applications/histogram_data_drift.py +1 -1
- mlrun/model_monitoring/db/stores/__init__.py +37 -24
- mlrun/model_monitoring/db/stores/base/store.py +40 -1
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +42 -87
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +27 -35
- mlrun/model_monitoring/db/tsdb/__init__.py +15 -15
- mlrun/model_monitoring/db/tsdb/base.py +1 -14
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +22 -18
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +86 -56
- mlrun/model_monitoring/helpers.py +34 -9
- mlrun/model_monitoring/stream_processing.py +12 -11
- mlrun/model_monitoring/writer.py +11 -11
- mlrun/projects/operations.py +5 -0
- mlrun/projects/pipelines.py +35 -21
- mlrun/projects/project.py +216 -107
- mlrun/render.py +10 -5
- mlrun/run.py +15 -5
- mlrun/runtimes/__init__.py +2 -0
- mlrun/runtimes/base.py +17 -4
- mlrun/runtimes/daskjob.py +8 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +1 -0
- mlrun/runtimes/local.py +23 -4
- mlrun/runtimes/nuclio/application/application.py +0 -2
- mlrun/runtimes/nuclio/function.py +31 -2
- mlrun/runtimes/nuclio/serving.py +9 -6
- mlrun/runtimes/pod.py +5 -29
- mlrun/runtimes/remotesparkjob.py +8 -2
- mlrun/serving/__init__.py +8 -1
- mlrun/serving/routers.py +75 -59
- mlrun/serving/server.py +11 -0
- mlrun/serving/states.py +80 -8
- mlrun/serving/utils.py +19 -11
- mlrun/serving/v2_serving.py +66 -39
- mlrun/utils/helpers.py +91 -11
- mlrun/utils/logger.py +36 -2
- mlrun/utils/notifications/notification/base.py +43 -7
- mlrun/utils/notifications/notification/git.py +21 -0
- mlrun/utils/notifications/notification/slack.py +9 -14
- mlrun/utils/notifications/notification/webhook.py +41 -1
- mlrun/utils/notifications/notification_pusher.py +3 -9
- mlrun/utils/regex.py +9 -0
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/METADATA +16 -9
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/RECORD +92 -91
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/WHEEL +1 -1
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py
CHANGED
|
@@ -51,6 +51,7 @@ import mlrun.runtimes.nuclio.api_gateway
|
|
|
51
51
|
import mlrun.runtimes.pod
|
|
52
52
|
import mlrun.runtimes.utils
|
|
53
53
|
import mlrun.serving
|
|
54
|
+
import mlrun.utils
|
|
54
55
|
import mlrun.utils.regex
|
|
55
56
|
from mlrun.alerts.alert import AlertConfig
|
|
56
57
|
from mlrun.common.schemas.alert import AlertTemplate
|
|
@@ -713,7 +714,8 @@ def _project_instance_from_struct(struct, name, allow_cross_project):
|
|
|
713
714
|
name_from_struct = struct.get("metadata", {}).get("name", "")
|
|
714
715
|
if name and name_from_struct and name_from_struct != name:
|
|
715
716
|
error_message = (
|
|
716
|
-
f"
|
|
717
|
+
f"Project name mismatch, {name_from_struct} != {name}, project is loaded from {name_from_struct} "
|
|
718
|
+
f"project yaml. To prevent/allow this, you can take one of the following actions:\n"
|
|
717
719
|
"1. Set the `allow_cross_project=True` when loading the project.\n"
|
|
718
720
|
f"2. Delete the existing project yaml, or ensure its name is equal to {name}.\n"
|
|
719
721
|
"3. Use different project context dir."
|
|
@@ -721,14 +723,14 @@ def _project_instance_from_struct(struct, name, allow_cross_project):
|
|
|
721
723
|
|
|
722
724
|
if allow_cross_project is None:
|
|
723
725
|
# TODO: Remove this warning in version 1.9.0 and also fix cli to support allow_cross_project
|
|
724
|
-
|
|
725
|
-
"Project name is different than specified on
|
|
726
|
-
"
|
|
727
|
-
description=error_message,
|
|
726
|
+
warnings.warn(
|
|
727
|
+
f"Project {name=} is different than specified on the context's project yaml. "
|
|
728
|
+
"This behavior is deprecated and will not be supported in version 1.9.0."
|
|
728
729
|
)
|
|
730
|
+
logger.warn(error_message)
|
|
729
731
|
elif allow_cross_project:
|
|
730
|
-
logger.
|
|
731
|
-
"Project name is different than specified on
|
|
732
|
+
logger.debug(
|
|
733
|
+
"Project name is different than specified on the context's project yaml. Overriding.",
|
|
732
734
|
existing_name=name_from_struct,
|
|
733
735
|
overriding_name=name,
|
|
734
736
|
)
|
|
@@ -993,15 +995,29 @@ class ProjectSpec(ModelObj):
|
|
|
993
995
|
|
|
994
996
|
artifacts_dict = {}
|
|
995
997
|
for artifact in artifacts:
|
|
996
|
-
|
|
998
|
+
invalid_object_type = not isinstance(artifact, dict) and not hasattr(
|
|
999
|
+
artifact, "to_dict"
|
|
1000
|
+
)
|
|
1001
|
+
is_artifact_model = not isinstance(artifact, dict) and hasattr(
|
|
1002
|
+
artifact, "to_dict"
|
|
1003
|
+
)
|
|
1004
|
+
|
|
1005
|
+
if invalid_object_type:
|
|
997
1006
|
raise ValueError("artifacts must be a dict or class")
|
|
998
|
-
|
|
999
|
-
key = artifact.get("metadata", {}).get("key", "")
|
|
1000
|
-
if not key:
|
|
1001
|
-
raise ValueError('artifacts "metadata.key" must be specified')
|
|
1002
|
-
else:
|
|
1007
|
+
elif is_artifact_model:
|
|
1003
1008
|
key = artifact.key
|
|
1004
1009
|
artifact = artifact.to_dict()
|
|
1010
|
+
else: # artifact is a dict
|
|
1011
|
+
# imported/legacy artifacts don't have metadata,spec,status fields
|
|
1012
|
+
key_field = (
|
|
1013
|
+
"key"
|
|
1014
|
+
if _is_imported_artifact(artifact)
|
|
1015
|
+
or mlrun.utils.is_legacy_artifact(artifact)
|
|
1016
|
+
else "metadata.key"
|
|
1017
|
+
)
|
|
1018
|
+
key = mlrun.utils.get_in(artifact, key_field, "")
|
|
1019
|
+
if not key:
|
|
1020
|
+
raise ValueError(f'artifacts "{key_field}" must be specified')
|
|
1005
1021
|
|
|
1006
1022
|
artifacts_dict[key] = artifact
|
|
1007
1023
|
|
|
@@ -1931,6 +1947,7 @@ class MlrunProject(ModelObj):
|
|
|
1931
1947
|
call `fn.deploy()` where `fn` is the object returned by this method.
|
|
1932
1948
|
|
|
1933
1949
|
examples::
|
|
1950
|
+
|
|
1934
1951
|
project.set_model_monitoring_function(
|
|
1935
1952
|
name="myApp", application_class="MyApp", image="mlrun/mlrun"
|
|
1936
1953
|
)
|
|
@@ -1996,6 +2013,7 @@ class MlrunProject(ModelObj):
|
|
|
1996
2013
|
Create a monitoring function object without setting it to the project
|
|
1997
2014
|
|
|
1998
2015
|
examples::
|
|
2016
|
+
|
|
1999
2017
|
project.create_model_monitoring_function(
|
|
2000
2018
|
application_class_name="MyApp", image="mlrun/mlrun", name="myApp"
|
|
2001
2019
|
)
|
|
@@ -2017,6 +2035,7 @@ class MlrunProject(ModelObj):
|
|
|
2017
2035
|
:param application_kwargs: Additional keyword arguments to be passed to the
|
|
2018
2036
|
monitoring application's constructor.
|
|
2019
2037
|
"""
|
|
2038
|
+
|
|
2020
2039
|
_, function_object, _ = self._instantiate_model_monitoring_function(
|
|
2021
2040
|
func,
|
|
2022
2041
|
application_class,
|
|
@@ -2113,6 +2132,8 @@ class MlrunProject(ModelObj):
|
|
|
2113
2132
|
*,
|
|
2114
2133
|
deploy_histogram_data_drift_app: bool = True,
|
|
2115
2134
|
wait_for_deployment: bool = False,
|
|
2135
|
+
rebuild_images: bool = False,
|
|
2136
|
+
fetch_credentials_from_sys_config: bool = False,
|
|
2116
2137
|
) -> None:
|
|
2117
2138
|
"""
|
|
2118
2139
|
Deploy model monitoring application controller, writer and stream functions.
|
|
@@ -2122,16 +2143,18 @@ class MlrunProject(ModelObj):
|
|
|
2122
2143
|
The stream function goal is to monitor the log of the data stream. It is triggered when a new log entry
|
|
2123
2144
|
is detected. It processes the new events into statistics that are then written to statistics databases.
|
|
2124
2145
|
|
|
2125
|
-
:param default_controller_image:
|
|
2126
|
-
:param base_period:
|
|
2127
|
-
|
|
2128
|
-
:param image:
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
:param deploy_histogram_data_drift_app:
|
|
2132
|
-
:param wait_for_deployment:
|
|
2133
|
-
|
|
2134
|
-
|
|
2146
|
+
:param default_controller_image: Deprecated.
|
|
2147
|
+
:param base_period: The time period in minutes in which the model monitoring controller
|
|
2148
|
+
function is triggered. By default, the base period is 10 minutes.
|
|
2149
|
+
:param image: The image of the model monitoring controller, writer, monitoring
|
|
2150
|
+
stream & histogram data drift functions, which are real time nuclio
|
|
2151
|
+
functions. By default, the image is mlrun/mlrun.
|
|
2152
|
+
:param deploy_histogram_data_drift_app: If true, deploy the default histogram-based data drift application.
|
|
2153
|
+
:param wait_for_deployment: If true, return only after the deployment is done on the backend.
|
|
2154
|
+
Otherwise, deploy the model monitoring infrastructure on the
|
|
2155
|
+
background, including the histogram data drift app if selected.
|
|
2156
|
+
:param rebuild_images: If true, force rebuild of model monitoring infrastructure images.
|
|
2157
|
+
:param fetch_credentials_from_sys_config: If true, fetch the credentials from the system configuration.
|
|
2135
2158
|
"""
|
|
2136
2159
|
if default_controller_image != "mlrun/mlrun":
|
|
2137
2160
|
# TODO: Remove this in 1.9.0
|
|
@@ -2147,6 +2170,8 @@ class MlrunProject(ModelObj):
|
|
|
2147
2170
|
image=image,
|
|
2148
2171
|
base_period=base_period,
|
|
2149
2172
|
deploy_histogram_data_drift_app=deploy_histogram_data_drift_app,
|
|
2173
|
+
rebuild_images=rebuild_images,
|
|
2174
|
+
fetch_credentials_from_sys_config=fetch_credentials_from_sys_config,
|
|
2150
2175
|
)
|
|
2151
2176
|
|
|
2152
2177
|
if wait_for_deployment:
|
|
@@ -2212,23 +2237,67 @@ class MlrunProject(ModelObj):
|
|
|
2212
2237
|
)
|
|
2213
2238
|
|
|
2214
2239
|
def disable_model_monitoring(
|
|
2215
|
-
self,
|
|
2240
|
+
self,
|
|
2241
|
+
*,
|
|
2242
|
+
delete_resources: bool = True,
|
|
2243
|
+
delete_stream_function: bool = False,
|
|
2244
|
+
delete_histogram_data_drift_app: bool = True,
|
|
2245
|
+
delete_user_applications: bool = False,
|
|
2246
|
+
user_application_list: list[str] = None,
|
|
2216
2247
|
) -> None:
|
|
2217
2248
|
"""
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
:param
|
|
2249
|
+
Disable model monitoring application controller, writer, stream, histogram data drift application
|
|
2250
|
+
and the user's applications functions, according to the given params.
|
|
2251
|
+
|
|
2252
|
+
:param delete_resources: If True, it would delete the model monitoring controller & writer
|
|
2253
|
+
functions. Default True
|
|
2254
|
+
:param delete_stream_function: If True, it would delete model monitoring stream function,
|
|
2255
|
+
need to use wisely because if you're deleting this function
|
|
2256
|
+
this can cause data loss in case you will want to
|
|
2257
|
+
enable the model monitoring capability to the project.
|
|
2258
|
+
Default False.
|
|
2259
|
+
:param delete_histogram_data_drift_app: If True, it would delete the default histogram-based data drift
|
|
2260
|
+
application. Default False.
|
|
2261
|
+
:param delete_user_applications: If True, it would delete the user's model monitoring
|
|
2262
|
+
application according to user_application_list, Default False.
|
|
2263
|
+
:param user_application_list: List of the user's model monitoring application to disable.
|
|
2264
|
+
Default all the applications.
|
|
2265
|
+
Note: you have to set delete_user_applications to True
|
|
2266
|
+
in order to delete the desired application.
|
|
2222
2267
|
"""
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
if delete_histogram_data_drift_app:
|
|
2227
|
-
db.delete_function(
|
|
2228
|
-
project=self.name,
|
|
2229
|
-
name=mm_constants.HistogramDataDriftApplicationConstants.NAME,
|
|
2268
|
+
if not delete_user_applications and user_application_list:
|
|
2269
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
2270
|
+
"user_application_list can be specified only if delete_user_applications is set to True"
|
|
2230
2271
|
)
|
|
2231
2272
|
|
|
2273
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
2274
|
+
succeed = db.disable_model_monitoring(
|
|
2275
|
+
project=self.name,
|
|
2276
|
+
delete_resources=delete_resources,
|
|
2277
|
+
delete_stream_function=delete_stream_function,
|
|
2278
|
+
delete_histogram_data_drift_app=delete_histogram_data_drift_app,
|
|
2279
|
+
delete_user_applications=delete_user_applications,
|
|
2280
|
+
user_application_list=user_application_list,
|
|
2281
|
+
)
|
|
2282
|
+
if succeed and delete_resources:
|
|
2283
|
+
if delete_resources:
|
|
2284
|
+
logger.info("Model Monitoring disabled", project=self.name)
|
|
2285
|
+
if delete_user_applications:
|
|
2286
|
+
logger.info(
|
|
2287
|
+
"All the desired monitoring application were deleted",
|
|
2288
|
+
project=self.name,
|
|
2289
|
+
)
|
|
2290
|
+
else:
|
|
2291
|
+
if delete_resources:
|
|
2292
|
+
logger.info(
|
|
2293
|
+
"Model Monitoring was not disabled properly", project=self.name
|
|
2294
|
+
)
|
|
2295
|
+
if delete_user_applications:
|
|
2296
|
+
logger.info(
|
|
2297
|
+
"Some of the desired monitoring application were not deleted",
|
|
2298
|
+
project=self.name,
|
|
2299
|
+
)
|
|
2300
|
+
|
|
2232
2301
|
def set_function(
|
|
2233
2302
|
self,
|
|
2234
2303
|
func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,
|
|
@@ -2277,7 +2346,8 @@ class MlrunProject(ModelObj):
|
|
|
2277
2346
|
Default: job
|
|
2278
2347
|
:param image: Docker image to be used, can also be specified in the function object/yaml
|
|
2279
2348
|
:param handler: Default function handler to invoke (can only be set with .py/.ipynb files)
|
|
2280
|
-
:param with_repo: Add (clone) the current repo to the build source
|
|
2349
|
+
:param with_repo: Add (clone) the current repo to the build source - use when the function code is in
|
|
2350
|
+
the project repo (project.spec.source).
|
|
2281
2351
|
:param tag: Function version tag to set (none for current or 'latest')
|
|
2282
2352
|
Specifying a tag as a parameter will update the project's tagged function
|
|
2283
2353
|
(myfunc:v1) and the untagged function (myfunc)
|
|
@@ -2423,22 +2493,39 @@ class MlrunProject(ModelObj):
|
|
|
2423
2493
|
"""
|
|
2424
2494
|
self.spec.remove_function(name)
|
|
2425
2495
|
|
|
2426
|
-
def remove_model_monitoring_function(self, name):
|
|
2427
|
-
"""
|
|
2496
|
+
def remove_model_monitoring_function(self, name: Union[str, list[str]]):
|
|
2497
|
+
"""delete the specified model-monitoring-app function/s
|
|
2428
2498
|
|
|
2429
|
-
:param name: name of the model-monitoring-
|
|
2499
|
+
:param name: name of the model-monitoring-function/s (under the project)
|
|
2430
2500
|
"""
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2501
|
+
# TODO: Remove this in 1.9.0
|
|
2502
|
+
warnings.warn(
|
|
2503
|
+
"'remove_model_monitoring_function' is deprecated and will be removed in 1.9.0. "
|
|
2504
|
+
"Please use `delete_model_monitoring_function` instead.",
|
|
2505
|
+
FutureWarning,
|
|
2506
|
+
)
|
|
2507
|
+
self.delete_model_monitoring_function(name)
|
|
2508
|
+
|
|
2509
|
+
def delete_model_monitoring_function(self, name: Union[str, list[str]]):
|
|
2510
|
+
"""delete the specified model-monitoring-app function/s
|
|
2511
|
+
|
|
2512
|
+
:param name: name of the model-monitoring-function/s (under the project)
|
|
2513
|
+
"""
|
|
2514
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
2515
|
+
succeed = db.delete_model_monitoring_function(
|
|
2516
|
+
project=self.name,
|
|
2517
|
+
functions=name if isinstance(name, list) else [name],
|
|
2518
|
+
)
|
|
2519
|
+
if succeed:
|
|
2520
|
+
logger.info(
|
|
2521
|
+
"All the desired monitoring functions were deleted",
|
|
2522
|
+
project=self.name,
|
|
2523
|
+
functions=name,
|
|
2524
|
+
)
|
|
2439
2525
|
else:
|
|
2440
|
-
|
|
2441
|
-
|
|
2526
|
+
logger.info(
|
|
2527
|
+
"Some of the desired monitoring functions were not deleted",
|
|
2528
|
+
project=self.name,
|
|
2442
2529
|
)
|
|
2443
2530
|
|
|
2444
2531
|
def get_function(
|
|
@@ -2546,10 +2633,10 @@ class MlrunProject(ModelObj):
|
|
|
2546
2633
|
def create_remote(self, url, name="origin", branch=None):
|
|
2547
2634
|
"""Create remote for the project git
|
|
2548
2635
|
|
|
2549
|
-
|
|
2550
|
-
|
|
2636
|
+
This method creates a new remote repository associated with the project's Git repository.
|
|
2637
|
+
If a remote with the specified name already exists, it will not be overwritten.
|
|
2551
2638
|
|
|
2552
|
-
|
|
2639
|
+
If you wish to update the URL of an existing remote, use the `set_remote` method instead.
|
|
2553
2640
|
|
|
2554
2641
|
:param url: remote git url
|
|
2555
2642
|
:param name: name for the remote (default is 'origin')
|
|
@@ -2904,9 +2991,10 @@ class MlrunProject(ModelObj):
|
|
|
2904
2991
|
For using the pre-defined workflow's schedule, set `schedule=True`
|
|
2905
2992
|
:param timeout: Timeout in seconds to wait for pipeline completion (watch will be activated)
|
|
2906
2993
|
:param source: Source to use instead of the actual `project.spec.source` (used when engine is remote).
|
|
2907
|
-
Can be
|
|
2908
|
-
|
|
2909
|
-
|
|
2994
|
+
Can be one of:
|
|
2995
|
+
|
|
2996
|
+
* Remote URL which is loaded dynamically to the workflow runner.
|
|
2997
|
+
* A path to the project's context on the workflow runner's image.
|
|
2910
2998
|
Path can be absolute or relative to `project.spec.build.source_code_target_dir` if defined
|
|
2911
2999
|
(enriched when building a project image with source, see `MlrunProject.build_image`).
|
|
2912
3000
|
For other engines the source is used to validate that the code is up-to-date.
|
|
@@ -3118,48 +3206,44 @@ class MlrunProject(ModelObj):
|
|
|
3118
3206
|
stream_path: Optional[str] = None,
|
|
3119
3207
|
tsdb_connection: Optional[str] = None,
|
|
3120
3208
|
):
|
|
3121
|
-
"""Set the credentials that will be used by the project's model monitoring
|
|
3122
|
-
infrastructure functions.
|
|
3123
|
-
|
|
3124
|
-
:param access_key: Model Monitoring access key for managing user permissions
|
|
3125
|
-
:param endpoint_store_connection: Endpoint store connection string
|
|
3126
|
-
:param stream_path: Path to the model monitoring stream
|
|
3127
|
-
:param tsdb_connection: Connection string to the time series database
|
|
3128
3209
|
"""
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3210
|
+
Set the credentials that will be used by the project's model monitoring
|
|
3211
|
+
infrastructure functions. Important to note that you have to set the credentials before deploying any
|
|
3212
|
+
model monitoring or serving function.
|
|
3213
|
+
|
|
3214
|
+
:param access_key: Model Monitoring access key for managing user permissions.
|
|
3215
|
+
:param endpoint_store_connection: Endpoint store connection string. By default, None.
|
|
3216
|
+
Options:
|
|
3217
|
+
1. None, will be set from the system configuration.
|
|
3218
|
+
2. v3io - for v3io endpoint store,
|
|
3219
|
+
pass `v3io` and the system will generate the exact path.
|
|
3220
|
+
3. MySQL/SQLite - for SQL endpoint store, please provide full
|
|
3221
|
+
connection string, for example
|
|
3222
|
+
mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>
|
|
3223
|
+
:param stream_path: Path to the model monitoring stream. By default, None.
|
|
3224
|
+
Options:
|
|
3225
|
+
1. None, will be set from the system configuration.
|
|
3226
|
+
2. v3io - for v3io stream,
|
|
3227
|
+
pass `v3io` and the system will generate the exact path.
|
|
3228
|
+
3. Kafka - for Kafka stream, please provide full connection string without
|
|
3229
|
+
custom topic, for example kafka://<some_kafka_broker>:<port>.
|
|
3230
|
+
:param tsdb_connection: Connection string to the time series database. By default, None.
|
|
3231
|
+
Options:
|
|
3232
|
+
1. None, will be set from the system configuration.
|
|
3233
|
+
2. v3io - for v3io stream,
|
|
3234
|
+
pass `v3io` and the system will generate the exact path.
|
|
3235
|
+
3. TDEngine - for TDEngine tsdb, please provide full websocket connection URL,
|
|
3236
|
+
for example taosws://<username>:<password>@<host>:<port>.
|
|
3237
|
+
"""
|
|
3238
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
3239
|
+
db.set_model_monitoring_credentials(
|
|
3240
|
+
project=self.name,
|
|
3241
|
+
credentials={
|
|
3242
|
+
"access_key": access_key,
|
|
3243
|
+
"endpoint_store_connection": endpoint_store_connection,
|
|
3244
|
+
"stream_path": stream_path,
|
|
3245
|
+
"tsdb_connection": tsdb_connection,
|
|
3246
|
+
},
|
|
3163
3247
|
)
|
|
3164
3248
|
|
|
3165
3249
|
def run_function(
|
|
@@ -3185,6 +3269,7 @@ class MlrunProject(ModelObj):
|
|
|
3185
3269
|
notifications: list[mlrun.model.Notification] = None,
|
|
3186
3270
|
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
3187
3271
|
builder_env: Optional[dict] = None,
|
|
3272
|
+
reset_on_run: bool = None,
|
|
3188
3273
|
) -> typing.Union[mlrun.model.RunObject, PipelineNodeWrapper]:
|
|
3189
3274
|
"""Run a local or remote task as part of a local/kubeflow pipeline
|
|
3190
3275
|
|
|
@@ -3241,6 +3326,10 @@ class MlrunProject(ModelObj):
|
|
|
3241
3326
|
artifact type can be given there. The artifact key must appear in the dictionary as
|
|
3242
3327
|
"key": "the_key".
|
|
3243
3328
|
:param builder_env: env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
|
|
3329
|
+
:param reset_on_run: When True, function python modules would reload prior to code execution.
|
|
3330
|
+
This ensures latest code changes are executed. This argument must be used in
|
|
3331
|
+
conjunction with the local=True argument.
|
|
3332
|
+
|
|
3244
3333
|
:return: MLRun RunObject or PipelineNodeWrapper
|
|
3245
3334
|
"""
|
|
3246
3335
|
return run_function(
|
|
@@ -3266,6 +3355,7 @@ class MlrunProject(ModelObj):
|
|
|
3266
3355
|
notifications=notifications,
|
|
3267
3356
|
returns=returns,
|
|
3268
3357
|
builder_env=builder_env,
|
|
3358
|
+
reset_on_run=reset_on_run,
|
|
3269
3359
|
)
|
|
3270
3360
|
|
|
3271
3361
|
def build_function(
|
|
@@ -3570,6 +3660,7 @@ class MlrunProject(ModelObj):
|
|
|
3570
3660
|
kind: str = None,
|
|
3571
3661
|
category: typing.Union[str, mlrun.common.schemas.ArtifactCategories] = None,
|
|
3572
3662
|
tree: str = None,
|
|
3663
|
+
limit: int = None,
|
|
3573
3664
|
) -> mlrun.lists.ArtifactList:
|
|
3574
3665
|
"""List artifacts filtered by various parameters.
|
|
3575
3666
|
|
|
@@ -3599,6 +3690,7 @@ class MlrunProject(ModelObj):
|
|
|
3599
3690
|
:param kind: Return artifacts of the requested kind.
|
|
3600
3691
|
:param category: Return artifacts of the requested category.
|
|
3601
3692
|
:param tree: Return artifacts of the requested tree.
|
|
3693
|
+
:param limit: Maximum number of artifacts to return.
|
|
3602
3694
|
"""
|
|
3603
3695
|
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
3604
3696
|
return db.list_artifacts(
|
|
@@ -3613,6 +3705,7 @@ class MlrunProject(ModelObj):
|
|
|
3613
3705
|
kind=kind,
|
|
3614
3706
|
category=category,
|
|
3615
3707
|
tree=tree,
|
|
3708
|
+
limit=limit,
|
|
3616
3709
|
)
|
|
3617
3710
|
|
|
3618
3711
|
def list_models(
|
|
@@ -3874,15 +3967,15 @@ class MlrunProject(ModelObj):
|
|
|
3874
3967
|
on MLRun and Nuclio sides, such as the 'host' attribute.
|
|
3875
3968
|
Nuclio docs here: https://docs.nuclio.io/en/latest/reference/api-gateway/http.html
|
|
3876
3969
|
|
|
3877
|
-
:param api_gateway:
|
|
3878
|
-
|
|
3879
|
-
:param wait_for_readiness: (Optional) A boolean indicating whether to wait for the API Gateway to become
|
|
3880
|
-
after creation or update (default is True)
|
|
3881
|
-
:param max_wait_time:
|
|
3970
|
+
:param api_gateway: An instance of :py:class:`~mlrun.runtimes.nuclio.APIGateway` representing the
|
|
3971
|
+
configuration of the API Gateway to be created or updated.
|
|
3972
|
+
:param wait_for_readiness: (Optional) A boolean indicating whether to wait for the API Gateway to become
|
|
3973
|
+
ready after creation or update (default is True).
|
|
3974
|
+
:param max_wait_time: (Optional) Maximum time to wait for API Gateway readiness in seconds (default is 90s)
|
|
3882
3975
|
|
|
3883
3976
|
|
|
3884
|
-
|
|
3885
|
-
|
|
3977
|
+
:returns: An instance of :py:class:`~mlrun.runtimes.nuclio.APIGateway` with all fields populated based on the
|
|
3978
|
+
information retrieved from the Nuclio API
|
|
3886
3979
|
"""
|
|
3887
3980
|
|
|
3888
3981
|
api_gateway_json = mlrun.db.get_run_db().store_api_gateway(
|
|
@@ -3904,8 +3997,8 @@ class MlrunProject(ModelObj):
|
|
|
3904
3997
|
"""
|
|
3905
3998
|
Retrieves a list of Nuclio API gateways associated with the project.
|
|
3906
3999
|
|
|
3907
|
-
|
|
3908
|
-
|
|
4000
|
+
:returns: List of :py:class:`~mlrun.runtimes.nuclio.api_gateway.APIGateway` objects representing
|
|
4001
|
+
the Nuclio API gateways associated with the project.
|
|
3909
4002
|
"""
|
|
3910
4003
|
gateways_list = mlrun.db.get_run_db().list_api_gateways(self.name)
|
|
3911
4004
|
return [
|
|
@@ -3946,6 +4039,7 @@ class MlrunProject(ModelObj):
|
|
|
3946
4039
|
) -> AlertConfig:
|
|
3947
4040
|
"""
|
|
3948
4041
|
Create/modify an alert.
|
|
4042
|
+
|
|
3949
4043
|
:param alert_data: The data of the alert.
|
|
3950
4044
|
:param alert_name: The name of the alert.
|
|
3951
4045
|
:return: the created/modified alert.
|
|
@@ -3958,6 +4052,7 @@ class MlrunProject(ModelObj):
|
|
|
3958
4052
|
def get_alert_config(self, alert_name: str) -> AlertConfig:
|
|
3959
4053
|
"""
|
|
3960
4054
|
Retrieve an alert.
|
|
4055
|
+
|
|
3961
4056
|
:param alert_name: The name of the alert to retrieve.
|
|
3962
4057
|
:return: The alert object.
|
|
3963
4058
|
"""
|
|
@@ -3967,6 +4062,7 @@ class MlrunProject(ModelObj):
|
|
|
3967
4062
|
def list_alerts_configs(self) -> list[AlertConfig]:
|
|
3968
4063
|
"""
|
|
3969
4064
|
Retrieve list of alerts of a project.
|
|
4065
|
+
|
|
3970
4066
|
:return: All the alerts objects of the project.
|
|
3971
4067
|
"""
|
|
3972
4068
|
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
@@ -3977,6 +4073,7 @@ class MlrunProject(ModelObj):
|
|
|
3977
4073
|
):
|
|
3978
4074
|
"""
|
|
3979
4075
|
Delete an alert.
|
|
4076
|
+
|
|
3980
4077
|
:param alert_data: The data of the alert.
|
|
3981
4078
|
:param alert_name: The name of the alert to delete.
|
|
3982
4079
|
"""
|
|
@@ -3996,6 +4093,7 @@ class MlrunProject(ModelObj):
|
|
|
3996
4093
|
):
|
|
3997
4094
|
"""
|
|
3998
4095
|
Reset an alert.
|
|
4096
|
+
|
|
3999
4097
|
:param alert_data: The data of the alert.
|
|
4000
4098
|
:param alert_name: The name of the alert to reset.
|
|
4001
4099
|
"""
|
|
@@ -4013,6 +4111,7 @@ class MlrunProject(ModelObj):
|
|
|
4013
4111
|
def get_alert_template(self, template_name: str) -> AlertTemplate:
|
|
4014
4112
|
"""
|
|
4015
4113
|
Retrieve a specific alert template.
|
|
4114
|
+
|
|
4016
4115
|
:param template_name: The name of the template to retrieve.
|
|
4017
4116
|
:return: The template object.
|
|
4018
4117
|
"""
|
|
@@ -4022,6 +4121,7 @@ class MlrunProject(ModelObj):
|
|
|
4022
4121
|
def list_alert_templates(self) -> list[AlertTemplate]:
|
|
4023
4122
|
"""
|
|
4024
4123
|
Retrieve list of all alert templates.
|
|
4124
|
+
|
|
4025
4125
|
:return: All the alert template objects in the database.
|
|
4026
4126
|
"""
|
|
4027
4127
|
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
@@ -4129,12 +4229,21 @@ class MlrunProject(ModelObj):
|
|
|
4129
4229
|
else:
|
|
4130
4230
|
producer_dict = artifact.spec.producer
|
|
4131
4231
|
|
|
4232
|
+
producer_tag = producer_dict.get("tag", None)
|
|
4233
|
+
producer_project = producer_dict.get("project", None)
|
|
4234
|
+
if not producer_tag or not producer_project:
|
|
4235
|
+
# try resolving the producer tag from the uri
|
|
4236
|
+
producer_uri = artifact.spec.producer.get("uri", "")
|
|
4237
|
+
producer_project, producer_tag, _ = ArtifactProducer.parse_uri(
|
|
4238
|
+
producer_uri
|
|
4239
|
+
)
|
|
4240
|
+
|
|
4132
4241
|
if producer_dict.get("kind", "") == "run":
|
|
4133
4242
|
return ArtifactProducer(
|
|
4134
4243
|
name=producer_dict.get("name", ""),
|
|
4135
4244
|
kind=producer_dict.get("kind", ""),
|
|
4136
|
-
project=
|
|
4137
|
-
tag=
|
|
4245
|
+
project=producer_project,
|
|
4246
|
+
tag=producer_tag,
|
|
4138
4247
|
), True
|
|
4139
4248
|
|
|
4140
4249
|
# do not retain the artifact's producer, replace it with the project as the producer
|
mlrun/render.py
CHANGED
|
@@ -283,9 +283,14 @@ function copyToClipboard(fld) {
|
|
|
283
283
|
}
|
|
284
284
|
function expandPanel(el) {
|
|
285
285
|
const panelName = "#" + el.getAttribute('paneName');
|
|
286
|
-
console.log(el.title);
|
|
287
286
|
|
|
288
|
-
|
|
287
|
+
// Get the base URL of the current notebook
|
|
288
|
+
var baseUrl = window.location.origin;
|
|
289
|
+
|
|
290
|
+
// Construct the full URL
|
|
291
|
+
var fullUrl = new URL(el.title, baseUrl).href;
|
|
292
|
+
|
|
293
|
+
document.querySelector(panelName + "-title").innerHTML = fullUrl
|
|
289
294
|
iframe = document.querySelector(panelName + "-body");
|
|
290
295
|
|
|
291
296
|
const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}
|
|
@@ -299,7 +304,7 @@ function expandPanel(el) {
|
|
|
299
304
|
}
|
|
300
305
|
|
|
301
306
|
function reqListener () {
|
|
302
|
-
if (
|
|
307
|
+
if (fullUrl.endsWith(".csv")) {
|
|
303
308
|
iframe.setAttribute("srcdoc", tblcss + csvToHtmlTable(this.responseText));
|
|
304
309
|
} else {
|
|
305
310
|
iframe.setAttribute("srcdoc", this.responseText);
|
|
@@ -309,11 +314,11 @@ function expandPanel(el) {
|
|
|
309
314
|
|
|
310
315
|
const oReq = new XMLHttpRequest();
|
|
311
316
|
oReq.addEventListener("load", reqListener);
|
|
312
|
-
oReq.open("GET",
|
|
317
|
+
oReq.open("GET", fullUrl);
|
|
313
318
|
oReq.send();
|
|
314
319
|
|
|
315
320
|
|
|
316
|
-
//iframe.src =
|
|
321
|
+
//iframe.src = fullUrl;
|
|
317
322
|
const resultPane = document.querySelector(panelName + "-pane");
|
|
318
323
|
if (resultPane.classList.contains("hidden")) {
|
|
319
324
|
resultPane.classList.remove("hidden");
|