mlrun 1.7.0rc5__py3-none-any.whl → 1.7.0rc6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/artifacts/base.py +2 -1
- mlrun/artifacts/plots.py +9 -5
- mlrun/config.py +8 -2
- mlrun/datastore/sources.py +5 -4
- mlrun/db/factory.py +1 -1
- mlrun/launcher/__init__.py +1 -1
- mlrun/launcher/base.py +1 -1
- mlrun/launcher/client.py +1 -1
- mlrun/launcher/factory.py +1 -1
- mlrun/launcher/local.py +1 -1
- mlrun/launcher/remote.py +1 -1
- mlrun/model_monitoring/api.py +6 -12
- mlrun/model_monitoring/application.py +21 -21
- mlrun/model_monitoring/applications/histogram_data_drift.py +130 -40
- mlrun/model_monitoring/batch.py +1 -42
- mlrun/model_monitoring/controller.py +1 -8
- mlrun/model_monitoring/features_drift_table.py +34 -22
- mlrun/model_monitoring/helpers.py +45 -4
- mlrun/model_monitoring/stream_processing.py +2 -0
- mlrun/projects/project.py +170 -16
- mlrun/run.py +69 -73
- mlrun/runtimes/__init__.py +35 -0
- mlrun/runtimes/base.py +1 -1
- mlrun/runtimes/nuclio/application/__init__.py +15 -0
- mlrun/runtimes/nuclio/application/application.py +283 -0
- mlrun/runtimes/nuclio/application/reverse_proxy.go +87 -0
- mlrun/runtimes/nuclio/function.py +50 -1
- mlrun/runtimes/pod.py +1 -1
- mlrun/serving/states.py +7 -19
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc5.dist-info → mlrun-1.7.0rc6.dist-info}/METADATA +1 -1
- {mlrun-1.7.0rc5.dist-info → mlrun-1.7.0rc6.dist-info}/RECORD +36 -33
- {mlrun-1.7.0rc5.dist-info → mlrun-1.7.0rc6.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc5.dist-info → mlrun-1.7.0rc6.dist-info}/WHEEL +0 -0
- {mlrun-1.7.0rc5.dist-info → mlrun-1.7.0rc6.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc5.dist-info → mlrun-1.7.0rc6.dist-info}/top_level.txt +0 -0
|
@@ -21,9 +21,34 @@ import plotly.graph_objects as go
|
|
|
21
21
|
from plotly.subplots import make_subplots
|
|
22
22
|
|
|
23
23
|
import mlrun.common.schemas.model_monitoring
|
|
24
|
+
from mlrun.artifacts import PlotlyArtifact
|
|
24
25
|
|
|
25
26
|
# A type for representing a drift result, a tuple of the status and the drift mean:
|
|
26
|
-
DriftResultType = tuple[
|
|
27
|
+
DriftResultType = tuple[
|
|
28
|
+
mlrun.common.schemas.model_monitoring.constants.ResultStatusApp, float
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class _PlotlyTableArtifact(PlotlyArtifact):
|
|
33
|
+
"""A custom class for plotly table artifacts"""
|
|
34
|
+
|
|
35
|
+
@staticmethod
|
|
36
|
+
def _disable_table_dragging(figure_html: str) -> str:
|
|
37
|
+
"""
|
|
38
|
+
Disable the table columns dragging by adding the following
|
|
39
|
+
JavaScript code
|
|
40
|
+
"""
|
|
41
|
+
start, end = figure_html.rsplit(";", 1)
|
|
42
|
+
middle = (
|
|
43
|
+
';for (const element of document.getElementsByClassName("table")) '
|
|
44
|
+
'{element.style.pointerEvents = "none";}'
|
|
45
|
+
)
|
|
46
|
+
figure_html = start + middle + end
|
|
47
|
+
return figure_html
|
|
48
|
+
|
|
49
|
+
def get_body(self) -> str:
|
|
50
|
+
"""Get the adjusted HTML representation of the figure"""
|
|
51
|
+
return self._disable_table_dragging(super().get_body())
|
|
27
52
|
|
|
28
53
|
|
|
29
54
|
class FeaturesDriftTablePlot:
|
|
@@ -62,9 +87,9 @@ class FeaturesDriftTablePlot:
|
|
|
62
87
|
|
|
63
88
|
# Status configurations:
|
|
64
89
|
_STATUS_COLORS = {
|
|
65
|
-
mlrun.common.schemas.model_monitoring.
|
|
66
|
-
mlrun.common.schemas.model_monitoring.
|
|
67
|
-
mlrun.common.schemas.model_monitoring.
|
|
90
|
+
mlrun.common.schemas.model_monitoring.constants.ResultStatusApp.no_detection: "rgb(0,176,80)", # Green
|
|
91
|
+
mlrun.common.schemas.model_monitoring.constants.ResultStatusApp.potential_detection: "rgb(255,192,0)", # Orange
|
|
92
|
+
mlrun.common.schemas.model_monitoring.constants.ResultStatusApp.detected: "rgb(208,0,106)", # Magenta
|
|
68
93
|
}
|
|
69
94
|
|
|
70
95
|
# Font configurations:
|
|
@@ -97,7 +122,7 @@ class FeaturesDriftTablePlot:
|
|
|
97
122
|
inputs_statistics: dict,
|
|
98
123
|
metrics: dict[str, Union[dict, float]],
|
|
99
124
|
drift_results: dict[str, DriftResultType],
|
|
100
|
-
) ->
|
|
125
|
+
) -> _PlotlyTableArtifact:
|
|
101
126
|
"""
|
|
102
127
|
Produce the html code of the table plot with the given information and the stored configurations in the class.
|
|
103
128
|
|
|
@@ -106,9 +131,8 @@ class FeaturesDriftTablePlot:
|
|
|
106
131
|
:param metrics: The drift detection metrics calculated on the sample set and inputs.
|
|
107
132
|
:param drift_results: The drift results per feature according to the rules of the monitor.
|
|
108
133
|
|
|
109
|
-
:return: The
|
|
134
|
+
:return: The drift table as a plotly artifact.
|
|
110
135
|
"""
|
|
111
|
-
# Plot the drift table:
|
|
112
136
|
figure = self._plot(
|
|
113
137
|
features=list(inputs_statistics.keys()),
|
|
114
138
|
sample_set_statistics=sample_set_statistics,
|
|
@@ -116,19 +140,7 @@ class FeaturesDriftTablePlot:
|
|
|
116
140
|
metrics=metrics,
|
|
117
141
|
drift_results=drift_results,
|
|
118
142
|
)
|
|
119
|
-
|
|
120
|
-
# Get its HTML representation:
|
|
121
|
-
figure_html = figure.to_html()
|
|
122
|
-
|
|
123
|
-
# Turn off the table columns dragging by injecting the following JavaScript code:
|
|
124
|
-
start, end = figure_html.rsplit(";", 1)
|
|
125
|
-
middle = (
|
|
126
|
-
';for (const element of document.getElementsByClassName("table")) '
|
|
127
|
-
'{element.style.pointerEvents = "none";}'
|
|
128
|
-
)
|
|
129
|
-
figure_html = start + middle + end
|
|
130
|
-
|
|
131
|
-
return figure_html
|
|
143
|
+
return _PlotlyTableArtifact(figure=figure, key="drift_table_plot")
|
|
132
144
|
|
|
133
145
|
def _read_columns_names(self, statistics_dictionary: dict, drift_metrics: dict):
|
|
134
146
|
"""
|
|
@@ -366,10 +378,10 @@ class FeaturesDriftTablePlot:
|
|
|
366
378
|
bins = np.array(bins)
|
|
367
379
|
if bins[0] == -sys.float_info.max:
|
|
368
380
|
bins[0] = bins[1] - (bins[2] - bins[1])
|
|
369
|
-
hovertext[0] = f"(
|
|
381
|
+
hovertext[0] = f"(-inf, {bins[1]})"
|
|
370
382
|
if bins[-1] == sys.float_info.max:
|
|
371
383
|
bins[-1] = bins[-2] + (bins[-2] - bins[-3])
|
|
372
|
-
hovertext[-1] = f"({bins[-2]},
|
|
384
|
+
hovertext[-1] = f"({bins[-2]}, inf)"
|
|
373
385
|
# Center the bins (leave the first one):
|
|
374
386
|
bins = 0.5 * (bins[:-1] + bins[1:])
|
|
375
387
|
# Plot the histogram as a line with filled background below it:
|
|
@@ -15,6 +15,9 @@
|
|
|
15
15
|
import datetime
|
|
16
16
|
import typing
|
|
17
17
|
|
|
18
|
+
import numpy as np
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
18
21
|
import mlrun
|
|
19
22
|
import mlrun.common.model_monitoring.helpers
|
|
20
23
|
import mlrun.common.schemas
|
|
@@ -36,10 +39,6 @@ class _BatchDict(typing.TypedDict):
|
|
|
36
39
|
days: int
|
|
37
40
|
|
|
38
41
|
|
|
39
|
-
class _MLRunNoRunsFoundError(Exception):
|
|
40
|
-
pass
|
|
41
|
-
|
|
42
|
-
|
|
43
42
|
def get_stream_path(
|
|
44
43
|
project: str = None,
|
|
45
44
|
function_name: str = mm_constants.MonitoringFunctionNames.STREAM,
|
|
@@ -212,3 +211,45 @@ def update_model_endpoint_last_request(
|
|
|
212
211
|
endpoint_id=model_endpoint.metadata.uid,
|
|
213
212
|
attributes={EventFieldType.LAST_REQUEST: bumped_last_request},
|
|
214
213
|
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def calculate_inputs_statistics(
|
|
217
|
+
sample_set_statistics: dict, inputs: pd.DataFrame
|
|
218
|
+
) -> dict:
|
|
219
|
+
"""
|
|
220
|
+
Calculate the inputs data statistics for drift monitoring purpose.
|
|
221
|
+
|
|
222
|
+
:param sample_set_statistics: The sample set (stored end point's dataset to reference) statistics. The bins of the
|
|
223
|
+
histograms of each feature will be used to recalculate the histograms of the inputs.
|
|
224
|
+
:param inputs: The inputs to calculate their statistics and later on - the drift with respect to the
|
|
225
|
+
sample set.
|
|
226
|
+
|
|
227
|
+
:returns: The calculated statistics of the inputs data.
|
|
228
|
+
"""
|
|
229
|
+
|
|
230
|
+
# Use `DFDataInfer` to calculate the statistics over the inputs:
|
|
231
|
+
inputs_statistics = mlrun.data_types.infer.DFDataInfer.get_stats(
|
|
232
|
+
df=inputs,
|
|
233
|
+
options=mlrun.data_types.infer.InferOptions.Histogram,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
# Recalculate the histograms over the bins that are set in the sample-set of the end point:
|
|
237
|
+
for feature in inputs_statistics.keys():
|
|
238
|
+
if feature in sample_set_statistics:
|
|
239
|
+
counts, bins = np.histogram(
|
|
240
|
+
inputs[feature].to_numpy(),
|
|
241
|
+
bins=sample_set_statistics[feature]["hist"][1],
|
|
242
|
+
)
|
|
243
|
+
inputs_statistics[feature]["hist"] = [
|
|
244
|
+
counts.tolist(),
|
|
245
|
+
bins.tolist(),
|
|
246
|
+
]
|
|
247
|
+
elif "hist" in inputs_statistics[feature]:
|
|
248
|
+
# Comply with the other common features' histogram length
|
|
249
|
+
mlrun.common.model_monitoring.helpers.pad_hist(
|
|
250
|
+
mlrun.common.model_monitoring.helpers.Histogram(
|
|
251
|
+
inputs_statistics[feature]["hist"]
|
|
252
|
+
)
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
return inputs_statistics
|
mlrun/projects/project.py
CHANGED
|
@@ -1375,14 +1375,7 @@ class MlrunProject(ModelObj):
|
|
|
1375
1375
|
artifact_path = mlrun.utils.helpers.template_artifact_path(
|
|
1376
1376
|
self.spec.artifact_path or mlrun.mlconf.artifact_path, self.metadata.name
|
|
1377
1377
|
)
|
|
1378
|
-
|
|
1379
|
-
# we need to maintain the different trees that generated them
|
|
1380
|
-
producer = ArtifactProducer(
|
|
1381
|
-
"project",
|
|
1382
|
-
self.metadata.name,
|
|
1383
|
-
self.metadata.name,
|
|
1384
|
-
tag=self._get_hexsha() or str(uuid.uuid4()),
|
|
1385
|
-
)
|
|
1378
|
+
project_tag = self._get_project_tag()
|
|
1386
1379
|
for artifact_dict in self.spec.artifacts:
|
|
1387
1380
|
if _is_imported_artifact(artifact_dict):
|
|
1388
1381
|
import_from = artifact_dict["import_from"]
|
|
@@ -1402,6 +1395,15 @@ class MlrunProject(ModelObj):
|
|
|
1402
1395
|
artifact.src_path = path.join(
|
|
1403
1396
|
self.spec.get_code_path(), artifact.src_path
|
|
1404
1397
|
)
|
|
1398
|
+
producer = self._resolve_artifact_producer(artifact, project_tag)
|
|
1399
|
+
# log the artifact only if it doesn't already exist
|
|
1400
|
+
if (
|
|
1401
|
+
producer.name != self.metadata.name
|
|
1402
|
+
and self._resolve_existing_artifact(
|
|
1403
|
+
artifact,
|
|
1404
|
+
)
|
|
1405
|
+
):
|
|
1406
|
+
continue
|
|
1405
1407
|
artifact_manager.log_artifact(
|
|
1406
1408
|
producer, artifact, artifact_path=artifact_path
|
|
1407
1409
|
)
|
|
@@ -1498,12 +1500,20 @@ class MlrunProject(ModelObj):
|
|
|
1498
1500
|
artifact_path = mlrun.utils.helpers.template_artifact_path(
|
|
1499
1501
|
artifact_path, self.metadata.name
|
|
1500
1502
|
)
|
|
1501
|
-
producer =
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
self.
|
|
1505
|
-
|
|
1506
|
-
|
|
1503
|
+
producer = self._resolve_artifact_producer(item)
|
|
1504
|
+
if producer.name != self.metadata.name:
|
|
1505
|
+
# the artifact producer is retained, log it only if it doesn't already exist
|
|
1506
|
+
if existing_artifact := self._resolve_existing_artifact(
|
|
1507
|
+
item,
|
|
1508
|
+
tag,
|
|
1509
|
+
):
|
|
1510
|
+
artifact_key = item if isinstance(item, str) else item.key
|
|
1511
|
+
logger.info(
|
|
1512
|
+
"Artifact already exists, skipping logging",
|
|
1513
|
+
key=artifact_key,
|
|
1514
|
+
tag=tag,
|
|
1515
|
+
)
|
|
1516
|
+
return existing_artifact
|
|
1507
1517
|
item = am.log_artifact(
|
|
1508
1518
|
producer,
|
|
1509
1519
|
item,
|
|
@@ -2403,13 +2413,47 @@ class MlrunProject(ModelObj):
|
|
|
2403
2413
|
clone_zip(url, self.spec.context, self._secrets)
|
|
2404
2414
|
|
|
2405
2415
|
def create_remote(self, url, name="origin", branch=None):
|
|
2406
|
-
"""
|
|
2416
|
+
"""Create remote for the project git
|
|
2417
|
+
|
|
2418
|
+
This method creates a new remote repository associated with the project's Git repository.
|
|
2419
|
+
If a remote with the specified name already exists, it will not be overwritten.
|
|
2420
|
+
|
|
2421
|
+
If you wish to update the URL of an existing remote, use the `set_remote` method instead.
|
|
2407
2422
|
|
|
2408
2423
|
:param url: remote git url
|
|
2409
2424
|
:param name: name for the remote (default is 'origin')
|
|
2410
2425
|
:param branch: Git branch to use as source
|
|
2411
2426
|
"""
|
|
2427
|
+
self.set_remote(url, name=name, branch=branch, overwrite=False)
|
|
2428
|
+
|
|
2429
|
+
def set_remote(self, url, name="origin", branch=None, overwrite=True):
|
|
2430
|
+
"""Create or update a remote for the project git repository.
|
|
2431
|
+
|
|
2432
|
+
This method allows you to manage remote repositories associated with the project.
|
|
2433
|
+
It checks if a remote with the specified name already exists.
|
|
2434
|
+
|
|
2435
|
+
If a remote with the same name does not exist, it will be created.
|
|
2436
|
+
If a remote with the same name already exists,
|
|
2437
|
+
the behavior depends on the value of the 'overwrite' flag.
|
|
2438
|
+
|
|
2439
|
+
:param url: remote git url
|
|
2440
|
+
:param name: name for the remote (default is 'origin')
|
|
2441
|
+
:param branch: Git branch to use as source
|
|
2442
|
+
:param overwrite: if True (default), updates the existing remote with the given URL if it already exists.
|
|
2443
|
+
if False, raises an error when attempting to create a remote with a name that already exists.
|
|
2444
|
+
:raises MLRunConflictError: If a remote with the same name already exists and overwrite
|
|
2445
|
+
is set to False.
|
|
2446
|
+
"""
|
|
2412
2447
|
self._ensure_git_repo()
|
|
2448
|
+
if self._remote_exists(name):
|
|
2449
|
+
if overwrite:
|
|
2450
|
+
self.spec.repo.delete_remote(name)
|
|
2451
|
+
else:
|
|
2452
|
+
raise mlrun.errors.MLRunConflictError(
|
|
2453
|
+
f"Remote '{name}' already exists in the project, "
|
|
2454
|
+
f"each remote in the project must have a unique name."
|
|
2455
|
+
"Use 'set_remote' with 'override=True' inorder to update the remote, or choose a different name."
|
|
2456
|
+
)
|
|
2413
2457
|
self.spec.repo.create_remote(name, url=url)
|
|
2414
2458
|
url = url.replace("https://", "git://")
|
|
2415
2459
|
if not branch:
|
|
@@ -2422,6 +2466,22 @@ class MlrunProject(ModelObj):
|
|
|
2422
2466
|
self.spec._source = self.spec.source or url
|
|
2423
2467
|
self.spec.origin_url = self.spec.origin_url or url
|
|
2424
2468
|
|
|
2469
|
+
def remove_remote(self, name):
|
|
2470
|
+
"""Remove a remote from the project's Git repository.
|
|
2471
|
+
|
|
2472
|
+
This method removes the remote repository associated with the specified name from the project's Git repository.
|
|
2473
|
+
|
|
2474
|
+
:param name: Name of the remote to remove.
|
|
2475
|
+
"""
|
|
2476
|
+
if self._remote_exists(name):
|
|
2477
|
+
self.spec.repo.delete_remote(name)
|
|
2478
|
+
else:
|
|
2479
|
+
logger.warning(f"The remote '{name}' does not exist. Nothing to remove.")
|
|
2480
|
+
|
|
2481
|
+
def _remote_exists(self, name):
|
|
2482
|
+
"""Check if a remote with the given name already exists"""
|
|
2483
|
+
return any(remote.name == name for remote in self.spec.repo.remotes)
|
|
2484
|
+
|
|
2425
2485
|
def _ensure_git_repo(self):
|
|
2426
2486
|
if self.spec.repo:
|
|
2427
2487
|
return
|
|
@@ -3333,7 +3393,12 @@ class MlrunProject(ModelObj):
|
|
|
3333
3393
|
artifact = db.read_artifact(
|
|
3334
3394
|
key, tag, iter=iter, project=self.metadata.name, tree=tree
|
|
3335
3395
|
)
|
|
3336
|
-
|
|
3396
|
+
|
|
3397
|
+
# in tests, if an artifact is not found, the db returns None
|
|
3398
|
+
# in real usage, the db should raise an exception
|
|
3399
|
+
if artifact:
|
|
3400
|
+
return dict_to_artifact(artifact)
|
|
3401
|
+
return None
|
|
3337
3402
|
|
|
3338
3403
|
def list_artifacts(
|
|
3339
3404
|
self,
|
|
@@ -3761,6 +3826,83 @@ class MlrunProject(ModelObj):
|
|
|
3761
3826
|
f"<project.spec.get_code_path()>/<{param_name}>)."
|
|
3762
3827
|
)
|
|
3763
3828
|
|
|
3829
|
+
def _resolve_artifact_producer(
|
|
3830
|
+
self,
|
|
3831
|
+
artifact: typing.Union[str, Artifact],
|
|
3832
|
+
project_producer_tag: str = None,
|
|
3833
|
+
) -> typing.Optional[ArtifactProducer]:
|
|
3834
|
+
"""
|
|
3835
|
+
Resolve the artifact producer of the given artifact.
|
|
3836
|
+
If the artifact's producer is a run, the artifact is registered with the original producer.
|
|
3837
|
+
Otherwise, the artifact is registered with the current project as the producer.
|
|
3838
|
+
|
|
3839
|
+
:param artifact: The artifact to resolve its producer.
|
|
3840
|
+
:param project_producer_tag: The tag to use for the project as the producer. If not provided, a tag will be
|
|
3841
|
+
generated for the project.
|
|
3842
|
+
:return: A tuple of the resolved producer and the resolved artifact.
|
|
3843
|
+
"""
|
|
3844
|
+
|
|
3845
|
+
if not isinstance(artifact, str) and artifact.producer:
|
|
3846
|
+
# if the artifact was imported from a yaml file, the producer can be a dict
|
|
3847
|
+
if isinstance(artifact.spec.producer, ArtifactProducer):
|
|
3848
|
+
producer_dict = artifact.spec.producer.get_meta()
|
|
3849
|
+
else:
|
|
3850
|
+
producer_dict = artifact.spec.producer
|
|
3851
|
+
|
|
3852
|
+
if producer_dict.get("kind", "") == "run":
|
|
3853
|
+
return ArtifactProducer(
|
|
3854
|
+
name=producer_dict.get("name", ""),
|
|
3855
|
+
kind=producer_dict.get("kind", ""),
|
|
3856
|
+
project=producer_dict.get("project", ""),
|
|
3857
|
+
tag=producer_dict.get("tag", ""),
|
|
3858
|
+
)
|
|
3859
|
+
|
|
3860
|
+
# do not retain the artifact's producer, replace it with the project as the producer
|
|
3861
|
+
project_producer_tag = project_producer_tag or self._get_project_tag()
|
|
3862
|
+
return ArtifactProducer(
|
|
3863
|
+
kind="project",
|
|
3864
|
+
name=self.metadata.name,
|
|
3865
|
+
project=self.metadata.name,
|
|
3866
|
+
tag=project_producer_tag,
|
|
3867
|
+
)
|
|
3868
|
+
|
|
3869
|
+
def _resolve_existing_artifact(
|
|
3870
|
+
self,
|
|
3871
|
+
item: typing.Union[str, Artifact],
|
|
3872
|
+
tag: str = None,
|
|
3873
|
+
) -> typing.Optional[Artifact]:
|
|
3874
|
+
"""
|
|
3875
|
+
Check if there is and existing artifact with the given item and tag.
|
|
3876
|
+
If there is, return the existing artifact. Otherwise, return None.
|
|
3877
|
+
|
|
3878
|
+
:param item: The item (or key) to check if there is an existing artifact for.
|
|
3879
|
+
:param tag: The tag to check if there is an existing artifact for.
|
|
3880
|
+
:return: The existing artifact if there is one, otherwise None.
|
|
3881
|
+
"""
|
|
3882
|
+
try:
|
|
3883
|
+
if isinstance(item, str):
|
|
3884
|
+
existing_artifact = self.get_artifact(key=item, tag=tag)
|
|
3885
|
+
else:
|
|
3886
|
+
existing_artifact = self.get_artifact(
|
|
3887
|
+
key=item.key,
|
|
3888
|
+
tag=item.tag,
|
|
3889
|
+
iter=item.iter,
|
|
3890
|
+
tree=item.tree,
|
|
3891
|
+
)
|
|
3892
|
+
if existing_artifact is not None:
|
|
3893
|
+
return existing_artifact.from_dict(existing_artifact)
|
|
3894
|
+
except mlrun.errors.MLRunNotFoundError:
|
|
3895
|
+
logger.debug(
|
|
3896
|
+
"No existing artifact was found",
|
|
3897
|
+
key=item if isinstance(item, str) else item.key,
|
|
3898
|
+
tag=tag if isinstance(item, str) else item.tag,
|
|
3899
|
+
tree=None if isinstance(item, str) else item.tree,
|
|
3900
|
+
)
|
|
3901
|
+
return None
|
|
3902
|
+
|
|
3903
|
+
def _get_project_tag(self):
|
|
3904
|
+
return self._get_hexsha() or str(uuid.uuid4())
|
|
3905
|
+
|
|
3764
3906
|
|
|
3765
3907
|
def _set_as_current_default_project(project: MlrunProject):
|
|
3766
3908
|
mlrun.mlconf.default_project = project.metadata.name
|
|
@@ -3846,6 +3988,18 @@ def _init_function_from_dict(
|
|
|
3846
3988
|
tag=tag,
|
|
3847
3989
|
)
|
|
3848
3990
|
|
|
3991
|
+
elif image and kind in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
|
|
3992
|
+
func = new_function(
|
|
3993
|
+
name,
|
|
3994
|
+
command=relative_url,
|
|
3995
|
+
image=image,
|
|
3996
|
+
kind=kind,
|
|
3997
|
+
handler=handler,
|
|
3998
|
+
tag=tag,
|
|
3999
|
+
)
|
|
4000
|
+
if kind != mlrun.runtimes.RuntimeKinds.application:
|
|
4001
|
+
logger.info("Function code not specified, setting entry point to image")
|
|
4002
|
+
func.from_image(image)
|
|
3849
4003
|
else:
|
|
3850
4004
|
raise ValueError(f"Unsupported function url:handler {url}:{handler} or no spec")
|
|
3851
4005
|
|
mlrun/run.py
CHANGED
|
@@ -34,7 +34,6 @@ import mlrun.common.schemas
|
|
|
34
34
|
import mlrun.errors
|
|
35
35
|
import mlrun.utils.helpers
|
|
36
36
|
from mlrun.kfpops import format_summary_from_kfp_run, show_kfp_run
|
|
37
|
-
from mlrun.runtimes.nuclio.serving import serving_subkind
|
|
38
37
|
|
|
39
38
|
from .common.helpers import parse_versioned_object_uri
|
|
40
39
|
from .config import config as mlconf
|
|
@@ -58,6 +57,7 @@ from .runtimes import (
|
|
|
58
57
|
)
|
|
59
58
|
from .runtimes.databricks_job.databricks_runtime import DatabricksRuntime
|
|
60
59
|
from .runtimes.funcdoc import update_function_entry_points
|
|
60
|
+
from .runtimes.nuclio.application import ApplicationRuntime
|
|
61
61
|
from .runtimes.utils import add_code_metadata, global_context
|
|
62
62
|
from .utils import (
|
|
63
63
|
extend_hub_uri_if_needed,
|
|
@@ -425,19 +425,19 @@ def import_function_to_dict(url, secrets=None):
|
|
|
425
425
|
|
|
426
426
|
|
|
427
427
|
def new_function(
|
|
428
|
-
name: str = "",
|
|
429
|
-
project: str = "",
|
|
430
|
-
tag: str = "",
|
|
431
|
-
kind: str = "",
|
|
432
|
-
command: str = "",
|
|
433
|
-
image: str = "",
|
|
434
|
-
args: list = None,
|
|
435
|
-
runtime=None,
|
|
436
|
-
mode=None,
|
|
437
|
-
handler: str = None,
|
|
438
|
-
source: str = None,
|
|
428
|
+
name: Optional[str] = "",
|
|
429
|
+
project: Optional[str] = "",
|
|
430
|
+
tag: Optional[str] = "",
|
|
431
|
+
kind: Optional[str] = "",
|
|
432
|
+
command: Optional[str] = "",
|
|
433
|
+
image: Optional[str] = "",
|
|
434
|
+
args: Optional[list] = None,
|
|
435
|
+
runtime: Optional[Union[mlrun.runtimes.BaseRuntime, dict]] = None,
|
|
436
|
+
mode: Optional[str] = None,
|
|
437
|
+
handler: Optional[str] = None,
|
|
438
|
+
source: Optional[str] = None,
|
|
439
439
|
requirements: Union[str, list[str]] = None,
|
|
440
|
-
kfp=None,
|
|
440
|
+
kfp: Optional[bool] = None,
|
|
441
441
|
requirements_file: str = "",
|
|
442
442
|
):
|
|
443
443
|
"""Create a new ML function from base properties
|
|
@@ -535,9 +535,9 @@ def new_function(
|
|
|
535
535
|
if source:
|
|
536
536
|
runner.spec.build.source = source
|
|
537
537
|
if handler:
|
|
538
|
-
if kind
|
|
538
|
+
if kind in [RuntimeKinds.serving, RuntimeKinds.application]:
|
|
539
539
|
raise MLRunInvalidArgumentError(
|
|
540
|
-
"
|
|
540
|
+
f"Handler is not supported for {kind} runtime"
|
|
541
541
|
)
|
|
542
542
|
elif kind in RuntimeKinds.nuclio_runtimes():
|
|
543
543
|
runner.spec.function_handler = handler
|
|
@@ -575,22 +575,22 @@ def _process_runtime(command, runtime, kind):
|
|
|
575
575
|
|
|
576
576
|
|
|
577
577
|
def code_to_function(
|
|
578
|
-
name: str = "",
|
|
579
|
-
project: str = "",
|
|
580
|
-
tag: str = "",
|
|
581
|
-
filename: str = "",
|
|
582
|
-
handler: str = "",
|
|
583
|
-
kind: str = "",
|
|
584
|
-
image: str = None,
|
|
585
|
-
code_output: str = "",
|
|
578
|
+
name: Optional[str] = "",
|
|
579
|
+
project: Optional[str] = "",
|
|
580
|
+
tag: Optional[str] = "",
|
|
581
|
+
filename: Optional[str] = "",
|
|
582
|
+
handler: Optional[str] = "",
|
|
583
|
+
kind: Optional[str] = "",
|
|
584
|
+
image: Optional[str] = None,
|
|
585
|
+
code_output: Optional[str] = "",
|
|
586
586
|
embed_code: bool = True,
|
|
587
|
-
description: str = "",
|
|
588
|
-
requirements: Union[str, list[str]] = None,
|
|
589
|
-
categories: list[str] = None,
|
|
590
|
-
labels: dict[str, str] = None,
|
|
591
|
-
with_doc: bool = True,
|
|
592
|
-
ignored_tags=None,
|
|
593
|
-
requirements_file: str = "",
|
|
587
|
+
description: Optional[str] = "",
|
|
588
|
+
requirements: Optional[Union[str, list[str]]] = None,
|
|
589
|
+
categories: Optional[list[str]] = None,
|
|
590
|
+
labels: Optional[dict[str, str]] = None,
|
|
591
|
+
with_doc: Optional[bool] = True,
|
|
592
|
+
ignored_tags: Optional[str] = None,
|
|
593
|
+
requirements_file: Optional[str] = "",
|
|
594
594
|
) -> Union[
|
|
595
595
|
MpiRuntimeV1Alpha1,
|
|
596
596
|
MpiRuntimeV1,
|
|
@@ -602,6 +602,7 @@ def code_to_function(
|
|
|
602
602
|
Spark3Runtime,
|
|
603
603
|
RemoteSparkRuntime,
|
|
604
604
|
DatabricksRuntime,
|
|
605
|
+
ApplicationRuntime,
|
|
605
606
|
]:
|
|
606
607
|
"""Convenience function to insert code and configure an mlrun runtime.
|
|
607
608
|
|
|
@@ -718,35 +719,34 @@ def code_to_function(
|
|
|
718
719
|
fn.metadata.categories = categories
|
|
719
720
|
fn.metadata.labels = labels or fn.metadata.labels
|
|
720
721
|
|
|
721
|
-
def resolve_nuclio_subkind(kind):
|
|
722
|
-
is_nuclio = kind.startswith("nuclio")
|
|
723
|
-
subkind = kind[kind.find(":") + 1 :] if is_nuclio and ":" in kind else None
|
|
724
|
-
if kind == RuntimeKinds.serving:
|
|
725
|
-
is_nuclio = True
|
|
726
|
-
subkind = serving_subkind
|
|
727
|
-
return is_nuclio, subkind
|
|
728
|
-
|
|
729
722
|
if (
|
|
730
723
|
not embed_code
|
|
731
724
|
and not code_output
|
|
732
725
|
and (not filename or filename.endswith(".ipynb"))
|
|
733
726
|
):
|
|
734
727
|
raise ValueError(
|
|
735
|
-
"
|
|
728
|
+
"A valid code file must be specified "
|
|
736
729
|
"when not using the embed_code option"
|
|
737
730
|
)
|
|
738
731
|
|
|
739
732
|
if kind == RuntimeKinds.databricks and not embed_code:
|
|
740
|
-
raise ValueError("
|
|
733
|
+
raise ValueError("Databricks tasks only support embed_code=True")
|
|
741
734
|
|
|
742
|
-
|
|
735
|
+
if kind == RuntimeKinds.application:
|
|
736
|
+
if handler:
|
|
737
|
+
raise MLRunInvalidArgumentError(
|
|
738
|
+
"Handler is not supported for application runtime"
|
|
739
|
+
)
|
|
740
|
+
filename, handler = ApplicationRuntime.get_filename_and_handler()
|
|
741
|
+
|
|
742
|
+
is_nuclio, sub_kind = RuntimeKinds.resolve_nuclio_sub_kind(kind)
|
|
743
743
|
code_origin = add_name(add_code_metadata(filename), name)
|
|
744
744
|
|
|
745
745
|
name, spec, code = nuclio.build_file(
|
|
746
746
|
filename,
|
|
747
747
|
name=name,
|
|
748
748
|
handler=handler or "handler",
|
|
749
|
-
kind=
|
|
749
|
+
kind=sub_kind,
|
|
750
750
|
ignored_tags=ignored_tags,
|
|
751
751
|
)
|
|
752
752
|
spec["spec"]["env"].append(
|
|
@@ -759,14 +759,14 @@ def code_to_function(
|
|
|
759
759
|
if not kind and spec_kind not in ["", "Function"]:
|
|
760
760
|
kind = spec_kind.lower()
|
|
761
761
|
|
|
762
|
-
# if its a nuclio
|
|
763
|
-
is_nuclio,
|
|
762
|
+
# if its a nuclio sub kind, redo nb parsing
|
|
763
|
+
is_nuclio, sub_kind = RuntimeKinds.resolve_nuclio_sub_kind(kind)
|
|
764
764
|
if is_nuclio:
|
|
765
765
|
name, spec, code = nuclio.build_file(
|
|
766
766
|
filename,
|
|
767
767
|
name=name,
|
|
768
768
|
handler=handler or "handler",
|
|
769
|
-
kind=
|
|
769
|
+
kind=sub_kind,
|
|
770
770
|
ignored_tags=ignored_tags,
|
|
771
771
|
)
|
|
772
772
|
|
|
@@ -780,33 +780,29 @@ def code_to_function(
|
|
|
780
780
|
raise ValueError("code_output option is only used with notebooks")
|
|
781
781
|
|
|
782
782
|
if is_nuclio:
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
else
|
|
786
|
-
|
|
787
|
-
r.spec.function_kind = subkind
|
|
788
|
-
# default_handler is only used in :mlrun subkind, determine the handler to invoke in function.run()
|
|
789
|
-
r.spec.default_handler = handler if subkind == "mlrun" else ""
|
|
790
|
-
r.spec.function_handler = (
|
|
783
|
+
runtime = RuntimeKinds.resolve_nuclio_runtime(kind, sub_kind)
|
|
784
|
+
# default_handler is only used in :mlrun sub kind, determine the handler to invoke in function.run()
|
|
785
|
+
runtime.spec.default_handler = handler if sub_kind == "mlrun" else ""
|
|
786
|
+
runtime.spec.function_handler = (
|
|
791
787
|
handler if handler and ":" in handler else get_in(spec, "spec.handler")
|
|
792
788
|
)
|
|
793
789
|
if not embed_code:
|
|
794
|
-
|
|
790
|
+
runtime.spec.source = filename
|
|
795
791
|
nuclio_runtime = get_in(spec, "spec.runtime")
|
|
796
792
|
if nuclio_runtime and not nuclio_runtime.startswith("py"):
|
|
797
|
-
|
|
793
|
+
runtime.spec.nuclio_runtime = nuclio_runtime
|
|
798
794
|
if not name:
|
|
799
|
-
raise ValueError("
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
update_common(
|
|
804
|
-
return
|
|
795
|
+
raise ValueError("Missing required parameter: name")
|
|
796
|
+
runtime.metadata.name = name
|
|
797
|
+
runtime.spec.build.code_origin = code_origin
|
|
798
|
+
runtime.spec.build.origin_filename = filename or (name + ".ipynb")
|
|
799
|
+
update_common(runtime, spec)
|
|
800
|
+
return runtime
|
|
805
801
|
|
|
806
802
|
if kind is None or kind in ["", "Function"]:
|
|
807
803
|
raise ValueError("please specify the function kind")
|
|
808
804
|
elif kind in RuntimeKinds.all():
|
|
809
|
-
|
|
805
|
+
runtime = get_runtime_class(kind)()
|
|
810
806
|
else:
|
|
811
807
|
raise ValueError(f"unsupported runtime ({kind})")
|
|
812
808
|
|
|
@@ -815,10 +811,10 @@ def code_to_function(
|
|
|
815
811
|
if not name:
|
|
816
812
|
raise ValueError("name must be specified")
|
|
817
813
|
h = get_in(spec, "spec.handler", "").split(":")
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
build =
|
|
814
|
+
runtime.handler = h[0] if len(h) <= 1 else h[1]
|
|
815
|
+
runtime.metadata = get_in(spec, "spec.metadata")
|
|
816
|
+
runtime.metadata.name = name
|
|
817
|
+
build = runtime.spec.build
|
|
822
818
|
build.code_origin = code_origin
|
|
823
819
|
build.origin_filename = filename or (name + ".ipynb")
|
|
824
820
|
build.extra = get_in(spec, "spec.build.extra")
|
|
@@ -826,18 +822,18 @@ def code_to_function(
|
|
|
826
822
|
build.builder_env = get_in(spec, "spec.build.builder_env")
|
|
827
823
|
if not embed_code:
|
|
828
824
|
if code_output:
|
|
829
|
-
|
|
825
|
+
runtime.spec.command = code_output
|
|
830
826
|
else:
|
|
831
|
-
|
|
827
|
+
runtime.spec.command = filename
|
|
832
828
|
|
|
833
829
|
build.image = get_in(spec, "spec.build.image")
|
|
834
|
-
update_common(
|
|
835
|
-
|
|
830
|
+
update_common(runtime, spec)
|
|
831
|
+
runtime.prepare_image_for_deploy()
|
|
836
832
|
|
|
837
833
|
if with_doc:
|
|
838
|
-
update_function_entry_points(
|
|
839
|
-
|
|
840
|
-
return
|
|
834
|
+
update_function_entry_points(runtime, code)
|
|
835
|
+
runtime.spec.default_handler = handler
|
|
836
|
+
return runtime
|
|
841
837
|
|
|
842
838
|
|
|
843
839
|
def _run_pipeline(
|