mlrun 1.7.0rc9__py3-none-any.whl → 1.7.0rc11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +1 -0
- mlrun/artifacts/model.py +29 -25
- mlrun/common/schemas/__init__.py +1 -0
- mlrun/common/schemas/alert.py +122 -0
- mlrun/common/schemas/auth.py +4 -0
- mlrun/common/schemas/client_spec.py +1 -0
- mlrun/common/schemas/model_monitoring/constants.py +3 -1
- mlrun/config.py +8 -4
- mlrun/datastore/base.py +6 -5
- mlrun/datastore/sources.py +9 -4
- mlrun/datastore/targets.py +11 -3
- mlrun/datastore/v3io.py +27 -50
- mlrun/db/base.py +44 -2
- mlrun/db/httpdb.py +192 -20
- mlrun/db/nopdb.py +36 -1
- mlrun/execution.py +21 -14
- mlrun/feature_store/api.py +6 -3
- mlrun/feature_store/feature_set.py +39 -23
- mlrun/feature_store/feature_vector.py +2 -1
- mlrun/feature_store/steps.py +30 -19
- mlrun/features.py +4 -13
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +2 -2
- mlrun/frameworks/lgbm/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
- mlrun/frameworks/lgbm/model_handler.py +1 -1
- mlrun/frameworks/pytorch/__init__.py +2 -2
- mlrun/frameworks/sklearn/__init__.py +1 -1
- mlrun/frameworks/tf_keras/__init__.py +1 -1
- mlrun/frameworks/xgboost/__init__.py +1 -1
- mlrun/model.py +2 -2
- mlrun/model_monitoring/application.py +11 -2
- mlrun/model_monitoring/applications/histogram_data_drift.py +3 -3
- mlrun/model_monitoring/controller.py +2 -3
- mlrun/model_monitoring/stream_processing.py +0 -1
- mlrun/model_monitoring/writer.py +32 -0
- mlrun/package/packagers_manager.py +1 -0
- mlrun/platforms/__init__.py +1 -1
- mlrun/platforms/other.py +1 -1
- mlrun/projects/operations.py +11 -4
- mlrun/projects/project.py +148 -52
- mlrun/run.py +72 -40
- mlrun/runtimes/mpijob/abstract.py +8 -8
- mlrun/runtimes/nuclio/function.py +9 -5
- mlrun/runtimes/nuclio/serving.py +9 -8
- mlrun/runtimes/pod.py +3 -3
- mlrun/secrets.py +6 -2
- mlrun/serving/routers.py +3 -1
- mlrun/serving/states.py +12 -33
- mlrun/serving/v2_serving.py +4 -4
- mlrun/utils/helpers.py +1 -1
- mlrun/utils/notifications/notification/base.py +12 -0
- mlrun/utils/notifications/notification/console.py +2 -0
- mlrun/utils/notifications/notification/git.py +3 -1
- mlrun/utils/notifications/notification/ipython.py +2 -0
- mlrun/utils/notifications/notification/slack.py +41 -13
- mlrun/utils/notifications/notification/webhook.py +11 -1
- mlrun/utils/retryer.py +2 -2
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/METADATA +1 -1
- {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/RECORD +64 -64
- mlrun/datastore/helpers.py +0 -18
- {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/WHEEL +0 -0
- {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py
CHANGED
|
@@ -47,6 +47,7 @@ import mlrun.runtimes.pod
|
|
|
47
47
|
import mlrun.runtimes.utils
|
|
48
48
|
import mlrun.serving
|
|
49
49
|
import mlrun.utils.regex
|
|
50
|
+
from mlrun.common.schemas import AlertConfig
|
|
50
51
|
from mlrun.datastore.datastore_profile import DatastoreProfile, DatastoreProfile2Json
|
|
51
52
|
from mlrun.runtimes.nuclio.function import RemoteRuntime
|
|
52
53
|
|
|
@@ -140,11 +141,15 @@ def new_project(
|
|
|
140
141
|
example::
|
|
141
142
|
|
|
142
143
|
# create a project with local and hub functions, a workflow, and an artifact
|
|
143
|
-
project = mlrun.new_project(
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
project.
|
|
147
|
-
|
|
144
|
+
project = mlrun.new_project(
|
|
145
|
+
"myproj", "./", init_git=True, description="my new project"
|
|
146
|
+
)
|
|
147
|
+
project.set_function(
|
|
148
|
+
"prep_data.py", "prep-data", image="mlrun/mlrun", handler="prep_data"
|
|
149
|
+
)
|
|
150
|
+
project.set_function("hub://auto-trainer", "train")
|
|
151
|
+
project.set_artifact("data", Artifact(target_path=data_url))
|
|
152
|
+
project.set_workflow("main", "./myflow.py")
|
|
148
153
|
project.save()
|
|
149
154
|
|
|
150
155
|
# run the "main" workflow (watch=True to wait for run completion)
|
|
@@ -154,19 +159,25 @@ def new_project(
|
|
|
154
159
|
|
|
155
160
|
# create a new project from a zip template (can also use yaml/git templates)
|
|
156
161
|
# initialize a local git, and register the git remote path
|
|
157
|
-
project = mlrun.new_project(
|
|
158
|
-
|
|
159
|
-
|
|
162
|
+
project = mlrun.new_project(
|
|
163
|
+
"myproj",
|
|
164
|
+
"./",
|
|
165
|
+
init_git=True,
|
|
166
|
+
remote="git://github.com/mlrun/project-demo.git",
|
|
167
|
+
from_template="http://mysite/proj.zip",
|
|
168
|
+
)
|
|
160
169
|
project.run("main", watch=True)
|
|
161
170
|
|
|
162
171
|
|
|
163
172
|
example using project_setup.py to init the project objects::
|
|
164
173
|
|
|
165
174
|
def setup(project):
|
|
166
|
-
project.set_function(
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
project.
|
|
175
|
+
project.set_function(
|
|
176
|
+
"prep_data.py", "prep-data", image="mlrun/mlrun", handler="prep_data"
|
|
177
|
+
)
|
|
178
|
+
project.set_function("hub://auto-trainer", "train")
|
|
179
|
+
project.set_artifact("data", Artifact(target_path=data_url))
|
|
180
|
+
project.set_workflow("main", "./myflow.py")
|
|
170
181
|
return project
|
|
171
182
|
|
|
172
183
|
|
|
@@ -298,7 +309,7 @@ def load_project(
|
|
|
298
309
|
# When using git as the url source the context directory must be an empty or
|
|
299
310
|
# non-existent folder as the git repo will be cloned there
|
|
300
311
|
project = load_project("./demo_proj", "git://github.com/mlrun/project-demo.git")
|
|
301
|
-
project.run("main", arguments={
|
|
312
|
+
project.run("main", arguments={"data": data_url})
|
|
302
313
|
|
|
303
314
|
|
|
304
315
|
project_setup.py example::
|
|
@@ -437,9 +448,11 @@ def get_or_create_project(
|
|
|
437
448
|
Usage example::
|
|
438
449
|
|
|
439
450
|
# load project from the DB (if exist) or the source repo
|
|
440
|
-
project = get_or_create_project(
|
|
451
|
+
project = get_or_create_project(
|
|
452
|
+
"myproj", "./", "git://github.com/mlrun/demo-xgb-project.git"
|
|
453
|
+
)
|
|
441
454
|
project.pull("development") # pull the latest code from git
|
|
442
|
-
project.run("main", arguments={
|
|
455
|
+
project.run("main", arguments={"data": data_url}) # run the workflow "main"
|
|
443
456
|
|
|
444
457
|
|
|
445
458
|
project_setup.py example::
|
|
@@ -1344,13 +1357,15 @@ class MlrunProject(ModelObj):
|
|
|
1344
1357
|
example::
|
|
1345
1358
|
|
|
1346
1359
|
# register a simple file artifact
|
|
1347
|
-
project.set_artifact(
|
|
1360
|
+
project.set_artifact("data", target_path=data_url)
|
|
1348
1361
|
# register a model artifact
|
|
1349
|
-
project.set_artifact(
|
|
1362
|
+
project.set_artifact(
|
|
1363
|
+
"model", ModelArtifact(model_file="model.pkl"), target_path=model_dir_url
|
|
1364
|
+
)
|
|
1350
1365
|
|
|
1351
1366
|
# register a path to artifact package (will be imported on project load)
|
|
1352
1367
|
# to generate such package use `artifact.export(target_path)`
|
|
1353
|
-
project.set_artifact(
|
|
1368
|
+
project.set_artifact("model", "https://mystuff.com/models/mymodel.zip")
|
|
1354
1369
|
|
|
1355
1370
|
:param key: artifact key/name
|
|
1356
1371
|
:param artifact: mlrun Artifact object/dict (or its subclasses) or path to artifact
|
|
@@ -1569,7 +1584,9 @@ class MlrunProject(ModelObj):
|
|
|
1569
1584
|
"age": [42, 52, 36, 24, 73],
|
|
1570
1585
|
"testScore": [25, 94, 57, 62, 70],
|
|
1571
1586
|
}
|
|
1572
|
-
df = pd.DataFrame(
|
|
1587
|
+
df = pd.DataFrame(
|
|
1588
|
+
raw_data, columns=["first_name", "last_name", "age", "testScore"]
|
|
1589
|
+
)
|
|
1573
1590
|
project.log_dataset("mydf", df=df, stats=True)
|
|
1574
1591
|
|
|
1575
1592
|
:param key: artifact key
|
|
@@ -1643,13 +1660,16 @@ class MlrunProject(ModelObj):
|
|
|
1643
1660
|
|
|
1644
1661
|
example::
|
|
1645
1662
|
|
|
1646
|
-
project.log_model(
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1663
|
+
project.log_model(
|
|
1664
|
+
"model",
|
|
1665
|
+
body=dumps(model),
|
|
1666
|
+
model_file="model.pkl",
|
|
1667
|
+
metrics=context.results,
|
|
1668
|
+
training_set=training_df,
|
|
1669
|
+
label_column="label",
|
|
1670
|
+
feature_vector=feature_vector_uri,
|
|
1671
|
+
labels={"app": "fraud"},
|
|
1672
|
+
)
|
|
1653
1673
|
|
|
1654
1674
|
:param key: artifact key or artifact class ()
|
|
1655
1675
|
:param body: will use the body as the artifact content
|
|
@@ -1898,8 +1918,9 @@ class MlrunProject(ModelObj):
|
|
|
1898
1918
|
Create a monitoring function object without setting it to the project
|
|
1899
1919
|
|
|
1900
1920
|
examples::
|
|
1901
|
-
project.create_model_monitoring_function(
|
|
1902
|
-
|
|
1921
|
+
project.create_model_monitoring_function(
|
|
1922
|
+
application_class_name="MyApp", image="mlrun/mlrun", name="myApp"
|
|
1923
|
+
)
|
|
1903
1924
|
|
|
1904
1925
|
:param func: Code url, None refers to current Notebook
|
|
1905
1926
|
:param name: Name of the function, can be specified with a tag to support
|
|
@@ -2086,7 +2107,8 @@ class MlrunProject(ModelObj):
|
|
|
2086
2107
|
db.delete_function(project=self.name, name=fn_name)
|
|
2087
2108
|
if delete_histogram_data_drift_app:
|
|
2088
2109
|
db.delete_function(
|
|
2089
|
-
project=self.name,
|
|
2110
|
+
project=self.name,
|
|
2111
|
+
name=mm_constants.HistogramDataDriftApplicationConstants.NAME,
|
|
2090
2112
|
)
|
|
2091
2113
|
|
|
2092
2114
|
def set_function(
|
|
@@ -2113,19 +2135,20 @@ class MlrunProject(ModelObj):
|
|
|
2113
2135
|
examples::
|
|
2114
2136
|
|
|
2115
2137
|
proj.set_function(func_object)
|
|
2116
|
-
proj.set_function(
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
proj.set_function(
|
|
2120
|
-
proj.set_function(
|
|
2138
|
+
proj.set_function(
|
|
2139
|
+
"./src/mycode.py", "ingest", image="myrepo/ing:latest", with_repo=True
|
|
2140
|
+
)
|
|
2141
|
+
proj.set_function("http://.../mynb.ipynb", "train")
|
|
2142
|
+
proj.set_function("./func.yaml")
|
|
2143
|
+
proj.set_function("hub://get_toy_data", "getdata")
|
|
2121
2144
|
|
|
2122
2145
|
# set function requirements
|
|
2123
2146
|
|
|
2124
2147
|
# by providing a list of packages
|
|
2125
|
-
proj.set_function(
|
|
2148
|
+
proj.set_function("my.py", requirements=["requests", "pandas"])
|
|
2126
2149
|
|
|
2127
2150
|
# by providing a path to a pip requirements file
|
|
2128
|
-
proj.set_function(
|
|
2151
|
+
proj.set_function("my.py", requirements="requirements.txt")
|
|
2129
2152
|
|
|
2130
2153
|
:param func: Function object or spec/code url, None refers to current Notebook
|
|
2131
2154
|
:param name: Name of the function (under the project), can be specified with a tag to support
|
|
@@ -2603,9 +2626,9 @@ class MlrunProject(ModelObj):
|
|
|
2603
2626
|
|
|
2604
2627
|
read secrets from a source provider to be used in workflows, example::
|
|
2605
2628
|
|
|
2606
|
-
proj.with_secrets(
|
|
2607
|
-
proj.with_secrets(
|
|
2608
|
-
proj.with_secrets(
|
|
2629
|
+
proj.with_secrets("file", "file.txt")
|
|
2630
|
+
proj.with_secrets("inline", {"key": "val"})
|
|
2631
|
+
proj.with_secrets("env", "ENV1,ENV2", prefix="PFX_")
|
|
2609
2632
|
|
|
2610
2633
|
Vault secret source has several options::
|
|
2611
2634
|
|
|
@@ -2616,7 +2639,7 @@ class MlrunProject(ModelObj):
|
|
|
2616
2639
|
The 2nd option uses the current project name as context.
|
|
2617
2640
|
Can also use empty secret list::
|
|
2618
2641
|
|
|
2619
|
-
proj.with_secrets(
|
|
2642
|
+
proj.with_secrets("vault", [])
|
|
2620
2643
|
|
|
2621
2644
|
This will enable access to all secrets in vault registered to the current project.
|
|
2622
2645
|
|
|
@@ -2647,17 +2670,20 @@ class MlrunProject(ModelObj):
|
|
|
2647
2670
|
file_path: str = None,
|
|
2648
2671
|
provider: typing.Union[str, mlrun.common.schemas.SecretProviderName] = None,
|
|
2649
2672
|
):
|
|
2650
|
-
"""
|
|
2673
|
+
"""
|
|
2674
|
+
Set project secrets from dict or secrets env file
|
|
2651
2675
|
when using a secrets file it should have lines in the form KEY=VALUE, comment line start with "#"
|
|
2652
2676
|
V3IO paths/credentials and MLrun service API address are dropped from the secrets
|
|
2653
2677
|
|
|
2654
|
-
example secrets file
|
|
2678
|
+
example secrets file:
|
|
2679
|
+
|
|
2680
|
+
.. code-block:: shell
|
|
2655
2681
|
|
|
2656
2682
|
# this is an env file
|
|
2657
|
-
AWS_ACCESS_KEY_ID
|
|
2683
|
+
AWS_ACCESS_KEY_ID=XXXX
|
|
2658
2684
|
AWS_SECRET_ACCESS_KEY=YYYY
|
|
2659
2685
|
|
|
2660
|
-
usage
|
|
2686
|
+
usage:
|
|
2661
2687
|
|
|
2662
2688
|
# read env vars from dict or file and set as project secrets
|
|
2663
2689
|
project.set_secrets({"SECRET1": "value"})
|
|
@@ -3032,8 +3058,11 @@ class MlrunProject(ModelObj):
|
|
|
3032
3058
|
|
|
3033
3059
|
# run functions (refer to them by name)
|
|
3034
3060
|
run1 = project.run_function("myfunc", params={"x": 7})
|
|
3035
|
-
run2 = project.run_function(
|
|
3036
|
-
|
|
3061
|
+
run2 = project.run_function(
|
|
3062
|
+
"train",
|
|
3063
|
+
params={"label_columns": LABELS},
|
|
3064
|
+
inputs={"dataset": run1.outputs["data"]},
|
|
3065
|
+
)
|
|
3037
3066
|
|
|
3038
3067
|
:param function: name of the function (in the project) or function object
|
|
3039
3068
|
:param handler: name of the function handler
|
|
@@ -3421,9 +3450,9 @@ class MlrunProject(ModelObj):
|
|
|
3421
3450
|
Examples::
|
|
3422
3451
|
|
|
3423
3452
|
# Get latest version of all artifacts in project
|
|
3424
|
-
latest_artifacts = project.list_artifacts(
|
|
3453
|
+
latest_artifacts = project.list_artifacts("", tag="latest")
|
|
3425
3454
|
# check different artifact versions for a specific artifact, return as objects list
|
|
3426
|
-
result_versions = project.list_artifacts(
|
|
3455
|
+
result_versions = project.list_artifacts("results", tag="*").to_objects()
|
|
3427
3456
|
|
|
3428
3457
|
:param name: Name of artifacts to retrieve. Name with '~' prefix is used as a like query, and is not
|
|
3429
3458
|
case-sensitive. This means that querying for ``~name`` may return artifacts named
|
|
@@ -3473,7 +3502,7 @@ class MlrunProject(ModelObj):
|
|
|
3473
3502
|
Examples::
|
|
3474
3503
|
|
|
3475
3504
|
# Get latest version of all models in project
|
|
3476
|
-
latest_models = project.list_models(
|
|
3505
|
+
latest_models = project.list_models("", tag="latest")
|
|
3477
3506
|
|
|
3478
3507
|
|
|
3479
3508
|
:param name: Name of artifacts to retrieve. Name with '~' prefix is used as a like query, and is not
|
|
@@ -3578,14 +3607,14 @@ class MlrunProject(ModelObj):
|
|
|
3578
3607
|
Example::
|
|
3579
3608
|
|
|
3580
3609
|
# return a list of runs matching the name and label and compare
|
|
3581
|
-
runs = project.list_runs(name=
|
|
3610
|
+
runs = project.list_runs(name="download", labels="owner=admin")
|
|
3582
3611
|
runs.compare()
|
|
3583
3612
|
|
|
3584
3613
|
# multi-label filter can also be provided
|
|
3585
|
-
runs = project.list_runs(name=
|
|
3614
|
+
runs = project.list_runs(name="download", labels=["kind=job", "owner=admin"])
|
|
3586
3615
|
|
|
3587
3616
|
# If running in Jupyter, can use the .show() function to display the results
|
|
3588
|
-
project.list_runs(name=
|
|
3617
|
+
project.list_runs(name="").show()
|
|
3589
3618
|
|
|
3590
3619
|
|
|
3591
3620
|
:param name: Name of the run to retrieve.
|
|
@@ -3772,6 +3801,73 @@ class MlrunProject(ModelObj):
|
|
|
3772
3801
|
|
|
3773
3802
|
mlrun.db.get_run_db().delete_api_gateway(name=name, project=self.name)
|
|
3774
3803
|
|
|
3804
|
+
def store_alert_config(self, alert_data: AlertConfig, alert_name=None):
|
|
3805
|
+
"""
|
|
3806
|
+
Create/modify an alert.
|
|
3807
|
+
:param alert_data: The data of the alert.
|
|
3808
|
+
:param alert_name: The name of the alert.
|
|
3809
|
+
:return: the created/modified alert.
|
|
3810
|
+
"""
|
|
3811
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
3812
|
+
if alert_name is None:
|
|
3813
|
+
alert_name = alert_data.name
|
|
3814
|
+
return db.store_alert_config(alert_name, alert_data.dict(), self.metadata.name)
|
|
3815
|
+
|
|
3816
|
+
def get_alert_config(self, alert_name: str) -> AlertConfig:
|
|
3817
|
+
"""
|
|
3818
|
+
Retrieve an alert.
|
|
3819
|
+
:param alert_name: The name of the alert to retrieve.
|
|
3820
|
+
:return: The alert object.
|
|
3821
|
+
"""
|
|
3822
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
3823
|
+
return db.get_alert_config(alert_name, self.metadata.name)
|
|
3824
|
+
|
|
3825
|
+
def list_alerts_configs(self):
|
|
3826
|
+
"""
|
|
3827
|
+
Retrieve list of alerts of a project.
|
|
3828
|
+
:return: All the alerts objects of the project.
|
|
3829
|
+
"""
|
|
3830
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
3831
|
+
return db.list_alerts_configs(self.metadata.name)
|
|
3832
|
+
|
|
3833
|
+
def delete_alert_config(
|
|
3834
|
+
self, alert_data: AlertConfig = None, alert_name: str = None
|
|
3835
|
+
):
|
|
3836
|
+
"""
|
|
3837
|
+
Delete an alert.
|
|
3838
|
+
:param alert_data: The data of the alert.
|
|
3839
|
+
:param alert_name: The name of the alert to delete.
|
|
3840
|
+
"""
|
|
3841
|
+
if alert_data is None and alert_name is None:
|
|
3842
|
+
raise ValueError(
|
|
3843
|
+
"At least one of alert_data or alert_name must be provided"
|
|
3844
|
+
)
|
|
3845
|
+
if alert_data and alert_name and alert_data.name != alert_name:
|
|
3846
|
+
raise ValueError("Alert_data name does not match the provided alert_name")
|
|
3847
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
3848
|
+
if alert_data:
|
|
3849
|
+
alert_name = alert_data.name
|
|
3850
|
+
db.delete_alert_config(alert_name, self.metadata.name)
|
|
3851
|
+
|
|
3852
|
+
def reset_alert_config(
|
|
3853
|
+
self, alert_data: AlertConfig = None, alert_name: str = None
|
|
3854
|
+
):
|
|
3855
|
+
"""
|
|
3856
|
+
Reset an alert.
|
|
3857
|
+
:param alert_data: The data of the alert.
|
|
3858
|
+
:param alert_name: The name of the alert to reset.
|
|
3859
|
+
"""
|
|
3860
|
+
if alert_data is None and alert_name is None:
|
|
3861
|
+
raise ValueError(
|
|
3862
|
+
"At least one of alert_data or alert_name must be provided"
|
|
3863
|
+
)
|
|
3864
|
+
if alert_data and alert_name and alert_data.name != alert_name:
|
|
3865
|
+
raise ValueError("Alert_data name does not match the provided alert_name")
|
|
3866
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
3867
|
+
if alert_data:
|
|
3868
|
+
alert_name = alert_data.name
|
|
3869
|
+
db.reset_alert_config(alert_name, self.metadata.name)
|
|
3870
|
+
|
|
3775
3871
|
def _run_authenticated_git_action(
|
|
3776
3872
|
self,
|
|
3777
3873
|
action: Callable,
|
mlrun/run.py
CHANGED
|
@@ -114,16 +114,18 @@ def function_to_module(code="", workdir=None, secrets=None, silent=False):
|
|
|
114
114
|
|
|
115
115
|
example::
|
|
116
116
|
|
|
117
|
-
mod = mlrun.function_to_module(
|
|
118
|
-
task = mlrun.new_task(inputs={
|
|
119
|
-
context = mlrun.get_or_create_ctx(
|
|
120
|
-
mod.my_job(context, p1=1, p2=
|
|
117
|
+
mod = mlrun.function_to_module("./examples/training.py")
|
|
118
|
+
task = mlrun.new_task(inputs={"infile.txt": "../examples/infile.txt"})
|
|
119
|
+
context = mlrun.get_or_create_ctx("myfunc", spec=task)
|
|
120
|
+
mod.my_job(context, p1=1, p2="x")
|
|
121
121
|
print(context.to_yaml())
|
|
122
122
|
|
|
123
|
-
fn = mlrun.import_function(
|
|
123
|
+
fn = mlrun.import_function("hub://open-archive")
|
|
124
124
|
mod = mlrun.function_to_module(fn)
|
|
125
|
-
data = mlrun.run.get_dataitem(
|
|
126
|
-
|
|
125
|
+
data = mlrun.run.get_dataitem(
|
|
126
|
+
"https://fpsignals-public.s3.amazonaws.com/catsndogs.tar.gz"
|
|
127
|
+
)
|
|
128
|
+
context = mlrun.get_or_create_ctx("myfunc")
|
|
127
129
|
mod.open_archive(context, archive_url=data)
|
|
128
130
|
print(context.to_yaml())
|
|
129
131
|
|
|
@@ -256,29 +258,31 @@ def get_or_create_ctx(
|
|
|
256
258
|
Examples::
|
|
257
259
|
|
|
258
260
|
# load MLRUN runtime context (will be set by the runtime framework e.g. KubeFlow)
|
|
259
|
-
context = get_or_create_ctx(
|
|
261
|
+
context = get_or_create_ctx("train")
|
|
260
262
|
|
|
261
263
|
# get parameters from the runtime context (or use defaults)
|
|
262
|
-
p1 = context.get_param(
|
|
263
|
-
p2 = context.get_param(
|
|
264
|
+
p1 = context.get_param("p1", 1)
|
|
265
|
+
p2 = context.get_param("p2", "a-string")
|
|
264
266
|
|
|
265
267
|
# access input metadata, values, files, and secrets (passwords)
|
|
266
|
-
print(f
|
|
267
|
-
print(f
|
|
268
|
+
print(f"Run: {context.name} (uid={context.uid})")
|
|
269
|
+
print(f"Params: p1={p1}, p2={p2}")
|
|
268
270
|
print(f'accesskey = {context.get_secret("ACCESS_KEY")}')
|
|
269
|
-
input_str = context.get_input(
|
|
270
|
-
print(f
|
|
271
|
+
input_str = context.get_input("infile.txt").get()
|
|
272
|
+
print(f"file: {input_str}")
|
|
271
273
|
|
|
272
274
|
# RUN some useful code e.g. ML training, data prep, etc.
|
|
273
275
|
|
|
274
276
|
# log scalar result values (job result metrics)
|
|
275
|
-
context.log_result(
|
|
276
|
-
context.log_result(
|
|
277
|
-
context.set_label(
|
|
277
|
+
context.log_result("accuracy", p1 * 2)
|
|
278
|
+
context.log_result("loss", p1 * 3)
|
|
279
|
+
context.set_label("framework", "sklearn")
|
|
278
280
|
|
|
279
281
|
# log various types of artifacts (file, web page, table), will be versioned and visible in the UI
|
|
280
|
-
context.log_artifact(
|
|
281
|
-
|
|
282
|
+
context.log_artifact(
|
|
283
|
+
"model.txt", body=b"abc is 123", labels={"framework": "xgboost"}
|
|
284
|
+
)
|
|
285
|
+
context.log_artifact("results.html", body=b"<b> Some HTML <b>", viewer="web-app")
|
|
282
286
|
|
|
283
287
|
"""
|
|
284
288
|
|
|
@@ -348,7 +352,9 @@ def import_function(url="", secrets=None, db="", project=None, new_name=None):
|
|
|
348
352
|
|
|
349
353
|
function = mlrun.import_function("hub://auto-trainer")
|
|
350
354
|
function = mlrun.import_function("./func.yaml")
|
|
351
|
-
function = mlrun.import_function(
|
|
355
|
+
function = mlrun.import_function(
|
|
356
|
+
"https://raw.githubusercontent.com/org/repo/func.yaml"
|
|
357
|
+
)
|
|
352
358
|
|
|
353
359
|
:param url: path/url to Function Hub, db or function YAML file
|
|
354
360
|
:param secrets: optional, credentials dict for DB or URL (s3, v3io, ...)
|
|
@@ -447,12 +453,18 @@ def new_function(
|
|
|
447
453
|
Example::
|
|
448
454
|
|
|
449
455
|
# define a container based function (the `training.py` must exist in the container workdir)
|
|
450
|
-
f = new_function(
|
|
456
|
+
f = new_function(
|
|
457
|
+
command="training.py -x {x}", image="myrepo/image:latest", kind="job"
|
|
458
|
+
)
|
|
451
459
|
f.run(params={"x": 5})
|
|
452
460
|
|
|
453
461
|
# define a container based function which reads its source from a git archive
|
|
454
|
-
f = new_function(
|
|
455
|
-
|
|
462
|
+
f = new_function(
|
|
463
|
+
command="training.py -x {x}",
|
|
464
|
+
image="myrepo/image:latest",
|
|
465
|
+
kind="job",
|
|
466
|
+
source="git://github.com/mlrun/something.git",
|
|
467
|
+
)
|
|
456
468
|
f.run(params={"x": 5})
|
|
457
469
|
|
|
458
470
|
# define a local handler function (execute a local function handler)
|
|
@@ -665,11 +677,15 @@ def code_to_function(
|
|
|
665
677
|
import mlrun
|
|
666
678
|
|
|
667
679
|
# create job function object from notebook code and add doc/metadata
|
|
668
|
-
fn = mlrun.code_to_function(
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
680
|
+
fn = mlrun.code_to_function(
|
|
681
|
+
"file_utils",
|
|
682
|
+
kind="job",
|
|
683
|
+
handler="open_archive",
|
|
684
|
+
image="mlrun/mlrun",
|
|
685
|
+
description="this function opens a zip archive into a local/mounted folder",
|
|
686
|
+
categories=["fileutils"],
|
|
687
|
+
labels={"author": "me"},
|
|
688
|
+
)
|
|
673
689
|
|
|
674
690
|
example::
|
|
675
691
|
|
|
@@ -680,11 +696,15 @@ def code_to_function(
|
|
|
680
696
|
Path("mover.py").touch()
|
|
681
697
|
|
|
682
698
|
# create nuclio function object from python module call mover.py
|
|
683
|
-
fn = mlrun.code_to_function(
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
699
|
+
fn = mlrun.code_to_function(
|
|
700
|
+
"nuclio-mover",
|
|
701
|
+
kind="nuclio",
|
|
702
|
+
filename="mover.py",
|
|
703
|
+
image="python:3.7",
|
|
704
|
+
description="this function moves files from one system to another",
|
|
705
|
+
requirements=["pandas"],
|
|
706
|
+
labels={"author": "me"},
|
|
707
|
+
)
|
|
688
708
|
|
|
689
709
|
"""
|
|
690
710
|
filebase, _ = path.splitext(path.basename(filename))
|
|
@@ -1098,13 +1118,25 @@ def wait_for_runs_completion(
|
|
|
1098
1118
|
example::
|
|
1099
1119
|
|
|
1100
1120
|
# run two training functions in parallel and wait for the results
|
|
1101
|
-
inputs = {
|
|
1102
|
-
run1 = train.run(
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1121
|
+
inputs = {"dataset": cleaned_data}
|
|
1122
|
+
run1 = train.run(
|
|
1123
|
+
name="train_lr",
|
|
1124
|
+
inputs=inputs,
|
|
1125
|
+
watch=False,
|
|
1126
|
+
params={
|
|
1127
|
+
"model_pkg_class": "sklearn.linear_model.LogisticRegression",
|
|
1128
|
+
"label_column": "label",
|
|
1129
|
+
},
|
|
1130
|
+
)
|
|
1131
|
+
run2 = train.run(
|
|
1132
|
+
name="train_lr",
|
|
1133
|
+
inputs=inputs,
|
|
1134
|
+
watch=False,
|
|
1135
|
+
params={
|
|
1136
|
+
"model_pkg_class": "sklearn.ensemble.RandomForestClassifier",
|
|
1137
|
+
"label_column": "label",
|
|
1138
|
+
},
|
|
1139
|
+
)
|
|
1108
1140
|
completed = wait_for_runs_completion([run1, run2])
|
|
1109
1141
|
|
|
1110
1142
|
:param runs: list of run objects (the returned values of function.run())
|
|
@@ -223,14 +223,14 @@ class AbstractMPIJobRuntime(KubejobRuntime, abc.ABC):
|
|
|
223
223
|
```
|
|
224
224
|
# Define the wanted MPI arguments
|
|
225
225
|
mpi_args = []
|
|
226
|
-
mpi_args.append(
|
|
227
|
-
mpi_args.append(
|
|
228
|
-
mpi_args.append(
|
|
229
|
-
mpi_args.append(
|
|
230
|
-
mpi_args.append(
|
|
231
|
-
mpi_args.append(
|
|
232
|
-
mpi_args.append(
|
|
233
|
-
mpi_args.append(
|
|
226
|
+
mpi_args.append("-x")
|
|
227
|
+
mpi_args.append("NCCL_DEBUG=INFO")
|
|
228
|
+
mpi_args.append("-x")
|
|
229
|
+
mpi_args.append("NCCL_SOCKET_NTHREADS=2")
|
|
230
|
+
mpi_args.append("-x")
|
|
231
|
+
mpi_args.append("NCCL_NSOCKS_PERTHREAD=8")
|
|
232
|
+
mpi_args.append("-x")
|
|
233
|
+
mpi_args.append("NCCL_MIN_NCHANNELS=4")
|
|
234
234
|
|
|
235
235
|
# Set the MPI arguments in the function
|
|
236
236
|
fn.set_mpi_args(mpi_args)
|
|
@@ -345,17 +345,21 @@ class RemoteRuntime(KubeResource):
|
|
|
345
345
|
|
|
346
346
|
git::
|
|
347
347
|
|
|
348
|
-
fn.with_source_archive(
|
|
349
|
-
|
|
350
|
-
|
|
348
|
+
fn.with_source_archive(
|
|
349
|
+
"git://github.com/org/repo#my-branch",
|
|
350
|
+
handler="main:handler",
|
|
351
|
+
workdir="path/inside/repo",
|
|
352
|
+
)
|
|
351
353
|
|
|
352
354
|
s3::
|
|
353
355
|
|
|
354
356
|
fn.spec.nuclio_runtime = "golang"
|
|
355
|
-
fn.with_source_archive(
|
|
357
|
+
fn.with_source_archive(
|
|
358
|
+
"s3://my-bucket/path/in/bucket/my-functions-archive",
|
|
356
359
|
handler="my_func:Handler",
|
|
357
360
|
workdir="path/inside/functions/archive",
|
|
358
|
-
runtime="golang"
|
|
361
|
+
runtime="golang",
|
|
362
|
+
)
|
|
359
363
|
"""
|
|
360
364
|
self.spec.build.source = source
|
|
361
365
|
# update handler in function_handler
|
mlrun/runtimes/nuclio/serving.py
CHANGED
|
@@ -295,9 +295,7 @@ class ServingRuntime(RemoteRuntime):
|
|
|
295
295
|
"provided class is not a router step, must provide a router class in router topology"
|
|
296
296
|
)
|
|
297
297
|
else:
|
|
298
|
-
step = RouterStep(
|
|
299
|
-
class_name=class_name, class_args=class_args, engine=engine
|
|
300
|
-
)
|
|
298
|
+
step = RouterStep(class_name=class_name, class_args=class_args)
|
|
301
299
|
self.spec.graph = step
|
|
302
300
|
elif topology == StepKinds.flow:
|
|
303
301
|
self.spec.graph = RootFlowStep(engine=engine)
|
|
@@ -367,8 +365,8 @@ class ServingRuntime(RemoteRuntime):
|
|
|
367
365
|
|
|
368
366
|
Example, create a function (from the notebook), add a model class, and deploy::
|
|
369
367
|
|
|
370
|
-
fn = code_to_function(kind=
|
|
371
|
-
fn.add_model(
|
|
368
|
+
fn = code_to_function(kind="serving")
|
|
369
|
+
fn.add_model("boost", model_path, model_class="MyClass", my_arg=5)
|
|
372
370
|
fn.deploy()
|
|
373
371
|
|
|
374
372
|
only works with router topology, for nested topologies (model under router under flow)
|
|
@@ -450,7 +448,7 @@ class ServingRuntime(RemoteRuntime):
|
|
|
450
448
|
|
|
451
449
|
example::
|
|
452
450
|
|
|
453
|
-
fn.add_child_function(
|
|
451
|
+
fn.add_child_function("enrich", "./enrich.ipynb", "mlrun/mlrun")
|
|
454
452
|
|
|
455
453
|
:param name: child function name
|
|
456
454
|
:param url: function/code url, support .py, .ipynb, .yaml extensions
|
|
@@ -731,8 +729,11 @@ class ServingRuntime(RemoteRuntime):
|
|
|
731
729
|
example::
|
|
732
730
|
|
|
733
731
|
serving_fn = mlrun.new_function("serving", image="mlrun/mlrun", kind="serving")
|
|
734
|
-
serving_fn.add_model(
|
|
735
|
-
|
|
732
|
+
serving_fn.add_model(
|
|
733
|
+
"my-classifier",
|
|
734
|
+
model_path=model_path,
|
|
735
|
+
class_name="mlrun.frameworks.sklearn.SklearnModelServer",
|
|
736
|
+
)
|
|
736
737
|
serving_fn.plot(rankdir="LR")
|
|
737
738
|
|
|
738
739
|
:param filename: target filepath for the image (None for the notebook)
|
mlrun/runtimes/pod.py
CHANGED
|
@@ -1278,9 +1278,9 @@ class KubeResource(BaseRuntime):
|
|
|
1278
1278
|
from kubernetes import client as k8s_client
|
|
1279
1279
|
|
|
1280
1280
|
security_context = k8s_client.V1SecurityContext(
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1281
|
+
run_as_user=1000,
|
|
1282
|
+
run_as_group=3000,
|
|
1283
|
+
)
|
|
1284
1284
|
function.with_security_context(security_context)
|
|
1285
1285
|
|
|
1286
1286
|
More info:
|