mlrun 1.7.0rc9__py3-none-any.whl → 1.7.0rc11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (65) hide show
  1. mlrun/__init__.py +1 -0
  2. mlrun/artifacts/model.py +29 -25
  3. mlrun/common/schemas/__init__.py +1 -0
  4. mlrun/common/schemas/alert.py +122 -0
  5. mlrun/common/schemas/auth.py +4 -0
  6. mlrun/common/schemas/client_spec.py +1 -0
  7. mlrun/common/schemas/model_monitoring/constants.py +3 -1
  8. mlrun/config.py +8 -4
  9. mlrun/datastore/base.py +6 -5
  10. mlrun/datastore/sources.py +9 -4
  11. mlrun/datastore/targets.py +11 -3
  12. mlrun/datastore/v3io.py +27 -50
  13. mlrun/db/base.py +44 -2
  14. mlrun/db/httpdb.py +192 -20
  15. mlrun/db/nopdb.py +36 -1
  16. mlrun/execution.py +21 -14
  17. mlrun/feature_store/api.py +6 -3
  18. mlrun/feature_store/feature_set.py +39 -23
  19. mlrun/feature_store/feature_vector.py +2 -1
  20. mlrun/feature_store/steps.py +30 -19
  21. mlrun/features.py +4 -13
  22. mlrun/frameworks/auto_mlrun/auto_mlrun.py +2 -2
  23. mlrun/frameworks/lgbm/__init__.py +1 -1
  24. mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
  25. mlrun/frameworks/lgbm/model_handler.py +1 -1
  26. mlrun/frameworks/pytorch/__init__.py +2 -2
  27. mlrun/frameworks/sklearn/__init__.py +1 -1
  28. mlrun/frameworks/tf_keras/__init__.py +1 -1
  29. mlrun/frameworks/xgboost/__init__.py +1 -1
  30. mlrun/model.py +2 -2
  31. mlrun/model_monitoring/application.py +11 -2
  32. mlrun/model_monitoring/applications/histogram_data_drift.py +3 -3
  33. mlrun/model_monitoring/controller.py +2 -3
  34. mlrun/model_monitoring/stream_processing.py +0 -1
  35. mlrun/model_monitoring/writer.py +32 -0
  36. mlrun/package/packagers_manager.py +1 -0
  37. mlrun/platforms/__init__.py +1 -1
  38. mlrun/platforms/other.py +1 -1
  39. mlrun/projects/operations.py +11 -4
  40. mlrun/projects/project.py +148 -52
  41. mlrun/run.py +72 -40
  42. mlrun/runtimes/mpijob/abstract.py +8 -8
  43. mlrun/runtimes/nuclio/function.py +9 -5
  44. mlrun/runtimes/nuclio/serving.py +9 -8
  45. mlrun/runtimes/pod.py +3 -3
  46. mlrun/secrets.py +6 -2
  47. mlrun/serving/routers.py +3 -1
  48. mlrun/serving/states.py +12 -33
  49. mlrun/serving/v2_serving.py +4 -4
  50. mlrun/utils/helpers.py +1 -1
  51. mlrun/utils/notifications/notification/base.py +12 -0
  52. mlrun/utils/notifications/notification/console.py +2 -0
  53. mlrun/utils/notifications/notification/git.py +3 -1
  54. mlrun/utils/notifications/notification/ipython.py +2 -0
  55. mlrun/utils/notifications/notification/slack.py +41 -13
  56. mlrun/utils/notifications/notification/webhook.py +11 -1
  57. mlrun/utils/retryer.py +2 -2
  58. mlrun/utils/version/version.json +2 -2
  59. {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/METADATA +1 -1
  60. {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/RECORD +64 -64
  61. mlrun/datastore/helpers.py +0 -18
  62. {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/LICENSE +0 -0
  63. {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/WHEEL +0 -0
  64. {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/entry_points.txt +0 -0
  65. {mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py CHANGED
@@ -47,6 +47,7 @@ import mlrun.runtimes.pod
47
47
  import mlrun.runtimes.utils
48
48
  import mlrun.serving
49
49
  import mlrun.utils.regex
50
+ from mlrun.common.schemas import AlertConfig
50
51
  from mlrun.datastore.datastore_profile import DatastoreProfile, DatastoreProfile2Json
51
52
  from mlrun.runtimes.nuclio.function import RemoteRuntime
52
53
 
@@ -140,11 +141,15 @@ def new_project(
140
141
  example::
141
142
 
142
143
  # create a project with local and hub functions, a workflow, and an artifact
143
- project = mlrun.new_project("myproj", "./", init_git=True, description="my new project")
144
- project.set_function('prep_data.py', 'prep-data', image='mlrun/mlrun', handler='prep_data')
145
- project.set_function('hub://auto-trainer', 'train')
146
- project.set_artifact('data', Artifact(target_path=data_url))
147
- project.set_workflow('main', "./myflow.py")
144
+ project = mlrun.new_project(
145
+ "myproj", "./", init_git=True, description="my new project"
146
+ )
147
+ project.set_function(
148
+ "prep_data.py", "prep-data", image="mlrun/mlrun", handler="prep_data"
149
+ )
150
+ project.set_function("hub://auto-trainer", "train")
151
+ project.set_artifact("data", Artifact(target_path=data_url))
152
+ project.set_workflow("main", "./myflow.py")
148
153
  project.save()
149
154
 
150
155
  # run the "main" workflow (watch=True to wait for run completion)
@@ -154,19 +159,25 @@ def new_project(
154
159
 
155
160
  # create a new project from a zip template (can also use yaml/git templates)
156
161
  # initialize a local git, and register the git remote path
157
- project = mlrun.new_project("myproj", "./", init_git=True,
158
- remote="git://github.com/mlrun/project-demo.git",
159
- from_template="http://mysite/proj.zip")
162
+ project = mlrun.new_project(
163
+ "myproj",
164
+ "./",
165
+ init_git=True,
166
+ remote="git://github.com/mlrun/project-demo.git",
167
+ from_template="http://mysite/proj.zip",
168
+ )
160
169
  project.run("main", watch=True)
161
170
 
162
171
 
163
172
  example using project_setup.py to init the project objects::
164
173
 
165
174
  def setup(project):
166
- project.set_function('prep_data.py', 'prep-data', image='mlrun/mlrun', handler='prep_data')
167
- project.set_function('hub://auto-trainer', 'train')
168
- project.set_artifact('data', Artifact(target_path=data_url))
169
- project.set_workflow('main', "./myflow.py")
175
+ project.set_function(
176
+ "prep_data.py", "prep-data", image="mlrun/mlrun", handler="prep_data"
177
+ )
178
+ project.set_function("hub://auto-trainer", "train")
179
+ project.set_artifact("data", Artifact(target_path=data_url))
180
+ project.set_workflow("main", "./myflow.py")
170
181
  return project
171
182
 
172
183
 
@@ -298,7 +309,7 @@ def load_project(
298
309
  # When using git as the url source the context directory must be an empty or
299
310
  # non-existent folder as the git repo will be cloned there
300
311
  project = load_project("./demo_proj", "git://github.com/mlrun/project-demo.git")
301
- project.run("main", arguments={'data': data_url})
312
+ project.run("main", arguments={"data": data_url})
302
313
 
303
314
 
304
315
  project_setup.py example::
@@ -437,9 +448,11 @@ def get_or_create_project(
437
448
  Usage example::
438
449
 
439
450
  # load project from the DB (if exist) or the source repo
440
- project = get_or_create_project("myproj", "./", "git://github.com/mlrun/demo-xgb-project.git")
451
+ project = get_or_create_project(
452
+ "myproj", "./", "git://github.com/mlrun/demo-xgb-project.git"
453
+ )
441
454
  project.pull("development") # pull the latest code from git
442
- project.run("main", arguments={'data': data_url}) # run the workflow "main"
455
+ project.run("main", arguments={"data": data_url}) # run the workflow "main"
443
456
 
444
457
 
445
458
  project_setup.py example::
@@ -1344,13 +1357,15 @@ class MlrunProject(ModelObj):
1344
1357
  example::
1345
1358
 
1346
1359
  # register a simple file artifact
1347
- project.set_artifact('data', target_path=data_url)
1360
+ project.set_artifact("data", target_path=data_url)
1348
1361
  # register a model artifact
1349
- project.set_artifact('model', ModelArtifact(model_file="model.pkl"), target_path=model_dir_url)
1362
+ project.set_artifact(
1363
+ "model", ModelArtifact(model_file="model.pkl"), target_path=model_dir_url
1364
+ )
1350
1365
 
1351
1366
  # register a path to artifact package (will be imported on project load)
1352
1367
  # to generate such package use `artifact.export(target_path)`
1353
- project.set_artifact('model', 'https://mystuff.com/models/mymodel.zip')
1368
+ project.set_artifact("model", "https://mystuff.com/models/mymodel.zip")
1354
1369
 
1355
1370
  :param key: artifact key/name
1356
1371
  :param artifact: mlrun Artifact object/dict (or its subclasses) or path to artifact
@@ -1569,7 +1584,9 @@ class MlrunProject(ModelObj):
1569
1584
  "age": [42, 52, 36, 24, 73],
1570
1585
  "testScore": [25, 94, 57, 62, 70],
1571
1586
  }
1572
- df = pd.DataFrame(raw_data, columns=["first_name", "last_name", "age", "testScore"])
1587
+ df = pd.DataFrame(
1588
+ raw_data, columns=["first_name", "last_name", "age", "testScore"]
1589
+ )
1573
1590
  project.log_dataset("mydf", df=df, stats=True)
1574
1591
 
1575
1592
  :param key: artifact key
@@ -1643,13 +1660,16 @@ class MlrunProject(ModelObj):
1643
1660
 
1644
1661
  example::
1645
1662
 
1646
- project.log_model("model", body=dumps(model),
1647
- model_file="model.pkl",
1648
- metrics=context.results,
1649
- training_set=training_df,
1650
- label_column='label',
1651
- feature_vector=feature_vector_uri,
1652
- labels={"app": "fraud"})
1663
+ project.log_model(
1664
+ "model",
1665
+ body=dumps(model),
1666
+ model_file="model.pkl",
1667
+ metrics=context.results,
1668
+ training_set=training_df,
1669
+ label_column="label",
1670
+ feature_vector=feature_vector_uri,
1671
+ labels={"app": "fraud"},
1672
+ )
1653
1673
 
1654
1674
  :param key: artifact key or artifact class ()
1655
1675
  :param body: will use the body as the artifact content
@@ -1898,8 +1918,9 @@ class MlrunProject(ModelObj):
1898
1918
  Create a monitoring function object without setting it to the project
1899
1919
 
1900
1920
  examples::
1901
- project.create_model_monitoring_function(application_class_name="MyApp",
1902
- image="mlrun/mlrun", name="myApp")
1921
+ project.create_model_monitoring_function(
1922
+ application_class_name="MyApp", image="mlrun/mlrun", name="myApp"
1923
+ )
1903
1924
 
1904
1925
  :param func: Code url, None refers to current Notebook
1905
1926
  :param name: Name of the function, can be specified with a tag to support
@@ -2086,7 +2107,8 @@ class MlrunProject(ModelObj):
2086
2107
  db.delete_function(project=self.name, name=fn_name)
2087
2108
  if delete_histogram_data_drift_app:
2088
2109
  db.delete_function(
2089
- project=self.name, name=mm_constants.MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME
2110
+ project=self.name,
2111
+ name=mm_constants.HistogramDataDriftApplicationConstants.NAME,
2090
2112
  )
2091
2113
 
2092
2114
  def set_function(
@@ -2113,19 +2135,20 @@ class MlrunProject(ModelObj):
2113
2135
  examples::
2114
2136
 
2115
2137
  proj.set_function(func_object)
2116
- proj.set_function('./src/mycode.py', 'ingest',
2117
- image='myrepo/ing:latest', with_repo=True)
2118
- proj.set_function('http://.../mynb.ipynb', 'train')
2119
- proj.set_function('./func.yaml')
2120
- proj.set_function('hub://get_toy_data', 'getdata')
2138
+ proj.set_function(
2139
+ "./src/mycode.py", "ingest", image="myrepo/ing:latest", with_repo=True
2140
+ )
2141
+ proj.set_function("http://.../mynb.ipynb", "train")
2142
+ proj.set_function("./func.yaml")
2143
+ proj.set_function("hub://get_toy_data", "getdata")
2121
2144
 
2122
2145
  # set function requirements
2123
2146
 
2124
2147
  # by providing a list of packages
2125
- proj.set_function('my.py', requirements=["requests", "pandas"])
2148
+ proj.set_function("my.py", requirements=["requests", "pandas"])
2126
2149
 
2127
2150
  # by providing a path to a pip requirements file
2128
- proj.set_function('my.py', requirements="requirements.txt")
2151
+ proj.set_function("my.py", requirements="requirements.txt")
2129
2152
 
2130
2153
  :param func: Function object or spec/code url, None refers to current Notebook
2131
2154
  :param name: Name of the function (under the project), can be specified with a tag to support
@@ -2603,9 +2626,9 @@ class MlrunProject(ModelObj):
2603
2626
 
2604
2627
  read secrets from a source provider to be used in workflows, example::
2605
2628
 
2606
- proj.with_secrets('file', 'file.txt')
2607
- proj.with_secrets('inline', {'key': 'val'})
2608
- proj.with_secrets('env', 'ENV1,ENV2', prefix='PFX_')
2629
+ proj.with_secrets("file", "file.txt")
2630
+ proj.with_secrets("inline", {"key": "val"})
2631
+ proj.with_secrets("env", "ENV1,ENV2", prefix="PFX_")
2609
2632
 
2610
2633
  Vault secret source has several options::
2611
2634
 
@@ -2616,7 +2639,7 @@ class MlrunProject(ModelObj):
2616
2639
  The 2nd option uses the current project name as context.
2617
2640
  Can also use empty secret list::
2618
2641
 
2619
- proj.with_secrets('vault', [])
2642
+ proj.with_secrets("vault", [])
2620
2643
 
2621
2644
  This will enable access to all secrets in vault registered to the current project.
2622
2645
 
@@ -2647,17 +2670,20 @@ class MlrunProject(ModelObj):
2647
2670
  file_path: str = None,
2648
2671
  provider: typing.Union[str, mlrun.common.schemas.SecretProviderName] = None,
2649
2672
  ):
2650
- """set project secrets from dict or secrets env file
2673
+ """
2674
+ Set project secrets from dict or secrets env file
2651
2675
  when using a secrets file it should have lines in the form KEY=VALUE, comment line start with "#"
2652
2676
  V3IO paths/credentials and MLrun service API address are dropped from the secrets
2653
2677
 
2654
- example secrets file::
2678
+ example secrets file:
2679
+
2680
+ .. code-block:: shell
2655
2681
 
2656
2682
  # this is an env file
2657
- AWS_ACCESS_KEY_ID-XXXX
2683
+ AWS_ACCESS_KEY_ID=XXXX
2658
2684
  AWS_SECRET_ACCESS_KEY=YYYY
2659
2685
 
2660
- usage::
2686
+ usage:
2661
2687
 
2662
2688
  # read env vars from dict or file and set as project secrets
2663
2689
  project.set_secrets({"SECRET1": "value"})
@@ -3032,8 +3058,11 @@ class MlrunProject(ModelObj):
3032
3058
 
3033
3059
  # run functions (refer to them by name)
3034
3060
  run1 = project.run_function("myfunc", params={"x": 7})
3035
- run2 = project.run_function("train", params={"label_columns": LABELS},
3036
- inputs={"dataset":run1.outputs["data"]})
3061
+ run2 = project.run_function(
3062
+ "train",
3063
+ params={"label_columns": LABELS},
3064
+ inputs={"dataset": run1.outputs["data"]},
3065
+ )
3037
3066
 
3038
3067
  :param function: name of the function (in the project) or function object
3039
3068
  :param handler: name of the function handler
@@ -3421,9 +3450,9 @@ class MlrunProject(ModelObj):
3421
3450
  Examples::
3422
3451
 
3423
3452
  # Get latest version of all artifacts in project
3424
- latest_artifacts = project.list_artifacts('', tag='latest')
3453
+ latest_artifacts = project.list_artifacts("", tag="latest")
3425
3454
  # check different artifact versions for a specific artifact, return as objects list
3426
- result_versions = project.list_artifacts('results', tag='*').to_objects()
3455
+ result_versions = project.list_artifacts("results", tag="*").to_objects()
3427
3456
 
3428
3457
  :param name: Name of artifacts to retrieve. Name with '~' prefix is used as a like query, and is not
3429
3458
  case-sensitive. This means that querying for ``~name`` may return artifacts named
@@ -3473,7 +3502,7 @@ class MlrunProject(ModelObj):
3473
3502
  Examples::
3474
3503
 
3475
3504
  # Get latest version of all models in project
3476
- latest_models = project.list_models('', tag='latest')
3505
+ latest_models = project.list_models("", tag="latest")
3477
3506
 
3478
3507
 
3479
3508
  :param name: Name of artifacts to retrieve. Name with '~' prefix is used as a like query, and is not
@@ -3578,14 +3607,14 @@ class MlrunProject(ModelObj):
3578
3607
  Example::
3579
3608
 
3580
3609
  # return a list of runs matching the name and label and compare
3581
- runs = project.list_runs(name='download', labels='owner=admin')
3610
+ runs = project.list_runs(name="download", labels="owner=admin")
3582
3611
  runs.compare()
3583
3612
 
3584
3613
  # multi-label filter can also be provided
3585
- runs = project.list_runs(name='download', labels=["kind=job", "owner=admin"])
3614
+ runs = project.list_runs(name="download", labels=["kind=job", "owner=admin"])
3586
3615
 
3587
3616
  # If running in Jupyter, can use the .show() function to display the results
3588
- project.list_runs(name='').show()
3617
+ project.list_runs(name="").show()
3589
3618
 
3590
3619
 
3591
3620
  :param name: Name of the run to retrieve.
@@ -3772,6 +3801,73 @@ class MlrunProject(ModelObj):
3772
3801
 
3773
3802
  mlrun.db.get_run_db().delete_api_gateway(name=name, project=self.name)
3774
3803
 
3804
+ def store_alert_config(self, alert_data: AlertConfig, alert_name=None):
3805
+ """
3806
+ Create/modify an alert.
3807
+ :param alert_data: The data of the alert.
3808
+ :param alert_name: The name of the alert.
3809
+ :return: the created/modified alert.
3810
+ """
3811
+ db = mlrun.db.get_run_db(secrets=self._secrets)
3812
+ if alert_name is None:
3813
+ alert_name = alert_data.name
3814
+ return db.store_alert_config(alert_name, alert_data.dict(), self.metadata.name)
3815
+
3816
+ def get_alert_config(self, alert_name: str) -> AlertConfig:
3817
+ """
3818
+ Retrieve an alert.
3819
+ :param alert_name: The name of the alert to retrieve.
3820
+ :return: The alert object.
3821
+ """
3822
+ db = mlrun.db.get_run_db(secrets=self._secrets)
3823
+ return db.get_alert_config(alert_name, self.metadata.name)
3824
+
3825
+ def list_alerts_configs(self):
3826
+ """
3827
+ Retrieve list of alerts of a project.
3828
+ :return: All the alerts objects of the project.
3829
+ """
3830
+ db = mlrun.db.get_run_db(secrets=self._secrets)
3831
+ return db.list_alerts_configs(self.metadata.name)
3832
+
3833
+ def delete_alert_config(
3834
+ self, alert_data: AlertConfig = None, alert_name: str = None
3835
+ ):
3836
+ """
3837
+ Delete an alert.
3838
+ :param alert_data: The data of the alert.
3839
+ :param alert_name: The name of the alert to delete.
3840
+ """
3841
+ if alert_data is None and alert_name is None:
3842
+ raise ValueError(
3843
+ "At least one of alert_data or alert_name must be provided"
3844
+ )
3845
+ if alert_data and alert_name and alert_data.name != alert_name:
3846
+ raise ValueError("Alert_data name does not match the provided alert_name")
3847
+ db = mlrun.db.get_run_db(secrets=self._secrets)
3848
+ if alert_data:
3849
+ alert_name = alert_data.name
3850
+ db.delete_alert_config(alert_name, self.metadata.name)
3851
+
3852
+ def reset_alert_config(
3853
+ self, alert_data: AlertConfig = None, alert_name: str = None
3854
+ ):
3855
+ """
3856
+ Reset an alert.
3857
+ :param alert_data: The data of the alert.
3858
+ :param alert_name: The name of the alert to reset.
3859
+ """
3860
+ if alert_data is None and alert_name is None:
3861
+ raise ValueError(
3862
+ "At least one of alert_data or alert_name must be provided"
3863
+ )
3864
+ if alert_data and alert_name and alert_data.name != alert_name:
3865
+ raise ValueError("Alert_data name does not match the provided alert_name")
3866
+ db = mlrun.db.get_run_db(secrets=self._secrets)
3867
+ if alert_data:
3868
+ alert_name = alert_data.name
3869
+ db.reset_alert_config(alert_name, self.metadata.name)
3870
+
3775
3871
  def _run_authenticated_git_action(
3776
3872
  self,
3777
3873
  action: Callable,
mlrun/run.py CHANGED
@@ -114,16 +114,18 @@ def function_to_module(code="", workdir=None, secrets=None, silent=False):
114
114
 
115
115
  example::
116
116
 
117
- mod = mlrun.function_to_module('./examples/training.py')
118
- task = mlrun.new_task(inputs={'infile.txt': '../examples/infile.txt'})
119
- context = mlrun.get_or_create_ctx('myfunc', spec=task)
120
- mod.my_job(context, p1=1, p2='x')
117
+ mod = mlrun.function_to_module("./examples/training.py")
118
+ task = mlrun.new_task(inputs={"infile.txt": "../examples/infile.txt"})
119
+ context = mlrun.get_or_create_ctx("myfunc", spec=task)
120
+ mod.my_job(context, p1=1, p2="x")
121
121
  print(context.to_yaml())
122
122
 
123
- fn = mlrun.import_function('hub://open-archive')
123
+ fn = mlrun.import_function("hub://open-archive")
124
124
  mod = mlrun.function_to_module(fn)
125
- data = mlrun.run.get_dataitem("https://fpsignals-public.s3.amazonaws.com/catsndogs.tar.gz")
126
- context = mlrun.get_or_create_ctx('myfunc')
125
+ data = mlrun.run.get_dataitem(
126
+ "https://fpsignals-public.s3.amazonaws.com/catsndogs.tar.gz"
127
+ )
128
+ context = mlrun.get_or_create_ctx("myfunc")
127
129
  mod.open_archive(context, archive_url=data)
128
130
  print(context.to_yaml())
129
131
 
@@ -256,29 +258,31 @@ def get_or_create_ctx(
256
258
  Examples::
257
259
 
258
260
  # load MLRUN runtime context (will be set by the runtime framework e.g. KubeFlow)
259
- context = get_or_create_ctx('train')
261
+ context = get_or_create_ctx("train")
260
262
 
261
263
  # get parameters from the runtime context (or use defaults)
262
- p1 = context.get_param('p1', 1)
263
- p2 = context.get_param('p2', 'a-string')
264
+ p1 = context.get_param("p1", 1)
265
+ p2 = context.get_param("p2", "a-string")
264
266
 
265
267
  # access input metadata, values, files, and secrets (passwords)
266
- print(f'Run: {context.name} (uid={context.uid})')
267
- print(f'Params: p1={p1}, p2={p2}')
268
+ print(f"Run: {context.name} (uid={context.uid})")
269
+ print(f"Params: p1={p1}, p2={p2}")
268
270
  print(f'accesskey = {context.get_secret("ACCESS_KEY")}')
269
- input_str = context.get_input('infile.txt').get()
270
- print(f'file: {input_str}')
271
+ input_str = context.get_input("infile.txt").get()
272
+ print(f"file: {input_str}")
271
273
 
272
274
  # RUN some useful code e.g. ML training, data prep, etc.
273
275
 
274
276
  # log scalar result values (job result metrics)
275
- context.log_result('accuracy', p1 * 2)
276
- context.log_result('loss', p1 * 3)
277
- context.set_label('framework', 'sklearn')
277
+ context.log_result("accuracy", p1 * 2)
278
+ context.log_result("loss", p1 * 3)
279
+ context.set_label("framework", "sklearn")
278
280
 
279
281
  # log various types of artifacts (file, web page, table), will be versioned and visible in the UI
280
- context.log_artifact('model.txt', body=b'abc is 123', labels={'framework': 'xgboost'})
281
- context.log_artifact('results.html', body=b'<b> Some HTML <b>', viewer='web-app')
282
+ context.log_artifact(
283
+ "model.txt", body=b"abc is 123", labels={"framework": "xgboost"}
284
+ )
285
+ context.log_artifact("results.html", body=b"<b> Some HTML <b>", viewer="web-app")
282
286
 
283
287
  """
284
288
 
@@ -348,7 +352,9 @@ def import_function(url="", secrets=None, db="", project=None, new_name=None):
348
352
 
349
353
  function = mlrun.import_function("hub://auto-trainer")
350
354
  function = mlrun.import_function("./func.yaml")
351
- function = mlrun.import_function("https://raw.githubusercontent.com/org/repo/func.yaml")
355
+ function = mlrun.import_function(
356
+ "https://raw.githubusercontent.com/org/repo/func.yaml"
357
+ )
352
358
 
353
359
  :param url: path/url to Function Hub, db or function YAML file
354
360
  :param secrets: optional, credentials dict for DB or URL (s3, v3io, ...)
@@ -447,12 +453,18 @@ def new_function(
447
453
  Example::
448
454
 
449
455
  # define a container based function (the `training.py` must exist in the container workdir)
450
- f = new_function(command='training.py -x {x}', image='myrepo/image:latest', kind='job')
456
+ f = new_function(
457
+ command="training.py -x {x}", image="myrepo/image:latest", kind="job"
458
+ )
451
459
  f.run(params={"x": 5})
452
460
 
453
461
  # define a container based function which reads its source from a git archive
454
- f = new_function(command='training.py -x {x}', image='myrepo/image:latest', kind='job',
455
- source='git://github.com/mlrun/something.git')
462
+ f = new_function(
463
+ command="training.py -x {x}",
464
+ image="myrepo/image:latest",
465
+ kind="job",
466
+ source="git://github.com/mlrun/something.git",
467
+ )
456
468
  f.run(params={"x": 5})
457
469
 
458
470
  # define a local handler function (execute a local function handler)
@@ -665,11 +677,15 @@ def code_to_function(
665
677
  import mlrun
666
678
 
667
679
  # create job function object from notebook code and add doc/metadata
668
- fn = mlrun.code_to_function("file_utils", kind="job",
669
- handler="open_archive", image="mlrun/mlrun",
670
- description = "this function opens a zip archive into a local/mounted folder",
671
- categories = ["fileutils"],
672
- labels = {"author": "me"})
680
+ fn = mlrun.code_to_function(
681
+ "file_utils",
682
+ kind="job",
683
+ handler="open_archive",
684
+ image="mlrun/mlrun",
685
+ description="this function opens a zip archive into a local/mounted folder",
686
+ categories=["fileutils"],
687
+ labels={"author": "me"},
688
+ )
673
689
 
674
690
  example::
675
691
 
@@ -680,11 +696,15 @@ def code_to_function(
680
696
  Path("mover.py").touch()
681
697
 
682
698
  # create nuclio function object from python module call mover.py
683
- fn = mlrun.code_to_function("nuclio-mover", kind="nuclio",
684
- filename="mover.py", image="python:3.7",
685
- description = "this function moves files from one system to another",
686
- requirements = ["pandas"],
687
- labels = {"author": "me"})
699
+ fn = mlrun.code_to_function(
700
+ "nuclio-mover",
701
+ kind="nuclio",
702
+ filename="mover.py",
703
+ image="python:3.7",
704
+ description="this function moves files from one system to another",
705
+ requirements=["pandas"],
706
+ labels={"author": "me"},
707
+ )
688
708
 
689
709
  """
690
710
  filebase, _ = path.splitext(path.basename(filename))
@@ -1098,13 +1118,25 @@ def wait_for_runs_completion(
1098
1118
  example::
1099
1119
 
1100
1120
  # run two training functions in parallel and wait for the results
1101
- inputs = {'dataset': cleaned_data}
1102
- run1 = train.run(name='train_lr', inputs=inputs, watch=False,
1103
- params={'model_pkg_class': 'sklearn.linear_model.LogisticRegression',
1104
- 'label_column': 'label'})
1105
- run2 = train.run(name='train_lr', inputs=inputs, watch=False,
1106
- params={'model_pkg_class': 'sklearn.ensemble.RandomForestClassifier',
1107
- 'label_column': 'label'})
1121
+ inputs = {"dataset": cleaned_data}
1122
+ run1 = train.run(
1123
+ name="train_lr",
1124
+ inputs=inputs,
1125
+ watch=False,
1126
+ params={
1127
+ "model_pkg_class": "sklearn.linear_model.LogisticRegression",
1128
+ "label_column": "label",
1129
+ },
1130
+ )
1131
+ run2 = train.run(
1132
+ name="train_lr",
1133
+ inputs=inputs,
1134
+ watch=False,
1135
+ params={
1136
+ "model_pkg_class": "sklearn.ensemble.RandomForestClassifier",
1137
+ "label_column": "label",
1138
+ },
1139
+ )
1108
1140
  completed = wait_for_runs_completion([run1, run2])
1109
1141
 
1110
1142
  :param runs: list of run objects (the returned values of function.run())
@@ -223,14 +223,14 @@ class AbstractMPIJobRuntime(KubejobRuntime, abc.ABC):
223
223
  ```
224
224
  # Define the wanted MPI arguments
225
225
  mpi_args = []
226
- mpi_args.append('-x')
227
- mpi_args.append('NCCL_DEBUG=INFO')
228
- mpi_args.append('-x')
229
- mpi_args.append('NCCL_SOCKET_NTHREADS=2')
230
- mpi_args.append('-x')
231
- mpi_args.append('NCCL_NSOCKS_PERTHREAD=8')
232
- mpi_args.append('-x')
233
- mpi_args.append('NCCL_MIN_NCHANNELS=4')
226
+ mpi_args.append("-x")
227
+ mpi_args.append("NCCL_DEBUG=INFO")
228
+ mpi_args.append("-x")
229
+ mpi_args.append("NCCL_SOCKET_NTHREADS=2")
230
+ mpi_args.append("-x")
231
+ mpi_args.append("NCCL_NSOCKS_PERTHREAD=8")
232
+ mpi_args.append("-x")
233
+ mpi_args.append("NCCL_MIN_NCHANNELS=4")
234
234
 
235
235
  # Set the MPI arguments in the function
236
236
  fn.set_mpi_args(mpi_args)
@@ -345,17 +345,21 @@ class RemoteRuntime(KubeResource):
345
345
 
346
346
  git::
347
347
 
348
- fn.with_source_archive("git://github.com/org/repo#my-branch",
349
- handler="main:handler",
350
- workdir="path/inside/repo")
348
+ fn.with_source_archive(
349
+ "git://github.com/org/repo#my-branch",
350
+ handler="main:handler",
351
+ workdir="path/inside/repo",
352
+ )
351
353
 
352
354
  s3::
353
355
 
354
356
  fn.spec.nuclio_runtime = "golang"
355
- fn.with_source_archive("s3://my-bucket/path/in/bucket/my-functions-archive",
357
+ fn.with_source_archive(
358
+ "s3://my-bucket/path/in/bucket/my-functions-archive",
356
359
  handler="my_func:Handler",
357
360
  workdir="path/inside/functions/archive",
358
- runtime="golang")
361
+ runtime="golang",
362
+ )
359
363
  """
360
364
  self.spec.build.source = source
361
365
  # update handler in function_handler
@@ -295,9 +295,7 @@ class ServingRuntime(RemoteRuntime):
295
295
  "provided class is not a router step, must provide a router class in router topology"
296
296
  )
297
297
  else:
298
- step = RouterStep(
299
- class_name=class_name, class_args=class_args, engine=engine
300
- )
298
+ step = RouterStep(class_name=class_name, class_args=class_args)
301
299
  self.spec.graph = step
302
300
  elif topology == StepKinds.flow:
303
301
  self.spec.graph = RootFlowStep(engine=engine)
@@ -367,8 +365,8 @@ class ServingRuntime(RemoteRuntime):
367
365
 
368
366
  Example, create a function (from the notebook), add a model class, and deploy::
369
367
 
370
- fn = code_to_function(kind='serving')
371
- fn.add_model('boost', model_path, model_class='MyClass', my_arg=5)
368
+ fn = code_to_function(kind="serving")
369
+ fn.add_model("boost", model_path, model_class="MyClass", my_arg=5)
372
370
  fn.deploy()
373
371
 
374
372
  only works with router topology, for nested topologies (model under router under flow)
@@ -450,7 +448,7 @@ class ServingRuntime(RemoteRuntime):
450
448
 
451
449
  example::
452
450
 
453
- fn.add_child_function('enrich', './enrich.ipynb', 'mlrun/mlrun')
451
+ fn.add_child_function("enrich", "./enrich.ipynb", "mlrun/mlrun")
454
452
 
455
453
  :param name: child function name
456
454
  :param url: function/code url, support .py, .ipynb, .yaml extensions
@@ -731,8 +729,11 @@ class ServingRuntime(RemoteRuntime):
731
729
  example::
732
730
 
733
731
  serving_fn = mlrun.new_function("serving", image="mlrun/mlrun", kind="serving")
734
- serving_fn.add_model('my-classifier',model_path=model_path,
735
- class_name='mlrun.frameworks.sklearn.SklearnModelServer')
732
+ serving_fn.add_model(
733
+ "my-classifier",
734
+ model_path=model_path,
735
+ class_name="mlrun.frameworks.sklearn.SklearnModelServer",
736
+ )
736
737
  serving_fn.plot(rankdir="LR")
737
738
 
738
739
  :param filename: target filepath for the image (None for the notebook)
mlrun/runtimes/pod.py CHANGED
@@ -1278,9 +1278,9 @@ class KubeResource(BaseRuntime):
1278
1278
  from kubernetes import client as k8s_client
1279
1279
 
1280
1280
  security_context = k8s_client.V1SecurityContext(
1281
- run_as_user=1000,
1282
- run_as_group=3000,
1283
- )
1281
+ run_as_user=1000,
1282
+ run_as_group=3000,
1283
+ )
1284
1284
  function.with_security_context(security_context)
1285
1285
 
1286
1286
  More info: