mlrun 1.9.0rc4__py3-none-any.whl → 1.10.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +13 -2
- mlrun/api/schemas/__init__.py +0 -1
- mlrun/common/__init__.py +0 -1
- mlrun/common/constants.py +7 -0
- mlrun/common/db/__init__.py +0 -1
- mlrun/common/db/sql_session.py +0 -1
- mlrun/common/formatters/__init__.py +0 -1
- mlrun/common/formatters/artifact.py +0 -1
- mlrun/common/formatters/base.py +0 -1
- mlrun/common/formatters/feature_set.py +0 -1
- mlrun/common/formatters/function.py +0 -1
- mlrun/common/formatters/model_endpoint.py +0 -1
- mlrun/common/formatters/pipeline.py +0 -1
- mlrun/common/formatters/project.py +0 -1
- mlrun/common/formatters/run.py +0 -2
- mlrun/common/runtimes/constants.py +9 -2
- mlrun/common/schemas/__init__.py +2 -0
- mlrun/common/schemas/alert.py +1 -1
- mlrun/common/schemas/api_gateway.py +1 -1
- mlrun/common/schemas/artifact.py +1 -1
- mlrun/common/schemas/auth.py +1 -1
- mlrun/common/schemas/background_task.py +1 -1
- mlrun/common/schemas/client_spec.py +1 -1
- mlrun/common/schemas/clusterization_spec.py +1 -1
- mlrun/common/schemas/constants.py +1 -1
- mlrun/common/schemas/datastore_profile.py +0 -1
- mlrun/common/schemas/events.py +1 -1
- mlrun/common/schemas/feature_store.py +1 -1
- mlrun/common/schemas/frontend_spec.py +1 -1
- mlrun/common/schemas/function.py +1 -1
- mlrun/common/schemas/http.py +1 -1
- mlrun/common/schemas/hub.py +1 -1
- mlrun/common/schemas/k8s.py +1 -1
- mlrun/common/schemas/memory_reports.py +0 -1
- mlrun/common/schemas/notification.py +4 -0
- mlrun/common/schemas/object.py +1 -1
- mlrun/common/schemas/partition.py +1 -1
- mlrun/common/schemas/pipeline.py +1 -1
- mlrun/common/schemas/project.py +1 -1
- mlrun/common/schemas/regex.py +1 -1
- mlrun/common/schemas/runtime_resource.py +1 -1
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/schemas/secret.py +1 -1
- mlrun/common/schemas/serving.py +17 -0
- mlrun/common/schemas/tag.py +0 -1
- mlrun/common/schemas/workflow.py +1 -1
- mlrun/common/secrets.py +0 -1
- mlrun/config.py +9 -11
- mlrun/data_types/infer.py +1 -1
- mlrun/data_types/spark.py +1 -1
- mlrun/datastore/datastore.py +1 -1
- mlrun/datastore/datastore_profile.py +5 -55
- mlrun/datastore/snowflake_utils.py +0 -1
- mlrun/datastore/sources.py +21 -13
- mlrun/datastore/spark_utils.py +0 -1
- mlrun/datastore/utils.py +20 -5
- mlrun/db/base.py +1 -1
- mlrun/db/httpdb.py +17 -12
- mlrun/db/nopdb.py +1 -2
- mlrun/errors.py +8 -1
- mlrun/execution.py +22 -1
- mlrun/feature_store/feature_set.py +0 -12
- mlrun/feature_store/retrieval/base.py +1 -1
- mlrun/feature_store/retrieval/dask_merger.py +1 -1
- mlrun/feature_store/retrieval/job.py +1 -1
- mlrun/feature_store/retrieval/spark_merger.py +0 -2
- mlrun/feature_store/steps.py +1 -1
- mlrun/features.py +1 -1
- mlrun/frameworks/_common/artifacts_library.py +1 -1
- mlrun/frameworks/_common/mlrun_interface.py +1 -1
- mlrun/frameworks/_common/model_handler.py +3 -3
- mlrun/frameworks/_common/producer.py +0 -1
- mlrun/frameworks/_common/utils.py +1 -1
- mlrun/frameworks/_dl_common/loggers/logger.py +0 -1
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
- mlrun/frameworks/_dl_common/model_handler.py +1 -1
- mlrun/frameworks/_dl_common/utils.py +1 -1
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
- mlrun/frameworks/_ml_common/loggers/logger.py +0 -1
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_ml_common/model_handler.py +1 -1
- mlrun/frameworks/_ml_common/pkl_model_server.py +1 -1
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +0 -1
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +0 -1
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
- mlrun/frameworks/_ml_common/producer.py +1 -1
- mlrun/frameworks/_ml_common/utils.py +1 -1
- mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +0 -1
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +0 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/model_handler.py +1 -1
- mlrun/frameworks/lgbm/model_server.py +1 -1
- mlrun/frameworks/lgbm/utils.py +1 -1
- mlrun/frameworks/onnx/dataset.py +1 -1
- mlrun/frameworks/onnx/mlrun_interface.py +1 -1
- mlrun/frameworks/onnx/model_handler.py +1 -1
- mlrun/frameworks/onnx/model_server.py +1 -1
- mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
- mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
- mlrun/frameworks/pytorch/model_handler.py +1 -1
- mlrun/frameworks/pytorch/model_server.py +1 -1
- mlrun/frameworks/pytorch/utils.py +1 -1
- mlrun/frameworks/sklearn/__init__.py +0 -14
- mlrun/frameworks/sklearn/estimator.py +1 -1
- mlrun/frameworks/sklearn/metric.py +1 -1
- mlrun/frameworks/sklearn/metrics_library.py +1 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
- mlrun/frameworks/sklearn/model_handler.py +1 -1
- mlrun/frameworks/sklearn/utils.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
- mlrun/frameworks/tf_keras/model_handler.py +1 -1
- mlrun/frameworks/tf_keras/model_server.py +1 -1
- mlrun/frameworks/tf_keras/utils.py +1 -1
- mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
- mlrun/frameworks/xgboost/model_handler.py +1 -1
- mlrun/frameworks/xgboost/utils.py +1 -1
- mlrun/k8s_utils.py +340 -0
- mlrun/launcher/base.py +3 -3
- mlrun/launcher/client.py +1 -1
- mlrun/launcher/local.py +2 -2
- mlrun/launcher/remote.py +2 -2
- mlrun/model.py +14 -0
- mlrun/model_monitoring/applications/__init__.py +0 -1
- mlrun/model_monitoring/applications/_application_steps.py +3 -1
- mlrun/model_monitoring/applications/evidently/base.py +59 -71
- mlrun/model_monitoring/controller.py +26 -13
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +13 -5
- mlrun/package/context_handler.py +1 -1
- mlrun/package/errors.py +1 -1
- mlrun/package/packager.py +1 -1
- mlrun/package/packagers/default_packager.py +1 -1
- mlrun/package/packagers/numpy_packagers.py +1 -1
- mlrun/package/packagers/pandas_packagers.py +1 -1
- mlrun/package/packagers/python_standard_library_packagers.py +1 -1
- mlrun/package/packagers_manager.py +1 -1
- mlrun/package/utils/_archiver.py +1 -1
- mlrun/package/utils/_formatter.py +1 -1
- mlrun/package/utils/_pickler.py +1 -1
- mlrun/package/utils/_supported_format.py +1 -1
- mlrun/package/utils/log_hint_utils.py +1 -1
- mlrun/package/utils/type_hint_utils.py +1 -1
- mlrun/projects/operations.py +36 -21
- mlrun/projects/project.py +147 -92
- mlrun/render.py +5 -9
- mlrun/run.py +1 -1
- mlrun/runtimes/base.py +17 -7
- mlrun/runtimes/daskjob.py +2 -1
- mlrun/runtimes/databricks_job/databricks_cancel_task.py +0 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +2 -1
- mlrun/runtimes/databricks_job/databricks_wrapper.py +0 -1
- mlrun/runtimes/mounts.py +2 -0
- mlrun/runtimes/nuclio/function.py +5 -0
- mlrun/runtimes/nuclio/serving.py +1 -1
- mlrun/runtimes/pod.py +4 -349
- mlrun/runtimes/sparkjob/spark3job.py +0 -12
- mlrun/runtimes/utils.py +25 -8
- mlrun/serving/merger.py +0 -1
- mlrun/serving/remote.py +1 -1
- mlrun/serving/serving_wrapper.py +1 -1
- mlrun/serving/states.py +142 -9
- mlrun/serving/utils.py +1 -1
- mlrun/utils/async_http.py +0 -1
- mlrun/utils/clones.py +1 -1
- mlrun/utils/db.py +1 -1
- mlrun/utils/helpers.py +3 -1
- mlrun/utils/http.py +0 -1
- mlrun/utils/regex.py +0 -1
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/vault.py +1 -1
- mlrun/utils/version/__init__.py +1 -1
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +1 -1
- {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/METADATA +12 -12
- mlrun-1.10.0rc2.dist-info/RECORD +351 -0
- {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/WHEEL +1 -1
- mlrun-1.9.0rc4.dist-info/RECORD +0 -351
- {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/entry_points.txt +0 -0
- {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/licenses/LICENSE +0 -0
- {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py
CHANGED
|
@@ -470,7 +470,8 @@ def get_or_create_project(
|
|
|
470
470
|
parameters: Optional[dict] = None,
|
|
471
471
|
allow_cross_project: Optional[bool] = None,
|
|
472
472
|
) -> "MlrunProject":
|
|
473
|
-
"""Load a project from MLRun DB, or create/import if it does not exist
|
|
473
|
+
"""Load a project from MLRun DB, or create/import if it does not exist.
|
|
474
|
+
The project will become the default project for the current session.
|
|
474
475
|
|
|
475
476
|
MLRun looks for a project.yaml file with project definition and objects in the project root path
|
|
476
477
|
and use it to initialize the project, in addition it runs the project_setup.py file (if it exists)
|
|
@@ -2143,35 +2144,35 @@ class MlrunProject(ModelObj):
|
|
|
2143
2144
|
),
|
|
2144
2145
|
reset_policy: mlrun.common.schemas.alert.ResetPolicy = mlrun.common.schemas.alert.ResetPolicy.AUTO,
|
|
2145
2146
|
) -> list[mlrun.alerts.alert.AlertConfig]:
|
|
2146
|
-
"""
|
|
2147
|
-
Generate alert configurations based on specified model endpoints and result names, which can be defined
|
|
2147
|
+
"""Generate alert configurations based on specified model endpoints and result names, which can be defined
|
|
2148
2148
|
explicitly or using regex patterns.
|
|
2149
2149
|
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2150
|
+
:param name: The name of the AlertConfig template. It will be combined with
|
|
2151
|
+
mep id, app name and result name to generate a unique name.
|
|
2152
|
+
:param summary: Summary of the alert, will be sent in the generated notifications
|
|
2153
|
+
:param endpoints: The endpoints from which metrics will be retrieved to configure
|
|
2154
|
+
the alerts.
|
|
2155
|
+
The ModelEndpointList object is obtained via the `list_model_endpoints`
|
|
2156
|
+
method or created manually using `ModelEndpoint` objects.
|
|
2157
|
+
:param events: AlertTrigger event types (EventKind).
|
|
2158
|
+
:param notifications: List of notifications to invoke once the alert is triggered
|
|
2159
|
+
:param result_names: Optional. Filters the result names used to create the alert
|
|
2160
|
+
configuration, constructed from the app and result_name regex.
|
|
2161
|
+
|
|
2162
|
+
For example:
|
|
2163
|
+
[`app1.result-*`, `*.result1`]
|
|
2164
|
+
will match "mep_uid1.app1.result.result-1" and
|
|
2165
|
+
"mep_uid1.app2.result.result1".
|
|
2166
|
+
A specific result_name (not a wildcard) will always create a new alert
|
|
2167
|
+
config, regardless of whether the result name exists.
|
|
2168
|
+
:param severity: Severity of the alert.
|
|
2169
|
+
:param criteria: The threshold for triggering the alert based on the
|
|
2170
|
+
specified number of events within the defined time period.
|
|
2171
|
+
:param reset_policy: When to clear the alert. Either "manual" for manual reset of the alert,
|
|
2172
|
+
or "auto" if the criteria contains a time period.
|
|
2173
|
+
|
|
2174
|
+
:returns: List of AlertConfig according to endpoints results,
|
|
2175
|
+
filtered by result_names.
|
|
2175
2176
|
"""
|
|
2176
2177
|
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
2177
2178
|
matching_results = []
|
|
@@ -2860,10 +2861,10 @@ class MlrunProject(ModelObj):
|
|
|
2860
2861
|
|
|
2861
2862
|
self.spec.set_function(name, function_object, func)
|
|
2862
2863
|
|
|
2863
|
-
# TODO: Remove this in 1.
|
|
2864
|
+
# TODO: Remove this in 1.11.0
|
|
2864
2865
|
@deprecated.deprecated(
|
|
2865
2866
|
version="1.8.0",
|
|
2866
|
-
reason="'remove_function' is deprecated and will be removed in 1.
|
|
2867
|
+
reason="'remove_function' is deprecated and will be removed in 1.11.0. "
|
|
2867
2868
|
"Please use `delete_function` instead.",
|
|
2868
2869
|
category=FutureWarning,
|
|
2869
2870
|
)
|
|
@@ -3709,7 +3710,7 @@ class MlrunProject(ModelObj):
|
|
|
3709
3710
|
brokers=["<kafka-broker-ip-address>:9094"],
|
|
3710
3711
|
topics=[], # Keep the topics list empty
|
|
3711
3712
|
## SASL is supported
|
|
3712
|
-
# sasl_user="
|
|
3713
|
+
# sasl_user="<kafka-sasl-user>",
|
|
3713
3714
|
# sasl_pass="<kafka-sasl-password>",
|
|
3714
3715
|
)
|
|
3715
3716
|
project.register_datastore_profile(stream_profile)
|
|
@@ -3742,6 +3743,29 @@ class MlrunProject(ModelObj):
|
|
|
3742
3743
|
|
|
3743
3744
|
In the V3IO datastore, you must provide an explicit access key to the stream, but not to the TSDB.
|
|
3744
3745
|
|
|
3746
|
+
An external Confluent Kafka stream is also supported. Here is an example:
|
|
3747
|
+
|
|
3748
|
+
.. code-block:: python
|
|
3749
|
+
|
|
3750
|
+
from mlrun.datastore.datastore_profile import DatastoreProfileKafkaSource
|
|
3751
|
+
|
|
3752
|
+
stream_profile = DatastoreProfileKafkaSource(
|
|
3753
|
+
name="confluent-kafka",
|
|
3754
|
+
brokers=["<server-domain-start>.confluent.cloud:9092"],
|
|
3755
|
+
topics=[],
|
|
3756
|
+
sasl_user="<API-key>",
|
|
3757
|
+
sasl_pass="<API-secret>",
|
|
3758
|
+
kwargs_public={
|
|
3759
|
+
"security_protocol": "SASL_SSL",
|
|
3760
|
+
"api_version_auto_timeout_ms": 15_000, # 15 seconds
|
|
3761
|
+
"tls": {"enable": True},
|
|
3762
|
+
"new_topic": {"replication_factor": 3},
|
|
3763
|
+
},
|
|
3764
|
+
)
|
|
3765
|
+
|
|
3766
|
+
The replication factor and timeout configuration might need to be adjusted according to your Confluent cluster
|
|
3767
|
+
type and settings.
|
|
3768
|
+
|
|
3745
3769
|
:param tsdb_profile_name: The datastore profile name of the time-series database to be used in model
|
|
3746
3770
|
monitoring. The supported profiles are:
|
|
3747
3771
|
|
|
@@ -3795,7 +3819,7 @@ class MlrunProject(ModelObj):
|
|
|
3795
3819
|
top_level: bool = False,
|
|
3796
3820
|
uids: Optional[list[str]] = None,
|
|
3797
3821
|
latest_only: bool = False,
|
|
3798
|
-
tsdb_metrics: bool =
|
|
3822
|
+
tsdb_metrics: bool = False,
|
|
3799
3823
|
metric_list: Optional[list[str]] = None,
|
|
3800
3824
|
) -> mlrun.common.schemas.ModelEndpointList:
|
|
3801
3825
|
"""
|
|
@@ -3880,6 +3904,7 @@ class MlrunProject(ModelObj):
|
|
|
3880
3904
|
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
3881
3905
|
builder_env: Optional[dict] = None,
|
|
3882
3906
|
reset_on_run: Optional[bool] = None,
|
|
3907
|
+
output_path: Optional[str] = None,
|
|
3883
3908
|
) -> typing.Union[mlrun.model.RunObject, PipelineNodeWrapper]:
|
|
3884
3909
|
"""Run a local or remote task as part of a local/kubeflow pipeline
|
|
3885
3910
|
|
|
@@ -3910,7 +3935,7 @@ class MlrunProject(ModelObj):
|
|
|
3910
3935
|
parsed during runtime from `mlrun.DataItem` to the given type hint. The type hint can be
|
|
3911
3936
|
given in the key field of the dictionary after a colon, e.g: "<key> : <type_hint>".
|
|
3912
3937
|
:param outputs: list of outputs which can pass in the workflow
|
|
3913
|
-
:param workdir: default
|
|
3938
|
+
:param workdir: working directory of the executed job and the default path for artifact inputs
|
|
3914
3939
|
:param labels: labels to tag the job/run with ({key:val, ..})
|
|
3915
3940
|
:param base_task: task object to use as base
|
|
3916
3941
|
:param watch: watch/follow run log, True by default
|
|
@@ -3922,7 +3947,8 @@ class MlrunProject(ModelObj):
|
|
|
3922
3947
|
(which will be converted to the class using its `from_crontab` constructor),
|
|
3923
3948
|
see this link for help:
|
|
3924
3949
|
https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
|
|
3925
|
-
:param artifact_path: path to store artifacts, when running in a workflow this will be set
|
|
3950
|
+
:param artifact_path: (deprecated) path to store artifacts, when running in a workflow this will be set
|
|
3951
|
+
automatically
|
|
3926
3952
|
:param notifications: list of notifications to push when the run is completed
|
|
3927
3953
|
:param returns: List of log hints - configurations for how to log the returning values from the
|
|
3928
3954
|
handler's run (as artifacts or results). The list's length must be equal to the amount
|
|
@@ -3940,34 +3966,47 @@ class MlrunProject(ModelObj):
|
|
|
3940
3966
|
:param reset_on_run: When True, function python modules would reload prior to code execution.
|
|
3941
3967
|
This ensures latest code changes are executed. This argument must be used in
|
|
3942
3968
|
conjunction with the local=True argument.
|
|
3969
|
+
:param output_path: path to store artifacts, when running in a workflow this will be set automatically
|
|
3943
3970
|
|
|
3944
3971
|
:return: MLRun RunObject or PipelineNodeWrapper
|
|
3945
3972
|
"""
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3962
|
-
|
|
3963
|
-
|
|
3964
|
-
|
|
3965
|
-
|
|
3966
|
-
|
|
3967
|
-
|
|
3968
|
-
|
|
3969
|
-
|
|
3970
|
-
|
|
3973
|
+
if artifact_path:
|
|
3974
|
+
warnings.warn(
|
|
3975
|
+
"'artifact_path' parameter is deprecated in 1.10.0 and will be removed in 1.12.0, "
|
|
3976
|
+
"use 'output_path' instead.",
|
|
3977
|
+
# TODO: Remove this in 1.12.0
|
|
3978
|
+
FutureWarning,
|
|
3979
|
+
)
|
|
3980
|
+
output_path = output_path or artifact_path
|
|
3981
|
+
|
|
3982
|
+
# remove this filter once the artifact_path parameter is deprecated in 1.12.0
|
|
3983
|
+
with warnings.catch_warnings():
|
|
3984
|
+
warnings.simplefilter("ignore", category=FutureWarning)
|
|
3985
|
+
return run_function(
|
|
3986
|
+
function,
|
|
3987
|
+
handler=handler,
|
|
3988
|
+
name=name,
|
|
3989
|
+
params=params,
|
|
3990
|
+
hyperparams=hyperparams,
|
|
3991
|
+
hyper_param_options=hyper_param_options,
|
|
3992
|
+
inputs=inputs,
|
|
3993
|
+
outputs=outputs,
|
|
3994
|
+
workdir=workdir,
|
|
3995
|
+
labels=labels,
|
|
3996
|
+
base_task=base_task,
|
|
3997
|
+
watch=watch,
|
|
3998
|
+
local=local,
|
|
3999
|
+
verbose=verbose,
|
|
4000
|
+
selector=selector,
|
|
4001
|
+
project_object=self,
|
|
4002
|
+
auto_build=auto_build,
|
|
4003
|
+
schedule=schedule,
|
|
4004
|
+
output_path=output_path,
|
|
4005
|
+
notifications=notifications,
|
|
4006
|
+
returns=returns,
|
|
4007
|
+
builder_env=builder_env,
|
|
4008
|
+
reset_on_run=reset_on_run,
|
|
4009
|
+
)
|
|
3971
4010
|
|
|
3972
4011
|
def build_function(
|
|
3973
4012
|
self,
|
|
@@ -4002,8 +4041,10 @@ class MlrunProject(ModelObj):
|
|
|
4002
4041
|
e.g. builder_env={"GIT_TOKEN": token}, does not work yet in KFP
|
|
4003
4042
|
:param overwrite_build_params: Overwrite existing build configuration (currently applies to
|
|
4004
4043
|
requirements and commands)
|
|
4044
|
+
|
|
4005
4045
|
* False: The new params are merged with the existing
|
|
4006
4046
|
* True: The existing params are replaced by the new ones
|
|
4047
|
+
|
|
4007
4048
|
:param extra_args: A string containing additional builder arguments in the format of command-line options,
|
|
4008
4049
|
e.g. extra_args="--skip-tls-verify --build-arg A=val"
|
|
4009
4050
|
:param force_build: force building the image, even when no changes were made
|
|
@@ -4054,8 +4095,10 @@ class MlrunProject(ModelObj):
|
|
|
4054
4095
|
:param requirements_file: requirements file to install on the built image
|
|
4055
4096
|
:param overwrite_build_params: Overwrite existing build configuration (currently applies to
|
|
4056
4097
|
requirements and commands)
|
|
4098
|
+
|
|
4057
4099
|
* False: The new params are merged with the existing
|
|
4058
4100
|
* True: The existing params are replaced by the new ones
|
|
4101
|
+
|
|
4059
4102
|
:param builder_env: Kaniko builder pod env vars dict (for config/credentials)
|
|
4060
4103
|
e.g. builder_env={"GIT_TOKEN": token}, does not work yet in KFP
|
|
4061
4104
|
:param extra_args: A string containing additional builder arguments in the format of command-line options,
|
|
@@ -4127,8 +4170,10 @@ class MlrunProject(ModelObj):
|
|
|
4127
4170
|
e.g. builder_env={"GIT_TOKEN": token}, does not work yet in KFP
|
|
4128
4171
|
:param overwrite_build_params: Overwrite existing build configuration (currently applies to
|
|
4129
4172
|
requirements and commands)
|
|
4173
|
+
|
|
4130
4174
|
* False: The new params are merged with the existing
|
|
4131
4175
|
* True: The existing params are replaced by the new ones
|
|
4176
|
+
|
|
4132
4177
|
:param extra_args: A string containing additional builder arguments in the format of command-line options,
|
|
4133
4178
|
e.g. extra_args="--skip-tls-verify --build-arg A=val"
|
|
4134
4179
|
:param target_dir: Path on the image where source code would be extracted (by default `/home/mlrun_code`)
|
|
@@ -4306,12 +4351,14 @@ class MlrunProject(ModelObj):
|
|
|
4306
4351
|
``my_Name_1`` or ``surname``.
|
|
4307
4352
|
:param tag: Return artifacts assigned this tag.
|
|
4308
4353
|
:param labels: Filter artifacts by label key-value pairs or key existence. This can be provided as:
|
|
4309
|
-
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4354
|
+
|
|
4355
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
4356
|
+
or `{"label": None}` to check for key existence.
|
|
4357
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
4358
|
+
or just `"label"` for key existence.
|
|
4359
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
4360
|
+
the specified key-value pairs or key existence.
|
|
4361
|
+
|
|
4315
4362
|
:param since: Not in use in :py:class:`HTTPRunDB`.
|
|
4316
4363
|
:param until: Not in use in :py:class:`HTTPRunDB`.
|
|
4317
4364
|
:param iter: Return artifacts from a specific iteration (where ``iter=0`` means the root iteration). If
|
|
@@ -4322,7 +4369,7 @@ class MlrunProject(ModelObj):
|
|
|
4322
4369
|
:param kind: Return artifacts of the requested kind.
|
|
4323
4370
|
:param category: Return artifacts of the requested category.
|
|
4324
4371
|
:param tree: Return artifacts of the requested tree.
|
|
4325
|
-
:param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.
|
|
4372
|
+
:param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
|
|
4326
4373
|
:param format_: The format in which to return the artifacts. Default is 'full'.
|
|
4327
4374
|
:param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
|
|
4328
4375
|
parameter must be provided as well.
|
|
@@ -4335,9 +4382,9 @@ class MlrunProject(ModelObj):
|
|
|
4335
4382
|
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
4336
4383
|
|
|
4337
4384
|
if limit:
|
|
4338
|
-
# TODO: Remove this in 1.
|
|
4385
|
+
# TODO: Remove this in 1.11.0
|
|
4339
4386
|
warnings.warn(
|
|
4340
|
-
"'limit' is deprecated and will be removed in 1.
|
|
4387
|
+
"'limit' is deprecated and will be removed in 1.11.0. Use 'page' and 'page_size' instead.",
|
|
4341
4388
|
FutureWarning,
|
|
4342
4389
|
)
|
|
4343
4390
|
|
|
@@ -4453,12 +4500,14 @@ class MlrunProject(ModelObj):
|
|
|
4453
4500
|
``my_Name_1`` or ``surname``.
|
|
4454
4501
|
:param tag: Return artifacts assigned this tag.
|
|
4455
4502
|
:param labels: Filter model artifacts by label key-value pairs or key existence. This can be provided as:
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
|
|
4461
|
-
|
|
4503
|
+
|
|
4504
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
4505
|
+
or `{"label": None}` to check for key existence.
|
|
4506
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
4507
|
+
or just `"label"` for key existence.
|
|
4508
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
4509
|
+
the specified key-value pairs or key existence.
|
|
4510
|
+
|
|
4462
4511
|
:param since: Not in use in :py:class:`HTTPRunDB`.
|
|
4463
4512
|
:param until: Not in use in :py:class:`HTTPRunDB`.
|
|
4464
4513
|
:param iter: Return artifacts from a specific iteration (where ``iter=0`` means the root iteration). If
|
|
@@ -4564,12 +4613,14 @@ class MlrunProject(ModelObj):
|
|
|
4564
4613
|
:param name: Return only functions with a specific name.
|
|
4565
4614
|
:param tag: Return function versions with specific tags. To return only tagged functions, set tag to ``"*"``.
|
|
4566
4615
|
:param labels: Filter functions by label key-value pairs or key existence. This can be provided as:
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4616
|
+
|
|
4617
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
4618
|
+
or `{"label": None}` to check for key existence.
|
|
4619
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
4620
|
+
or just `"label"` for key existence.
|
|
4621
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
4622
|
+
the specified key-value pairs or key existence.
|
|
4623
|
+
|
|
4573
4624
|
:param kind: Return functions of the specified kind. If not provided, all function kinds will be returned.
|
|
4574
4625
|
:param format_: The format in which to return the functions. Default is 'full'.
|
|
4575
4626
|
:returns: List of function objects.
|
|
@@ -4663,12 +4714,14 @@ class MlrunProject(ModelObj):
|
|
|
4663
4714
|
:param name: Return only functions with a specific name.
|
|
4664
4715
|
:param tag: Return function versions with specific tags.
|
|
4665
4716
|
:param labels: Filter functions by label key-value pairs or key existence. This can be provided as:
|
|
4666
|
-
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
|
|
4671
|
-
|
|
4717
|
+
|
|
4718
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
4719
|
+
or `{"label": None}` to check for key existence.
|
|
4720
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
4721
|
+
or just `"label"` for key existence.
|
|
4722
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
4723
|
+
the specified key-value pairs or key existence.
|
|
4724
|
+
|
|
4672
4725
|
:returns: List of function objects.
|
|
4673
4726
|
"""
|
|
4674
4727
|
|
|
@@ -4724,12 +4777,14 @@ class MlrunProject(ModelObj):
|
|
|
4724
4777
|
:param name: Name of the run to retrieve.
|
|
4725
4778
|
:param uid: Unique ID of the run.
|
|
4726
4779
|
:param labels: Filter runs by label key-value pairs or key existence. This can be provided as:
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
4731
|
-
|
|
4732
|
-
|
|
4780
|
+
|
|
4781
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
4782
|
+
or `{"label": None}` to check for key existence.
|
|
4783
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
4784
|
+
or just `"label"` for key existence.
|
|
4785
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
4786
|
+
the specified key-value pairs or key existence.
|
|
4787
|
+
|
|
4733
4788
|
:param state: Deprecated - List only runs whose state is specified.
|
|
4734
4789
|
:param states: List only runs whose state is one of the provided states.
|
|
4735
4790
|
:param sort: Whether to sort the result according to their start time. Otherwise, results will be
|
mlrun/render.py
CHANGED
|
@@ -361,9 +361,6 @@ def get_tblframe(df, display, classes=None):
|
|
|
361
361
|
return ipython_display(html, display)
|
|
362
362
|
|
|
363
363
|
|
|
364
|
-
uid_template = '<div title="{}"><a href="{}/{}/{}/jobs/monitor/{}/overview" target="_blank" >...{}</a></div>'
|
|
365
|
-
|
|
366
|
-
|
|
367
364
|
def runs_to_html(
|
|
368
365
|
df: pd.DataFrame,
|
|
369
366
|
display: bool = True,
|
|
@@ -379,15 +376,14 @@ def runs_to_html(
|
|
|
379
376
|
df["results"] = df["results"].apply(dict_html)
|
|
380
377
|
df["start"] = df["start"].apply(time_str)
|
|
381
378
|
df["parameters"] = df["parameters"].apply(dict_html)
|
|
379
|
+
uid_template = '<div title="{}"><a href="{}" target="_blank" >...{}</a></div>'
|
|
380
|
+
|
|
382
381
|
if config.resolve_ui_url():
|
|
383
382
|
df["uid"] = df.apply(
|
|
384
383
|
lambda x: uid_template.format(
|
|
385
|
-
x
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
x.project,
|
|
389
|
-
x.uid,
|
|
390
|
-
x.uid[-8:],
|
|
384
|
+
x["uid"],
|
|
385
|
+
mlrun.utils.get_run_url(x["project"], x["uid"], x["name"]),
|
|
386
|
+
x["uid"][-8:],
|
|
391
387
|
),
|
|
392
388
|
axis=1,
|
|
393
389
|
)
|
mlrun/run.py
CHANGED
|
@@ -637,7 +637,7 @@ def code_to_function(
|
|
|
637
637
|
- databricks: run code on Databricks cluster (python scripts, Spark etc.)
|
|
638
638
|
- application: run a long living application (e.g. a web server, UI, etc.)
|
|
639
639
|
|
|
640
|
-
Learn more about
|
|
640
|
+
Learn more about :doc:`../../concepts/functions-overview`
|
|
641
641
|
|
|
642
642
|
:param name: function name, typically best to use hyphen-case
|
|
643
643
|
:param project: project used to namespace the function, defaults to 'default'
|
mlrun/runtimes/base.py
CHANGED
|
@@ -345,6 +345,7 @@ class BaseRuntime(ModelObj):
|
|
|
345
345
|
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
346
346
|
state_thresholds: Optional[dict[str, int]] = None,
|
|
347
347
|
reset_on_run: Optional[bool] = None,
|
|
348
|
+
output_path: Optional[str] = "",
|
|
348
349
|
**launcher_kwargs,
|
|
349
350
|
) -> RunObject:
|
|
350
351
|
"""
|
|
@@ -358,9 +359,9 @@ class BaseRuntime(ModelObj):
|
|
|
358
359
|
:param inputs: Input objects to pass to the handler. Type hints can be given so the input will be parsed
|
|
359
360
|
during runtime from `mlrun.DataItem` to the given type hint. The type hint can be given
|
|
360
361
|
in the key field of the dictionary after a colon, e.g: "<key> : <type_hint>".
|
|
361
|
-
:param out_path: Default artifact output path.
|
|
362
|
-
:param artifact_path: Default artifact output path (will replace out_path).
|
|
363
|
-
:param workdir:
|
|
362
|
+
:param out_path: (deprecated) Default artifact output path.
|
|
363
|
+
:param artifact_path: (deprecated) Default artifact output path (will replace out_path).
|
|
364
|
+
:param workdir: Working directory of the executed job and the default path for artifact inputs
|
|
364
365
|
:param watch: Watch/follow run log.
|
|
365
366
|
:param schedule: ScheduleCronTrigger class instance or a standard crontab expression string
|
|
366
367
|
(which will be converted to the class using its `from_crontab` constructor),
|
|
@@ -402,8 +403,18 @@ class BaseRuntime(ModelObj):
|
|
|
402
403
|
:param reset_on_run: When True, function python modules would reload prior to code execution.
|
|
403
404
|
This ensures latest code changes are executed. This argument must be used in
|
|
404
405
|
conjunction with the local=True argument.
|
|
406
|
+
:param output_path: Default artifact output path.
|
|
405
407
|
:return: Run context object (RunObject) with run metadata, results and status
|
|
406
408
|
"""
|
|
409
|
+
if artifact_path or out_path:
|
|
410
|
+
deprecated_param = "artifact_path" if artifact_path else "out_path"
|
|
411
|
+
warnings.warn(
|
|
412
|
+
f"'{deprecated_param}' parameter is deprecated in 1.10.0 and will be removed in 1.12.0, "
|
|
413
|
+
"use 'output_path' instead.",
|
|
414
|
+
# TODO: Remove this in 1.12.0
|
|
415
|
+
FutureWarning,
|
|
416
|
+
)
|
|
417
|
+
output_path = output_path or out_path or artifact_path
|
|
407
418
|
launcher = mlrun.launcher.factory.LauncherFactory().create_launcher(
|
|
408
419
|
self._is_remote, local=local, **launcher_kwargs
|
|
409
420
|
)
|
|
@@ -415,9 +426,8 @@ class BaseRuntime(ModelObj):
|
|
|
415
426
|
project=project,
|
|
416
427
|
params=params,
|
|
417
428
|
inputs=inputs,
|
|
418
|
-
out_path=out_path,
|
|
419
429
|
workdir=workdir,
|
|
420
|
-
|
|
430
|
+
output_path=output_path,
|
|
421
431
|
watch=watch,
|
|
422
432
|
schedule=schedule,
|
|
423
433
|
hyperparams=hyperparams,
|
|
@@ -489,7 +499,7 @@ class BaseRuntime(ModelObj):
|
|
|
489
499
|
def _store_function(self, runspec, meta, db):
|
|
490
500
|
meta.labels["kind"] = self.kind
|
|
491
501
|
mlrun.runtimes.utils.enrich_run_labels(
|
|
492
|
-
meta.labels, [
|
|
502
|
+
meta.labels, [mlrun_constants.MLRunInternalLabels.owner]
|
|
493
503
|
)
|
|
494
504
|
if runspec.spec.output_path:
|
|
495
505
|
runspec.spec.output_path = runspec.spec.output_path.replace(
|
|
@@ -711,7 +721,7 @@ class BaseRuntime(ModelObj):
|
|
|
711
721
|
given in the key field of the dictionary after a colon, e.g: "<key> : <type_hint>".
|
|
712
722
|
:param outputs: list of outputs which can pass in the workflow
|
|
713
723
|
:param artifact_path: default artifact output path (replace out_path)
|
|
714
|
-
:param workdir: default
|
|
724
|
+
:param workdir: working directory of the executed job and the default path for artifact inputs
|
|
715
725
|
:param image: container image to use
|
|
716
726
|
:param labels: labels to tag the job/run with ({key:val, ..})
|
|
717
727
|
:param use_db: save function spec in the db (vs the workflow file)
|
mlrun/runtimes/daskjob.py
CHANGED
|
@@ -495,6 +495,7 @@ class DaskCluster(KubejobRuntime):
|
|
|
495
495
|
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
496
496
|
state_thresholds: Optional[dict[str, int]] = None,
|
|
497
497
|
reset_on_run: Optional[bool] = None,
|
|
498
|
+
output_path: Optional[str] = "",
|
|
498
499
|
**launcher_kwargs,
|
|
499
500
|
) -> RunObject:
|
|
500
501
|
if state_thresholds:
|
|
@@ -510,7 +511,7 @@ class DaskCluster(KubejobRuntime):
|
|
|
510
511
|
inputs=inputs,
|
|
511
512
|
out_path=out_path,
|
|
512
513
|
workdir=workdir,
|
|
513
|
-
|
|
514
|
+
output_path=output_path,
|
|
514
515
|
watch=watch,
|
|
515
516
|
schedule=schedule,
|
|
516
517
|
hyperparams=hyperparams,
|
|
@@ -234,6 +234,7 @@ def run_mlrun_databricks_job(context,task_parameters: dict, **kwargs):
|
|
|
234
234
|
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
235
235
|
state_thresholds: Optional[dict[str, int]] = None,
|
|
236
236
|
reset_on_run: Optional[bool] = None,
|
|
237
|
+
output_path: Optional[str] = "",
|
|
237
238
|
**launcher_kwargs,
|
|
238
239
|
) -> RunObject:
|
|
239
240
|
if local:
|
|
@@ -247,7 +248,7 @@ def run_mlrun_databricks_job(context,task_parameters: dict, **kwargs):
|
|
|
247
248
|
inputs=inputs,
|
|
248
249
|
out_path=out_path,
|
|
249
250
|
workdir=workdir,
|
|
250
|
-
|
|
251
|
+
output_path=output_path,
|
|
251
252
|
watch=watch,
|
|
252
253
|
schedule=schedule,
|
|
253
254
|
hyperparams=hyperparams,
|
mlrun/runtimes/mounts.py
CHANGED
|
@@ -352,10 +352,12 @@ def auto_mount(
|
|
|
352
352
|
"""Choose the mount based on env variables and params
|
|
353
353
|
|
|
354
354
|
Volume will be selected by the following order:
|
|
355
|
+
|
|
355
356
|
- k8s PVC volume when both pvc_name and volume_mount_path are set
|
|
356
357
|
- k8s PVC volume when env var is set: MLRUN_PVC_MOUNT=<pvc-name>:<mount-path>
|
|
357
358
|
- k8s PVC volume if it's configured as the auto mount type
|
|
358
359
|
- iguazio v3io volume when V3IO_ACCESS_KEY and V3IO_USERNAME env vars are set
|
|
360
|
+
|
|
359
361
|
"""
|
|
360
362
|
if pvc_name and volume_mount_path:
|
|
361
363
|
return mount_pvc(
|
|
@@ -651,6 +651,11 @@ class RemoteRuntime(KubeResource):
|
|
|
651
651
|
logger.info("Starting remote function deploy")
|
|
652
652
|
data = db.deploy_nuclio_function(func=self, builder_env=builder_env)
|
|
653
653
|
self.status = data["data"].get("status")
|
|
654
|
+
|
|
655
|
+
# Extract the spec to avoid overwriting server-side updates during the later save in
|
|
656
|
+
# _enrich_command_from_status.
|
|
657
|
+
self.spec = data["data"].get("spec")
|
|
658
|
+
|
|
654
659
|
self._update_credentials_from_remote_build(data["data"])
|
|
655
660
|
|
|
656
661
|
# when a function is deployed, we wait for it to be ready by default
|
mlrun/runtimes/nuclio/serving.py
CHANGED
|
@@ -800,7 +800,7 @@ class ServingRuntime(RemoteRuntime):
|
|
|
800
800
|
serving_fn.add_model(
|
|
801
801
|
"my-classifier",
|
|
802
802
|
model_path=model_path,
|
|
803
|
-
class_name="mlrun.frameworks.sklearn.
|
|
803
|
+
class_name="mlrun.frameworks.sklearn.SKLearnModelServer",
|
|
804
804
|
)
|
|
805
805
|
serving_fn.plot(rankdir="LR")
|
|
806
806
|
|