mlrun 1.6.4rc7__py3-none-any.whl → 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +11 -1
- mlrun/__main__.py +40 -122
- mlrun/alerts/__init__.py +15 -0
- mlrun/alerts/alert.py +248 -0
- mlrun/api/schemas/__init__.py +5 -4
- mlrun/artifacts/__init__.py +8 -3
- mlrun/artifacts/base.py +47 -257
- mlrun/artifacts/dataset.py +11 -192
- mlrun/artifacts/manager.py +79 -47
- mlrun/artifacts/model.py +31 -159
- mlrun/artifacts/plots.py +23 -380
- mlrun/common/constants.py +74 -1
- mlrun/common/db/sql_session.py +5 -5
- mlrun/common/formatters/__init__.py +21 -0
- mlrun/common/formatters/artifact.py +45 -0
- mlrun/common/formatters/base.py +113 -0
- mlrun/common/formatters/feature_set.py +33 -0
- mlrun/common/formatters/function.py +46 -0
- mlrun/common/formatters/pipeline.py +53 -0
- mlrun/common/formatters/project.py +51 -0
- mlrun/common/formatters/run.py +29 -0
- mlrun/common/helpers.py +12 -3
- mlrun/common/model_monitoring/helpers.py +9 -5
- mlrun/{runtimes → common/runtimes}/constants.py +37 -9
- mlrun/common/schemas/__init__.py +31 -5
- mlrun/common/schemas/alert.py +202 -0
- mlrun/common/schemas/api_gateway.py +196 -0
- mlrun/common/schemas/artifact.py +25 -4
- mlrun/common/schemas/auth.py +16 -5
- mlrun/common/schemas/background_task.py +1 -1
- mlrun/common/schemas/client_spec.py +4 -2
- mlrun/common/schemas/common.py +7 -4
- mlrun/common/schemas/constants.py +3 -0
- mlrun/common/schemas/feature_store.py +74 -44
- mlrun/common/schemas/frontend_spec.py +15 -7
- mlrun/common/schemas/function.py +12 -1
- mlrun/common/schemas/hub.py +11 -18
- mlrun/common/schemas/memory_reports.py +2 -2
- mlrun/common/schemas/model_monitoring/__init__.py +20 -4
- mlrun/common/schemas/model_monitoring/constants.py +123 -42
- mlrun/common/schemas/model_monitoring/grafana.py +13 -9
- mlrun/common/schemas/model_monitoring/model_endpoints.py +101 -54
- mlrun/common/schemas/notification.py +71 -14
- mlrun/common/schemas/object.py +2 -2
- mlrun/{model_monitoring/controller_handler.py → common/schemas/pagination.py} +9 -12
- mlrun/common/schemas/pipeline.py +8 -1
- mlrun/common/schemas/project.py +69 -18
- mlrun/common/schemas/runs.py +7 -1
- mlrun/common/schemas/runtime_resource.py +8 -12
- mlrun/common/schemas/schedule.py +4 -4
- mlrun/common/schemas/tag.py +1 -2
- mlrun/common/schemas/workflow.py +12 -4
- mlrun/common/types.py +14 -1
- mlrun/config.py +154 -69
- mlrun/data_types/data_types.py +6 -1
- mlrun/data_types/spark.py +2 -2
- mlrun/data_types/to_pandas.py +67 -37
- mlrun/datastore/__init__.py +6 -8
- mlrun/datastore/alibaba_oss.py +131 -0
- mlrun/datastore/azure_blob.py +143 -42
- mlrun/datastore/base.py +102 -58
- mlrun/datastore/datastore.py +34 -13
- mlrun/datastore/datastore_profile.py +146 -20
- mlrun/datastore/dbfs_store.py +3 -7
- mlrun/datastore/filestore.py +1 -4
- mlrun/datastore/google_cloud_storage.py +97 -33
- mlrun/datastore/hdfs.py +56 -0
- mlrun/datastore/inmem.py +6 -3
- mlrun/datastore/redis.py +7 -2
- mlrun/datastore/s3.py +34 -12
- mlrun/datastore/snowflake_utils.py +45 -0
- mlrun/datastore/sources.py +303 -111
- mlrun/datastore/spark_utils.py +31 -2
- mlrun/datastore/store_resources.py +9 -7
- mlrun/datastore/storeytargets.py +151 -0
- mlrun/datastore/targets.py +453 -176
- mlrun/datastore/utils.py +72 -58
- mlrun/datastore/v3io.py +6 -1
- mlrun/db/base.py +274 -41
- mlrun/db/factory.py +1 -1
- mlrun/db/httpdb.py +893 -225
- mlrun/db/nopdb.py +291 -33
- mlrun/errors.py +36 -6
- mlrun/execution.py +115 -42
- mlrun/feature_store/__init__.py +0 -2
- mlrun/feature_store/api.py +65 -73
- mlrun/feature_store/common.py +7 -12
- mlrun/feature_store/feature_set.py +76 -55
- mlrun/feature_store/feature_vector.py +39 -31
- mlrun/feature_store/ingestion.py +7 -6
- mlrun/feature_store/retrieval/base.py +16 -11
- mlrun/feature_store/retrieval/dask_merger.py +2 -0
- mlrun/feature_store/retrieval/job.py +13 -4
- mlrun/feature_store/retrieval/local_merger.py +2 -0
- mlrun/feature_store/retrieval/spark_merger.py +24 -32
- mlrun/feature_store/steps.py +45 -34
- mlrun/features.py +11 -21
- mlrun/frameworks/_common/artifacts_library.py +9 -9
- mlrun/frameworks/_common/mlrun_interface.py +5 -5
- mlrun/frameworks/_common/model_handler.py +48 -48
- mlrun/frameworks/_common/plan.py +5 -6
- mlrun/frameworks/_common/producer.py +3 -4
- mlrun/frameworks/_common/utils.py +5 -5
- mlrun/frameworks/_dl_common/loggers/logger.py +6 -7
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +9 -9
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +23 -47
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -2
- mlrun/frameworks/_ml_common/loggers/logger.py +3 -4
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +4 -5
- mlrun/frameworks/_ml_common/model_handler.py +24 -24
- mlrun/frameworks/_ml_common/pkl_model_server.py +2 -2
- mlrun/frameworks/_ml_common/plan.py +2 -2
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +2 -3
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +2 -3
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
- mlrun/frameworks/_ml_common/utils.py +4 -4
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +9 -9
- mlrun/frameworks/huggingface/model_server.py +4 -4
- mlrun/frameworks/lgbm/__init__.py +33 -33
- mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -5
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -5
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -3
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +6 -6
- mlrun/frameworks/lgbm/model_handler.py +10 -10
- mlrun/frameworks/lgbm/model_server.py +6 -6
- mlrun/frameworks/lgbm/utils.py +5 -5
- mlrun/frameworks/onnx/dataset.py +8 -8
- mlrun/frameworks/onnx/mlrun_interface.py +3 -3
- mlrun/frameworks/onnx/model_handler.py +6 -6
- mlrun/frameworks/onnx/model_server.py +7 -7
- mlrun/frameworks/parallel_coordinates.py +6 -6
- mlrun/frameworks/pytorch/__init__.py +18 -18
- mlrun/frameworks/pytorch/callbacks/callback.py +4 -5
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +17 -17
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +11 -11
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +23 -29
- mlrun/frameworks/pytorch/callbacks_handler.py +38 -38
- mlrun/frameworks/pytorch/mlrun_interface.py +20 -20
- mlrun/frameworks/pytorch/model_handler.py +17 -17
- mlrun/frameworks/pytorch/model_server.py +7 -7
- mlrun/frameworks/sklearn/__init__.py +13 -13
- mlrun/frameworks/sklearn/estimator.py +4 -4
- mlrun/frameworks/sklearn/metrics_library.py +14 -14
- mlrun/frameworks/sklearn/mlrun_interface.py +16 -9
- mlrun/frameworks/sklearn/model_handler.py +2 -2
- mlrun/frameworks/tf_keras/__init__.py +10 -7
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +15 -15
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +11 -11
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +19 -23
- mlrun/frameworks/tf_keras/mlrun_interface.py +9 -11
- mlrun/frameworks/tf_keras/model_handler.py +14 -14
- mlrun/frameworks/tf_keras/model_server.py +6 -6
- mlrun/frameworks/xgboost/__init__.py +13 -13
- mlrun/frameworks/xgboost/model_handler.py +6 -6
- mlrun/k8s_utils.py +61 -17
- mlrun/launcher/__init__.py +1 -1
- mlrun/launcher/base.py +16 -15
- mlrun/launcher/client.py +13 -11
- mlrun/launcher/factory.py +1 -1
- mlrun/launcher/local.py +23 -13
- mlrun/launcher/remote.py +17 -10
- mlrun/lists.py +7 -6
- mlrun/model.py +478 -103
- mlrun/model_monitoring/__init__.py +1 -1
- mlrun/model_monitoring/api.py +163 -371
- mlrun/{runtimes/mpijob/v1alpha1.py → model_monitoring/applications/__init__.py} +9 -15
- mlrun/model_monitoring/applications/_application_steps.py +188 -0
- mlrun/model_monitoring/applications/base.py +108 -0
- mlrun/model_monitoring/applications/context.py +341 -0
- mlrun/model_monitoring/{evidently_application.py → applications/evidently_base.py} +27 -22
- mlrun/model_monitoring/applications/histogram_data_drift.py +354 -0
- mlrun/model_monitoring/applications/results.py +99 -0
- mlrun/model_monitoring/controller.py +131 -278
- mlrun/model_monitoring/db/__init__.py +18 -0
- mlrun/model_monitoring/db/stores/__init__.py +136 -0
- mlrun/model_monitoring/db/stores/base/__init__.py +15 -0
- mlrun/model_monitoring/db/stores/base/store.py +213 -0
- mlrun/model_monitoring/db/stores/sqldb/__init__.py +13 -0
- mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +71 -0
- mlrun/model_monitoring/db/stores/sqldb/models/base.py +190 -0
- mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +103 -0
- mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py} +19 -13
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +659 -0
- mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +13 -0
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +726 -0
- mlrun/model_monitoring/db/tsdb/__init__.py +105 -0
- mlrun/model_monitoring/db/tsdb/base.py +448 -0
- mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
- mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +279 -0
- mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +42 -0
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +507 -0
- mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +158 -0
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +849 -0
- mlrun/model_monitoring/features_drift_table.py +134 -106
- mlrun/model_monitoring/helpers.py +199 -55
- mlrun/model_monitoring/metrics/__init__.py +13 -0
- mlrun/model_monitoring/metrics/histogram_distance.py +127 -0
- mlrun/model_monitoring/model_endpoint.py +3 -2
- mlrun/model_monitoring/stream_processing.py +131 -398
- mlrun/model_monitoring/tracking_policy.py +9 -2
- mlrun/model_monitoring/writer.py +161 -125
- mlrun/package/__init__.py +6 -6
- mlrun/package/context_handler.py +5 -5
- mlrun/package/packager.py +7 -7
- mlrun/package/packagers/default_packager.py +8 -8
- mlrun/package/packagers/numpy_packagers.py +15 -15
- mlrun/package/packagers/pandas_packagers.py +5 -5
- mlrun/package/packagers/python_standard_library_packagers.py +10 -10
- mlrun/package/packagers_manager.py +19 -23
- mlrun/package/utils/_formatter.py +6 -6
- mlrun/package/utils/_pickler.py +2 -2
- mlrun/package/utils/_supported_format.py +4 -4
- mlrun/package/utils/log_hint_utils.py +2 -2
- mlrun/package/utils/type_hint_utils.py +4 -9
- mlrun/platforms/__init__.py +11 -10
- mlrun/platforms/iguazio.py +24 -203
- mlrun/projects/operations.py +52 -25
- mlrun/projects/pipelines.py +191 -197
- mlrun/projects/project.py +1227 -400
- mlrun/render.py +16 -19
- mlrun/run.py +209 -184
- mlrun/runtimes/__init__.py +83 -15
- mlrun/runtimes/base.py +51 -35
- mlrun/runtimes/daskjob.py +17 -10
- mlrun/runtimes/databricks_job/databricks_cancel_task.py +1 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +8 -7
- mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
- mlrun/runtimes/funcdoc.py +1 -29
- mlrun/runtimes/function_reference.py +1 -1
- mlrun/runtimes/kubejob.py +34 -128
- mlrun/runtimes/local.py +40 -11
- mlrun/runtimes/mpijob/__init__.py +0 -20
- mlrun/runtimes/mpijob/abstract.py +9 -10
- mlrun/runtimes/mpijob/v1.py +1 -1
- mlrun/{model_monitoring/stores/models/sqlite.py → runtimes/nuclio/__init__.py} +7 -9
- mlrun/runtimes/nuclio/api_gateway.py +769 -0
- mlrun/runtimes/nuclio/application/__init__.py +15 -0
- mlrun/runtimes/nuclio/application/application.py +758 -0
- mlrun/runtimes/nuclio/application/reverse_proxy.go +95 -0
- mlrun/runtimes/{function.py → nuclio/function.py} +200 -83
- mlrun/runtimes/{nuclio.py → nuclio/nuclio.py} +6 -6
- mlrun/runtimes/{serving.py → nuclio/serving.py} +65 -68
- mlrun/runtimes/pod.py +281 -101
- mlrun/runtimes/remotesparkjob.py +12 -9
- mlrun/runtimes/sparkjob/spark3job.py +67 -51
- mlrun/runtimes/utils.py +41 -75
- mlrun/secrets.py +9 -5
- mlrun/serving/__init__.py +8 -1
- mlrun/serving/remote.py +2 -7
- mlrun/serving/routers.py +85 -69
- mlrun/serving/server.py +69 -44
- mlrun/serving/states.py +209 -36
- mlrun/serving/utils.py +22 -14
- mlrun/serving/v1_serving.py +6 -7
- mlrun/serving/v2_serving.py +129 -54
- mlrun/track/tracker.py +2 -1
- mlrun/track/tracker_manager.py +3 -3
- mlrun/track/trackers/mlflow_tracker.py +6 -2
- mlrun/utils/async_http.py +6 -8
- mlrun/utils/azure_vault.py +1 -1
- mlrun/utils/clones.py +1 -2
- mlrun/utils/condition_evaluator.py +3 -3
- mlrun/utils/db.py +21 -3
- mlrun/utils/helpers.py +405 -225
- mlrun/utils/http.py +3 -6
- mlrun/utils/logger.py +112 -16
- mlrun/utils/notifications/notification/__init__.py +17 -13
- mlrun/utils/notifications/notification/base.py +50 -2
- mlrun/utils/notifications/notification/console.py +2 -0
- mlrun/utils/notifications/notification/git.py +24 -1
- mlrun/utils/notifications/notification/ipython.py +3 -1
- mlrun/utils/notifications/notification/slack.py +96 -21
- mlrun/utils/notifications/notification/webhook.py +59 -2
- mlrun/utils/notifications/notification_pusher.py +149 -30
- mlrun/utils/regex.py +9 -0
- mlrun/utils/retryer.py +208 -0
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/v3io_clients.py +4 -6
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +2 -6
- mlrun-1.7.0.dist-info/METADATA +378 -0
- mlrun-1.7.0.dist-info/RECORD +351 -0
- {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/WHEEL +1 -1
- mlrun/feature_store/retrieval/conversion.py +0 -273
- mlrun/kfpops.py +0 -868
- mlrun/model_monitoring/application.py +0 -310
- mlrun/model_monitoring/batch.py +0 -1095
- mlrun/model_monitoring/prometheus.py +0 -219
- mlrun/model_monitoring/stores/__init__.py +0 -111
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +0 -576
- mlrun/model_monitoring/stores/model_endpoint_store.py +0 -147
- mlrun/model_monitoring/stores/models/__init__.py +0 -27
- mlrun/model_monitoring/stores/models/base.py +0 -84
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -384
- mlrun/platforms/other.py +0 -306
- mlrun-1.6.4rc7.dist-info/METADATA +0 -272
- mlrun-1.6.4rc7.dist-info/RECORD +0 -314
- {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/LICENSE +0 -0
- {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/entry_points.txt +0 -0
- {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/top_level.txt +0 -0
mlrun/runtimes/__init__.py
CHANGED
|
@@ -26,21 +26,31 @@ __all__ = [
|
|
|
26
26
|
"Spark3Runtime",
|
|
27
27
|
"DatabricksRuntime",
|
|
28
28
|
"KubeResource",
|
|
29
|
+
"ApplicationRuntime",
|
|
30
|
+
"MpiRuntimeV1",
|
|
29
31
|
]
|
|
30
32
|
|
|
33
|
+
import typing
|
|
34
|
+
|
|
31
35
|
from mlrun.runtimes.utils import resolve_spark_operator_version
|
|
32
36
|
|
|
37
|
+
from ..common.runtimes.constants import MPIJobCRDVersions
|
|
33
38
|
from .base import BaseRuntime, RunError, RuntimeClassMode # noqa
|
|
34
|
-
from .constants import MPIJobCRDVersions
|
|
35
39
|
from .daskjob import DaskCluster # noqa
|
|
36
40
|
from .databricks_job.databricks_runtime import DatabricksRuntime
|
|
37
|
-
from .function import RemoteRuntime
|
|
38
41
|
from .kubejob import KubejobRuntime, KubeResource # noqa
|
|
39
42
|
from .local import HandlerRuntime, LocalRuntime # noqa
|
|
40
|
-
from .mpijob import
|
|
41
|
-
from .nuclio import
|
|
43
|
+
from .mpijob import MpiRuntimeV1 # noqa
|
|
44
|
+
from .nuclio import (
|
|
45
|
+
RemoteRuntime,
|
|
46
|
+
ServingRuntime,
|
|
47
|
+
new_v2_model_server,
|
|
48
|
+
nuclio_init_hook,
|
|
49
|
+
)
|
|
50
|
+
from .nuclio.api_gateway import APIGateway
|
|
51
|
+
from .nuclio.application import ApplicationRuntime
|
|
52
|
+
from .nuclio.serving import serving_subkind
|
|
42
53
|
from .remotesparkjob import RemoteSparkRuntime
|
|
43
|
-
from .serving import ServingRuntime, new_v2_model_server
|
|
44
54
|
from .sparkjob import Spark3Runtime
|
|
45
55
|
|
|
46
56
|
# for legacy imports (MLModelServer moved from here to /serving)
|
|
@@ -86,7 +96,7 @@ def new_model_server(
|
|
|
86
96
|
)
|
|
87
97
|
|
|
88
98
|
|
|
89
|
-
class RuntimeKinds
|
|
99
|
+
class RuntimeKinds:
|
|
90
100
|
remote = "remote"
|
|
91
101
|
nuclio = "nuclio"
|
|
92
102
|
dask = "dask"
|
|
@@ -98,6 +108,7 @@ class RuntimeKinds(object):
|
|
|
98
108
|
local = "local"
|
|
99
109
|
handler = "handler"
|
|
100
110
|
databricks = "databricks"
|
|
111
|
+
application = "application"
|
|
101
112
|
|
|
102
113
|
@staticmethod
|
|
103
114
|
def all():
|
|
@@ -112,6 +123,7 @@ class RuntimeKinds(object):
|
|
|
112
123
|
RuntimeKinds.mpijob,
|
|
113
124
|
RuntimeKinds.local,
|
|
114
125
|
RuntimeKinds.databricks,
|
|
126
|
+
RuntimeKinds.application,
|
|
115
127
|
]
|
|
116
128
|
|
|
117
129
|
@staticmethod
|
|
@@ -144,6 +156,23 @@ class RuntimeKinds(object):
|
|
|
144
156
|
RuntimeKinds.remote,
|
|
145
157
|
RuntimeKinds.nuclio,
|
|
146
158
|
RuntimeKinds.serving,
|
|
159
|
+
RuntimeKinds.application,
|
|
160
|
+
]
|
|
161
|
+
|
|
162
|
+
@staticmethod
|
|
163
|
+
def pure_nuclio_deployed_runtimes():
|
|
164
|
+
return [
|
|
165
|
+
RuntimeKinds.remote,
|
|
166
|
+
RuntimeKinds.nuclio,
|
|
167
|
+
RuntimeKinds.serving,
|
|
168
|
+
]
|
|
169
|
+
|
|
170
|
+
@staticmethod
|
|
171
|
+
def handlerless_runtimes():
|
|
172
|
+
return [
|
|
173
|
+
RuntimeKinds.serving,
|
|
174
|
+
# Application runtime handler is internal reverse proxy
|
|
175
|
+
RuntimeKinds.application,
|
|
147
176
|
]
|
|
148
177
|
|
|
149
178
|
@staticmethod
|
|
@@ -154,7 +183,7 @@ class RuntimeKinds(object):
|
|
|
154
183
|
]
|
|
155
184
|
|
|
156
185
|
@staticmethod
|
|
157
|
-
def is_log_collectable_runtime(kind: str):
|
|
186
|
+
def is_log_collectable_runtime(kind: typing.Optional[str]):
|
|
158
187
|
"""
|
|
159
188
|
whether log collector can collect logs for that runtime
|
|
160
189
|
:param kind: kind name
|
|
@@ -165,13 +194,18 @@ class RuntimeKinds(object):
|
|
|
165
194
|
if RuntimeKinds.is_local_runtime(kind):
|
|
166
195
|
return False
|
|
167
196
|
|
|
168
|
-
if
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
197
|
+
if (
|
|
198
|
+
kind
|
|
199
|
+
not in [
|
|
200
|
+
# dask implementation is different from other runtimes, because few runs can be run against the same
|
|
201
|
+
# runtime resource, so collecting logs on that runtime resource won't be correct, the way we collect
|
|
202
|
+
# logs for dask is by using `log_std` on client side after we execute the code against the cluster,
|
|
203
|
+
# as submitting the run with the dask client will return the run stdout.
|
|
204
|
+
# For more information head to `DaskCluster._run`.
|
|
205
|
+
RuntimeKinds.dask
|
|
206
|
+
]
|
|
207
|
+
+ RuntimeKinds.nuclio_runtimes()
|
|
208
|
+
):
|
|
175
209
|
return True
|
|
176
210
|
|
|
177
211
|
return False
|
|
@@ -208,10 +242,43 @@ class RuntimeKinds(object):
|
|
|
208
242
|
# both spark and remote spark uses different mechanism for assigning images
|
|
209
243
|
return kind not in [RuntimeKinds.spark, RuntimeKinds.remotespark]
|
|
210
244
|
|
|
245
|
+
@staticmethod
|
|
246
|
+
def supports_from_notebook(kind):
|
|
247
|
+
return kind not in [RuntimeKinds.application]
|
|
248
|
+
|
|
249
|
+
@staticmethod
|
|
250
|
+
def resolve_nuclio_runtime(kind: str, sub_kind: str):
|
|
251
|
+
kind = kind.split(":")[0]
|
|
252
|
+
if kind not in RuntimeKinds.nuclio_runtimes():
|
|
253
|
+
raise ValueError(
|
|
254
|
+
f"Kind {kind} is not a nuclio runtime, available runtimes are {RuntimeKinds.nuclio_runtimes()}"
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
if sub_kind == serving_subkind:
|
|
258
|
+
return ServingRuntime()
|
|
259
|
+
|
|
260
|
+
if kind == RuntimeKinds.application:
|
|
261
|
+
return ApplicationRuntime()
|
|
262
|
+
|
|
263
|
+
runtime = RemoteRuntime()
|
|
264
|
+
runtime.spec.function_kind = sub_kind
|
|
265
|
+
return runtime
|
|
266
|
+
|
|
267
|
+
@staticmethod
|
|
268
|
+
def resolve_nuclio_sub_kind(kind):
|
|
269
|
+
is_nuclio = kind.startswith("nuclio")
|
|
270
|
+
sub_kind = kind[kind.find(":") + 1 :] if is_nuclio and ":" in kind else None
|
|
271
|
+
if kind == RuntimeKinds.serving:
|
|
272
|
+
is_nuclio = True
|
|
273
|
+
sub_kind = serving_subkind
|
|
274
|
+
elif kind == RuntimeKinds.application:
|
|
275
|
+
is_nuclio = True
|
|
276
|
+
return is_nuclio, sub_kind
|
|
277
|
+
|
|
211
278
|
|
|
212
279
|
def get_runtime_class(kind: str):
|
|
213
280
|
if kind == RuntimeKinds.mpijob:
|
|
214
|
-
return
|
|
281
|
+
return MpiRuntimeV1
|
|
215
282
|
|
|
216
283
|
if kind == RuntimeKinds.spark:
|
|
217
284
|
return Spark3Runtime
|
|
@@ -225,6 +292,7 @@ def get_runtime_class(kind: str):
|
|
|
225
292
|
RuntimeKinds.local: LocalRuntime,
|
|
226
293
|
RuntimeKinds.remotespark: RemoteSparkRuntime,
|
|
227
294
|
RuntimeKinds.databricks: DatabricksRuntime,
|
|
295
|
+
RuntimeKinds.application: ApplicationRuntime,
|
|
228
296
|
}
|
|
229
297
|
|
|
230
298
|
return kind_runtime_map[kind]
|
mlrun/runtimes/base.py
CHANGED
|
@@ -18,11 +18,14 @@ import typing
|
|
|
18
18
|
import warnings
|
|
19
19
|
from base64 import b64encode
|
|
20
20
|
from os import environ
|
|
21
|
-
from typing import Callable,
|
|
21
|
+
from typing import Callable, Optional, Union
|
|
22
22
|
|
|
23
23
|
import requests.exceptions
|
|
24
|
+
from mlrun_pipelines.common.ops import mlrun_op
|
|
24
25
|
from nuclio.build import mlrun_footer
|
|
25
26
|
|
|
27
|
+
import mlrun.common.constants
|
|
28
|
+
import mlrun.common.constants as mlrun_constants
|
|
26
29
|
import mlrun.common.schemas
|
|
27
30
|
import mlrun.common.schemas.model_monitoring.constants as mm_constants
|
|
28
31
|
import mlrun.db
|
|
@@ -36,7 +39,6 @@ from mlrun.utils.helpers import generate_object_uri, verify_field_regex
|
|
|
36
39
|
from ..config import config
|
|
37
40
|
from ..datastore import store_manager
|
|
38
41
|
from ..errors import err_to_str
|
|
39
|
-
from ..kfpops import mlrun_op
|
|
40
42
|
from ..lists import RunList
|
|
41
43
|
from ..model import BaseMetadata, HyperParamOptions, ImageBuilder, ModelObj, RunObject
|
|
42
44
|
from ..utils import (
|
|
@@ -66,6 +68,7 @@ spec_fields = [
|
|
|
66
68
|
"disable_auto_mount",
|
|
67
69
|
"allow_empty_resources",
|
|
68
70
|
"clone_target_dir",
|
|
71
|
+
"reset_on_run",
|
|
69
72
|
]
|
|
70
73
|
|
|
71
74
|
|
|
@@ -92,6 +95,7 @@ class FunctionStatus(ModelObj):
|
|
|
92
95
|
|
|
93
96
|
class FunctionSpec(ModelObj):
|
|
94
97
|
_dict_fields = spec_fields
|
|
98
|
+
_default_fields_to_strip = []
|
|
95
99
|
|
|
96
100
|
def __init__(
|
|
97
101
|
self,
|
|
@@ -123,7 +127,7 @@ class FunctionSpec(ModelObj):
|
|
|
123
127
|
self.entry_points = entry_points or {}
|
|
124
128
|
self.disable_auto_mount = disable_auto_mount
|
|
125
129
|
self.allow_empty_resources = None
|
|
126
|
-
#
|
|
130
|
+
# The build.source is cloned/extracted to the specified clone_target_dir
|
|
127
131
|
# if a relative path is specified, it will be enriched with a temp dir path
|
|
128
132
|
self._clone_target_dir = clone_target_dir or None
|
|
129
133
|
|
|
@@ -169,6 +173,9 @@ class BaseRuntime(ModelObj):
|
|
|
169
173
|
_is_nested = False
|
|
170
174
|
_is_remote = False
|
|
171
175
|
_dict_fields = ["kind", "metadata", "spec", "status", "verbose"]
|
|
176
|
+
_default_fields_to_strip = ModelObj._default_fields_to_strip + [
|
|
177
|
+
"status", # Function status describes the state rather than configuration
|
|
178
|
+
]
|
|
172
179
|
|
|
173
180
|
def __init__(self, metadata=None, spec=None):
|
|
174
181
|
self._metadata = None
|
|
@@ -218,7 +225,7 @@ class BaseRuntime(ModelObj):
|
|
|
218
225
|
self.metadata.labels[key] = str(value)
|
|
219
226
|
return self
|
|
220
227
|
|
|
221
|
-
def set_categories(self, categories:
|
|
228
|
+
def set_categories(self, categories: list[str]):
|
|
222
229
|
self.metadata.categories = mlrun.utils.helpers.as_list(categories)
|
|
223
230
|
|
|
224
231
|
@property
|
|
@@ -292,7 +299,7 @@ class BaseRuntime(ModelObj):
|
|
|
292
299
|
mlrun.model.Credentials.generate_access_key
|
|
293
300
|
)
|
|
294
301
|
|
|
295
|
-
def generate_runtime_k8s_env(self, runobj: RunObject = None) ->
|
|
302
|
+
def generate_runtime_k8s_env(self, runobj: RunObject = None) -> list[dict]:
|
|
296
303
|
"""
|
|
297
304
|
Prepares a runtime environment as it's expected by kubernetes.models.V1Container
|
|
298
305
|
|
|
@@ -313,23 +320,24 @@ class BaseRuntime(ModelObj):
|
|
|
313
320
|
name: Optional[str] = "",
|
|
314
321
|
project: Optional[str] = "",
|
|
315
322
|
params: Optional[dict] = None,
|
|
316
|
-
inputs: Optional[
|
|
323
|
+
inputs: Optional[dict[str, str]] = None,
|
|
317
324
|
out_path: Optional[str] = "",
|
|
318
325
|
workdir: Optional[str] = "",
|
|
319
326
|
artifact_path: Optional[str] = "",
|
|
320
327
|
watch: Optional[bool] = True,
|
|
321
328
|
schedule: Optional[Union[str, mlrun.common.schemas.ScheduleCronTrigger]] = None,
|
|
322
|
-
hyperparams: Optional[
|
|
329
|
+
hyperparams: Optional[dict[str, list]] = None,
|
|
323
330
|
hyper_param_options: Optional[HyperParamOptions] = None,
|
|
324
331
|
verbose: Optional[bool] = None,
|
|
325
332
|
scrape_metrics: Optional[bool] = None,
|
|
326
333
|
local: Optional[bool] = False,
|
|
327
334
|
local_code_path: Optional[str] = None,
|
|
328
335
|
auto_build: Optional[bool] = None,
|
|
329
|
-
param_file_secrets: Optional[
|
|
330
|
-
notifications: Optional[
|
|
331
|
-
returns: Optional[
|
|
332
|
-
state_thresholds: Optional[
|
|
336
|
+
param_file_secrets: Optional[dict[str, str]] = None,
|
|
337
|
+
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
338
|
+
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
339
|
+
state_thresholds: Optional[dict[str, int]] = None,
|
|
340
|
+
reset_on_run: Optional[bool] = None,
|
|
333
341
|
**launcher_kwargs,
|
|
334
342
|
) -> RunObject:
|
|
335
343
|
"""
|
|
@@ -384,6 +392,9 @@ class BaseRuntime(ModelObj):
|
|
|
384
392
|
standards and is at least 1 minute (-1 for infinite).
|
|
385
393
|
If the phase is active for longer than the threshold, the run will be aborted.
|
|
386
394
|
See mlconf.function.spec.state_thresholds for the state options and default values.
|
|
395
|
+
:param reset_on_run: When True, function python modules would reload prior to code execution.
|
|
396
|
+
This ensures latest code changes are executed. This argument must be used in
|
|
397
|
+
conjunction with the local=True argument.
|
|
387
398
|
:return: Run context object (RunObject) with run metadata, results and status
|
|
388
399
|
"""
|
|
389
400
|
launcher = mlrun.launcher.factory.LauncherFactory().create_launcher(
|
|
@@ -412,12 +423,13 @@ class BaseRuntime(ModelObj):
|
|
|
412
423
|
notifications=notifications,
|
|
413
424
|
returns=returns,
|
|
414
425
|
state_thresholds=state_thresholds,
|
|
426
|
+
reset_on_run=reset_on_run,
|
|
415
427
|
)
|
|
416
428
|
|
|
417
429
|
def _get_db_run(
|
|
418
430
|
self,
|
|
419
431
|
task: RunObject = None,
|
|
420
|
-
run_format: mlrun.common.
|
|
432
|
+
run_format: mlrun.common.formatters.RunFormat = mlrun.common.formatters.RunFormat.full,
|
|
421
433
|
):
|
|
422
434
|
if self._get_db() and task:
|
|
423
435
|
project = task.metadata.project
|
|
@@ -432,7 +444,7 @@ class BaseRuntime(ModelObj):
|
|
|
432
444
|
if task:
|
|
433
445
|
return task.to_dict()
|
|
434
446
|
|
|
435
|
-
def _generate_runtime_env(self, runobj: RunObject = None) ->
|
|
447
|
+
def _generate_runtime_env(self, runobj: RunObject = None) -> dict:
|
|
436
448
|
"""
|
|
437
449
|
Prepares all available environment variables for usage on a runtime
|
|
438
450
|
Data will be extracted from several sources and most of them are not guaranteed to be available
|
|
@@ -470,11 +482,11 @@ class BaseRuntime(ModelObj):
|
|
|
470
482
|
def _store_function(self, runspec, meta, db):
|
|
471
483
|
meta.labels["kind"] = self.kind
|
|
472
484
|
mlrun.runtimes.utils.enrich_run_labels(
|
|
473
|
-
meta.labels, [mlrun.runtimes.constants.RunLabels.owner]
|
|
485
|
+
meta.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
|
|
474
486
|
)
|
|
475
487
|
if runspec.spec.output_path:
|
|
476
488
|
runspec.spec.output_path = runspec.spec.output_path.replace(
|
|
477
|
-
"{{run.user}}", meta.labels[
|
|
489
|
+
"{{run.user}}", meta.labels[mlrun_constants.MLRunInternalLabels.owner]
|
|
478
490
|
)
|
|
479
491
|
|
|
480
492
|
if db and self.kind != "handler":
|
|
@@ -544,7 +556,7 @@ class BaseRuntime(ModelObj):
|
|
|
544
556
|
resp: dict = None,
|
|
545
557
|
task: RunObject = None,
|
|
546
558
|
err: Union[Exception, str] = None,
|
|
547
|
-
run_format: mlrun.common.
|
|
559
|
+
run_format: mlrun.common.formatters.RunFormat = mlrun.common.formatters.RunFormat.full,
|
|
548
560
|
) -> typing.Optional[dict]:
|
|
549
561
|
"""update the task state in the DB"""
|
|
550
562
|
was_none = False
|
|
@@ -582,9 +594,9 @@ class BaseRuntime(ModelObj):
|
|
|
582
594
|
|
|
583
595
|
elif (
|
|
584
596
|
not was_none
|
|
585
|
-
and last_state != mlrun.runtimes.constants.RunStates.completed
|
|
597
|
+
and last_state != mlrun.common.runtimes.constants.RunStates.completed
|
|
586
598
|
and last_state
|
|
587
|
-
not in mlrun.runtimes.constants.RunStates.error_and_abortion_states()
|
|
599
|
+
not in mlrun.common.runtimes.constants.RunStates.error_and_abortion_states()
|
|
588
600
|
):
|
|
589
601
|
try:
|
|
590
602
|
runtime_cls = mlrun.runtimes.get_runtime_class(kind)
|
|
@@ -637,7 +649,9 @@ class BaseRuntime(ModelObj):
|
|
|
637
649
|
image = image or self.spec.image or ""
|
|
638
650
|
|
|
639
651
|
image = enrich_image_url(image, client_version, client_python_version)
|
|
640
|
-
if not image.startswith(
|
|
652
|
+
if not image.startswith(
|
|
653
|
+
mlrun.common.constants.IMAGE_NAME_ENRICH_REGISTRY_PREFIX
|
|
654
|
+
):
|
|
641
655
|
return image
|
|
642
656
|
registry, repository = get_parsed_docker_registry()
|
|
643
657
|
if registry:
|
|
@@ -660,7 +674,7 @@ class BaseRuntime(ModelObj):
|
|
|
660
674
|
selector="",
|
|
661
675
|
hyper_param_options: HyperParamOptions = None,
|
|
662
676
|
inputs: dict = None,
|
|
663
|
-
outputs:
|
|
677
|
+
outputs: list = None,
|
|
664
678
|
workdir: str = "",
|
|
665
679
|
artifact_path: str = "",
|
|
666
680
|
image: str = "",
|
|
@@ -668,7 +682,7 @@ class BaseRuntime(ModelObj):
|
|
|
668
682
|
use_db=True,
|
|
669
683
|
verbose=None,
|
|
670
684
|
scrape_metrics=False,
|
|
671
|
-
returns: Optional[
|
|
685
|
+
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
672
686
|
auto_build: bool = False,
|
|
673
687
|
):
|
|
674
688
|
"""Run a local or remote task.
|
|
@@ -707,11 +721,11 @@ class BaseRuntime(ModelObj):
|
|
|
707
721
|
"key": "the_key".
|
|
708
722
|
:param auto_build: when set to True and the function require build it will be built on the first
|
|
709
723
|
function run, use only if you dont plan on changing the build config between runs
|
|
710
|
-
:return:
|
|
724
|
+
:return: mlrun_pipelines.models.PipelineNodeWrapper
|
|
711
725
|
"""
|
|
712
726
|
|
|
713
727
|
# if the function contain KFP PipelineParams (futures) pass the full spec to the
|
|
714
|
-
#
|
|
728
|
+
# PipelineNodeWrapper this way KFP will substitute the params with previous step outputs
|
|
715
729
|
if use_db and not self._has_pipeline_param():
|
|
716
730
|
# if the same function is built as part of the pipeline we do not use the versioned function
|
|
717
731
|
# rather the latest function w the same tag so we can pick up the updated image/status
|
|
@@ -774,7 +788,7 @@ class BaseRuntime(ModelObj):
|
|
|
774
788
|
body = fp.read()
|
|
775
789
|
if self.kind == mlrun.runtimes.RuntimeKinds.serving:
|
|
776
790
|
body = body + mlrun_footer.format(
|
|
777
|
-
mlrun.runtimes.serving.serving_subkind
|
|
791
|
+
mlrun.runtimes.nuclio.serving.serving_subkind
|
|
778
792
|
)
|
|
779
793
|
|
|
780
794
|
self.spec.build.functionSourceCode = b64encode(body.encode("utf-8")).decode(
|
|
@@ -786,10 +800,10 @@ class BaseRuntime(ModelObj):
|
|
|
786
800
|
|
|
787
801
|
def with_requirements(
|
|
788
802
|
self,
|
|
789
|
-
requirements: Optional[
|
|
803
|
+
requirements: Optional[list[str]] = None,
|
|
790
804
|
overwrite: bool = False,
|
|
791
805
|
prepare_image_for_deploy: bool = True,
|
|
792
|
-
requirements_file: str = "",
|
|
806
|
+
requirements_file: Optional[str] = "",
|
|
793
807
|
):
|
|
794
808
|
"""add package requirements from file or list to build spec.
|
|
795
809
|
|
|
@@ -808,7 +822,7 @@ class BaseRuntime(ModelObj):
|
|
|
808
822
|
|
|
809
823
|
def with_commands(
|
|
810
824
|
self,
|
|
811
|
-
commands:
|
|
825
|
+
commands: list[str],
|
|
812
826
|
overwrite: bool = False,
|
|
813
827
|
prepare_image_for_deploy: bool = True,
|
|
814
828
|
):
|
|
@@ -843,6 +857,12 @@ class BaseRuntime(ModelObj):
|
|
|
843
857
|
or (build.source and not build.load_source_on_run)
|
|
844
858
|
)
|
|
845
859
|
|
|
860
|
+
def enrich_runtime_spec(
|
|
861
|
+
self,
|
|
862
|
+
project_node_selector: dict[str, str],
|
|
863
|
+
):
|
|
864
|
+
pass
|
|
865
|
+
|
|
846
866
|
def prepare_image_for_deploy(self):
|
|
847
867
|
"""
|
|
848
868
|
if a function has a 'spec.image' it is considered to be deployed,
|
|
@@ -877,7 +897,7 @@ class BaseRuntime(ModelObj):
|
|
|
877
897
|
data = dict_to_json(struct)
|
|
878
898
|
stores = store_manager.set(secrets)
|
|
879
899
|
target = target or "function.yaml"
|
|
880
|
-
datastore, subpath = stores.get_or_create_store(target)
|
|
900
|
+
datastore, subpath, url = stores.get_or_create_store(target)
|
|
881
901
|
datastore.put(subpath, data)
|
|
882
902
|
logger.info(f"function spec saved to path: {target}")
|
|
883
903
|
return self
|
|
@@ -890,13 +910,6 @@ class BaseRuntime(ModelObj):
|
|
|
890
910
|
self, tag=tag, versioned=versioned, refresh=refresh
|
|
891
911
|
)
|
|
892
912
|
|
|
893
|
-
def to_dict(self, fields=None, exclude=None, strip=False):
|
|
894
|
-
struct = super().to_dict(fields, exclude=exclude)
|
|
895
|
-
if strip:
|
|
896
|
-
if "status" in struct:
|
|
897
|
-
del struct["status"]
|
|
898
|
-
return struct
|
|
899
|
-
|
|
900
913
|
def doc(self):
|
|
901
914
|
print("function:", self.metadata.name)
|
|
902
915
|
print(self.spec.description)
|
|
@@ -916,3 +929,6 @@ class BaseRuntime(ModelObj):
|
|
|
916
929
|
if "default" in p:
|
|
917
930
|
line += f", default={p['default']}"
|
|
918
931
|
print(" " + line)
|
|
932
|
+
|
|
933
|
+
def skip_image_enrichment(self):
|
|
934
|
+
return False
|
mlrun/runtimes/daskjob.py
CHANGED
|
@@ -16,7 +16,7 @@ import inspect
|
|
|
16
16
|
import socket
|
|
17
17
|
import time
|
|
18
18
|
from os import environ
|
|
19
|
-
from typing import Callable,
|
|
19
|
+
from typing import Callable, Optional, Union
|
|
20
20
|
|
|
21
21
|
import mlrun.common.schemas
|
|
22
22
|
import mlrun.errors
|
|
@@ -379,7 +379,7 @@ class DaskCluster(KubejobRuntime):
|
|
|
379
379
|
:param show_on_failure: show logs only in case of build failure
|
|
380
380
|
:param force_build: force building the image, even when no changes were made
|
|
381
381
|
|
|
382
|
-
:return
|
|
382
|
+
:return: True if the function is ready (deployed)
|
|
383
383
|
"""
|
|
384
384
|
return super().deploy(
|
|
385
385
|
watch,
|
|
@@ -461,7 +461,7 @@ class DaskCluster(KubejobRuntime):
|
|
|
461
461
|
|
|
462
462
|
def set_state_thresholds(
|
|
463
463
|
self,
|
|
464
|
-
state_thresholds:
|
|
464
|
+
state_thresholds: dict[str, str],
|
|
465
465
|
patch: bool = True,
|
|
466
466
|
):
|
|
467
467
|
raise NotImplementedError(
|
|
@@ -477,23 +477,24 @@ class DaskCluster(KubejobRuntime):
|
|
|
477
477
|
name: Optional[str] = "",
|
|
478
478
|
project: Optional[str] = "",
|
|
479
479
|
params: Optional[dict] = None,
|
|
480
|
-
inputs: Optional[
|
|
480
|
+
inputs: Optional[dict[str, str]] = None,
|
|
481
481
|
out_path: Optional[str] = "",
|
|
482
482
|
workdir: Optional[str] = "",
|
|
483
483
|
artifact_path: Optional[str] = "",
|
|
484
484
|
watch: Optional[bool] = True,
|
|
485
485
|
schedule: Optional[Union[str, mlrun.common.schemas.ScheduleCronTrigger]] = None,
|
|
486
|
-
hyperparams: Optional[
|
|
486
|
+
hyperparams: Optional[dict[str, list]] = None,
|
|
487
487
|
hyper_param_options: Optional[mlrun.model.HyperParamOptions] = None,
|
|
488
488
|
verbose: Optional[bool] = None,
|
|
489
489
|
scrape_metrics: Optional[bool] = None,
|
|
490
490
|
local: Optional[bool] = False,
|
|
491
491
|
local_code_path: Optional[str] = None,
|
|
492
492
|
auto_build: Optional[bool] = None,
|
|
493
|
-
param_file_secrets: Optional[
|
|
494
|
-
notifications: Optional[
|
|
495
|
-
returns: Optional[
|
|
496
|
-
state_thresholds: Optional[
|
|
493
|
+
param_file_secrets: Optional[dict[str, str]] = None,
|
|
494
|
+
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
495
|
+
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
496
|
+
state_thresholds: Optional[dict[str, int]] = None,
|
|
497
|
+
reset_on_run: Optional[bool] = None,
|
|
497
498
|
**launcher_kwargs,
|
|
498
499
|
) -> RunObject:
|
|
499
500
|
if state_thresholds:
|
|
@@ -547,7 +548,13 @@ class DaskCluster(KubejobRuntime):
|
|
|
547
548
|
"specified handler (string) without command "
|
|
548
549
|
"(py file path), specify command or use handler pointer"
|
|
549
550
|
)
|
|
550
|
-
|
|
551
|
+
# Do not embed the module in system as it is not persistent with the dask cluster
|
|
552
|
+
handler = load_module(
|
|
553
|
+
self.spec.command,
|
|
554
|
+
handler,
|
|
555
|
+
context=context,
|
|
556
|
+
embed_in_sys=False,
|
|
557
|
+
)
|
|
551
558
|
client = self.client
|
|
552
559
|
setattr(context, "dask_client", client)
|
|
553
560
|
sout, serr = exec_from_params(handler, runobj, context)
|
|
@@ -33,7 +33,7 @@ def main():
|
|
|
33
33
|
"The Databricks credentials path does not exist."
|
|
34
34
|
" Please manually cancel the job from the Databricks environment."
|
|
35
35
|
)
|
|
36
|
-
with open(credentials_path
|
|
36
|
+
with open(credentials_path) as yaml_file:
|
|
37
37
|
loaded_data = yaml.safe_load(yaml_file)
|
|
38
38
|
# use for flat yaml only
|
|
39
39
|
for key, value in loaded_data.items():
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
from ast import FunctionDef, parse, unparse
|
|
16
16
|
from base64 import b64decode, b64encode
|
|
17
|
-
from typing import Callable,
|
|
17
|
+
from typing import Callable, Optional, Union
|
|
18
18
|
|
|
19
19
|
import mlrun
|
|
20
20
|
import mlrun.runtimes.kubejob as kubejob
|
|
@@ -215,23 +215,24 @@ def run_mlrun_databricks_job(context,task_parameters: dict, **kwargs):
|
|
|
215
215
|
name: Optional[str] = "",
|
|
216
216
|
project: Optional[str] = "",
|
|
217
217
|
params: Optional[dict] = None,
|
|
218
|
-
inputs: Optional[
|
|
218
|
+
inputs: Optional[dict[str, str]] = None,
|
|
219
219
|
out_path: Optional[str] = "",
|
|
220
220
|
workdir: Optional[str] = "",
|
|
221
221
|
artifact_path: Optional[str] = "",
|
|
222
222
|
watch: Optional[bool] = True,
|
|
223
223
|
schedule: Optional[Union[str, mlrun.common.schemas.ScheduleCronTrigger]] = None,
|
|
224
|
-
hyperparams: Optional[
|
|
224
|
+
hyperparams: Optional[dict[str, list]] = None,
|
|
225
225
|
hyper_param_options: Optional[HyperParamOptions] = None,
|
|
226
226
|
verbose: Optional[bool] = None,
|
|
227
227
|
scrape_metrics: Optional[bool] = None,
|
|
228
228
|
local: Optional[bool] = False,
|
|
229
229
|
local_code_path: Optional[str] = None,
|
|
230
230
|
auto_build: Optional[bool] = None,
|
|
231
|
-
param_file_secrets: Optional[
|
|
232
|
-
notifications: Optional[
|
|
233
|
-
returns: Optional[
|
|
234
|
-
state_thresholds: Optional[
|
|
231
|
+
param_file_secrets: Optional[dict[str, str]] = None,
|
|
232
|
+
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
233
|
+
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
234
|
+
state_thresholds: Optional[dict[str, int]] = None,
|
|
235
|
+
reset_on_run: Optional[bool] = None,
|
|
235
236
|
**launcher_kwargs,
|
|
236
237
|
) -> RunObject:
|
|
237
238
|
if local:
|
|
@@ -99,7 +99,7 @@ def save_credentials(
|
|
|
99
99
|
credentials["DATABRICKS_CLUSTER_ID"] = cluster_id
|
|
100
100
|
|
|
101
101
|
with open(credentials_path, "w") as yaml_file:
|
|
102
|
-
yaml.
|
|
102
|
+
yaml.safe_dump(credentials, yaml_file, default_flow_style=False)
|
|
103
103
|
|
|
104
104
|
|
|
105
105
|
def run_mlrun_databricks_job(
|
mlrun/runtimes/funcdoc.py
CHANGED
|
@@ -16,8 +16,6 @@ import ast
|
|
|
16
16
|
import inspect
|
|
17
17
|
import re
|
|
18
18
|
|
|
19
|
-
from deprecated import deprecated
|
|
20
|
-
|
|
21
19
|
from mlrun.model import FunctionEntrypoint
|
|
22
20
|
|
|
23
21
|
|
|
@@ -73,32 +71,6 @@ def func_dict(
|
|
|
73
71
|
}
|
|
74
72
|
|
|
75
73
|
|
|
76
|
-
# TODO: remove in 1.7.0
|
|
77
|
-
@deprecated(
|
|
78
|
-
version="1.5.0",
|
|
79
|
-
reason="'func_info' is deprecated and will be removed in 1.7.0, use 'ast_func_info' instead",
|
|
80
|
-
category=FutureWarning,
|
|
81
|
-
)
|
|
82
|
-
def func_info(fn) -> dict:
|
|
83
|
-
sig = inspect.signature(fn)
|
|
84
|
-
doc = inspect.getdoc(fn) or ""
|
|
85
|
-
|
|
86
|
-
out = func_dict(
|
|
87
|
-
name=fn.__name__,
|
|
88
|
-
doc=doc,
|
|
89
|
-
params=[inspect_param(p) for p in sig.parameters.values()],
|
|
90
|
-
returns=param_dict(
|
|
91
|
-
type=type_name(sig.return_annotation, empty_is_none=True), default=None
|
|
92
|
-
),
|
|
93
|
-
lineno=func_lineno(fn),
|
|
94
|
-
)
|
|
95
|
-
|
|
96
|
-
if not fn.__doc__ or not fn.__doc__.strip():
|
|
97
|
-
return out
|
|
98
|
-
|
|
99
|
-
return merge_doc(out, doc)
|
|
100
|
-
|
|
101
|
-
|
|
102
74
|
def func_lineno(fn):
|
|
103
75
|
try:
|
|
104
76
|
return inspect.getsourcelines(fn)[1]
|
|
@@ -275,7 +247,7 @@ class ASTVisitor(ast.NodeVisitor):
|
|
|
275
247
|
self.exprs.append(node)
|
|
276
248
|
super().generic_visit(node)
|
|
277
249
|
|
|
278
|
-
def visit_FunctionDef(self, node):
|
|
250
|
+
def visit_FunctionDef(self, node): # noqa: N802
|
|
279
251
|
self.funcs.append(node)
|
|
280
252
|
self.generic_visit(node)
|
|
281
253
|
|
|
@@ -111,7 +111,7 @@ class FunctionReference(ModelObj):
|
|
|
111
111
|
code = self.code
|
|
112
112
|
if kind == mlrun.runtimes.RuntimeKinds.serving:
|
|
113
113
|
code = code + mlrun_footer.format(
|
|
114
|
-
mlrun.runtimes.serving.serving_subkind
|
|
114
|
+
mlrun.runtimes.nuclio.serving.serving_subkind
|
|
115
115
|
)
|
|
116
116
|
func = mlrun.new_function(
|
|
117
117
|
self.name, kind=kind, image=self.image or default_image
|