mlrun 1.7.0rc5__py3-none-any.whl → 1.7.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +11 -1
- mlrun/__main__.py +39 -121
- mlrun/{datastore/helpers.py → alerts/__init__.py} +2 -5
- mlrun/alerts/alert.py +248 -0
- mlrun/api/schemas/__init__.py +4 -3
- mlrun/artifacts/__init__.py +8 -3
- mlrun/artifacts/base.py +39 -254
- mlrun/artifacts/dataset.py +9 -190
- mlrun/artifacts/manager.py +73 -46
- mlrun/artifacts/model.py +30 -158
- mlrun/artifacts/plots.py +23 -380
- mlrun/common/constants.py +73 -2
- mlrun/common/db/sql_session.py +3 -2
- mlrun/common/formatters/__init__.py +21 -0
- mlrun/common/formatters/artifact.py +46 -0
- mlrun/common/formatters/base.py +113 -0
- mlrun/common/formatters/feature_set.py +44 -0
- mlrun/common/formatters/function.py +46 -0
- mlrun/common/formatters/pipeline.py +53 -0
- mlrun/common/formatters/project.py +51 -0
- mlrun/common/formatters/run.py +29 -0
- mlrun/common/helpers.py +11 -1
- mlrun/{runtimes → common/runtimes}/constants.py +32 -4
- mlrun/common/schemas/__init__.py +21 -4
- mlrun/common/schemas/alert.py +202 -0
- mlrun/common/schemas/api_gateway.py +113 -2
- mlrun/common/schemas/artifact.py +28 -1
- mlrun/common/schemas/auth.py +11 -0
- mlrun/common/schemas/client_spec.py +2 -1
- mlrun/common/schemas/common.py +7 -4
- mlrun/common/schemas/constants.py +3 -0
- mlrun/common/schemas/feature_store.py +58 -28
- mlrun/common/schemas/frontend_spec.py +8 -0
- mlrun/common/schemas/function.py +11 -0
- mlrun/common/schemas/hub.py +7 -9
- mlrun/common/schemas/model_monitoring/__init__.py +21 -4
- mlrun/common/schemas/model_monitoring/constants.py +136 -42
- mlrun/common/schemas/model_monitoring/grafana.py +9 -5
- mlrun/common/schemas/model_monitoring/model_endpoints.py +89 -41
- mlrun/common/schemas/notification.py +69 -12
- mlrun/{runtimes/mpijob/v1alpha1.py → common/schemas/pagination.py} +10 -13
- mlrun/common/schemas/pipeline.py +7 -0
- mlrun/common/schemas/project.py +67 -16
- mlrun/common/schemas/runs.py +17 -0
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/schemas/workflow.py +10 -2
- mlrun/common/types.py +14 -1
- mlrun/config.py +224 -58
- mlrun/data_types/data_types.py +11 -1
- mlrun/data_types/spark.py +5 -4
- mlrun/data_types/to_pandas.py +75 -34
- mlrun/datastore/__init__.py +8 -10
- mlrun/datastore/alibaba_oss.py +131 -0
- mlrun/datastore/azure_blob.py +131 -43
- mlrun/datastore/base.py +107 -47
- mlrun/datastore/datastore.py +17 -7
- mlrun/datastore/datastore_profile.py +91 -7
- mlrun/datastore/dbfs_store.py +3 -7
- mlrun/datastore/filestore.py +1 -3
- mlrun/datastore/google_cloud_storage.py +92 -32
- mlrun/datastore/hdfs.py +5 -0
- mlrun/datastore/inmem.py +6 -3
- mlrun/datastore/redis.py +3 -2
- mlrun/datastore/s3.py +30 -12
- mlrun/datastore/snowflake_utils.py +45 -0
- mlrun/datastore/sources.py +274 -59
- mlrun/datastore/spark_utils.py +30 -0
- mlrun/datastore/store_resources.py +9 -7
- mlrun/datastore/storeytargets.py +151 -0
- mlrun/datastore/targets.py +374 -102
- mlrun/datastore/utils.py +68 -5
- mlrun/datastore/v3io.py +28 -50
- mlrun/db/auth_utils.py +152 -0
- mlrun/db/base.py +231 -22
- mlrun/db/factory.py +1 -4
- mlrun/db/httpdb.py +864 -228
- mlrun/db/nopdb.py +268 -16
- mlrun/errors.py +35 -5
- mlrun/execution.py +111 -38
- mlrun/feature_store/__init__.py +0 -2
- mlrun/feature_store/api.py +46 -53
- mlrun/feature_store/common.py +6 -11
- mlrun/feature_store/feature_set.py +48 -23
- mlrun/feature_store/feature_vector.py +13 -2
- mlrun/feature_store/ingestion.py +7 -6
- mlrun/feature_store/retrieval/base.py +9 -4
- mlrun/feature_store/retrieval/dask_merger.py +2 -0
- mlrun/feature_store/retrieval/job.py +13 -4
- mlrun/feature_store/retrieval/local_merger.py +2 -0
- mlrun/feature_store/retrieval/spark_merger.py +24 -32
- mlrun/feature_store/steps.py +38 -19
- mlrun/features.py +6 -14
- mlrun/frameworks/_common/plan.py +3 -3
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +7 -12
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +2 -2
- mlrun/frameworks/lgbm/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
- mlrun/frameworks/lgbm/model_handler.py +1 -1
- mlrun/frameworks/parallel_coordinates.py +4 -4
- mlrun/frameworks/pytorch/__init__.py +2 -2
- mlrun/frameworks/sklearn/__init__.py +1 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +13 -3
- mlrun/frameworks/tf_keras/__init__.py +5 -2
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/mlrun_interface.py +2 -2
- mlrun/frameworks/xgboost/__init__.py +1 -1
- mlrun/k8s_utils.py +57 -12
- mlrun/launcher/__init__.py +1 -1
- mlrun/launcher/base.py +6 -5
- mlrun/launcher/client.py +13 -11
- mlrun/launcher/factory.py +1 -1
- mlrun/launcher/local.py +15 -5
- mlrun/launcher/remote.py +10 -3
- mlrun/lists.py +6 -2
- mlrun/model.py +297 -48
- mlrun/model_monitoring/__init__.py +1 -1
- mlrun/model_monitoring/api.py +152 -357
- mlrun/model_monitoring/applications/__init__.py +10 -0
- mlrun/model_monitoring/applications/_application_steps.py +190 -0
- mlrun/model_monitoring/applications/base.py +108 -0
- mlrun/model_monitoring/applications/context.py +341 -0
- mlrun/model_monitoring/{evidently_application.py → applications/evidently_base.py} +27 -22
- mlrun/model_monitoring/applications/histogram_data_drift.py +227 -91
- mlrun/model_monitoring/applications/results.py +99 -0
- mlrun/model_monitoring/controller.py +130 -303
- mlrun/model_monitoring/{stores/models/sqlite.py → db/__init__.py} +5 -10
- mlrun/model_monitoring/db/stores/__init__.py +136 -0
- mlrun/model_monitoring/db/stores/base/__init__.py +15 -0
- mlrun/model_monitoring/db/stores/base/store.py +213 -0
- mlrun/model_monitoring/db/stores/sqldb/__init__.py +13 -0
- mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +71 -0
- mlrun/model_monitoring/db/stores/sqldb/models/base.py +190 -0
- mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +103 -0
- mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py} +19 -13
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +659 -0
- mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +13 -0
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +726 -0
- mlrun/model_monitoring/db/tsdb/__init__.py +105 -0
- mlrun/model_monitoring/db/tsdb/base.py +448 -0
- mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
- mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +298 -0
- mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +42 -0
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +522 -0
- mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +158 -0
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +849 -0
- mlrun/model_monitoring/features_drift_table.py +34 -22
- mlrun/model_monitoring/helpers.py +177 -39
- mlrun/model_monitoring/model_endpoint.py +3 -2
- mlrun/model_monitoring/stream_processing.py +165 -398
- mlrun/model_monitoring/tracking_policy.py +7 -1
- mlrun/model_monitoring/writer.py +161 -125
- mlrun/package/packagers/default_packager.py +2 -2
- mlrun/package/packagers_manager.py +1 -0
- mlrun/package/utils/_formatter.py +2 -2
- mlrun/platforms/__init__.py +11 -10
- mlrun/platforms/iguazio.py +67 -228
- mlrun/projects/__init__.py +6 -1
- mlrun/projects/operations.py +47 -20
- mlrun/projects/pipelines.py +396 -249
- mlrun/projects/project.py +1125 -414
- mlrun/render.py +28 -22
- mlrun/run.py +207 -180
- mlrun/runtimes/__init__.py +76 -11
- mlrun/runtimes/base.py +40 -14
- mlrun/runtimes/daskjob.py +9 -2
- mlrun/runtimes/databricks_job/databricks_runtime.py +1 -0
- mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
- mlrun/runtimes/funcdoc.py +1 -29
- mlrun/runtimes/kubejob.py +34 -128
- mlrun/runtimes/local.py +39 -10
- mlrun/runtimes/mpijob/__init__.py +0 -20
- mlrun/runtimes/mpijob/abstract.py +8 -8
- mlrun/runtimes/mpijob/v1.py +1 -1
- mlrun/runtimes/nuclio/api_gateway.py +646 -177
- mlrun/runtimes/nuclio/application/__init__.py +15 -0
- mlrun/runtimes/nuclio/application/application.py +758 -0
- mlrun/runtimes/nuclio/application/reverse_proxy.go +95 -0
- mlrun/runtimes/nuclio/function.py +188 -68
- mlrun/runtimes/nuclio/serving.py +57 -60
- mlrun/runtimes/pod.py +191 -58
- mlrun/runtimes/remotesparkjob.py +11 -8
- mlrun/runtimes/sparkjob/spark3job.py +17 -18
- mlrun/runtimes/utils.py +40 -73
- mlrun/secrets.py +6 -2
- mlrun/serving/__init__.py +8 -1
- mlrun/serving/remote.py +2 -3
- mlrun/serving/routers.py +89 -64
- mlrun/serving/server.py +54 -26
- mlrun/serving/states.py +187 -56
- mlrun/serving/utils.py +19 -11
- mlrun/serving/v2_serving.py +136 -63
- mlrun/track/tracker.py +2 -1
- mlrun/track/trackers/mlflow_tracker.py +5 -0
- mlrun/utils/async_http.py +26 -6
- mlrun/utils/db.py +18 -0
- mlrun/utils/helpers.py +375 -105
- mlrun/utils/http.py +2 -2
- mlrun/utils/logger.py +75 -9
- mlrun/utils/notifications/notification/__init__.py +14 -10
- mlrun/utils/notifications/notification/base.py +48 -0
- mlrun/utils/notifications/notification/console.py +2 -0
- mlrun/utils/notifications/notification/git.py +24 -1
- mlrun/utils/notifications/notification/ipython.py +2 -0
- mlrun/utils/notifications/notification/slack.py +96 -21
- mlrun/utils/notifications/notification/webhook.py +63 -2
- mlrun/utils/notifications/notification_pusher.py +146 -16
- mlrun/utils/regex.py +9 -0
- mlrun/utils/retryer.py +3 -2
- mlrun/utils/v3io_clients.py +2 -3
- mlrun/utils/version/version.json +2 -2
- mlrun-1.7.2.dist-info/METADATA +390 -0
- mlrun-1.7.2.dist-info/RECORD +351 -0
- {mlrun-1.7.0rc5.dist-info → mlrun-1.7.2.dist-info}/WHEEL +1 -1
- mlrun/feature_store/retrieval/conversion.py +0 -271
- mlrun/kfpops.py +0 -868
- mlrun/model_monitoring/application.py +0 -310
- mlrun/model_monitoring/batch.py +0 -974
- mlrun/model_monitoring/controller_handler.py +0 -37
- mlrun/model_monitoring/prometheus.py +0 -216
- mlrun/model_monitoring/stores/__init__.py +0 -111
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +0 -574
- mlrun/model_monitoring/stores/model_endpoint_store.py +0 -145
- mlrun/model_monitoring/stores/models/__init__.py +0 -27
- mlrun/model_monitoring/stores/models/base.py +0 -84
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -382
- mlrun/platforms/other.py +0 -305
- mlrun-1.7.0rc5.dist-info/METADATA +0 -269
- mlrun-1.7.0rc5.dist-info/RECORD +0 -323
- {mlrun-1.7.0rc5.dist-info → mlrun-1.7.2.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc5.dist-info → mlrun-1.7.2.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc5.dist-info → mlrun-1.7.2.dist-info}/top_level.txt +0 -0
mlrun/frameworks/_common/plan.py
CHANGED
|
@@ -11,12 +11,12 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
from abc import ABC, abstractmethod
|
|
16
16
|
|
|
17
17
|
import mlrun
|
|
18
18
|
from mlrun.artifacts import Artifact
|
|
19
|
-
from mlrun.utils.helpers import
|
|
19
|
+
from mlrun.utils.helpers import is_jupyter
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
class Plan(ABC):
|
|
@@ -84,7 +84,7 @@ class Plan(ABC):
|
|
|
84
84
|
return
|
|
85
85
|
|
|
86
86
|
# Call the correct display method according to the kernel:
|
|
87
|
-
if
|
|
87
|
+
if is_jupyter:
|
|
88
88
|
self._gui_display()
|
|
89
89
|
else:
|
|
90
90
|
self._cli_display()
|
|
@@ -547,9 +547,9 @@ class TensorboardLogger(Logger, Generic[DLTypes.WeightType]):
|
|
|
547
547
|
"inputs",
|
|
548
548
|
"parameters",
|
|
549
549
|
]:
|
|
550
|
-
text +=
|
|
551
|
-
property_name.capitalize()
|
|
552
|
-
self._markdown_print(value=property_value, tabs=2)
|
|
550
|
+
text += (
|
|
551
|
+
f"\n * **{property_name.capitalize()}**: "
|
|
552
|
+
f"{self._markdown_print(value=property_value, tabs=2)}"
|
|
553
553
|
)
|
|
554
554
|
else:
|
|
555
555
|
for property_name, property_value in self._extract_epoch_results().items():
|
|
@@ -614,13 +614,8 @@ class TensorboardLogger(Logger, Generic[DLTypes.WeightType]):
|
|
|
614
614
|
:return: The generated link.
|
|
615
615
|
"""
|
|
616
616
|
return (
|
|
617
|
-
'<a href="{}/{}/{}
|
|
618
|
-
|
|
619
|
-
config.ui.projects_prefix,
|
|
620
|
-
context.project,
|
|
621
|
-
context.uid,
|
|
622
|
-
link_text,
|
|
623
|
-
)
|
|
617
|
+
f'<a href="{config.resolve_ui_url()}/{config.ui.projects_prefix}/{context.project}'
|
|
618
|
+
f'/jobs/monitor/{context.uid}/overview" target="_blank">{link_text}</a>'
|
|
624
619
|
)
|
|
625
620
|
|
|
626
621
|
@staticmethod
|
|
@@ -653,13 +648,13 @@ class TensorboardLogger(Logger, Generic[DLTypes.WeightType]):
|
|
|
653
648
|
if isinstance(value, list):
|
|
654
649
|
if len(value) == 0:
|
|
655
650
|
return ""
|
|
656
|
-
text = "\n" + yaml.
|
|
651
|
+
text = "\n" + yaml.safe_dump(value)
|
|
657
652
|
text = " \n".join([" " * tabs + line for line in text.splitlines()])
|
|
658
653
|
return text
|
|
659
654
|
if isinstance(value, dict):
|
|
660
655
|
if len(value) == 0:
|
|
661
656
|
return ""
|
|
662
|
-
text = yaml.
|
|
657
|
+
text = yaml.safe_dump(value)
|
|
663
658
|
text = " \n".join(
|
|
664
659
|
[" " * tabs + "- " + line for line in text.splitlines()]
|
|
665
660
|
)
|
|
@@ -363,7 +363,7 @@ class AutoMLRun:
|
|
|
363
363
|
|
|
364
364
|
{
|
|
365
365
|
"/.../custom_model.py": "MyModel",
|
|
366
|
-
"/.../custom_objects.py": ["object1", "object2"]
|
|
366
|
+
"/.../custom_objects.py": ["object1", "object2"],
|
|
367
367
|
}
|
|
368
368
|
|
|
369
369
|
All the paths will be accessed from the given 'custom_objects_directory',
|
|
@@ -464,7 +464,7 @@ class AutoMLRun:
|
|
|
464
464
|
|
|
465
465
|
{
|
|
466
466
|
"/.../custom_model.py": "MyModel",
|
|
467
|
-
"/.../custom_objects.py": ["object1", "object2"]
|
|
467
|
+
"/.../custom_objects.py": ["object1", "object2"],
|
|
468
468
|
}
|
|
469
469
|
|
|
470
470
|
All the paths will be accessed from the given 'custom_objects_directory',
|
|
@@ -241,7 +241,7 @@ def apply_mlrun(
|
|
|
241
241
|
|
|
242
242
|
{
|
|
243
243
|
"/.../custom_model.py": "MyModel",
|
|
244
|
-
"/.../custom_objects.py": ["object1", "object2"]
|
|
244
|
+
"/.../custom_objects.py": ["object1", "object2"],
|
|
245
245
|
}
|
|
246
246
|
|
|
247
247
|
All the paths will be accessed from the given 'custom_objects_directory', meaning
|
|
@@ -63,11 +63,9 @@ class Callback(ABC):
|
|
|
63
63
|
def on_train_end(self):
|
|
64
64
|
print("{self.name}: Done training!")
|
|
65
65
|
|
|
66
|
+
|
|
66
67
|
apply_mlrun()
|
|
67
|
-
lgb.train(
|
|
68
|
-
...,
|
|
69
|
-
callbacks=[ExampleCallback(name="Example")]
|
|
70
|
-
)
|
|
68
|
+
lgb.train(..., callbacks=[ExampleCallback(name="Example")])
|
|
71
69
|
"""
|
|
72
70
|
|
|
73
71
|
def __init__(self, order: int = 10, before_iteration: bool = False):
|
|
@@ -103,7 +103,7 @@ class LGBMModelHandler(MLModelHandler):
|
|
|
103
103
|
|
|
104
104
|
{
|
|
105
105
|
"/.../custom_model.py": "MyModel",
|
|
106
|
-
"/.../custom_objects.py": ["object1", "object2"]
|
|
106
|
+
"/.../custom_objects.py": ["object1", "object2"],
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
All the paths will be accessed from the given 'custom_objects_directory',
|
|
@@ -18,8 +18,7 @@ from typing import Union
|
|
|
18
18
|
|
|
19
19
|
import numpy as np
|
|
20
20
|
import pandas as pd
|
|
21
|
-
from IPython.
|
|
22
|
-
from IPython.display import display
|
|
21
|
+
from IPython.display import HTML, display
|
|
23
22
|
from pandas.api.types import is_numeric_dtype, is_string_dtype
|
|
24
23
|
|
|
25
24
|
import mlrun
|
|
@@ -216,7 +215,7 @@ def _show_and_export_html(html: str, show=None, filename=None, runs_list=None):
|
|
|
216
215
|
fp.write("</body></html>")
|
|
217
216
|
else:
|
|
218
217
|
fp.write(html)
|
|
219
|
-
if show or (show is None and mlrun.utils.
|
|
218
|
+
if show or (show is None and mlrun.utils.is_jupyter):
|
|
220
219
|
display(HTML(html))
|
|
221
220
|
if runs_list and len(runs_list) <= max_table_rows:
|
|
222
221
|
display(HTML(html_table))
|
|
@@ -295,7 +294,7 @@ def compare_db_runs(
|
|
|
295
294
|
iter=False,
|
|
296
295
|
start_time_from: datetime = None,
|
|
297
296
|
hide_identical: bool = True,
|
|
298
|
-
exclude: list =
|
|
297
|
+
exclude: list = None,
|
|
299
298
|
show=None,
|
|
300
299
|
colorscale: str = "Blues",
|
|
301
300
|
filename=None,
|
|
@@ -332,6 +331,7 @@ def compare_db_runs(
|
|
|
332
331
|
**query_args,
|
|
333
332
|
)
|
|
334
333
|
|
|
334
|
+
exclude = exclude or []
|
|
335
335
|
runs_df = _runs_list_to_df(runs_list)
|
|
336
336
|
plot_as_html = gen_pcp_plot(
|
|
337
337
|
runs_df,
|
|
@@ -112,7 +112,7 @@ def train(
|
|
|
112
112
|
|
|
113
113
|
{
|
|
114
114
|
"/.../custom_optimizer.py": "optimizer",
|
|
115
|
-
"/.../custom_layers.py": ["layer1", "layer2"]
|
|
115
|
+
"/.../custom_layers.py": ["layer1", "layer2"],
|
|
116
116
|
}
|
|
117
117
|
|
|
118
118
|
All the paths will be accessed from the given 'custom_objects_directory',
|
|
@@ -264,7 +264,7 @@ def evaluate(
|
|
|
264
264
|
|
|
265
265
|
{
|
|
266
266
|
"/.../custom_optimizer.py": "optimizer",
|
|
267
|
-
"/.../custom_layers.py": ["layer1", "layer2"]
|
|
267
|
+
"/.../custom_layers.py": ["layer1", "layer2"],
|
|
268
268
|
}
|
|
269
269
|
|
|
270
270
|
All the paths will be accessed from the given 'custom_objects_directory', meaning
|
|
@@ -92,7 +92,7 @@ def apply_mlrun(
|
|
|
92
92
|
|
|
93
93
|
{
|
|
94
94
|
"/.../custom_model.py": "MyModel",
|
|
95
|
-
"/.../custom_objects.py": ["object1", "object2"]
|
|
95
|
+
"/.../custom_objects.py": ["object1", "object2"],
|
|
96
96
|
}
|
|
97
97
|
|
|
98
98
|
All the paths will be accessed from the given 'custom_objects_directory', meaning
|
|
@@ -97,7 +97,7 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
|
|
|
97
97
|
|
|
98
98
|
def wrapper(
|
|
99
99
|
self: SKLearnTypes.ModelType,
|
|
100
|
-
X: SKLearnTypes.DatasetType,
|
|
100
|
+
X: SKLearnTypes.DatasetType, # noqa: N803 - should be lowercase "x", kept for BC
|
|
101
101
|
y: SKLearnTypes.DatasetType = None,
|
|
102
102
|
*args,
|
|
103
103
|
**kwargs,
|
|
@@ -124,7 +124,12 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
|
|
|
124
124
|
|
|
125
125
|
return wrapper
|
|
126
126
|
|
|
127
|
-
def mlrun_predict(
|
|
127
|
+
def mlrun_predict(
|
|
128
|
+
self,
|
|
129
|
+
X: SKLearnTypes.DatasetType, # noqa: N803 - should be lowercase "x", kept for BC
|
|
130
|
+
*args,
|
|
131
|
+
**kwargs,
|
|
132
|
+
):
|
|
128
133
|
"""
|
|
129
134
|
MLRun's wrapper for the common ML API predict method.
|
|
130
135
|
"""
|
|
@@ -136,7 +141,12 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
|
|
|
136
141
|
|
|
137
142
|
return y_pred
|
|
138
143
|
|
|
139
|
-
def mlrun_predict_proba(
|
|
144
|
+
def mlrun_predict_proba(
|
|
145
|
+
self,
|
|
146
|
+
X: SKLearnTypes.DatasetType, # noqa: N803 - should be lowercase "x", kept for BC
|
|
147
|
+
*args,
|
|
148
|
+
**kwargs,
|
|
149
|
+
):
|
|
140
150
|
"""
|
|
141
151
|
MLRun's wrapper for the common ML API predict_proba method.
|
|
142
152
|
"""
|
|
@@ -18,6 +18,7 @@ from typing import Any, Union
|
|
|
18
18
|
from tensorflow import keras
|
|
19
19
|
|
|
20
20
|
import mlrun
|
|
21
|
+
import mlrun.common.constants as mlrun_constants
|
|
21
22
|
|
|
22
23
|
from .callbacks import MLRunLoggingCallback, TensorboardLoggingCallback
|
|
23
24
|
from .mlrun_interface import TFKerasMLRunInterface
|
|
@@ -85,7 +86,7 @@ def apply_mlrun(
|
|
|
85
86
|
|
|
86
87
|
{
|
|
87
88
|
"/.../custom_optimizer.py": "optimizer",
|
|
88
|
-
"/.../custom_layers.py": ["layer1", "layer2"]
|
|
89
|
+
"/.../custom_layers.py": ["layer1", "layer2"],
|
|
89
90
|
}
|
|
90
91
|
|
|
91
92
|
All the paths will be accessed from the given 'custom_objects_directory',
|
|
@@ -126,7 +127,9 @@ def apply_mlrun(
|
|
|
126
127
|
# # Use horovod:
|
|
127
128
|
if use_horovod is None:
|
|
128
129
|
use_horovod = (
|
|
129
|
-
context.labels.get(
|
|
130
|
+
context.labels.get(mlrun_constants.MLRunInternalLabels.kind, "") == "mpijob"
|
|
131
|
+
if context is not None
|
|
132
|
+
else False
|
|
130
133
|
)
|
|
131
134
|
|
|
132
135
|
# Create a model handler:
|
|
@@ -19,7 +19,8 @@ from typing import Union
|
|
|
19
19
|
|
|
20
20
|
import tensorflow as tf
|
|
21
21
|
from tensorflow import keras
|
|
22
|
-
from tensorflow.keras.
|
|
22
|
+
from tensorflow.keras.optimizers import Optimizer
|
|
23
|
+
from tensorflow.python.keras.callbacks import (
|
|
23
24
|
BaseLogger,
|
|
24
25
|
Callback,
|
|
25
26
|
CSVLogger,
|
|
@@ -27,7 +28,6 @@ from tensorflow.keras.callbacks import (
|
|
|
27
28
|
ProgbarLogger,
|
|
28
29
|
TensorBoard,
|
|
29
30
|
)
|
|
30
|
-
from tensorflow.keras.optimizers import Optimizer
|
|
31
31
|
|
|
32
32
|
import mlrun
|
|
33
33
|
|
|
@@ -90,7 +90,7 @@ def apply_mlrun(
|
|
|
90
90
|
|
|
91
91
|
{
|
|
92
92
|
"/.../custom_model.py": "MyModel",
|
|
93
|
-
"/.../custom_objects.py": ["object1", "object2"]
|
|
93
|
+
"/.../custom_objects.py": ["object1", "object2"],
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
All the paths will be accessed from the given 'custom_objects_directory', meaning
|
mlrun/k8s_utils.py
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import re
|
|
15
|
+
import warnings
|
|
15
16
|
|
|
16
17
|
import kubernetes.client
|
|
17
18
|
|
|
@@ -133,7 +134,7 @@ def sanitize_label_value(value: str) -> str:
|
|
|
133
134
|
return re.sub(r"([^a-zA-Z0-9_.-]|^[^a-zA-Z0-9]|[^a-zA-Z0-9]$)", "-", value[:63])
|
|
134
135
|
|
|
135
136
|
|
|
136
|
-
def verify_label_key(key: str):
|
|
137
|
+
def verify_label_key(key: str, allow_k8s_prefix: bool = False):
|
|
137
138
|
"""
|
|
138
139
|
Verify that the label key is valid for Kubernetes.
|
|
139
140
|
Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
|
|
@@ -141,22 +142,15 @@ def verify_label_key(key: str):
|
|
|
141
142
|
if not key:
|
|
142
143
|
raise mlrun.errors.MLRunInvalidArgumentError("label key cannot be empty")
|
|
143
144
|
|
|
144
|
-
mlrun.utils.helpers.verify_field_regex(
|
|
145
|
-
f"project.metadata.labels.'{key}'",
|
|
146
|
-
key,
|
|
147
|
-
mlrun.utils.regex.k8s_character_limit,
|
|
148
|
-
)
|
|
149
|
-
|
|
150
|
-
if key.startswith("k8s.io/") or key.startswith("kubernetes.io/"):
|
|
151
|
-
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
152
|
-
"Labels cannot start with 'k8s.io/' or 'kubernetes.io/'"
|
|
153
|
-
)
|
|
154
|
-
|
|
155
145
|
parts = key.split("/")
|
|
156
146
|
if len(parts) == 1:
|
|
157
147
|
name = parts[0]
|
|
158
148
|
elif len(parts) == 2:
|
|
159
149
|
prefix, name = parts
|
|
150
|
+
if len(name) == 0:
|
|
151
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
152
|
+
"Label key name cannot be empty when a prefix is set"
|
|
153
|
+
)
|
|
160
154
|
if len(prefix) == 0:
|
|
161
155
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
162
156
|
"Label key prefix cannot be empty"
|
|
@@ -173,12 +167,28 @@ def verify_label_key(key: str):
|
|
|
173
167
|
"Label key can only contain one '/'"
|
|
174
168
|
)
|
|
175
169
|
|
|
170
|
+
mlrun.utils.helpers.verify_field_regex(
|
|
171
|
+
f"project.metadata.labels.'{key}'",
|
|
172
|
+
name,
|
|
173
|
+
mlrun.utils.regex.k8s_character_limit,
|
|
174
|
+
)
|
|
176
175
|
mlrun.utils.helpers.verify_field_regex(
|
|
177
176
|
f"project.metadata.labels.'{key}'",
|
|
178
177
|
name,
|
|
179
178
|
mlrun.utils.regex.qualified_name,
|
|
180
179
|
)
|
|
181
180
|
|
|
181
|
+
# Allow the use of Kubernetes reserved prefixes ('k8s.io/' or 'kubernetes.io/')
|
|
182
|
+
# only when setting node selectors, not when adding new labels.
|
|
183
|
+
if (
|
|
184
|
+
key.startswith("k8s.io/")
|
|
185
|
+
or key.startswith("kubernetes.io/")
|
|
186
|
+
and not allow_k8s_prefix
|
|
187
|
+
):
|
|
188
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
189
|
+
"Labels cannot start with 'k8s.io/' or 'kubernetes.io/'"
|
|
190
|
+
)
|
|
191
|
+
|
|
182
192
|
|
|
183
193
|
def verify_label_value(value, label_key):
|
|
184
194
|
mlrun.utils.helpers.verify_field_regex(
|
|
@@ -186,3 +196,38 @@ def verify_label_value(value, label_key):
|
|
|
186
196
|
value,
|
|
187
197
|
mlrun.utils.regex.label_value,
|
|
188
198
|
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def validate_node_selectors(
|
|
202
|
+
node_selectors: dict[str, str], raise_on_error: bool = True
|
|
203
|
+
) -> bool:
|
|
204
|
+
"""
|
|
205
|
+
Ensures that user-defined node selectors adhere to Kubernetes label standards:
|
|
206
|
+
- Validates that each key conforms to Kubernetes naming conventions, with specific rules for name and prefix.
|
|
207
|
+
- Ensures values comply with Kubernetes label value rules.
|
|
208
|
+
- If raise_on_error is True, raises errors for invalid selectors.
|
|
209
|
+
- If raise_on_error is False, logs warnings for invalid selectors.
|
|
210
|
+
"""
|
|
211
|
+
|
|
212
|
+
# Helper function for handling errors or warnings
|
|
213
|
+
def handle_invalid(message):
|
|
214
|
+
if raise_on_error:
|
|
215
|
+
raise
|
|
216
|
+
else:
|
|
217
|
+
warnings.warn(
|
|
218
|
+
f"{message}\n"
|
|
219
|
+
f"The node selector you’ve set does not meet the validation rules for the current Kubernetes version. "
|
|
220
|
+
f"Please note that invalid node selectors may cause issues with function scheduling."
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
node_selectors = node_selectors or {}
|
|
224
|
+
for key, value in node_selectors.items():
|
|
225
|
+
try:
|
|
226
|
+
verify_label_key(key, allow_k8s_prefix=True)
|
|
227
|
+
verify_label_value(value, label_key=key)
|
|
228
|
+
except mlrun.errors.MLRunInvalidArgumentError as err:
|
|
229
|
+
# An error or warning is raised by handle_invalid due to validation failure.
|
|
230
|
+
# Returning False indicates validation failed, allowing us to exit the function.
|
|
231
|
+
handle_invalid(str(err))
|
|
232
|
+
return False
|
|
233
|
+
return True
|
mlrun/launcher/__init__.py
CHANGED
mlrun/launcher/base.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2023
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -18,10 +18,11 @@ import os
|
|
|
18
18
|
import uuid
|
|
19
19
|
from typing import Any, Callable, Optional, Union
|
|
20
20
|
|
|
21
|
+
import mlrun_pipelines.common.ops
|
|
22
|
+
|
|
21
23
|
import mlrun.common.schemas
|
|
22
24
|
import mlrun.config
|
|
23
25
|
import mlrun.errors
|
|
24
|
-
import mlrun.kfpops
|
|
25
26
|
import mlrun.lists
|
|
26
27
|
import mlrun.model
|
|
27
28
|
import mlrun.runtimes
|
|
@@ -353,7 +354,7 @@ class BaseLauncher(abc.ABC):
|
|
|
353
354
|
or {}
|
|
354
355
|
)
|
|
355
356
|
state_thresholds = (
|
|
356
|
-
mlrun.
|
|
357
|
+
mlrun.mlconf.function.spec.state_thresholds.default.to_dict()
|
|
357
358
|
| state_thresholds
|
|
358
359
|
)
|
|
359
360
|
run.spec.state_thresholds = state_thresholds or run.spec.state_thresholds
|
|
@@ -390,7 +391,7 @@ class BaseLauncher(abc.ABC):
|
|
|
390
391
|
return
|
|
391
392
|
|
|
392
393
|
if result and runtime.kfp and err is None:
|
|
393
|
-
|
|
394
|
+
mlrun_pipelines.common.ops.write_kfpmeta(result)
|
|
394
395
|
|
|
395
396
|
self._log_track_results(runtime.is_child, result, run)
|
|
396
397
|
|
|
@@ -403,7 +404,7 @@ class BaseLauncher(abc.ABC):
|
|
|
403
404
|
)
|
|
404
405
|
if (
|
|
405
406
|
run.status.state
|
|
406
|
-
in mlrun.runtimes.constants.RunStates.error_and_abortion_states()
|
|
407
|
+
in mlrun.common.runtimes.constants.RunStates.error_and_abortion_states()
|
|
407
408
|
):
|
|
408
409
|
if runtime._is_remote and not runtime.is_child:
|
|
409
410
|
logger.error(
|
mlrun/launcher/client.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2023
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -14,14 +14,15 @@
|
|
|
14
14
|
import abc
|
|
15
15
|
from typing import Optional
|
|
16
16
|
|
|
17
|
-
import IPython
|
|
17
|
+
import IPython.display
|
|
18
18
|
|
|
19
|
+
import mlrun.common.constants as mlrun_constants
|
|
19
20
|
import mlrun.errors
|
|
20
21
|
import mlrun.launcher.base as launcher
|
|
21
22
|
import mlrun.lists
|
|
22
23
|
import mlrun.model
|
|
23
24
|
import mlrun.runtimes
|
|
24
|
-
|
|
25
|
+
import mlrun.utils
|
|
25
26
|
|
|
26
27
|
|
|
27
28
|
class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
|
|
@@ -47,7 +48,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
|
|
|
47
48
|
If build is needed, set the image as the base_image for the build.
|
|
48
49
|
If image is not given set the default one.
|
|
49
50
|
"""
|
|
50
|
-
if runtime.kind in mlrun.runtimes.RuntimeKinds.
|
|
51
|
+
if runtime.kind in mlrun.runtimes.RuntimeKinds.pure_nuclio_deployed_runtimes():
|
|
51
52
|
return
|
|
52
53
|
|
|
53
54
|
require_build = runtime.requires_build()
|
|
@@ -69,13 +70,14 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
|
|
|
69
70
|
def _store_function(
|
|
70
71
|
runtime: "mlrun.runtimes.BaseRuntime", run: "mlrun.run.RunObject"
|
|
71
72
|
):
|
|
72
|
-
run.metadata.labels[
|
|
73
|
+
run.metadata.labels[mlrun_constants.MLRunInternalLabels.kind] = runtime.kind
|
|
73
74
|
mlrun.runtimes.utils.enrich_run_labels(
|
|
74
|
-
run.metadata.labels, [mlrun.runtimes.constants.RunLabels.owner]
|
|
75
|
+
run.metadata.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
|
|
75
76
|
)
|
|
76
77
|
if run.spec.output_path:
|
|
77
78
|
run.spec.output_path = run.spec.output_path.replace(
|
|
78
|
-
"{{run.user}}",
|
|
79
|
+
"{{run.user}}",
|
|
80
|
+
run.metadata.labels[mlrun_constants.MLRunInternalLabels.owner],
|
|
79
81
|
)
|
|
80
82
|
db = runtime._get_db()
|
|
81
83
|
if db and runtime.kind != "handler":
|
|
@@ -126,10 +128,10 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
|
|
|
126
128
|
if result:
|
|
127
129
|
results_tbl.append(result)
|
|
128
130
|
else:
|
|
129
|
-
logger.info("no returned result (job may still be in progress)")
|
|
131
|
+
mlrun.utils.logger.info("no returned result (job may still be in progress)")
|
|
130
132
|
results_tbl.append(run.to_dict())
|
|
131
133
|
|
|
132
|
-
if mlrun.utils.
|
|
134
|
+
if mlrun.utils.is_jupyter and mlrun.mlconf.ipython_widget:
|
|
133
135
|
results_tbl.show()
|
|
134
136
|
print()
|
|
135
137
|
ui_url = mlrun.utils.get_ui_url(project, uid)
|
|
@@ -145,9 +147,9 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
|
|
|
145
147
|
project_flag = f"-p {project}" if project else ""
|
|
146
148
|
info_cmd = f"mlrun get run {uid} {project_flag}"
|
|
147
149
|
logs_cmd = f"mlrun logs {uid} {project_flag}"
|
|
148
|
-
logger.info(
|
|
150
|
+
mlrun.utils.logger.info(
|
|
149
151
|
"To track results use the CLI", info_cmd=info_cmd, logs_cmd=logs_cmd
|
|
150
152
|
)
|
|
151
153
|
ui_url = mlrun.utils.get_ui_url(project, uid)
|
|
152
154
|
if ui_url:
|
|
153
|
-
logger.info("Or click for UI", ui_url=ui_url)
|
|
155
|
+
mlrun.utils.logger.info("Or click for UI", ui_url=ui_url)
|
mlrun/launcher/factory.py
CHANGED
mlrun/launcher/local.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2023
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -15,6 +15,7 @@ import os
|
|
|
15
15
|
import pathlib
|
|
16
16
|
from typing import Callable, Optional, Union
|
|
17
17
|
|
|
18
|
+
import mlrun.common.constants as mlrun_constants
|
|
18
19
|
import mlrun.common.schemas.schedule
|
|
19
20
|
import mlrun.errors
|
|
20
21
|
import mlrun.launcher.client as launcher
|
|
@@ -68,11 +69,12 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
|
|
|
68
69
|
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
69
70
|
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
70
71
|
state_thresholds: Optional[dict[str, int]] = None,
|
|
72
|
+
reset_on_run: Optional[bool] = None,
|
|
71
73
|
) -> "mlrun.run.RunObject":
|
|
72
74
|
# do not allow local function to be scheduled
|
|
73
|
-
if
|
|
75
|
+
if schedule is not None:
|
|
74
76
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
75
|
-
"
|
|
77
|
+
f"Unexpected {schedule=} parameter for local function execution"
|
|
76
78
|
)
|
|
77
79
|
|
|
78
80
|
self.enrich_runtime(runtime, project)
|
|
@@ -87,6 +89,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
|
|
|
87
89
|
name=name,
|
|
88
90
|
workdir=workdir,
|
|
89
91
|
handler=handler,
|
|
92
|
+
reset_on_run=reset_on_run,
|
|
90
93
|
)
|
|
91
94
|
|
|
92
95
|
# sanity check
|
|
@@ -132,8 +135,13 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
|
|
|
132
135
|
runtime: "mlrun.runtimes.BaseRuntime",
|
|
133
136
|
run: Optional[Union["mlrun.run.RunTemplate", "mlrun.run.RunObject"]] = None,
|
|
134
137
|
):
|
|
135
|
-
if
|
|
136
|
-
|
|
138
|
+
if (
|
|
139
|
+
"V3IO_USERNAME" in os.environ
|
|
140
|
+
and mlrun_constants.MLRunInternalLabels.v3io_user not in run.metadata.labels
|
|
141
|
+
):
|
|
142
|
+
run.metadata.labels[mlrun_constants.MLRunInternalLabels.v3io_user] = (
|
|
143
|
+
os.environ.get("V3IO_USERNAME")
|
|
144
|
+
)
|
|
137
145
|
|
|
138
146
|
# store function object in db unless running from within a run pod
|
|
139
147
|
if not runtime.is_child:
|
|
@@ -206,6 +214,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
|
|
|
206
214
|
name: Optional[str] = "",
|
|
207
215
|
workdir: Optional[str] = "",
|
|
208
216
|
handler: Optional[str] = None,
|
|
217
|
+
reset_on_run: Optional[bool] = None,
|
|
209
218
|
):
|
|
210
219
|
project = project or runtime.metadata.project
|
|
211
220
|
function_name = name or runtime.metadata.name
|
|
@@ -244,6 +253,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
|
|
|
244
253
|
fn.spec.build = runtime.spec.build
|
|
245
254
|
|
|
246
255
|
run.spec.handler = handler
|
|
256
|
+
run.spec.reset_on_run = reset_on_run
|
|
247
257
|
return fn
|
|
248
258
|
|
|
249
259
|
@staticmethod
|
mlrun/launcher/remote.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2023
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -17,6 +17,7 @@ from typing import Optional, Union
|
|
|
17
17
|
import pandas as pd
|
|
18
18
|
import requests
|
|
19
19
|
|
|
20
|
+
import mlrun.common.constants as mlrun_constants
|
|
20
21
|
import mlrun.common.schemas.schedule
|
|
21
22
|
import mlrun.db
|
|
22
23
|
import mlrun.errors
|
|
@@ -58,6 +59,7 @@ class ClientRemoteLauncher(launcher.ClientBaseLauncher):
|
|
|
58
59
|
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
59
60
|
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
60
61
|
state_thresholds: Optional[dict[str, int]] = None,
|
|
62
|
+
reset_on_run: Optional[bool] = None,
|
|
61
63
|
) -> "mlrun.run.RunObject":
|
|
62
64
|
self.enrich_runtime(runtime, project)
|
|
63
65
|
run = self._create_run_object(task)
|
|
@@ -100,8 +102,13 @@ class ClientRemoteLauncher(launcher.ClientBaseLauncher):
|
|
|
100
102
|
if runtime.verbose:
|
|
101
103
|
logger.info(f"runspec:\n{run.to_yaml()}")
|
|
102
104
|
|
|
103
|
-
if
|
|
104
|
-
|
|
105
|
+
if (
|
|
106
|
+
"V3IO_USERNAME" in os.environ
|
|
107
|
+
and mlrun_constants.MLRunInternalLabels.v3io_user not in run.metadata.labels
|
|
108
|
+
):
|
|
109
|
+
run.metadata.labels[mlrun_constants.MLRunInternalLabels.v3io_user] = (
|
|
110
|
+
os.environ.get("V3IO_USERNAME")
|
|
111
|
+
)
|
|
105
112
|
|
|
106
113
|
logger.info(
|
|
107
114
|
"Storing function",
|