mlrun 1.6.4rc7__py3-none-any.whl → 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +11 -1
- mlrun/__main__.py +40 -122
- mlrun/alerts/__init__.py +15 -0
- mlrun/alerts/alert.py +248 -0
- mlrun/api/schemas/__init__.py +5 -4
- mlrun/artifacts/__init__.py +8 -3
- mlrun/artifacts/base.py +47 -257
- mlrun/artifacts/dataset.py +11 -192
- mlrun/artifacts/manager.py +79 -47
- mlrun/artifacts/model.py +31 -159
- mlrun/artifacts/plots.py +23 -380
- mlrun/common/constants.py +74 -1
- mlrun/common/db/sql_session.py +5 -5
- mlrun/common/formatters/__init__.py +21 -0
- mlrun/common/formatters/artifact.py +45 -0
- mlrun/common/formatters/base.py +113 -0
- mlrun/common/formatters/feature_set.py +33 -0
- mlrun/common/formatters/function.py +46 -0
- mlrun/common/formatters/pipeline.py +53 -0
- mlrun/common/formatters/project.py +51 -0
- mlrun/common/formatters/run.py +29 -0
- mlrun/common/helpers.py +12 -3
- mlrun/common/model_monitoring/helpers.py +9 -5
- mlrun/{runtimes → common/runtimes}/constants.py +37 -9
- mlrun/common/schemas/__init__.py +31 -5
- mlrun/common/schemas/alert.py +202 -0
- mlrun/common/schemas/api_gateway.py +196 -0
- mlrun/common/schemas/artifact.py +25 -4
- mlrun/common/schemas/auth.py +16 -5
- mlrun/common/schemas/background_task.py +1 -1
- mlrun/common/schemas/client_spec.py +4 -2
- mlrun/common/schemas/common.py +7 -4
- mlrun/common/schemas/constants.py +3 -0
- mlrun/common/schemas/feature_store.py +74 -44
- mlrun/common/schemas/frontend_spec.py +15 -7
- mlrun/common/schemas/function.py +12 -1
- mlrun/common/schemas/hub.py +11 -18
- mlrun/common/schemas/memory_reports.py +2 -2
- mlrun/common/schemas/model_monitoring/__init__.py +20 -4
- mlrun/common/schemas/model_monitoring/constants.py +123 -42
- mlrun/common/schemas/model_monitoring/grafana.py +13 -9
- mlrun/common/schemas/model_monitoring/model_endpoints.py +101 -54
- mlrun/common/schemas/notification.py +71 -14
- mlrun/common/schemas/object.py +2 -2
- mlrun/{model_monitoring/controller_handler.py → common/schemas/pagination.py} +9 -12
- mlrun/common/schemas/pipeline.py +8 -1
- mlrun/common/schemas/project.py +69 -18
- mlrun/common/schemas/runs.py +7 -1
- mlrun/common/schemas/runtime_resource.py +8 -12
- mlrun/common/schemas/schedule.py +4 -4
- mlrun/common/schemas/tag.py +1 -2
- mlrun/common/schemas/workflow.py +12 -4
- mlrun/common/types.py +14 -1
- mlrun/config.py +154 -69
- mlrun/data_types/data_types.py +6 -1
- mlrun/data_types/spark.py +2 -2
- mlrun/data_types/to_pandas.py +67 -37
- mlrun/datastore/__init__.py +6 -8
- mlrun/datastore/alibaba_oss.py +131 -0
- mlrun/datastore/azure_blob.py +143 -42
- mlrun/datastore/base.py +102 -58
- mlrun/datastore/datastore.py +34 -13
- mlrun/datastore/datastore_profile.py +146 -20
- mlrun/datastore/dbfs_store.py +3 -7
- mlrun/datastore/filestore.py +1 -4
- mlrun/datastore/google_cloud_storage.py +97 -33
- mlrun/datastore/hdfs.py +56 -0
- mlrun/datastore/inmem.py +6 -3
- mlrun/datastore/redis.py +7 -2
- mlrun/datastore/s3.py +34 -12
- mlrun/datastore/snowflake_utils.py +45 -0
- mlrun/datastore/sources.py +303 -111
- mlrun/datastore/spark_utils.py +31 -2
- mlrun/datastore/store_resources.py +9 -7
- mlrun/datastore/storeytargets.py +151 -0
- mlrun/datastore/targets.py +453 -176
- mlrun/datastore/utils.py +72 -58
- mlrun/datastore/v3io.py +6 -1
- mlrun/db/base.py +274 -41
- mlrun/db/factory.py +1 -1
- mlrun/db/httpdb.py +893 -225
- mlrun/db/nopdb.py +291 -33
- mlrun/errors.py +36 -6
- mlrun/execution.py +115 -42
- mlrun/feature_store/__init__.py +0 -2
- mlrun/feature_store/api.py +65 -73
- mlrun/feature_store/common.py +7 -12
- mlrun/feature_store/feature_set.py +76 -55
- mlrun/feature_store/feature_vector.py +39 -31
- mlrun/feature_store/ingestion.py +7 -6
- mlrun/feature_store/retrieval/base.py +16 -11
- mlrun/feature_store/retrieval/dask_merger.py +2 -0
- mlrun/feature_store/retrieval/job.py +13 -4
- mlrun/feature_store/retrieval/local_merger.py +2 -0
- mlrun/feature_store/retrieval/spark_merger.py +24 -32
- mlrun/feature_store/steps.py +45 -34
- mlrun/features.py +11 -21
- mlrun/frameworks/_common/artifacts_library.py +9 -9
- mlrun/frameworks/_common/mlrun_interface.py +5 -5
- mlrun/frameworks/_common/model_handler.py +48 -48
- mlrun/frameworks/_common/plan.py +5 -6
- mlrun/frameworks/_common/producer.py +3 -4
- mlrun/frameworks/_common/utils.py +5 -5
- mlrun/frameworks/_dl_common/loggers/logger.py +6 -7
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +9 -9
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +23 -47
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -2
- mlrun/frameworks/_ml_common/loggers/logger.py +3 -4
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +4 -5
- mlrun/frameworks/_ml_common/model_handler.py +24 -24
- mlrun/frameworks/_ml_common/pkl_model_server.py +2 -2
- mlrun/frameworks/_ml_common/plan.py +2 -2
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +2 -3
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +2 -3
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
- mlrun/frameworks/_ml_common/utils.py +4 -4
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +9 -9
- mlrun/frameworks/huggingface/model_server.py +4 -4
- mlrun/frameworks/lgbm/__init__.py +33 -33
- mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -5
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -5
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -3
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +6 -6
- mlrun/frameworks/lgbm/model_handler.py +10 -10
- mlrun/frameworks/lgbm/model_server.py +6 -6
- mlrun/frameworks/lgbm/utils.py +5 -5
- mlrun/frameworks/onnx/dataset.py +8 -8
- mlrun/frameworks/onnx/mlrun_interface.py +3 -3
- mlrun/frameworks/onnx/model_handler.py +6 -6
- mlrun/frameworks/onnx/model_server.py +7 -7
- mlrun/frameworks/parallel_coordinates.py +6 -6
- mlrun/frameworks/pytorch/__init__.py +18 -18
- mlrun/frameworks/pytorch/callbacks/callback.py +4 -5
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +17 -17
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +11 -11
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +23 -29
- mlrun/frameworks/pytorch/callbacks_handler.py +38 -38
- mlrun/frameworks/pytorch/mlrun_interface.py +20 -20
- mlrun/frameworks/pytorch/model_handler.py +17 -17
- mlrun/frameworks/pytorch/model_server.py +7 -7
- mlrun/frameworks/sklearn/__init__.py +13 -13
- mlrun/frameworks/sklearn/estimator.py +4 -4
- mlrun/frameworks/sklearn/metrics_library.py +14 -14
- mlrun/frameworks/sklearn/mlrun_interface.py +16 -9
- mlrun/frameworks/sklearn/model_handler.py +2 -2
- mlrun/frameworks/tf_keras/__init__.py +10 -7
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +15 -15
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +11 -11
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +19 -23
- mlrun/frameworks/tf_keras/mlrun_interface.py +9 -11
- mlrun/frameworks/tf_keras/model_handler.py +14 -14
- mlrun/frameworks/tf_keras/model_server.py +6 -6
- mlrun/frameworks/xgboost/__init__.py +13 -13
- mlrun/frameworks/xgboost/model_handler.py +6 -6
- mlrun/k8s_utils.py +61 -17
- mlrun/launcher/__init__.py +1 -1
- mlrun/launcher/base.py +16 -15
- mlrun/launcher/client.py +13 -11
- mlrun/launcher/factory.py +1 -1
- mlrun/launcher/local.py +23 -13
- mlrun/launcher/remote.py +17 -10
- mlrun/lists.py +7 -6
- mlrun/model.py +478 -103
- mlrun/model_monitoring/__init__.py +1 -1
- mlrun/model_monitoring/api.py +163 -371
- mlrun/{runtimes/mpijob/v1alpha1.py → model_monitoring/applications/__init__.py} +9 -15
- mlrun/model_monitoring/applications/_application_steps.py +188 -0
- mlrun/model_monitoring/applications/base.py +108 -0
- mlrun/model_monitoring/applications/context.py +341 -0
- mlrun/model_monitoring/{evidently_application.py → applications/evidently_base.py} +27 -22
- mlrun/model_monitoring/applications/histogram_data_drift.py +354 -0
- mlrun/model_monitoring/applications/results.py +99 -0
- mlrun/model_monitoring/controller.py +131 -278
- mlrun/model_monitoring/db/__init__.py +18 -0
- mlrun/model_monitoring/db/stores/__init__.py +136 -0
- mlrun/model_monitoring/db/stores/base/__init__.py +15 -0
- mlrun/model_monitoring/db/stores/base/store.py +213 -0
- mlrun/model_monitoring/db/stores/sqldb/__init__.py +13 -0
- mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +71 -0
- mlrun/model_monitoring/db/stores/sqldb/models/base.py +190 -0
- mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +103 -0
- mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py} +19 -13
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +659 -0
- mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +13 -0
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +726 -0
- mlrun/model_monitoring/db/tsdb/__init__.py +105 -0
- mlrun/model_monitoring/db/tsdb/base.py +448 -0
- mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
- mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +279 -0
- mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +42 -0
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +507 -0
- mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +158 -0
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +849 -0
- mlrun/model_monitoring/features_drift_table.py +134 -106
- mlrun/model_monitoring/helpers.py +199 -55
- mlrun/model_monitoring/metrics/__init__.py +13 -0
- mlrun/model_monitoring/metrics/histogram_distance.py +127 -0
- mlrun/model_monitoring/model_endpoint.py +3 -2
- mlrun/model_monitoring/stream_processing.py +131 -398
- mlrun/model_monitoring/tracking_policy.py +9 -2
- mlrun/model_monitoring/writer.py +161 -125
- mlrun/package/__init__.py +6 -6
- mlrun/package/context_handler.py +5 -5
- mlrun/package/packager.py +7 -7
- mlrun/package/packagers/default_packager.py +8 -8
- mlrun/package/packagers/numpy_packagers.py +15 -15
- mlrun/package/packagers/pandas_packagers.py +5 -5
- mlrun/package/packagers/python_standard_library_packagers.py +10 -10
- mlrun/package/packagers_manager.py +19 -23
- mlrun/package/utils/_formatter.py +6 -6
- mlrun/package/utils/_pickler.py +2 -2
- mlrun/package/utils/_supported_format.py +4 -4
- mlrun/package/utils/log_hint_utils.py +2 -2
- mlrun/package/utils/type_hint_utils.py +4 -9
- mlrun/platforms/__init__.py +11 -10
- mlrun/platforms/iguazio.py +24 -203
- mlrun/projects/operations.py +52 -25
- mlrun/projects/pipelines.py +191 -197
- mlrun/projects/project.py +1227 -400
- mlrun/render.py +16 -19
- mlrun/run.py +209 -184
- mlrun/runtimes/__init__.py +83 -15
- mlrun/runtimes/base.py +51 -35
- mlrun/runtimes/daskjob.py +17 -10
- mlrun/runtimes/databricks_job/databricks_cancel_task.py +1 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +8 -7
- mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
- mlrun/runtimes/funcdoc.py +1 -29
- mlrun/runtimes/function_reference.py +1 -1
- mlrun/runtimes/kubejob.py +34 -128
- mlrun/runtimes/local.py +40 -11
- mlrun/runtimes/mpijob/__init__.py +0 -20
- mlrun/runtimes/mpijob/abstract.py +9 -10
- mlrun/runtimes/mpijob/v1.py +1 -1
- mlrun/{model_monitoring/stores/models/sqlite.py → runtimes/nuclio/__init__.py} +7 -9
- mlrun/runtimes/nuclio/api_gateway.py +769 -0
- mlrun/runtimes/nuclio/application/__init__.py +15 -0
- mlrun/runtimes/nuclio/application/application.py +758 -0
- mlrun/runtimes/nuclio/application/reverse_proxy.go +95 -0
- mlrun/runtimes/{function.py → nuclio/function.py} +200 -83
- mlrun/runtimes/{nuclio.py → nuclio/nuclio.py} +6 -6
- mlrun/runtimes/{serving.py → nuclio/serving.py} +65 -68
- mlrun/runtimes/pod.py +281 -101
- mlrun/runtimes/remotesparkjob.py +12 -9
- mlrun/runtimes/sparkjob/spark3job.py +67 -51
- mlrun/runtimes/utils.py +41 -75
- mlrun/secrets.py +9 -5
- mlrun/serving/__init__.py +8 -1
- mlrun/serving/remote.py +2 -7
- mlrun/serving/routers.py +85 -69
- mlrun/serving/server.py +69 -44
- mlrun/serving/states.py +209 -36
- mlrun/serving/utils.py +22 -14
- mlrun/serving/v1_serving.py +6 -7
- mlrun/serving/v2_serving.py +129 -54
- mlrun/track/tracker.py +2 -1
- mlrun/track/tracker_manager.py +3 -3
- mlrun/track/trackers/mlflow_tracker.py +6 -2
- mlrun/utils/async_http.py +6 -8
- mlrun/utils/azure_vault.py +1 -1
- mlrun/utils/clones.py +1 -2
- mlrun/utils/condition_evaluator.py +3 -3
- mlrun/utils/db.py +21 -3
- mlrun/utils/helpers.py +405 -225
- mlrun/utils/http.py +3 -6
- mlrun/utils/logger.py +112 -16
- mlrun/utils/notifications/notification/__init__.py +17 -13
- mlrun/utils/notifications/notification/base.py +50 -2
- mlrun/utils/notifications/notification/console.py +2 -0
- mlrun/utils/notifications/notification/git.py +24 -1
- mlrun/utils/notifications/notification/ipython.py +3 -1
- mlrun/utils/notifications/notification/slack.py +96 -21
- mlrun/utils/notifications/notification/webhook.py +59 -2
- mlrun/utils/notifications/notification_pusher.py +149 -30
- mlrun/utils/regex.py +9 -0
- mlrun/utils/retryer.py +208 -0
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/v3io_clients.py +4 -6
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +2 -6
- mlrun-1.7.0.dist-info/METADATA +378 -0
- mlrun-1.7.0.dist-info/RECORD +351 -0
- {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/WHEEL +1 -1
- mlrun/feature_store/retrieval/conversion.py +0 -273
- mlrun/kfpops.py +0 -868
- mlrun/model_monitoring/application.py +0 -310
- mlrun/model_monitoring/batch.py +0 -1095
- mlrun/model_monitoring/prometheus.py +0 -219
- mlrun/model_monitoring/stores/__init__.py +0 -111
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +0 -576
- mlrun/model_monitoring/stores/model_endpoint_store.py +0 -147
- mlrun/model_monitoring/stores/models/__init__.py +0 -27
- mlrun/model_monitoring/stores/models/base.py +0 -84
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -384
- mlrun/platforms/other.py +0 -306
- mlrun-1.6.4rc7.dist-info/METADATA +0 -272
- mlrun-1.6.4rc7.dist-info/RECORD +0 -314
- {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/LICENSE +0 -0
- {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/entry_points.txt +0 -0
- {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/top_level.txt +0 -0
|
@@ -16,7 +16,7 @@ import os
|
|
|
16
16
|
import pathlib
|
|
17
17
|
import tempfile
|
|
18
18
|
from abc import ABC, abstractmethod
|
|
19
|
-
from typing import Any,
|
|
19
|
+
from typing import Any, Union
|
|
20
20
|
|
|
21
21
|
import numpy as np
|
|
22
22
|
import pandas as pd
|
|
@@ -29,7 +29,7 @@ from ..utils import ArtifactType, SupportedFormat
|
|
|
29
29
|
from .default_packager import DefaultPackager
|
|
30
30
|
|
|
31
31
|
# Type for collection of numpy arrays (list / dict of arrays):
|
|
32
|
-
NumPyArrayCollectionType = Union[
|
|
32
|
+
NumPyArrayCollectionType = Union[list[np.ndarray], dict[str, np.ndarray]]
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
class _Formatter(ABC):
|
|
@@ -194,7 +194,7 @@ class _NPZFormatter(_Formatter):
|
|
|
194
194
|
save_function(file_path, **obj)
|
|
195
195
|
|
|
196
196
|
@classmethod
|
|
197
|
-
def load(cls, file_path: str, **load_kwargs: dict) ->
|
|
197
|
+
def load(cls, file_path: str, **load_kwargs: dict) -> dict[str, np.ndarray]:
|
|
198
198
|
"""
|
|
199
199
|
Load the arrays from the given 'npz' file path.
|
|
200
200
|
|
|
@@ -226,7 +226,7 @@ class NumPySupportedFormat(SupportedFormat[_Formatter]):
|
|
|
226
226
|
}
|
|
227
227
|
|
|
228
228
|
@classmethod
|
|
229
|
-
def get_single_array_formats(cls) ->
|
|
229
|
+
def get_single_array_formats(cls) -> list[str]:
|
|
230
230
|
"""
|
|
231
231
|
Get the supported formats for saving one numpy array.
|
|
232
232
|
|
|
@@ -235,7 +235,7 @@ class NumPySupportedFormat(SupportedFormat[_Formatter]):
|
|
|
235
235
|
return [cls.NPY, cls.TXT, cls.GZ, cls.CSV]
|
|
236
236
|
|
|
237
237
|
@classmethod
|
|
238
|
-
def get_multi_array_formats(cls) ->
|
|
238
|
+
def get_multi_array_formats(cls) -> list[str]:
|
|
239
239
|
"""
|
|
240
240
|
Get the supported formats for saving a collection (multiple) numpy arrays - e.g. list of arrays or dictionary of
|
|
241
241
|
arrays.
|
|
@@ -310,7 +310,7 @@ class NumPyNDArrayPackager(DefaultPackager):
|
|
|
310
310
|
key: str,
|
|
311
311
|
file_format: str = DEFAULT_NUMPY_ARRAY_FORMAT,
|
|
312
312
|
**save_kwargs,
|
|
313
|
-
) ->
|
|
313
|
+
) -> tuple[Artifact, dict]:
|
|
314
314
|
"""
|
|
315
315
|
Pack an array as a file by the given format.
|
|
316
316
|
|
|
@@ -342,7 +342,7 @@ class NumPyNDArrayPackager(DefaultPackager):
|
|
|
342
342
|
obj: np.ndarray,
|
|
343
343
|
key: str,
|
|
344
344
|
file_format: str = "",
|
|
345
|
-
) ->
|
|
345
|
+
) -> tuple[Artifact, dict]:
|
|
346
346
|
"""
|
|
347
347
|
Pack an array as a dataset.
|
|
348
348
|
|
|
@@ -442,7 +442,7 @@ class _NumPyNDArrayCollectionPackager(DefaultPackager):
|
|
|
442
442
|
key: str,
|
|
443
443
|
file_format: str = DEFAULT_NUMPPY_ARRAY_COLLECTION_FORMAT,
|
|
444
444
|
**save_kwargs,
|
|
445
|
-
) ->
|
|
445
|
+
) -> tuple[Artifact, dict]:
|
|
446
446
|
"""
|
|
447
447
|
Pack an array collection as a file by the given format.
|
|
448
448
|
|
|
@@ -476,7 +476,7 @@ class _NumPyNDArrayCollectionPackager(DefaultPackager):
|
|
|
476
476
|
data_item: DataItem,
|
|
477
477
|
file_format: str = None,
|
|
478
478
|
allow_pickle: bool = False,
|
|
479
|
-
) ->
|
|
479
|
+
) -> dict[str, np.ndarray]:
|
|
480
480
|
"""
|
|
481
481
|
Unpack a numppy array collection from file.
|
|
482
482
|
|
|
@@ -545,7 +545,7 @@ class NumPyNDArrayDictPackager(_NumPyNDArrayCollectionPackager):
|
|
|
545
545
|
``dict[str, numpy.ndarray]`` packager.
|
|
546
546
|
"""
|
|
547
547
|
|
|
548
|
-
PACKABLE_OBJECT_TYPE =
|
|
548
|
+
PACKABLE_OBJECT_TYPE = dict[str, np.ndarray]
|
|
549
549
|
|
|
550
550
|
def is_packable(
|
|
551
551
|
self, obj: Any, artifact_type: str = None, configurations: dict = None
|
|
@@ -583,7 +583,7 @@ class NumPyNDArrayDictPackager(_NumPyNDArrayCollectionPackager):
|
|
|
583
583
|
|
|
584
584
|
return True
|
|
585
585
|
|
|
586
|
-
def pack_result(self, obj:
|
|
586
|
+
def pack_result(self, obj: dict[str, np.ndarray], key: str) -> dict:
|
|
587
587
|
"""
|
|
588
588
|
Pack a dictionary of numpy arrays as a result.
|
|
589
589
|
|
|
@@ -604,7 +604,7 @@ class NumPyNDArrayDictPackager(_NumPyNDArrayCollectionPackager):
|
|
|
604
604
|
data_item: DataItem,
|
|
605
605
|
file_format: str = None,
|
|
606
606
|
allow_pickle: bool = False,
|
|
607
|
-
) ->
|
|
607
|
+
) -> dict[str, np.ndarray]:
|
|
608
608
|
"""
|
|
609
609
|
Unpack a numppy array dictionary from file.
|
|
610
610
|
|
|
@@ -630,7 +630,7 @@ class NumPyNDArrayListPackager(_NumPyNDArrayCollectionPackager):
|
|
|
630
630
|
``list[numpy.ndarray]`` packager.
|
|
631
631
|
"""
|
|
632
632
|
|
|
633
|
-
PACKABLE_OBJECT_TYPE =
|
|
633
|
+
PACKABLE_OBJECT_TYPE = list[np.ndarray]
|
|
634
634
|
|
|
635
635
|
def is_packable(
|
|
636
636
|
self, obj: Any, artifact_type: str = None, configurations: dict = None
|
|
@@ -665,7 +665,7 @@ class NumPyNDArrayListPackager(_NumPyNDArrayCollectionPackager):
|
|
|
665
665
|
|
|
666
666
|
return True
|
|
667
667
|
|
|
668
|
-
def pack_result(self, obj:
|
|
668
|
+
def pack_result(self, obj: list[np.ndarray], key: str) -> dict:
|
|
669
669
|
"""
|
|
670
670
|
Pack a list of numpy arrays as a result.
|
|
671
671
|
|
|
@@ -681,7 +681,7 @@ class NumPyNDArrayListPackager(_NumPyNDArrayCollectionPackager):
|
|
|
681
681
|
data_item: DataItem,
|
|
682
682
|
file_format: str = None,
|
|
683
683
|
allow_pickle: bool = False,
|
|
684
|
-
) ->
|
|
684
|
+
) -> list[np.ndarray]:
|
|
685
685
|
"""
|
|
686
686
|
Unpack a numppy array list from file.
|
|
687
687
|
|
|
@@ -17,7 +17,7 @@ import os
|
|
|
17
17
|
import pathlib
|
|
18
18
|
import tempfile
|
|
19
19
|
from abc import ABC, abstractmethod
|
|
20
|
-
from typing import Any,
|
|
20
|
+
from typing import Any, Union
|
|
21
21
|
|
|
22
22
|
import pandas as pd
|
|
23
23
|
|
|
@@ -70,7 +70,7 @@ class _Formatter(ABC):
|
|
|
70
70
|
pass
|
|
71
71
|
|
|
72
72
|
@staticmethod
|
|
73
|
-
def _flatten_dataframe(dataframe: pd.DataFrame) ->
|
|
73
|
+
def _flatten_dataframe(dataframe: pd.DataFrame) -> tuple[pd.DataFrame, dict]:
|
|
74
74
|
"""
|
|
75
75
|
Flatten the dataframe: moving all indexes to be columns at the start (from column 0) and lowering the columns
|
|
76
76
|
levels to 1, renaming them from tuples. All columns and index info is stored so it can be unflatten later on.
|
|
@@ -733,7 +733,7 @@ class PandasDataFramePackager(DefaultPackager):
|
|
|
733
733
|
file_format: str = None,
|
|
734
734
|
flatten: bool = True,
|
|
735
735
|
**to_kwargs,
|
|
736
|
-
) ->
|
|
736
|
+
) -> tuple[Artifact, dict]:
|
|
737
737
|
"""
|
|
738
738
|
Pack a dataframe as a file by the given format.
|
|
739
739
|
|
|
@@ -857,7 +857,7 @@ class PandasSeriesPackager(PandasDataFramePackager):
|
|
|
857
857
|
PACKABLE_OBJECT_TYPE = pd.Series
|
|
858
858
|
DEFAULT_PACKING_ARTIFACT_TYPE = ArtifactType.FILE
|
|
859
859
|
|
|
860
|
-
def get_supported_artifact_types(self) ->
|
|
860
|
+
def get_supported_artifact_types(self) -> list[str]:
|
|
861
861
|
"""
|
|
862
862
|
Get all the supported artifact types on this packager. It will be the same as `PandasDataFramePackager` but
|
|
863
863
|
without the 'dataset' artifact type support.
|
|
@@ -886,7 +886,7 @@ class PandasSeriesPackager(PandasDataFramePackager):
|
|
|
886
886
|
file_format: str = None,
|
|
887
887
|
flatten: bool = True,
|
|
888
888
|
**to_kwargs,
|
|
889
|
-
) ->
|
|
889
|
+
) -> tuple[Artifact, dict]:
|
|
890
890
|
"""
|
|
891
891
|
Pack a series as a file by the given format.
|
|
892
892
|
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import os
|
|
16
16
|
import pathlib
|
|
17
17
|
import tempfile
|
|
18
|
-
from typing import
|
|
18
|
+
from typing import Union
|
|
19
19
|
|
|
20
20
|
from mlrun.artifacts import Artifact
|
|
21
21
|
from mlrun.datastore import DataItem
|
|
@@ -45,7 +45,7 @@ class NonePackager(DefaultPackager):
|
|
|
45
45
|
DEFAULT_PACKING_ARTIFACT_TYPE = ArtifactType.RESULT
|
|
46
46
|
|
|
47
47
|
# TODO: `None` as pickle will be available from Python 3.10, so this method can be removed once we move to 3.10.
|
|
48
|
-
def get_supported_artifact_types(self) ->
|
|
48
|
+
def get_supported_artifact_types(self) -> list[str]:
|
|
49
49
|
"""
|
|
50
50
|
Get all the supported artifact types on this packager. It will be the same as `DefaultPackager` but without the
|
|
51
51
|
'object' artifact type support (None cannot be pickled, only from Python 3.10, and it should not be pickled
|
|
@@ -96,7 +96,7 @@ class StrPackager(DefaultPackager):
|
|
|
96
96
|
|
|
97
97
|
def pack_path(
|
|
98
98
|
self, obj: str, key: str, archive_format: str = DEFAULT_ARCHIVE_FORMAT
|
|
99
|
-
) ->
|
|
99
|
+
) -> tuple[Artifact, dict]:
|
|
100
100
|
"""
|
|
101
101
|
Pack a path string value content (pack the file or directory in that path).
|
|
102
102
|
|
|
@@ -198,7 +198,7 @@ class _BuiltinCollectionPackager(DefaultPackager):
|
|
|
198
198
|
obj: Union[dict, list],
|
|
199
199
|
key: str,
|
|
200
200
|
file_format: str = DEFAULT_STRUCT_FILE_FORMAT,
|
|
201
|
-
) ->
|
|
201
|
+
) -> tuple[Artifact, dict]:
|
|
202
202
|
"""
|
|
203
203
|
Pack a builtin collection as a file by the given format.
|
|
204
204
|
|
|
@@ -343,7 +343,7 @@ class TuplePackager(ListPackager):
|
|
|
343
343
|
|
|
344
344
|
def pack_file(
|
|
345
345
|
self, obj: tuple, key: str, file_format: str = DEFAULT_STRUCT_FILE_FORMAT
|
|
346
|
-
) ->
|
|
346
|
+
) -> tuple[Artifact, dict]:
|
|
347
347
|
"""
|
|
348
348
|
Pack a tuple as a file by the given format.
|
|
349
349
|
|
|
@@ -388,7 +388,7 @@ class SetPackager(ListPackager):
|
|
|
388
388
|
|
|
389
389
|
def pack_file(
|
|
390
390
|
self, obj: set, key: str, file_format: str = DEFAULT_STRUCT_FILE_FORMAT
|
|
391
|
-
) ->
|
|
391
|
+
) -> tuple[Artifact, dict]:
|
|
392
392
|
"""
|
|
393
393
|
Pack a set as a file by the given format.
|
|
394
394
|
|
|
@@ -422,7 +422,7 @@ class FrozensetPackager(SetPackager):
|
|
|
422
422
|
|
|
423
423
|
def pack_file(
|
|
424
424
|
self, obj: frozenset, key: str, file_format: str = DEFAULT_STRUCT_FILE_FORMAT
|
|
425
|
-
) ->
|
|
425
|
+
) -> tuple[Artifact, dict]:
|
|
426
426
|
"""
|
|
427
427
|
Pack a frozenset as a file by the given format.
|
|
428
428
|
|
|
@@ -469,7 +469,7 @@ class BytesPackager(ListPackager):
|
|
|
469
469
|
|
|
470
470
|
def pack_file(
|
|
471
471
|
self, obj: bytes, key: str, file_format: str = DEFAULT_STRUCT_FILE_FORMAT
|
|
472
|
-
) ->
|
|
472
|
+
) -> tuple[Artifact, dict]:
|
|
473
473
|
"""
|
|
474
474
|
Pack a bytes as a file by the given format.
|
|
475
475
|
|
|
@@ -514,7 +514,7 @@ class BytearrayPackager(BytesPackager):
|
|
|
514
514
|
|
|
515
515
|
def pack_file(
|
|
516
516
|
self, obj: bytearray, key: str, file_format: str = DEFAULT_STRUCT_FILE_FORMAT
|
|
517
|
-
) ->
|
|
517
|
+
) -> tuple[Artifact, dict]:
|
|
518
518
|
"""
|
|
519
519
|
Pack a bytearray as a file by the given format.
|
|
520
520
|
|
|
@@ -569,7 +569,7 @@ class PathPackager(StrPackager):
|
|
|
569
569
|
|
|
570
570
|
def pack_path(
|
|
571
571
|
self, obj: pathlib.Path, key: str, archive_format: str = DEFAULT_ARCHIVE_FORMAT
|
|
572
|
-
) ->
|
|
572
|
+
) -> tuple[Artifact, dict]:
|
|
573
573
|
"""
|
|
574
574
|
Pack a `Path` value (pack the file or directory in that path).
|
|
575
575
|
|
|
@@ -16,9 +16,8 @@ import importlib
|
|
|
16
16
|
import inspect
|
|
17
17
|
import os
|
|
18
18
|
import shutil
|
|
19
|
-
import sys
|
|
20
19
|
import traceback
|
|
21
|
-
from typing import Any,
|
|
20
|
+
from typing import Any, Union
|
|
22
21
|
|
|
23
22
|
from mlrun.artifacts import Artifact
|
|
24
23
|
from mlrun.datastore import DataItem, store_manager
|
|
@@ -42,7 +41,7 @@ class PackagersManager:
|
|
|
42
41
|
It prepares the instructions / log hint configurations and then looks for the first packager that fits the task.
|
|
43
42
|
"""
|
|
44
43
|
|
|
45
|
-
def __init__(self, default_packager:
|
|
44
|
+
def __init__(self, default_packager: type[Packager] = None):
|
|
46
45
|
"""
|
|
47
46
|
Initialize a packagers manager.
|
|
48
47
|
|
|
@@ -54,15 +53,15 @@ class PackagersManager:
|
|
|
54
53
|
self._default_packager = (default_packager or DefaultPackager)()
|
|
55
54
|
|
|
56
55
|
# Initialize the packagers list (with the default packager in it):
|
|
57
|
-
self._packagers:
|
|
56
|
+
self._packagers: list[Packager] = []
|
|
58
57
|
|
|
59
58
|
# Set an artifacts list and results dictionary to collect all packed objects (will be used later to write extra
|
|
60
59
|
# data if noted by the user using the log hint key "extra_data")
|
|
61
|
-
self._artifacts:
|
|
60
|
+
self._artifacts: list[Artifact] = []
|
|
62
61
|
self._results = {}
|
|
63
62
|
|
|
64
63
|
@property
|
|
65
|
-
def artifacts(self) ->
|
|
64
|
+
def artifacts(self) -> list[Artifact]:
|
|
66
65
|
"""
|
|
67
66
|
Get the artifacts that were packed by the manager.
|
|
68
67
|
|
|
@@ -80,7 +79,7 @@ class PackagersManager:
|
|
|
80
79
|
return self._results
|
|
81
80
|
|
|
82
81
|
def collect_packagers(
|
|
83
|
-
self, packagers:
|
|
82
|
+
self, packagers: list[Union[type[Packager], str]], default_priority: int = 5
|
|
84
83
|
):
|
|
85
84
|
"""
|
|
86
85
|
Collect the provided packagers. Packagers passed as module paths are imported and validated to be of type
|
|
@@ -93,6 +92,7 @@ class PackagersManager:
|
|
|
93
92
|
from mlrun import Packager
|
|
94
93
|
from x import XPackager
|
|
95
94
|
|
|
95
|
+
|
|
96
96
|
class YPackager(Packager):
|
|
97
97
|
pass
|
|
98
98
|
|
|
@@ -171,8 +171,8 @@ class PackagersManager:
|
|
|
171
171
|
self._packagers.sort()
|
|
172
172
|
|
|
173
173
|
def pack(
|
|
174
|
-
self, obj: Any, log_hint:
|
|
175
|
-
) -> Union[Artifact, dict, None,
|
|
174
|
+
self, obj: Any, log_hint: dict[str, str]
|
|
175
|
+
) -> Union[Artifact, dict, None, list[Union[Artifact, dict, None]]]:
|
|
176
176
|
"""
|
|
177
177
|
Pack an object using one of the manager's packagers. A `dict` ("**") or `list` ("*") unpacking syntax in the
|
|
178
178
|
log hint key packs the objects within them in separate packages.
|
|
@@ -244,7 +244,7 @@ class PackagersManager:
|
|
|
244
244
|
# If multiple packages were packed, return a list, otherwise return the single package:
|
|
245
245
|
return packages if len(packages) > 1 else packages[0]
|
|
246
246
|
|
|
247
|
-
def unpack(self, data_item: DataItem, type_hint:
|
|
247
|
+
def unpack(self, data_item: DataItem, type_hint: type) -> Any:
|
|
248
248
|
"""
|
|
249
249
|
Unpack an object using one of the manager's packagers. The data item can be unpacked in two ways:
|
|
250
250
|
|
|
@@ -264,11 +264,7 @@ class PackagersManager:
|
|
|
264
264
|
:return: The unpacked object parsed as type hinted.
|
|
265
265
|
"""
|
|
266
266
|
# Check if `DataItem` is hinted - meaning the user can expect a data item and do not want to unpack it:
|
|
267
|
-
|
|
268
|
-
if sys.version_info[1] < 8:
|
|
269
|
-
if self._get_type_name(typ=DataItem) in str(type_hint):
|
|
270
|
-
return data_item
|
|
271
|
-
elif TypeHintUtils.is_matching(object_type=DataItem, type_hint=type_hint):
|
|
267
|
+
if TypeHintUtils.is_matching(object_type=DataItem, type_hint=type_hint):
|
|
272
268
|
return data_item
|
|
273
269
|
|
|
274
270
|
# Set variables to hold the manager notes and packager instructions:
|
|
@@ -306,7 +302,7 @@ class PackagersManager:
|
|
|
306
302
|
|
|
307
303
|
def link_packages(
|
|
308
304
|
self,
|
|
309
|
-
additional_artifacts:
|
|
305
|
+
additional_artifacts: list[Artifact],
|
|
310
306
|
additional_results: dict,
|
|
311
307
|
):
|
|
312
308
|
"""
|
|
@@ -371,7 +367,7 @@ class PackagersManager:
|
|
|
371
367
|
ARTIFACT_TYPE = "artifact_type"
|
|
372
368
|
INSTRUCTIONS = "instructions"
|
|
373
369
|
|
|
374
|
-
def _get_packagers_with_default_packager(self) ->
|
|
370
|
+
def _get_packagers_with_default_packager(self) -> list[Packager]:
|
|
375
371
|
"""
|
|
376
372
|
Get the full list of packagers - the collected packagers and the default packager (located at last place in the
|
|
377
373
|
list - the lowest priority).
|
|
@@ -635,7 +631,7 @@ class PackagersManager:
|
|
|
635
631
|
)
|
|
636
632
|
return self._unpack_data_item(data_item=data_item, type_hint=type_hint)
|
|
637
633
|
|
|
638
|
-
def _unpack_data_item(self, data_item: DataItem, type_hint:
|
|
634
|
+
def _unpack_data_item(self, data_item: DataItem, type_hint: type):
|
|
639
635
|
"""
|
|
640
636
|
Unpack a data item to the desired hinted type. In case the type hint includes multiple types (as in the case of
|
|
641
637
|
`typing.Union`), the manager goes over the types, and reduces them while looking for the first packager that
|
|
@@ -649,7 +645,7 @@ class PackagersManager:
|
|
|
649
645
|
:raise MLRunPackageUnpackingError: If there is no packager that supports the provided type hint.
|
|
650
646
|
"""
|
|
651
647
|
# Prepare a list of a packager and exception string for all the failures in case there was no fitting packager:
|
|
652
|
-
found_packagers:
|
|
648
|
+
found_packagers: list[tuple[Packager, str]] = []
|
|
653
649
|
|
|
654
650
|
# Try to unpack as one of the possible types in the type hint:
|
|
655
651
|
possible_type_hints = {type_hint}
|
|
@@ -718,7 +714,7 @@ class PackagersManager:
|
|
|
718
714
|
@staticmethod
|
|
719
715
|
def _look_for_extra_data(
|
|
720
716
|
key: str,
|
|
721
|
-
artifacts:
|
|
717
|
+
artifacts: list[Artifact],
|
|
722
718
|
results: dict,
|
|
723
719
|
) -> Union[Artifact, str, int, float, None]:
|
|
724
720
|
"""
|
|
@@ -739,7 +735,7 @@ class PackagersManager:
|
|
|
739
735
|
return results.get(key, None)
|
|
740
736
|
|
|
741
737
|
@staticmethod
|
|
742
|
-
def _split_module_path(module_path: str) ->
|
|
738
|
+
def _split_module_path(module_path: str) -> tuple[str, str]:
|
|
743
739
|
"""
|
|
744
740
|
Split a module path to the module name and the class name. Inner classes are not supported.
|
|
745
741
|
|
|
@@ -756,7 +752,7 @@ class PackagersManager:
|
|
|
756
752
|
return module_name, class_name
|
|
757
753
|
|
|
758
754
|
@staticmethod
|
|
759
|
-
def _get_type_name(typ:
|
|
755
|
+
def _get_type_name(typ: type) -> str:
|
|
760
756
|
"""
|
|
761
757
|
Get an object type full name - its module path. For example, the name of a pandas data frame is "DataFrame"
|
|
762
758
|
but its full name (module path) is: "pandas.core.frame.DataFrame".
|
|
@@ -777,7 +773,7 @@ class PackagersManager:
|
|
|
777
773
|
return f"{module_name}.{class_name}" if module_name else class_name
|
|
778
774
|
|
|
779
775
|
@staticmethod
|
|
780
|
-
def _get_type_from_name(type_name: str) ->
|
|
776
|
+
def _get_type_from_name(type_name: str) -> type:
|
|
781
777
|
"""
|
|
782
778
|
Get the type object out of the given module path. The module must be a full module path (for example:
|
|
783
779
|
"pandas.DataFrame" and not "DataFrame") otherwise it assumes to be from the local run module - __main__.
|
|
@@ -82,7 +82,7 @@ class _JSONFormatter(_Formatter):
|
|
|
82
82
|
|
|
83
83
|
:return: The read object.
|
|
84
84
|
"""
|
|
85
|
-
with open(file_path
|
|
85
|
+
with open(file_path) as file:
|
|
86
86
|
obj = json.load(file)
|
|
87
87
|
return obj
|
|
88
88
|
|
|
@@ -117,7 +117,7 @@ class _JSONLFormatter(_Formatter):
|
|
|
117
117
|
|
|
118
118
|
:return: The read object.
|
|
119
119
|
"""
|
|
120
|
-
with open(file_path
|
|
120
|
+
with open(file_path) as file:
|
|
121
121
|
lines = file.readlines()
|
|
122
122
|
|
|
123
123
|
obj = []
|
|
@@ -142,11 +142,11 @@ class _YAMLFormatter(_Formatter):
|
|
|
142
142
|
|
|
143
143
|
:param obj: The object to write.
|
|
144
144
|
:param file_path: The file path to write to.
|
|
145
|
-
:param dump_kwargs: Additional keyword arguments to pass to the `yaml.
|
|
145
|
+
:param dump_kwargs: Additional keyword arguments to pass to the `yaml.safe_dump` method of the formatter in use.
|
|
146
146
|
"""
|
|
147
147
|
dump_kwargs = dump_kwargs or cls.DEFAULT_DUMP_KWARGS
|
|
148
148
|
with open(file_path, "w") as file:
|
|
149
|
-
yaml.
|
|
149
|
+
yaml.safe_dump(obj, file, **dump_kwargs)
|
|
150
150
|
|
|
151
151
|
@classmethod
|
|
152
152
|
def read(cls, file_path: str) -> Union[list, dict]:
|
|
@@ -157,7 +157,7 @@ class _YAMLFormatter(_Formatter):
|
|
|
157
157
|
|
|
158
158
|
:return: The read object.
|
|
159
159
|
"""
|
|
160
|
-
with open(file_path
|
|
160
|
+
with open(file_path) as file:
|
|
161
161
|
obj = yaml.safe_load(file)
|
|
162
162
|
return obj
|
|
163
163
|
|
|
@@ -188,7 +188,7 @@ class _TXTFormatter(_Formatter):
|
|
|
188
188
|
|
|
189
189
|
:return: The read object.
|
|
190
190
|
"""
|
|
191
|
-
with open(file_path
|
|
191
|
+
with open(file_path) as file:
|
|
192
192
|
obj = ast.literal_eval(file.read())
|
|
193
193
|
return obj
|
|
194
194
|
|
mlrun/package/utils/_pickler.py
CHANGED
|
@@ -19,7 +19,7 @@ import sys
|
|
|
19
19
|
import tempfile
|
|
20
20
|
import warnings
|
|
21
21
|
from types import ModuleType
|
|
22
|
-
from typing import Any,
|
|
22
|
+
from typing import Any, Union
|
|
23
23
|
|
|
24
24
|
from mlrun.errors import MLRunInvalidArgumentError
|
|
25
25
|
from mlrun.utils import logger
|
|
@@ -35,7 +35,7 @@ class Pickler:
|
|
|
35
35
|
@staticmethod
|
|
36
36
|
def pickle(
|
|
37
37
|
obj: Any, pickle_module_name: str, output_path: str = None
|
|
38
|
-
) ->
|
|
38
|
+
) -> tuple[str, dict[str, Union[str, None]]]:
|
|
39
39
|
"""
|
|
40
40
|
Pickle an object using the given module. The pickled object will be saved to file to the given output path.
|
|
41
41
|
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
15
|
from abc import ABC
|
|
16
|
-
from typing import
|
|
16
|
+
from typing import Generic, TypeVar, Union
|
|
17
17
|
|
|
18
18
|
# A generic type for a supported format handler class type:
|
|
19
19
|
FileHandlerType = TypeVar("FileHandlerType")
|
|
@@ -29,10 +29,10 @@ class SupportedFormat(ABC, Generic[FileHandlerType]):
|
|
|
29
29
|
|
|
30
30
|
# The map to use in the method `get_format_handler`. A dictionary of string key to a class type to handle that
|
|
31
31
|
# format. New supported formats and handlers should be added to it:
|
|
32
|
-
_FORMAT_HANDLERS_MAP:
|
|
32
|
+
_FORMAT_HANDLERS_MAP: dict[str, type[FileHandlerType]] = {}
|
|
33
33
|
|
|
34
34
|
@classmethod
|
|
35
|
-
def get_all_formats(cls) ->
|
|
35
|
+
def get_all_formats(cls) -> list[str]:
|
|
36
36
|
"""
|
|
37
37
|
Get all supported formats.
|
|
38
38
|
|
|
@@ -45,7 +45,7 @@ class SupportedFormat(ABC, Generic[FileHandlerType]):
|
|
|
45
45
|
]
|
|
46
46
|
|
|
47
47
|
@classmethod
|
|
48
|
-
def get_format_handler(cls, fmt: str) ->
|
|
48
|
+
def get_format_handler(cls, fmt: str) -> type[FileHandlerType]:
|
|
49
49
|
"""
|
|
50
50
|
Get the format handler to the provided format (file extension):
|
|
51
51
|
|
|
@@ -35,8 +35,8 @@ class LogHintUtils:
|
|
|
35
35
|
|
|
36
36
|
@staticmethod
|
|
37
37
|
def parse_log_hint(
|
|
38
|
-
log_hint: typing.Union[
|
|
39
|
-
) -> typing.Union[
|
|
38
|
+
log_hint: typing.Union[dict[str, str], str, None],
|
|
39
|
+
) -> typing.Union[dict[str, str], None]:
|
|
40
40
|
"""
|
|
41
41
|
Parse a given log hint from string to a logging configuration dictionary. The string will be read as the
|
|
42
42
|
artifact key ('key' in the dictionary) and if the string have a single colon, the following structure is
|
|
@@ -16,7 +16,6 @@ import builtins
|
|
|
16
16
|
import importlib
|
|
17
17
|
import itertools
|
|
18
18
|
import re
|
|
19
|
-
import sys
|
|
20
19
|
import typing
|
|
21
20
|
|
|
22
21
|
from mlrun.errors import MLRunInvalidArgumentError
|
|
@@ -151,7 +150,7 @@ class TypeHintUtils:
|
|
|
151
150
|
@staticmethod
|
|
152
151
|
def is_matching(
|
|
153
152
|
object_type: type,
|
|
154
|
-
type_hint: typing.Union[type,
|
|
153
|
+
type_hint: typing.Union[type, set[type]],
|
|
155
154
|
include_subclasses: bool = True,
|
|
156
155
|
reduce_type_hint: bool = True,
|
|
157
156
|
) -> bool:
|
|
@@ -189,8 +188,8 @@ class TypeHintUtils:
|
|
|
189
188
|
|
|
190
189
|
@staticmethod
|
|
191
190
|
def reduce_type_hint(
|
|
192
|
-
type_hint: typing.Union[type,
|
|
193
|
-
) ->
|
|
191
|
+
type_hint: typing.Union[type, set[type]],
|
|
192
|
+
) -> set[type]:
|
|
194
193
|
"""
|
|
195
194
|
Reduce a type hint (or a set of type hints) using the `_reduce_type_hint` function.
|
|
196
195
|
|
|
@@ -212,7 +211,7 @@ class TypeHintUtils:
|
|
|
212
211
|
)
|
|
213
212
|
|
|
214
213
|
@staticmethod
|
|
215
|
-
def _reduce_type_hint(type_hint: type) ->
|
|
214
|
+
def _reduce_type_hint(type_hint: type) -> list[type]:
|
|
216
215
|
"""
|
|
217
216
|
Reduce a type hint. If the type hint is a `typing` module, it will be reduced to its original hinted types. For
|
|
218
217
|
example: `typing.Union[int, float, typing.List[int]]` will return `[int, float, List[int]]` and
|
|
@@ -225,10 +224,6 @@ class TypeHintUtils:
|
|
|
225
224
|
|
|
226
225
|
:return: The reduced type hint as list of hinted types or an empty list if the type hint could not be reduced.
|
|
227
226
|
"""
|
|
228
|
-
# TODO: Remove when we'll no longer support Python 3.7:
|
|
229
|
-
if sys.version_info[1] < 8:
|
|
230
|
-
return []
|
|
231
|
-
|
|
232
227
|
# If it's not a typing type (meaning it's an actual object type) then we can't reduce it further:
|
|
233
228
|
if not TypeHintUtils.is_typing_type(type_hint=type_hint):
|
|
234
229
|
return []
|
mlrun/platforms/__init__.py
CHANGED
|
@@ -17,22 +17,23 @@ import json
|
|
|
17
17
|
from pprint import pprint
|
|
18
18
|
from time import sleep
|
|
19
19
|
|
|
20
|
-
from .
|
|
21
|
-
|
|
22
|
-
VolumeMount,
|
|
23
|
-
add_or_refresh_credentials,
|
|
24
|
-
is_iguazio_session_cookie,
|
|
25
|
-
mount_v3io,
|
|
26
|
-
v3io_cred,
|
|
27
|
-
)
|
|
28
|
-
from .other import (
|
|
20
|
+
from mlrun_pipelines.common.mounts import VolumeMount
|
|
21
|
+
from mlrun_pipelines.mounts import (
|
|
29
22
|
auto_mount,
|
|
30
23
|
mount_configmap,
|
|
31
24
|
mount_hostpath,
|
|
32
25
|
mount_pvc,
|
|
33
26
|
mount_s3,
|
|
34
27
|
mount_secret,
|
|
28
|
+
mount_v3io,
|
|
35
29
|
set_env_variables,
|
|
30
|
+
v3io_cred,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
from .iguazio import (
|
|
34
|
+
V3ioStreamClient,
|
|
35
|
+
add_or_refresh_credentials,
|
|
36
|
+
is_iguazio_session_cookie,
|
|
36
37
|
)
|
|
37
38
|
|
|
38
39
|
|
|
@@ -48,7 +49,7 @@ def watch_stream(
|
|
|
48
49
|
|
|
49
50
|
example::
|
|
50
51
|
|
|
51
|
-
watch_stream(
|
|
52
|
+
watch_stream("v3io:///users/admin/mystream")
|
|
52
53
|
|
|
53
54
|
:param url: stream url
|
|
54
55
|
:param shard_ids: range or list of shard IDs
|