mlrun 1.6.0rc35__py3-none-any.whl → 1.7.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +3 -3
- mlrun/api/schemas/__init__.py +1 -1
- mlrun/artifacts/base.py +11 -6
- mlrun/artifacts/dataset.py +2 -2
- mlrun/artifacts/model.py +30 -24
- mlrun/artifacts/plots.py +2 -2
- mlrun/common/db/sql_session.py +5 -3
- mlrun/common/helpers.py +1 -2
- mlrun/common/schemas/artifact.py +3 -3
- mlrun/common/schemas/auth.py +3 -3
- mlrun/common/schemas/background_task.py +1 -1
- mlrun/common/schemas/client_spec.py +1 -1
- mlrun/common/schemas/feature_store.py +16 -16
- mlrun/common/schemas/frontend_spec.py +7 -7
- mlrun/common/schemas/function.py +1 -1
- mlrun/common/schemas/hub.py +4 -9
- mlrun/common/schemas/memory_reports.py +2 -2
- mlrun/common/schemas/model_monitoring/grafana.py +4 -4
- mlrun/common/schemas/model_monitoring/model_endpoints.py +14 -15
- mlrun/common/schemas/notification.py +4 -4
- mlrun/common/schemas/object.py +2 -2
- mlrun/common/schemas/pipeline.py +1 -1
- mlrun/common/schemas/project.py +3 -3
- mlrun/common/schemas/runtime_resource.py +8 -12
- mlrun/common/schemas/schedule.py +3 -3
- mlrun/common/schemas/tag.py +1 -2
- mlrun/common/schemas/workflow.py +2 -2
- mlrun/config.py +8 -4
- mlrun/data_types/to_pandas.py +1 -3
- mlrun/datastore/base.py +0 -28
- mlrun/datastore/datastore_profile.py +9 -9
- mlrun/datastore/filestore.py +0 -1
- mlrun/datastore/google_cloud_storage.py +1 -1
- mlrun/datastore/sources.py +7 -11
- mlrun/datastore/spark_utils.py +1 -2
- mlrun/datastore/targets.py +31 -31
- mlrun/datastore/utils.py +4 -6
- mlrun/datastore/v3io.py +70 -46
- mlrun/db/base.py +22 -23
- mlrun/db/httpdb.py +34 -34
- mlrun/db/nopdb.py +19 -19
- mlrun/errors.py +1 -1
- mlrun/execution.py +4 -4
- mlrun/feature_store/api.py +20 -21
- mlrun/feature_store/common.py +1 -1
- mlrun/feature_store/feature_set.py +28 -32
- mlrun/feature_store/feature_vector.py +24 -27
- mlrun/feature_store/retrieval/base.py +7 -7
- mlrun/feature_store/retrieval/conversion.py +2 -4
- mlrun/feature_store/steps.py +7 -15
- mlrun/features.py +5 -7
- mlrun/frameworks/_common/artifacts_library.py +9 -9
- mlrun/frameworks/_common/mlrun_interface.py +5 -5
- mlrun/frameworks/_common/model_handler.py +48 -48
- mlrun/frameworks/_common/plan.py +2 -3
- mlrun/frameworks/_common/producer.py +3 -4
- mlrun/frameworks/_common/utils.py +5 -5
- mlrun/frameworks/_dl_common/loggers/logger.py +6 -7
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +9 -9
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +16 -35
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -2
- mlrun/frameworks/_ml_common/loggers/logger.py +3 -4
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +4 -5
- mlrun/frameworks/_ml_common/model_handler.py +24 -24
- mlrun/frameworks/_ml_common/pkl_model_server.py +2 -2
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +2 -3
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +2 -3
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
- mlrun/frameworks/_ml_common/utils.py +4 -4
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +7 -7
- mlrun/frameworks/huggingface/model_server.py +4 -4
- mlrun/frameworks/lgbm/__init__.py +32 -32
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -5
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -5
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -3
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +6 -6
- mlrun/frameworks/lgbm/model_handler.py +9 -9
- mlrun/frameworks/lgbm/model_server.py +6 -6
- mlrun/frameworks/lgbm/utils.py +5 -5
- mlrun/frameworks/onnx/dataset.py +8 -8
- mlrun/frameworks/onnx/mlrun_interface.py +3 -3
- mlrun/frameworks/onnx/model_handler.py +6 -6
- mlrun/frameworks/onnx/model_server.py +7 -7
- mlrun/frameworks/parallel_coordinates.py +2 -2
- mlrun/frameworks/pytorch/__init__.py +16 -16
- mlrun/frameworks/pytorch/callbacks/callback.py +4 -5
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +17 -17
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +11 -11
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +23 -29
- mlrun/frameworks/pytorch/callbacks_handler.py +38 -38
- mlrun/frameworks/pytorch/mlrun_interface.py +20 -20
- mlrun/frameworks/pytorch/model_handler.py +17 -17
- mlrun/frameworks/pytorch/model_server.py +7 -7
- mlrun/frameworks/sklearn/__init__.py +12 -12
- mlrun/frameworks/sklearn/estimator.py +4 -4
- mlrun/frameworks/sklearn/metrics_library.py +14 -14
- mlrun/frameworks/sklearn/mlrun_interface.py +3 -6
- mlrun/frameworks/sklearn/model_handler.py +2 -2
- mlrun/frameworks/tf_keras/__init__.py +5 -5
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +14 -14
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +11 -11
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +19 -23
- mlrun/frameworks/tf_keras/mlrun_interface.py +7 -9
- mlrun/frameworks/tf_keras/model_handler.py +14 -14
- mlrun/frameworks/tf_keras/model_server.py +6 -6
- mlrun/frameworks/xgboost/__init__.py +12 -12
- mlrun/frameworks/xgboost/model_handler.py +6 -6
- mlrun/k8s_utils.py +4 -5
- mlrun/kfpops.py +2 -2
- mlrun/launcher/base.py +10 -10
- mlrun/launcher/local.py +8 -8
- mlrun/launcher/remote.py +7 -7
- mlrun/lists.py +3 -4
- mlrun/model.py +205 -55
- mlrun/model_monitoring/api.py +21 -24
- mlrun/model_monitoring/application.py +4 -4
- mlrun/model_monitoring/batch.py +17 -17
- mlrun/model_monitoring/controller.py +2 -1
- mlrun/model_monitoring/features_drift_table.py +44 -31
- mlrun/model_monitoring/prometheus.py +1 -4
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +11 -13
- mlrun/model_monitoring/stores/model_endpoint_store.py +9 -11
- mlrun/model_monitoring/stores/models/__init__.py +2 -2
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +11 -13
- mlrun/model_monitoring/stream_processing.py +16 -34
- mlrun/model_monitoring/tracking_policy.py +2 -1
- mlrun/package/__init__.py +6 -6
- mlrun/package/context_handler.py +5 -5
- mlrun/package/packager.py +7 -7
- mlrun/package/packagers/default_packager.py +6 -6
- mlrun/package/packagers/numpy_packagers.py +15 -15
- mlrun/package/packagers/pandas_packagers.py +5 -5
- mlrun/package/packagers/python_standard_library_packagers.py +10 -10
- mlrun/package/packagers_manager.py +18 -23
- mlrun/package/utils/_formatter.py +4 -4
- mlrun/package/utils/_pickler.py +2 -2
- mlrun/package/utils/_supported_format.py +4 -4
- mlrun/package/utils/log_hint_utils.py +2 -2
- mlrun/package/utils/type_hint_utils.py +4 -9
- mlrun/platforms/other.py +1 -2
- mlrun/projects/operations.py +5 -5
- mlrun/projects/pipelines.py +9 -9
- mlrun/projects/project.py +58 -46
- mlrun/render.py +1 -1
- mlrun/run.py +9 -9
- mlrun/runtimes/__init__.py +7 -4
- mlrun/runtimes/base.py +20 -23
- mlrun/runtimes/constants.py +5 -5
- mlrun/runtimes/daskjob.py +8 -8
- mlrun/runtimes/databricks_job/databricks_cancel_task.py +1 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +7 -7
- mlrun/runtimes/function_reference.py +1 -1
- mlrun/runtimes/local.py +1 -1
- mlrun/runtimes/mpijob/abstract.py +1 -2
- mlrun/runtimes/nuclio/__init__.py +20 -0
- mlrun/runtimes/{function.py → nuclio/function.py} +15 -16
- mlrun/runtimes/{nuclio.py → nuclio/nuclio.py} +6 -6
- mlrun/runtimes/{serving.py → nuclio/serving.py} +13 -12
- mlrun/runtimes/pod.py +95 -48
- mlrun/runtimes/remotesparkjob.py +1 -1
- mlrun/runtimes/sparkjob/spark3job.py +50 -33
- mlrun/runtimes/utils.py +1 -2
- mlrun/secrets.py +3 -3
- mlrun/serving/remote.py +0 -4
- mlrun/serving/routers.py +6 -6
- mlrun/serving/server.py +4 -4
- mlrun/serving/states.py +29 -0
- mlrun/serving/utils.py +3 -3
- mlrun/serving/v1_serving.py +6 -7
- mlrun/serving/v2_serving.py +50 -8
- mlrun/track/tracker_manager.py +3 -3
- mlrun/track/trackers/mlflow_tracker.py +1 -2
- mlrun/utils/async_http.py +5 -7
- mlrun/utils/azure_vault.py +1 -1
- mlrun/utils/clones.py +1 -2
- mlrun/utils/condition_evaluator.py +3 -3
- mlrun/utils/db.py +3 -3
- mlrun/utils/helpers.py +37 -119
- mlrun/utils/http.py +1 -4
- mlrun/utils/logger.py +49 -14
- mlrun/utils/notifications/notification/__init__.py +3 -3
- mlrun/utils/notifications/notification/base.py +2 -2
- mlrun/utils/notifications/notification/ipython.py +1 -1
- mlrun/utils/notifications/notification_pusher.py +8 -14
- mlrun/utils/retryer.py +207 -0
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/v3io_clients.py +2 -3
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +2 -6
- {mlrun-1.6.0rc35.dist-info → mlrun-1.7.0rc2.dist-info}/METADATA +9 -9
- mlrun-1.7.0rc2.dist-info/RECORD +315 -0
- mlrun-1.6.0rc35.dist-info/RECORD +0 -313
- {mlrun-1.6.0rc35.dist-info → mlrun-1.7.0rc2.dist-info}/LICENSE +0 -0
- {mlrun-1.6.0rc35.dist-info → mlrun-1.7.0rc2.dist-info}/WHEEL +0 -0
- {mlrun-1.6.0rc35.dist-info → mlrun-1.7.0rc2.dist-info}/entry_points.txt +0 -0
- {mlrun-1.6.0rc35.dist-info → mlrun-1.7.0rc2.dist-info}/top_level.txt +0 -0
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
|
-
from typing import Callable,
|
|
15
|
+
from typing import Callable, Union
|
|
16
16
|
|
|
17
17
|
import mlrun
|
|
18
18
|
from mlrun.artifacts import Artifact
|
|
@@ -50,16 +50,16 @@ class MLRunLoggingCallback(LoggingCallback):
|
|
|
50
50
|
context: mlrun.MLClientCtx,
|
|
51
51
|
model_handler: TFKerasModelHandler,
|
|
52
52
|
log_model_tag: str = "",
|
|
53
|
-
log_model_labels:
|
|
54
|
-
log_model_parameters:
|
|
55
|
-
log_model_extra_data:
|
|
53
|
+
log_model_labels: dict[str, TFKerasTypes.TrackableType] = None,
|
|
54
|
+
log_model_parameters: dict[str, TFKerasTypes.TrackableType] = None,
|
|
55
|
+
log_model_extra_data: dict[
|
|
56
56
|
str, Union[TFKerasTypes.TrackableType, Artifact]
|
|
57
57
|
] = None,
|
|
58
|
-
dynamic_hyperparameters:
|
|
59
|
-
str, Union[
|
|
58
|
+
dynamic_hyperparameters: dict[
|
|
59
|
+
str, Union[list[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]]
|
|
60
60
|
] = None,
|
|
61
|
-
static_hyperparameters:
|
|
62
|
-
str, Union[TFKerasTypes,
|
|
61
|
+
static_hyperparameters: dict[
|
|
62
|
+
str, Union[TFKerasTypes, list[Union[str, int]]]
|
|
63
63
|
] = None,
|
|
64
64
|
auto_log: bool = False,
|
|
65
65
|
):
|
|
@@ -97,7 +97,7 @@ class MLRunLoggingCallback(LoggingCallback):
|
|
|
97
97
|
trying to track common static and dynamic hyperparameters such as learning
|
|
98
98
|
rate.
|
|
99
99
|
"""
|
|
100
|
-
super(
|
|
100
|
+
super().__init__(
|
|
101
101
|
dynamic_hyperparameters=dynamic_hyperparameters,
|
|
102
102
|
static_hyperparameters=static_hyperparameters,
|
|
103
103
|
auto_log=auto_log,
|
|
@@ -134,7 +134,7 @@ class MLRunLoggingCallback(LoggingCallback):
|
|
|
134
134
|
:param logs: Currently no data is passed to this argument for this method but that may change in the
|
|
135
135
|
future.
|
|
136
136
|
"""
|
|
137
|
-
super(
|
|
137
|
+
super().on_test_end(logs=logs)
|
|
138
138
|
|
|
139
139
|
# Check if its part of evaluation. If so, end the run:
|
|
140
140
|
if self._logger.mode == LoggingMode.EVALUATION:
|
|
@@ -151,7 +151,7 @@ class MLRunLoggingCallback(LoggingCallback):
|
|
|
151
151
|
performed. Validation result keys are prefixed with `val_`. For training epoch, the values of the
|
|
152
152
|
`Model`'s metrics are returned. Example : `{'loss': 0.2, 'acc': 0.7}`.
|
|
153
153
|
"""
|
|
154
|
-
super(
|
|
154
|
+
super().on_epoch_end(epoch=epoch)
|
|
155
155
|
|
|
156
156
|
# Log the current epoch's results:
|
|
157
157
|
self._logger.log_epoch_to_context(epoch=epoch)
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
15
|
from datetime import datetime
|
|
16
|
-
from typing import Callable,
|
|
16
|
+
from typing import Callable, Union
|
|
17
17
|
|
|
18
18
|
import tensorflow as tf
|
|
19
19
|
from packaging import version
|
|
@@ -38,7 +38,7 @@ class _TFKerasTensorboardLogger(TensorboardLogger):
|
|
|
38
38
|
|
|
39
39
|
def __init__(
|
|
40
40
|
self,
|
|
41
|
-
statistics_functions:
|
|
41
|
+
statistics_functions: list[Callable[[Union[Variable]], Union[float, Variable]]],
|
|
42
42
|
context: mlrun.MLClientCtx = None,
|
|
43
43
|
tensorboard_directory: str = None,
|
|
44
44
|
run_name: str = None,
|
|
@@ -67,7 +67,7 @@ class _TFKerasTensorboardLogger(TensorboardLogger):
|
|
|
67
67
|
update. Notice that writing to tensorboard too frequently may cause the training
|
|
68
68
|
to be slower. Default: 'epoch'.
|
|
69
69
|
"""
|
|
70
|
-
super(
|
|
70
|
+
super().__init__(
|
|
71
71
|
statistics_functions=statistics_functions,
|
|
72
72
|
context=context,
|
|
73
73
|
tensorboard_directory=tensorboard_directory,
|
|
@@ -255,15 +255,15 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
255
255
|
context: mlrun.MLClientCtx = None,
|
|
256
256
|
tensorboard_directory: str = None,
|
|
257
257
|
run_name: str = None,
|
|
258
|
-
weights: Union[bool,
|
|
259
|
-
statistics_functions:
|
|
258
|
+
weights: Union[bool, list[str]] = False,
|
|
259
|
+
statistics_functions: list[
|
|
260
260
|
Callable[[Union[Variable, Tensor]], Union[float, Tensor]]
|
|
261
261
|
] = None,
|
|
262
|
-
dynamic_hyperparameters:
|
|
263
|
-
str, Union[
|
|
262
|
+
dynamic_hyperparameters: dict[
|
|
263
|
+
str, Union[list[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]]
|
|
264
264
|
] = None,
|
|
265
|
-
static_hyperparameters:
|
|
266
|
-
str, Union[TFKerasTypes.TrackableType,
|
|
265
|
+
static_hyperparameters: dict[
|
|
266
|
+
str, Union[TFKerasTypes.TrackableType, list[Union[str, int]]]
|
|
267
267
|
] = None,
|
|
268
268
|
update_frequency: Union[int, str] = "epoch",
|
|
269
269
|
auto_log: bool = False,
|
|
@@ -325,7 +325,7 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
325
325
|
:raise MLRunInvalidArgumentError: In case both 'context' and 'tensorboard_directory' parameters were not given
|
|
326
326
|
or the 'update_frequency' was incorrect.
|
|
327
327
|
"""
|
|
328
|
-
super(
|
|
328
|
+
super().__init__(
|
|
329
329
|
dynamic_hyperparameters=dynamic_hyperparameters,
|
|
330
330
|
static_hyperparameters=static_hyperparameters,
|
|
331
331
|
auto_log=auto_log,
|
|
@@ -352,7 +352,7 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
352
352
|
self._logged_model = False
|
|
353
353
|
self._logged_hyperparameters = False
|
|
354
354
|
|
|
355
|
-
def get_weights(self) ->
|
|
355
|
+
def get_weights(self) -> dict[str, Variable]:
|
|
356
356
|
"""
|
|
357
357
|
Get the weights tensors tracked. The weights will be stored in a dictionary where each key is the weight's name
|
|
358
358
|
and the value is the weight's parameter (tensor).
|
|
@@ -361,7 +361,7 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
361
361
|
"""
|
|
362
362
|
return self._logger.weights
|
|
363
363
|
|
|
364
|
-
def get_weights_statistics(self) ->
|
|
364
|
+
def get_weights_statistics(self) -> dict[str, dict[str, list[float]]]:
|
|
365
365
|
"""
|
|
366
366
|
Get the weights mean results logged. The results will be stored in a dictionary where each key is the weight's
|
|
367
367
|
name and the value is a list of mean values per epoch.
|
|
@@ -408,7 +408,7 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
408
408
|
:param logs: Currently the output of the last call to `on_epoch_end()` is passed to this argument for this
|
|
409
409
|
method but that may change in the future.
|
|
410
410
|
"""
|
|
411
|
-
super(
|
|
411
|
+
super().on_train_end()
|
|
412
412
|
|
|
413
413
|
# Write the final run summary:
|
|
414
414
|
self._logger.write_final_summary_text()
|
|
@@ -453,7 +453,7 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
453
453
|
:param logs: Currently no data is passed to this argument for this method but that may change in the
|
|
454
454
|
future.
|
|
455
455
|
"""
|
|
456
|
-
super(
|
|
456
|
+
super().on_test_end(logs=logs)
|
|
457
457
|
|
|
458
458
|
# Check if needed to end the run (in case of evaluation and not training):
|
|
459
459
|
if not self._is_training:
|
|
@@ -477,7 +477,7 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
477
477
|
`Model`'s metrics are returned. Example : `{'loss': 0.2, 'acc': 0.7}`.
|
|
478
478
|
"""
|
|
479
479
|
# Update the dynamic hyperparameters
|
|
480
|
-
super(
|
|
480
|
+
super().on_epoch_end(epoch=epoch)
|
|
481
481
|
|
|
482
482
|
# Log the weights statistics:
|
|
483
483
|
self._logger.log_weights_statistics()
|
|
@@ -515,9 +515,7 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
515
515
|
:param logs: Aggregated metric results up until this batch.
|
|
516
516
|
"""
|
|
517
517
|
# Log the batch's results:
|
|
518
|
-
super(
|
|
519
|
-
batch=batch, logs=logs
|
|
520
|
-
)
|
|
518
|
+
super().on_train_batch_end(batch=batch, logs=logs)
|
|
521
519
|
|
|
522
520
|
# Write the batch loss and metrics results to their graphs:
|
|
523
521
|
self._logger.write_training_results()
|
|
@@ -540,9 +538,7 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
540
538
|
:param logs: Aggregated metric results up until this batch.
|
|
541
539
|
"""
|
|
542
540
|
# Log the batch's results:
|
|
543
|
-
super(
|
|
544
|
-
batch=batch, logs=logs
|
|
545
|
-
)
|
|
541
|
+
super().on_test_batch_end(batch=batch, logs=logs)
|
|
546
542
|
|
|
547
543
|
# Write the batch loss and metrics results to their graphs:
|
|
548
544
|
self._logger.write_validation_results()
|
|
@@ -555,7 +551,7 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
555
551
|
|
|
556
552
|
@staticmethod
|
|
557
553
|
def get_default_weight_statistics_list() -> (
|
|
558
|
-
|
|
554
|
+
list[Callable[[Union[Variable, Tensor]], Union[float, Tensor]]]
|
|
559
555
|
):
|
|
560
556
|
"""
|
|
561
557
|
Get the default list of statistics functions being applied on the tracked weights each epoch.
|
|
@@ -569,7 +565,7 @@ class TensorboardLoggingCallback(LoggingCallback):
|
|
|
569
565
|
After the trainer / evaluator run begins, this method will be called to setup the results, hyperparameters
|
|
570
566
|
and weights dictionaries for logging.
|
|
571
567
|
"""
|
|
572
|
-
super(
|
|
568
|
+
super()._setup_run()
|
|
573
569
|
|
|
574
570
|
# Check if needed to track weights:
|
|
575
571
|
if self._tracked_weights is False:
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import importlib
|
|
16
16
|
import os
|
|
17
17
|
from abc import ABC
|
|
18
|
-
from typing import
|
|
18
|
+
from typing import Union
|
|
19
19
|
|
|
20
20
|
import tensorflow as tf
|
|
21
21
|
from tensorflow import keras
|
|
@@ -88,9 +88,7 @@ class TFKerasMLRunInterface(MLRunInterface, ABC):
|
|
|
88
88
|
:param restoration: Restoration information tuple as returned from 'remove_interface' in order to
|
|
89
89
|
add the interface in a certain state.
|
|
90
90
|
"""
|
|
91
|
-
super(
|
|
92
|
-
obj=obj, restoration=restoration
|
|
93
|
-
)
|
|
91
|
+
super().add_interface(obj=obj, restoration=restoration)
|
|
94
92
|
|
|
95
93
|
def mlrun_compile(self, *args, **kwargs):
|
|
96
94
|
"""
|
|
@@ -237,7 +235,7 @@ class TFKerasMLRunInterface(MLRunInterface, ABC):
|
|
|
237
235
|
"""
|
|
238
236
|
self._RANK_0_ONLY_CALLBACKS.add(callback_name)
|
|
239
237
|
|
|
240
|
-
def _pre_compile(self, optimizer: Optimizer) ->
|
|
238
|
+
def _pre_compile(self, optimizer: Optimizer) -> tuple[Optimizer, Union[bool, None]]:
|
|
241
239
|
"""
|
|
242
240
|
Method to call before calling 'compile' to setup the run and inputs for using horovod.
|
|
243
241
|
|
|
@@ -295,11 +293,11 @@ class TFKerasMLRunInterface(MLRunInterface, ABC):
|
|
|
295
293
|
|
|
296
294
|
def _pre_fit(
|
|
297
295
|
self,
|
|
298
|
-
callbacks:
|
|
296
|
+
callbacks: list[Callback],
|
|
299
297
|
verbose: int,
|
|
300
298
|
steps_per_epoch: Union[int, None],
|
|
301
299
|
validation_steps: Union[int, None],
|
|
302
|
-
) ->
|
|
300
|
+
) -> tuple[list[Callback], int, Union[int, None], Union[int, None]]:
|
|
303
301
|
"""
|
|
304
302
|
Method to call before calling 'fit' to setup the run and inputs for using horovod.
|
|
305
303
|
|
|
@@ -366,9 +364,9 @@ class TFKerasMLRunInterface(MLRunInterface, ABC):
|
|
|
366
364
|
|
|
367
365
|
def _pre_evaluate(
|
|
368
366
|
self,
|
|
369
|
-
callbacks:
|
|
367
|
+
callbacks: list[Callback],
|
|
370
368
|
steps: Union[int, None],
|
|
371
|
-
) ->
|
|
369
|
+
) -> tuple[list[Callback], Union[int, None]]:
|
|
372
370
|
"""
|
|
373
371
|
Method to call before calling 'evaluate' to setup the run and inputs for using horovod.
|
|
374
372
|
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import os
|
|
16
16
|
import shutil
|
|
17
17
|
import zipfile
|
|
18
|
-
from typing import
|
|
18
|
+
from typing import Union
|
|
19
19
|
|
|
20
20
|
import numpy as np
|
|
21
21
|
import tensorflow as tf
|
|
@@ -67,8 +67,8 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
67
67
|
model_name: str = None,
|
|
68
68
|
model_format: str = ModelFormats.SAVED_MODEL,
|
|
69
69
|
context: mlrun.MLClientCtx = None,
|
|
70
|
-
modules_map: Union[
|
|
71
|
-
custom_objects_map: Union[
|
|
70
|
+
modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
|
|
71
|
+
custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
|
|
72
72
|
custom_objects_directory: str = None,
|
|
73
73
|
save_traces: bool = False,
|
|
74
74
|
**kwargs,
|
|
@@ -174,7 +174,7 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
174
174
|
self._weights_file = None # type: str
|
|
175
175
|
|
|
176
176
|
# Setup the base handler class:
|
|
177
|
-
super(
|
|
177
|
+
super().__init__(
|
|
178
178
|
model=model,
|
|
179
179
|
model_path=model_path,
|
|
180
180
|
model_name=model_name,
|
|
@@ -190,8 +190,8 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
190
190
|
|
|
191
191
|
def set_labels(
|
|
192
192
|
self,
|
|
193
|
-
to_add:
|
|
194
|
-
to_remove:
|
|
193
|
+
to_add: dict[str, Union[str, int, float]] = None,
|
|
194
|
+
to_remove: list[str] = None,
|
|
195
195
|
):
|
|
196
196
|
"""
|
|
197
197
|
Update the labels dictionary of this model artifact. There are required labels that cannot be edited or removed.
|
|
@@ -200,7 +200,7 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
200
200
|
:param to_remove: A list of labels keys to remove.
|
|
201
201
|
"""
|
|
202
202
|
# Update the user's labels:
|
|
203
|
-
super(
|
|
203
|
+
super().set_labels(to_add=to_add, to_remove=to_remove)
|
|
204
204
|
|
|
205
205
|
# Set the required labels:
|
|
206
206
|
self._labels[self._LabelKeys.MODEL_FORMAT] = self._model_format
|
|
@@ -211,7 +211,7 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
211
211
|
@without_mlrun_interface(interface=TFKerasMLRunInterface)
|
|
212
212
|
def save(
|
|
213
213
|
self, output_path: str = None, **kwargs
|
|
214
|
-
) -> Union[
|
|
214
|
+
) -> Union[dict[str, Artifact], None]:
|
|
215
215
|
"""
|
|
216
216
|
Save the handled model at the given output path. If a MLRun context is available, the saved model files will be
|
|
217
217
|
logged and returned as artifacts.
|
|
@@ -221,7 +221,7 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
221
221
|
|
|
222
222
|
:return The saved model additional artifacts (if needed) dictionary if context is available and None otherwise.
|
|
223
223
|
"""
|
|
224
|
-
super(
|
|
224
|
+
super().save(output_path=output_path)
|
|
225
225
|
|
|
226
226
|
# Setup the returning model artifacts list:
|
|
227
227
|
artifacts = {} # type: Dict[str, Artifact]
|
|
@@ -291,7 +291,7 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
291
291
|
"Loading a model using checkpoint is not yet implemented."
|
|
292
292
|
)
|
|
293
293
|
|
|
294
|
-
super(
|
|
294
|
+
super().load()
|
|
295
295
|
|
|
296
296
|
# ModelFormats.H5 - Load from a h5 file:
|
|
297
297
|
if self._model_format == TFKerasModelHandler.ModelFormats.H5:
|
|
@@ -308,7 +308,7 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
308
308
|
# ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS - Load from a json architecture file and a h5 weights file:
|
|
309
309
|
else:
|
|
310
310
|
# Load the model architecture (json):
|
|
311
|
-
with open(self._model_file
|
|
311
|
+
with open(self._model_file) as json_file:
|
|
312
312
|
model_architecture = json_file.read()
|
|
313
313
|
self._model = keras.models.model_from_json(
|
|
314
314
|
model_architecture, custom_objects=self._custom_objects
|
|
@@ -321,7 +321,7 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
321
321
|
model_name: str = None,
|
|
322
322
|
optimize: bool = True,
|
|
323
323
|
input_signature: Union[
|
|
324
|
-
|
|
324
|
+
list[tf.TensorSpec], list[np.ndarray], tf.TensorSpec, np.ndarray
|
|
325
325
|
] = None,
|
|
326
326
|
output_path: str = None,
|
|
327
327
|
log: bool = None,
|
|
@@ -487,7 +487,7 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
487
487
|
].local()
|
|
488
488
|
|
|
489
489
|
# Continue collecting from abstract class:
|
|
490
|
-
super(
|
|
490
|
+
super()._collect_files_from_store_object()
|
|
491
491
|
|
|
492
492
|
def _collect_files_from_local_path(self):
|
|
493
493
|
"""
|
|
@@ -554,7 +554,7 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
554
554
|
"""
|
|
555
555
|
# Supported types:
|
|
556
556
|
if isinstance(sample, np.ndarray):
|
|
557
|
-
return super(
|
|
557
|
+
return super()._read_sample(sample=sample)
|
|
558
558
|
elif isinstance(sample, tf.TensorSpec):
|
|
559
559
|
return Feature(
|
|
560
560
|
name=sample.name,
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
|
-
from typing import Any,
|
|
15
|
+
from typing import Any, Union
|
|
16
16
|
|
|
17
17
|
import numpy as np
|
|
18
18
|
from tensorflow import keras
|
|
@@ -36,8 +36,8 @@ class TFKerasModelServer(V2ModelServer):
|
|
|
36
36
|
model: keras.Model = None,
|
|
37
37
|
model_path: str = None,
|
|
38
38
|
model_name: str = None,
|
|
39
|
-
modules_map: Union[
|
|
40
|
-
custom_objects_map: Union[
|
|
39
|
+
modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
|
|
40
|
+
custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
|
|
41
41
|
custom_objects_directory: str = None,
|
|
42
42
|
model_format: str = TFKerasModelHandler.ModelFormats.SAVED_MODEL,
|
|
43
43
|
to_list: bool = False,
|
|
@@ -103,7 +103,7 @@ class TFKerasModelServer(V2ModelServer):
|
|
|
103
103
|
:param protocol: -
|
|
104
104
|
:param class_args: -
|
|
105
105
|
"""
|
|
106
|
-
super(
|
|
106
|
+
super().__init__(
|
|
107
107
|
context=context,
|
|
108
108
|
name=name,
|
|
109
109
|
model_path=model_path,
|
|
@@ -146,7 +146,7 @@ class TFKerasModelServer(V2ModelServer):
|
|
|
146
146
|
self._model_handler.load()
|
|
147
147
|
self.model = self._model_handler.model
|
|
148
148
|
|
|
149
|
-
def predict(self, request:
|
|
149
|
+
def predict(self, request: dict[str, Any]) -> Union[np.ndarray, list]:
|
|
150
150
|
"""
|
|
151
151
|
Infer the inputs through the model using 'keras.Model.predict' and return its output. The inferred data will be
|
|
152
152
|
read from the "inputs" key of the request.
|
|
@@ -165,7 +165,7 @@ class TFKerasModelServer(V2ModelServer):
|
|
|
165
165
|
# Return as list if required:
|
|
166
166
|
return prediction if not self.to_list else prediction.tolist()
|
|
167
167
|
|
|
168
|
-
def explain(self, request:
|
|
168
|
+
def explain(self, request: dict[str, Any]) -> str:
|
|
169
169
|
"""
|
|
170
170
|
Return a string explaining what model is being serve in this serving function and the function name.
|
|
171
171
|
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
15
|
# flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
|
|
16
|
-
from typing import
|
|
16
|
+
from typing import Union
|
|
17
17
|
|
|
18
18
|
import xgboost as xgb
|
|
19
19
|
|
|
@@ -35,25 +35,25 @@ def apply_mlrun(
|
|
|
35
35
|
model_name: str = "model",
|
|
36
36
|
tag: str = "",
|
|
37
37
|
model_path: str = None,
|
|
38
|
-
modules_map: Union[
|
|
39
|
-
custom_objects_map: Union[
|
|
38
|
+
modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
|
|
39
|
+
custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
|
|
40
40
|
custom_objects_directory: str = None,
|
|
41
41
|
context: mlrun.MLClientCtx = None,
|
|
42
|
-
artifacts: Union[
|
|
42
|
+
artifacts: Union[list[MLPlan], list[str], dict[str, dict]] = None,
|
|
43
43
|
metrics: Union[
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
44
|
+
list[Metric],
|
|
45
|
+
list[XGBoostTypes.MetricEntryType],
|
|
46
|
+
dict[str, XGBoostTypes.MetricEntryType],
|
|
47
47
|
] = None,
|
|
48
48
|
x_test: XGBoostTypes.DatasetType = None,
|
|
49
49
|
y_test: XGBoostTypes.DatasetType = None,
|
|
50
50
|
sample_set: Union[XGBoostTypes.DatasetType, mlrun.DataItem, str] = None,
|
|
51
|
-
y_columns: Union[
|
|
51
|
+
y_columns: Union[list[str], list[int]] = None,
|
|
52
52
|
feature_vector: str = None,
|
|
53
|
-
feature_weights:
|
|
54
|
-
labels:
|
|
55
|
-
parameters:
|
|
56
|
-
extra_data:
|
|
53
|
+
feature_weights: list[float] = None,
|
|
54
|
+
labels: dict[str, Union[str, int, float]] = None,
|
|
55
|
+
parameters: dict[str, Union[str, int, float]] = None,
|
|
56
|
+
extra_data: dict[str, XGBoostTypes.ExtraDataType] = None,
|
|
57
57
|
auto_log: bool = True,
|
|
58
58
|
**kwargs,
|
|
59
59
|
) -> XGBoostModelHandler:
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
#
|
|
15
15
|
import os
|
|
16
16
|
import pickle
|
|
17
|
-
from typing import
|
|
17
|
+
from typing import Union
|
|
18
18
|
|
|
19
19
|
import cloudpickle
|
|
20
20
|
|
|
@@ -47,8 +47,8 @@ class XGBoostModelHandler(MLModelHandler):
|
|
|
47
47
|
model: XGBoostTypes.ModelType = None,
|
|
48
48
|
model_path: str = None,
|
|
49
49
|
model_name: str = None,
|
|
50
|
-
modules_map: Union[
|
|
51
|
-
custom_objects_map: Union[
|
|
50
|
+
modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
|
|
51
|
+
custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
|
|
52
52
|
custom_objects_directory: str = None,
|
|
53
53
|
context: mlrun.MLClientCtx = None,
|
|
54
54
|
model_format: str = ModelFormats.PKL,
|
|
@@ -124,7 +124,7 @@ class XGBoostModelHandler(MLModelHandler):
|
|
|
124
124
|
# Store the configuration:
|
|
125
125
|
self._model_format = model_format
|
|
126
126
|
|
|
127
|
-
super(
|
|
127
|
+
super().__init__(
|
|
128
128
|
model=model,
|
|
129
129
|
model_path=model_path,
|
|
130
130
|
model_name=model_name,
|
|
@@ -162,7 +162,7 @@ class XGBoostModelHandler(MLModelHandler):
|
|
|
162
162
|
|
|
163
163
|
:return The saved model additional artifacts (if needed) dictionary if context is available and None otherwise.
|
|
164
164
|
"""
|
|
165
|
-
super(
|
|
165
|
+
super().save(output_path=output_path)
|
|
166
166
|
|
|
167
167
|
# ModelFormats.PICKLE - Save from a pkl file:
|
|
168
168
|
if self._model_format == XGBoostModelHandler.ModelFormats.PKL:
|
|
@@ -177,7 +177,7 @@ class XGBoostModelHandler(MLModelHandler):
|
|
|
177
177
|
Load the specified model in this handler. Additional parameters for the class initializer can be passed via the
|
|
178
178
|
kwargs dictionary.
|
|
179
179
|
"""
|
|
180
|
-
super(
|
|
180
|
+
super().load()
|
|
181
181
|
|
|
182
182
|
# ModelFormats.PICKLE - Load from a pkl file:
|
|
183
183
|
if self._model_format == XGBoostModelHandler.ModelFormats.PKL:
|
mlrun/k8s_utils.py
CHANGED
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import re
|
|
15
|
-
import typing
|
|
16
15
|
|
|
17
16
|
import kubernetes.client
|
|
18
17
|
|
|
@@ -38,7 +37,7 @@ def is_running_inside_kubernetes_cluster():
|
|
|
38
37
|
|
|
39
38
|
def generate_preemptible_node_selector_requirements(
|
|
40
39
|
node_selector_operator: str,
|
|
41
|
-
) ->
|
|
40
|
+
) -> list[kubernetes.client.V1NodeSelectorRequirement]:
|
|
42
41
|
"""
|
|
43
42
|
Generate node selector requirements based on the pre-configured node selector of the preemptible nodes.
|
|
44
43
|
node selector operator represents a key's relationship to a set of values.
|
|
@@ -62,7 +61,7 @@ def generate_preemptible_node_selector_requirements(
|
|
|
62
61
|
|
|
63
62
|
|
|
64
63
|
def generate_preemptible_nodes_anti_affinity_terms() -> (
|
|
65
|
-
|
|
64
|
+
list[kubernetes.client.V1NodeSelectorTerm]
|
|
66
65
|
):
|
|
67
66
|
"""
|
|
68
67
|
Generate node selector term containing anti-affinity expressions based on the
|
|
@@ -84,7 +83,7 @@ def generate_preemptible_nodes_anti_affinity_terms() -> (
|
|
|
84
83
|
|
|
85
84
|
|
|
86
85
|
def generate_preemptible_nodes_affinity_terms() -> (
|
|
87
|
-
|
|
86
|
+
list[kubernetes.client.V1NodeSelectorTerm]
|
|
88
87
|
):
|
|
89
88
|
"""
|
|
90
89
|
Use for purpose of scheduling on node having at least one of the node selectors.
|
|
@@ -105,7 +104,7 @@ def generate_preemptible_nodes_affinity_terms() -> (
|
|
|
105
104
|
return node_selector_terms
|
|
106
105
|
|
|
107
106
|
|
|
108
|
-
def generate_preemptible_tolerations() ->
|
|
107
|
+
def generate_preemptible_tolerations() -> list[kubernetes.client.V1Toleration]:
|
|
109
108
|
tolerations = mlconfig.get_preemptible_tolerations()
|
|
110
109
|
|
|
111
110
|
toleration_objects = []
|
mlrun/kfpops.py
CHANGED
|
@@ -15,7 +15,7 @@ import json
|
|
|
15
15
|
import os
|
|
16
16
|
import os.path
|
|
17
17
|
from copy import deepcopy
|
|
18
|
-
from typing import
|
|
18
|
+
from typing import Union
|
|
19
19
|
|
|
20
20
|
import inflection
|
|
21
21
|
from kfp import dsl
|
|
@@ -195,7 +195,7 @@ def mlrun_op(
|
|
|
195
195
|
hyper_param_options=None,
|
|
196
196
|
verbose=None,
|
|
197
197
|
scrape_metrics=False,
|
|
198
|
-
returns:
|
|
198
|
+
returns: list[Union[str, dict[str, str]]] = None,
|
|
199
199
|
auto_build: bool = False,
|
|
200
200
|
):
|
|
201
201
|
"""mlrun KubeFlow pipelines operator, use to form pipeline steps
|
mlrun/launcher/base.py
CHANGED
|
@@ -16,7 +16,7 @@ import ast
|
|
|
16
16
|
import copy
|
|
17
17
|
import os
|
|
18
18
|
import uuid
|
|
19
|
-
from typing import Any, Callable,
|
|
19
|
+
from typing import Any, Callable, Optional, Union
|
|
20
20
|
|
|
21
21
|
import mlrun.common.schemas
|
|
22
22
|
import mlrun.config
|
|
@@ -53,7 +53,7 @@ class BaseLauncher(abc.ABC):
|
|
|
53
53
|
name: Optional[str] = "",
|
|
54
54
|
project: Optional[str] = "",
|
|
55
55
|
params: Optional[dict] = None,
|
|
56
|
-
inputs: Optional[
|
|
56
|
+
inputs: Optional[dict[str, str]] = None,
|
|
57
57
|
out_path: Optional[str] = "",
|
|
58
58
|
workdir: Optional[str] = "",
|
|
59
59
|
artifact_path: Optional[str] = "",
|
|
@@ -61,16 +61,16 @@ class BaseLauncher(abc.ABC):
|
|
|
61
61
|
schedule: Optional[
|
|
62
62
|
Union[str, mlrun.common.schemas.schedule.ScheduleCronTrigger]
|
|
63
63
|
] = None,
|
|
64
|
-
hyperparams:
|
|
64
|
+
hyperparams: dict[str, list] = None,
|
|
65
65
|
hyper_param_options: Optional[mlrun.model.HyperParamOptions] = None,
|
|
66
66
|
verbose: Optional[bool] = None,
|
|
67
67
|
scrape_metrics: Optional[bool] = None,
|
|
68
68
|
local_code_path: Optional[str] = None,
|
|
69
69
|
auto_build: Optional[bool] = None,
|
|
70
|
-
param_file_secrets: Optional[
|
|
71
|
-
notifications: Optional[
|
|
72
|
-
returns: Optional[
|
|
73
|
-
state_thresholds: Optional[
|
|
70
|
+
param_file_secrets: Optional[dict[str, str]] = None,
|
|
71
|
+
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
72
|
+
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
73
|
+
state_thresholds: Optional[dict[str, int]] = None,
|
|
74
74
|
) -> "mlrun.run.RunObject":
|
|
75
75
|
"""run the function from the server/client[local/remote]"""
|
|
76
76
|
pass
|
|
@@ -175,7 +175,7 @@ class BaseLauncher(abc.ABC):
|
|
|
175
175
|
if message:
|
|
176
176
|
logger.warning(message, output_path=run.spec.output_path)
|
|
177
177
|
|
|
178
|
-
def _validate_run_params(self, parameters:
|
|
178
|
+
def _validate_run_params(self, parameters: dict[str, Any]):
|
|
179
179
|
for param_name, param_value in parameters.items():
|
|
180
180
|
if isinstance(param_value, dict):
|
|
181
181
|
# if the parameter is a dict, we might have some nested parameters,
|
|
@@ -237,8 +237,8 @@ class BaseLauncher(abc.ABC):
|
|
|
237
237
|
out_path=None,
|
|
238
238
|
artifact_path=None,
|
|
239
239
|
workdir=None,
|
|
240
|
-
notifications:
|
|
241
|
-
state_thresholds: Optional[
|
|
240
|
+
notifications: list[mlrun.model.Notification] = None,
|
|
241
|
+
state_thresholds: Optional[dict[str, int]] = None,
|
|
242
242
|
):
|
|
243
243
|
run.spec.handler = (
|
|
244
244
|
handler or run.spec.handler or runtime.spec.default_handler or ""
|