mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +11 -1
- mlrun/__main__.py +26 -112
- mlrun/alerts/__init__.py +15 -0
- mlrun/alerts/alert.py +144 -0
- mlrun/api/schemas/__init__.py +5 -4
- mlrun/artifacts/__init__.py +8 -3
- mlrun/artifacts/base.py +46 -257
- mlrun/artifacts/dataset.py +11 -192
- mlrun/artifacts/manager.py +47 -48
- mlrun/artifacts/model.py +31 -159
- mlrun/artifacts/plots.py +23 -380
- mlrun/common/constants.py +69 -0
- mlrun/common/db/sql_session.py +2 -3
- mlrun/common/formatters/__init__.py +19 -0
- mlrun/common/formatters/artifact.py +21 -0
- mlrun/common/formatters/base.py +78 -0
- mlrun/common/formatters/function.py +41 -0
- mlrun/common/formatters/pipeline.py +53 -0
- mlrun/common/formatters/project.py +51 -0
- mlrun/common/helpers.py +1 -2
- mlrun/common/model_monitoring/helpers.py +9 -5
- mlrun/{runtimes → common/runtimes}/constants.py +37 -9
- mlrun/common/schemas/__init__.py +24 -4
- mlrun/common/schemas/alert.py +203 -0
- mlrun/common/schemas/api_gateway.py +148 -0
- mlrun/common/schemas/artifact.py +18 -8
- mlrun/common/schemas/auth.py +11 -5
- mlrun/common/schemas/background_task.py +1 -1
- mlrun/common/schemas/client_spec.py +4 -1
- mlrun/common/schemas/feature_store.py +16 -16
- mlrun/common/schemas/frontend_spec.py +8 -7
- mlrun/common/schemas/function.py +5 -1
- mlrun/common/schemas/hub.py +11 -18
- mlrun/common/schemas/memory_reports.py +2 -2
- mlrun/common/schemas/model_monitoring/__init__.py +18 -3
- mlrun/common/schemas/model_monitoring/constants.py +83 -26
- mlrun/common/schemas/model_monitoring/grafana.py +13 -9
- mlrun/common/schemas/model_monitoring/model_endpoints.py +99 -16
- mlrun/common/schemas/notification.py +4 -4
- mlrun/common/schemas/object.py +2 -2
- mlrun/{runtimes/mpijob/v1alpha1.py → common/schemas/pagination.py} +10 -13
- mlrun/common/schemas/pipeline.py +1 -10
- mlrun/common/schemas/project.py +24 -23
- mlrun/common/schemas/runtime_resource.py +8 -12
- mlrun/common/schemas/schedule.py +3 -3
- mlrun/common/schemas/tag.py +1 -2
- mlrun/common/schemas/workflow.py +2 -2
- mlrun/common/types.py +7 -1
- mlrun/config.py +54 -17
- mlrun/data_types/to_pandas.py +10 -12
- mlrun/datastore/__init__.py +5 -8
- mlrun/datastore/alibaba_oss.py +130 -0
- mlrun/datastore/azure_blob.py +17 -5
- mlrun/datastore/base.py +62 -39
- mlrun/datastore/datastore.py +28 -9
- mlrun/datastore/datastore_profile.py +146 -20
- mlrun/datastore/filestore.py +0 -1
- mlrun/datastore/google_cloud_storage.py +6 -2
- mlrun/datastore/hdfs.py +56 -0
- mlrun/datastore/inmem.py +2 -2
- mlrun/datastore/redis.py +6 -2
- mlrun/datastore/s3.py +9 -0
- mlrun/datastore/snowflake_utils.py +43 -0
- mlrun/datastore/sources.py +201 -96
- mlrun/datastore/spark_utils.py +1 -2
- mlrun/datastore/store_resources.py +7 -7
- mlrun/datastore/targets.py +358 -104
- mlrun/datastore/utils.py +72 -58
- mlrun/datastore/v3io.py +5 -1
- mlrun/db/base.py +185 -35
- mlrun/db/factory.py +1 -1
- mlrun/db/httpdb.py +614 -179
- mlrun/db/nopdb.py +210 -26
- mlrun/errors.py +12 -1
- mlrun/execution.py +41 -24
- mlrun/feature_store/__init__.py +0 -2
- mlrun/feature_store/api.py +40 -72
- mlrun/feature_store/common.py +1 -1
- mlrun/feature_store/feature_set.py +76 -55
- mlrun/feature_store/feature_vector.py +28 -30
- mlrun/feature_store/ingestion.py +7 -6
- mlrun/feature_store/retrieval/base.py +16 -11
- mlrun/feature_store/retrieval/conversion.py +11 -13
- mlrun/feature_store/retrieval/dask_merger.py +2 -0
- mlrun/feature_store/retrieval/job.py +9 -3
- mlrun/feature_store/retrieval/local_merger.py +2 -0
- mlrun/feature_store/retrieval/spark_merger.py +34 -24
- mlrun/feature_store/steps.py +37 -34
- mlrun/features.py +9 -20
- mlrun/frameworks/_common/artifacts_library.py +9 -9
- mlrun/frameworks/_common/mlrun_interface.py +5 -5
- mlrun/frameworks/_common/model_handler.py +48 -48
- mlrun/frameworks/_common/plan.py +2 -3
- mlrun/frameworks/_common/producer.py +3 -4
- mlrun/frameworks/_common/utils.py +5 -5
- mlrun/frameworks/_dl_common/loggers/logger.py +6 -7
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +9 -9
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +23 -47
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -2
- mlrun/frameworks/_ml_common/loggers/logger.py +3 -4
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +4 -5
- mlrun/frameworks/_ml_common/model_handler.py +24 -24
- mlrun/frameworks/_ml_common/pkl_model_server.py +2 -2
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +2 -3
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +2 -3
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
- mlrun/frameworks/_ml_common/utils.py +4 -4
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +9 -9
- mlrun/frameworks/huggingface/model_server.py +4 -4
- mlrun/frameworks/lgbm/__init__.py +33 -33
- mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -5
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -5
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -3
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +6 -6
- mlrun/frameworks/lgbm/model_handler.py +10 -10
- mlrun/frameworks/lgbm/model_server.py +6 -6
- mlrun/frameworks/lgbm/utils.py +5 -5
- mlrun/frameworks/onnx/dataset.py +8 -8
- mlrun/frameworks/onnx/mlrun_interface.py +3 -3
- mlrun/frameworks/onnx/model_handler.py +6 -6
- mlrun/frameworks/onnx/model_server.py +7 -7
- mlrun/frameworks/parallel_coordinates.py +4 -3
- mlrun/frameworks/pytorch/__init__.py +18 -18
- mlrun/frameworks/pytorch/callbacks/callback.py +4 -5
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +17 -17
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +11 -11
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +23 -29
- mlrun/frameworks/pytorch/callbacks_handler.py +38 -38
- mlrun/frameworks/pytorch/mlrun_interface.py +20 -20
- mlrun/frameworks/pytorch/model_handler.py +17 -17
- mlrun/frameworks/pytorch/model_server.py +7 -7
- mlrun/frameworks/sklearn/__init__.py +13 -13
- mlrun/frameworks/sklearn/estimator.py +4 -4
- mlrun/frameworks/sklearn/metrics_library.py +14 -14
- mlrun/frameworks/sklearn/mlrun_interface.py +3 -6
- mlrun/frameworks/sklearn/model_handler.py +2 -2
- mlrun/frameworks/tf_keras/__init__.py +10 -7
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +15 -15
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +11 -11
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +19 -23
- mlrun/frameworks/tf_keras/mlrun_interface.py +9 -11
- mlrun/frameworks/tf_keras/model_handler.py +14 -14
- mlrun/frameworks/tf_keras/model_server.py +6 -6
- mlrun/frameworks/xgboost/__init__.py +13 -13
- mlrun/frameworks/xgboost/model_handler.py +6 -6
- mlrun/k8s_utils.py +14 -16
- mlrun/launcher/__init__.py +1 -1
- mlrun/launcher/base.py +16 -15
- mlrun/launcher/client.py +8 -6
- mlrun/launcher/factory.py +1 -1
- mlrun/launcher/local.py +17 -11
- mlrun/launcher/remote.py +16 -10
- mlrun/lists.py +7 -6
- mlrun/model.py +238 -73
- mlrun/model_monitoring/__init__.py +1 -1
- mlrun/model_monitoring/api.py +138 -315
- mlrun/model_monitoring/application.py +5 -296
- mlrun/model_monitoring/applications/__init__.py +24 -0
- mlrun/model_monitoring/applications/_application_steps.py +157 -0
- mlrun/model_monitoring/applications/base.py +282 -0
- mlrun/model_monitoring/applications/context.py +214 -0
- mlrun/model_monitoring/applications/evidently_base.py +211 -0
- mlrun/model_monitoring/applications/histogram_data_drift.py +349 -0
- mlrun/model_monitoring/applications/results.py +99 -0
- mlrun/model_monitoring/controller.py +104 -84
- mlrun/model_monitoring/controller_handler.py +13 -5
- mlrun/model_monitoring/db/__init__.py +18 -0
- mlrun/model_monitoring/{stores → db/stores}/__init__.py +43 -36
- mlrun/model_monitoring/db/stores/base/__init__.py +15 -0
- mlrun/model_monitoring/{stores/model_endpoint_store.py → db/stores/base/store.py} +64 -40
- mlrun/model_monitoring/db/stores/sqldb/__init__.py +13 -0
- mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +71 -0
- mlrun/model_monitoring/{stores → db/stores/sqldb}/models/base.py +109 -5
- mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +88 -0
- mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py} +19 -13
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +684 -0
- mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +13 -0
- mlrun/model_monitoring/{stores/kv_model_endpoint_store.py → db/stores/v3io_kv/kv_store.py} +310 -165
- mlrun/model_monitoring/db/tsdb/__init__.py +100 -0
- mlrun/model_monitoring/db/tsdb/base.py +329 -0
- mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
- mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +240 -0
- mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +45 -0
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +397 -0
- mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +117 -0
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +630 -0
- mlrun/model_monitoring/evidently_application.py +6 -118
- mlrun/model_monitoring/features_drift_table.py +134 -106
- mlrun/model_monitoring/helpers.py +127 -28
- mlrun/model_monitoring/metrics/__init__.py +13 -0
- mlrun/model_monitoring/metrics/histogram_distance.py +127 -0
- mlrun/model_monitoring/model_endpoint.py +3 -2
- mlrun/model_monitoring/prometheus.py +1 -4
- mlrun/model_monitoring/stream_processing.py +62 -231
- mlrun/model_monitoring/tracking_policy.py +9 -2
- mlrun/model_monitoring/writer.py +152 -124
- mlrun/package/__init__.py +6 -6
- mlrun/package/context_handler.py +5 -5
- mlrun/package/packager.py +7 -7
- mlrun/package/packagers/default_packager.py +6 -6
- mlrun/package/packagers/numpy_packagers.py +15 -15
- mlrun/package/packagers/pandas_packagers.py +5 -5
- mlrun/package/packagers/python_standard_library_packagers.py +10 -10
- mlrun/package/packagers_manager.py +19 -23
- mlrun/package/utils/_formatter.py +6 -6
- mlrun/package/utils/_pickler.py +2 -2
- mlrun/package/utils/_supported_format.py +4 -4
- mlrun/package/utils/log_hint_utils.py +2 -2
- mlrun/package/utils/type_hint_utils.py +4 -9
- mlrun/platforms/__init__.py +11 -10
- mlrun/platforms/iguazio.py +24 -203
- mlrun/projects/operations.py +35 -21
- mlrun/projects/pipelines.py +68 -99
- mlrun/projects/project.py +830 -266
- mlrun/render.py +3 -11
- mlrun/run.py +162 -166
- mlrun/runtimes/__init__.py +62 -7
- mlrun/runtimes/base.py +39 -32
- mlrun/runtimes/daskjob.py +8 -8
- mlrun/runtimes/databricks_job/databricks_cancel_task.py +1 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +7 -7
- mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
- mlrun/runtimes/funcdoc.py +0 -28
- mlrun/runtimes/function_reference.py +1 -1
- mlrun/runtimes/kubejob.py +28 -122
- mlrun/runtimes/local.py +6 -3
- mlrun/runtimes/mpijob/__init__.py +0 -20
- mlrun/runtimes/mpijob/abstract.py +9 -10
- mlrun/runtimes/mpijob/v1.py +1 -1
- mlrun/{model_monitoring/stores/models/sqlite.py → runtimes/nuclio/__init__.py} +7 -9
- mlrun/runtimes/nuclio/api_gateway.py +709 -0
- mlrun/runtimes/nuclio/application/__init__.py +15 -0
- mlrun/runtimes/nuclio/application/application.py +523 -0
- mlrun/runtimes/nuclio/application/reverse_proxy.go +95 -0
- mlrun/runtimes/{function.py → nuclio/function.py} +112 -73
- mlrun/runtimes/{nuclio.py → nuclio/nuclio.py} +6 -6
- mlrun/runtimes/{serving.py → nuclio/serving.py} +45 -51
- mlrun/runtimes/pod.py +286 -88
- mlrun/runtimes/remotesparkjob.py +2 -2
- mlrun/runtimes/sparkjob/spark3job.py +51 -34
- mlrun/runtimes/utils.py +7 -75
- mlrun/secrets.py +9 -5
- mlrun/serving/remote.py +2 -7
- mlrun/serving/routers.py +13 -10
- mlrun/serving/server.py +22 -26
- mlrun/serving/states.py +99 -25
- mlrun/serving/utils.py +3 -3
- mlrun/serving/v1_serving.py +6 -7
- mlrun/serving/v2_serving.py +59 -20
- mlrun/track/tracker.py +2 -1
- mlrun/track/tracker_manager.py +3 -3
- mlrun/track/trackers/mlflow_tracker.py +1 -2
- mlrun/utils/async_http.py +5 -7
- mlrun/utils/azure_vault.py +1 -1
- mlrun/utils/clones.py +1 -2
- mlrun/utils/condition_evaluator.py +3 -3
- mlrun/utils/db.py +3 -3
- mlrun/utils/helpers.py +183 -197
- mlrun/utils/http.py +2 -5
- mlrun/utils/logger.py +76 -14
- mlrun/utils/notifications/notification/__init__.py +17 -12
- mlrun/utils/notifications/notification/base.py +14 -2
- mlrun/utils/notifications/notification/console.py +2 -0
- mlrun/utils/notifications/notification/git.py +3 -1
- mlrun/utils/notifications/notification/ipython.py +3 -1
- mlrun/utils/notifications/notification/slack.py +101 -21
- mlrun/utils/notifications/notification/webhook.py +11 -1
- mlrun/utils/notifications/notification_pusher.py +155 -30
- mlrun/utils/retryer.py +208 -0
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/v3io_clients.py +2 -4
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +2 -6
- {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/METADATA +31 -19
- mlrun-1.7.0rc20.dist-info/RECORD +353 -0
- mlrun/kfpops.py +0 -868
- mlrun/model_monitoring/batch.py +0 -1095
- mlrun/model_monitoring/stores/models/__init__.py +0 -27
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -384
- mlrun/platforms/other.py +0 -306
- mlrun-1.6.4rc2.dist-info/RECORD +0 -314
- {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/LICENSE +0 -0
- {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/WHEEL +0 -0
- {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/entry_points.txt +0 -0
- {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/top_level.txt +0 -0
mlrun/feature_store/steps.py
CHANGED
|
@@ -16,7 +16,7 @@ import math
|
|
|
16
16
|
import re
|
|
17
17
|
import uuid
|
|
18
18
|
from collections import OrderedDict
|
|
19
|
-
from typing import Any,
|
|
19
|
+
from typing import Any, Optional, Union
|
|
20
20
|
|
|
21
21
|
import numpy as np
|
|
22
22
|
import pandas as pd
|
|
@@ -92,8 +92,6 @@ class MLRunStep(MapClass):
|
|
|
92
92
|
|
|
93
93
|
|
|
94
94
|
class FeaturesetValidator(StepToDict, MLRunStep):
|
|
95
|
-
"""Validate feature values according to the feature set validation policy"""
|
|
96
|
-
|
|
97
95
|
def __init__(self, featureset=None, columns=None, name=None, **kwargs):
|
|
98
96
|
"""Validate feature values according to the feature set validation policy
|
|
99
97
|
|
|
@@ -152,11 +150,9 @@ class FeaturesetValidator(StepToDict, MLRunStep):
|
|
|
152
150
|
|
|
153
151
|
|
|
154
152
|
class MapValues(StepToDict, MLRunStep):
|
|
155
|
-
"""Map column values to new values"""
|
|
156
|
-
|
|
157
153
|
def __init__(
|
|
158
154
|
self,
|
|
159
|
-
mapping:
|
|
155
|
+
mapping: dict[str, dict[Union[str, int, bool], Any]],
|
|
160
156
|
with_original_features: bool = False,
|
|
161
157
|
suffix: str = "mapped",
|
|
162
158
|
**kwargs,
|
|
@@ -166,13 +162,19 @@ class MapValues(StepToDict, MLRunStep):
|
|
|
166
162
|
example::
|
|
167
163
|
|
|
168
164
|
# replace the value "U" with '0' in the age column
|
|
169
|
-
graph.to(MapValues(mapping={
|
|
165
|
+
graph.to(MapValues(mapping={"age": {"U": "0"}}, with_original_features=True))
|
|
170
166
|
|
|
171
167
|
# replace integers, example
|
|
172
|
-
graph.to(MapValues(mapping={
|
|
168
|
+
graph.to(MapValues(mapping={"not": {0: 1, 1: 0}}))
|
|
173
169
|
|
|
174
170
|
# replace by range, use -inf and inf for extended range
|
|
175
|
-
graph.to(
|
|
171
|
+
graph.to(
|
|
172
|
+
MapValues(
|
|
173
|
+
mapping={
|
|
174
|
+
"numbers": {"ranges": {"negative": [-inf, 0], "positive": [0, inf]}}
|
|
175
|
+
}
|
|
176
|
+
)
|
|
177
|
+
)
|
|
176
178
|
|
|
177
179
|
:param mapping: a dict with entry per column and the associated old/new values map
|
|
178
180
|
:param with_original_features: set to True to keep the original features
|
|
@@ -377,7 +379,7 @@ class Imputer(StepToDict, MLRunStep):
|
|
|
377
379
|
self,
|
|
378
380
|
method: str = "avg",
|
|
379
381
|
default_value=None,
|
|
380
|
-
mapping:
|
|
382
|
+
mapping: dict[str, Any] = None,
|
|
381
383
|
**kwargs,
|
|
382
384
|
):
|
|
383
385
|
"""Replace None values with default values
|
|
@@ -423,13 +425,15 @@ class Imputer(StepToDict, MLRunStep):
|
|
|
423
425
|
|
|
424
426
|
|
|
425
427
|
class OneHotEncoder(StepToDict, MLRunStep):
|
|
426
|
-
def __init__(self, mapping:
|
|
428
|
+
def __init__(self, mapping: dict[str, list[Union[int, str]]], **kwargs):
|
|
427
429
|
"""Create new binary fields, one per category (one hot encoded)
|
|
428
430
|
|
|
429
431
|
example::
|
|
430
432
|
|
|
431
|
-
mapping = {
|
|
432
|
-
|
|
433
|
+
mapping = {
|
|
434
|
+
"category": ["food", "health", "transportation"],
|
|
435
|
+
"gender": ["male", "female"],
|
|
436
|
+
}
|
|
433
437
|
graph.to(OneHotEncoder(mapping=one_hot_encoder_mapping))
|
|
434
438
|
|
|
435
439
|
:param mapping: a dict of per column categories (to map to binary fields)
|
|
@@ -510,15 +514,13 @@ class OneHotEncoder(StepToDict, MLRunStep):
|
|
|
510
514
|
|
|
511
515
|
|
|
512
516
|
class DateExtractor(StepToDict, MLRunStep):
|
|
513
|
-
"""Date Extractor allows you to extract a date-time component"""
|
|
514
|
-
|
|
515
517
|
def __init__(
|
|
516
518
|
self,
|
|
517
|
-
parts: Union[
|
|
519
|
+
parts: Union[dict[str, str], list[str]],
|
|
518
520
|
timestamp_col: str = None,
|
|
519
521
|
**kwargs,
|
|
520
522
|
):
|
|
521
|
-
"""Date Extractor
|
|
523
|
+
"""Date Extractor extracts a date-time component into new columns
|
|
522
524
|
|
|
523
525
|
The extracted date part will appear as `<timestamp_col>_<date_part>` feature.
|
|
524
526
|
|
|
@@ -548,10 +550,12 @@ class DateExtractor(StepToDict, MLRunStep):
|
|
|
548
550
|
|
|
549
551
|
# (taken from the fraud-detection end-to-end feature store demo)
|
|
550
552
|
# Define the Transactions FeatureSet
|
|
551
|
-
transaction_set = fstore.FeatureSet(
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
553
|
+
transaction_set = fstore.FeatureSet(
|
|
554
|
+
"transactions",
|
|
555
|
+
entities=[fstore.Entity("source")],
|
|
556
|
+
timestamp_key="timestamp",
|
|
557
|
+
description="transactions feature set",
|
|
558
|
+
)
|
|
555
559
|
|
|
556
560
|
# Get FeatureSet computation graph
|
|
557
561
|
transaction_graph = transaction_set.graph
|
|
@@ -559,11 +563,11 @@ class DateExtractor(StepToDict, MLRunStep):
|
|
|
559
563
|
# Add the custom `DateExtractor` step
|
|
560
564
|
# to the computation graph
|
|
561
565
|
transaction_graph.to(
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
566
|
+
class_name="DateExtractor",
|
|
567
|
+
name="Extract Dates",
|
|
568
|
+
parts=["hour", "day_of_week"],
|
|
569
|
+
timestamp_col="timestamp",
|
|
570
|
+
)
|
|
567
571
|
|
|
568
572
|
:param parts: list of pandas style date-time parts you want to extract.
|
|
569
573
|
:param timestamp_col: The name of the column containing the timestamps to extract from,
|
|
@@ -629,8 +633,6 @@ class DateExtractor(StepToDict, MLRunStep):
|
|
|
629
633
|
|
|
630
634
|
|
|
631
635
|
class SetEventMetadata(MapClass):
|
|
632
|
-
"""Set the event metadata (id and key) from the event body"""
|
|
633
|
-
|
|
634
636
|
def __init__(
|
|
635
637
|
self,
|
|
636
638
|
id_path: Optional[str] = None,
|
|
@@ -695,18 +697,19 @@ class SetEventMetadata(MapClass):
|
|
|
695
697
|
|
|
696
698
|
|
|
697
699
|
class DropFeatures(StepToDict, MLRunStep):
|
|
698
|
-
def __init__(self, features:
|
|
700
|
+
def __init__(self, features: list[str], **kwargs):
|
|
699
701
|
"""Drop all the features from feature list
|
|
700
702
|
|
|
701
703
|
:param features: string list of the features names to drop
|
|
702
704
|
|
|
703
705
|
example::
|
|
704
706
|
|
|
705
|
-
feature_set = fstore.FeatureSet(
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
707
|
+
feature_set = fstore.FeatureSet(
|
|
708
|
+
"fs-new",
|
|
709
|
+
entities=[fstore.Entity("id")],
|
|
710
|
+
description="feature set",
|
|
711
|
+
engine="pandas",
|
|
712
|
+
)
|
|
710
713
|
# Pre-processing graph steps
|
|
711
714
|
feature_set.graph.to(DropFeatures(features=["age"]))
|
|
712
715
|
df_pandas = feature_set.ingest(data)
|
mlrun/features.py
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
#
|
|
15
15
|
import math
|
|
16
16
|
import re
|
|
17
|
-
from typing import
|
|
17
|
+
from typing import Optional, Union
|
|
18
18
|
|
|
19
19
|
from .data_types import ValueType, python_type_to_value_type
|
|
20
20
|
from .errors import MLRunRuntimeError, err_to_str
|
|
@@ -44,7 +44,7 @@ class Entity(ModelObj):
|
|
|
44
44
|
name: str = None,
|
|
45
45
|
value_type: Union[ValueType, str] = None,
|
|
46
46
|
description: str = None,
|
|
47
|
-
labels: Optional[
|
|
47
|
+
labels: Optional[dict[str, str]] = None,
|
|
48
48
|
):
|
|
49
49
|
"""data entity (index key)
|
|
50
50
|
|
|
@@ -65,8 +65,6 @@ class Entity(ModelObj):
|
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
class Feature(ModelObj):
|
|
68
|
-
"""data feature"""
|
|
69
|
-
|
|
70
68
|
_dict_fields = [
|
|
71
69
|
"name",
|
|
72
70
|
"description",
|
|
@@ -82,13 +80,13 @@ class Feature(ModelObj):
|
|
|
82
80
|
def __init__(
|
|
83
81
|
self,
|
|
84
82
|
value_type: Union[ValueType, str] = None,
|
|
85
|
-
dims:
|
|
83
|
+
dims: list[int] = None,
|
|
86
84
|
description: str = None,
|
|
87
85
|
aggregate: bool = None,
|
|
88
86
|
name: str = None,
|
|
89
87
|
validator=None,
|
|
90
88
|
default: str = None,
|
|
91
|
-
labels:
|
|
89
|
+
labels: dict[str, str] = None,
|
|
92
90
|
):
|
|
93
91
|
"""data feature
|
|
94
92
|
|
|
@@ -240,10 +238,7 @@ class Validator(ModelObj):
|
|
|
240
238
|
from mlrun.features import Validator
|
|
241
239
|
|
|
242
240
|
# Add validator to the feature 'bid' with check type
|
|
243
|
-
quotes_set["bid"].validator = Validator(
|
|
244
|
-
check_type=True,
|
|
245
|
-
severity="info"
|
|
246
|
-
)
|
|
241
|
+
quotes_set["bid"].validator = Validator(check_type=True, severity="info")
|
|
247
242
|
|
|
248
243
|
:param check_type: check feature type e.g. True, False
|
|
249
244
|
:param severity: severity name e.g. info, warning, etc.
|
|
@@ -282,10 +277,7 @@ class MinMaxValidator(Validator):
|
|
|
282
277
|
|
|
283
278
|
# Add validator to the feature 'bid', where valid
|
|
284
279
|
# minimal value is 52
|
|
285
|
-
quotes_set["bid"].validator = MinMaxValidator(
|
|
286
|
-
min=52,
|
|
287
|
-
severity="info"
|
|
288
|
-
)
|
|
280
|
+
quotes_set["bid"].validator = MinMaxValidator(min=52, severity="info")
|
|
289
281
|
|
|
290
282
|
:param check_type: check feature type e.g. True, False
|
|
291
283
|
:param severity: severity name e.g. info, warning, etc.
|
|
@@ -346,9 +338,7 @@ class MinMaxLenValidator(Validator):
|
|
|
346
338
|
# Add length validator to the feature 'ticker', where valid
|
|
347
339
|
# minimal length is 1 and maximal length is 10
|
|
348
340
|
quotes_set["ticker"].validator = MinMaxLenValidator(
|
|
349
|
-
min=1,
|
|
350
|
-
max=10,
|
|
351
|
-
severity="info"
|
|
341
|
+
min=1, max=10, severity="info"
|
|
352
342
|
)
|
|
353
343
|
|
|
354
344
|
:param check_type: check feature type e.g. True, False
|
|
@@ -410,8 +400,7 @@ class RegexValidator(Validator):
|
|
|
410
400
|
# expression '(\b[A-Za-z]{1}[0-9]{7}\b)' where valid values are
|
|
411
401
|
# e.g. A1234567, z9874563, etc.
|
|
412
402
|
quotes_set["name"].validator = RegexValidator(
|
|
413
|
-
regex=r"(\b[A-Za-z]{1}[0-9]{7}\b)",
|
|
414
|
-
severity="info"
|
|
403
|
+
regex=r"(\b[A-Za-z]{1}[0-9]{7}\b)", severity="info"
|
|
415
404
|
)
|
|
416
405
|
|
|
417
406
|
:param check_type: check feature type e.g. True, False
|
|
@@ -445,7 +434,7 @@ class RegexValidator(Validator):
|
|
|
445
434
|
|
|
446
435
|
@classmethod
|
|
447
436
|
def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
|
|
448
|
-
new_obj = super(
|
|
437
|
+
new_obj = super().from_dict(
|
|
449
438
|
struct=struct, fields=fields, deprecated_fields=deprecated_fields
|
|
450
439
|
)
|
|
451
440
|
if hasattr(new_obj, "regex"):
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
15
|
from abc import ABC, abstractmethod
|
|
16
|
-
from typing import
|
|
16
|
+
from typing import Union
|
|
17
17
|
|
|
18
18
|
import mlrun
|
|
19
19
|
|
|
@@ -39,12 +39,12 @@ class ArtifactsLibrary(ABC):
|
|
|
39
39
|
@classmethod
|
|
40
40
|
def get_plans(
|
|
41
41
|
cls,
|
|
42
|
-
artifacts: Union[
|
|
42
|
+
artifacts: Union[list[Plan], dict[str, dict], list[str]] = None,
|
|
43
43
|
context: mlrun.MLClientCtx = None,
|
|
44
44
|
include_default: bool = True,
|
|
45
45
|
# custom_plans: dict = None, :param custom_plans: Custom user plans objects to initialize from.
|
|
46
46
|
**default_kwargs,
|
|
47
|
-
) ->
|
|
47
|
+
) -> list[Plan]:
|
|
48
48
|
"""
|
|
49
49
|
Get plans for a run. The plans will be taken from the provided artifacts / configuration via code, from provided
|
|
50
50
|
configuration via MLRun context and if the 'include_default' is True, from the framework artifact library's
|
|
@@ -97,7 +97,7 @@ class ArtifactsLibrary(ABC):
|
|
|
97
97
|
|
|
98
98
|
@classmethod
|
|
99
99
|
@abstractmethod
|
|
100
|
-
def default(cls, **kwargs) ->
|
|
100
|
+
def default(cls, **kwargs) -> list[Plan]:
|
|
101
101
|
"""
|
|
102
102
|
Get the default artifacts plans list of this framework's library.
|
|
103
103
|
|
|
@@ -106,7 +106,7 @@ class ArtifactsLibrary(ABC):
|
|
|
106
106
|
pass
|
|
107
107
|
|
|
108
108
|
@classmethod
|
|
109
|
-
def _get_library_plans(cls) ->
|
|
109
|
+
def _get_library_plans(cls) -> dict[str, type[Plan]]:
|
|
110
110
|
"""
|
|
111
111
|
Get all the supported plans in this library.
|
|
112
112
|
|
|
@@ -120,8 +120,8 @@ class ArtifactsLibrary(ABC):
|
|
|
120
120
|
|
|
121
121
|
@staticmethod
|
|
122
122
|
def _from_dict(
|
|
123
|
-
requested_plans:
|
|
124
|
-
) ->
|
|
123
|
+
requested_plans: dict[str, dict], available_plans: dict[str, type[Plan]]
|
|
124
|
+
) -> list[Plan]:
|
|
125
125
|
"""
|
|
126
126
|
Initialize a list of plans from a given configuration dictionary. The configuration is expected to be a
|
|
127
127
|
dictionary of plans and their initialization parameters in the following format:
|
|
@@ -162,8 +162,8 @@ class ArtifactsLibrary(ABC):
|
|
|
162
162
|
|
|
163
163
|
@staticmethod
|
|
164
164
|
def _from_list(
|
|
165
|
-
requested_plans:
|
|
166
|
-
) ->
|
|
165
|
+
requested_plans: list[str], available_plans: dict[str, type[Plan]]
|
|
166
|
+
) -> list[Plan]:
|
|
167
167
|
"""
|
|
168
168
|
Initialize a list of plans from a given configuration list. The configuration is expected to be a list of plans
|
|
169
169
|
names to be initialized with their default configuration.
|
|
@@ -17,7 +17,7 @@ import functools
|
|
|
17
17
|
import inspect
|
|
18
18
|
from abc import ABC
|
|
19
19
|
from types import FunctionType, MethodType
|
|
20
|
-
from typing import Any,
|
|
20
|
+
from typing import Any, Generic, Union
|
|
21
21
|
|
|
22
22
|
from .utils import CommonTypes
|
|
23
23
|
|
|
@@ -173,7 +173,7 @@ class MLRunInterface(ABC, Generic[CommonTypes.MLRunInterfaceableType]):
|
|
|
173
173
|
def _insert_properties(
|
|
174
174
|
cls,
|
|
175
175
|
obj: CommonTypes.MLRunInterfaceableType,
|
|
176
|
-
properties:
|
|
176
|
+
properties: dict[str, Any] = None,
|
|
177
177
|
):
|
|
178
178
|
"""
|
|
179
179
|
Insert the properties of the interface to the object. The properties default values are being copied (not deep
|
|
@@ -238,7 +238,7 @@ class MLRunInterface(ABC, Generic[CommonTypes.MLRunInterfaceableType]):
|
|
|
238
238
|
|
|
239
239
|
@classmethod
|
|
240
240
|
def _replace_properties(
|
|
241
|
-
cls, obj: CommonTypes.MLRunInterfaceableType, properties:
|
|
241
|
+
cls, obj: CommonTypes.MLRunInterfaceableType, properties: dict[str, Any] = None
|
|
242
242
|
):
|
|
243
243
|
"""
|
|
244
244
|
Replace the properties of the given object according to the configuration in the MLRun interface.
|
|
@@ -282,7 +282,7 @@ class MLRunInterface(ABC, Generic[CommonTypes.MLRunInterfaceableType]):
|
|
|
282
282
|
|
|
283
283
|
@classmethod
|
|
284
284
|
def _replace_functions(
|
|
285
|
-
cls, obj: CommonTypes.MLRunInterfaceableType, functions:
|
|
285
|
+
cls, obj: CommonTypes.MLRunInterfaceableType, functions: list[str] = None
|
|
286
286
|
):
|
|
287
287
|
"""
|
|
288
288
|
Replace the functions / methods of the given object according to the configuration in the MLRun interface.
|
|
@@ -420,7 +420,7 @@ class MLRunInterface(ABC, Generic[CommonTypes.MLRunInterfaceableType]):
|
|
|
420
420
|
passed_args: tuple = None,
|
|
421
421
|
passed_kwargs: dict = None,
|
|
422
422
|
default_value: Any = None,
|
|
423
|
-
) ->
|
|
423
|
+
) -> tuple[Any, Union[str, int, None]]:
|
|
424
424
|
"""
|
|
425
425
|
Get a passed argument (from *args or **kwargs) to a function. If the argument was not found the default value
|
|
426
426
|
will be returned. In addition, the keyword of the argument in `kwargs` or the index of the argument in `args`
|
|
@@ -20,7 +20,7 @@ import sys
|
|
|
20
20
|
import zipfile
|
|
21
21
|
from abc import ABC, abstractmethod
|
|
22
22
|
from types import MethodType
|
|
23
|
-
from typing import Any,
|
|
23
|
+
from typing import Any, Generic, Union
|
|
24
24
|
|
|
25
25
|
import numpy as np
|
|
26
26
|
|
|
@@ -57,10 +57,10 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
57
57
|
model_path: CommonTypes.PathType = None,
|
|
58
58
|
model_name: str = None,
|
|
59
59
|
modules_map: Union[
|
|
60
|
-
|
|
60
|
+
dict[str, Union[None, str, list[str]]], CommonTypes.PathType
|
|
61
61
|
] = None,
|
|
62
62
|
custom_objects_map: Union[
|
|
63
|
-
|
|
63
|
+
dict[str, Union[str, list[str]]], CommonTypes.PathType
|
|
64
64
|
] = None,
|
|
65
65
|
custom_objects_directory: CommonTypes.PathType = None,
|
|
66
66
|
context: MLClientCtx = None,
|
|
@@ -224,7 +224,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
224
224
|
return self._tag
|
|
225
225
|
|
|
226
226
|
@property
|
|
227
|
-
def inputs(self) -> Union[
|
|
227
|
+
def inputs(self) -> Union[list[Feature], None]:
|
|
228
228
|
"""
|
|
229
229
|
Get the input ports features list of this model's artifact. If the inputs are not set, None will be returned.
|
|
230
230
|
|
|
@@ -233,7 +233,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
233
233
|
return self._inputs
|
|
234
234
|
|
|
235
235
|
@property
|
|
236
|
-
def outputs(self) -> Union[
|
|
236
|
+
def outputs(self) -> Union[list[Feature], None]:
|
|
237
237
|
"""
|
|
238
238
|
Get the output ports features list of this model's artifact. If the outputs are not set, None will be returned.
|
|
239
239
|
|
|
@@ -242,7 +242,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
242
242
|
return self._outputs
|
|
243
243
|
|
|
244
244
|
@property
|
|
245
|
-
def labels(self) ->
|
|
245
|
+
def labels(self) -> dict[str, str]:
|
|
246
246
|
"""
|
|
247
247
|
Get the labels dictionary of this model's artifact. These will be the labels that will be logged with the model.
|
|
248
248
|
|
|
@@ -251,7 +251,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
251
251
|
return self._labels
|
|
252
252
|
|
|
253
253
|
@property
|
|
254
|
-
def parameters(self) ->
|
|
254
|
+
def parameters(self) -> dict[str, str]:
|
|
255
255
|
"""
|
|
256
256
|
Get the parameters dictionary of this model's artifact. These will be the parameters that will be logged with
|
|
257
257
|
the model.
|
|
@@ -262,7 +262,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
262
262
|
|
|
263
263
|
def get_artifacts(
|
|
264
264
|
self, committed_only: bool = False
|
|
265
|
-
) ->
|
|
265
|
+
) -> dict[str, CommonTypes.ExtraDataType]:
|
|
266
266
|
"""
|
|
267
267
|
Get the registered artifacts of this model's artifact. By default all the artifacts (logged and to be logged -
|
|
268
268
|
committed only) will be returned. To get only the artifacts registered in the current run whom are committed and
|
|
@@ -306,7 +306,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
306
306
|
def set_inputs(
|
|
307
307
|
self,
|
|
308
308
|
from_sample: CommonTypes.IOSampleType = None,
|
|
309
|
-
features:
|
|
309
|
+
features: list[Feature] = None,
|
|
310
310
|
**kwargs,
|
|
311
311
|
):
|
|
312
312
|
"""
|
|
@@ -335,7 +335,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
335
335
|
def set_outputs(
|
|
336
336
|
self,
|
|
337
337
|
from_sample: CommonTypes.IOSampleType = None,
|
|
338
|
-
features:
|
|
338
|
+
features: list[Feature] = None,
|
|
339
339
|
**kwargs,
|
|
340
340
|
):
|
|
341
341
|
"""
|
|
@@ -363,8 +363,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
363
363
|
|
|
364
364
|
def set_labels(
|
|
365
365
|
self,
|
|
366
|
-
to_add:
|
|
367
|
-
to_remove:
|
|
366
|
+
to_add: dict[str, Union[str, int, float]] = None,
|
|
367
|
+
to_remove: list[str] = None,
|
|
368
368
|
):
|
|
369
369
|
"""
|
|
370
370
|
Update the labels dictionary of this model artifact.
|
|
@@ -383,8 +383,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
383
383
|
|
|
384
384
|
def set_parameters(
|
|
385
385
|
self,
|
|
386
|
-
to_add:
|
|
387
|
-
to_remove:
|
|
386
|
+
to_add: dict[str, Union[str, int, float]] = None,
|
|
387
|
+
to_remove: list[str] = None,
|
|
388
388
|
):
|
|
389
389
|
"""
|
|
390
390
|
Update the parameters dictionary of this model artifact.
|
|
@@ -403,8 +403,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
403
403
|
|
|
404
404
|
def set_metrics(
|
|
405
405
|
self,
|
|
406
|
-
to_add:
|
|
407
|
-
to_remove:
|
|
406
|
+
to_add: dict[str, CommonTypes.ExtraDataType] = None,
|
|
407
|
+
to_remove: list[str] = None,
|
|
408
408
|
):
|
|
409
409
|
"""
|
|
410
410
|
Update the metrics dictionary of this model artifact.
|
|
@@ -423,8 +423,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
423
423
|
|
|
424
424
|
def set_extra_data(
|
|
425
425
|
self,
|
|
426
|
-
to_add:
|
|
427
|
-
to_remove:
|
|
426
|
+
to_add: dict[str, CommonTypes.ExtraDataType] = None,
|
|
427
|
+
to_remove: list[str] = None,
|
|
428
428
|
):
|
|
429
429
|
"""
|
|
430
430
|
Update the extra data dictionary of this model artifact.
|
|
@@ -442,7 +442,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
442
442
|
self._extra_data.pop(label)
|
|
443
443
|
|
|
444
444
|
def register_artifacts(
|
|
445
|
-
self, artifacts: Union[Artifact,
|
|
445
|
+
self, artifacts: Union[Artifact, list[Artifact], dict[str, Artifact]]
|
|
446
446
|
):
|
|
447
447
|
"""
|
|
448
448
|
Register the given artifacts, so they will be logged as extra data with the model of this handler. Notice: The
|
|
@@ -466,7 +466,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
466
466
|
@abstractmethod
|
|
467
467
|
def save(
|
|
468
468
|
self, output_path: CommonTypes.PathType = None, **kwargs
|
|
469
|
-
) -> Union[
|
|
469
|
+
) -> Union[dict[str, Artifact], None]:
|
|
470
470
|
"""
|
|
471
471
|
Save the handled model at the given output path.
|
|
472
472
|
|
|
@@ -525,13 +525,13 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
525
525
|
def log(
|
|
526
526
|
self,
|
|
527
527
|
tag: str = "",
|
|
528
|
-
labels:
|
|
529
|
-
parameters:
|
|
530
|
-
inputs:
|
|
531
|
-
outputs:
|
|
532
|
-
metrics:
|
|
533
|
-
artifacts:
|
|
534
|
-
extra_data:
|
|
528
|
+
labels: dict[str, Union[str, int, float]] = None,
|
|
529
|
+
parameters: dict[str, Union[str, int, float]] = None,
|
|
530
|
+
inputs: list[Feature] = None,
|
|
531
|
+
outputs: list[Feature] = None,
|
|
532
|
+
metrics: dict[str, Union[int, float]] = None,
|
|
533
|
+
artifacts: dict[str, Artifact] = None,
|
|
534
|
+
extra_data: dict[str, CommonTypes.ExtraDataType] = None,
|
|
535
535
|
**kwargs,
|
|
536
536
|
):
|
|
537
537
|
"""
|
|
@@ -630,13 +630,13 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
630
630
|
|
|
631
631
|
def update(
|
|
632
632
|
self,
|
|
633
|
-
labels:
|
|
634
|
-
parameters:
|
|
635
|
-
inputs:
|
|
636
|
-
outputs:
|
|
637
|
-
metrics:
|
|
638
|
-
artifacts:
|
|
639
|
-
extra_data:
|
|
633
|
+
labels: dict[str, Union[str, int, float]] = None,
|
|
634
|
+
parameters: dict[str, Union[str, int, float]] = None,
|
|
635
|
+
inputs: list[Feature] = None,
|
|
636
|
+
outputs: list[Feature] = None,
|
|
637
|
+
metrics: dict[str, Union[int, float]] = None,
|
|
638
|
+
artifacts: dict[str, Artifact] = None,
|
|
639
|
+
extra_data: dict[str, CommonTypes.ExtraDataType] = None,
|
|
640
640
|
**kwargs,
|
|
641
641
|
):
|
|
642
642
|
"""
|
|
@@ -857,7 +857,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
857
857
|
|
|
858
858
|
# Read the modules map if given as a json:
|
|
859
859
|
if isinstance(self._modules_map, str):
|
|
860
|
-
with open(self._modules_map
|
|
860
|
+
with open(self._modules_map) as map_json_file:
|
|
861
861
|
self._modules_map = json.loads(map_json_file.read())
|
|
862
862
|
|
|
863
863
|
# Start importing the modules according to the map:
|
|
@@ -887,7 +887,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
887
887
|
|
|
888
888
|
# Read the custom objects map if given as a json:
|
|
889
889
|
if isinstance(self._custom_objects_map, str):
|
|
890
|
-
with open(self._custom_objects_map
|
|
890
|
+
with open(self._custom_objects_map) as map_json_file:
|
|
891
891
|
self._custom_objects_map = json.loads(map_json_file.read())
|
|
892
892
|
|
|
893
893
|
# Unzip the custom objects files if the directory was given as a zip:
|
|
@@ -917,7 +917,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
917
917
|
),
|
|
918
918
|
}
|
|
919
919
|
|
|
920
|
-
def _log_modules(self) ->
|
|
920
|
+
def _log_modules(self) -> dict[str, Artifact]:
|
|
921
921
|
"""
|
|
922
922
|
Log the modules, returning the modules map json file logged as an artifact.
|
|
923
923
|
|
|
@@ -946,7 +946,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
946
946
|
|
|
947
947
|
return artifacts
|
|
948
948
|
|
|
949
|
-
def _log_custom_objects(self) ->
|
|
949
|
+
def _log_custom_objects(self) -> dict[str, Artifact]:
|
|
950
950
|
"""
|
|
951
951
|
Log the custom objects, returning their artifacts:
|
|
952
952
|
|
|
@@ -1002,8 +1002,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
1002
1002
|
|
|
1003
1003
|
def _read_io_samples(
|
|
1004
1004
|
self,
|
|
1005
|
-
samples: Union[CommonTypes.IOSampleType,
|
|
1006
|
-
) ->
|
|
1005
|
+
samples: Union[CommonTypes.IOSampleType, list[CommonTypes.IOSampleType]],
|
|
1006
|
+
) -> list[Feature]:
|
|
1007
1007
|
"""
|
|
1008
1008
|
Read the given inputs / output sample to / from the model into a list of MLRun Features (ports) to log in
|
|
1009
1009
|
the model's artifact.
|
|
@@ -1062,7 +1062,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
1062
1062
|
|
|
1063
1063
|
@staticmethod
|
|
1064
1064
|
def _validate_modules_parameter(
|
|
1065
|
-
modules_map: Union[
|
|
1065
|
+
modules_map: Union[dict[str, Union[None, str, list[str]]], str],
|
|
1066
1066
|
):
|
|
1067
1067
|
"""
|
|
1068
1068
|
Validate the given modules parameter.
|
|
@@ -1090,7 +1090,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
1090
1090
|
|
|
1091
1091
|
@staticmethod
|
|
1092
1092
|
def _validate_custom_objects_parameters(
|
|
1093
|
-
custom_objects_map: Union[
|
|
1093
|
+
custom_objects_map: Union[dict[str, Union[str, list[str]]], str],
|
|
1094
1094
|
custom_objects_directory: str,
|
|
1095
1095
|
):
|
|
1096
1096
|
"""
|
|
@@ -1154,8 +1154,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
1154
1154
|
|
|
1155
1155
|
@staticmethod
|
|
1156
1156
|
def _import_module(
|
|
1157
|
-
module_path: str, objects_names: Union[
|
|
1158
|
-
) ->
|
|
1157
|
+
module_path: str, objects_names: Union[list[str], None]
|
|
1158
|
+
) -> dict[str, Any]:
|
|
1159
1159
|
"""
|
|
1160
1160
|
Import the given objects by their names from the given module path by the following rules:
|
|
1161
1161
|
|
|
@@ -1199,8 +1199,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
1199
1199
|
|
|
1200
1200
|
@staticmethod
|
|
1201
1201
|
def _import_custom_object(
|
|
1202
|
-
py_file_path: str, objects_names:
|
|
1203
|
-
) ->
|
|
1202
|
+
py_file_path: str, objects_names: list[str]
|
|
1203
|
+
) -> dict[str, Any]:
|
|
1204
1204
|
"""
|
|
1205
1205
|
Import the given objects by their names from the given python file as: from 'py_file_path' import 'object_name'.
|
|
1206
1206
|
If an object specified is already imported, a reference would simply be returned.
|
|
@@ -1232,7 +1232,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
1232
1232
|
return objects_imports
|
|
1233
1233
|
|
|
1234
1234
|
|
|
1235
|
-
def with_mlrun_interface(interface:
|
|
1235
|
+
def with_mlrun_interface(interface: type[MLRunInterface]):
|
|
1236
1236
|
"""
|
|
1237
1237
|
Decorator configure for decorating a ModelHandler method (expecting 'self' to be the first argument) to add the
|
|
1238
1238
|
given MLRun interface into the model before executing the method and remove it afterwards.
|
|
@@ -1261,7 +1261,7 @@ def with_mlrun_interface(interface: Type[MLRunInterface]):
|
|
|
1261
1261
|
return decorator
|
|
1262
1262
|
|
|
1263
1263
|
|
|
1264
|
-
def without_mlrun_interface(interface:
|
|
1264
|
+
def without_mlrun_interface(interface: type[MLRunInterface]):
|
|
1265
1265
|
"""
|
|
1266
1266
|
Decorator configure for decorating a ModelHandler method (expecting 'self' to be the first argument) to remove the
|
|
1267
1267
|
given MLRun interface from the model before executing the method and restore it afterwards.
|