mlrun 1.10.0rc18__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +24 -3
- mlrun/__main__.py +0 -4
- mlrun/artifacts/dataset.py +2 -2
- mlrun/artifacts/document.py +6 -1
- mlrun/artifacts/llm_prompt.py +21 -15
- mlrun/artifacts/model.py +3 -3
- mlrun/artifacts/plots.py +1 -1
- mlrun/{model_monitoring/db/tsdb/tdengine → auth}/__init__.py +2 -3
- mlrun/auth/nuclio.py +89 -0
- mlrun/auth/providers.py +429 -0
- mlrun/auth/utils.py +415 -0
- mlrun/common/constants.py +14 -0
- mlrun/common/model_monitoring/helpers.py +123 -0
- mlrun/common/runtimes/constants.py +28 -0
- mlrun/common/schemas/__init__.py +14 -3
- mlrun/common/schemas/alert.py +2 -2
- mlrun/common/schemas/api_gateway.py +3 -0
- mlrun/common/schemas/auth.py +12 -10
- mlrun/common/schemas/client_spec.py +4 -0
- mlrun/common/schemas/constants.py +25 -0
- mlrun/common/schemas/frontend_spec.py +1 -8
- mlrun/common/schemas/function.py +34 -0
- mlrun/common/schemas/hub.py +33 -20
- mlrun/common/schemas/model_monitoring/__init__.py +2 -1
- mlrun/common/schemas/model_monitoring/constants.py +12 -15
- mlrun/common/schemas/model_monitoring/functions.py +13 -4
- mlrun/common/schemas/model_monitoring/model_endpoints.py +11 -0
- mlrun/common/schemas/pipeline.py +1 -1
- mlrun/common/schemas/secret.py +17 -2
- mlrun/common/secrets.py +95 -1
- mlrun/common/types.py +10 -10
- mlrun/config.py +69 -19
- mlrun/data_types/infer.py +2 -2
- mlrun/datastore/__init__.py +12 -5
- mlrun/datastore/azure_blob.py +162 -47
- mlrun/datastore/base.py +274 -10
- mlrun/datastore/datastore.py +7 -2
- mlrun/datastore/datastore_profile.py +84 -22
- mlrun/datastore/model_provider/huggingface_provider.py +225 -41
- mlrun/datastore/model_provider/mock_model_provider.py +87 -0
- mlrun/datastore/model_provider/model_provider.py +206 -74
- mlrun/datastore/model_provider/openai_provider.py +226 -66
- mlrun/datastore/s3.py +39 -18
- mlrun/datastore/sources.py +1 -1
- mlrun/datastore/store_resources.py +4 -4
- mlrun/datastore/storeytargets.py +17 -12
- mlrun/datastore/targets.py +1 -1
- mlrun/datastore/utils.py +25 -6
- mlrun/datastore/v3io.py +1 -1
- mlrun/db/base.py +63 -32
- mlrun/db/httpdb.py +373 -153
- mlrun/db/nopdb.py +54 -21
- mlrun/errors.py +4 -2
- mlrun/execution.py +66 -25
- mlrun/feature_store/api.py +1 -1
- mlrun/feature_store/common.py +1 -1
- mlrun/feature_store/feature_vector_utils.py +1 -1
- mlrun/feature_store/steps.py +8 -6
- mlrun/frameworks/_common/utils.py +3 -3
- mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +2 -1
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_ml_common/utils.py +2 -1
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +4 -3
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +2 -1
- mlrun/frameworks/onnx/dataset.py +2 -1
- mlrun/frameworks/onnx/mlrun_interface.py +2 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +5 -4
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +2 -1
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +2 -1
- mlrun/frameworks/pytorch/utils.py +2 -1
- mlrun/frameworks/sklearn/metric.py +2 -1
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +5 -4
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +2 -1
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +2 -1
- mlrun/hub/__init__.py +52 -0
- mlrun/hub/base.py +142 -0
- mlrun/hub/module.py +172 -0
- mlrun/hub/step.py +113 -0
- mlrun/k8s_utils.py +105 -16
- mlrun/launcher/base.py +15 -7
- mlrun/launcher/local.py +4 -1
- mlrun/model.py +14 -4
- mlrun/model_monitoring/__init__.py +0 -1
- mlrun/model_monitoring/api.py +65 -28
- mlrun/model_monitoring/applications/__init__.py +1 -1
- mlrun/model_monitoring/applications/base.py +299 -128
- mlrun/model_monitoring/applications/context.py +2 -4
- mlrun/model_monitoring/controller.py +132 -58
- mlrun/model_monitoring/db/_schedules.py +38 -29
- mlrun/model_monitoring/db/_stats.py +6 -16
- mlrun/model_monitoring/db/tsdb/__init__.py +9 -7
- mlrun/model_monitoring/db/tsdb/base.py +29 -9
- mlrun/model_monitoring/db/tsdb/preaggregate.py +234 -0
- mlrun/model_monitoring/db/tsdb/stream_graph_steps.py +63 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_metrics_queries.py +414 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_predictions_queries.py +376 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/queries/timescaledb_results_queries.py +590 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_connection.py +434 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_connector.py +541 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_operations.py +808 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_schema.py +502 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_stream.py +163 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/timescaledb_stream_graph_steps.py +60 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/utils/timescaledb_dataframe_processor.py +141 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/utils/timescaledb_query_builder.py +585 -0
- mlrun/model_monitoring/db/tsdb/timescaledb/writer_graph_steps.py +73 -0
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +20 -9
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +235 -51
- mlrun/model_monitoring/features_drift_table.py +2 -1
- mlrun/model_monitoring/helpers.py +30 -6
- mlrun/model_monitoring/stream_processing.py +34 -28
- mlrun/model_monitoring/writer.py +224 -4
- mlrun/package/__init__.py +2 -1
- mlrun/platforms/__init__.py +0 -43
- mlrun/platforms/iguazio.py +8 -4
- mlrun/projects/operations.py +17 -11
- mlrun/projects/pipelines.py +2 -2
- mlrun/projects/project.py +187 -123
- mlrun/run.py +95 -21
- mlrun/runtimes/__init__.py +2 -186
- mlrun/runtimes/base.py +103 -25
- mlrun/runtimes/constants.py +225 -0
- mlrun/runtimes/daskjob.py +5 -2
- mlrun/runtimes/databricks_job/databricks_runtime.py +2 -1
- mlrun/runtimes/local.py +5 -2
- mlrun/runtimes/mounts.py +20 -2
- mlrun/runtimes/nuclio/__init__.py +12 -7
- mlrun/runtimes/nuclio/api_gateway.py +36 -6
- mlrun/runtimes/nuclio/application/application.py +339 -40
- mlrun/runtimes/nuclio/function.py +222 -72
- mlrun/runtimes/nuclio/serving.py +132 -42
- mlrun/runtimes/pod.py +213 -21
- mlrun/runtimes/utils.py +49 -9
- mlrun/secrets.py +99 -14
- mlrun/serving/__init__.py +2 -0
- mlrun/serving/remote.py +84 -11
- mlrun/serving/routers.py +26 -44
- mlrun/serving/server.py +138 -51
- mlrun/serving/serving_wrapper.py +6 -2
- mlrun/serving/states.py +997 -283
- mlrun/serving/steps.py +62 -0
- mlrun/serving/system_steps.py +149 -95
- mlrun/serving/v2_serving.py +9 -10
- mlrun/track/trackers/mlflow_tracker.py +29 -31
- mlrun/utils/helpers.py +292 -94
- mlrun/utils/http.py +9 -2
- mlrun/utils/notifications/notification/base.py +18 -0
- mlrun/utils/notifications/notification/git.py +3 -5
- mlrun/utils/notifications/notification/mail.py +39 -16
- mlrun/utils/notifications/notification/slack.py +2 -4
- mlrun/utils/notifications/notification/webhook.py +2 -5
- mlrun/utils/notifications/notification_pusher.py +3 -3
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +3 -4
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/METADATA +63 -74
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/RECORD +161 -143
- mlrun/api/schemas/__init__.py +0 -259
- mlrun/db/auth_utils.py +0 -152
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +0 -344
- mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +0 -75
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +0 -281
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +0 -1266
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/WHEEL +0 -0
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/entry_points.txt +0 -0
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/licenses/LICENSE +0 -0
- {mlrun-1.10.0rc18.dist-info → mlrun-1.11.0rc16.dist-info}/top_level.txt +0 -0
mlrun/db/nopdb.py
CHANGED
|
@@ -212,7 +212,6 @@ class NopDB(RunDBInterface):
|
|
|
212
212
|
tree: Optional[str] = None,
|
|
213
213
|
parent: Optional[str] = None,
|
|
214
214
|
format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
|
|
215
|
-
limit: Optional[int] = None,
|
|
216
215
|
partition_by: Optional[
|
|
217
216
|
Union[mlrun.common.schemas.ArtifactPartitionByField, str]
|
|
218
217
|
] = None,
|
|
@@ -376,16 +375,6 @@ class NopDB(RunDBInterface):
|
|
|
376
375
|
) -> dict:
|
|
377
376
|
pass
|
|
378
377
|
|
|
379
|
-
def list_features(
|
|
380
|
-
self,
|
|
381
|
-
project: str,
|
|
382
|
-
name: Optional[str] = None,
|
|
383
|
-
tag: Optional[str] = None,
|
|
384
|
-
entities: Optional[list[str]] = None,
|
|
385
|
-
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
386
|
-
) -> mlrun.common.schemas.FeaturesOutput:
|
|
387
|
-
pass
|
|
388
|
-
|
|
389
378
|
def list_features_v2(
|
|
390
379
|
self,
|
|
391
380
|
project: str,
|
|
@@ -593,16 +582,6 @@ class NopDB(RunDBInterface):
|
|
|
593
582
|
):
|
|
594
583
|
pass
|
|
595
584
|
|
|
596
|
-
def create_user_secrets(
|
|
597
|
-
self,
|
|
598
|
-
user: str,
|
|
599
|
-
provider: Union[
|
|
600
|
-
str, mlrun.common.schemas.SecretProviderName
|
|
601
|
-
] = mlrun.common.schemas.SecretProviderName.vault,
|
|
602
|
-
secrets: Optional[dict] = None,
|
|
603
|
-
):
|
|
604
|
-
pass
|
|
605
|
-
|
|
606
585
|
def create_model_endpoint(
|
|
607
586
|
self,
|
|
608
587
|
model_endpoint: mlrun.common.schemas.ModelEndpoint,
|
|
@@ -636,6 +615,9 @@ class NopDB(RunDBInterface):
|
|
|
636
615
|
tsdb_metrics: bool = False,
|
|
637
616
|
metric_list: Optional[list[str]] = None,
|
|
638
617
|
top_level: bool = False,
|
|
618
|
+
modes: Optional[
|
|
619
|
+
Union[mm_constants.EndpointMode, list[mm_constants.EndpointMode]]
|
|
620
|
+
] = None,
|
|
639
621
|
uids: Optional[list[str]] = None,
|
|
640
622
|
latest_only: bool = False,
|
|
641
623
|
) -> mlrun.common.schemas.ModelEndpointList:
|
|
@@ -682,6 +664,7 @@ class NopDB(RunDBInterface):
|
|
|
682
664
|
item_name: Optional[str] = None,
|
|
683
665
|
tag: Optional[str] = None,
|
|
684
666
|
version: Optional[str] = None,
|
|
667
|
+
item_type: mlrun.common.schemas.hub.HubSourceType = mlrun.common.schemas.hub.HubSourceType.functions,
|
|
685
668
|
):
|
|
686
669
|
pass
|
|
687
670
|
|
|
@@ -698,6 +681,7 @@ class NopDB(RunDBInterface):
|
|
|
698
681
|
version: Optional[str] = None,
|
|
699
682
|
tag: Optional[str] = None,
|
|
700
683
|
force_refresh: bool = False,
|
|
684
|
+
object_type: mlrun.common.schemas.hub.HubSourceType = mlrun.common.schemas.hub.HubSourceType.functions,
|
|
701
685
|
):
|
|
702
686
|
pass
|
|
703
687
|
|
|
@@ -709,6 +693,18 @@ class NopDB(RunDBInterface):
|
|
|
709
693
|
version: Optional[str] = None,
|
|
710
694
|
tag: str = "latest",
|
|
711
695
|
force_refresh: bool = False,
|
|
696
|
+
item_type: mlrun.common.schemas.hub.HubSourceType = mlrun.common.schemas.hub.HubSourceType.functions,
|
|
697
|
+
):
|
|
698
|
+
pass
|
|
699
|
+
|
|
700
|
+
def get_hub_asset(
|
|
701
|
+
self,
|
|
702
|
+
source_name: str,
|
|
703
|
+
item_name: str,
|
|
704
|
+
asset_name: str,
|
|
705
|
+
version: Optional[str] = None,
|
|
706
|
+
tag: str = "latest",
|
|
707
|
+
item_type: mlrun.common.schemas.hub.HubSourceType = mlrun.common.schemas.hub.HubSourceType.functions,
|
|
712
708
|
):
|
|
713
709
|
pass
|
|
714
710
|
|
|
@@ -895,6 +891,14 @@ class NopDB(RunDBInterface):
|
|
|
895
891
|
) -> None:
|
|
896
892
|
pass
|
|
897
893
|
|
|
894
|
+
def delete_model_monitoring_metrics(
|
|
895
|
+
self,
|
|
896
|
+
project: str,
|
|
897
|
+
application_name: str,
|
|
898
|
+
endpoint_ids: Optional[list[str]] = None,
|
|
899
|
+
) -> None:
|
|
900
|
+
pass
|
|
901
|
+
|
|
898
902
|
def get_monitoring_function_summaries(
|
|
899
903
|
self,
|
|
900
904
|
project: str,
|
|
@@ -993,3 +997,32 @@ class NopDB(RunDBInterface):
|
|
|
993
997
|
end: Optional[datetime.datetime] = None,
|
|
994
998
|
) -> mlrun.common.schemas.model_monitoring.ModelEndpointDriftValues:
|
|
995
999
|
pass
|
|
1000
|
+
|
|
1001
|
+
def store_secret_token(
|
|
1002
|
+
self,
|
|
1003
|
+
secret_token: mlrun.common.schemas.SecretToken,
|
|
1004
|
+
log_warning: bool = True,
|
|
1005
|
+
force: bool = False,
|
|
1006
|
+
) -> mlrun.common.schemas.StoreSecretTokensResponse:
|
|
1007
|
+
pass
|
|
1008
|
+
|
|
1009
|
+
def store_secret_tokens(
|
|
1010
|
+
self,
|
|
1011
|
+
secret_tokens: list[mlrun.common.schemas.SecretToken],
|
|
1012
|
+
log_warning: bool = True,
|
|
1013
|
+
force: bool = False,
|
|
1014
|
+
) -> mlrun.common.schemas.StoreSecretTokensResponse:
|
|
1015
|
+
pass
|
|
1016
|
+
|
|
1017
|
+
def list_secret_tokens(self) -> mlrun.common.schemas.ListSecretTokensResponse:
|
|
1018
|
+
pass
|
|
1019
|
+
|
|
1020
|
+
def revoke_secret_token(self, token_name: str) -> None:
|
|
1021
|
+
pass
|
|
1022
|
+
|
|
1023
|
+
def get_secret_token(
|
|
1024
|
+
self,
|
|
1025
|
+
token_name: str,
|
|
1026
|
+
username: Optional[str] = None,
|
|
1027
|
+
) -> mlrun.common.schemas.SecretToken:
|
|
1028
|
+
pass
|
mlrun/errors.py
CHANGED
|
@@ -264,12 +264,14 @@ class MLRunFatalFailureError(Exception):
|
|
|
264
264
|
|
|
265
265
|
|
|
266
266
|
class ModelRunnerError(MLRunBaseError):
|
|
267
|
-
def __init__(self, models_errors: dict[str
|
|
267
|
+
def __init__(self, models_errors: dict[str, str], *args) -> None:
|
|
268
268
|
self.models_errors = models_errors
|
|
269
269
|
super().__init__(self.__repr__(), *args)
|
|
270
270
|
|
|
271
271
|
def __repr__(self):
|
|
272
|
-
return
|
|
272
|
+
return "ModelRunnerError: " + ";\n".join(
|
|
273
|
+
f"{model} {msg}" for model, msg in self.models_errors.items()
|
|
274
|
+
)
|
|
273
275
|
|
|
274
276
|
def __copy__(self):
|
|
275
277
|
return type(self)(models_errors=self.models_errors)
|
mlrun/execution.py
CHANGED
|
@@ -15,7 +15,6 @@
|
|
|
15
15
|
import logging
|
|
16
16
|
import os
|
|
17
17
|
import uuid
|
|
18
|
-
import warnings
|
|
19
18
|
from copy import deepcopy
|
|
20
19
|
from typing import Optional, Union, cast
|
|
21
20
|
|
|
@@ -101,6 +100,7 @@ class MLClientCtx:
|
|
|
101
100
|
self._node_selector = {}
|
|
102
101
|
self._tolerations = {}
|
|
103
102
|
self._affinity = {}
|
|
103
|
+
self._auth = {}
|
|
104
104
|
|
|
105
105
|
self._function = ""
|
|
106
106
|
self._parameters = {}
|
|
@@ -234,6 +234,11 @@ class MLClientCtx:
|
|
|
234
234
|
"""Dictionary with node selectors (read-only)"""
|
|
235
235
|
return deepcopy(self._node_selector)
|
|
236
236
|
|
|
237
|
+
@property
|
|
238
|
+
def auth(self):
|
|
239
|
+
"""Dictionary with auth (read-only)"""
|
|
240
|
+
return deepcopy(self._auth)
|
|
241
|
+
|
|
237
242
|
@property
|
|
238
243
|
def tolerations(self):
|
|
239
244
|
"""Dictionary with tolerations (read-only)"""
|
|
@@ -437,6 +442,7 @@ class MLClientCtx:
|
|
|
437
442
|
self._affinity = spec.get("affinity", self._affinity)
|
|
438
443
|
self._reset_on_run = spec.get("reset_on_run", self._reset_on_run)
|
|
439
444
|
self._retry_spec = spec.get("retry", self._retry_spec)
|
|
445
|
+
self._auth = spec.get("auth", self._auth)
|
|
440
446
|
|
|
441
447
|
self._init_dbs(rundb)
|
|
442
448
|
|
|
@@ -917,7 +923,7 @@ class MLClientCtx:
|
|
|
917
923
|
prompt_path: Optional[str] = None,
|
|
918
924
|
prompt_legend: Optional[dict] = None,
|
|
919
925
|
model_artifact: Union[ModelArtifact, str] = None,
|
|
920
|
-
|
|
926
|
+
invocation_config: Optional[dict] = None,
|
|
921
927
|
description: Optional[str] = None,
|
|
922
928
|
target_path: Optional[str] = None,
|
|
923
929
|
artifact_path: Optional[str] = None,
|
|
@@ -934,14 +940,51 @@ class MLClientCtx:
|
|
|
934
940
|
|
|
935
941
|
Examples::
|
|
936
942
|
|
|
937
|
-
# Log an inline prompt
|
|
943
|
+
# Log directly with an inline prompt template
|
|
944
|
+
context.log_llm_prompt(
|
|
945
|
+
key="customer_support_prompt",
|
|
946
|
+
prompt_template=[
|
|
947
|
+
{
|
|
948
|
+
"role": "system",
|
|
949
|
+
"content": "You are a helpful customer support assistant.",
|
|
950
|
+
},
|
|
951
|
+
{
|
|
952
|
+
"role": "user",
|
|
953
|
+
"content": "The customer reports: {issue_description}",
|
|
954
|
+
},
|
|
955
|
+
],
|
|
956
|
+
prompt_legend={
|
|
957
|
+
"issue_description": {
|
|
958
|
+
"field": "user_issue",
|
|
959
|
+
"description": "Detailed description of the customer's issue",
|
|
960
|
+
},
|
|
961
|
+
"solution": {
|
|
962
|
+
"field": "proposed_solution",
|
|
963
|
+
"description": "Suggested fix for the customer's issue",
|
|
964
|
+
},
|
|
965
|
+
},
|
|
966
|
+
model_artifact=model,
|
|
967
|
+
invocation_config={"temperature": 0.5, "max_tokens": 200},
|
|
968
|
+
description="Prompt for handling customer support queries",
|
|
969
|
+
tag="support-v1",
|
|
970
|
+
labels={"domain": "support"},
|
|
971
|
+
)
|
|
972
|
+
|
|
973
|
+
# Log a prompt from file
|
|
938
974
|
context.log_llm_prompt(
|
|
939
|
-
key="
|
|
940
|
-
|
|
975
|
+
key="qa_prompt",
|
|
976
|
+
prompt_path="prompts/template.json",
|
|
977
|
+
prompt_legend={
|
|
978
|
+
"question": {
|
|
979
|
+
"field": "user_question",
|
|
980
|
+
"description": "The actual question asked by the user",
|
|
981
|
+
}
|
|
982
|
+
},
|
|
941
983
|
model_artifact=model,
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
tag="
|
|
984
|
+
invocation_config={"temperature": 0.7, "max_tokens": 256},
|
|
985
|
+
description="Q&A prompt template with user-provided question",
|
|
986
|
+
tag="v2",
|
|
987
|
+
labels={"task": "qa", "stage": "experiment"},
|
|
945
988
|
)
|
|
946
989
|
|
|
947
990
|
:param key: Unique name of the artifact.
|
|
@@ -950,17 +993,22 @@ class MLClientCtx:
|
|
|
950
993
|
"role": "user", "content": "I need your help with {profession}"]. only "role" and "content" keys allow in any
|
|
951
994
|
str format (upper/lower case), keys will be modified to lower case.
|
|
952
995
|
Cannot be used with `prompt_path`.
|
|
953
|
-
:param prompt_path: Path to a file containing the prompt
|
|
996
|
+
:param prompt_path: Path to a JSON file containing the prompt template.
|
|
997
|
+
Cannot be used together with `prompt_template`.
|
|
998
|
+
The file should define a list of dictionaries in the same format
|
|
999
|
+
supported by `prompt_template`.
|
|
954
1000
|
:param prompt_legend: A dictionary where each key is a placeholder in the prompt (e.g., ``{user_name}``)
|
|
955
1001
|
and the value is a dictionary holding two keys, "field", "description". "field" points to the field in
|
|
956
1002
|
the event where the value of the place-holder inside the event, if None or not exist will be replaced
|
|
957
1003
|
with the place-holder name. "description" will point to explanation of what that placeholder represents.
|
|
958
1004
|
Useful for documenting and clarifying dynamic parts of the prompt.
|
|
959
1005
|
:param model_artifact: Reference to the parent model (either `ModelArtifact` or model URI string).
|
|
960
|
-
:param
|
|
961
|
-
:param description:
|
|
962
|
-
:param target_path:
|
|
963
|
-
:param artifact_path:
|
|
1006
|
+
:param invocation_config: Dictionary of generation parameters (e.g., temperature, max_tokens).
|
|
1007
|
+
:param description: Optional description of the prompt.
|
|
1008
|
+
:param target_path: Absolute target path (instead of using artifact_path + local_path)
|
|
1009
|
+
:param artifact_path: Target artifact path (when not using the default)
|
|
1010
|
+
To define a subpath under the default location use:
|
|
1011
|
+
`artifact_path=context.artifact_subpath('data')`
|
|
964
1012
|
:param tag: Tag/version to assign to the prompt artifact.
|
|
965
1013
|
:param labels: Labels to tag the artifact (e.g., list or dict of key-value pairs).
|
|
966
1014
|
:param upload: Whether to upload the artifact to the store (defaults to True).
|
|
@@ -981,7 +1029,7 @@ class MLClientCtx:
|
|
|
981
1029
|
prompt_path=prompt_path,
|
|
982
1030
|
prompt_legend=prompt_legend,
|
|
983
1031
|
model_artifact=model_artifact,
|
|
984
|
-
|
|
1032
|
+
invocation_config=invocation_config,
|
|
985
1033
|
target_path=target_path,
|
|
986
1034
|
description=description,
|
|
987
1035
|
**kwargs,
|
|
@@ -1099,14 +1147,6 @@ class MLClientCtx:
|
|
|
1099
1147
|
self._update_run()
|
|
1100
1148
|
return item
|
|
1101
1149
|
|
|
1102
|
-
def get_cached_artifact(self, key):
|
|
1103
|
-
"""Return a logged artifact from cache (for potential updates)"""
|
|
1104
|
-
warnings.warn(
|
|
1105
|
-
"get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.11.0. Use get_artifact instead.",
|
|
1106
|
-
FutureWarning,
|
|
1107
|
-
)
|
|
1108
|
-
return self.get_artifact(key)
|
|
1109
|
-
|
|
1110
1150
|
def get_artifact(
|
|
1111
1151
|
self, key, tag=None, iter=None, tree=None, uid=None
|
|
1112
1152
|
) -> Optional[Artifact]:
|
|
@@ -1268,6 +1308,7 @@ class MLClientCtx:
|
|
|
1268
1308
|
"node_selector": self._node_selector,
|
|
1269
1309
|
"tolerations": self._tolerations,
|
|
1270
1310
|
"affinity": self._affinity,
|
|
1311
|
+
"auth": self._auth,
|
|
1271
1312
|
"retry": self._retry_spec,
|
|
1272
1313
|
},
|
|
1273
1314
|
"status": {
|
|
@@ -1469,15 +1510,15 @@ class MLClientCtx:
|
|
|
1469
1510
|
|
|
1470
1511
|
|
|
1471
1512
|
def _cast_result(value):
|
|
1472
|
-
if isinstance(value,
|
|
1513
|
+
if isinstance(value, int | str | float):
|
|
1473
1514
|
return value
|
|
1474
1515
|
if isinstance(value, list):
|
|
1475
1516
|
return [_cast_result(v) for v in value]
|
|
1476
1517
|
if isinstance(value, dict):
|
|
1477
1518
|
return {k: _cast_result(v) for k, v in value.items()}
|
|
1478
|
-
if isinstance(value,
|
|
1519
|
+
if isinstance(value, np.int64 | np.integer):
|
|
1479
1520
|
return int(value)
|
|
1480
|
-
if isinstance(value,
|
|
1521
|
+
if isinstance(value, np.floating | np.float64):
|
|
1481
1522
|
return float(value)
|
|
1482
1523
|
if isinstance(value, np.ndarray):
|
|
1483
1524
|
return value.tolist()
|
mlrun/feature_store/api.py
CHANGED
|
@@ -361,7 +361,7 @@ def _ingest(
|
|
|
361
361
|
import pyspark.sql
|
|
362
362
|
|
|
363
363
|
if (
|
|
364
|
-
isinstance(source,
|
|
364
|
+
isinstance(source, pd.DataFrame | pyspark.sql.DataFrame)
|
|
365
365
|
and run_config is not None
|
|
366
366
|
):
|
|
367
367
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
mlrun/feature_store/common.py
CHANGED
|
@@ -256,7 +256,7 @@ class RunConfig:
|
|
|
256
256
|
@function.setter
|
|
257
257
|
def function(self, function):
|
|
258
258
|
if function and not (
|
|
259
|
-
isinstance(function,
|
|
259
|
+
isinstance(function, str | FunctionReference) or hasattr(function, "apply")
|
|
260
260
|
):
|
|
261
261
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
262
262
|
"function must be a uri (string) or mlrun function object/reference"
|
|
@@ -370,7 +370,7 @@ class OnlineVectorService:
|
|
|
370
370
|
if (
|
|
371
371
|
not entity_rows
|
|
372
372
|
or not isinstance(entity_rows, list)
|
|
373
|
-
or not isinstance(entity_rows[0],
|
|
373
|
+
or not isinstance(entity_rows[0], list | dict)
|
|
374
374
|
):
|
|
375
375
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
376
376
|
f"input data is of type {type(entity_rows)}. must be a list of lists or list of dicts"
|
mlrun/feature_store/steps.py
CHANGED
|
@@ -279,11 +279,11 @@ class MapValues(StepToDict, MLRunStep):
|
|
|
279
279
|
new_col_type = df.schema[new_column_name].dataType
|
|
280
280
|
# in order to avoid exception at isna on non-decimal/float columns -
|
|
281
281
|
# we need to check their types before filtering.
|
|
282
|
-
if isinstance(col_type,
|
|
282
|
+
if isinstance(col_type, FloatType | DoubleType | DecimalType):
|
|
283
283
|
column_filter = (~isnull(col(column))) & (~isnan(col(column)))
|
|
284
284
|
else:
|
|
285
285
|
column_filter = ~isnull(col(column))
|
|
286
|
-
if isinstance(new_col_type,
|
|
286
|
+
if isinstance(new_col_type, FloatType | DoubleType | DecimalType):
|
|
287
287
|
new_column_filter = isnull(col(new_column_name)) | isnan(
|
|
288
288
|
col(new_column_name)
|
|
289
289
|
)
|
|
@@ -295,7 +295,7 @@ class MapValues(StepToDict, MLRunStep):
|
|
|
295
295
|
for k, v in column_map.items()
|
|
296
296
|
if v is None
|
|
297
297
|
or (
|
|
298
|
-
isinstance(v,
|
|
298
|
+
isinstance(v, float | np.float64 | np.float32 | np.float16)
|
|
299
299
|
and math.isnan(v)
|
|
300
300
|
)
|
|
301
301
|
]
|
|
@@ -338,7 +338,7 @@ class MapValues(StepToDict, MLRunStep):
|
|
|
338
338
|
for val in column_map.values()
|
|
339
339
|
if type(val) is not None
|
|
340
340
|
and not (
|
|
341
|
-
isinstance(val,
|
|
341
|
+
isinstance(val, float | np.float64 | np.float32 | np.float16)
|
|
342
342
|
and math.isnan(val)
|
|
343
343
|
)
|
|
344
344
|
)
|
|
@@ -358,7 +358,9 @@ class MapValues(StepToDict, MLRunStep):
|
|
|
358
358
|
and val != "-inf"
|
|
359
359
|
and val != "inf"
|
|
360
360
|
and not (
|
|
361
|
-
isinstance(
|
|
361
|
+
isinstance(
|
|
362
|
+
val, float | np.float64 | np.float32 | np.float16
|
|
363
|
+
)
|
|
362
364
|
and math.isnan(val)
|
|
363
365
|
)
|
|
364
366
|
)
|
|
@@ -443,7 +445,7 @@ class OneHotEncoder(StepToDict, MLRunStep):
|
|
|
443
445
|
self.mapping = mapping
|
|
444
446
|
for key, values in mapping.items():
|
|
445
447
|
for val in values:
|
|
446
|
-
if not (isinstance(val, str) or isinstance(val,
|
|
448
|
+
if not (isinstance(val, str) or isinstance(val, int | np.integer)):
|
|
447
449
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
448
450
|
"For OneHotEncoder you must provide int or string mapping list"
|
|
449
451
|
)
|
|
@@ -97,9 +97,9 @@ class CommonUtils(ABC):
|
|
|
97
97
|
"""
|
|
98
98
|
if isinstance(dataset, np.ndarray):
|
|
99
99
|
return dataset
|
|
100
|
-
if isinstance(dataset,
|
|
100
|
+
if isinstance(dataset, pd.DataFrame | pd.Series):
|
|
101
101
|
return dataset.to_numpy()
|
|
102
|
-
if isinstance(dataset,
|
|
102
|
+
if isinstance(dataset, list | tuple):
|
|
103
103
|
return np.array(dataset)
|
|
104
104
|
if isinstance(dataset, dict):
|
|
105
105
|
return np.array(list(dataset.values()))
|
|
@@ -133,7 +133,7 @@ class CommonUtils(ABC):
|
|
|
133
133
|
"""
|
|
134
134
|
if isinstance(dataset, pd.DataFrame):
|
|
135
135
|
return dataset
|
|
136
|
-
if isinstance(dataset,
|
|
136
|
+
if isinstance(dataset, np.ndarray | pd.Series | list | tuple | dict):
|
|
137
137
|
return pd.DataFrame(dataset)
|
|
138
138
|
try:
|
|
139
139
|
# SciPy is not in MLRun's requirements but common to all frameworks.
|
|
@@ -281,7 +281,7 @@ class Logger:
|
|
|
281
281
|
"""
|
|
282
282
|
for parameter_name, parameter_value in self._context.parameters.items():
|
|
283
283
|
# Check if the parameter is a trackable value:
|
|
284
|
-
if isinstance(parameter_value,
|
|
284
|
+
if isinstance(parameter_value, str | bool | float | int):
|
|
285
285
|
self.log_static_hyperparameter(
|
|
286
286
|
parameter_name=parameter_name, value=parameter_value
|
|
287
287
|
)
|
|
@@ -14,8 +14,9 @@
|
|
|
14
14
|
|
|
15
15
|
import os
|
|
16
16
|
from abc import abstractmethod
|
|
17
|
+
from collections.abc import Callable
|
|
17
18
|
from datetime import datetime
|
|
18
|
-
from typing import Any,
|
|
19
|
+
from typing import Any, Generic, Optional, Union
|
|
19
20
|
|
|
20
21
|
import yaml
|
|
21
22
|
|
|
@@ -73,7 +73,7 @@ class MLRunLogger(Logger):
|
|
|
73
73
|
"""
|
|
74
74
|
for parameter_name, parameter_value in self._context.parameters.items():
|
|
75
75
|
# Check if the parameter is a trackable value:
|
|
76
|
-
if isinstance(parameter_value,
|
|
76
|
+
if isinstance(parameter_value, str | bool | float | int):
|
|
77
77
|
self.log_static_hyperparameter(
|
|
78
78
|
parameter_name=parameter_name, value=parameter_value
|
|
79
79
|
)
|
|
@@ -13,8 +13,9 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
from abc import ABC
|
|
16
|
+
from collections.abc import Callable
|
|
16
17
|
from enum import Enum
|
|
17
|
-
from typing import
|
|
18
|
+
from typing import Union
|
|
18
19
|
|
|
19
20
|
import pandas as pd
|
|
20
21
|
from sklearn.base import is_classifier, is_regressor
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from typing import Optional, Union
|
|
16
17
|
|
|
17
18
|
import mlrun
|
|
18
19
|
from mlrun.artifacts import get_model
|
|
@@ -58,7 +59,7 @@ def get_framework_by_instance(model: CommonTypes.ModelType) -> str:
|
|
|
58
59
|
|
|
59
60
|
from mlrun.frameworks.xgboost import XGBoostModelHandler
|
|
60
61
|
|
|
61
|
-
if isinstance(model,
|
|
62
|
+
if isinstance(model, XGBModel | Booster):
|
|
62
63
|
return XGBoostModelHandler.FRAMEWORK_NAME
|
|
63
64
|
except ModuleNotFoundError:
|
|
64
65
|
pass
|
|
@@ -69,7 +70,7 @@ def get_framework_by_instance(model: CommonTypes.ModelType) -> str:
|
|
|
69
70
|
|
|
70
71
|
from mlrun.frameworks.lgbm import LGBMModelHandler
|
|
71
72
|
|
|
72
|
-
if isinstance(model,
|
|
73
|
+
if isinstance(model, LGBMModel | Booster):
|
|
73
74
|
return LGBMModelHandler.FRAMEWORK_NAME
|
|
74
75
|
except ModuleNotFoundError:
|
|
75
76
|
pass
|
mlrun/frameworks/onnx/dataset.py
CHANGED
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from typing import Optional
|
|
16
17
|
|
|
17
18
|
import numpy as np
|
|
18
19
|
import onnx
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from typing import Optional, Union
|
|
16
17
|
|
|
17
18
|
import numpy as np
|
|
18
19
|
from torch import Tensor
|
|
@@ -496,7 +497,7 @@ class LoggingCallback(Callback):
|
|
|
496
497
|
value = self._objects[source]
|
|
497
498
|
for key in key_chain:
|
|
498
499
|
try:
|
|
499
|
-
if isinstance(value,
|
|
500
|
+
if isinstance(value, dict | list | tuple):
|
|
500
501
|
value = value[key]
|
|
501
502
|
else:
|
|
502
503
|
value = getattr(value, key)
|
|
@@ -507,7 +508,7 @@ class LoggingCallback(Callback):
|
|
|
507
508
|
)
|
|
508
509
|
|
|
509
510
|
# Parse the value:
|
|
510
|
-
if isinstance(value,
|
|
511
|
+
if isinstance(value, Tensor | Parameter):
|
|
511
512
|
if value.numel() == 1:
|
|
512
513
|
value = float(value)
|
|
513
514
|
else:
|
|
@@ -523,7 +524,7 @@ class LoggingCallback(Callback):
|
|
|
523
524
|
f"The parameter with the following key chain: {key_chain} is a numpy.ndarray with {value.size} "
|
|
524
525
|
f"elements. numpy arrays are trackable only if they have 1 element."
|
|
525
526
|
)
|
|
526
|
-
elif not isinstance(value,
|
|
527
|
+
elif not isinstance(value, float | int | str | bool):
|
|
527
528
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
528
529
|
f"The parameter with the following key chain: {key_chain} is of type '{type(value)}'. "
|
|
529
530
|
f"The only trackable types are: float, int, str and bool."
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from typing import Optional, Union
|
|
16
17
|
|
|
17
18
|
import torch
|
|
18
19
|
from torch import Tensor
|
|
@@ -12,8 +12,9 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
from collections.abc import Callable
|
|
15
16
|
from datetime import datetime
|
|
16
|
-
from typing import
|
|
17
|
+
from typing import Optional, Union
|
|
17
18
|
|
|
18
19
|
import torch
|
|
19
20
|
from torch import Tensor
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from typing import Union
|
|
16
17
|
|
|
17
18
|
import numpy as np
|
|
18
19
|
import torch
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from typing import Optional, Union
|
|
16
17
|
|
|
17
18
|
import numpy as np
|
|
18
19
|
import tensorflow as tf
|
|
@@ -441,8 +442,8 @@ class LoggingCallback(Callback):
|
|
|
441
442
|
)
|
|
442
443
|
|
|
443
444
|
# Parse the value:
|
|
444
|
-
if isinstance(value,
|
|
445
|
-
is_keras_3() and isinstance(value,
|
|
445
|
+
if isinstance(value, tf.Tensor | tf.Variable) or (
|
|
446
|
+
is_keras_3() and isinstance(value, keras.KerasTensor | keras.Variable)
|
|
446
447
|
):
|
|
447
448
|
if int(tf.size(value)) == 1:
|
|
448
449
|
value = float(value)
|
|
@@ -459,7 +460,7 @@ class LoggingCallback(Callback):
|
|
|
459
460
|
f"The parameter with the following key chain: {key_chain} is a numpy.ndarray with {value.size} "
|
|
460
461
|
f"elements. numpy arrays are trackable only if they have 1 element."
|
|
461
462
|
)
|
|
462
|
-
elif not (isinstance(value,
|
|
463
|
+
elif not (isinstance(value, float | int | str | bool)):
|
|
463
464
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
464
465
|
f"The parameter with the following key chain: {key_chain} is of type '{type(value)}'. The only "
|
|
465
466
|
f"trackable types are: float, int, str and bool."
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from typing import Optional, Union
|
|
16
17
|
|
|
17
18
|
import mlrun
|
|
18
19
|
from mlrun.artifacts import Artifact
|