mlrun 1.9.0rc3__py3-none-any.whl → 1.10.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/api/schemas/__init__.py +0 -1
- mlrun/common/__init__.py +0 -1
- mlrun/common/db/__init__.py +0 -1
- mlrun/common/db/sql_session.py +0 -1
- mlrun/common/formatters/__init__.py +0 -1
- mlrun/common/formatters/artifact.py +0 -1
- mlrun/common/formatters/base.py +0 -1
- mlrun/common/formatters/feature_set.py +0 -1
- mlrun/common/formatters/function.py +0 -1
- mlrun/common/formatters/model_endpoint.py +0 -1
- mlrun/common/formatters/pipeline.py +0 -1
- mlrun/common/formatters/project.py +0 -1
- mlrun/common/formatters/run.py +0 -2
- mlrun/common/runtimes/constants.py +1 -1
- mlrun/common/schemas/__init__.py +1 -0
- mlrun/common/schemas/alert.py +1 -1
- mlrun/common/schemas/api_gateway.py +1 -1
- mlrun/common/schemas/artifact.py +1 -1
- mlrun/common/schemas/auth.py +1 -1
- mlrun/common/schemas/background_task.py +1 -1
- mlrun/common/schemas/client_spec.py +1 -1
- mlrun/common/schemas/clusterization_spec.py +1 -1
- mlrun/common/schemas/constants.py +1 -1
- mlrun/common/schemas/datastore_profile.py +0 -1
- mlrun/common/schemas/events.py +1 -1
- mlrun/common/schemas/feature_store.py +1 -1
- mlrun/common/schemas/frontend_spec.py +1 -1
- mlrun/common/schemas/function.py +1 -1
- mlrun/common/schemas/http.py +1 -1
- mlrun/common/schemas/hub.py +1 -1
- mlrun/common/schemas/k8s.py +1 -1
- mlrun/common/schemas/memory_reports.py +0 -1
- mlrun/common/schemas/model_monitoring/model_endpoints.py +32 -8
- mlrun/common/schemas/notification.py +4 -0
- mlrun/common/schemas/object.py +1 -1
- mlrun/common/schemas/partition.py +1 -1
- mlrun/common/schemas/pipeline.py +1 -1
- mlrun/common/schemas/project.py +1 -1
- mlrun/common/schemas/regex.py +1 -1
- mlrun/common/schemas/runtime_resource.py +1 -1
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/schemas/secret.py +1 -1
- mlrun/common/schemas/tag.py +0 -1
- mlrun/common/schemas/workflow.py +1 -1
- mlrun/common/secrets.py +0 -1
- mlrun/config.py +9 -17
- mlrun/data_types/infer.py +1 -1
- mlrun/data_types/spark.py +1 -1
- mlrun/datastore/datastore.py +1 -1
- mlrun/datastore/snowflake_utils.py +0 -1
- mlrun/datastore/spark_utils.py +0 -1
- mlrun/datastore/utils.py +1 -1
- mlrun/db/base.py +2 -0
- mlrun/db/httpdb.py +29 -19
- mlrun/db/nopdb.py +2 -1
- mlrun/errors.py +1 -1
- mlrun/execution.py +21 -9
- mlrun/feature_store/feature_set.py +0 -12
- mlrun/feature_store/retrieval/base.py +1 -1
- mlrun/feature_store/retrieval/dask_merger.py +1 -1
- mlrun/feature_store/retrieval/job.py +1 -1
- mlrun/feature_store/retrieval/spark_merger.py +0 -2
- mlrun/feature_store/steps.py +1 -1
- mlrun/features.py +1 -1
- mlrun/frameworks/_common/artifacts_library.py +1 -1
- mlrun/frameworks/_common/mlrun_interface.py +1 -1
- mlrun/frameworks/_common/model_handler.py +3 -3
- mlrun/frameworks/_common/producer.py +0 -1
- mlrun/frameworks/_common/utils.py +1 -1
- mlrun/frameworks/_dl_common/loggers/logger.py +0 -1
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
- mlrun/frameworks/_dl_common/model_handler.py +1 -1
- mlrun/frameworks/_dl_common/utils.py +1 -1
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
- mlrun/frameworks/_ml_common/loggers/logger.py +0 -1
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_ml_common/model_handler.py +1 -1
- mlrun/frameworks/_ml_common/pkl_model_server.py +1 -1
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +0 -1
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +0 -1
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
- mlrun/frameworks/_ml_common/producer.py +1 -1
- mlrun/frameworks/_ml_common/utils.py +1 -1
- mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +0 -1
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +0 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/model_handler.py +1 -1
- mlrun/frameworks/lgbm/model_server.py +1 -1
- mlrun/frameworks/lgbm/utils.py +1 -1
- mlrun/frameworks/onnx/dataset.py +1 -1
- mlrun/frameworks/onnx/mlrun_interface.py +1 -1
- mlrun/frameworks/onnx/model_handler.py +1 -1
- mlrun/frameworks/onnx/model_server.py +1 -1
- mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
- mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
- mlrun/frameworks/pytorch/model_handler.py +1 -1
- mlrun/frameworks/pytorch/model_server.py +1 -1
- mlrun/frameworks/pytorch/utils.py +1 -1
- mlrun/frameworks/sklearn/__init__.py +0 -14
- mlrun/frameworks/sklearn/estimator.py +1 -1
- mlrun/frameworks/sklearn/metric.py +1 -1
- mlrun/frameworks/sklearn/metrics_library.py +1 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
- mlrun/frameworks/sklearn/model_handler.py +1 -1
- mlrun/frameworks/sklearn/utils.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
- mlrun/frameworks/tf_keras/model_handler.py +1 -1
- mlrun/frameworks/tf_keras/model_server.py +1 -1
- mlrun/frameworks/tf_keras/utils.py +1 -1
- mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
- mlrun/frameworks/xgboost/model_handler.py +1 -1
- mlrun/frameworks/xgboost/utils.py +1 -1
- mlrun/k8s_utils.py +340 -0
- mlrun/launcher/base.py +3 -3
- mlrun/launcher/local.py +2 -2
- mlrun/launcher/remote.py +2 -2
- mlrun/model.py +14 -0
- mlrun/model_monitoring/applications/__init__.py +0 -1
- mlrun/model_monitoring/applications/_application_steps.py +3 -1
- mlrun/model_monitoring/controller.py +3 -1
- mlrun/model_monitoring/db/tsdb/base.py +3 -1
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +213 -0
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +27 -49
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +35 -30
- mlrun/package/context_handler.py +1 -1
- mlrun/package/errors.py +1 -1
- mlrun/package/packager.py +1 -1
- mlrun/package/packagers/default_packager.py +1 -1
- mlrun/package/packagers/numpy_packagers.py +1 -1
- mlrun/package/packagers/pandas_packagers.py +1 -1
- mlrun/package/packagers/python_standard_library_packagers.py +1 -1
- mlrun/package/packagers_manager.py +1 -1
- mlrun/package/utils/_archiver.py +1 -1
- mlrun/package/utils/_formatter.py +1 -1
- mlrun/package/utils/_pickler.py +1 -1
- mlrun/package/utils/_supported_format.py +1 -1
- mlrun/package/utils/log_hint_utils.py +1 -1
- mlrun/package/utils/type_hint_utils.py +1 -1
- mlrun/projects/operations.py +36 -21
- mlrun/projects/project.py +82 -74
- mlrun/run.py +1 -1
- mlrun/runtimes/base.py +16 -6
- mlrun/runtimes/daskjob.py +2 -1
- mlrun/runtimes/databricks_job/databricks_cancel_task.py +0 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +2 -1
- mlrun/runtimes/databricks_job/databricks_wrapper.py +0 -1
- mlrun/runtimes/mounts.py +2 -0
- mlrun/runtimes/nuclio/function.py +6 -1
- mlrun/runtimes/nuclio/serving.py +1 -1
- mlrun/runtimes/pod.py +4 -349
- mlrun/runtimes/sparkjob/spark3job.py +0 -12
- mlrun/serving/merger.py +0 -1
- mlrun/serving/remote.py +1 -1
- mlrun/serving/serving_wrapper.py +1 -1
- mlrun/serving/utils.py +1 -1
- mlrun/utils/async_http.py +0 -1
- mlrun/utils/clones.py +1 -1
- mlrun/utils/db.py +1 -1
- mlrun/utils/helpers.py +3 -1
- mlrun/utils/http.py +0 -1
- mlrun/utils/notifications/notification/webhook.py +18 -2
- mlrun/utils/regex.py +0 -1
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/vault.py +1 -1
- mlrun/utils/version/__init__.py +1 -1
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +1 -1
- {mlrun-1.9.0rc3.dist-info → mlrun-1.10.0rc1.dist-info}/METADATA +6 -10
- mlrun-1.10.0rc1.dist-info/RECORD +351 -0
- {mlrun-1.9.0rc3.dist-info → mlrun-1.10.0rc1.dist-info}/WHEEL +1 -1
- mlrun-1.9.0rc3.dist-info/RECORD +0 -350
- {mlrun-1.9.0rc3.dist-info → mlrun-1.10.0rc1.dist-info}/entry_points.txt +0 -0
- {mlrun-1.9.0rc3.dist-info → mlrun-1.10.0rc1.dist-info}/licenses/LICENSE +0 -0
- {mlrun-1.9.0rc3.dist-info → mlrun-1.10.0rc1.dist-info}/top_level.txt +0 -0
mlrun/runtimes/pod.py
CHANGED
|
@@ -29,17 +29,12 @@ import mlrun.errors
|
|
|
29
29
|
import mlrun.runtimes.mounts
|
|
30
30
|
import mlrun.utils.regex
|
|
31
31
|
from mlrun.common.schemas import (
|
|
32
|
-
NodeSelectorOperator,
|
|
33
32
|
PreemptionModes,
|
|
34
33
|
SecurityContextEnrichmentModes,
|
|
35
34
|
)
|
|
36
35
|
|
|
37
36
|
from ..config import config as mlconf
|
|
38
37
|
from ..k8s_utils import (
|
|
39
|
-
generate_preemptible_node_selector_requirements,
|
|
40
|
-
generate_preemptible_nodes_affinity_terms,
|
|
41
|
-
generate_preemptible_nodes_anti_affinity_terms,
|
|
42
|
-
generate_preemptible_tolerations,
|
|
43
38
|
validate_node_selectors,
|
|
44
39
|
)
|
|
45
40
|
from ..utils import logger, update_in
|
|
@@ -292,7 +287,6 @@ class KubeResourceSpec(FunctionSpec):
|
|
|
292
287
|
@preemption_mode.setter
|
|
293
288
|
def preemption_mode(self, mode):
|
|
294
289
|
self._preemption_mode = mode or mlconf.function_defaults.preemption_mode
|
|
295
|
-
self.enrich_function_preemption_spec()
|
|
296
290
|
|
|
297
291
|
@property
|
|
298
292
|
def security_context(self) -> k8s_client.V1SecurityContext:
|
|
@@ -560,300 +554,6 @@ class KubeResourceSpec(FunctionSpec):
|
|
|
560
554
|
return {}
|
|
561
555
|
return resources
|
|
562
556
|
|
|
563
|
-
def _merge_node_selector(self, node_selector: dict[str, str]):
|
|
564
|
-
if not node_selector:
|
|
565
|
-
return
|
|
566
|
-
|
|
567
|
-
# merge node selectors - precedence to existing node selector
|
|
568
|
-
self.node_selector = mlrun.utils.helpers.merge_dicts_with_precedence(
|
|
569
|
-
node_selector, self.node_selector
|
|
570
|
-
)
|
|
571
|
-
|
|
572
|
-
def _merge_tolerations(
|
|
573
|
-
self,
|
|
574
|
-
tolerations: list[k8s_client.V1Toleration],
|
|
575
|
-
tolerations_field_name: str,
|
|
576
|
-
):
|
|
577
|
-
if not tolerations:
|
|
578
|
-
return
|
|
579
|
-
# In case function has no toleration, take all from input
|
|
580
|
-
self_tolerations = getattr(self, tolerations_field_name)
|
|
581
|
-
if not self_tolerations:
|
|
582
|
-
setattr(self, tolerations_field_name, tolerations)
|
|
583
|
-
return
|
|
584
|
-
tolerations_to_add = []
|
|
585
|
-
|
|
586
|
-
# Only add non-matching tolerations to avoid duplications
|
|
587
|
-
for toleration in tolerations:
|
|
588
|
-
to_add = True
|
|
589
|
-
for function_toleration in self_tolerations:
|
|
590
|
-
if function_toleration == toleration:
|
|
591
|
-
to_add = False
|
|
592
|
-
break
|
|
593
|
-
if to_add:
|
|
594
|
-
tolerations_to_add.append(toleration)
|
|
595
|
-
|
|
596
|
-
if len(tolerations_to_add) > 0:
|
|
597
|
-
self_tolerations.extend(tolerations_to_add)
|
|
598
|
-
|
|
599
|
-
def _override_required_during_scheduling_ignored_during_execution(
|
|
600
|
-
self,
|
|
601
|
-
node_selector: k8s_client.V1NodeSelector,
|
|
602
|
-
affinity_field_name: str,
|
|
603
|
-
):
|
|
604
|
-
self._initialize_affinity(affinity_field_name)
|
|
605
|
-
self._initialize_node_affinity(affinity_field_name)
|
|
606
|
-
|
|
607
|
-
self_affinity = getattr(self, affinity_field_name)
|
|
608
|
-
self_affinity.node_affinity.required_during_scheduling_ignored_during_execution = node_selector
|
|
609
|
-
|
|
610
|
-
def enrich_function_preemption_spec(
|
|
611
|
-
self,
|
|
612
|
-
preemption_mode_field_name: str = "preemption_mode",
|
|
613
|
-
tolerations_field_name: str = "tolerations",
|
|
614
|
-
affinity_field_name: str = "affinity",
|
|
615
|
-
node_selector_field_name: str = "node_selector",
|
|
616
|
-
):
|
|
617
|
-
"""
|
|
618
|
-
Enriches function pod with the below described spec.
|
|
619
|
-
If no preemptible node configuration is provided, do nothing.
|
|
620
|
-
`allow` - Adds Tolerations if configured.
|
|
621
|
-
otherwise, assume pods can be scheduled on preemptible nodes.
|
|
622
|
-
> Purges any `affinity` / `anti-affinity` preemption related configuration
|
|
623
|
-
> Purges preemptible node selector
|
|
624
|
-
`constrain` - Uses node-affinity to make sure pods are assigned using OR on the configured
|
|
625
|
-
node label selectors.
|
|
626
|
-
> Merges tolerations with preemptible tolerations.
|
|
627
|
-
> Purges any `anti-affinity` preemption related configuration
|
|
628
|
-
`prevent` - Prevention is done either using taints (if Tolerations were configured) or anti-affinity.
|
|
629
|
-
> Purges any `tolerations` preemption related configuration
|
|
630
|
-
> Purges any `affinity` preemption related configuration
|
|
631
|
-
> Purges preemptible node selector
|
|
632
|
-
> Sets anti-affinity and overrides any affinity if no tolerations were configured
|
|
633
|
-
`none` - Doesn't apply any preemptible node selection configuration.
|
|
634
|
-
"""
|
|
635
|
-
# nothing to do here, configuration is not populated
|
|
636
|
-
if not mlconf.is_preemption_nodes_configured():
|
|
637
|
-
return
|
|
638
|
-
|
|
639
|
-
if not getattr(self, preemption_mode_field_name):
|
|
640
|
-
# We're not supposed to get here, but if we do, we'll set the private attribute to
|
|
641
|
-
# avoid triggering circular enrichment.
|
|
642
|
-
setattr(
|
|
643
|
-
self,
|
|
644
|
-
f"_{preemption_mode_field_name}",
|
|
645
|
-
mlconf.function_defaults.preemption_mode,
|
|
646
|
-
)
|
|
647
|
-
logger.debug(
|
|
648
|
-
"No preemption mode was given, using the default preemption mode",
|
|
649
|
-
default_preemption_mode=getattr(self, preemption_mode_field_name),
|
|
650
|
-
)
|
|
651
|
-
self_preemption_mode = getattr(self, preemption_mode_field_name)
|
|
652
|
-
# don't enrich with preemption configuration.
|
|
653
|
-
if self_preemption_mode == PreemptionModes.none.value:
|
|
654
|
-
return
|
|
655
|
-
# remove preemptible tolerations and remove preemption related configuration
|
|
656
|
-
# and enrich with anti-affinity if preemptible tolerations configuration haven't been provided
|
|
657
|
-
if self_preemption_mode == PreemptionModes.prevent.value:
|
|
658
|
-
# ensure no preemptible node tolerations
|
|
659
|
-
self._prune_tolerations(
|
|
660
|
-
generate_preemptible_tolerations(),
|
|
661
|
-
tolerations_field_name=tolerations_field_name,
|
|
662
|
-
)
|
|
663
|
-
|
|
664
|
-
# purge affinity preemption related configuration
|
|
665
|
-
self._prune_affinity_node_selector_requirement(
|
|
666
|
-
generate_preemptible_node_selector_requirements(
|
|
667
|
-
NodeSelectorOperator.node_selector_op_in.value
|
|
668
|
-
),
|
|
669
|
-
affinity_field_name=affinity_field_name,
|
|
670
|
-
)
|
|
671
|
-
# remove preemptible nodes constrain
|
|
672
|
-
self._prune_node_selector(
|
|
673
|
-
mlconf.get_preemptible_node_selector(),
|
|
674
|
-
node_selector_field_name=node_selector_field_name,
|
|
675
|
-
)
|
|
676
|
-
|
|
677
|
-
# if tolerations are configured, simply pruning tolerations is sufficient because functions
|
|
678
|
-
# cannot be scheduled without tolerations on tainted nodes.
|
|
679
|
-
# however, if preemptible tolerations are not configured, we must use anti-affinity on preemptible nodes
|
|
680
|
-
# to ensure that the function is not scheduled on the nodes.
|
|
681
|
-
if not generate_preemptible_tolerations():
|
|
682
|
-
# using a single term with potentially multiple expressions to ensure anti-affinity
|
|
683
|
-
self._override_required_during_scheduling_ignored_during_execution(
|
|
684
|
-
k8s_client.V1NodeSelector(
|
|
685
|
-
node_selector_terms=generate_preemptible_nodes_anti_affinity_terms()
|
|
686
|
-
),
|
|
687
|
-
affinity_field_name=affinity_field_name,
|
|
688
|
-
)
|
|
689
|
-
# enrich tolerations and override all node selector terms with preemptible node selector terms
|
|
690
|
-
elif self_preemption_mode == PreemptionModes.constrain.value:
|
|
691
|
-
# enrich with tolerations
|
|
692
|
-
self._merge_tolerations(
|
|
693
|
-
generate_preemptible_tolerations(),
|
|
694
|
-
tolerations_field_name=tolerations_field_name,
|
|
695
|
-
)
|
|
696
|
-
|
|
697
|
-
# setting required_during_scheduling_ignored_during_execution
|
|
698
|
-
# overriding other terms that have been set, and only setting terms for preemptible nodes
|
|
699
|
-
# when having multiple terms, pod scheduling is succeeded if at least one term is satisfied
|
|
700
|
-
self._override_required_during_scheduling_ignored_during_execution(
|
|
701
|
-
k8s_client.V1NodeSelector(
|
|
702
|
-
node_selector_terms=generate_preemptible_nodes_affinity_terms()
|
|
703
|
-
),
|
|
704
|
-
affinity_field_name=affinity_field_name,
|
|
705
|
-
)
|
|
706
|
-
# purge any affinity / anti-affinity preemption related configuration and enrich with preemptible tolerations
|
|
707
|
-
elif self_preemption_mode == PreemptionModes.allow.value:
|
|
708
|
-
# remove preemptible anti-affinity
|
|
709
|
-
self._prune_affinity_node_selector_requirement(
|
|
710
|
-
generate_preemptible_node_selector_requirements(
|
|
711
|
-
NodeSelectorOperator.node_selector_op_not_in.value
|
|
712
|
-
),
|
|
713
|
-
affinity_field_name=affinity_field_name,
|
|
714
|
-
)
|
|
715
|
-
# remove preemptible affinity
|
|
716
|
-
self._prune_affinity_node_selector_requirement(
|
|
717
|
-
generate_preemptible_node_selector_requirements(
|
|
718
|
-
NodeSelectorOperator.node_selector_op_in.value
|
|
719
|
-
),
|
|
720
|
-
affinity_field_name=affinity_field_name,
|
|
721
|
-
)
|
|
722
|
-
|
|
723
|
-
# remove preemptible nodes constrain
|
|
724
|
-
self._prune_node_selector(
|
|
725
|
-
mlconf.get_preemptible_node_selector(),
|
|
726
|
-
node_selector_field_name=node_selector_field_name,
|
|
727
|
-
)
|
|
728
|
-
|
|
729
|
-
# enrich with tolerations
|
|
730
|
-
self._merge_tolerations(
|
|
731
|
-
generate_preemptible_tolerations(),
|
|
732
|
-
tolerations_field_name=tolerations_field_name,
|
|
733
|
-
)
|
|
734
|
-
|
|
735
|
-
self._clear_affinity_if_initialized_but_empty(
|
|
736
|
-
affinity_field_name=affinity_field_name
|
|
737
|
-
)
|
|
738
|
-
self._clear_tolerations_if_initialized_but_empty(
|
|
739
|
-
tolerations_field_name=tolerations_field_name
|
|
740
|
-
)
|
|
741
|
-
|
|
742
|
-
def _clear_affinity_if_initialized_but_empty(self, affinity_field_name: str):
|
|
743
|
-
self_affinity = getattr(self, affinity_field_name)
|
|
744
|
-
if not getattr(self, affinity_field_name):
|
|
745
|
-
setattr(self, affinity_field_name, None)
|
|
746
|
-
elif (
|
|
747
|
-
not self_affinity.node_affinity
|
|
748
|
-
and not self_affinity.pod_affinity
|
|
749
|
-
and not self_affinity.pod_anti_affinity
|
|
750
|
-
):
|
|
751
|
-
setattr(self, affinity_field_name, None)
|
|
752
|
-
|
|
753
|
-
def _clear_tolerations_if_initialized_but_empty(self, tolerations_field_name: str):
|
|
754
|
-
if not getattr(self, tolerations_field_name):
|
|
755
|
-
setattr(self, tolerations_field_name, None)
|
|
756
|
-
|
|
757
|
-
def _merge_node_selector_term_to_node_affinity(
|
|
758
|
-
self,
|
|
759
|
-
node_selector_terms: list[k8s_client.V1NodeSelectorTerm],
|
|
760
|
-
affinity_field_name: str,
|
|
761
|
-
):
|
|
762
|
-
if not node_selector_terms:
|
|
763
|
-
return
|
|
764
|
-
|
|
765
|
-
self._initialize_affinity(affinity_field_name)
|
|
766
|
-
self._initialize_node_affinity(affinity_field_name)
|
|
767
|
-
|
|
768
|
-
self_affinity = getattr(self, affinity_field_name)
|
|
769
|
-
if not self_affinity.node_affinity.required_during_scheduling_ignored_during_execution:
|
|
770
|
-
self_affinity.node_affinity.required_during_scheduling_ignored_during_execution = k8s_client.V1NodeSelector(
|
|
771
|
-
node_selector_terms=node_selector_terms
|
|
772
|
-
)
|
|
773
|
-
return
|
|
774
|
-
|
|
775
|
-
node_selector = self_affinity.node_affinity.required_during_scheduling_ignored_during_execution
|
|
776
|
-
new_node_selector_terms = []
|
|
777
|
-
|
|
778
|
-
for node_selector_term_to_add in node_selector_terms:
|
|
779
|
-
to_add = True
|
|
780
|
-
for node_selector_term in node_selector.node_selector_terms:
|
|
781
|
-
if node_selector_term == node_selector_term_to_add:
|
|
782
|
-
to_add = False
|
|
783
|
-
break
|
|
784
|
-
if to_add:
|
|
785
|
-
new_node_selector_terms.append(node_selector_term_to_add)
|
|
786
|
-
|
|
787
|
-
if new_node_selector_terms:
|
|
788
|
-
node_selector.node_selector_terms += new_node_selector_terms
|
|
789
|
-
|
|
790
|
-
def _initialize_affinity(self, affinity_field_name: str):
|
|
791
|
-
if not getattr(self, affinity_field_name):
|
|
792
|
-
setattr(self, affinity_field_name, k8s_client.V1Affinity())
|
|
793
|
-
|
|
794
|
-
def _initialize_node_affinity(self, affinity_field_name: str):
|
|
795
|
-
if not getattr(getattr(self, affinity_field_name), "node_affinity"):
|
|
796
|
-
# self.affinity.node_affinity:
|
|
797
|
-
getattr(
|
|
798
|
-
self, affinity_field_name
|
|
799
|
-
).node_affinity = k8s_client.V1NodeAffinity()
|
|
800
|
-
# self.affinity.node_affinity = k8s_client.V1NodeAffinity()
|
|
801
|
-
|
|
802
|
-
def _prune_affinity_node_selector_requirement(
|
|
803
|
-
self,
|
|
804
|
-
node_selector_requirements: list[k8s_client.V1NodeSelectorRequirement],
|
|
805
|
-
affinity_field_name: str = "affinity",
|
|
806
|
-
):
|
|
807
|
-
"""
|
|
808
|
-
Prunes given node selector requirements from affinity.
|
|
809
|
-
We are only editing required_during_scheduling_ignored_during_execution because the scheduler can't schedule
|
|
810
|
-
the pod unless the rule is met.
|
|
811
|
-
:param node_selector_requirements:
|
|
812
|
-
:return:
|
|
813
|
-
"""
|
|
814
|
-
# both needs to exist to prune required affinity from spec affinity
|
|
815
|
-
self_affinity = getattr(self, affinity_field_name)
|
|
816
|
-
if not self_affinity or not node_selector_requirements:
|
|
817
|
-
return
|
|
818
|
-
if self_affinity.node_affinity:
|
|
819
|
-
node_affinity: k8s_client.V1NodeAffinity = self_affinity.node_affinity
|
|
820
|
-
|
|
821
|
-
new_required_during_scheduling_ignored_during_execution = None
|
|
822
|
-
if node_affinity.required_during_scheduling_ignored_during_execution:
|
|
823
|
-
node_selector: k8s_client.V1NodeSelector = (
|
|
824
|
-
node_affinity.required_during_scheduling_ignored_during_execution
|
|
825
|
-
)
|
|
826
|
-
new_node_selector_terms = (
|
|
827
|
-
self._prune_node_selector_requirements_from_node_selector_terms(
|
|
828
|
-
node_selector_terms=node_selector.node_selector_terms,
|
|
829
|
-
node_selector_requirements_to_prune=node_selector_requirements,
|
|
830
|
-
)
|
|
831
|
-
)
|
|
832
|
-
# check whether there are node selector terms to add to the new list of required terms
|
|
833
|
-
if len(new_node_selector_terms) > 0:
|
|
834
|
-
new_required_during_scheduling_ignored_during_execution = (
|
|
835
|
-
k8s_client.V1NodeSelector(
|
|
836
|
-
node_selector_terms=new_node_selector_terms
|
|
837
|
-
)
|
|
838
|
-
)
|
|
839
|
-
# if both preferred and new required are empty, clean node_affinity
|
|
840
|
-
if (
|
|
841
|
-
not node_affinity.preferred_during_scheduling_ignored_during_execution
|
|
842
|
-
and not new_required_during_scheduling_ignored_during_execution
|
|
843
|
-
):
|
|
844
|
-
setattr(self_affinity, "node_affinity", None)
|
|
845
|
-
# self.affinity.node_affinity = None
|
|
846
|
-
return
|
|
847
|
-
|
|
848
|
-
self._initialize_affinity(affinity_field_name)
|
|
849
|
-
self._initialize_node_affinity(affinity_field_name)
|
|
850
|
-
|
|
851
|
-
# fmt: off
|
|
852
|
-
self_affinity.node_affinity.required_during_scheduling_ignored_during_execution = (
|
|
853
|
-
new_required_during_scheduling_ignored_during_execution
|
|
854
|
-
)
|
|
855
|
-
# fmt: on
|
|
856
|
-
|
|
857
557
|
@staticmethod
|
|
858
558
|
def _prune_node_selector_requirements_from_node_selector_terms(
|
|
859
559
|
node_selector_terms: list[k8s_client.V1NodeSelectorTerm],
|
|
@@ -894,55 +594,6 @@ class KubeResourceSpec(FunctionSpec):
|
|
|
894
594
|
)
|
|
895
595
|
return new_node_selector_terms
|
|
896
596
|
|
|
897
|
-
def _prune_tolerations(
|
|
898
|
-
self,
|
|
899
|
-
tolerations: list[k8s_client.V1Toleration],
|
|
900
|
-
tolerations_field_name: str = "tolerations",
|
|
901
|
-
):
|
|
902
|
-
"""
|
|
903
|
-
Prunes given tolerations from function spec
|
|
904
|
-
:param tolerations: tolerations to prune
|
|
905
|
-
"""
|
|
906
|
-
self_tolerations = getattr(self, tolerations_field_name)
|
|
907
|
-
# both needs to exist to prune required tolerations from spec tolerations
|
|
908
|
-
if not tolerations or not self_tolerations:
|
|
909
|
-
return
|
|
910
|
-
|
|
911
|
-
# generate a list of tolerations without tolerations to prune
|
|
912
|
-
new_tolerations = []
|
|
913
|
-
for toleration in self_tolerations:
|
|
914
|
-
to_prune = False
|
|
915
|
-
for toleration_to_delete in tolerations:
|
|
916
|
-
if toleration == toleration_to_delete:
|
|
917
|
-
to_prune = True
|
|
918
|
-
# no need to keep going over the list provided for the current toleration
|
|
919
|
-
break
|
|
920
|
-
if not to_prune:
|
|
921
|
-
new_tolerations.append(toleration)
|
|
922
|
-
|
|
923
|
-
# Set tolerations without tolerations to prune
|
|
924
|
-
setattr(self, tolerations_field_name, new_tolerations)
|
|
925
|
-
|
|
926
|
-
def _prune_node_selector(
|
|
927
|
-
self,
|
|
928
|
-
node_selector: dict[str, str],
|
|
929
|
-
node_selector_field_name: str,
|
|
930
|
-
):
|
|
931
|
-
"""
|
|
932
|
-
Prunes given node_selector key from function spec if their key and value are matching
|
|
933
|
-
:param node_selector: node selectors to prune
|
|
934
|
-
"""
|
|
935
|
-
self_node_selector = getattr(self, node_selector_field_name)
|
|
936
|
-
# both needs to exists to prune required node_selector from the spec node selector
|
|
937
|
-
if not node_selector or not self_node_selector:
|
|
938
|
-
return
|
|
939
|
-
|
|
940
|
-
for key, value in node_selector.items():
|
|
941
|
-
if value:
|
|
942
|
-
spec_value = self_node_selector.get(key)
|
|
943
|
-
if spec_value and spec_value == value:
|
|
944
|
-
self_node_selector.pop(key)
|
|
945
|
-
|
|
946
597
|
|
|
947
598
|
class AutoMountType(str, Enum):
|
|
948
599
|
none = "none"
|
|
@@ -1646,6 +1297,10 @@ def get_sanitized_attribute(spec, attribute_name: str):
|
|
|
1646
1297
|
if _resolve_if_type_sanitized(attribute_name, attribute[0]):
|
|
1647
1298
|
return attribute
|
|
1648
1299
|
|
|
1300
|
+
return sanitize_attribute(attribute)
|
|
1301
|
+
|
|
1302
|
+
|
|
1303
|
+
def sanitize_attribute(attribute):
|
|
1649
1304
|
api = k8s_client.ApiClient()
|
|
1650
1305
|
return api.sanitize_for_serialization(attribute)
|
|
1651
1306
|
|
|
@@ -288,12 +288,6 @@ class Spark3JobSpec(KubeResourceSpec):
|
|
|
288
288
|
self._driver_preemption_mode = (
|
|
289
289
|
mode or mlrun.mlconf.function_defaults.preemption_mode
|
|
290
290
|
)
|
|
291
|
-
self.enrich_function_preemption_spec(
|
|
292
|
-
preemption_mode_field_name="driver_preemption_mode",
|
|
293
|
-
tolerations_field_name="driver_tolerations",
|
|
294
|
-
affinity_field_name="driver_affinity",
|
|
295
|
-
node_selector_field_name="driver_node_selector",
|
|
296
|
-
)
|
|
297
291
|
|
|
298
292
|
@property
|
|
299
293
|
def executor_preemption_mode(self) -> str:
|
|
@@ -304,12 +298,6 @@ class Spark3JobSpec(KubeResourceSpec):
|
|
|
304
298
|
self._executor_preemption_mode = (
|
|
305
299
|
mode or mlrun.mlconf.function_defaults.preemption_mode
|
|
306
300
|
)
|
|
307
|
-
self.enrich_function_preemption_spec(
|
|
308
|
-
preemption_mode_field_name="executor_preemption_mode",
|
|
309
|
-
tolerations_field_name="executor_tolerations",
|
|
310
|
-
affinity_field_name="executor_affinity",
|
|
311
|
-
node_selector_field_name="executor_node_selector",
|
|
312
|
-
)
|
|
313
301
|
|
|
314
302
|
@property
|
|
315
303
|
def driver_volume_mounts(self) -> list:
|
mlrun/serving/merger.py
CHANGED
mlrun/serving/remote.py
CHANGED
mlrun/serving/serving_wrapper.py
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
# serving runtime hooks, used in empty serving functions
|
|
16
16
|
from mlrun.runtimes import nuclio_init_hook
|
|
17
17
|
|
mlrun/serving/utils.py
CHANGED
mlrun/utils/async_http.py
CHANGED
mlrun/utils/clones.py
CHANGED
mlrun/utils/db.py
CHANGED
mlrun/utils/helpers.py
CHANGED
|
@@ -1502,7 +1502,9 @@ def _fill_project_path_template(artifact_path, project):
|
|
|
1502
1502
|
|
|
1503
1503
|
|
|
1504
1504
|
def to_non_empty_values_dict(input_dict: dict) -> dict:
|
|
1505
|
-
return
|
|
1505
|
+
return (
|
|
1506
|
+
{key: value for key, value in input_dict.items() if value} if input_dict else {}
|
|
1507
|
+
)
|
|
1506
1508
|
|
|
1507
1509
|
|
|
1508
1510
|
def get_enriched_gpu_limits(function_limits: dict) -> dict[str, int]:
|
mlrun/utils/http.py
CHANGED
|
@@ -16,6 +16,7 @@ import re
|
|
|
16
16
|
import typing
|
|
17
17
|
|
|
18
18
|
import aiohttp
|
|
19
|
+
import orjson
|
|
19
20
|
|
|
20
21
|
import mlrun.common.schemas
|
|
21
22
|
import mlrun.lists
|
|
@@ -86,9 +87,14 @@ class WebhookNotification(NotificationBase):
|
|
|
86
87
|
# we automatically handle it as `ssl=None` for their convenience.
|
|
87
88
|
verify_ssl = verify_ssl and None if url.startswith("https") else None
|
|
88
89
|
|
|
89
|
-
async with aiohttp.ClientSession(
|
|
90
|
+
async with aiohttp.ClientSession(
|
|
91
|
+
json_serialize=self._encoder,
|
|
92
|
+
) as session:
|
|
90
93
|
response = await getattr(session, method)(
|
|
91
|
-
url,
|
|
94
|
+
url,
|
|
95
|
+
headers=headers,
|
|
96
|
+
json=request_body,
|
|
97
|
+
ssl=verify_ssl,
|
|
92
98
|
)
|
|
93
99
|
response.raise_for_status()
|
|
94
100
|
|
|
@@ -128,3 +134,13 @@ class WebhookNotification(NotificationBase):
|
|
|
128
134
|
)
|
|
129
135
|
|
|
130
136
|
return override_body
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def _encoder(self):
|
|
140
|
+
return lambda body: orjson.dumps(
|
|
141
|
+
body,
|
|
142
|
+
option=orjson.OPT_NAIVE_UTC
|
|
143
|
+
| orjson.OPT_SERIALIZE_NUMPY
|
|
144
|
+
| orjson.OPT_NON_STR_KEYS
|
|
145
|
+
| orjson.OPT_SORT_KEYS,
|
|
146
|
+
).decode()
|
mlrun/utils/regex.py
CHANGED
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
#
|
|
15
14
|
|
|
16
15
|
# pipeline param format which is passed when running a pipeline (e.g. {{pipelineparam:op=;name=mem}})
|
|
17
16
|
# https://github.com/kubeflow/pipelines/blob/16edebf4eaf84cd7478e2601ef4878ab339a7854/sdk/python/kfp/dsl/_pipeline_param.py#L213
|
mlrun/utils/singleton.py
CHANGED
mlrun/utils/vault.py
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
# import json
|
|
16
16
|
# import os
|
|
17
17
|
# from os.path import expanduser
|
mlrun/utils/version/__init__.py
CHANGED
mlrun/utils/version/version.json
CHANGED
mlrun/utils/version/version.py
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
import json
|
|
16
16
|
import sys
|
|
17
17
|
from importlib.resources import read_text
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mlrun
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.10.0rc1
|
|
4
4
|
Summary: Tracking and config of machine learning runs
|
|
5
5
|
Home-page: https://github.com/mlrun/mlrun
|
|
6
6
|
Author: Yaron Haviv
|
|
@@ -51,8 +51,8 @@ Requires-Dist: setuptools>=75.2
|
|
|
51
51
|
Requires-Dist: deprecated~=1.2
|
|
52
52
|
Requires-Dist: jinja2>=3.1.3,~=3.1
|
|
53
53
|
Requires-Dist: orjson<4,>=3.9.15
|
|
54
|
-
Requires-Dist: mlrun-pipelines-kfp-common~=0.
|
|
55
|
-
Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.
|
|
54
|
+
Requires-Dist: mlrun-pipelines-kfp-common~=0.5.1
|
|
55
|
+
Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.1
|
|
56
56
|
Requires-Dist: docstring_parser~=0.16
|
|
57
57
|
Requires-Dist: aiosmtplib~=3.0
|
|
58
58
|
Provides-Extra: s3
|
|
@@ -99,11 +99,10 @@ Requires-Dist: ossfs==2023.12.0; extra == "alibaba-oss"
|
|
|
99
99
|
Requires-Dist: oss2==2.18.1; extra == "alibaba-oss"
|
|
100
100
|
Provides-Extra: tdengine
|
|
101
101
|
Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
|
|
102
|
-
Requires-Dist: taoswswrap~=0.3.5; extra == "tdengine"
|
|
103
102
|
Provides-Extra: snowflake
|
|
104
103
|
Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
|
|
105
104
|
Provides-Extra: kfp18
|
|
106
|
-
Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.
|
|
105
|
+
Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.5.0; python_version < "3.11" and extra == "kfp18"
|
|
107
106
|
Provides-Extra: api
|
|
108
107
|
Requires-Dist: uvicorn~=0.32.1; extra == "api"
|
|
109
108
|
Requires-Dist: dask-kubernetes~=0.11.0; extra == "api"
|
|
@@ -119,7 +118,7 @@ Requires-Dist: timelength~=1.1; extra == "api"
|
|
|
119
118
|
Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
|
|
120
119
|
Requires-Dist: aiosmtplib~=3.0; extra == "api"
|
|
121
120
|
Requires-Dist: pydantic<2,>=1; extra == "api"
|
|
122
|
-
Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.
|
|
121
|
+
Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.1; extra == "api"
|
|
123
122
|
Requires-Dist: grpcio~=1.70.0; extra == "api"
|
|
124
123
|
Provides-Extra: all
|
|
125
124
|
Requires-Dist: adlfs==2023.9.0; extra == "all"
|
|
@@ -152,7 +151,6 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "all"
|
|
|
152
151
|
Requires-Dist: snowflake-connector-python~=3.7; extra == "all"
|
|
153
152
|
Requires-Dist: sqlalchemy~=1.4; extra == "all"
|
|
154
153
|
Requires-Dist: taos-ws-py==0.3.2; extra == "all"
|
|
155
|
-
Requires-Dist: taoswswrap~=0.3.5; extra == "all"
|
|
156
154
|
Provides-Extra: complete
|
|
157
155
|
Requires-Dist: adlfs==2023.9.0; extra == "complete"
|
|
158
156
|
Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete"
|
|
@@ -184,7 +182,6 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete"
|
|
|
184
182
|
Requires-Dist: snowflake-connector-python~=3.7; extra == "complete"
|
|
185
183
|
Requires-Dist: sqlalchemy~=1.4; extra == "complete"
|
|
186
184
|
Requires-Dist: taos-ws-py==0.3.2; extra == "complete"
|
|
187
|
-
Requires-Dist: taoswswrap~=0.3.5; extra == "complete"
|
|
188
185
|
Provides-Extra: complete-api
|
|
189
186
|
Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
|
|
190
187
|
Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
|
|
@@ -215,7 +212,7 @@ Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
|
|
|
215
212
|
Requires-Dist: kafka-python~=2.0; extra == "complete-api"
|
|
216
213
|
Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "complete-api"
|
|
217
214
|
Requires-Dist: mlflow~=2.16; extra == "complete-api"
|
|
218
|
-
Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.
|
|
215
|
+
Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.1; extra == "complete-api"
|
|
219
216
|
Requires-Dist: msrest~=0.6.21; extra == "complete-api"
|
|
220
217
|
Requires-Dist: objgraph~=3.6; extra == "complete-api"
|
|
221
218
|
Requires-Dist: oss2==2.18.1; extra == "complete-api"
|
|
@@ -229,7 +226,6 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete-api"
|
|
|
229
226
|
Requires-Dist: snowflake-connector-python~=3.7; extra == "complete-api"
|
|
230
227
|
Requires-Dist: sqlalchemy~=1.4; extra == "complete-api"
|
|
231
228
|
Requires-Dist: taos-ws-py==0.3.2; extra == "complete-api"
|
|
232
|
-
Requires-Dist: taoswswrap~=0.3.5; extra == "complete-api"
|
|
233
229
|
Requires-Dist: timelength~=1.1; extra == "complete-api"
|
|
234
230
|
Requires-Dist: uvicorn~=0.32.1; extra == "complete-api"
|
|
235
231
|
Dynamic: author
|