mlrun 1.9.0rc3__py3-none-any.whl → 1.10.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (188) hide show
  1. mlrun/api/schemas/__init__.py +0 -1
  2. mlrun/common/__init__.py +0 -1
  3. mlrun/common/db/__init__.py +0 -1
  4. mlrun/common/db/sql_session.py +0 -1
  5. mlrun/common/formatters/__init__.py +0 -1
  6. mlrun/common/formatters/artifact.py +0 -1
  7. mlrun/common/formatters/base.py +0 -1
  8. mlrun/common/formatters/feature_set.py +0 -1
  9. mlrun/common/formatters/function.py +0 -1
  10. mlrun/common/formatters/model_endpoint.py +0 -1
  11. mlrun/common/formatters/pipeline.py +0 -1
  12. mlrun/common/formatters/project.py +0 -1
  13. mlrun/common/formatters/run.py +0 -2
  14. mlrun/common/runtimes/constants.py +1 -1
  15. mlrun/common/schemas/__init__.py +1 -0
  16. mlrun/common/schemas/alert.py +1 -1
  17. mlrun/common/schemas/api_gateway.py +1 -1
  18. mlrun/common/schemas/artifact.py +1 -1
  19. mlrun/common/schemas/auth.py +1 -1
  20. mlrun/common/schemas/background_task.py +1 -1
  21. mlrun/common/schemas/client_spec.py +1 -1
  22. mlrun/common/schemas/clusterization_spec.py +1 -1
  23. mlrun/common/schemas/constants.py +1 -1
  24. mlrun/common/schemas/datastore_profile.py +0 -1
  25. mlrun/common/schemas/events.py +1 -1
  26. mlrun/common/schemas/feature_store.py +1 -1
  27. mlrun/common/schemas/frontend_spec.py +1 -1
  28. mlrun/common/schemas/function.py +1 -1
  29. mlrun/common/schemas/http.py +1 -1
  30. mlrun/common/schemas/hub.py +1 -1
  31. mlrun/common/schemas/k8s.py +1 -1
  32. mlrun/common/schemas/memory_reports.py +0 -1
  33. mlrun/common/schemas/model_monitoring/model_endpoints.py +32 -8
  34. mlrun/common/schemas/notification.py +4 -0
  35. mlrun/common/schemas/object.py +1 -1
  36. mlrun/common/schemas/partition.py +1 -1
  37. mlrun/common/schemas/pipeline.py +1 -1
  38. mlrun/common/schemas/project.py +1 -1
  39. mlrun/common/schemas/regex.py +1 -1
  40. mlrun/common/schemas/runtime_resource.py +1 -1
  41. mlrun/common/schemas/schedule.py +1 -1
  42. mlrun/common/schemas/secret.py +1 -1
  43. mlrun/common/schemas/tag.py +0 -1
  44. mlrun/common/schemas/workflow.py +1 -1
  45. mlrun/common/secrets.py +0 -1
  46. mlrun/config.py +9 -17
  47. mlrun/data_types/infer.py +1 -1
  48. mlrun/data_types/spark.py +1 -1
  49. mlrun/datastore/datastore.py +1 -1
  50. mlrun/datastore/snowflake_utils.py +0 -1
  51. mlrun/datastore/spark_utils.py +0 -1
  52. mlrun/datastore/utils.py +1 -1
  53. mlrun/db/base.py +2 -0
  54. mlrun/db/httpdb.py +29 -19
  55. mlrun/db/nopdb.py +2 -1
  56. mlrun/errors.py +1 -1
  57. mlrun/execution.py +21 -9
  58. mlrun/feature_store/feature_set.py +0 -12
  59. mlrun/feature_store/retrieval/base.py +1 -1
  60. mlrun/feature_store/retrieval/dask_merger.py +1 -1
  61. mlrun/feature_store/retrieval/job.py +1 -1
  62. mlrun/feature_store/retrieval/spark_merger.py +0 -2
  63. mlrun/feature_store/steps.py +1 -1
  64. mlrun/features.py +1 -1
  65. mlrun/frameworks/_common/artifacts_library.py +1 -1
  66. mlrun/frameworks/_common/mlrun_interface.py +1 -1
  67. mlrun/frameworks/_common/model_handler.py +3 -3
  68. mlrun/frameworks/_common/producer.py +0 -1
  69. mlrun/frameworks/_common/utils.py +1 -1
  70. mlrun/frameworks/_dl_common/loggers/logger.py +0 -1
  71. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
  72. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
  73. mlrun/frameworks/_dl_common/model_handler.py +1 -1
  74. mlrun/frameworks/_dl_common/utils.py +1 -1
  75. mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
  76. mlrun/frameworks/_ml_common/loggers/logger.py +0 -1
  77. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  78. mlrun/frameworks/_ml_common/model_handler.py +1 -1
  79. mlrun/frameworks/_ml_common/pkl_model_server.py +1 -1
  80. mlrun/frameworks/_ml_common/plan.py +1 -1
  81. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +0 -1
  82. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +0 -1
  83. mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
  84. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
  85. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
  86. mlrun/frameworks/_ml_common/producer.py +1 -1
  87. mlrun/frameworks/_ml_common/utils.py +1 -1
  88. mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
  89. mlrun/frameworks/lgbm/callbacks/logging_callback.py +0 -1
  90. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +0 -1
  91. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
  92. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
  93. mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
  94. mlrun/frameworks/lgbm/model_handler.py +1 -1
  95. mlrun/frameworks/lgbm/model_server.py +1 -1
  96. mlrun/frameworks/lgbm/utils.py +1 -1
  97. mlrun/frameworks/onnx/dataset.py +1 -1
  98. mlrun/frameworks/onnx/mlrun_interface.py +1 -1
  99. mlrun/frameworks/onnx/model_handler.py +1 -1
  100. mlrun/frameworks/onnx/model_server.py +1 -1
  101. mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
  102. mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
  103. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
  104. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
  105. mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
  106. mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
  107. mlrun/frameworks/pytorch/model_handler.py +1 -1
  108. mlrun/frameworks/pytorch/model_server.py +1 -1
  109. mlrun/frameworks/pytorch/utils.py +1 -1
  110. mlrun/frameworks/sklearn/__init__.py +0 -14
  111. mlrun/frameworks/sklearn/estimator.py +1 -1
  112. mlrun/frameworks/sklearn/metric.py +1 -1
  113. mlrun/frameworks/sklearn/metrics_library.py +1 -1
  114. mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
  115. mlrun/frameworks/sklearn/model_handler.py +1 -1
  116. mlrun/frameworks/sklearn/utils.py +1 -1
  117. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  118. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
  119. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
  120. mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
  121. mlrun/frameworks/tf_keras/model_handler.py +1 -1
  122. mlrun/frameworks/tf_keras/model_server.py +1 -1
  123. mlrun/frameworks/tf_keras/utils.py +1 -1
  124. mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
  125. mlrun/frameworks/xgboost/model_handler.py +1 -1
  126. mlrun/frameworks/xgboost/utils.py +1 -1
  127. mlrun/k8s_utils.py +340 -0
  128. mlrun/launcher/base.py +3 -3
  129. mlrun/launcher/local.py +2 -2
  130. mlrun/launcher/remote.py +2 -2
  131. mlrun/model.py +14 -0
  132. mlrun/model_monitoring/applications/__init__.py +0 -1
  133. mlrun/model_monitoring/applications/_application_steps.py +3 -1
  134. mlrun/model_monitoring/controller.py +3 -1
  135. mlrun/model_monitoring/db/tsdb/base.py +3 -1
  136. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +213 -0
  137. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +27 -49
  138. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +35 -30
  139. mlrun/package/context_handler.py +1 -1
  140. mlrun/package/errors.py +1 -1
  141. mlrun/package/packager.py +1 -1
  142. mlrun/package/packagers/default_packager.py +1 -1
  143. mlrun/package/packagers/numpy_packagers.py +1 -1
  144. mlrun/package/packagers/pandas_packagers.py +1 -1
  145. mlrun/package/packagers/python_standard_library_packagers.py +1 -1
  146. mlrun/package/packagers_manager.py +1 -1
  147. mlrun/package/utils/_archiver.py +1 -1
  148. mlrun/package/utils/_formatter.py +1 -1
  149. mlrun/package/utils/_pickler.py +1 -1
  150. mlrun/package/utils/_supported_format.py +1 -1
  151. mlrun/package/utils/log_hint_utils.py +1 -1
  152. mlrun/package/utils/type_hint_utils.py +1 -1
  153. mlrun/projects/operations.py +36 -21
  154. mlrun/projects/project.py +82 -74
  155. mlrun/run.py +1 -1
  156. mlrun/runtimes/base.py +16 -6
  157. mlrun/runtimes/daskjob.py +2 -1
  158. mlrun/runtimes/databricks_job/databricks_cancel_task.py +0 -1
  159. mlrun/runtimes/databricks_job/databricks_runtime.py +2 -1
  160. mlrun/runtimes/databricks_job/databricks_wrapper.py +0 -1
  161. mlrun/runtimes/mounts.py +2 -0
  162. mlrun/runtimes/nuclio/function.py +6 -1
  163. mlrun/runtimes/nuclio/serving.py +1 -1
  164. mlrun/runtimes/pod.py +4 -349
  165. mlrun/runtimes/sparkjob/spark3job.py +0 -12
  166. mlrun/serving/merger.py +0 -1
  167. mlrun/serving/remote.py +1 -1
  168. mlrun/serving/serving_wrapper.py +1 -1
  169. mlrun/serving/utils.py +1 -1
  170. mlrun/utils/async_http.py +0 -1
  171. mlrun/utils/clones.py +1 -1
  172. mlrun/utils/db.py +1 -1
  173. mlrun/utils/helpers.py +3 -1
  174. mlrun/utils/http.py +0 -1
  175. mlrun/utils/notifications/notification/webhook.py +18 -2
  176. mlrun/utils/regex.py +0 -1
  177. mlrun/utils/singleton.py +1 -1
  178. mlrun/utils/vault.py +1 -1
  179. mlrun/utils/version/__init__.py +1 -1
  180. mlrun/utils/version/version.json +2 -2
  181. mlrun/utils/version/version.py +1 -1
  182. {mlrun-1.9.0rc3.dist-info → mlrun-1.10.0rc1.dist-info}/METADATA +6 -10
  183. mlrun-1.10.0rc1.dist-info/RECORD +351 -0
  184. {mlrun-1.9.0rc3.dist-info → mlrun-1.10.0rc1.dist-info}/WHEEL +1 -1
  185. mlrun-1.9.0rc3.dist-info/RECORD +0 -350
  186. {mlrun-1.9.0rc3.dist-info → mlrun-1.10.0rc1.dist-info}/entry_points.txt +0 -0
  187. {mlrun-1.9.0rc3.dist-info → mlrun-1.10.0rc1.dist-info}/licenses/LICENSE +0 -0
  188. {mlrun-1.9.0rc3.dist-info → mlrun-1.10.0rc1.dist-info}/top_level.txt +0 -0
mlrun/runtimes/pod.py CHANGED
@@ -29,17 +29,12 @@ import mlrun.errors
29
29
  import mlrun.runtimes.mounts
30
30
  import mlrun.utils.regex
31
31
  from mlrun.common.schemas import (
32
- NodeSelectorOperator,
33
32
  PreemptionModes,
34
33
  SecurityContextEnrichmentModes,
35
34
  )
36
35
 
37
36
  from ..config import config as mlconf
38
37
  from ..k8s_utils import (
39
- generate_preemptible_node_selector_requirements,
40
- generate_preemptible_nodes_affinity_terms,
41
- generate_preemptible_nodes_anti_affinity_terms,
42
- generate_preemptible_tolerations,
43
38
  validate_node_selectors,
44
39
  )
45
40
  from ..utils import logger, update_in
@@ -292,7 +287,6 @@ class KubeResourceSpec(FunctionSpec):
292
287
  @preemption_mode.setter
293
288
  def preemption_mode(self, mode):
294
289
  self._preemption_mode = mode or mlconf.function_defaults.preemption_mode
295
- self.enrich_function_preemption_spec()
296
290
 
297
291
  @property
298
292
  def security_context(self) -> k8s_client.V1SecurityContext:
@@ -560,300 +554,6 @@ class KubeResourceSpec(FunctionSpec):
560
554
  return {}
561
555
  return resources
562
556
 
563
- def _merge_node_selector(self, node_selector: dict[str, str]):
564
- if not node_selector:
565
- return
566
-
567
- # merge node selectors - precedence to existing node selector
568
- self.node_selector = mlrun.utils.helpers.merge_dicts_with_precedence(
569
- node_selector, self.node_selector
570
- )
571
-
572
- def _merge_tolerations(
573
- self,
574
- tolerations: list[k8s_client.V1Toleration],
575
- tolerations_field_name: str,
576
- ):
577
- if not tolerations:
578
- return
579
- # In case function has no toleration, take all from input
580
- self_tolerations = getattr(self, tolerations_field_name)
581
- if not self_tolerations:
582
- setattr(self, tolerations_field_name, tolerations)
583
- return
584
- tolerations_to_add = []
585
-
586
- # Only add non-matching tolerations to avoid duplications
587
- for toleration in tolerations:
588
- to_add = True
589
- for function_toleration in self_tolerations:
590
- if function_toleration == toleration:
591
- to_add = False
592
- break
593
- if to_add:
594
- tolerations_to_add.append(toleration)
595
-
596
- if len(tolerations_to_add) > 0:
597
- self_tolerations.extend(tolerations_to_add)
598
-
599
- def _override_required_during_scheduling_ignored_during_execution(
600
- self,
601
- node_selector: k8s_client.V1NodeSelector,
602
- affinity_field_name: str,
603
- ):
604
- self._initialize_affinity(affinity_field_name)
605
- self._initialize_node_affinity(affinity_field_name)
606
-
607
- self_affinity = getattr(self, affinity_field_name)
608
- self_affinity.node_affinity.required_during_scheduling_ignored_during_execution = node_selector
609
-
610
- def enrich_function_preemption_spec(
611
- self,
612
- preemption_mode_field_name: str = "preemption_mode",
613
- tolerations_field_name: str = "tolerations",
614
- affinity_field_name: str = "affinity",
615
- node_selector_field_name: str = "node_selector",
616
- ):
617
- """
618
- Enriches function pod with the below described spec.
619
- If no preemptible node configuration is provided, do nothing.
620
- `allow` - Adds Tolerations if configured.
621
- otherwise, assume pods can be scheduled on preemptible nodes.
622
- > Purges any `affinity` / `anti-affinity` preemption related configuration
623
- > Purges preemptible node selector
624
- `constrain` - Uses node-affinity to make sure pods are assigned using OR on the configured
625
- node label selectors.
626
- > Merges tolerations with preemptible tolerations.
627
- > Purges any `anti-affinity` preemption related configuration
628
- `prevent` - Prevention is done either using taints (if Tolerations were configured) or anti-affinity.
629
- > Purges any `tolerations` preemption related configuration
630
- > Purges any `affinity` preemption related configuration
631
- > Purges preemptible node selector
632
- > Sets anti-affinity and overrides any affinity if no tolerations were configured
633
- `none` - Doesn't apply any preemptible node selection configuration.
634
- """
635
- # nothing to do here, configuration is not populated
636
- if not mlconf.is_preemption_nodes_configured():
637
- return
638
-
639
- if not getattr(self, preemption_mode_field_name):
640
- # We're not supposed to get here, but if we do, we'll set the private attribute to
641
- # avoid triggering circular enrichment.
642
- setattr(
643
- self,
644
- f"_{preemption_mode_field_name}",
645
- mlconf.function_defaults.preemption_mode,
646
- )
647
- logger.debug(
648
- "No preemption mode was given, using the default preemption mode",
649
- default_preemption_mode=getattr(self, preemption_mode_field_name),
650
- )
651
- self_preemption_mode = getattr(self, preemption_mode_field_name)
652
- # don't enrich with preemption configuration.
653
- if self_preemption_mode == PreemptionModes.none.value:
654
- return
655
- # remove preemptible tolerations and remove preemption related configuration
656
- # and enrich with anti-affinity if preemptible tolerations configuration haven't been provided
657
- if self_preemption_mode == PreemptionModes.prevent.value:
658
- # ensure no preemptible node tolerations
659
- self._prune_tolerations(
660
- generate_preemptible_tolerations(),
661
- tolerations_field_name=tolerations_field_name,
662
- )
663
-
664
- # purge affinity preemption related configuration
665
- self._prune_affinity_node_selector_requirement(
666
- generate_preemptible_node_selector_requirements(
667
- NodeSelectorOperator.node_selector_op_in.value
668
- ),
669
- affinity_field_name=affinity_field_name,
670
- )
671
- # remove preemptible nodes constrain
672
- self._prune_node_selector(
673
- mlconf.get_preemptible_node_selector(),
674
- node_selector_field_name=node_selector_field_name,
675
- )
676
-
677
- # if tolerations are configured, simply pruning tolerations is sufficient because functions
678
- # cannot be scheduled without tolerations on tainted nodes.
679
- # however, if preemptible tolerations are not configured, we must use anti-affinity on preemptible nodes
680
- # to ensure that the function is not scheduled on the nodes.
681
- if not generate_preemptible_tolerations():
682
- # using a single term with potentially multiple expressions to ensure anti-affinity
683
- self._override_required_during_scheduling_ignored_during_execution(
684
- k8s_client.V1NodeSelector(
685
- node_selector_terms=generate_preemptible_nodes_anti_affinity_terms()
686
- ),
687
- affinity_field_name=affinity_field_name,
688
- )
689
- # enrich tolerations and override all node selector terms with preemptible node selector terms
690
- elif self_preemption_mode == PreemptionModes.constrain.value:
691
- # enrich with tolerations
692
- self._merge_tolerations(
693
- generate_preemptible_tolerations(),
694
- tolerations_field_name=tolerations_field_name,
695
- )
696
-
697
- # setting required_during_scheduling_ignored_during_execution
698
- # overriding other terms that have been set, and only setting terms for preemptible nodes
699
- # when having multiple terms, pod scheduling is succeeded if at least one term is satisfied
700
- self._override_required_during_scheduling_ignored_during_execution(
701
- k8s_client.V1NodeSelector(
702
- node_selector_terms=generate_preemptible_nodes_affinity_terms()
703
- ),
704
- affinity_field_name=affinity_field_name,
705
- )
706
- # purge any affinity / anti-affinity preemption related configuration and enrich with preemptible tolerations
707
- elif self_preemption_mode == PreemptionModes.allow.value:
708
- # remove preemptible anti-affinity
709
- self._prune_affinity_node_selector_requirement(
710
- generate_preemptible_node_selector_requirements(
711
- NodeSelectorOperator.node_selector_op_not_in.value
712
- ),
713
- affinity_field_name=affinity_field_name,
714
- )
715
- # remove preemptible affinity
716
- self._prune_affinity_node_selector_requirement(
717
- generate_preemptible_node_selector_requirements(
718
- NodeSelectorOperator.node_selector_op_in.value
719
- ),
720
- affinity_field_name=affinity_field_name,
721
- )
722
-
723
- # remove preemptible nodes constrain
724
- self._prune_node_selector(
725
- mlconf.get_preemptible_node_selector(),
726
- node_selector_field_name=node_selector_field_name,
727
- )
728
-
729
- # enrich with tolerations
730
- self._merge_tolerations(
731
- generate_preemptible_tolerations(),
732
- tolerations_field_name=tolerations_field_name,
733
- )
734
-
735
- self._clear_affinity_if_initialized_but_empty(
736
- affinity_field_name=affinity_field_name
737
- )
738
- self._clear_tolerations_if_initialized_but_empty(
739
- tolerations_field_name=tolerations_field_name
740
- )
741
-
742
- def _clear_affinity_if_initialized_but_empty(self, affinity_field_name: str):
743
- self_affinity = getattr(self, affinity_field_name)
744
- if not getattr(self, affinity_field_name):
745
- setattr(self, affinity_field_name, None)
746
- elif (
747
- not self_affinity.node_affinity
748
- and not self_affinity.pod_affinity
749
- and not self_affinity.pod_anti_affinity
750
- ):
751
- setattr(self, affinity_field_name, None)
752
-
753
- def _clear_tolerations_if_initialized_but_empty(self, tolerations_field_name: str):
754
- if not getattr(self, tolerations_field_name):
755
- setattr(self, tolerations_field_name, None)
756
-
757
- def _merge_node_selector_term_to_node_affinity(
758
- self,
759
- node_selector_terms: list[k8s_client.V1NodeSelectorTerm],
760
- affinity_field_name: str,
761
- ):
762
- if not node_selector_terms:
763
- return
764
-
765
- self._initialize_affinity(affinity_field_name)
766
- self._initialize_node_affinity(affinity_field_name)
767
-
768
- self_affinity = getattr(self, affinity_field_name)
769
- if not self_affinity.node_affinity.required_during_scheduling_ignored_during_execution:
770
- self_affinity.node_affinity.required_during_scheduling_ignored_during_execution = k8s_client.V1NodeSelector(
771
- node_selector_terms=node_selector_terms
772
- )
773
- return
774
-
775
- node_selector = self_affinity.node_affinity.required_during_scheduling_ignored_during_execution
776
- new_node_selector_terms = []
777
-
778
- for node_selector_term_to_add in node_selector_terms:
779
- to_add = True
780
- for node_selector_term in node_selector.node_selector_terms:
781
- if node_selector_term == node_selector_term_to_add:
782
- to_add = False
783
- break
784
- if to_add:
785
- new_node_selector_terms.append(node_selector_term_to_add)
786
-
787
- if new_node_selector_terms:
788
- node_selector.node_selector_terms += new_node_selector_terms
789
-
790
- def _initialize_affinity(self, affinity_field_name: str):
791
- if not getattr(self, affinity_field_name):
792
- setattr(self, affinity_field_name, k8s_client.V1Affinity())
793
-
794
- def _initialize_node_affinity(self, affinity_field_name: str):
795
- if not getattr(getattr(self, affinity_field_name), "node_affinity"):
796
- # self.affinity.node_affinity:
797
- getattr(
798
- self, affinity_field_name
799
- ).node_affinity = k8s_client.V1NodeAffinity()
800
- # self.affinity.node_affinity = k8s_client.V1NodeAffinity()
801
-
802
- def _prune_affinity_node_selector_requirement(
803
- self,
804
- node_selector_requirements: list[k8s_client.V1NodeSelectorRequirement],
805
- affinity_field_name: str = "affinity",
806
- ):
807
- """
808
- Prunes given node selector requirements from affinity.
809
- We are only editing required_during_scheduling_ignored_during_execution because the scheduler can't schedule
810
- the pod unless the rule is met.
811
- :param node_selector_requirements:
812
- :return:
813
- """
814
- # both needs to exist to prune required affinity from spec affinity
815
- self_affinity = getattr(self, affinity_field_name)
816
- if not self_affinity or not node_selector_requirements:
817
- return
818
- if self_affinity.node_affinity:
819
- node_affinity: k8s_client.V1NodeAffinity = self_affinity.node_affinity
820
-
821
- new_required_during_scheduling_ignored_during_execution = None
822
- if node_affinity.required_during_scheduling_ignored_during_execution:
823
- node_selector: k8s_client.V1NodeSelector = (
824
- node_affinity.required_during_scheduling_ignored_during_execution
825
- )
826
- new_node_selector_terms = (
827
- self._prune_node_selector_requirements_from_node_selector_terms(
828
- node_selector_terms=node_selector.node_selector_terms,
829
- node_selector_requirements_to_prune=node_selector_requirements,
830
- )
831
- )
832
- # check whether there are node selector terms to add to the new list of required terms
833
- if len(new_node_selector_terms) > 0:
834
- new_required_during_scheduling_ignored_during_execution = (
835
- k8s_client.V1NodeSelector(
836
- node_selector_terms=new_node_selector_terms
837
- )
838
- )
839
- # if both preferred and new required are empty, clean node_affinity
840
- if (
841
- not node_affinity.preferred_during_scheduling_ignored_during_execution
842
- and not new_required_during_scheduling_ignored_during_execution
843
- ):
844
- setattr(self_affinity, "node_affinity", None)
845
- # self.affinity.node_affinity = None
846
- return
847
-
848
- self._initialize_affinity(affinity_field_name)
849
- self._initialize_node_affinity(affinity_field_name)
850
-
851
- # fmt: off
852
- self_affinity.node_affinity.required_during_scheduling_ignored_during_execution = (
853
- new_required_during_scheduling_ignored_during_execution
854
- )
855
- # fmt: on
856
-
857
557
  @staticmethod
858
558
  def _prune_node_selector_requirements_from_node_selector_terms(
859
559
  node_selector_terms: list[k8s_client.V1NodeSelectorTerm],
@@ -894,55 +594,6 @@ class KubeResourceSpec(FunctionSpec):
894
594
  )
895
595
  return new_node_selector_terms
896
596
 
897
- def _prune_tolerations(
898
- self,
899
- tolerations: list[k8s_client.V1Toleration],
900
- tolerations_field_name: str = "tolerations",
901
- ):
902
- """
903
- Prunes given tolerations from function spec
904
- :param tolerations: tolerations to prune
905
- """
906
- self_tolerations = getattr(self, tolerations_field_name)
907
- # both needs to exist to prune required tolerations from spec tolerations
908
- if not tolerations or not self_tolerations:
909
- return
910
-
911
- # generate a list of tolerations without tolerations to prune
912
- new_tolerations = []
913
- for toleration in self_tolerations:
914
- to_prune = False
915
- for toleration_to_delete in tolerations:
916
- if toleration == toleration_to_delete:
917
- to_prune = True
918
- # no need to keep going over the list provided for the current toleration
919
- break
920
- if not to_prune:
921
- new_tolerations.append(toleration)
922
-
923
- # Set tolerations without tolerations to prune
924
- setattr(self, tolerations_field_name, new_tolerations)
925
-
926
- def _prune_node_selector(
927
- self,
928
- node_selector: dict[str, str],
929
- node_selector_field_name: str,
930
- ):
931
- """
932
- Prunes given node_selector key from function spec if their key and value are matching
933
- :param node_selector: node selectors to prune
934
- """
935
- self_node_selector = getattr(self, node_selector_field_name)
936
- # both needs to exists to prune required node_selector from the spec node selector
937
- if not node_selector or not self_node_selector:
938
- return
939
-
940
- for key, value in node_selector.items():
941
- if value:
942
- spec_value = self_node_selector.get(key)
943
- if spec_value and spec_value == value:
944
- self_node_selector.pop(key)
945
-
946
597
 
947
598
  class AutoMountType(str, Enum):
948
599
  none = "none"
@@ -1646,6 +1297,10 @@ def get_sanitized_attribute(spec, attribute_name: str):
1646
1297
  if _resolve_if_type_sanitized(attribute_name, attribute[0]):
1647
1298
  return attribute
1648
1299
 
1300
+ return sanitize_attribute(attribute)
1301
+
1302
+
1303
+ def sanitize_attribute(attribute):
1649
1304
  api = k8s_client.ApiClient()
1650
1305
  return api.sanitize_for_serialization(attribute)
1651
1306
 
@@ -288,12 +288,6 @@ class Spark3JobSpec(KubeResourceSpec):
288
288
  self._driver_preemption_mode = (
289
289
  mode or mlrun.mlconf.function_defaults.preemption_mode
290
290
  )
291
- self.enrich_function_preemption_spec(
292
- preemption_mode_field_name="driver_preemption_mode",
293
- tolerations_field_name="driver_tolerations",
294
- affinity_field_name="driver_affinity",
295
- node_selector_field_name="driver_node_selector",
296
- )
297
291
 
298
292
  @property
299
293
  def executor_preemption_mode(self) -> str:
@@ -304,12 +298,6 @@ class Spark3JobSpec(KubeResourceSpec):
304
298
  self._executor_preemption_mode = (
305
299
  mode or mlrun.mlconf.function_defaults.preemption_mode
306
300
  )
307
- self.enrich_function_preemption_spec(
308
- preemption_mode_field_name="executor_preemption_mode",
309
- tolerations_field_name="executor_tolerations",
310
- affinity_field_name="executor_affinity",
311
- node_selector_field_name="executor_node_selector",
312
- )
313
301
 
314
302
  @property
315
303
  def driver_volume_mounts(self) -> list:
mlrun/serving/merger.py CHANGED
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
14
 
16
15
  from typing import Optional
17
16
 
mlrun/serving/remote.py CHANGED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import asyncio
16
16
  import json
17
17
  from copy import copy
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  # serving runtime hooks, used in empty serving functions
16
16
  from mlrun.runtimes import nuclio_init_hook
17
17
 
mlrun/serving/utils.py CHANGED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import inspect
16
16
  from typing import Optional
17
17
 
mlrun/utils/async_http.py CHANGED
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
14
 
16
15
  import asyncio
17
16
  import logging
mlrun/utils/clones.py CHANGED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import os
16
16
  import shutil
17
17
  import tarfile
mlrun/utils/db.py CHANGED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import abc
16
16
  import pickle
17
17
  from datetime import datetime
mlrun/utils/helpers.py CHANGED
@@ -1502,7 +1502,9 @@ def _fill_project_path_template(artifact_path, project):
1502
1502
 
1503
1503
 
1504
1504
  def to_non_empty_values_dict(input_dict: dict) -> dict:
1505
- return {key: value for key, value in input_dict.items() if value}
1505
+ return (
1506
+ {key: value for key, value in input_dict.items() if value} if input_dict else {}
1507
+ )
1506
1508
 
1507
1509
 
1508
1510
  def get_enriched_gpu_limits(function_limits: dict) -> dict[str, int]:
mlrun/utils/http.py CHANGED
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
14
 
16
15
  import time
17
16
 
@@ -16,6 +16,7 @@ import re
16
16
  import typing
17
17
 
18
18
  import aiohttp
19
+ import orjson
19
20
 
20
21
  import mlrun.common.schemas
21
22
  import mlrun.lists
@@ -86,9 +87,14 @@ class WebhookNotification(NotificationBase):
86
87
  # we automatically handle it as `ssl=None` for their convenience.
87
88
  verify_ssl = verify_ssl and None if url.startswith("https") else None
88
89
 
89
- async with aiohttp.ClientSession() as session:
90
+ async with aiohttp.ClientSession(
91
+ json_serialize=self._encoder,
92
+ ) as session:
90
93
  response = await getattr(session, method)(
91
- url, headers=headers, json=request_body, ssl=verify_ssl
94
+ url,
95
+ headers=headers,
96
+ json=request_body,
97
+ ssl=verify_ssl,
92
98
  )
93
99
  response.raise_for_status()
94
100
 
@@ -128,3 +134,13 @@ class WebhookNotification(NotificationBase):
128
134
  )
129
135
 
130
136
  return override_body
137
+
138
+ @property
139
+ def _encoder(self):
140
+ return lambda body: orjson.dumps(
141
+ body,
142
+ option=orjson.OPT_NAIVE_UTC
143
+ | orjson.OPT_SERIALIZE_NUMPY
144
+ | orjson.OPT_NON_STR_KEYS
145
+ | orjson.OPT_SORT_KEYS,
146
+ ).decode()
mlrun/utils/regex.py CHANGED
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
14
 
16
15
  # pipeline param format which is passed when running a pipeline (e.g. {{pipelineparam:op=;name=mem}})
17
16
  # https://github.com/kubeflow/pipelines/blob/16edebf4eaf84cd7478e2601ef4878ab339a7854/sdk/python/kfp/dsl/_pipeline_param.py#L213
mlrun/utils/singleton.py CHANGED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import abc
16
16
 
17
17
 
mlrun/utils/vault.py CHANGED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  # import json
16
16
  # import os
17
17
  # from os.path import expanduser
@@ -11,5 +11,5 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  from .version import Version # noqa: F401
@@ -1,4 +1,4 @@
1
1
  {
2
- "git_commit": "847c2d4b159d98e5e456cbc4898272b9dac77d65",
3
- "version": "1.9.0-rc3"
2
+ "git_commit": "4d532f1a4e8dd427fd5f870863084fa564680bd6",
3
+ "version": "1.10.0-rc1"
4
4
  }
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import json
16
16
  import sys
17
17
  from importlib.resources import read_text
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mlrun
3
- Version: 1.9.0rc3
3
+ Version: 1.10.0rc1
4
4
  Summary: Tracking and config of machine learning runs
5
5
  Home-page: https://github.com/mlrun/mlrun
6
6
  Author: Yaron Haviv
@@ -51,8 +51,8 @@ Requires-Dist: setuptools>=75.2
51
51
  Requires-Dist: deprecated~=1.2
52
52
  Requires-Dist: jinja2>=3.1.3,~=3.1
53
53
  Requires-Dist: orjson<4,>=3.9.15
54
- Requires-Dist: mlrun-pipelines-kfp-common~=0.4.1
55
- Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.4.0
54
+ Requires-Dist: mlrun-pipelines-kfp-common~=0.5.1
55
+ Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.1
56
56
  Requires-Dist: docstring_parser~=0.16
57
57
  Requires-Dist: aiosmtplib~=3.0
58
58
  Provides-Extra: s3
@@ -99,11 +99,10 @@ Requires-Dist: ossfs==2023.12.0; extra == "alibaba-oss"
99
99
  Requires-Dist: oss2==2.18.1; extra == "alibaba-oss"
100
100
  Provides-Extra: tdengine
101
101
  Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
102
- Requires-Dist: taoswswrap~=0.3.5; extra == "tdengine"
103
102
  Provides-Extra: snowflake
104
103
  Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
105
104
  Provides-Extra: kfp18
106
- Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.4.0; python_version < "3.11" and extra == "kfp18"
105
+ Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.5.0; python_version < "3.11" and extra == "kfp18"
107
106
  Provides-Extra: api
108
107
  Requires-Dist: uvicorn~=0.32.1; extra == "api"
109
108
  Requires-Dist: dask-kubernetes~=0.11.0; extra == "api"
@@ -119,7 +118,7 @@ Requires-Dist: timelength~=1.1; extra == "api"
119
118
  Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
120
119
  Requires-Dist: aiosmtplib~=3.0; extra == "api"
121
120
  Requires-Dist: pydantic<2,>=1; extra == "api"
122
- Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.4.0; extra == "api"
121
+ Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.1; extra == "api"
123
122
  Requires-Dist: grpcio~=1.70.0; extra == "api"
124
123
  Provides-Extra: all
125
124
  Requires-Dist: adlfs==2023.9.0; extra == "all"
@@ -152,7 +151,6 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "all"
152
151
  Requires-Dist: snowflake-connector-python~=3.7; extra == "all"
153
152
  Requires-Dist: sqlalchemy~=1.4; extra == "all"
154
153
  Requires-Dist: taos-ws-py==0.3.2; extra == "all"
155
- Requires-Dist: taoswswrap~=0.3.5; extra == "all"
156
154
  Provides-Extra: complete
157
155
  Requires-Dist: adlfs==2023.9.0; extra == "complete"
158
156
  Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete"
@@ -184,7 +182,6 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete"
184
182
  Requires-Dist: snowflake-connector-python~=3.7; extra == "complete"
185
183
  Requires-Dist: sqlalchemy~=1.4; extra == "complete"
186
184
  Requires-Dist: taos-ws-py==0.3.2; extra == "complete"
187
- Requires-Dist: taoswswrap~=0.3.5; extra == "complete"
188
185
  Provides-Extra: complete-api
189
186
  Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
190
187
  Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
@@ -215,7 +212,7 @@ Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
215
212
  Requires-Dist: kafka-python~=2.0; extra == "complete-api"
216
213
  Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "complete-api"
217
214
  Requires-Dist: mlflow~=2.16; extra == "complete-api"
218
- Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.4.0; extra == "complete-api"
215
+ Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.1; extra == "complete-api"
219
216
  Requires-Dist: msrest~=0.6.21; extra == "complete-api"
220
217
  Requires-Dist: objgraph~=3.6; extra == "complete-api"
221
218
  Requires-Dist: oss2==2.18.1; extra == "complete-api"
@@ -229,7 +226,6 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete-api"
229
226
  Requires-Dist: snowflake-connector-python~=3.7; extra == "complete-api"
230
227
  Requires-Dist: sqlalchemy~=1.4; extra == "complete-api"
231
228
  Requires-Dist: taos-ws-py==0.3.2; extra == "complete-api"
232
- Requires-Dist: taoswswrap~=0.3.5; extra == "complete-api"
233
229
  Requires-Dist: timelength~=1.1; extra == "complete-api"
234
230
  Requires-Dist: uvicorn~=0.32.1; extra == "complete-api"
235
231
  Dynamic: author