mlrun 1.6.0rc35__py3-none-any.whl → 1.7.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (199) hide show
  1. mlrun/__main__.py +3 -3
  2. mlrun/api/schemas/__init__.py +1 -1
  3. mlrun/artifacts/base.py +11 -6
  4. mlrun/artifacts/dataset.py +2 -2
  5. mlrun/artifacts/model.py +30 -24
  6. mlrun/artifacts/plots.py +2 -2
  7. mlrun/common/db/sql_session.py +5 -3
  8. mlrun/common/helpers.py +1 -2
  9. mlrun/common/schemas/artifact.py +3 -3
  10. mlrun/common/schemas/auth.py +3 -3
  11. mlrun/common/schemas/background_task.py +1 -1
  12. mlrun/common/schemas/client_spec.py +1 -1
  13. mlrun/common/schemas/feature_store.py +16 -16
  14. mlrun/common/schemas/frontend_spec.py +7 -7
  15. mlrun/common/schemas/function.py +1 -1
  16. mlrun/common/schemas/hub.py +4 -9
  17. mlrun/common/schemas/memory_reports.py +2 -2
  18. mlrun/common/schemas/model_monitoring/grafana.py +4 -4
  19. mlrun/common/schemas/model_monitoring/model_endpoints.py +14 -15
  20. mlrun/common/schemas/notification.py +4 -4
  21. mlrun/common/schemas/object.py +2 -2
  22. mlrun/common/schemas/pipeline.py +1 -1
  23. mlrun/common/schemas/project.py +3 -3
  24. mlrun/common/schemas/runtime_resource.py +8 -12
  25. mlrun/common/schemas/schedule.py +3 -3
  26. mlrun/common/schemas/tag.py +1 -2
  27. mlrun/common/schemas/workflow.py +2 -2
  28. mlrun/config.py +8 -4
  29. mlrun/data_types/to_pandas.py +1 -3
  30. mlrun/datastore/base.py +0 -28
  31. mlrun/datastore/datastore_profile.py +9 -9
  32. mlrun/datastore/filestore.py +0 -1
  33. mlrun/datastore/google_cloud_storage.py +1 -1
  34. mlrun/datastore/sources.py +7 -11
  35. mlrun/datastore/spark_utils.py +1 -2
  36. mlrun/datastore/targets.py +31 -31
  37. mlrun/datastore/utils.py +4 -6
  38. mlrun/datastore/v3io.py +70 -46
  39. mlrun/db/base.py +22 -23
  40. mlrun/db/httpdb.py +34 -34
  41. mlrun/db/nopdb.py +19 -19
  42. mlrun/errors.py +1 -1
  43. mlrun/execution.py +4 -4
  44. mlrun/feature_store/api.py +20 -21
  45. mlrun/feature_store/common.py +1 -1
  46. mlrun/feature_store/feature_set.py +28 -32
  47. mlrun/feature_store/feature_vector.py +24 -27
  48. mlrun/feature_store/retrieval/base.py +7 -7
  49. mlrun/feature_store/retrieval/conversion.py +2 -4
  50. mlrun/feature_store/steps.py +7 -15
  51. mlrun/features.py +5 -7
  52. mlrun/frameworks/_common/artifacts_library.py +9 -9
  53. mlrun/frameworks/_common/mlrun_interface.py +5 -5
  54. mlrun/frameworks/_common/model_handler.py +48 -48
  55. mlrun/frameworks/_common/plan.py +2 -3
  56. mlrun/frameworks/_common/producer.py +3 -4
  57. mlrun/frameworks/_common/utils.py +5 -5
  58. mlrun/frameworks/_dl_common/loggers/logger.py +6 -7
  59. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +9 -9
  60. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +16 -35
  61. mlrun/frameworks/_ml_common/artifacts_library.py +1 -2
  62. mlrun/frameworks/_ml_common/loggers/logger.py +3 -4
  63. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +4 -5
  64. mlrun/frameworks/_ml_common/model_handler.py +24 -24
  65. mlrun/frameworks/_ml_common/pkl_model_server.py +2 -2
  66. mlrun/frameworks/_ml_common/plan.py +1 -1
  67. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +2 -3
  68. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +2 -3
  69. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  70. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +3 -3
  71. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  72. mlrun/frameworks/_ml_common/utils.py +4 -4
  73. mlrun/frameworks/auto_mlrun/auto_mlrun.py +7 -7
  74. mlrun/frameworks/huggingface/model_server.py +4 -4
  75. mlrun/frameworks/lgbm/__init__.py +32 -32
  76. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -5
  77. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -5
  78. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -3
  79. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +6 -6
  80. mlrun/frameworks/lgbm/model_handler.py +9 -9
  81. mlrun/frameworks/lgbm/model_server.py +6 -6
  82. mlrun/frameworks/lgbm/utils.py +5 -5
  83. mlrun/frameworks/onnx/dataset.py +8 -8
  84. mlrun/frameworks/onnx/mlrun_interface.py +3 -3
  85. mlrun/frameworks/onnx/model_handler.py +6 -6
  86. mlrun/frameworks/onnx/model_server.py +7 -7
  87. mlrun/frameworks/parallel_coordinates.py +2 -2
  88. mlrun/frameworks/pytorch/__init__.py +16 -16
  89. mlrun/frameworks/pytorch/callbacks/callback.py +4 -5
  90. mlrun/frameworks/pytorch/callbacks/logging_callback.py +17 -17
  91. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +11 -11
  92. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +23 -29
  93. mlrun/frameworks/pytorch/callbacks_handler.py +38 -38
  94. mlrun/frameworks/pytorch/mlrun_interface.py +20 -20
  95. mlrun/frameworks/pytorch/model_handler.py +17 -17
  96. mlrun/frameworks/pytorch/model_server.py +7 -7
  97. mlrun/frameworks/sklearn/__init__.py +12 -12
  98. mlrun/frameworks/sklearn/estimator.py +4 -4
  99. mlrun/frameworks/sklearn/metrics_library.py +14 -14
  100. mlrun/frameworks/sklearn/mlrun_interface.py +3 -6
  101. mlrun/frameworks/sklearn/model_handler.py +2 -2
  102. mlrun/frameworks/tf_keras/__init__.py +5 -5
  103. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +14 -14
  104. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +11 -11
  105. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +19 -23
  106. mlrun/frameworks/tf_keras/mlrun_interface.py +7 -9
  107. mlrun/frameworks/tf_keras/model_handler.py +14 -14
  108. mlrun/frameworks/tf_keras/model_server.py +6 -6
  109. mlrun/frameworks/xgboost/__init__.py +12 -12
  110. mlrun/frameworks/xgboost/model_handler.py +6 -6
  111. mlrun/k8s_utils.py +4 -5
  112. mlrun/kfpops.py +2 -2
  113. mlrun/launcher/base.py +10 -10
  114. mlrun/launcher/local.py +8 -8
  115. mlrun/launcher/remote.py +7 -7
  116. mlrun/lists.py +3 -4
  117. mlrun/model.py +205 -55
  118. mlrun/model_monitoring/api.py +21 -24
  119. mlrun/model_monitoring/application.py +4 -4
  120. mlrun/model_monitoring/batch.py +17 -17
  121. mlrun/model_monitoring/controller.py +2 -1
  122. mlrun/model_monitoring/features_drift_table.py +44 -31
  123. mlrun/model_monitoring/prometheus.py +1 -4
  124. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +11 -13
  125. mlrun/model_monitoring/stores/model_endpoint_store.py +9 -11
  126. mlrun/model_monitoring/stores/models/__init__.py +2 -2
  127. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +11 -13
  128. mlrun/model_monitoring/stream_processing.py +16 -34
  129. mlrun/model_monitoring/tracking_policy.py +2 -1
  130. mlrun/package/__init__.py +6 -6
  131. mlrun/package/context_handler.py +5 -5
  132. mlrun/package/packager.py +7 -7
  133. mlrun/package/packagers/default_packager.py +6 -6
  134. mlrun/package/packagers/numpy_packagers.py +15 -15
  135. mlrun/package/packagers/pandas_packagers.py +5 -5
  136. mlrun/package/packagers/python_standard_library_packagers.py +10 -10
  137. mlrun/package/packagers_manager.py +18 -23
  138. mlrun/package/utils/_formatter.py +4 -4
  139. mlrun/package/utils/_pickler.py +2 -2
  140. mlrun/package/utils/_supported_format.py +4 -4
  141. mlrun/package/utils/log_hint_utils.py +2 -2
  142. mlrun/package/utils/type_hint_utils.py +4 -9
  143. mlrun/platforms/other.py +1 -2
  144. mlrun/projects/operations.py +5 -5
  145. mlrun/projects/pipelines.py +9 -9
  146. mlrun/projects/project.py +58 -46
  147. mlrun/render.py +1 -1
  148. mlrun/run.py +9 -9
  149. mlrun/runtimes/__init__.py +7 -4
  150. mlrun/runtimes/base.py +20 -23
  151. mlrun/runtimes/constants.py +5 -5
  152. mlrun/runtimes/daskjob.py +8 -8
  153. mlrun/runtimes/databricks_job/databricks_cancel_task.py +1 -1
  154. mlrun/runtimes/databricks_job/databricks_runtime.py +7 -7
  155. mlrun/runtimes/function_reference.py +1 -1
  156. mlrun/runtimes/local.py +1 -1
  157. mlrun/runtimes/mpijob/abstract.py +1 -2
  158. mlrun/runtimes/nuclio/__init__.py +20 -0
  159. mlrun/runtimes/{function.py → nuclio/function.py} +15 -16
  160. mlrun/runtimes/{nuclio.py → nuclio/nuclio.py} +6 -6
  161. mlrun/runtimes/{serving.py → nuclio/serving.py} +13 -12
  162. mlrun/runtimes/pod.py +95 -48
  163. mlrun/runtimes/remotesparkjob.py +1 -1
  164. mlrun/runtimes/sparkjob/spark3job.py +50 -33
  165. mlrun/runtimes/utils.py +1 -2
  166. mlrun/secrets.py +3 -3
  167. mlrun/serving/remote.py +0 -4
  168. mlrun/serving/routers.py +6 -6
  169. mlrun/serving/server.py +4 -4
  170. mlrun/serving/states.py +29 -0
  171. mlrun/serving/utils.py +3 -3
  172. mlrun/serving/v1_serving.py +6 -7
  173. mlrun/serving/v2_serving.py +50 -8
  174. mlrun/track/tracker_manager.py +3 -3
  175. mlrun/track/trackers/mlflow_tracker.py +1 -2
  176. mlrun/utils/async_http.py +5 -7
  177. mlrun/utils/azure_vault.py +1 -1
  178. mlrun/utils/clones.py +1 -2
  179. mlrun/utils/condition_evaluator.py +3 -3
  180. mlrun/utils/db.py +3 -3
  181. mlrun/utils/helpers.py +37 -119
  182. mlrun/utils/http.py +1 -4
  183. mlrun/utils/logger.py +49 -14
  184. mlrun/utils/notifications/notification/__init__.py +3 -3
  185. mlrun/utils/notifications/notification/base.py +2 -2
  186. mlrun/utils/notifications/notification/ipython.py +1 -1
  187. mlrun/utils/notifications/notification_pusher.py +8 -14
  188. mlrun/utils/retryer.py +207 -0
  189. mlrun/utils/singleton.py +1 -1
  190. mlrun/utils/v3io_clients.py +2 -3
  191. mlrun/utils/version/version.json +2 -2
  192. mlrun/utils/version/version.py +2 -6
  193. {mlrun-1.6.0rc35.dist-info → mlrun-1.7.0rc2.dist-info}/METADATA +9 -9
  194. mlrun-1.7.0rc2.dist-info/RECORD +315 -0
  195. mlrun-1.6.0rc35.dist-info/RECORD +0 -313
  196. {mlrun-1.6.0rc35.dist-info → mlrun-1.7.0rc2.dist-info}/LICENSE +0 -0
  197. {mlrun-1.6.0rc35.dist-info → mlrun-1.7.0rc2.dist-info}/WHEEL +0 -0
  198. {mlrun-1.6.0rc35.dist-info → mlrun-1.7.0rc2.dist-info}/entry_points.txt +0 -0
  199. {mlrun-1.6.0rc35.dist-info → mlrun-1.7.0rc2.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py CHANGED
@@ -25,7 +25,7 @@ import uuid
25
25
  import warnings
26
26
  import zipfile
27
27
  from os import environ, makedirs, path, remove
28
- from typing import Callable, Dict, List, Optional, Union
28
+ from typing import Callable, Optional, Union
29
29
 
30
30
  import dotenv
31
31
  import git
@@ -46,6 +46,7 @@ import mlrun.runtimes.pod
46
46
  import mlrun.runtimes.utils
47
47
  import mlrun.utils.regex
48
48
  from mlrun.datastore.datastore_profile import DatastoreProfile, DatastoreProfile2Json
49
+ from mlrun.runtimes.nuclio.function import RemoteRuntime
49
50
 
50
51
  from ..artifacts import Artifact, ArtifactProducer, DatasetArtifact, ModelArtifact
51
52
  from ..artifacts.manager import ArtifactManager, dict_to_artifact, extend_artifact_path
@@ -57,7 +58,6 @@ from ..model_monitoring.application import (
57
58
  PushToMonitoringWriter,
58
59
  )
59
60
  from ..run import code_to_function, get_object, import_function, new_function
60
- from ..runtimes.function import RemoteRuntime
61
61
  from ..secrets import SecretsStore
62
62
  from ..utils import (
63
63
  is_ipython,
@@ -580,24 +580,36 @@ def _run_project_setup(
580
580
 
581
581
  def _load_project_dir(context, name="", subpath=""):
582
582
  subpath_str = subpath or ""
583
- fpath = path.join(context, subpath_str, "project.yaml")
583
+
584
+ # support both .yaml and .yml file extensions
585
+ project_file_path = path.join(context, subpath_str, "project.y*ml")
586
+ function_file_path = path.join(context, subpath_str, "function.y*ml")
584
587
  setup_file_path = path.join(context, subpath_str, "project_setup.py")
585
- if path.isfile(fpath):
586
- with open(fpath) as fp:
588
+
589
+ if project_files := glob.glob(project_file_path):
590
+ # if there are multiple project files, use the first one
591
+ project_file_path = project_files[0]
592
+ with open(project_file_path) as fp:
587
593
  data = fp.read()
588
594
  struct = yaml.load(data, Loader=yaml.FullLoader)
589
595
  project = _project_instance_from_struct(struct, name)
590
596
  project.spec.context = context
591
-
592
- elif path.isfile(path.join(context, subpath_str, "function.yaml")):
593
- func = import_function(path.join(context, subpath_str, "function.yaml"))
597
+ elif function_files := glob.glob(function_file_path):
598
+ function_path = function_files[0]
599
+ func = import_function(function_path)
600
+ function_file_name = path.basename(path.normpath(function_path))
594
601
  project = MlrunProject.from_dict(
595
602
  {
596
603
  "metadata": {
597
604
  "name": func.metadata.project,
598
605
  },
599
606
  "spec": {
600
- "functions": [{"url": "function.yaml", "name": func.metadata.name}],
607
+ "functions": [
608
+ {
609
+ "url": function_file_name,
610
+ "name": func.metadata.name,
611
+ },
612
+ ],
601
613
  },
602
614
  }
603
615
  )
@@ -735,14 +747,14 @@ class ProjectSpec(ModelObj):
735
747
  origin_url=None,
736
748
  goals=None,
737
749
  load_source_on_run=None,
738
- default_requirements: typing.Union[str, typing.List[str]] = None,
750
+ default_requirements: typing.Union[str, list[str]] = None,
739
751
  desired_state=mlrun.common.schemas.ProjectState.online.value,
740
752
  owner=None,
741
753
  disable_auto_mount=None,
742
754
  workdir=None,
743
755
  default_image=None,
744
756
  build=None,
745
- custom_packagers: typing.List[typing.Tuple[str, bool]] = None,
757
+ custom_packagers: list[tuple[str, bool]] = None,
746
758
  ):
747
759
  self.repo = None
748
760
 
@@ -859,14 +871,14 @@ class ProjectSpec(ModelObj):
859
871
  del self._function_definitions[name]
860
872
 
861
873
  @property
862
- def workflows(self) -> typing.List[dict]:
874
+ def workflows(self) -> list[dict]:
863
875
  """
864
876
  :returns: list of workflows specs dicts used in this project
865
877
  """
866
878
  return [workflow.to_dict() for workflow in self._workflows.values()]
867
879
 
868
880
  @workflows.setter
869
- def workflows(self, workflows: typing.List[typing.Union[dict, WorkflowSpec]]):
881
+ def workflows(self, workflows: list[typing.Union[dict, WorkflowSpec]]):
870
882
  if not workflows:
871
883
  workflows = []
872
884
  if not isinstance(workflows, list):
@@ -982,7 +994,7 @@ class ProjectSpec(ModelObj):
982
994
  :raise MLRunInvalidArgumentError: In case the packager was not in the list.
983
995
  """
984
996
  # Look for the packager tuple in the list to remove it:
985
- packager_tuple: typing.Tuple[str, bool] = None
997
+ packager_tuple: tuple[str, bool] = None
986
998
  for custom_packager in self.custom_packagers:
987
999
  if custom_packager[0] == packager:
988
1000
  packager_tuple = custom_packager
@@ -1033,8 +1045,8 @@ class MlrunProject(ModelObj):
1033
1045
 
1034
1046
  def __init__(
1035
1047
  self,
1036
- metadata: Optional[Union[ProjectMetadata, Dict]] = None,
1037
- spec: Optional[Union[ProjectSpec, Dict]] = None,
1048
+ metadata: Optional[Union[ProjectMetadata, dict]] = None,
1049
+ spec: Optional[Union[ProjectSpec, dict]] = None,
1038
1050
  ):
1039
1051
  self.metadata: ProjectMetadata = metadata
1040
1052
  self.spec: ProjectSpec = spec
@@ -1237,7 +1249,7 @@ class MlrunProject(ModelObj):
1237
1249
  workflow_path: str,
1238
1250
  embed=False,
1239
1251
  engine=None,
1240
- args_schema: typing.List[EntrypointParam] = None,
1252
+ args_schema: list[EntrypointParam] = None,
1241
1253
  handler=None,
1242
1254
  schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
1243
1255
  ttl=None,
@@ -1282,7 +1294,7 @@ class MlrunProject(ModelObj):
1282
1294
  and not workflow_path.startswith(self.context)
1283
1295
  ):
1284
1296
  workflow_path = path.join(self.context, workflow_path)
1285
- with open(workflow_path, "r") as fp:
1297
+ with open(workflow_path) as fp:
1286
1298
  txt = fp.read()
1287
1299
  workflow = {"name": name, "code": txt}
1288
1300
  else:
@@ -1409,7 +1421,7 @@ class MlrunProject(ModelObj):
1409
1421
  self,
1410
1422
  url: str,
1411
1423
  check_path_in_context: bool = False,
1412
- ) -> typing.Tuple[str, bool]:
1424
+ ) -> tuple[str, bool]:
1413
1425
  """
1414
1426
  Get the absolute path of the artifact or function file
1415
1427
  :param url: remote url, absolute path or relative path
@@ -1591,8 +1603,8 @@ class MlrunProject(ModelObj):
1591
1603
  artifact_path=None,
1592
1604
  upload=None,
1593
1605
  labels=None,
1594
- inputs: typing.List[Feature] = None,
1595
- outputs: typing.List[Feature] = None,
1606
+ inputs: list[Feature] = None,
1607
+ outputs: list[Feature] = None,
1596
1608
  feature_vector: str = None,
1597
1609
  feature_weights: list = None,
1598
1610
  training_set=None,
@@ -1716,7 +1728,7 @@ class MlrunProject(ModelObj):
1716
1728
  with tempfile.TemporaryDirectory() as temp_dir:
1717
1729
  with zipfile.ZipFile(item_file, "r") as zf:
1718
1730
  zf.extractall(temp_dir)
1719
- with open(f"{temp_dir}/_spec.yaml", "r") as fp:
1731
+ with open(f"{temp_dir}/_spec.yaml") as fp:
1720
1732
  data = fp.read()
1721
1733
  spec = yaml.load(data, Loader=yaml.FullLoader)
1722
1734
  artifact = get_artifact(spec)
@@ -1785,7 +1797,7 @@ class MlrunProject(ModelObj):
1785
1797
  handler=None,
1786
1798
  with_repo: bool = None,
1787
1799
  tag: str = None,
1788
- requirements: typing.Union[str, typing.List[str]] = None,
1800
+ requirements: typing.Union[str, list[str]] = None,
1789
1801
  requirements_file: str = "",
1790
1802
  **application_kwargs,
1791
1803
  ) -> mlrun.runtimes.BaseRuntime:
@@ -1855,7 +1867,7 @@ class MlrunProject(ModelObj):
1855
1867
  handler: str = None,
1856
1868
  with_repo: bool = None,
1857
1869
  tag: str = None,
1858
- requirements: typing.Union[str, typing.List[str]] = None,
1870
+ requirements: typing.Union[str, list[str]] = None,
1859
1871
  requirements_file: str = "",
1860
1872
  **application_kwargs,
1861
1873
  ) -> mlrun.runtimes.BaseRuntime:
@@ -1906,10 +1918,10 @@ class MlrunProject(ModelObj):
1906
1918
  handler: str = None,
1907
1919
  with_repo: bool = None,
1908
1920
  tag: str = None,
1909
- requirements: typing.Union[str, typing.List[str]] = None,
1921
+ requirements: typing.Union[str, list[str]] = None,
1910
1922
  requirements_file: str = "",
1911
1923
  **application_kwargs,
1912
- ) -> typing.Tuple[str, mlrun.runtimes.BaseRuntime, dict]:
1924
+ ) -> tuple[str, mlrun.runtimes.BaseRuntime, dict]:
1913
1925
  function_object: RemoteRuntime = None
1914
1926
  kind = None
1915
1927
  if (isinstance(func, str) or func is None) and application_class is not None:
@@ -2013,7 +2025,7 @@ class MlrunProject(ModelObj):
2013
2025
  handler: str = None,
2014
2026
  with_repo: bool = None,
2015
2027
  tag: str = None,
2016
- requirements: typing.Union[str, typing.List[str]] = None,
2028
+ requirements: typing.Union[str, list[str]] = None,
2017
2029
  requirements_file: str = "",
2018
2030
  ) -> mlrun.runtimes.BaseRuntime:
2019
2031
  """update or add a function object to the project
@@ -2089,9 +2101,9 @@ class MlrunProject(ModelObj):
2089
2101
  handler: str = None,
2090
2102
  with_repo: bool = None,
2091
2103
  tag: str = None,
2092
- requirements: typing.Union[str, typing.List[str]] = None,
2104
+ requirements: typing.Union[str, list[str]] = None,
2093
2105
  requirements_file: str = "",
2094
- ) -> typing.Tuple[str, str, mlrun.runtimes.BaseRuntime, dict]:
2106
+ ) -> tuple[str, str, mlrun.runtimes.BaseRuntime, dict]:
2095
2107
  if func is None and not _has_module(handler, kind):
2096
2108
  # if function path is not provided and it is not a module (no ".")
2097
2109
  # use the current notebook as default
@@ -2288,7 +2300,7 @@ class MlrunProject(ModelObj):
2288
2300
  self.sync_functions()
2289
2301
  return FunctionsDict(self)
2290
2302
 
2291
- def get_function_names(self) -> typing.List[str]:
2303
+ def get_function_names(self) -> list[str]:
2292
2304
  """get a list of all the project function names"""
2293
2305
  return [func["name"] for func in self.spec.functions]
2294
2306
 
@@ -2578,7 +2590,7 @@ class MlrunProject(ModelObj):
2578
2590
  self,
2579
2591
  name: str = None,
2580
2592
  workflow_path: str = None,
2581
- arguments: typing.Dict[str, typing.Any] = None,
2593
+ arguments: dict[str, typing.Any] = None,
2582
2594
  artifact_path: str = None,
2583
2595
  workflow_handler: typing.Union[str, typing.Callable] = None,
2584
2596
  namespace: str = None,
@@ -2593,7 +2605,7 @@ class MlrunProject(ModelObj):
2593
2605
  timeout: int = None,
2594
2606
  source: str = None,
2595
2607
  cleanup_ttl: int = None,
2596
- notifications: typing.List[mlrun.model.Notification] = None,
2608
+ notifications: list[mlrun.model.Notification] = None,
2597
2609
  ) -> _PipelineRunStatus:
2598
2610
  """run a workflow using kubeflow pipelines
2599
2611
 
@@ -2863,7 +2875,7 @@ class MlrunProject(ModelObj):
2863
2875
  hyperparams: dict = None,
2864
2876
  hyper_param_options: mlrun.model.HyperParamOptions = None,
2865
2877
  inputs: dict = None,
2866
- outputs: typing.List[str] = None,
2878
+ outputs: list[str] = None,
2867
2879
  workdir: str = "",
2868
2880
  labels: dict = None,
2869
2881
  base_task: mlrun.model.RunTemplate = None,
@@ -2874,8 +2886,8 @@ class MlrunProject(ModelObj):
2874
2886
  auto_build: bool = None,
2875
2887
  schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
2876
2888
  artifact_path: str = None,
2877
- notifications: typing.List[mlrun.model.Notification] = None,
2878
- returns: Optional[List[Union[str, Dict[str, str]]]] = None,
2889
+ notifications: list[mlrun.model.Notification] = None,
2890
+ returns: Optional[list[Union[str, dict[str, str]]]] = None,
2879
2891
  builder_env: Optional[dict] = None,
2880
2892
  ) -> typing.Union[mlrun.model.RunObject, kfp.dsl.ContainerOp]:
2881
2893
  """Run a local or remote task as part of a local/kubeflow pipeline
@@ -2966,7 +2978,7 @@ class MlrunProject(ModelObj):
2966
2978
  base_image: str = None,
2967
2979
  commands: list = None,
2968
2980
  secret_name: str = None,
2969
- requirements: typing.Union[str, typing.List[str]] = None,
2981
+ requirements: typing.Union[str, list[str]] = None,
2970
2982
  mlrun_version_specifier: str = None,
2971
2983
  builder_env: dict = None,
2972
2984
  overwrite_build_params: bool = False,
@@ -3022,7 +3034,7 @@ class MlrunProject(ModelObj):
3022
3034
  base_image: str = None,
3023
3035
  commands: list = None,
3024
3036
  secret_name: str = None,
3025
- requirements: typing.Union[str, typing.List[str]] = None,
3037
+ requirements: typing.Union[str, list[str]] = None,
3026
3038
  overwrite_build_params: bool = False,
3027
3039
  requirements_file: str = None,
3028
3040
  builder_env: dict = None,
@@ -3084,7 +3096,7 @@ class MlrunProject(ModelObj):
3084
3096
  base_image: str = None,
3085
3097
  commands: list = None,
3086
3098
  secret_name: str = None,
3087
- requirements: typing.Union[str, typing.List[str]] = None,
3099
+ requirements: typing.Union[str, list[str]] = None,
3088
3100
  mlrun_version_specifier: str = None,
3089
3101
  builder_env: dict = None,
3090
3102
  overwrite_build_params: bool = False,
@@ -3247,7 +3259,7 @@ class MlrunProject(ModelObj):
3247
3259
  self,
3248
3260
  name=None,
3249
3261
  tag=None,
3250
- labels: Optional[Union[Dict[str, str], List[str]]] = None,
3262
+ labels: Optional[Union[dict[str, str], list[str]]] = None,
3251
3263
  since=None,
3252
3264
  until=None,
3253
3265
  iter: int = None,
@@ -3304,7 +3316,7 @@ class MlrunProject(ModelObj):
3304
3316
  self,
3305
3317
  name=None,
3306
3318
  tag=None,
3307
- labels: Optional[Union[Dict[str, str], List[str]]] = None,
3319
+ labels: Optional[Union[dict[str, str], list[str]]] = None,
3308
3320
  since=None,
3309
3321
  until=None,
3310
3322
  iter: int = None,
@@ -3400,8 +3412,8 @@ class MlrunProject(ModelObj):
3400
3412
  def list_runs(
3401
3413
  self,
3402
3414
  name: Optional[str] = None,
3403
- uid: Optional[Union[str, List[str]]] = None,
3404
- labels: Optional[Union[str, List[str]]] = None,
3415
+ uid: Optional[Union[str, list[str]]] = None,
3416
+ labels: Optional[Union[str, list[str]]] = None,
3405
3417
  state: Optional[str] = None,
3406
3418
  sort: bool = True,
3407
3419
  last: int = 0,
@@ -3489,7 +3501,7 @@ class MlrunProject(ModelObj):
3489
3501
  profile, self.name
3490
3502
  )
3491
3503
 
3492
- def list_datastore_profiles(self) -> List[DatastoreProfile]:
3504
+ def list_datastore_profiles(self) -> list[DatastoreProfile]:
3493
3505
  """
3494
3506
  Returns a list of datastore profiles associated with the project.
3495
3507
  The information excludes private details, showcasing only public data.
@@ -3498,7 +3510,7 @@ class MlrunProject(ModelObj):
3498
3510
  self.name
3499
3511
  )
3500
3512
 
3501
- def get_custom_packagers(self) -> typing.List[typing.Tuple[str, bool]]:
3513
+ def get_custom_packagers(self) -> list[tuple[str, bool]]:
3502
3514
  """
3503
3515
  Get the custom packagers registered in the project.
3504
3516
 
@@ -3621,7 +3633,7 @@ def _init_function_from_dict(
3621
3633
  f: dict,
3622
3634
  project: MlrunProject,
3623
3635
  name: typing.Optional[str] = None,
3624
- ) -> typing.Tuple[str, mlrun.runtimes.BaseRuntime]:
3636
+ ) -> tuple[str, mlrun.runtimes.BaseRuntime]:
3625
3637
  name = name or f.get("name", "")
3626
3638
  url = f.get("url", "")
3627
3639
  kind = f.get("kind", "")
@@ -3716,7 +3728,7 @@ def _init_function_from_obj(
3716
3728
  func: mlrun.runtimes.BaseRuntime,
3717
3729
  project: MlrunProject,
3718
3730
  name: typing.Optional[str] = None,
3719
- ) -> typing.Tuple[str, mlrun.runtimes.BaseRuntime]:
3731
+ ) -> tuple[str, mlrun.runtimes.BaseRuntime]:
3720
3732
  build = func.spec.build
3721
3733
  if project.spec.origin_url:
3722
3734
  origin = project.spec.origin_url
mlrun/render.py CHANGED
@@ -105,7 +105,7 @@ def link_html(text, link=""):
105
105
 
106
106
 
107
107
  def artifacts_html(
108
- artifacts: typing.List[dict],
108
+ artifacts: list[dict],
109
109
  attribute_name: str = "path",
110
110
  ):
111
111
  """
mlrun/run.py CHANGED
@@ -24,7 +24,7 @@ from base64 import b64decode
24
24
  from copy import deepcopy
25
25
  from os import environ, makedirs, path
26
26
  from pathlib import Path
27
- from typing import Dict, List, Optional, Tuple, Union
27
+ from typing import Optional, Union
28
28
 
29
29
  import nuclio
30
30
  import yaml
@@ -34,6 +34,7 @@ import mlrun.common.schemas
34
34
  import mlrun.errors
35
35
  import mlrun.utils.helpers
36
36
  from mlrun.kfpops import format_summary_from_kfp_run, show_kfp_run
37
+ from mlrun.runtimes.nuclio.serving import serving_subkind
37
38
 
38
39
  from .common.helpers import parse_versioned_object_uri
39
40
  from .config import config as mlconf
@@ -57,7 +58,6 @@ from .runtimes import (
57
58
  )
58
59
  from .runtimes.databricks_job.databricks_runtime import DatabricksRuntime
59
60
  from .runtimes.funcdoc import update_function_entry_points
60
- from .runtimes.serving import serving_subkind
61
61
  from .runtimes.utils import add_code_metadata, global_context
62
62
  from .utils import (
63
63
  extend_hub_uri_if_needed,
@@ -69,7 +69,7 @@ from .utils import (
69
69
  )
70
70
 
71
71
 
72
- class RunStatuses(object):
72
+ class RunStatuses:
73
73
  succeeded = "Succeeded"
74
74
  failed = "Failed"
75
75
  skipped = "Skipped"
@@ -436,7 +436,7 @@ def new_function(
436
436
  mode=None,
437
437
  handler: str = None,
438
438
  source: str = None,
439
- requirements: Union[str, List[str]] = None,
439
+ requirements: Union[str, list[str]] = None,
440
440
  kfp=None,
441
441
  requirements_file: str = "",
442
442
  ):
@@ -585,9 +585,9 @@ def code_to_function(
585
585
  code_output: str = "",
586
586
  embed_code: bool = True,
587
587
  description: str = "",
588
- requirements: Union[str, List[str]] = None,
589
- categories: List[str] = None,
590
- labels: Dict[str, str] = None,
588
+ requirements: Union[str, list[str]] = None,
589
+ categories: list[str] = None,
590
+ labels: dict[str, str] = None,
591
591
  with_doc: bool = True,
592
592
  ignored_tags=None,
593
593
  requirements_file: str = "",
@@ -896,7 +896,7 @@ def _run_pipeline(
896
896
  def wait_for_pipeline_completion(
897
897
  run_id,
898
898
  timeout=60 * 60,
899
- expected_statuses: List[str] = None,
899
+ expected_statuses: list[str] = None,
900
900
  namespace=None,
901
901
  remote=True,
902
902
  project: str = None,
@@ -1041,7 +1041,7 @@ def list_pipelines(
1041
1041
  namespace=None,
1042
1042
  project="*",
1043
1043
  format_: mlrun.common.schemas.PipelinesFormat = mlrun.common.schemas.PipelinesFormat.metadata_only,
1044
- ) -> Tuple[int, Optional[int], List[dict]]:
1044
+ ) -> tuple[int, Optional[int], list[dict]]:
1045
1045
  """List pipelines
1046
1046
 
1047
1047
  :param full: Deprecated, use `format_` instead. if True will set `format_` to full, otherwise `format_` will
@@ -34,13 +34,16 @@ from .base import BaseRuntime, RunError, RuntimeClassMode # noqa
34
34
  from .constants import MPIJobCRDVersions
35
35
  from .daskjob import DaskCluster # noqa
36
36
  from .databricks_job.databricks_runtime import DatabricksRuntime
37
- from .function import RemoteRuntime
38
37
  from .kubejob import KubejobRuntime, KubeResource # noqa
39
38
  from .local import HandlerRuntime, LocalRuntime # noqa
40
39
  from .mpijob import MpiRuntimeContainer, MpiRuntimeV1, MpiRuntimeV1Alpha1 # noqa
41
- from .nuclio import nuclio_init_hook
40
+ from .nuclio import (
41
+ RemoteRuntime,
42
+ ServingRuntime,
43
+ new_v2_model_server,
44
+ nuclio_init_hook,
45
+ )
42
46
  from .remotesparkjob import RemoteSparkRuntime
43
- from .serving import ServingRuntime, new_v2_model_server
44
47
  from .sparkjob import Spark3Runtime
45
48
 
46
49
  # for legacy imports (MLModelServer moved from here to /serving)
@@ -86,7 +89,7 @@ def new_model_server(
86
89
  )
87
90
 
88
91
 
89
- class RuntimeKinds(object):
92
+ class RuntimeKinds:
90
93
  remote = "remote"
91
94
  nuclio = "nuclio"
92
95
  dask = "dask"
mlrun/runtimes/base.py CHANGED
@@ -17,7 +17,7 @@ import re
17
17
  import typing
18
18
  from base64 import b64encode
19
19
  from os import environ
20
- from typing import Callable, Dict, List, Optional, Union
20
+ from typing import Callable, Optional, Union
21
21
 
22
22
  import requests.exceptions
23
23
  from nuclio.build import mlrun_footer
@@ -91,6 +91,7 @@ class FunctionStatus(ModelObj):
91
91
 
92
92
  class FunctionSpec(ModelObj):
93
93
  _dict_fields = spec_fields
94
+ _default_fields_to_strip = []
94
95
 
95
96
  def __init__(
96
97
  self,
@@ -122,9 +123,9 @@ class FunctionSpec(ModelObj):
122
123
  self.entry_points = entry_points or {}
123
124
  self.disable_auto_mount = disable_auto_mount
124
125
  self.allow_empty_resources = None
125
- # the build.source is cloned/extracted to the specified clone_target_dir
126
+ # The build.source is cloned/extracted to the specified clone_target_dir
126
127
  # if a relative path is specified, it will be enriched with a temp dir path
127
- self.clone_target_dir = clone_target_dir or ""
128
+ self.clone_target_dir = clone_target_dir or None
128
129
 
129
130
  @property
130
131
  def build(self) -> ImageBuilder:
@@ -146,6 +147,9 @@ class BaseRuntime(ModelObj):
146
147
  _is_nested = False
147
148
  _is_remote = False
148
149
  _dict_fields = ["kind", "metadata", "spec", "status", "verbose"]
150
+ _default_fields_to_strip = ModelObj._default_fields_to_strip + [
151
+ "status", # Function status describes the state rather than configuration
152
+ ]
149
153
 
150
154
  def __init__(self, metadata=None, spec=None):
151
155
  self._metadata = None
@@ -195,7 +199,7 @@ class BaseRuntime(ModelObj):
195
199
  self.metadata.labels[key] = str(value)
196
200
  return self
197
201
 
198
- def set_categories(self, categories: List[str]):
202
+ def set_categories(self, categories: list[str]):
199
203
  self.metadata.categories = mlrun.utils.helpers.as_list(categories)
200
204
 
201
205
  @property
@@ -269,7 +273,7 @@ class BaseRuntime(ModelObj):
269
273
  mlrun.model.Credentials.generate_access_key
270
274
  )
271
275
 
272
- def generate_runtime_k8s_env(self, runobj: RunObject = None) -> List[Dict]:
276
+ def generate_runtime_k8s_env(self, runobj: RunObject = None) -> list[dict]:
273
277
  """
274
278
  Prepares a runtime environment as it's expected by kubernetes.models.V1Container
275
279
 
@@ -290,23 +294,23 @@ class BaseRuntime(ModelObj):
290
294
  name: Optional[str] = "",
291
295
  project: Optional[str] = "",
292
296
  params: Optional[dict] = None,
293
- inputs: Optional[Dict[str, str]] = None,
297
+ inputs: Optional[dict[str, str]] = None,
294
298
  out_path: Optional[str] = "",
295
299
  workdir: Optional[str] = "",
296
300
  artifact_path: Optional[str] = "",
297
301
  watch: Optional[bool] = True,
298
302
  schedule: Optional[Union[str, mlrun.common.schemas.ScheduleCronTrigger]] = None,
299
- hyperparams: Optional[Dict[str, list]] = None,
303
+ hyperparams: Optional[dict[str, list]] = None,
300
304
  hyper_param_options: Optional[HyperParamOptions] = None,
301
305
  verbose: Optional[bool] = None,
302
306
  scrape_metrics: Optional[bool] = None,
303
307
  local: Optional[bool] = False,
304
308
  local_code_path: Optional[str] = None,
305
309
  auto_build: Optional[bool] = None,
306
- param_file_secrets: Optional[Dict[str, str]] = None,
307
- notifications: Optional[List[mlrun.model.Notification]] = None,
308
- returns: Optional[List[Union[str, Dict[str, str]]]] = None,
309
- state_thresholds: Optional[Dict[str, int]] = None,
310
+ param_file_secrets: Optional[dict[str, str]] = None,
311
+ notifications: Optional[list[mlrun.model.Notification]] = None,
312
+ returns: Optional[list[Union[str, dict[str, str]]]] = None,
313
+ state_thresholds: Optional[dict[str, int]] = None,
310
314
  **launcher_kwargs,
311
315
  ) -> RunObject:
312
316
  """
@@ -403,7 +407,7 @@ class BaseRuntime(ModelObj):
403
407
  if task:
404
408
  return task.to_dict()
405
409
 
406
- def _generate_runtime_env(self, runobj: RunObject = None) -> Dict:
410
+ def _generate_runtime_env(self, runobj: RunObject = None) -> dict:
407
411
  """
408
412
  Prepares all available environment variables for usage on a runtime
409
413
  Data will be extracted from several sources and most of them are not guaranteed to be available
@@ -638,7 +642,7 @@ class BaseRuntime(ModelObj):
638
642
  use_db=True,
639
643
  verbose=None,
640
644
  scrape_metrics=False,
641
- returns: Optional[List[Union[str, Dict[str, str]]]] = None,
645
+ returns: Optional[list[Union[str, dict[str, str]]]] = None,
642
646
  auto_build: bool = False,
643
647
  ):
644
648
  """Run a local or remote task.
@@ -744,7 +748,7 @@ class BaseRuntime(ModelObj):
744
748
  body = fp.read()
745
749
  if self.kind == mlrun.runtimes.RuntimeKinds.serving:
746
750
  body = body + mlrun_footer.format(
747
- mlrun.runtimes.serving.serving_subkind
751
+ mlrun.runtimes.nuclio.serving.serving_subkind
748
752
  )
749
753
 
750
754
  self.spec.build.functionSourceCode = b64encode(body.encode("utf-8")).decode(
@@ -756,7 +760,7 @@ class BaseRuntime(ModelObj):
756
760
 
757
761
  def with_requirements(
758
762
  self,
759
- requirements: Optional[List[str]] = None,
763
+ requirements: Optional[list[str]] = None,
760
764
  overwrite: bool = False,
761
765
  prepare_image_for_deploy: bool = True,
762
766
  requirements_file: str = "",
@@ -778,7 +782,7 @@ class BaseRuntime(ModelObj):
778
782
 
779
783
  def with_commands(
780
784
  self,
781
- commands: List[str],
785
+ commands: list[str],
782
786
  overwrite: bool = False,
783
787
  prepare_image_for_deploy: bool = True,
784
788
  ):
@@ -860,13 +864,6 @@ class BaseRuntime(ModelObj):
860
864
  self, tag=tag, versioned=versioned, refresh=refresh
861
865
  )
862
866
 
863
- def to_dict(self, fields=None, exclude=None, strip=False):
864
- struct = super().to_dict(fields, exclude=exclude)
865
- if strip:
866
- if "status" in struct:
867
- del struct["status"]
868
- return struct
869
-
870
867
  def doc(self):
871
868
  print("function:", self.metadata.name)
872
869
  print(self.spec.description)
@@ -75,7 +75,7 @@ class ThresholdStates:
75
75
  ]
76
76
 
77
77
  @staticmethod
78
- def from_pod_phase(pod_phase: str, pod: typing.Dict) -> typing.Optional[str]:
78
+ def from_pod_phase(pod_phase: str, pod: dict) -> typing.Optional[str]:
79
79
  if pod_phase == PodPhases.pending:
80
80
  if ThresholdStates.is_pod_in_image_pull_backoff(pod):
81
81
  return ThresholdStates.image_pull_backoff
@@ -90,7 +90,7 @@ class ThresholdStates:
90
90
  return None
91
91
 
92
92
  @staticmethod
93
- def is_pod_scheduled(pod: typing.Dict):
93
+ def is_pod_scheduled(pod: dict):
94
94
  conditions = pod["status"].get("conditions", []) or []
95
95
  for condition in conditions:
96
96
  if condition["type"] == "PodScheduled" and condition["status"] == "True":
@@ -98,7 +98,7 @@ class ThresholdStates:
98
98
  return False
99
99
 
100
100
  @staticmethod
101
- def is_pod_in_image_pull_backoff(pod: typing.Dict):
101
+ def is_pod_in_image_pull_backoff(pod: dict):
102
102
  container_statuses = pod.get("status").get("container_statuses", []) or []
103
103
  for container_status in container_statuses:
104
104
  state_waiting = container_status.get("state", {}).get("waiting", {}) or {}
@@ -107,7 +107,7 @@ class ThresholdStates:
107
107
  return False
108
108
 
109
109
 
110
- class MPIJobCRDVersions(object):
110
+ class MPIJobCRDVersions:
111
111
  v1 = "v1"
112
112
  v1alpha1 = "v1alpha1"
113
113
 
@@ -127,7 +127,7 @@ class MPIJobCRDVersions(object):
127
127
  }[version]
128
128
 
129
129
 
130
- class RunStates(object):
130
+ class RunStates:
131
131
  completed = "completed"
132
132
  error = "error"
133
133
  running = "running"