mlrun 1.7.0rc13__py3-none-any.whl → 1.7.0rc15__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (85) hide show
  1. mlrun/__main__.py +0 -105
  2. mlrun/artifacts/__init__.py +1 -2
  3. mlrun/artifacts/base.py +8 -250
  4. mlrun/artifacts/dataset.py +1 -190
  5. mlrun/artifacts/manager.py +2 -41
  6. mlrun/artifacts/model.py +1 -140
  7. mlrun/artifacts/plots.py +1 -375
  8. mlrun/common/schemas/model_monitoring/__init__.py +4 -0
  9. mlrun/common/schemas/model_monitoring/constants.py +24 -3
  10. mlrun/common/schemas/model_monitoring/model_endpoints.py +13 -1
  11. mlrun/common/schemas/project.py +1 -0
  12. mlrun/config.py +14 -4
  13. mlrun/data_types/to_pandas.py +4 -4
  14. mlrun/datastore/base.py +41 -9
  15. mlrun/datastore/datastore_profile.py +50 -3
  16. mlrun/datastore/hdfs.py +5 -0
  17. mlrun/datastore/inmem.py +2 -2
  18. mlrun/datastore/sources.py +43 -2
  19. mlrun/datastore/store_resources.py +2 -6
  20. mlrun/datastore/targets.py +125 -6
  21. mlrun/datastore/v3io.py +1 -1
  22. mlrun/db/auth_utils.py +152 -0
  23. mlrun/db/base.py +1 -1
  24. mlrun/db/httpdb.py +69 -33
  25. mlrun/feature_store/__init__.py +0 -2
  26. mlrun/feature_store/api.py +12 -47
  27. mlrun/feature_store/feature_set.py +9 -0
  28. mlrun/feature_store/retrieval/base.py +9 -4
  29. mlrun/feature_store/retrieval/conversion.py +4 -4
  30. mlrun/feature_store/retrieval/dask_merger.py +2 -0
  31. mlrun/feature_store/retrieval/job.py +2 -0
  32. mlrun/feature_store/retrieval/local_merger.py +2 -0
  33. mlrun/feature_store/retrieval/spark_merger.py +5 -0
  34. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +5 -10
  35. mlrun/kfpops.py +5 -10
  36. mlrun/launcher/base.py +1 -1
  37. mlrun/launcher/client.py +1 -1
  38. mlrun/lists.py +2 -2
  39. mlrun/model.py +36 -9
  40. mlrun/model_monitoring/api.py +41 -18
  41. mlrun/model_monitoring/application.py +5 -305
  42. mlrun/model_monitoring/applications/__init__.py +11 -0
  43. mlrun/model_monitoring/applications/_application_steps.py +158 -0
  44. mlrun/model_monitoring/applications/base.py +282 -0
  45. mlrun/model_monitoring/applications/context.py +214 -0
  46. mlrun/model_monitoring/applications/evidently_base.py +211 -0
  47. mlrun/model_monitoring/applications/histogram_data_drift.py +92 -77
  48. mlrun/model_monitoring/applications/results.py +99 -0
  49. mlrun/model_monitoring/controller.py +3 -1
  50. mlrun/model_monitoring/db/stores/sqldb/models/base.py +7 -6
  51. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +1 -1
  52. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +67 -4
  53. mlrun/model_monitoring/evidently_application.py +6 -118
  54. mlrun/model_monitoring/helpers.py +1 -1
  55. mlrun/model_monitoring/model_endpoint.py +3 -2
  56. mlrun/model_monitoring/stream_processing.py +2 -3
  57. mlrun/model_monitoring/writer.py +69 -39
  58. mlrun/platforms/iguazio.py +2 -2
  59. mlrun/projects/pipelines.py +24 -7
  60. mlrun/projects/project.py +130 -65
  61. mlrun/render.py +2 -10
  62. mlrun/run.py +1 -4
  63. mlrun/runtimes/__init__.py +3 -3
  64. mlrun/runtimes/base.py +3 -3
  65. mlrun/runtimes/funcdoc.py +0 -28
  66. mlrun/runtimes/local.py +1 -1
  67. mlrun/runtimes/mpijob/__init__.py +0 -20
  68. mlrun/runtimes/mpijob/v1.py +1 -1
  69. mlrun/runtimes/nuclio/api_gateway.py +275 -153
  70. mlrun/runtimes/nuclio/function.py +1 -1
  71. mlrun/runtimes/pod.py +5 -5
  72. mlrun/runtimes/utils.py +1 -1
  73. mlrun/serving/states.py +53 -2
  74. mlrun/utils/helpers.py +27 -40
  75. mlrun/utils/notifications/notification/slack.py +31 -8
  76. mlrun/utils/notifications/notification_pusher.py +133 -14
  77. mlrun/utils/version/version.json +2 -2
  78. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/METADATA +2 -2
  79. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/RECORD +84 -79
  80. mlrun/runtimes/mpijob/v1alpha1.py +0 -29
  81. /mlrun/{runtimes → common/runtimes}/constants.py +0 -0
  82. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/LICENSE +0 -0
  83. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/WHEEL +0 -0
  84. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/entry_points.txt +0 -0
  85. {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py CHANGED
@@ -41,6 +41,7 @@ import mlrun.common.schemas.model_monitoring.constants as mm_constants
41
41
  import mlrun.db
42
42
  import mlrun.errors
43
43
  import mlrun.k8s_utils
44
+ import mlrun.model_monitoring.applications as mm_app
44
45
  import mlrun.runtimes
45
46
  import mlrun.runtimes.nuclio.api_gateway
46
47
  import mlrun.runtimes.pod
@@ -56,14 +57,10 @@ from ..artifacts.manager import ArtifactManager, dict_to_artifact, extend_artifa
56
57
  from ..datastore import store_manager
57
58
  from ..features import Feature
58
59
  from ..model import EntrypointParam, ImageBuilder, ModelObj
59
- from ..model_monitoring.application import (
60
- ModelMonitoringApplicationBase,
61
- )
62
60
  from ..run import code_to_function, get_object, import_function, new_function
63
61
  from ..secrets import SecretsStore
64
62
  from ..utils import (
65
63
  is_ipython,
66
- is_legacy_artifact,
67
64
  is_relative_path,
68
65
  is_yaml_path,
69
66
  logger,
@@ -207,14 +204,16 @@ def new_project(
207
204
  "Unsupported option, cannot use subpath argument with project templates"
208
205
  )
209
206
  if from_template.endswith(".yaml"):
210
- project = _load_project_file(from_template, name, secrets)
207
+ project = _load_project_file(
208
+ from_template, name, secrets, allow_cross_project=True
209
+ )
211
210
  elif from_template.startswith("git://"):
212
211
  clone_git(from_template, context, secrets, clone=True)
213
212
  shutil.rmtree(path.join(context, ".git"))
214
- project = _load_project_dir(context, name)
213
+ project = _load_project_dir(context, name, allow_cross_project=True)
215
214
  elif from_template.endswith(".zip"):
216
215
  clone_zip(from_template, context, secrets)
217
- project = _load_project_dir(context, name)
216
+ project = _load_project_dir(context, name, allow_cross_project=True)
218
217
  else:
219
218
  raise ValueError("template must be a path to .yaml or .zip file")
220
219
  project.metadata.name = name
@@ -296,6 +295,7 @@ def load_project(
296
295
  save: bool = True,
297
296
  sync_functions: bool = False,
298
297
  parameters: dict = None,
298
+ allow_cross_project: bool = None,
299
299
  ) -> "MlrunProject":
300
300
  """Load an MLRun project from git or tar or dir
301
301
 
@@ -342,6 +342,8 @@ def load_project(
342
342
  :param save: whether to save the created project and artifact in the DB
343
343
  :param sync_functions: sync the project's functions into the project object (will be saved to the DB if save=True)
344
344
  :param parameters: key/value pairs to add to the project.spec.params
345
+ :param allow_cross_project: if True, override the loaded project name. This flag ensures awareness of
346
+ loading an existing project yaml as a baseline for a new project with a different name
345
347
 
346
348
  :returns: project object
347
349
  """
@@ -357,7 +359,7 @@ def load_project(
357
359
  if url:
358
360
  url = str(url) # to support path objects
359
361
  if is_yaml_path(url):
360
- project = _load_project_file(url, name, secrets)
362
+ project = _load_project_file(url, name, secrets, allow_cross_project)
361
363
  project.spec.context = context
362
364
  elif url.startswith("git://"):
363
365
  url, repo = clone_git(url, context, secrets, clone)
@@ -384,7 +386,7 @@ def load_project(
384
386
  repo, url = init_repo(context, url, init_git)
385
387
 
386
388
  if not project:
387
- project = _load_project_dir(context, name, subpath)
389
+ project = _load_project_dir(context, name, subpath, allow_cross_project)
388
390
 
389
391
  if not project.metadata.name:
390
392
  raise ValueError("Project name must be specified")
@@ -438,6 +440,7 @@ def get_or_create_project(
438
440
  from_template: str = None,
439
441
  save: bool = True,
440
442
  parameters: dict = None,
443
+ allow_cross_project: bool = None,
441
444
  ) -> "MlrunProject":
442
445
  """Load a project from MLRun DB, or create/import if it does not exist
443
446
 
@@ -482,12 +485,12 @@ def get_or_create_project(
482
485
  :param from_template: path to project YAML file that will be used as from_template (for new projects)
483
486
  :param save: whether to save the created project in the DB
484
487
  :param parameters: key/value pairs to add to the project.spec.params
488
+ :param allow_cross_project: if True, override the loaded project name. This flag ensures awareness of
489
+ loading an existing project yaml as a baseline for a new project with a different name
485
490
 
486
491
  :returns: project object
487
492
  """
488
493
  context = context or "./"
489
- spec_path = path.join(context, subpath or "", "project.yaml")
490
- load_from_path = url or path.isfile(spec_path)
491
494
  try:
492
495
  # load project from the DB.
493
496
  # use `name` as `url` as we load the project from the DB
@@ -503,13 +506,15 @@ def get_or_create_project(
503
506
  # only loading project from db so no need to save it
504
507
  save=False,
505
508
  parameters=parameters,
509
+ allow_cross_project=allow_cross_project,
506
510
  )
507
511
  logger.info("Project loaded successfully", project_name=name)
508
512
  return project
509
-
510
513
  except mlrun.errors.MLRunNotFoundError:
511
514
  logger.debug("Project not found in db", project_name=name)
512
515
 
516
+ spec_path = path.join(context, subpath or "", "project.yaml")
517
+ load_from_path = url or path.isfile(spec_path)
513
518
  # do not nest under "try" or else the exceptions raised below will be logged along with the "not found" message
514
519
  if load_from_path:
515
520
  # loads a project from archive or local project.yaml
@@ -525,6 +530,7 @@ def get_or_create_project(
525
530
  user_project=user_project,
526
531
  save=save,
527
532
  parameters=parameters,
533
+ allow_cross_project=allow_cross_project,
528
534
  )
529
535
 
530
536
  logger.info(
@@ -599,7 +605,7 @@ def _run_project_setup(
599
605
  return project
600
606
 
601
607
 
602
- def _load_project_dir(context, name="", subpath=""):
608
+ def _load_project_dir(context, name="", subpath="", allow_cross_project=None):
603
609
  subpath_str = subpath or ""
604
610
 
605
611
  # support both .yaml and .yml file extensions
@@ -613,7 +619,7 @@ def _load_project_dir(context, name="", subpath=""):
613
619
  with open(project_file_path) as fp:
614
620
  data = fp.read()
615
621
  struct = yaml.load(data, Loader=yaml.FullLoader)
616
- project = _project_instance_from_struct(struct, name)
622
+ project = _project_instance_from_struct(struct, name, allow_cross_project)
617
623
  project.spec.context = context
618
624
  elif function_files := glob.glob(function_file_path):
619
625
  function_path = function_files[0]
@@ -686,19 +692,41 @@ def _delete_project_from_db(project_name, secrets, deletion_strategy):
686
692
  return db.delete_project(project_name, deletion_strategy=deletion_strategy)
687
693
 
688
694
 
689
- def _load_project_file(url, name="", secrets=None):
695
+ def _load_project_file(url, name="", secrets=None, allow_cross_project=None):
690
696
  try:
691
697
  obj = get_object(url, secrets)
692
698
  except FileNotFoundError as exc:
693
699
  raise FileNotFoundError(f"cant find project file at {url}") from exc
694
700
  struct = yaml.load(obj, Loader=yaml.FullLoader)
695
- return _project_instance_from_struct(struct, name)
701
+ return _project_instance_from_struct(struct, name, allow_cross_project)
696
702
 
697
703
 
698
- def _project_instance_from_struct(struct, name):
699
- struct.setdefault("metadata", {})["name"] = name or struct.get("metadata", {}).get(
700
- "name", ""
701
- )
704
+ def _project_instance_from_struct(struct, name, allow_cross_project):
705
+ name_from_struct = struct.get("metadata", {}).get("name", "")
706
+ if name and name_from_struct and name_from_struct != name:
707
+ error_message = (
708
+ f"project name mismatch, {name_from_struct} != {name}, please do one of the following:\n"
709
+ "1. Set the `allow_cross_project=True` when loading the project.\n"
710
+ f"2. Delete the existing project yaml, or ensure its name is equal to {name}.\n"
711
+ "3. Use different project context dir."
712
+ )
713
+
714
+ if allow_cross_project is None:
715
+ # TODO: Remove this warning in version 1.9.0 and also fix cli to support allow_cross_project
716
+ logger.warn(
717
+ "Project name is different than specified on its project yaml."
718
+ "You should fix it until version 1.9.0",
719
+ description=error_message,
720
+ )
721
+ elif allow_cross_project:
722
+ logger.warn(
723
+ "Project name is different than specified on its project yaml. Overriding.",
724
+ existing_name=name_from_struct,
725
+ overriding_name=name,
726
+ )
727
+ else:
728
+ raise ValueError(error_message)
729
+ struct.setdefault("metadata", {})["name"] = name or name_from_struct
702
730
  return MlrunProject.from_dict(struct)
703
731
 
704
732
 
@@ -960,13 +988,9 @@ class ProjectSpec(ModelObj):
960
988
  if not isinstance(artifact, dict) and not hasattr(artifact, "to_dict"):
961
989
  raise ValueError("artifacts must be a dict or class")
962
990
  if isinstance(artifact, dict):
963
- # Support legacy artifacts
964
- if is_legacy_artifact(artifact) or _is_imported_artifact(artifact):
965
- key = artifact.get("key")
966
- else:
967
- key = artifact.get("metadata").get("key", "")
991
+ key = artifact.get("metadata", {}).get("key", "")
968
992
  if not key:
969
- raise ValueError('artifacts "key" must be specified')
993
+ raise ValueError('artifacts "metadata.key" must be specified')
970
994
  else:
971
995
  key = artifact.key
972
996
  artifact = artifact.to_dict()
@@ -1814,10 +1838,18 @@ class MlrunProject(ModelObj):
1814
1838
  """
1815
1839
  context = context or self.spec.context
1816
1840
  if context:
1817
- project = _load_project_dir(context, self.metadata.name, self.spec.subpath)
1841
+ project = _load_project_dir(
1842
+ context,
1843
+ self.metadata.name,
1844
+ self.spec.subpath,
1845
+ allow_cross_project=False,
1846
+ )
1818
1847
  else:
1819
1848
  project = _load_project_file(
1820
- self.spec.origin_url, self.metadata.name, self._secrets
1849
+ self.spec.origin_url,
1850
+ self.metadata.name,
1851
+ self._secrets,
1852
+ allow_cross_project=None,
1821
1853
  )
1822
1854
  project.spec.source = self.spec.source
1823
1855
  project.spec.repo = self.spec.repo
@@ -1846,7 +1878,11 @@ class MlrunProject(ModelObj):
1846
1878
  def set_model_monitoring_function(
1847
1879
  self,
1848
1880
  func: typing.Union[str, mlrun.runtimes.BaseRuntime, None] = None,
1849
- application_class: typing.Union[str, ModelMonitoringApplicationBase] = None,
1881
+ application_class: typing.Union[
1882
+ str,
1883
+ mm_app.ModelMonitoringApplicationBase,
1884
+ mm_app.ModelMonitoringApplicationBaseV2,
1885
+ ] = None,
1850
1886
  name: str = None,
1851
1887
  image: str = None,
1852
1888
  handler=None,
@@ -1884,11 +1920,6 @@ class MlrunProject(ModelObj):
1884
1920
  monitoring application's constructor.
1885
1921
  """
1886
1922
 
1887
- if name in mm_constants.MonitoringFunctionNames.list():
1888
- raise mlrun.errors.MLRunInvalidArgumentError(
1889
- f"An application cannot have the following names: "
1890
- f"{mm_constants.MonitoringFunctionNames.list()}"
1891
- )
1892
1923
  function_object: RemoteRuntime = None
1893
1924
  (
1894
1925
  resolved_function_name,
@@ -1914,7 +1945,11 @@ class MlrunProject(ModelObj):
1914
1945
  def create_model_monitoring_function(
1915
1946
  self,
1916
1947
  func: str = None,
1917
- application_class: typing.Union[str, ModelMonitoringApplicationBase] = None,
1948
+ application_class: typing.Union[
1949
+ str,
1950
+ mm_app.ModelMonitoringApplicationBase,
1951
+ mm_app.ModelMonitoringApplicationBaseV2,
1952
+ ] = None,
1918
1953
  name: str = None,
1919
1954
  image: str = None,
1920
1955
  handler: str = None,
@@ -1967,7 +2002,10 @@ class MlrunProject(ModelObj):
1967
2002
  self,
1968
2003
  func: typing.Union[str, mlrun.runtimes.BaseRuntime, None] = None,
1969
2004
  application_class: typing.Union[
1970
- str, ModelMonitoringApplicationBase, None
2005
+ str,
2006
+ mm_app.ModelMonitoringApplicationBase,
2007
+ mm_app.ModelMonitoringApplicationBaseV2,
2008
+ None,
1971
2009
  ] = None,
1972
2010
  name: typing.Optional[str] = None,
1973
2011
  image: typing.Optional[str] = None,
@@ -2024,12 +2062,24 @@ class MlrunProject(ModelObj):
2024
2062
 
2025
2063
  return resolved_function_name, function_object, func
2026
2064
 
2065
+ def _wait_for_functions_deployment(self, function_names: list[str]) -> None:
2066
+ """
2067
+ Wait for the deployment of functions on the backend.
2068
+
2069
+ :param function_names: A list of function names.
2070
+ """
2071
+ for fn_name in function_names:
2072
+ fn = typing.cast(RemoteRuntime, self.get_function(key=fn_name))
2073
+ fn._wait_for_function_deployment(db=fn._get_db())
2074
+
2027
2075
  def enable_model_monitoring(
2028
2076
  self,
2029
2077
  default_controller_image: str = "mlrun/mlrun",
2030
2078
  base_period: int = 10,
2031
2079
  image: str = "mlrun/mlrun",
2080
+ *,
2032
2081
  deploy_histogram_data_drift_app: bool = True,
2082
+ wait_for_deployment: bool = False,
2033
2083
  ) -> None:
2034
2084
  """
2035
2085
  Deploy model monitoring application controller, writer and stream functions.
@@ -2039,7 +2089,6 @@ class MlrunProject(ModelObj):
2039
2089
  The stream function goal is to monitor the log of the data stream. It is triggered when a new log entry
2040
2090
  is detected. It processes the new events into statistics that are then written to statistics databases.
2041
2091
 
2042
-
2043
2092
  :param default_controller_image: Deprecated.
2044
2093
  :param base_period: The time period in minutes in which the model monitoring controller
2045
2094
  function is triggered. By default, the base period is 10 minutes.
@@ -2047,6 +2096,9 @@ class MlrunProject(ModelObj):
2047
2096
  stream & histogram data drift functions, which are real time nuclio
2048
2097
  functions. By default, the image is mlrun/mlrun.
2049
2098
  :param deploy_histogram_data_drift_app: If true, deploy the default histogram-based data drift application.
2099
+ :param wait_for_deployment: If true, return only after the deployment is done on the backend.
2100
+ Otherwise, deploy the model monitoring infrastructure on the
2101
+ background, including the histogram data drift app if selected.
2050
2102
  """
2051
2103
  if default_controller_image != "mlrun/mlrun":
2052
2104
  # TODO: Remove this in 1.9.0
@@ -2064,37 +2116,55 @@ class MlrunProject(ModelObj):
2064
2116
  deploy_histogram_data_drift_app=deploy_histogram_data_drift_app,
2065
2117
  )
2066
2118
 
2119
+ if wait_for_deployment:
2120
+ deployment_functions = mm_constants.MonitoringFunctionNames.list()
2121
+ if deploy_histogram_data_drift_app:
2122
+ deployment_functions.append(
2123
+ mm_constants.HistogramDataDriftApplicationConstants.NAME
2124
+ )
2125
+ self._wait_for_functions_deployment(deployment_functions)
2126
+
2067
2127
  def deploy_histogram_data_drift_app(
2068
2128
  self,
2069
2129
  *,
2070
2130
  image: str = "mlrun/mlrun",
2071
2131
  db: Optional[mlrun.db.RunDBInterface] = None,
2132
+ wait_for_deployment: bool = False,
2072
2133
  ) -> None:
2073
2134
  """
2074
2135
  Deploy the histogram data drift application.
2075
2136
 
2076
- :param image: The image on which the application will run.
2077
- :param db: An optional DB object.
2137
+ :param image: The image on which the application will run.
2138
+ :param db: An optional DB object.
2139
+ :param wait_for_deployment: If true, return only after the deployment is done on the backend.
2140
+ Otherwise, deploy the application on the background.
2078
2141
  """
2079
2142
  if db is None:
2080
2143
  db = mlrun.db.get_run_db(secrets=self._secrets)
2081
2144
  db.deploy_histogram_data_drift_app(project=self.name, image=image)
2082
2145
 
2146
+ if wait_for_deployment:
2147
+ self._wait_for_functions_deployment(
2148
+ [mm_constants.HistogramDataDriftApplicationConstants.NAME]
2149
+ )
2150
+
2083
2151
  def update_model_monitoring_controller(
2084
2152
  self,
2085
2153
  base_period: int = 10,
2086
2154
  image: str = "mlrun/mlrun",
2155
+ *,
2156
+ wait_for_deployment: bool = False,
2087
2157
  ) -> None:
2088
2158
  """
2089
2159
  Redeploy model monitoring application controller functions.
2090
2160
 
2091
-
2092
- :param base_period: The time period in minutes in which the model monitoring controller function
2093
- is triggered. By default, the base period is 10 minutes.
2094
- :param image: The image of the model monitoring controller, writer & monitoring
2095
- stream functions, which are real time nuclio functions.
2096
- By default, the image is mlrun/mlrun.
2097
- :returns: model monitoring controller job as a dictionary.
2161
+ :param base_period: The time period in minutes in which the model monitoring controller function
2162
+ is triggered. By default, the base period is 10 minutes.
2163
+ :param image: The image of the model monitoring controller, writer & monitoring
2164
+ stream functions, which are real time nuclio functions.
2165
+ By default, the image is mlrun/mlrun.
2166
+ :param wait_for_deployment: If true, return only after the deployment is done on the backend.
2167
+ Otherwise, deploy the controller on the background.
2098
2168
  """
2099
2169
  db = mlrun.db.get_run_db(secrets=self._secrets)
2100
2170
  db.update_model_monitoring_controller(
@@ -2103,6 +2173,11 @@ class MlrunProject(ModelObj):
2103
2173
  image=image,
2104
2174
  )
2105
2175
 
2176
+ if wait_for_deployment:
2177
+ self._wait_for_functions_deployment(
2178
+ [mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER]
2179
+ )
2180
+
2106
2181
  def disable_model_monitoring(
2107
2182
  self, *, delete_histogram_data_drift_app: bool = True
2108
2183
  ) -> None:
@@ -2824,12 +2899,14 @@ class MlrunProject(ModelObj):
2824
2899
  "Remote repo is not defined, use .create_remote() + push()"
2825
2900
  )
2826
2901
 
2827
- self.sync_functions(always=sync)
2828
- if not self.spec._function_objects:
2829
- raise ValueError(
2830
- "There are no functions in the project."
2831
- " Make sure you've set your functions with project.set_function()."
2832
- )
2902
+ if engine not in ["remote"]:
2903
+ # for remote runs we don't require the functions to be synced as they can be loaded dynamically during run
2904
+ self.sync_functions(always=sync)
2905
+ if not self.spec._function_objects:
2906
+ raise ValueError(
2907
+ "There are no functions in the project."
2908
+ " Make sure you've set your functions with project.set_function()."
2909
+ )
2833
2910
 
2834
2911
  if not name and not workflow_path and not workflow_handler:
2835
2912
  raise ValueError("Workflow name, path, or handler must be specified")
@@ -3266,7 +3343,6 @@ class MlrunProject(ModelObj):
3266
3343
  image: str = None,
3267
3344
  set_as_default: bool = True,
3268
3345
  with_mlrun: bool = None,
3269
- skip_deployed: bool = False,
3270
3346
  base_image: str = None,
3271
3347
  commands: list = None,
3272
3348
  secret_name: str = None,
@@ -3287,7 +3363,6 @@ class MlrunProject(ModelObj):
3287
3363
  used. If not set, the `mlconf.default_project_image_name` value will be used
3288
3364
  :param set_as_default: set `image` to be the project's default image (default False)
3289
3365
  :param with_mlrun: add the current mlrun package to the container build
3290
- :param skip_deployed: *Deprecated* parameter is ignored
3291
3366
  :param base_image: base image name/path (commands and source code will be added to it) defaults to
3292
3367
  mlrun.mlconf.default_base_image
3293
3368
  :param commands: list of docker build (RUN) commands e.g. ['pip install pandas']
@@ -3312,14 +3387,6 @@ class MlrunProject(ModelObj):
3312
3387
  base_image=base_image,
3313
3388
  )
3314
3389
 
3315
- if skip_deployed:
3316
- warnings.warn(
3317
- "The 'skip_deployed' parameter is deprecated and will be removed in 1.7.0. "
3318
- "This parameter is ignored.",
3319
- # TODO: remove in 1.7.0
3320
- FutureWarning,
3321
- )
3322
-
3323
3390
  if not overwrite_build_params:
3324
3391
  # TODO: change overwrite_build_params default to True in 1.8.0
3325
3392
  warnings.warn(
@@ -3582,9 +3649,7 @@ class MlrunProject(ModelObj):
3582
3649
  :returns: List of function objects.
3583
3650
  """
3584
3651
 
3585
- model_monitoring_labels_list = [
3586
- f"{mm_constants.ModelMonitoringAppLabel.KEY}={mm_constants.ModelMonitoringAppLabel.VAL}"
3587
- ]
3652
+ model_monitoring_labels_list = [str(mm_constants.ModelMonitoringAppLabel())]
3588
3653
  if labels:
3589
3654
  model_monitoring_labels_list += labels
3590
3655
  return self.list_functions(
mlrun/render.py CHANGED
@@ -121,16 +121,8 @@ def artifacts_html(
121
121
  html = ""
122
122
 
123
123
  for artifact in artifacts:
124
- # TODO: remove this in 1.7.0 once we no longer support legacy format
125
- if mlrun.utils.is_legacy_artifact(artifact):
126
- attribute_value = artifact.get(attribute_name)
127
- else:
128
- attribute_value = artifact["spec"].get(attribute_name)
129
-
130
- if mlrun.utils.is_legacy_artifact(artifact):
131
- key = artifact["key"]
132
- else:
133
- key = artifact["metadata"]["key"]
124
+ attribute_value = artifact["spec"].get(attribute_name)
125
+ key = artifact["metadata"]["key"]
134
126
 
135
127
  if not attribute_value:
136
128
  mlrun.utils.logger.warning(
mlrun/run.py CHANGED
@@ -47,7 +47,6 @@ from .runtimes import (
47
47
  KubejobRuntime,
48
48
  LocalRuntime,
49
49
  MpiRuntimeV1,
50
- MpiRuntimeV1Alpha1,
51
50
  RemoteRuntime,
52
51
  RemoteSparkRuntime,
53
52
  RuntimeKinds,
@@ -606,7 +605,6 @@ def code_to_function(
606
605
  ignored_tags: Optional[str] = None,
607
606
  requirements_file: Optional[str] = "",
608
607
  ) -> Union[
609
- MpiRuntimeV1Alpha1,
610
608
  MpiRuntimeV1,
611
609
  RemoteRuntime,
612
610
  ServingRuntime,
@@ -661,7 +659,6 @@ def code_to_function(
661
659
  :param embed_code: indicates whether or not to inject the code directly into the function runtime spec,
662
660
  defaults to True
663
661
  :param description: short function description, defaults to ''
664
- :param requirements: list of python packages or pip requirements file path, defaults to None
665
662
  :param requirements: a list of python packages
666
663
  :param requirements_file: path to a python requirements file
667
664
  :param categories: list of categories for mlrun Function Hub, defaults to None
@@ -1151,7 +1148,7 @@ def wait_for_runs_completion(
1151
1148
  running = []
1152
1149
  for run in runs:
1153
1150
  state = run.state()
1154
- if state in mlrun.runtimes.constants.RunStates.terminal_states():
1151
+ if state in mlrun.common.runtimes.constants.RunStates.terminal_states():
1155
1152
  completed.append(run)
1156
1153
  else:
1157
1154
  running.append(run)
@@ -30,13 +30,13 @@ __all__ = [
30
30
 
31
31
  from mlrun.runtimes.utils import resolve_spark_operator_version
32
32
 
33
+ from ..common.runtimes.constants import MPIJobCRDVersions
33
34
  from .base import BaseRuntime, RunError, RuntimeClassMode # noqa
34
- from .constants import MPIJobCRDVersions
35
35
  from .daskjob import DaskCluster # noqa
36
36
  from .databricks_job.databricks_runtime import DatabricksRuntime
37
37
  from .kubejob import KubejobRuntime, KubeResource # noqa
38
38
  from .local import HandlerRuntime, LocalRuntime # noqa
39
- from .mpijob import MpiRuntimeContainer, MpiRuntimeV1, MpiRuntimeV1Alpha1 # noqa
39
+ from .mpijob import MpiRuntimeV1 # noqa
40
40
  from .nuclio import (
41
41
  RemoteRuntime,
42
42
  ServingRuntime,
@@ -264,7 +264,7 @@ class RuntimeKinds:
264
264
 
265
265
  def get_runtime_class(kind: str):
266
266
  if kind == RuntimeKinds.mpijob:
267
- return MpiRuntimeContainer.selector()
267
+ return MpiRuntimeV1
268
268
 
269
269
  if kind == RuntimeKinds.spark:
270
270
  return Spark3Runtime
mlrun/runtimes/base.py CHANGED
@@ -469,7 +469,7 @@ class BaseRuntime(ModelObj):
469
469
  def _store_function(self, runspec, meta, db):
470
470
  meta.labels["kind"] = self.kind
471
471
  mlrun.runtimes.utils.enrich_run_labels(
472
- meta.labels, [mlrun.runtimes.constants.RunLabels.owner]
472
+ meta.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
473
473
  )
474
474
  if runspec.spec.output_path:
475
475
  runspec.spec.output_path = runspec.spec.output_path.replace(
@@ -580,9 +580,9 @@ class BaseRuntime(ModelObj):
580
580
 
581
581
  elif (
582
582
  not was_none
583
- and last_state != mlrun.runtimes.constants.RunStates.completed
583
+ and last_state != mlrun.common.runtimes.constants.RunStates.completed
584
584
  and last_state
585
- not in mlrun.runtimes.constants.RunStates.error_and_abortion_states()
585
+ not in mlrun.common.runtimes.constants.RunStates.error_and_abortion_states()
586
586
  ):
587
587
  try:
588
588
  runtime_cls = mlrun.runtimes.get_runtime_class(kind)
mlrun/runtimes/funcdoc.py CHANGED
@@ -16,8 +16,6 @@ import ast
16
16
  import inspect
17
17
  import re
18
18
 
19
- from deprecated import deprecated
20
-
21
19
  from mlrun.model import FunctionEntrypoint
22
20
 
23
21
 
@@ -73,32 +71,6 @@ def func_dict(
73
71
  }
74
72
 
75
73
 
76
- # TODO: remove in 1.7.0
77
- @deprecated(
78
- version="1.5.0",
79
- reason="'func_info' is deprecated and will be removed in 1.7.0, use 'ast_func_info' instead",
80
- category=FutureWarning,
81
- )
82
- def func_info(fn) -> dict:
83
- sig = inspect.signature(fn)
84
- doc = inspect.getdoc(fn) or ""
85
-
86
- out = func_dict(
87
- name=fn.__name__,
88
- doc=doc,
89
- params=[inspect_param(p) for p in sig.parameters.values()],
90
- returns=param_dict(
91
- type=type_name(sig.return_annotation, empty_is_none=True), default=None
92
- ),
93
- lineno=func_lineno(fn),
94
- )
95
-
96
- if not fn.__doc__ or not fn.__doc__.strip():
97
- return out
98
-
99
- return merge_doc(out, doc)
100
-
101
-
102
74
  def func_lineno(fn):
103
75
  try:
104
76
  return inspect.getsourcelines(fn)[1]
mlrun/runtimes/local.py CHANGED
@@ -493,7 +493,7 @@ def exec_from_params(handler, runobj: RunObject, context: MLClientCtx, cwd=None)
493
493
  logger.warning("Run was aborted", err=err_to_str(exc))
494
494
  # Run was aborted, the state run state is updated by the abort job, no need to commit again
495
495
  context.set_state(
496
- mlrun.runtimes.constants.RunStates.aborted, commit=False
496
+ mlrun.common.runtimes.constants.RunStates.aborted, commit=False
497
497
  )
498
498
  commit = False
499
499
  except Exception as exc:
@@ -21,28 +21,8 @@ from mlrun.config import config
21
21
  from .. import MPIJobCRDVersions
22
22
  from .abstract import AbstractMPIJobRuntime
23
23
  from .v1 import MpiRuntimeV1
24
- from .v1alpha1 import MpiRuntimeV1Alpha1
25
24
 
26
25
 
27
26
  def _resolve_mpijob_crd_version():
28
27
  # config is expected to get enriched from the API through the client-spec
29
28
  return config.mpijob_crd_version or MPIJobCRDVersions.default()
30
-
31
-
32
- class MpiRuntimeContainer(containers.DeclarativeContainer):
33
- resolver = providers.Callable(
34
- _resolve_mpijob_crd_version,
35
- )
36
-
37
- selector = providers.Selector(
38
- resolver,
39
- v1=providers.Object(MpiRuntimeV1),
40
- v1alpha1=providers.Object(MpiRuntimeV1Alpha1),
41
- )
42
-
43
- # An empty selector to be overriden by the API
44
- handler_selector = providers.Selector(
45
- resolver,
46
- v1=providers.Object(None),
47
- v1alpha1=providers.Object(None),
48
- )
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- from mlrun.runtimes.constants import MPIJobCRDVersions, MPIJobV1CleanPodPolicies
14
+ from mlrun.common.runtimes.constants import MPIJobCRDVersions, MPIJobV1CleanPodPolicies
15
15
  from mlrun.runtimes.mpijob.abstract import AbstractMPIJobRuntime, MPIResourceSpec
16
16
 
17
17