mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/runtimes/constants.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/runtimes/daskjob.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,8 +23,9 @@ from deprecated import deprecated
 from kubernetes.client.rest import ApiException
 from sqlalchemy.orm import Session
 
-import mlrun.api.schemas
+import mlrun.common.schemas
 import mlrun.errors
+import mlrun.k8s_utils
 import mlrun.utils
 import mlrun.utils.regex
 from mlrun.api.db.base import DBInterface
@@ -33,7 +34,6 @@ from mlrun.runtimes.base import BaseRuntimeHandler
 
 from ..config import config
 from ..execution import MLClientCtx
-from ..k8s_utils import get_k8s_helper
 from ..model import RunObject
 from ..render import ipython_display
 from ..utils import logger, normalize_name, update_in
@@ -41,7 +41,7 @@ from .base import FunctionStatus, RuntimeClassMode
 from .kubejob import KubejobRuntime
 from .local import exec_from_params, load_module
 from .pod import KubeResourceSpec, kube_resource_spec_to_pod_spec
-from .utils import RunError, get_func_selector, get_resource_labels, log_std
+from .utils import RunError, get_func_selector, get_k8s, get_resource_labels, log_std
 
 
 def get_dask_resource():
@@ -106,6 +106,7 @@ class DaskSpec(KubeResourceSpec):
         tolerations=None,
         preemption_mode=None,
         security_context=None,
+        clone_target_dir=None,
     ):
 
         super().__init__(
@@ -135,6 +136,7 @@ class DaskSpec(KubeResourceSpec):
             tolerations=tolerations,
             preemption_mode=preemption_mode,
             security_context=security_context,
+            clone_target_dir=clone_target_dir,
         )
         self.args = args
 
@@ -201,9 +203,7 @@ class DaskCluster(KubejobRuntime):
     def __init__(self, spec=None, metadata=None):
         super().__init__(spec, metadata)
         self._cluster = None
-        self.use_remote = not get_k8s_helper(
-            silent=True
-        ).is_running_inside_kubernetes_cluster()
+        self.use_remote = not mlrun.k8s_utils.is_running_inside_kubernetes_cluster()
         self.spec.build.base_image = self.spec.build.base_image or "daskdev/dask:latest"
 
     @property
@@ -271,11 +271,11 @@ class DaskCluster(KubejobRuntime):
                     )
                     if (
                         background_task.status.state
-                        in mlrun.api.schemas.BackgroundTaskState.terminal_states()
+                        in mlrun.common.schemas.BackgroundTaskState.terminal_states()
                     ):
                         if (
                             background_task.status.state
-                            == mlrun.api.schemas.BackgroundTaskState.failed
+                            == mlrun.common.schemas.BackgroundTaskState.failed
                         ):
                             raise mlrun.errors.MLRunRuntimeError(
                                 "Failed bringing up dask cluster"
@@ -352,10 +352,6 @@ class DaskCluster(KubejobRuntime):
                     f"remote scheduler at {addr} not ready, will try to restart {err_to_str(exc)}"
                 )
 
-                # todo: figure out if test is needed
-                # if self._is_remote_api():
-                #     raise Exception('no access to Kubernetes API')
-
                 status = self.get_status()
                 if status != "running":
                     self._start()
@@ -671,7 +667,9 @@ def get_obj_status(selector=None, namespace=None):
     if selector is None:
         selector = []
 
-    k8s = get_k8s_helper()
+    import mlrun.api.utils.singletons.k8s
+
+    k8s = mlrun.api.utils.singletons.k8s.get_k8s_helper()
     namespace = namespace or config.namespace
     selector = ",".join(["dask.org/component=scheduler"] + selector)
     pods = k8s.list_pods(namespace, selector=selector)
@@ -730,17 +728,19 @@ class DaskRuntimeHandler(BaseRuntimeHandler):
     def _enrich_list_resources_response(
         self,
         response: Union[
-            mlrun.api.schemas.RuntimeResources,
-            mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
-            mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput,
+            mlrun.common.schemas.RuntimeResources,
+            mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+            mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
         ],
         namespace: str,
         label_selector: str = None,
-        group_by: Optional[mlrun.api.schemas.ListRuntimeResourcesGroupByField] = None,
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
     ) -> Union[
-        mlrun.api.schemas.RuntimeResources,
-        mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
-        mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput,
+        mlrun.common.schemas.RuntimeResources,
+        mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+        mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
     ]:
         """
         Handling listing service resources
@@ -748,14 +748,13 @@ class DaskRuntimeHandler(BaseRuntimeHandler):
         enrich_needed = self._validate_if_enrich_is_needed_by_group_by(group_by)
         if not enrich_needed:
             return response
-        k8s_helper = get_k8s_helper()
-        services = k8s_helper.v1api.list_namespaced_service(
+        services = get_k8s().v1api.list_namespaced_service(
             namespace, label_selector=label_selector
         )
         service_resources = []
         for service in services.items:
             service_resources.append(
-                mlrun.api.schemas.RuntimeResource(
+                mlrun.common.schemas.RuntimeResource(
                     name=service.metadata.name, labels=service.metadata.labels
                 )
             )
@@ -766,12 +765,14 @@ class DaskRuntimeHandler(BaseRuntimeHandler):
     def _build_output_from_runtime_resources(
         self,
         response: Union[
-            mlrun.api.schemas.RuntimeResources,
-            mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
-            mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput,
+            mlrun.common.schemas.RuntimeResources,
+            mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+            mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
         ],
-        runtime_resources_list: List[mlrun.api.schemas.RuntimeResources],
-        group_by: Optional[mlrun.api.schemas.ListRuntimeResourcesGroupByField] = None,
+        runtime_resources_list: List[mlrun.common.schemas.RuntimeResources],
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
     ):
         enrich_needed = self._validate_if_enrich_is_needed_by_group_by(group_by)
         if not enrich_needed:
@@ -786,13 +787,15 @@ class DaskRuntimeHandler(BaseRuntimeHandler):
 
     def _validate_if_enrich_is_needed_by_group_by(
         self,
-        group_by: Optional[mlrun.api.schemas.ListRuntimeResourcesGroupByField] = None,
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
     ) -> bool:
         # Dask runtime resources are per function (and not per job) therefore, when grouping by job we're simply
         # omitting the dask runtime resources
-        if group_by == mlrun.api.schemas.ListRuntimeResourcesGroupByField.job:
+        if group_by == mlrun.common.schemas.ListRuntimeResourcesGroupByField.job:
             return False
-        elif group_by == mlrun.api.schemas.ListRuntimeResourcesGroupByField.project:
+        elif group_by == mlrun.common.schemas.ListRuntimeResourcesGroupByField.project:
             return True
         elif group_by is not None:
             raise NotImplementedError(
@@ -803,14 +806,16 @@ class DaskRuntimeHandler(BaseRuntimeHandler):
     def _enrich_service_resources_in_response(
         self,
         response: Union[
-            mlrun.api.schemas.RuntimeResources,
-            mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
-            mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput,
+            mlrun.common.schemas.RuntimeResources,
+            mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+            mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
         ],
-        service_resources: List[mlrun.api.schemas.RuntimeResource],
-        group_by: Optional[mlrun.api.schemas.ListRuntimeResourcesGroupByField] = None,
+        service_resources: List[mlrun.common.schemas.RuntimeResource],
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
     ):
-        if group_by == mlrun.api.schemas.ListRuntimeResourcesGroupByField.project:
+        if group_by == mlrun.common.schemas.ListRuntimeResourcesGroupByField.project:
             for service_resource in service_resources:
                 self._add_resource_to_grouped_by_project_resources_response(
                     response, "service_resources", service_resource
@@ -845,14 +850,13 @@ class DaskRuntimeHandler(BaseRuntimeHandler):
             if dask_component == "scheduler" and cluster_name:
                 service_names.append(cluster_name)
 
-        k8s_helper = get_k8s_helper()
-        services = k8s_helper.v1api.list_namespaced_service(
+        services = get_k8s().v1api.list_namespaced_service(
             namespace, label_selector=label_selector
         )
         for service in services.items:
             try:
                 if force or service.metadata.name in service_names:
-                    k8s_helper.v1api.delete_namespaced_service(
+                    get_k8s().v1api.delete_namespaced_service(
                         service.metadata.name, namespace
                     )
                     logger.info(f"Deleted service: {service.metadata.name}")

mlrun/runtimes/funcdoc.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 import ast
 import inspect
 import re
+import sys
 
 from mlrun.model import FunctionEntrypoint
 
@@ -49,13 +50,23 @@ def param_dict(name="", type="", doc="", default=""):
     }
 
 
-def func_dict(name, doc, params, returns, lineno):
+def func_dict(
+    name,
+    doc,
+    params,
+    returns,
+    lineno,
+    has_varargs: bool = False,
+    has_kwargs: bool = False,
+):
     return {
         "name": name,
         "doc": doc,
         "params": params,
         "return": returns,
         "lineno": lineno,
+        "has_varargs": has_varargs,
+        "has_kwargs": has_kwargs,
     }
 
 
@@ -165,6 +176,9 @@ def ast_func_info(func: ast.FunctionDef):
     doc = ast.get_docstring(func) or ""
     rtype = getattr(func.returns, "id", "")
     params = [ast_param_dict(p) for p in func.args.args]
+    # adds info about *args and **kwargs to the function doc
+    has_varargs = func.args.vararg is not None
+    has_kwargs = func.args.kwarg is not None
     defaults = func.args.defaults
     if defaults:
         for param, default in zip(params[-len(defaults) :], defaults):
@@ -176,6 +190,8 @@ def ast_func_info(func: ast.FunctionDef):
         params=params,
         returns=param_dict(type=rtype),
         lineno=func.lineno,
+        has_varargs=has_varargs,
+        has_kwargs=has_kwargs,
     )
 
     if not doc.strip():
@@ -195,16 +211,33 @@ def ast_param_dict(param: ast.arg) -> dict:
 
 def ann_type(ann):
     if hasattr(ann, "slice"):
-        name = ann.value.id
+        if isinstance(ann.value, ast.Attribute):
+            # value is an attribute, e.g. b of a.b - get the full path
+            name = get_attr_path(ann.value)
+        else:
+            name = ann.value.id
         inner = ", ".join(ann_type(e) for e in iter_elems(ann.slice))
         return f"{name}[{inner}]"
 
     if isinstance(ann, ast.Attribute):
+        if isinstance(ann.value, ast.Attribute):
+            # value is an attribute, e.g. b of a.b - get the full path
+            return get_attr_path(ann)
+
         return ann.attr
 
     return getattr(ann, "id", "")
 
 
+def get_attr_path(ann: ast.Attribute):
+    if isinstance(ann.value, ast.Attribute):
+        # value is an attribute, e.g. b of a.b - get the full path
+        return f"{get_attr_path(ann.value)}.{ann.attr}"
+
+    # value can be a subscript or name - get its annotation type and append the attribute
+    return f"{ann_type(ann.value)}.{ann.attr}"
+
+
 def iter_elems(ann):
     """
     Gets the elements of an ast.Subscript.slice, e.g. Union[int, str] -> [int, str]
@@ -219,10 +252,13 @@ def iter_elems(ann):
         return [ann.value]
 
     # From python 3.9, slice is an expr and we should evaluate it recursively. Left this for backward compatibility.
-    elif hasattr(ann.slice, "elts"):
-        return ann.slice.elts
-    elif hasattr(ann.slice, "value"):
-        return [ann.slice.value]
+    # TODO: Remove this in 1.5.0 when we drop support for python 3.7
+    if sys.version_info < (3, 9):
+        if hasattr(ann.slice, "elts"):
+            return ann.slice.elts
+        elif hasattr(ann.slice, "value"):
+            return [ann.slice.value]
+
     return [ann]