mlrun 1.3.1rc5__py3-none-any.whl → 1.4.0rc2__py3-none-any.whl

mlrun/feature_store/steps.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import math
 import re
 import uuid
 import warnings
@@ -55,7 +56,7 @@ class MLRunStep(MapClass):
         engine = get_engine(event)
         self.do = self._engine_to_do_method.get(engine, None)
         if self.do is None:
-            raise mlrun.errors.InvalidArgummentError(
+            raise mlrun.errors.MLRunInvalidArgumentError(
                 f"Unrecognized engine: {engine}. Available engines are: pandas, spark and storey"
             )
 
@@ -136,7 +137,7 @@ class MapValues(StepToDict, MLRunStep):
 
     def __init__(
         self,
-        mapping: Dict[str, Dict[str, Any]],
+        mapping: Dict[str, Dict[Union[str, int, bool], Any]],
         with_original_features: bool = False,
         suffix: str = "mapped",
         **kwargs,
@@ -226,34 +227,130 @@ class MapValues(StepToDict, MLRunStep):
     def _do_spark(self, event):
         from itertools import chain
 
-        from pyspark.sql.functions import col, create_map, lit, when
+        from pyspark.sql.functions import col, create_map, isnan, isnull, lit, when
+        from pyspark.sql.types import DecimalType, DoubleType, FloatType
+        from pyspark.sql.utils import AnalysisException
 
+        df = event
+        source_column_names = df.columns
         for column, column_map in self.mapping.items():
             new_column_name = self._get_feature_name(column)
-            if "ranges" not in column_map:
+            if not self.get_ranges_key() in column_map:
+                if column not in source_column_names:
+                    continue
                 mapping_expr = create_map([lit(x) for x in chain(*column_map.items())])
-                event = event.withColumn(
-                    new_column_name, mapping_expr.getItem(col(column))
-                )
+                try:
+                    df = df.withColumn(
+                        new_column_name,
+                        when(
+                            col(column).isin(list(column_map.keys())),
+                            mapping_expr.getItem(col(column)),
+                        ).otherwise(col(column)),
+                    )
+                #  if failed to use otherwise it is probably because the new column has different type
+                #  then the original column.
+                #  we will try to replace the values without using 'otherwise'.
+                except AnalysisException:
+                    df = df.withColumn(
+                        new_column_name, mapping_expr.getItem(col(column))
+                    )
+                    col_type = df.schema[column].dataType
+                    new_col_type = df.schema[new_column_name].dataType
+                    #  in order to avoid exception at isna on non-decimal/float columns -
+                    #  we need to check their types before filtering.
+                    if isinstance(col_type, (FloatType, DoubleType, DecimalType)):
+                        column_filter = (~isnull(col(column))) & (~isnan(col(column)))
+                    else:
+                        column_filter = ~isnull(col(column))
+                    if isinstance(new_col_type, (FloatType, DoubleType, DecimalType)):
+                        new_column_filter = isnull(col(new_column_name)) | isnan(
+                            col(new_column_name)
+                        )
+                    else:
+                        #  we need to check that every value replaced if we changed column type - except None or NaN.
+                        new_column_filter = isnull(col(new_column_name))
+                    mapping_to_null = [
+                        k
+                        for k, v in column_map.items()
+                        if v is None
+                        or (
+                            isinstance(v, (float, np.float64, np.float32, np.float16))
+                            and math.isnan(v)
+                        )
+                    ]
+                    turned_to_none_values = df.filter(
+                        column_filter & new_column_filter
+                    ).filter(~col(column).isin(mapping_to_null))
+
+                    if len(turned_to_none_values.head(1)) > 0:
+                        raise mlrun.errors.MLRunInvalidArgumentError(
+                            f"MapValues - mapping that changes column type must change all values accordingly,"
+                            f" which is not the case for column '{column}'"
+                        )
             else:
                 for val, val_range in column_map["ranges"].items():
                     min_val = val_range[0] if val_range[0] != "-inf" else -np.inf
                     max_val = val_range[1] if val_range[1] != "inf" else np.inf
                     otherwise = ""
-                    if new_column_name in event.columns:
-                        otherwise = event[new_column_name]
-                    event = event.withColumn(
+                    if new_column_name in df.columns:
+                        otherwise = df[new_column_name]
+                    df = df.withColumn(
                         new_column_name,
                         when(
-                            (event[column] < max_val) & (event[column] >= min_val),
+                            (df[column] < max_val) & (df[column] >= min_val),
                             lit(val),
                         ).otherwise(otherwise),
                     )
 
         if not self.with_original_features:
-            event = event.select(*self.mapping.keys())
+            df = df.select(*self.mapping.keys())
 
-        return event
+        return df
+
+    @classmethod
+    def validate_args(cls, feature_set, **kwargs):
+        mapping = kwargs.get("mapping", [])
+        for column, column_map in mapping.items():
+            if not cls.get_ranges_key() in column_map:
+                types = set(
+                    type(val)
+                    for val in column_map.values()
+                    if type(val) is not None
+                    and not (
+                        isinstance(val, (float, np.float64, np.float32, np.float16))
+                        and math.isnan(val)
+                    )
+                )
+            else:
+                if len(column_map) > 1:
+                    raise mlrun.errors.MLRunInvalidArgumentError(
+                        f"MapValues - mapping values of the same column can not combine ranges and "
+                        f"single replacement, which is the case for column '{column}'"
+                    )
+                ranges_dict = column_map[cls.get_ranges_key()]
+                types = set()
+                for ranges_mapping_values in ranges_dict.values():
+                    range_types = set(
+                        type(val)
+                        for val in ranges_mapping_values
+                        if type(val) is not None
+                        and val != "-inf"
+                        and val != "inf"
+                        and not (
+                            isinstance(val, (float, np.float64, np.float32, np.float16))
+                            and math.isnan(val)
+                        )
+                    )
+                    types = types.union(range_types)
+            if len(types) > 1:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"MapValues - mapping values of the same column must be in the"
+                    f" same type, which was not the case for Column '{column}'"
+                )
+
+    @staticmethod
+    def get_ranges_key():
+        return "ranges"
 
 
 class Imputer(StepToDict, MLRunStep):

mlrun/k8s_utils.py

@@ -508,6 +508,16 @@ class K8sHelper:
         k8s_secret.data = secret_data
         self.v1api.replace_namespaced_secret(secret_name, namespace, k8s_secret)
 
+    def load_secret(self, secret_name, namespace=""):
+        namespace = namespace or self.resolve_namespace(namespace)
+
+        try:
+            k8s_secret = self.v1api.read_namespaced_secret(secret_name, namespace)
+        except ApiException:
+            return None
+
+        return k8s_secret.data
+
     def delete_project_secrets(self, project, secrets, namespace=""):
         secret_name = self.get_project_secret_name(project)
         self.delete_secrets(secret_name, secrets, namespace)

mlrun/model.py

@@ -338,6 +338,7 @@ class ImageBuilder(ModelObj):
         origin_filename=None,
         with_mlrun=None,
         auto_build=None,
+        requirements: list = None,
     ):
         self.functionSourceCode = functionSourceCode  #: functionSourceCode
         self.codeEntryType = ""  #: codeEntryType
@@ -355,6 +356,7 @@ class ImageBuilder(ModelObj):
         self.with_mlrun = with_mlrun  #: with_mlrun
         self.auto_build = auto_build  #: auto_build
         self.build_pod = None
+        self.requirements = requirements or []  #: pip requirements
 
     @property
     def source(self):
@@ -376,6 +378,32 @@ class ImageBuilder(ModelObj):
         self._source = source
 
 
+class Notification(ModelObj):
+    """Notification specification"""
+
+    def __init__(
+        self,
+        kind=None,
+        name=None,
+        message=None,
+        severity=None,
+        when=None,
+        condition=None,
+        params=None,
+        status=None,
+        sent_time=None,
+    ):
+        self.kind = kind
+        self.name = name
+        self.message = message
+        self.severity = severity
+        self.when = when
+        self.condition = condition
+        self.params = params or {}
+        self.status = status
+        self.sent_time = sent_time
+
+
 class RunMetadata(ModelObj):
     """Run metadata"""
 
@@ -492,6 +520,7 @@ class RunSpec(ModelObj):
         allow_empty_resources=None,
         inputs_type_hints=None,
         returns=None,
+        notifications=None,
     ):
         # A dictionary of parsing configurations that will be read from the inputs the user set. The keys are the inputs
         # keys (parameter names) and the values are the type hint given in the input keys after the colon.
@@ -526,6 +555,7 @@ class RunSpec(ModelObj):
         self.verbose = verbose
         self.scrape_metrics = scrape_metrics
         self.allow_empty_resources = allow_empty_resources
+        self._notifications = notifications or []
 
     def to_dict(self, fields=None, exclude=None):
         struct = super().to_dict(fields, exclude=["handler"])
@@ -678,6 +708,19 @@ class RunSpec(ModelObj):
                 return str(self.handler)
         return ""
 
+    @property
+    def notifications(self):
+        return self._notifications
+
+    @notifications.setter
+    def notifications(self, notifications):
+        if isinstance(notifications, list):
+            self._notifications = ObjectList.from_list(Notification, notifications)
+        elif isinstance(notifications, ObjectList):
+            self._notifications = notifications
+        else:
+            raise ValueError("Notifications must be a list")
+
     def extract_type_hints_from_inputs(self):
         """
         This method extracts the type hints from the inputs keys in the input dictionary.

mlrun/projects/operations.py

@@ -72,6 +72,7 @@ def run_function(
     auto_build: bool = None,
     schedule: Union[str, mlrun.api.schemas.ScheduleCronTrigger] = None,
     artifact_path: str = None,
+    notifications: List[mlrun.model.Notification] = None,
     returns: Optional[List[Union[str, Dict[str, str]]]] = None,
 ) -> Union[mlrun.model.RunObject, kfp.dsl.ContainerOp]:
     """Run a local or remote task as part of a local/kubeflow pipeline
@@ -91,7 +92,7 @@ def run_function(
         LABELS = "is_error"
         MODEL_CLASS = "sklearn.ensemble.RandomForestClassifier"
         DATA_PATH = "s3://bigdata/data.parquet"
-        function = mlrun.import_function("hub://auto_trainer")
+        function = mlrun.import_function("hub://auto-trainer")
         run1 = run_function(function, params={"label_columns": LABELS, "model_class": MODEL_CLASS},
                                       inputs={"dataset": DATA_PATH})
 
@@ -100,7 +101,7 @@ def run_function(
         # create a project with two functions (local and from marketplace)
         project = mlrun.new_project(project_name, "./proj)
         project.set_function("mycode.py", "myfunc", image="mlrun/mlrun")
-        project.set_function("hub://auto_trainer", "train")
+        project.set_function("hub://auto-trainer", "train")
 
         # run functions (refer to them by name)
         run1 = run_function("myfunc", params={"x": 7})
@@ -143,6 +144,7 @@ def run_function(
                             see this link for help:
                             https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
     :param artifact_path:   path to store artifacts, when running in a workflow this will be set automatically
+    :param notifications:   list of notifications to push when the run is completed
     :param returns:         List of log hints - configurations for how to log the returning values from the handler's
                             run (as artifacts or results). The list's length must be equal to the amount of returning
                             objects. A log hint may be given as:
@@ -198,6 +200,7 @@ def run_function(
             or (project.artifact_path if project else None),
             auto_build=auto_build,
             schedule=schedule,
+            notifications=notifications,
         )
         if run_result:
             run_result._notified = False

mlrun/projects/pipelines.py

@@ -705,7 +705,7 @@ class _LocalRunner(_PipelineRunner):
             trace = traceback.format_exc()
             logger.error(trace)
             project.notifiers.push(
-                f"Workflow {workflow_id} run failed!, error: {e}\n{trace}", "error"
+                f":x: Workflow {workflow_id} run failed!, error: {e}\n{trace}", "error"
             )
             state = mlrun.run.RunStatuses.failed
         mlrun.run.wait_for_runs_completion(pipeline_context.runs_map.values())
@@ -874,7 +874,8 @@ class _RemoteRunner(_PipelineRunner):
             trace = traceback.format_exc()
             logger.error(trace)
             project.notifiers.push(
-                f"Workflow {workflow_name} run failed!, error: {e}\n{trace}", "error"
+                f":x: Workflow {workflow_name} run failed!, error: {e}\n{trace}",
+                "error",
             )
             state = mlrun.run.RunStatuses.failed
             return _PipelineRunStatus(
@@ -1033,7 +1034,7 @@ def load_and_run(
             try:
                 notification_pusher.push(
                     message=message,
-                    severity=mlrun.utils.notifications.NotificationSeverity.ERROR,
+                    severity=mlrun.api.schemas.NotificationSeverity.ERROR,
                 )
 
             except Exception as exc:

mlrun/projects/project.py

@@ -117,7 +117,7 @@ def new_project(
         # create a project with local and marketplace functions, a workflow, and an artifact
         project = mlrun.new_project("myproj", "./", init_git=True, description="my new project")
         project.set_function('prep_data.py', 'prep-data', image='mlrun/mlrun', handler='prep_data')
-        project.set_function('hub://auto_trainer', 'train')
+        project.set_function('hub://auto-trainer', 'train')
         project.set_artifact('data', Artifact(target_path=data_url))
         project.set_workflow('main', "./myflow.py")
         project.save()
@@ -169,7 +169,13 @@ def new_project(
         # Remove original owner name for avoiding possible conflicts
         project.spec.owner = None
     else:
-        project = MlrunProject(name=name)
+        project = MlrunProject.from_dict(
+            {
+                "metadata": {
+                    "name": name,
+                }
+            }
+        )
     project.spec.context = context
     project.spec.subpath = subpath or project.spec.subpath
 
@@ -419,9 +425,15 @@ def _load_project_dir(context, name="", subpath=""):
 
     elif path.isfile(path.join(context, subpath_str, "function.yaml")):
         func = import_function(path.join(context, subpath_str, "function.yaml"))
-        project = MlrunProject(
-            name=func.metadata.project,
-            functions=[{"url": "function.yaml", "name": func.metadata.name}],
+        project = MlrunProject.from_dict(
+            {
+                "metadata": {
+                    "name": func.metadata.project,
+                },
+                "spec": {
+                    "functions": [{"url": "function.yaml", "name": func.metadata.name}],
+                },
+            }
         )
     else:
         raise mlrun.errors.MLRunNotFoundError(
@@ -769,6 +781,7 @@ class MlrunProject(ModelObj):
 
     def __init__(
         self,
+        # TODO: remove all arguments except metadata and spec in 1.6.0
         name=None,
         description=None,
         params=None,
@@ -777,7 +790,7 @@ class MlrunProject(ModelObj):
         artifacts=None,
         artifact_path=None,
         conda=None,
-        # all except these 2 are for backwards compatibility with MlrunProjectLegacy
+        # all except these metadata and spec are for backwards compatibility with MlrunProjectLegacy
         metadata=None,
         spec=None,
         default_requirements: typing.Union[str, typing.List[str]] = None,
@@ -789,6 +802,26 @@ class MlrunProject(ModelObj):
         self._status = None
         self.status = None
 
+        if any(
+            [
+                name,
+                description,
+                params,
+                functions,
+                workflows,
+                artifacts,
+                artifact_path,
+                conda,
+                default_requirements,
+            ]
+        ):
+            # TODO: remove in 1.6.0 along with all arguments except metadata and spec
+            warnings.warn(
+                "Project constructor arguments are deprecated in 1.4.0 and will be removed in 1.6.0,"
+                " use metadata and spec instead",
+                FutureWarning,
+            )
+
         # Handling the fields given in the legacy way
         self.metadata.name = name or self.metadata.name
         self.spec.description = description or self.spec.description
@@ -1436,12 +1469,15 @@ class MlrunProject(ModelObj):
                 with open(f"{temp_dir}/_body", "rb") as fp:
                     artifact.spec._body = fp.read()
                 artifact.target_path = ""
+
+                # if the dataitem is not a file, it means we downloaded it from a remote source to a temp file,
+                # so we need to remove it after we're done with it
+                dataitem.remove_local()
+
                 return self.log_artifact(
                     artifact, local_path=temp_dir, artifact_path=artifact_path
                 )
 
-            if dataitem.kind != "file":
-                remove(item_file)
         else:
             raise ValueError("unsupported file suffix, use .yaml, .json, or .zip")
 
@@ -1489,7 +1525,7 @@ class MlrunProject(ModelObj):
 
             object (s3://, v3io://, ..)
             MLRun DB e.g. db://project/func:ver
-            functions hub/market: e.g. hub://auto_trainer:master
+            functions hub/market: e.g. hub://auto-trainer:master
 
         examples::
 
@@ -1536,6 +1572,7 @@ class MlrunProject(ModelObj):
                 func = path.relpath(func, self.spec.context)
 
         func = func or ""
+        name = mlrun.utils.normalize_name(name) if name else name
         if isinstance(func, str):
             # in hub or db functions name defaults to the function name
             if not name and not (func.startswith("db://") or func.startswith("hub://")):
@@ -2151,6 +2188,7 @@ class MlrunProject(ModelObj):
         auto_build: bool = None,
         schedule: typing.Union[str, mlrun.api.schemas.ScheduleCronTrigger] = None,
         artifact_path: str = None,
+        notifications: typing.List[mlrun.model.Notification] = None,
         returns: Optional[List[Union[str, Dict[str, str]]]] = None,
     ) -> typing.Union[mlrun.model.RunObject, kfp.dsl.ContainerOp]:
         """Run a local or remote task as part of a local/kubeflow pipeline
@@ -2160,7 +2198,7 @@ class MlrunProject(ModelObj):
             # create a project with two functions (local and from marketplace)
             project = mlrun.new_project(project_name, "./proj")
             project.set_function("mycode.py", "myfunc", image="mlrun/mlrun")
-            project.set_function("hub://auto_trainer", "train")
+            project.set_function("hub://auto-trainer", "train")
 
             # run functions (refer to them by name)
             run1 = project.run_function("myfunc", params={"x": 7})
@@ -2192,6 +2230,7 @@ class MlrunProject(ModelObj):
                                 see this link for help:
                                 https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
         :param artifact_path:   path to store artifacts, when running in a workflow this will be set automatically
+        :param notifications:   list of notifications to push when the run is completed
         :param returns:         List of log hints - configurations for how to log the returning values from the
                                 handler's run (as artifacts or results). The list's length must be equal to the amount
                                 of returning objects. A log hint may be given as:
@@ -2226,6 +2265,7 @@ class MlrunProject(ModelObj):
             auto_build=auto_build,
             schedule=schedule,
             artifact_path=artifact_path,
+            notifications=notifications,
             returns=returns,
         )
 

mlrun/run.py

@@ -57,7 +57,6 @@ from .runtimes import (
     RemoteSparkRuntime,
     RuntimeKinds,
     ServingRuntime,
-    Spark2Runtime,
     Spark3Runtime,
     get_runtime_class,
 )
@@ -127,6 +126,7 @@ def run_local(
     artifact_path: str = "",
     mode: str = None,
     allow_empty_resources=None,
+    notifications: List[mlrun.model.Notification] = None,
     returns: list = None,
 ):
     """Run a task on function/code (.py, .ipynb or .yaml) locally,
@@ -216,6 +216,7 @@ def run_local(
         inputs=inputs,
         returns=returns,
         artifact_path=artifact_path,
+        notifications=notifications,
     )
 
 
@@ -235,7 +236,7 @@ def function_to_module(code="", workdir=None, secrets=None, silent=False):
         mod.my_job(context, p1=1, p2='x')
         print(context.to_yaml())
 
-        fn = mlrun.import_function('hub://open_archive')
+        fn = mlrun.import_function('hub://open-archive')
         mod = mlrun.function_to_module(fn)
         data = mlrun.run.get_dataitem("https://fpsignals-public.s3.amazonaws.com/catsndogs.tar.gz")
         context = mlrun.get_or_create_ctx('myfunc')
@@ -457,7 +458,7 @@ def import_function(url="", secrets=None, db="", project=None, new_name=None):
 
     examples::
 
-        function = mlrun.import_function("hub://auto_trainer")
+        function = mlrun.import_function("hub://auto-trainer")
         function = mlrun.import_function("./func.yaml")
         function = mlrun.import_function("https://raw.githubusercontent.com/org/repo/func.yaml")
 
@@ -700,7 +701,6 @@ def code_to_function(
     DaskCluster,
     KubejobRuntime,
     LocalRuntime,
-    Spark2Runtime,
     Spark3Runtime,
     RemoteSparkRuntime,
 ]:
@@ -793,6 +793,7 @@ def code_to_function(
 
     def update_common(fn, spec):
         fn.spec.image = image or get_in(spec, "spec.image", "")
+        fn.spec.filename = filename or get_in(spec, "spec.filename", "")
         fn.spec.build.base_image = get_in(spec, "spec.build.baseImage")
         fn.spec.build.commands = get_in(spec, "spec.build.commands")
         fn.spec.build.secret = get_in(spec, "spec.build.secret")

mlrun/runtimes/__init__.py

@@ -47,7 +47,7 @@ from .mpijob import (  # noqa
 from .nuclio import nuclio_init_hook
 from .remotesparkjob import RemoteSparkRuntime, RemoteSparkRuntimeHandler
 from .serving import ServingRuntime, new_v2_model_server
-from .sparkjob import Spark2Runtime, Spark3Runtime, SparkRuntimeHandler
+from .sparkjob import Spark3Runtime, SparkRuntimeHandler
 
 # for legacy imports (MLModelServer moved from here to /serving)
 from ..serving import MLModelServer, new_v1_model_server  # noqa isort: skip
@@ -260,11 +260,7 @@ def get_runtime_class(kind: str):
         return crd_version_to_runtime[mpijob_crd_version]
 
     if kind == RuntimeKinds.spark:
-        spark_operator_version = resolve_spark_operator_version()
-        if spark_operator_version == 2:
-            return Spark2Runtime
-        elif spark_operator_version == 3:
-            return Spark3Runtime
+        return Spark3Runtime
 
     kind_runtime_map = {
         RuntimeKinds.remote: RemoteRuntime,