mlrun 1.4.0rc25__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

mlrun/api/crud/notifications.py

@@ -34,13 +34,18 @@ class Notifications(
         notification_objects: typing.List[mlrun.model.Notification],
         run_uid: str,
         project: str = None,
+        mask_params: bool = True,
     ):
         project = project or mlrun.mlconf.default_project
-        notification_objects_to_store = (
-            mlrun.api.api.utils.validate_and_mask_notification_list(
-                notification_objects, run_uid, project
+
+        # we don't mask the notification params when it's a status update as they are already masked
+        notification_objects_to_store = notification_objects
+        if mask_params:
+            notification_objects_to_store = (
+                mlrun.api.api.utils.validate_and_mask_notification_list(
+                    notification_objects, run_uid, project
+                )
             )
-        )
 
         mlrun.api.utils.singletons.db.get_db().store_run_notifications(
             session, notification_objects_to_store, run_uid, project

mlrun/api/crud/pipelines.py

@@ -25,6 +25,7 @@ import sqlalchemy.orm
 
 import mlrun
 import mlrun.api.api.utils
+import mlrun.common.helpers
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.kfpops
@@ -94,7 +95,7 @@ class Pipelines(
             runs = [run.to_dict() for run in response.runs or []]
             total_size = response.total_size
             next_page_token = response.next_page_token
-        runs = self._format_runs(db_session, runs, format_)
+        runs = self._format_runs(runs, format_)
 
         return total_size, next_page_token, runs
 
@@ -130,7 +131,6 @@ class Pipelines(
 
     def get_pipeline(
         self,
-        db_session: sqlalchemy.orm.Session,
         run_id: str,
         project: typing.Optional[str] = None,
         namespace: typing.Optional[str] = None,
@@ -148,9 +148,7 @@ class Pipelines(
                         raise mlrun.errors.MLRunNotFoundError(
                             f"Pipeline run with id {run_id} is not of project {project}"
                         )
-                run = self._format_run(
-                    db_session, run, format_, api_run_detail.to_dict()
-                )
+                run = self._format_run(run, format_, api_run_detail.to_dict())
         except kfp_server_api.ApiException as exc:
             mlrun.errors.raise_for_status_code(int(exc.status), err_to_str(exc))
         except mlrun.errors.MLRunHTTPStatusError:
@@ -227,18 +225,16 @@ class Pipelines(
 
     def _format_runs(
         self,
-        db_session: sqlalchemy.orm.Session,
         runs: typing.List[dict],
         format_: mlrun.common.schemas.PipelinesFormat = mlrun.common.schemas.PipelinesFormat.metadata_only,
     ) -> typing.List[dict]:
         formatted_runs = []
         for run in runs:
-            formatted_runs.append(self._format_run(db_session, run, format_))
+            formatted_runs.append(self._format_run(run, format_))
         return formatted_runs
 
     def _format_run(
         self,
-        db_session: sqlalchemy.orm.Session,
         run: dict,
         format_: mlrun.common.schemas.PipelinesFormat = mlrun.common.schemas.PipelinesFormat.metadata_only,
         api_run_detail: typing.Optional[dict] = None,
@@ -257,7 +253,7 @@ class Pipelines(
                     "The full kfp api_run_detail object is needed to generate the summary format"
                 )
             return mlrun.kfpops.format_summary_from_kfp_run(
-                api_run_detail, run["project"], db_session
+                api_run_detail, run["project"]
             )
         else:
             raise NotImplementedError(
@@ -290,7 +286,7 @@ class Pipelines(
                             _,
                             _,
                             _,
-                        ) = mlrun.utils.helpers.parse_versioned_object_uri(
+                        ) = mlrun.common.helpers.parse_versioned_object_uri(
                             function_url[len("db://") :]
                         )
                         if project:
@@ -316,7 +312,6 @@ class Pipelines(
         return None
 
     def resolve_project_from_pipeline(self, pipeline):
-
         workflow_manifest = json.loads(
             pipeline.get("pipeline_spec", {}).get("workflow_manifest") or "{}"
         )

mlrun/api/crud/projects.py

@@ -24,7 +24,7 @@ import sqlalchemy.orm
 import mlrun.api.crud
 import mlrun.api.db.session
 import mlrun.api.utils.events.events_factory as events_factory
-import mlrun.api.utils.projects.remotes.follower
+import mlrun.api.utils.projects.remotes.follower as project_follower
 import mlrun.api.utils.singletons.db
 import mlrun.api.utils.singletons.k8s
 import mlrun.api.utils.singletons.scheduler
@@ -35,7 +35,7 @@ from mlrun.utils import logger
 
 
 class Projects(
-    mlrun.api.utils.projects.remotes.follower.Member,
+    project_follower.Member,
     metaclass=mlrun.utils.singleton.AbstractSingleton,
 ):
     def __init__(self) -> None:

mlrun/api/crud/runtime_resources.py

@@ -18,6 +18,7 @@ import mergedeep
 import sqlalchemy.orm
 
 import mlrun.api.api.utils
+import mlrun.api.runtime_handlers
 import mlrun.api.utils.projects.remotes.follower
 import mlrun.api.utils.singletons.db
 import mlrun.common.schemas
@@ -50,7 +51,7 @@ class RuntimeResources(
             self.validate_runtime_resources_kind(kind)
             kinds = [kind]
         for kind in kinds:
-            runtime_handler = mlrun.runtimes.get_runtime_handler(kind)
+            runtime_handler = mlrun.api.runtime_handlers.get_runtime_handler(kind)
             resources = runtime_handler.list_resources(
                 project, object_id, label_selector, group_by
             )
@@ -88,7 +89,7 @@ class RuntimeResources(
                     )
         runtimes_resources_output = [] if group_by is None else {}
         for kind, runtime_resources_list in runtime_resources_by_kind.items():
-            runtime_handler = mlrun.runtimes.get_runtime_handler(kind)
+            runtime_handler = mlrun.api.runtime_handlers.get_runtime_handler(kind)
             resources = runtime_handler.build_output_from_runtime_resources(
                 runtime_resources_list, group_by
             )
@@ -122,7 +123,7 @@ class RuntimeResources(
             self.validate_runtime_resources_kind(kind)
             kinds = [kind]
         for kind in kinds:
-            runtime_handler = mlrun.runtimes.get_runtime_handler(kind)
+            runtime_handler = mlrun.api.runtime_handlers.get_runtime_handler(kind)
             if object_id:
                 runtime_handler.delete_runtime_object_resources(
                     mlrun.api.utils.singletons.db.get_db(),

mlrun/api/crud/runtimes/nuclio/helpers.py

@@ -17,6 +17,8 @@ import urllib.parse
 import semver
 
 import mlrun
+import mlrun.api.utils.clients.nuclio
+import mlrun.api.utils.runtimes.nuclio
 import mlrun.api.utils.singletons.k8s
 import mlrun.runtimes
 from mlrun.utils import logger
@@ -190,7 +192,9 @@ def is_nuclio_version_in_range(min_version: str, max_version: str) -> bool:
     try:
         parsed_min_version = semver.VersionInfo.parse(min_version)
         parsed_max_version = semver.VersionInfo.parse(max_version)
-        resolved_nuclio_version = mlrun.runtimes.utils.resolve_nuclio_version()
+        resolved_nuclio_version = (
+            mlrun.api.utils.runtimes.nuclio.resolve_nuclio_version()
+        )
         parsed_current_version = semver.VersionInfo.parse(resolved_nuclio_version)
     except ValueError:
         logger.warning(

mlrun/api/crud/secrets.py

@@ -17,9 +17,11 @@ import json
 import typing
 import uuid
 
+import mlrun.api
 import mlrun.api.utils.clients.iguazio
 import mlrun.api.utils.events.events_factory as events_factory
 import mlrun.api.utils.singletons.k8s
+import mlrun.common
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.helpers
@@ -531,3 +533,22 @@ class Secrets(
     @staticmethod
     def _generate_uuid() -> str:
         return str(uuid.uuid4())
+
+
+def get_project_secret_provider(project: str) -> typing.Callable:
+    """Implement secret provider for handle the related project secret on the API side.
+
+    :param project: Project name.
+
+    :return: A secret provider function.
+    """
+
+    def secret_provider(key: str):
+        return mlrun.api.crud.secrets.Secrets().get_project_secret(
+            project=project,
+            provider=mlrun.common.schemas.secret.SecretProviderName.kubernetes,
+            allow_secrets_from_k8s=True,
+            secret_key=key,
+        )
+
+    return secret_provider

mlrun/api/crud/workflows.py

@@ -0,0 +1,352 @@
+# Copyright 2018 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import uuid
+from typing import Dict
+
+from sqlalchemy.orm import Session
+
+import mlrun.common.schemas
+import mlrun.utils.singleton
+from mlrun.api.api.utils import (
+    apply_enrichment_and_validation_on_function,
+    get_run_db_instance,
+    get_scheduler,
+)
+from mlrun.config import config
+from mlrun.model import Credentials, RunMetadata, RunObject, RunSpec
+
+
+class WorkflowRunners(
+    metaclass=mlrun.utils.singleton.Singleton,
+):
+    @staticmethod
+    def create_runner(
+        run_name: str,
+        project: str,
+        db_session: Session,
+        auth_info: mlrun.common.schemas.AuthInfo,
+        image: str,
+    ) -> mlrun.run.KubejobRuntime:
+        """
+        Creating the base object for the workflow runner function with
+        all the necessary metadata to create it on server-side.
+
+        :param run_name:    workflow-runner function name
+        :param project:     project name
+        :param db_session:  session that manages the current dialog with the database
+        :param auth_info:   auth info of the request
+        :param image:       image for the workflow runner job
+
+        :returns: workflow runner object
+        """
+        runner = mlrun.new_function(
+            name=run_name,
+            project=project,
+            kind=mlrun.runtimes.RuntimeKinds.job,
+            # For preventing deployment:
+            image=image,
+        )
+
+        runner.set_db_connection(get_run_db_instance(db_session))
+
+        # Enrichment and validation requires access key
+        runner.metadata.credentials.access_key = Credentials.generate_access_key
+
+        apply_enrichment_and_validation_on_function(
+            function=runner,
+            auth_info=auth_info,
+        )
+
+        runner.save()
+        return runner
+
+    def schedule(
+        self,
+        runner: mlrun.run.KubejobRuntime,
+        project: mlrun.common.schemas.Project,
+        workflow_request: mlrun.common.schemas.WorkflowRequest,
+        db_session: Session = None,
+        auth_info: mlrun.common.schemas.AuthInfo = None,
+    ):
+        """
+        Schedule workflow runner.
+
+        :param runner:              workflow runner function object
+        :param project:             MLRun project
+        :param workflow_request:    contains the workflow spec, that will be scheduled
+        :param db_session:          session that manages the current dialog with the database
+        :param auth_info:           auth info of the request
+        """
+        labels = {
+            "job-type": "workflow-runner",
+            "workflow": workflow_request.spec.name,
+        }
+
+        run_spec = self._prepare_run_object_for_scheduling(
+            project=project,
+            workflow_request=workflow_request,
+            labels=labels,
+        )
+        # this includes filling the spec.function which is required for submit run
+        runner._store_function(
+            runspec=run_spec, meta=run_spec.metadata, db=runner._get_db()
+        )
+        workflow_spec = workflow_request.spec
+        schedule = workflow_spec.schedule
+        scheduled_object = {
+            "task": run_spec.to_dict(),
+            "schedule": schedule,
+        }
+
+        get_scheduler().store_schedule(
+            db_session=db_session,
+            auth_info=auth_info,
+            project=project.metadata.name,
+            name=workflow_spec.name,
+            scheduled_object=scheduled_object,
+            cron_trigger=schedule,
+            labels=runner.metadata.labels,
+            kind=mlrun.common.schemas.ScheduleKinds.job,
+        )
+
+    def _prepare_run_object_for_scheduling(
+        self,
+        project: mlrun.common.schemas.Project,
+        workflow_request: mlrun.common.schemas.WorkflowRequest,
+        labels: Dict[str, str],
+    ) -> mlrun.run.RunObject:
+        """
+        Preparing all the necessary metadata and specifications for scheduling workflow from server-side.
+
+        :param project:             MLRun project
+        :param workflow_request:    contains the workflow spec and extra data for the run object
+        :param labels:              dict of the task labels
+
+        :returns: RunObject ready for schedule.
+        """
+        meta_uid = uuid.uuid4().hex
+
+        save = self._set_source(project, workflow_request.source)
+        workflow_spec = workflow_request.spec
+        run_object = RunObject(
+            spec=RunSpec(
+                parameters=dict(
+                    url=project.spec.source,
+                    project_name=project.metadata.name,
+                    workflow_name=workflow_spec.name,
+                    workflow_path=workflow_spec.path,
+                    workflow_arguments=workflow_spec.args,
+                    artifact_path=workflow_request.artifact_path,
+                    workflow_handler=workflow_spec.handler,
+                    namespace=workflow_request.namespace,
+                    ttl=workflow_spec.ttl,
+                    engine=workflow_spec.engine,
+                    local=workflow_spec.run_local,
+                    save=save,
+                    subpath=project.spec.subpath,
+                ),
+                handler="mlrun.projects.load_and_run",
+                scrape_metrics=config.scrape_metrics,
+                output_path=(
+                    workflow_request.artifact_path or config.artifact_path
+                ).replace("{{run.uid}}", meta_uid),
+            ),
+            metadata=RunMetadata(
+                uid=meta_uid, name=workflow_spec.name, project=project.metadata.name
+            ),
+        )
+
+        # Setting labels:
+        return self._label_run_object(run_object, labels)
+
+    def run(
+        self,
+        runner: mlrun.run.KubejobRuntime,
+        project: mlrun.common.schemas.Project,
+        workflow_request: mlrun.common.schemas.WorkflowRequest = None,
+        load_only: bool = False,
+    ) -> RunObject:
+        """
+        Run workflow runner.
+
+        :param runner:              workflow runner function object
+        :param project:             MLRun project
+        :param workflow_request:    contains the workflow spec, that will be executed
+        :param load_only:           If True, will only load the project remotely (without running workflow)
+
+        :returns: run context object (RunObject) with run metadata, results and status
+        """
+        labels = {"project": project.metadata.name}
+        if load_only:
+            labels["job-type"] = "project-loader"
+        else:
+            labels["job-type"] = "workflow-runner"
+            labels["workflow"] = runner.metadata.name
+
+        run_spec = self._prepare_run_object_for_single_run(
+            project=project,
+            labels=labels,
+            workflow_request=workflow_request,
+            run_name=runner.metadata.name,
+            load_only=load_only,
+        )
+
+        artifact_path = workflow_request.artifact_path if workflow_request else ""
+        return runner.run(
+            runspec=run_spec,
+            artifact_path=artifact_path,
+            local=False,
+            watch=False,
+        )
+
+    @staticmethod
+    def get_workflow_id(
+        uid: str, project: str, engine: str, db_session: Session
+    ) -> mlrun.common.schemas.GetWorkflowResponse:
+        """
+        Retrieving the actual workflow id form the workflow runner
+
+        :param uid:         the id of the workflow runner job
+        :param project:     name of the project
+        :param engine:      pipeline runner, for example: "kfp"
+        :param db_session:  session that manages the current dialog with the database
+
+        :return: The id of the workflow.
+        """
+        # Reading run:
+        run = mlrun.api.crud.Runs().get_run(
+            db_session=db_session, uid=uid, iter=0, project=project
+        )
+        run_object = RunObject.from_dict(run)
+        state = run_object.status.state
+        workflow_id = None
+        if isinstance(run_object.status.results, dict):
+            workflow_id = run_object.status.results.get("workflow_id", None)
+
+        if workflow_id is None:
+            if (
+                engine == "local"
+                and state.casefold() == mlrun.run.RunStatuses.running.casefold()
+            ):
+                workflow_id = ""
+            else:
+                raise mlrun.errors.MLRunNotFoundError(
+                    f"workflow id of run {project}:{uid} not found"
+                )
+
+        return mlrun.common.schemas.GetWorkflowResponse(workflow_id=workflow_id)
+
+    def _prepare_run_object_for_single_run(
+        self,
+        project: mlrun.common.schemas.Project,
+        labels: Dict[str, str],
+        workflow_request: mlrun.common.schemas.WorkflowRequest = None,
+        run_name: str = None,
+        load_only: bool = False,
+    ) -> mlrun.run.RunObject:
+        """
+        Preparing all the necessary metadata and specifications for running workflow from server-side.
+
+        :param project:             MLRun project
+        :param labels:              dict of the task labels
+        :param workflow_request:    contains the workflow spec and extra data for the run object
+        :param run_name:            workflow-runner function name
+        :param load_only:           if True, will only load the project remotely (without running workflow)
+
+        :returns: RunObject ready for execution.
+        """
+        source = workflow_request.source if workflow_request else ""
+        save = self._set_source(project, source, load_only)
+        run_object = RunObject(
+            spec=RunSpec(
+                parameters=dict(
+                    url=project.spec.source,
+                    project_name=project.metadata.name,
+                    load_only=load_only,
+                    save=save,
+                ),
+                handler="mlrun.projects.load_and_run",
+            ),
+            metadata=RunMetadata(name=run_name),
+        )
+
+        if not load_only:
+            workflow_spec = workflow_request.spec
+            run_object.spec.parameters.update(
+                dict(
+                    workflow_name=workflow_spec.name,
+                    workflow_path=workflow_spec.path,
+                    workflow_arguments=workflow_spec.args,
+                    artifact_path=workflow_request.artifact_path,
+                    workflow_handler=workflow_spec.handler,
+                    namespace=workflow_request.namespace,
+                    ttl=workflow_spec.ttl,
+                    engine=workflow_spec.engine,
+                    local=workflow_spec.run_local,
+                    subpath=project.spec.subpath,
+                )
+            )
+
+        # Setting labels:
+        return self._label_run_object(run_object, labels)
+
+    @staticmethod
+    def _set_source(
+        project: mlrun.common.schemas.Project, source: str, load_only: bool = False
+    ) -> bool:
+        """
+        Setting the project source.
+        In case the user provided a source we want to load the project from the source
+        (like from a specific commit/branch from git repo) without changing the source of the project (save=False).
+
+        :param project:     MLRun project
+        :param source:      the source of the project, needs to be a remote URL that contains the project yaml file.
+        :param load_only:   if we only load the project, it must be saved to ensure we are not running a pipeline
+                            without a project as it's not supported.
+
+        :returns: True if the project need to be saved afterward.
+        """
+
+        save = True
+        if source and not load_only:
+            save = False
+            project.spec.source = source
+
+        if "://" not in project.spec.source:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Remote workflows can only be performed by a project with remote source (e.g git:// or http://),"
+                f" but the specified source '{project.spec.source}' is not remote. "
+                f"Either put your code in Git, or archive it and then set a source to it."
+                f" For more details, read"
+                f" https://docs.mlrun.org/en/latest/concepts/scheduled-jobs.html#scheduling-a-workflow"
+            )
+        return save
+
+    @staticmethod
+    def _label_run_object(
+        run_object: mlrun.run.RunObject,
+        labels: Dict[str, str],
+    ) -> mlrun.run.RunObject:
+        """
+        Setting labels to the task
+
+        :param run_object:  run object to set labels on
+        :param labels:      dict of labels
+
+        :returns: labeled RunObject
+        """
+        for key, value in labels.items():
+            run_object = run_object.set_label(key, value)
+        return run_object

mlrun/api/db/base.py

@@ -259,6 +259,21 @@ class DBInterface(ABC):
     ):
         pass
 
+    def store_schedule(
+        self,
+        session,
+        project: str,
+        name: str,
+        kind: mlrun.common.schemas.ScheduleKinds = None,
+        scheduled_object: Any = None,
+        cron_trigger: mlrun.common.schemas.ScheduleCronTrigger = None,
+        labels: Dict = None,
+        last_run_uri: str = None,
+        concurrency_limit: int = None,
+        next_run_time: datetime = None,
+    ):
+        pass
+
     @abstractmethod
     def list_schedules(
         self,
@@ -272,7 +287,7 @@ class DBInterface(ABC):
 
     @abstractmethod
     def get_schedule(
-        self, session, project: str, name: str
+        self, session, project: str, name: str, raise_on_not_found: bool = True
     ) -> mlrun.common.schemas.ScheduleRecord:
         pass
 

mlrun/api/db/init_db.py

@@ -14,10 +14,8 @@
 #
 
 from mlrun.api.db.sqldb.models import Base
-from mlrun.api.db.sqldb.session import get_engine
-from mlrun.config import config
+from mlrun.common.db.sql_session import get_engine
 
 
 def init_db() -> None:
-    if config.httpdb.db_type != "filedb":
-        Base.metadata.create_all(bind=get_engine())
+    Base.metadata.create_all(bind=get_engine())

mlrun/api/db/session.py

@@ -14,7 +14,7 @@
 #
 from sqlalchemy.orm import Session
 
-from mlrun.api.db.sqldb.session import create_session as sqldb_create_session
+from mlrun.common.db.sql_session import create_session as sqldb_create_session
 
 
 def create_session() -> Session: