mlrun 1.5.0rc1__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

mlrun/api/crud/pipelines.py

@@ -95,7 +95,7 @@ class Pipelines(
             runs = [run.to_dict() for run in response.runs or []]
             total_size = response.total_size
             next_page_token = response.next_page_token
-        runs = self._format_runs(db_session, runs, format_)
+        runs = self._format_runs(runs, format_)
 
         return total_size, next_page_token, runs
 
@@ -131,7 +131,6 @@ class Pipelines(
 
     def get_pipeline(
         self,
-        db_session: sqlalchemy.orm.Session,
         run_id: str,
         project: typing.Optional[str] = None,
         namespace: typing.Optional[str] = None,
@@ -149,9 +148,7 @@ class Pipelines(
                         raise mlrun.errors.MLRunNotFoundError(
                             f"Pipeline run with id {run_id} is not of project {project}"
                         )
-                run = self._format_run(
-                    db_session, run, format_, api_run_detail.to_dict()
-                )
+                run = self._format_run(run, format_, api_run_detail.to_dict())
         except kfp_server_api.ApiException as exc:
             mlrun.errors.raise_for_status_code(int(exc.status), err_to_str(exc))
         except mlrun.errors.MLRunHTTPStatusError:
@@ -228,18 +225,16 @@ class Pipelines(
 
     def _format_runs(
         self,
-        db_session: sqlalchemy.orm.Session,
         runs: typing.List[dict],
         format_: mlrun.common.schemas.PipelinesFormat = mlrun.common.schemas.PipelinesFormat.metadata_only,
     ) -> typing.List[dict]:
         formatted_runs = []
         for run in runs:
-            formatted_runs.append(self._format_run(db_session, run, format_))
+            formatted_runs.append(self._format_run(run, format_))
         return formatted_runs
 
     def _format_run(
         self,
-        db_session: sqlalchemy.orm.Session,
         run: dict,
         format_: mlrun.common.schemas.PipelinesFormat = mlrun.common.schemas.PipelinesFormat.metadata_only,
         api_run_detail: typing.Optional[dict] = None,
@@ -258,7 +253,7 @@ class Pipelines(
                     "The full kfp api_run_detail object is needed to generate the summary format"
                 )
             return mlrun.kfpops.format_summary_from_kfp_run(
-                api_run_detail, run["project"], db_session
+                api_run_detail, run["project"]
             )
         else:
             raise NotImplementedError(

mlrun/api/crud/runtime_resources.py

@@ -18,6 +18,7 @@ import mergedeep
 import sqlalchemy.orm
 
 import mlrun.api.api.utils
+import mlrun.api.runtime_handlers
 import mlrun.api.utils.projects.remotes.follower
 import mlrun.api.utils.singletons.db
 import mlrun.common.schemas
@@ -50,7 +51,7 @@ class RuntimeResources(
             self.validate_runtime_resources_kind(kind)
             kinds = [kind]
         for kind in kinds:
-            runtime_handler = mlrun.runtimes.get_runtime_handler(kind)
+            runtime_handler = mlrun.api.runtime_handlers.get_runtime_handler(kind)
             resources = runtime_handler.list_resources(
                 project, object_id, label_selector, group_by
             )
@@ -88,7 +89,7 @@ class RuntimeResources(
                     )
         runtimes_resources_output = [] if group_by is None else {}
         for kind, runtime_resources_list in runtime_resources_by_kind.items():
-            runtime_handler = mlrun.runtimes.get_runtime_handler(kind)
+            runtime_handler = mlrun.api.runtime_handlers.get_runtime_handler(kind)
             resources = runtime_handler.build_output_from_runtime_resources(
                 runtime_resources_list, group_by
             )
@@ -122,7 +123,7 @@ class RuntimeResources(
             self.validate_runtime_resources_kind(kind)
             kinds = [kind]
         for kind in kinds:
-            runtime_handler = mlrun.runtimes.get_runtime_handler(kind)
+            runtime_handler = mlrun.api.runtime_handlers.get_runtime_handler(kind)
             if object_id:
                 runtime_handler.delete_runtime_object_resources(
                     mlrun.api.utils.singletons.db.get_db(),

mlrun/api/crud/secrets.py

@@ -17,9 +17,11 @@ import json
 import typing
 import uuid
 
+import mlrun.api
 import mlrun.api.utils.clients.iguazio
 import mlrun.api.utils.events.events_factory as events_factory
 import mlrun.api.utils.singletons.k8s
+import mlrun.common
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.helpers
@@ -531,3 +533,22 @@ class Secrets(
     @staticmethod
     def _generate_uuid() -> str:
         return str(uuid.uuid4())
+
+
+def get_project_secret_provider(project: str) -> typing.Callable:
+    """Implement secret provider for handle the related project secret on the API side.
+
+    :param project: Project name.
+
+    :return: A secret provider function.
+    """
+
+    def secret_provider(key: str):
+        return mlrun.api.crud.secrets.Secrets().get_project_secret(
+            project=project,
+            provider=mlrun.common.schemas.secret.SecretProviderName.kubernetes,
+            allow_secrets_from_k8s=True,
+            secret_key=key,
+        )
+
+    return secret_provider

mlrun/api/crud/workflows.py

@@ -0,0 +1,352 @@
+# Copyright 2018 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import uuid
+from typing import Dict
+
+from sqlalchemy.orm import Session
+
+import mlrun.common.schemas
+import mlrun.utils.singleton
+from mlrun.api.api.utils import (
+    apply_enrichment_and_validation_on_function,
+    get_run_db_instance,
+    get_scheduler,
+)
+from mlrun.config import config
+from mlrun.model import Credentials, RunMetadata, RunObject, RunSpec
+
+
+class WorkflowRunners(
+    metaclass=mlrun.utils.singleton.Singleton,
+):
+    @staticmethod
+    def create_runner(
+        run_name: str,
+        project: str,
+        db_session: Session,
+        auth_info: mlrun.common.schemas.AuthInfo,
+        image: str,
+    ) -> mlrun.run.KubejobRuntime:
+        """
+        Creating the base object for the workflow runner function with
+        all the necessary metadata to create it on server-side.
+
+        :param run_name:    workflow-runner function name
+        :param project:     project name
+        :param db_session:  session that manages the current dialog with the database
+        :param auth_info:   auth info of the request
+        :param image:       image for the workflow runner job
+
+        :returns: workflow runner object
+        """
+        runner = mlrun.new_function(
+            name=run_name,
+            project=project,
+            kind=mlrun.runtimes.RuntimeKinds.job,
+            # For preventing deployment:
+            image=image,
+        )
+
+        runner.set_db_connection(get_run_db_instance(db_session))
+
+        # Enrichment and validation requires access key
+        runner.metadata.credentials.access_key = Credentials.generate_access_key
+
+        apply_enrichment_and_validation_on_function(
+            function=runner,
+            auth_info=auth_info,
+        )
+
+        runner.save()
+        return runner
+
+    def schedule(
+        self,
+        runner: mlrun.run.KubejobRuntime,
+        project: mlrun.common.schemas.Project,
+        workflow_request: mlrun.common.schemas.WorkflowRequest,
+        db_session: Session = None,
+        auth_info: mlrun.common.schemas.AuthInfo = None,
+    ):
+        """
+        Schedule workflow runner.
+
+        :param runner:              workflow runner function object
+        :param project:             MLRun project
+        :param workflow_request:    contains the workflow spec, that will be scheduled
+        :param db_session:          session that manages the current dialog with the database
+        :param auth_info:           auth info of the request
+        """
+        labels = {
+            "job-type": "workflow-runner",
+            "workflow": workflow_request.spec.name,
+        }
+
+        run_spec = self._prepare_run_object_for_scheduling(
+            project=project,
+            workflow_request=workflow_request,
+            labels=labels,
+        )
+        # this includes filling the spec.function which is required for submit run
+        runner._store_function(
+            runspec=run_spec, meta=run_spec.metadata, db=runner._get_db()
+        )
+        workflow_spec = workflow_request.spec
+        schedule = workflow_spec.schedule
+        scheduled_object = {
+            "task": run_spec.to_dict(),
+            "schedule": schedule,
+        }
+
+        get_scheduler().store_schedule(
+            db_session=db_session,
+            auth_info=auth_info,
+            project=project.metadata.name,
+            name=workflow_spec.name,
+            scheduled_object=scheduled_object,
+            cron_trigger=schedule,
+            labels=runner.metadata.labels,
+            kind=mlrun.common.schemas.ScheduleKinds.job,
+        )
+
+    def _prepare_run_object_for_scheduling(
+        self,
+        project: mlrun.common.schemas.Project,
+        workflow_request: mlrun.common.schemas.WorkflowRequest,
+        labels: Dict[str, str],
+    ) -> mlrun.run.RunObject:
+        """
+        Preparing all the necessary metadata and specifications for scheduling workflow from server-side.
+
+        :param project:             MLRun project
+        :param workflow_request:    contains the workflow spec and extra data for the run object
+        :param labels:              dict of the task labels
+
+        :returns: RunObject ready for schedule.
+        """
+        meta_uid = uuid.uuid4().hex
+
+        save = self._set_source(project, workflow_request.source)
+        workflow_spec = workflow_request.spec
+        run_object = RunObject(
+            spec=RunSpec(
+                parameters=dict(
+                    url=project.spec.source,
+                    project_name=project.metadata.name,
+                    workflow_name=workflow_spec.name,
+                    workflow_path=workflow_spec.path,
+                    workflow_arguments=workflow_spec.args,
+                    artifact_path=workflow_request.artifact_path,
+                    workflow_handler=workflow_spec.handler,
+                    namespace=workflow_request.namespace,
+                    ttl=workflow_spec.ttl,
+                    engine=workflow_spec.engine,
+                    local=workflow_spec.run_local,
+                    save=save,
+                    subpath=project.spec.subpath,
+                ),
+                handler="mlrun.projects.load_and_run",
+                scrape_metrics=config.scrape_metrics,
+                output_path=(
+                    workflow_request.artifact_path or config.artifact_path
+                ).replace("{{run.uid}}", meta_uid),
+            ),
+            metadata=RunMetadata(
+                uid=meta_uid, name=workflow_spec.name, project=project.metadata.name
+            ),
+        )
+
+        # Setting labels:
+        return self._label_run_object(run_object, labels)
+
+    def run(
+        self,
+        runner: mlrun.run.KubejobRuntime,
+        project: mlrun.common.schemas.Project,
+        workflow_request: mlrun.common.schemas.WorkflowRequest = None,
+        load_only: bool = False,
+    ) -> RunObject:
+        """
+        Run workflow runner.
+
+        :param runner:              workflow runner function object
+        :param project:             MLRun project
+        :param workflow_request:    contains the workflow spec, that will be executed
+        :param load_only:           If True, will only load the project remotely (without running workflow)
+
+        :returns: run context object (RunObject) with run metadata, results and status
+        """
+        labels = {"project": project.metadata.name}
+        if load_only:
+            labels["job-type"] = "project-loader"
+        else:
+            labels["job-type"] = "workflow-runner"
+            labels["workflow"] = runner.metadata.name
+
+        run_spec = self._prepare_run_object_for_single_run(
+            project=project,
+            labels=labels,
+            workflow_request=workflow_request,
+            run_name=runner.metadata.name,
+            load_only=load_only,
+        )
+
+        artifact_path = workflow_request.artifact_path if workflow_request else ""
+        return runner.run(
+            runspec=run_spec,
+            artifact_path=artifact_path,
+            local=False,
+            watch=False,
+        )
+
+    @staticmethod
+    def get_workflow_id(
+        uid: str, project: str, engine: str, db_session: Session
+    ) -> mlrun.common.schemas.GetWorkflowResponse:
+        """
+        Retrieving the actual workflow id form the workflow runner
+
+        :param uid:         the id of the workflow runner job
+        :param project:     name of the project
+        :param engine:      pipeline runner, for example: "kfp"
+        :param db_session:  session that manages the current dialog with the database
+
+        :return: The id of the workflow.
+        """
+        # Reading run:
+        run = mlrun.api.crud.Runs().get_run(
+            db_session=db_session, uid=uid, iter=0, project=project
+        )
+        run_object = RunObject.from_dict(run)
+        state = run_object.status.state
+        workflow_id = None
+        if isinstance(run_object.status.results, dict):
+            workflow_id = run_object.status.results.get("workflow_id", None)
+
+        if workflow_id is None:
+            if (
+                engine == "local"
+                and state.casefold() == mlrun.run.RunStatuses.running.casefold()
+            ):
+                workflow_id = ""
+            else:
+                raise mlrun.errors.MLRunNotFoundError(
+                    f"workflow id of run {project}:{uid} not found"
+                )
+
+        return mlrun.common.schemas.GetWorkflowResponse(workflow_id=workflow_id)
+
+    def _prepare_run_object_for_single_run(
+        self,
+        project: mlrun.common.schemas.Project,
+        labels: Dict[str, str],
+        workflow_request: mlrun.common.schemas.WorkflowRequest = None,
+        run_name: str = None,
+        load_only: bool = False,
+    ) -> mlrun.run.RunObject:
+        """
+        Preparing all the necessary metadata and specifications for running workflow from server-side.
+
+        :param project:             MLRun project
+        :param labels:              dict of the task labels
+        :param workflow_request:    contains the workflow spec and extra data for the run object
+        :param run_name:            workflow-runner function name
+        :param load_only:           if True, will only load the project remotely (without running workflow)
+
+        :returns: RunObject ready for execution.
+        """
+        source = workflow_request.source if workflow_request else ""
+        save = self._set_source(project, source, load_only)
+        run_object = RunObject(
+            spec=RunSpec(
+                parameters=dict(
+                    url=project.spec.source,
+                    project_name=project.metadata.name,
+                    load_only=load_only,
+                    save=save,
+                ),
+                handler="mlrun.projects.load_and_run",
+            ),
+            metadata=RunMetadata(name=run_name),
+        )
+
+        if not load_only:
+            workflow_spec = workflow_request.spec
+            run_object.spec.parameters.update(
+                dict(
+                    workflow_name=workflow_spec.name,
+                    workflow_path=workflow_spec.path,
+                    workflow_arguments=workflow_spec.args,
+                    artifact_path=workflow_request.artifact_path,
+                    workflow_handler=workflow_spec.handler,
+                    namespace=workflow_request.namespace,
+                    ttl=workflow_spec.ttl,
+                    engine=workflow_spec.engine,
+                    local=workflow_spec.run_local,
+                    subpath=project.spec.subpath,
+                )
+            )
+
+        # Setting labels:
+        return self._label_run_object(run_object, labels)
+
+    @staticmethod
+    def _set_source(
+        project: mlrun.common.schemas.Project, source: str, load_only: bool = False
+    ) -> bool:
+        """
+        Setting the project source.
+        In case the user provided a source we want to load the project from the source
+        (like from a specific commit/branch from git repo) without changing the source of the project (save=False).
+
+        :param project:     MLRun project
+        :param source:      the source of the project, needs to be a remote URL that contains the project yaml file.
+        :param load_only:   if we only load the project, it must be saved to ensure we are not running a pipeline
+                            without a project as it's not supported.
+
+        :returns: True if the project need to be saved afterward.
+        """
+
+        save = True
+        if source and not load_only:
+            save = False
+            project.spec.source = source
+
+        if "://" not in project.spec.source:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Remote workflows can only be performed by a project with remote source (e.g git:// or http://),"
+                f" but the specified source '{project.spec.source}' is not remote. "
+                f"Either put your code in Git, or archive it and then set a source to it."
+                f" For more details, read"
+                f" https://docs.mlrun.org/en/latest/concepts/scheduled-jobs.html#scheduling-a-workflow"
+            )
+        return save
+
+    @staticmethod
+    def _label_run_object(
+        run_object: mlrun.run.RunObject,
+        labels: Dict[str, str],
+    ) -> mlrun.run.RunObject:
+        """
+        Setting labels to the task
+
+        :param run_object:  run object to set labels on
+        :param labels:      dict of labels
+
+        :returns: labeled RunObject
+        """
+        for key, value in labels.items():
+            run_object = run_object.set_label(key, value)
+        return run_object

mlrun/api/db/base.py

@@ -259,6 +259,21 @@ class DBInterface(ABC):
     ):
         pass
 
+    def store_schedule(
+        self,
+        session,
+        project: str,
+        name: str,
+        kind: mlrun.common.schemas.ScheduleKinds = None,
+        scheduled_object: Any = None,
+        cron_trigger: mlrun.common.schemas.ScheduleCronTrigger = None,
+        labels: Dict = None,
+        last_run_uri: str = None,
+        concurrency_limit: int = None,
+        next_run_time: datetime = None,
+    ):
+        pass
+
     @abstractmethod
     def list_schedules(
         self,
@@ -272,7 +287,7 @@ class DBInterface(ABC):
 
     @abstractmethod
     def get_schedule(
-        self, session, project: str, name: str
+        self, session, project: str, name: str, raise_on_not_found: bool = True
     ) -> mlrun.common.schemas.ScheduleRecord:
         pass
 

mlrun/api/db/sqldb/db.py

@@ -1174,6 +1174,63 @@ class SQLDB(DBInterface):
 
         return results
 
+    def store_schedule(
+        self,
+        session: Session,
+        project: str,
+        name: str,
+        kind: mlrun.common.schemas.ScheduleKinds = None,
+        scheduled_object: Any = None,
+        cron_trigger: mlrun.common.schemas.ScheduleCronTrigger = None,
+        labels: Dict = None,
+        last_run_uri: str = None,
+        concurrency_limit: int = None,
+        next_run_time: datetime = None,
+    ) -> typing.Optional[mlrun.common.schemas.ScheduleRecord]:
+        schedule = self.get_schedule(
+            session=session, project=project, name=name, raise_on_not_found=False
+        )
+        schedule_exist = schedule is not None
+
+        if not schedule_exist:
+            schedule = Schedule(
+                project=project,
+                name=name,
+                kind=kind.value,
+                creation_time=datetime.now(timezone.utc),
+                concurrency_limit=concurrency_limit,
+                next_run_time=next_run_time,
+                scheduled_object=scheduled_object,
+                cron_trigger=cron_trigger,
+            )
+            labels = labels or {}
+
+        self._update_schedule_body(
+            schedule=schedule,
+            scheduled_object=scheduled_object,
+            cron_trigger=cron_trigger,
+            labels=labels,
+            last_run_uri=last_run_uri,
+            concurrency_limit=concurrency_limit,
+            next_run_time=next_run_time,
+        )
+
+        logger.debug(
+            "Storing schedule to db",
+            project=project,
+            name=name,
+            kind=kind,
+            cron_trigger=cron_trigger,
+            labels=labels,
+            concurrency_limit=concurrency_limit,
+            scheduled_object=scheduled_object,
+        )
+
+        self._upsert(session, [schedule])
+
+        if schedule_exist:
+            return schedule
+
     def create_schedule(
         self,
         session: Session,
@@ -1232,6 +1289,37 @@ class SQLDB(DBInterface):
     ):
         schedule = self._get_schedule_record(session, project, name)
 
+        self._update_schedule_body(
+            schedule=schedule,
+            scheduled_object=scheduled_object,
+            cron_trigger=cron_trigger,
+            labels=labels,
+            last_run_uri=last_run_uri,
+            concurrency_limit=concurrency_limit,
+            next_run_time=next_run_time,
+        )
+
+        logger.debug(
+            "Updating schedule in db",
+            project=project,
+            name=name,
+            cron_trigger=cron_trigger,
+            labels=labels,
+            concurrency_limit=concurrency_limit,
+            next_run_time=next_run_time,
+        )
+        self._upsert(session, [schedule])
+
+    @staticmethod
+    def _update_schedule_body(
+        schedule: mlrun.common.schemas.ScheduleRecord,
+        scheduled_object: Any = None,
+        cron_trigger: mlrun.common.schemas.ScheduleCronTrigger = None,
+        labels: Dict = None,
+        last_run_uri: str = None,
+        concurrency_limit: int = None,
+        next_run_time: datetime = None,
+    ):
         # explicitly ensure the updated fields are not None, as they can be empty strings/dictionaries etc.
         if scheduled_object is not None:
             schedule.scheduled_object = scheduled_object
@@ -1253,17 +1341,6 @@ class SQLDB(DBInterface):
             # saved in the DB in UTC timezone, therefore we transform next_run_time to UTC as well.
             schedule.next_run_time = next_run_time.astimezone(pytz.utc)
 
-        logger.debug(
-            "Updating schedule in db",
-            project=project,
-            name=name,
-            cron_trigger=cron_trigger,
-            labels=labels,
-            concurrency_limit=concurrency_limit,
-            next_run_time=next_run_time,
-        )
-        self._upsert(session, [schedule])
-
     def list_schedules(
         self,
         session: Session,
@@ -1286,19 +1363,23 @@ class SQLDB(DBInterface):
         return schedules
 
     def get_schedule(
-        self, session: Session, project: str, name: str
-    ) -> mlrun.common.schemas.ScheduleRecord:
+        self, session: Session, project: str, name: str, raise_on_not_found: bool = True
+    ) -> typing.Optional[mlrun.common.schemas.ScheduleRecord]:
         logger.debug("Getting schedule from db", project=project, name=name)
-        schedule_record = self._get_schedule_record(session, project, name)
+        schedule_record = self._get_schedule_record(
+            session, project, name, raise_on_not_found
+        )
+        if not schedule_record:
+            return
         schedule = self._transform_schedule_record_to_scheme(schedule_record)
         return schedule
 
     def _get_schedule_record(
-        self, session: Session, project: str, name: str
+        self, session: Session, project: str, name: str, raise_on_not_found: bool = True
     ) -> mlrun.common.schemas.ScheduleRecord:
         query = self._query(session, Schedule, project=project, name=name)
         schedule_record = query.one_or_none()
-        if not schedule_record:
+        if not schedule_record and raise_on_not_found:
             raise mlrun.errors.MLRunNotFoundError(
                 f"Schedule not found: project={project}, name={name}"
             )