mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/api/api/endpoints/pipelines.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,32 +23,30 @@ from fastapi.concurrency import run_in_threadpool
 from sqlalchemy.orm import Session
 
 import mlrun.api.crud
-import mlrun.api.schemas
 import mlrun.api.utils.auth.verifier
+import mlrun.api.utils.singletons.k8s
+import mlrun.common.schemas
 import mlrun.errors
 from mlrun.api.api import deps
 from mlrun.api.api.utils import log_and_raise
 from mlrun.config import config
-from mlrun.k8s_utils import get_k8s_helper
 from mlrun.utils import logger
 
-router = APIRouter()
+router = APIRouter(prefix="/projects/{project}/pipelines")
 
 
-@router.get(
-    "/projects/{project}/pipelines", response_model=mlrun.api.schemas.PipelinesOutput
-)
+@router.get("", response_model=mlrun.common.schemas.PipelinesOutput)
 async def list_pipelines(
     project: str,
     namespace: str = None,
     sort_by: str = "",
     page_token: str = "",
     filter_: str = Query("", alias="filter"),
-    format_: mlrun.api.schemas.PipelinesFormat = Query(
-        mlrun.api.schemas.PipelinesFormat.metadata_only, alias="format"
+    format_: mlrun.common.schemas.PipelinesFormat = Query(
+        mlrun.common.schemas.PipelinesFormat.metadata_only, alias="format"
     ),
     page_size: int = Query(None, gt=0, le=200),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
+    auth_info: mlrun.common.schemas.AuthInfo = Depends(
         mlrun.api.api.deps.authenticate_request
     ),
     db_session: Session = Depends(deps.get_db_session),
@@ -58,16 +56,18 @@ async def list_pipelines(
     if project != "*":
         await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
             project,
-            mlrun.api.schemas.AuthorizationAction.read,
+            mlrun.common.schemas.AuthorizationAction.read,
             auth_info,
         )
     total_size, next_page_token, runs = None, None, []
-    if get_k8s_helper(silent=True).is_running_inside_kubernetes_cluster():
+    if mlrun.api.utils.singletons.k8s.get_k8s_helper(
+        silent=True
+    ).is_running_inside_kubernetes_cluster():
         # we need to resolve the project from the returned run for the opa enforcement (project query param might be
         # "*"), so we can't really get back only the names here
         computed_format = (
-            mlrun.api.schemas.PipelinesFormat.metadata_only
-            if format_ == mlrun.api.schemas.PipelinesFormat.name_only
+            mlrun.common.schemas.PipelinesFormat.metadata_only
+            if format_ == mlrun.common.schemas.PipelinesFormat.name_only
             else format_
         )
         total_size, next_page_token, runs = await run_in_threadpool(
@@ -82,7 +82,7 @@ async def list_pipelines(
             page_size,
         )
     allowed_runs = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.pipeline,
+        mlrun.common.schemas.AuthorizationResourceTypes.pipeline,
         runs,
         lambda run: (
             run["project"],
@@ -90,23 +90,23 @@ async def list_pipelines(
         ),
         auth_info,
     )
-    if format_ == mlrun.api.schemas.PipelinesFormat.name_only:
+    if format_ == mlrun.common.schemas.PipelinesFormat.name_only:
         allowed_runs = [run["name"] for run in allowed_runs]
-    return mlrun.api.schemas.PipelinesOutput(
+    return mlrun.common.schemas.PipelinesOutput(
         runs=allowed_runs,
         total_size=total_size or 0,
         next_page_token=next_page_token or None,
     )
 
 
-@router.post("/projects/{project}/pipelines")
+@router.post("")
 async def create_pipeline(
     project: str,
     request: Request,
     namespace: str = None,
     experiment_name: str = Query("Default", alias="experiment"),
     run_name: str = Query("", alias="run"),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
+    auth_info: mlrun.common.schemas.AuthInfo = Depends(
         mlrun.api.api.deps.authenticate_request
     ),
 ):
@@ -118,8 +118,76 @@ async def create_pipeline(
     return response
 
 
+@router.get("/{run_id}")
+async def get_pipeline(
+    run_id: str,
+    project: str,
+    namespace: str = Query(config.namespace),
+    format_: mlrun.common.schemas.PipelinesFormat = Query(
+        mlrun.common.schemas.PipelinesFormat.summary, alias="format"
+    ),
+    auth_info: mlrun.common.schemas.AuthInfo = Depends(
+        mlrun.api.api.deps.authenticate_request
+    ),
+    db_session: Session = Depends(deps.get_db_session),
+):
+    pipeline = await run_in_threadpool(
+        mlrun.api.crud.Pipelines().get_pipeline,
+        db_session,
+        run_id,
+        project,
+        namespace,
+        format_,
+    )
+    if project == "*":
+        # In some flows the user may use SDK functions that won't require them to specify the pipeline's project (for
+        # backwards compatibility reasons), so the client will just send * in the project, in that case we use the
+        # legacy flow in which we first get the pipeline, resolve the project out of it, and only then query permissions
+        # we don't use the return value from this function since the user may have asked for a different format than
+        # summary which is the one used inside
+        await _get_pipeline_without_project(db_session, auth_info, run_id, namespace)
+    else:
+        await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
+            mlrun.common.schemas.AuthorizationResourceTypes.pipeline,
+            project,
+            run_id,
+            mlrun.common.schemas.AuthorizationAction.read,
+            auth_info,
+        )
+    return pipeline
+
+
+async def _get_pipeline_without_project(
+    db_session: Session,
+    auth_info: mlrun.common.schemas.AuthInfo,
+    run_id: str,
+    namespace: str,
+):
+    """
+    This function is for when we receive a get pipeline request without the client specifying the project
+    So we first get the pipeline, resolve the project out of it, and now that we know the project, we can verify
+    permissions
+    """
+    run = await run_in_threadpool(
+        mlrun.api.crud.Pipelines().get_pipeline,
+        db_session,
+        run_id,
+        namespace=namespace,
+        # minimal format that includes the project
+        format_=mlrun.common.schemas.PipelinesFormat.summary,
+    )
+    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
+        mlrun.common.schemas.AuthorizationResourceTypes.pipeline,
+        run["run"]["project"],
+        run["run"]["id"],
+        mlrun.common.schemas.AuthorizationAction.read,
+        auth_info,
+    )
+    return run
+
+
 async def _create_pipeline(
-    auth_info: mlrun.api.schemas.AuthInfo,
+    auth_info: mlrun.common.schemas.AuthInfo,
     request: Request,
     namespace: str,
     experiment_name: str,
@@ -129,10 +197,10 @@ async def _create_pipeline(
     # If we have the project (new clients from 0.7.0 uses the new endpoint in which it's mandatory) - check auth now
     if project:
         await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
-            mlrun.api.schemas.AuthorizationResourceTypes.pipeline,
+            mlrun.common.schemas.AuthorizationResourceTypes.pipeline,
             project,
             "",
-            mlrun.api.schemas.AuthorizationAction.create,
+            mlrun.common.schemas.AuthorizationAction.create,
             auth_info,
         )
     run_name = run_name or experiment_name + " " + datetime.now().strftime(
@@ -153,16 +221,16 @@ async def _create_pipeline(
         )
     else:
         await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
-            mlrun.api.schemas.AuthorizationResourceTypes.pipeline,
+            mlrun.common.schemas.AuthorizationResourceTypes.pipeline,
             project,
             "",
-            mlrun.api.schemas.AuthorizationAction.create,
+            mlrun.common.schemas.AuthorizationAction.create,
             auth_info,
         )
 
     arguments = {}
     arguments_data = request.headers.get(
-        mlrun.api.schemas.HeaderNames.pipeline_arguments
+        mlrun.common.schemas.HeaderNames.pipeline_arguments
     )
     if arguments_data:
         arguments = ast.literal_eval(arguments_data)
@@ -196,66 +264,3 @@ def _try_resolve_project_from_body(
     return mlrun.api.crud.Pipelines().resolve_project_from_workflow_manifest(
         workflow_manifest
     )
-
-
-@router.get("/projects/{project}/pipelines/{run_id}")
-async def get_pipeline(
-    run_id: str,
-    project: str,
-    namespace: str = Query(config.namespace),
-    format_: mlrun.api.schemas.PipelinesFormat = Query(
-        mlrun.api.schemas.PipelinesFormat.summary, alias="format"
-    ),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
-        mlrun.api.api.deps.authenticate_request
-    ),
-    db_session: Session = Depends(deps.get_db_session),
-):
-    pipeline = mlrun.api.crud.Pipelines().get_pipeline(
-        db_session, run_id, project, namespace, format_
-    )
-    if project == "*":
-        # In some flows the user may use SDK functions that won't require them to specify the pipeline's project (for
-        # backwards compatibility reasons), so the client will just send * in the project, in that case we use the
-        # legacy flow in which we first get the pipeline, resolve the project out of it, and only then query permissions
-        # we don't use the return value from this function since the user may have asked for a different format than
-        # summary which is the one used inside
-        await _get_pipeline_without_project(db_session, auth_info, run_id, namespace)
-    else:
-        await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
-            mlrun.api.schemas.AuthorizationResourceTypes.pipeline,
-            project,
-            run_id,
-            mlrun.api.schemas.AuthorizationAction.read,
-            auth_info,
-        )
-    return pipeline
-
-
-async def _get_pipeline_without_project(
-    db_session: Session,
-    auth_info: mlrun.api.schemas.AuthInfo,
-    run_id: str,
-    namespace: str,
-):
-    """
-    This function is for when we receive a get pipeline request without the client specifying the project
-    So we first get the pipeline, resolve the project out of it, and now that we know the project, we can verify
-    permissions
-    """
-    run = await run_in_threadpool(
-        mlrun.api.crud.Pipelines().get_pipeline,
-        db_session,
-        run_id,
-        namespace=namespace,
-        # minimal format that includes the project
-        format_=mlrun.api.schemas.PipelinesFormat.summary,
-    )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.pipeline,
-        run["run"]["project"],
-        run["run"]["id"],
-        mlrun.api.schemas.AuthorizationAction.read,
-        auth_info,
-    )
-    return run

mlrun/api/api/endpoints/projects.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,9 +20,9 @@ import sqlalchemy.orm
 from fastapi.concurrency import run_in_threadpool
 
 import mlrun.api.api.deps
-import mlrun.api.schemas
 import mlrun.api.utils.auth.verifier
 import mlrun.api.utils.clients.chief
+import mlrun.common.schemas
 from mlrun.api.utils.singletons.project_member import get_project_member
 from mlrun.utils import logger
 
@@ -32,17 +32,17 @@ router = fastapi.APIRouter()
 @router.post(
     "/projects",
     responses={
-        http.HTTPStatus.CREATED.value: {"model": mlrun.api.schemas.Project},
+        http.HTTPStatus.CREATED.value: {"model": mlrun.common.schemas.Project},
         http.HTTPStatus.ACCEPTED.value: {},
     },
 )
 def create_project(
-    project: mlrun.api.schemas.Project,
+    project: mlrun.common.schemas.Project,
     response: fastapi.Response,
     # TODO: we're in a http request context here, therefore it doesn't make sense that by default it will hold the
     #  request until the process will be completed - after UI supports waiting - change default to False
     wait_for_completion: bool = fastapi.Query(True, alias="wait-for-completion"),
-    auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
+    auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
         mlrun.api.api.deps.authenticate_request
     ),
     db_session: sqlalchemy.orm.Session = fastapi.Depends(
@@ -65,17 +65,17 @@ def create_project(
 @router.put(
     "/projects/{name}",
     responses={
-        http.HTTPStatus.OK.value: {"model": mlrun.api.schemas.Project},
+        http.HTTPStatus.OK.value: {"model": mlrun.common.schemas.Project},
         http.HTTPStatus.ACCEPTED.value: {},
     },
 )
 def store_project(
-    project: mlrun.api.schemas.Project,
+    project: mlrun.common.schemas.Project,
     name: str,
     # TODO: we're in a http request context here, therefore it doesn't make sense that by default it will hold the
     #  request until the process will be completed - after UI supports waiting - change default to False
     wait_for_completion: bool = fastapi.Query(True, alias="wait-for-completion"),
-    auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
+    auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
         mlrun.api.api.deps.authenticate_request
     ),
     db_session: sqlalchemy.orm.Session = fastapi.Depends(
@@ -98,21 +98,21 @@ def store_project(
 @router.patch(
     "/projects/{name}",
     responses={
-        http.HTTPStatus.OK.value: {"model": mlrun.api.schemas.Project},
+        http.HTTPStatus.OK.value: {"model": mlrun.common.schemas.Project},
         http.HTTPStatus.ACCEPTED.value: {},
     },
 )
 def patch_project(
     project: dict,
     name: str,
-    patch_mode: mlrun.api.schemas.PatchMode = fastapi.Header(
-        mlrun.api.schemas.PatchMode.replace,
-        alias=mlrun.api.schemas.HeaderNames.patch_mode,
+    patch_mode: mlrun.common.schemas.PatchMode = fastapi.Header(
+        mlrun.common.schemas.PatchMode.replace,
+        alias=mlrun.common.schemas.HeaderNames.patch_mode,
     ),
     # TODO: we're in a http request context here, therefore it doesn't make sense that by default it will hold the
     #  request until the process will be completed - after UI supports waiting - change default to False
     wait_for_completion: bool = fastapi.Query(True, alias="wait-for-completion"),
-    auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
+    auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
         mlrun.api.api.deps.authenticate_request
     ),
     db_session: sqlalchemy.orm.Session = fastapi.Depends(
@@ -133,13 +133,13 @@ def patch_project(
     return project
 
 
-@router.get("/projects/{name}", response_model=mlrun.api.schemas.Project)
+@router.get("/projects/{name}", response_model=mlrun.common.schemas.Project)
 async def get_project(
     name: str,
     db_session: sqlalchemy.orm.Session = fastapi.Depends(
         mlrun.api.api.deps.get_db_session
     ),
-    auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
+    auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
         mlrun.api.api.deps.authenticate_request
     ),
 ):
@@ -150,7 +150,7 @@ async def get_project(
     if not _is_request_from_leader(auth_info.projects_role):
         await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
             name,
-            mlrun.api.schemas.AuthorizationAction.read,
+            mlrun.common.schemas.AuthorizationAction.read,
             auth_info,
         )
     return project
@@ -166,14 +166,14 @@ async def get_project(
 async def delete_project(
     name: str,
     request: fastapi.Request,
-    deletion_strategy: mlrun.api.schemas.DeletionStrategy = fastapi.Header(
-        mlrun.api.schemas.DeletionStrategy.default(),
-        alias=mlrun.api.schemas.HeaderNames.deletion_strategy,
+    deletion_strategy: mlrun.common.schemas.DeletionStrategy = fastapi.Header(
+        mlrun.common.schemas.DeletionStrategy.default(),
+        alias=mlrun.common.schemas.HeaderNames.deletion_strategy,
     ),
     # TODO: we're in a http request context here, therefore it doesn't make sense that by default it will hold the
     #  request until the process will be completed - after UI supports waiting - change default to False
     wait_for_completion: bool = fastapi.Query(True, alias="wait-for-completion"),
-    auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
+    auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
         mlrun.api.api.deps.authenticate_request
     ),
     db_session: sqlalchemy.orm.Session = fastapi.Depends(
@@ -184,7 +184,7 @@ async def delete_project(
     # that is why we re-route requests to chief
     if (
         mlrun.mlconf.httpdb.clusterization.role
-        != mlrun.api.schemas.ClusterizationRole.chief
+        != mlrun.common.schemas.ClusterizationRole.chief
     ):
         logger.info(
             "Requesting to delete project, re-routing to chief",
@@ -209,15 +209,15 @@ async def delete_project(
     return fastapi.Response(status_code=http.HTTPStatus.NO_CONTENT.value)
 
 
-@router.get("/projects", response_model=mlrun.api.schemas.ProjectsOutput)
+@router.get("/projects", response_model=mlrun.common.schemas.ProjectsOutput)
 async def list_projects(
-    format_: mlrun.api.schemas.ProjectsFormat = fastapi.Query(
-        mlrun.api.schemas.ProjectsFormat.full, alias="format"
+    format_: mlrun.common.schemas.ProjectsFormat = fastapi.Query(
+        mlrun.common.schemas.ProjectsFormat.full, alias="format"
     ),
     owner: str = None,
     labels: typing.List[str] = fastapi.Query(None, alias="label"),
-    state: mlrun.api.schemas.ProjectState = None,
-    auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
+    state: mlrun.common.schemas.ProjectState = None,
+    auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
         mlrun.api.api.deps.authenticate_request
     ),
     db_session: sqlalchemy.orm.Session = fastapi.Depends(
@@ -231,7 +231,7 @@ async def list_projects(
             get_project_member().list_projects,
             db_session,
             owner,
-            mlrun.api.schemas.ProjectsFormat.name_only,
+            mlrun.common.schemas.ProjectsFormat.name_only,
             labels,
             state,
             auth_info.projects_role,
@@ -257,13 +257,13 @@ async def list_projects(
 
 
 @router.get(
-    "/project-summaries", response_model=mlrun.api.schemas.ProjectSummariesOutput
+    "/project-summaries", response_model=mlrun.common.schemas.ProjectSummariesOutput
 )
 async def list_project_summaries(
     owner: str = None,
     labels: typing.List[str] = fastapi.Query(None, alias="label"),
-    state: mlrun.api.schemas.ProjectState = None,
-    auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
+    state: mlrun.common.schemas.ProjectState = None,
+    auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
         mlrun.api.api.deps.authenticate_request
     ),
     db_session: sqlalchemy.orm.Session = fastapi.Depends(
@@ -274,7 +274,7 @@ async def list_project_summaries(
         get_project_member().list_projects,
         db_session,
         owner,
-        mlrun.api.schemas.ProjectsFormat.name_only,
+        mlrun.common.schemas.ProjectsFormat.name_only,
         labels,
         state,
         auth_info.projects_role,
@@ -299,14 +299,14 @@ async def list_project_summaries(
 
 
 @router.get(
-    "/project-summaries/{name}", response_model=mlrun.api.schemas.ProjectSummary
+    "/project-summaries/{name}", response_model=mlrun.common.schemas.ProjectSummary
 )
 async def get_project_summary(
     name: str,
     db_session: sqlalchemy.orm.Session = fastapi.Depends(
         mlrun.api.api.deps.get_db_session
     ),
-    auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
+    auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
         mlrun.api.api.deps.authenticate_request
     ),
 ):
@@ -317,14 +317,14 @@ async def get_project_summary(
     if not _is_request_from_leader(auth_info.projects_role):
         await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
             name,
-            mlrun.api.schemas.AuthorizationAction.read,
+            mlrun.common.schemas.AuthorizationAction.read,
             auth_info,
         )
     return project_summary
 
 
 def _is_request_from_leader(
-    projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole],
+    projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole],
 ) -> bool:
     if projects_role and projects_role.value == mlrun.mlconf.httpdb.projects.leader:
         return True