mlrun 1.5.0rc4__py3-none-any.whl → 1.5.0rc6__py3-none-any.whl

mlrun/api/api/endpoints/datastore_profile.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 #
 
+from http import HTTPStatus
 
 from fastapi import APIRouter, Depends
 from fastapi.concurrency import run_in_threadpool
@@ -24,12 +25,13 @@ import mlrun.api.crud
 import mlrun.api.utils.auth.verifier
 import mlrun.api.utils.singletons.db
 import mlrun.common.schemas
+from mlrun.api.api.utils import log_and_raise
 
 router = APIRouter()
 
 
 @router.put(
-    path="/projects/{project_name}/datastore_profiles",
+    path="/projects/{project_name}/datastore-profiles",
 )
 async def store_datastore_profile(
     project_name: str,
@@ -45,13 +47,20 @@ async def store_datastore_profile(
         project_name,
         auth_info.session,
     )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
+    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
         mlrun.common.schemas.AuthorizationResourceTypes.datastore_profile,
-        mlrun.common.schemas.AuthorizationAction.create,
+        project_name,
+        info.name,
+        mlrun.common.schemas.AuthorizationAction.store,
         auth_info,
     )
     # overwrite the project
-    info.project = project_name
+    if info.project != project_name:
+        log_and_raise(
+            HTTPStatus.BAD_REQUEST.value,
+            reason="The project name provided in the URI does not match the one specified in the DatastoreProfile",
+        )
+
     await run_in_threadpool(
         mlrun.api.utils.singletons.db.get_db().store_datastore_profile,
         db_session,
@@ -67,7 +76,7 @@ async def store_datastore_profile(
 
 
 @router.get(
-    path="/projects/{project_name}/datastore_profiles",
+    path="/projects/{project_name}/datastore-profiles",
 )
 async def list_datastore_profiles(
     project_name: str,
@@ -82,20 +91,29 @@ async def list_datastore_profiles(
         project_name,
         auth_info.session,
     )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
-        mlrun.common.schemas.AuthorizationResourceTypes.datastore_profile,
+    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
+        project_name,
         mlrun.common.schemas.AuthorizationAction.read,
         auth_info,
     )
-    return await run_in_threadpool(
+    profiles = await run_in_threadpool(
         mlrun.api.utils.singletons.db.get_db().list_datastore_profiles,
         db_session,
         project_name,
     )
+    if len(profiles) == 0:
+        return profiles
+    filtered_data = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
+        mlrun.common.schemas.AuthorizationResourceTypes.datastore_profile,
+        profiles,
+        lambda profile: (project_name, profile.name),
+        auth_info,
+    )
+    return filtered_data
 
 
 @router.get(
-    path="/projects/{project_name}/datastore_profiles/{profile}",
+    path="/projects/{project_name}/datastore-profiles/{profile}",
 )
 async def get_datastore_profile(
     project_name: str,
@@ -111,8 +129,10 @@ async def get_datastore_profile(
         project_name,
         auth_info.session,
     )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
+    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
         mlrun.common.schemas.AuthorizationResourceTypes.datastore_profile,
+        project_name,
+        profile,
         mlrun.common.schemas.AuthorizationAction.read,
         auth_info,
     )
@@ -125,7 +145,7 @@ async def get_datastore_profile(
 
 
 @router.delete(
-    path="/projects/{project_name}/datastore_profiles/{profile}",
+    path="/projects/{project_name}/datastore-profiles/{profile}",
 )
 async def delete_datastore_profile(
     project_name: str,
@@ -141,9 +161,11 @@ async def delete_datastore_profile(
         project_name,
         auth_info.session,
     )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
+    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
         mlrun.common.schemas.AuthorizationResourceTypes.datastore_profile,
-        mlrun.common.schemas.AuthorizationAction.read,
+        project_name,
+        profile,
+        mlrun.common.schemas.AuthorizationAction.delete,
         auth_info,
     )
     return await run_in_threadpool(

mlrun/api/api/endpoints/files.py

@@ -42,7 +42,7 @@ def get_files(
     schema: str = "",
     objpath: str = fastapi.Query("", alias="path"),
     user: str = "",
-    size: int = 0,
+    size: int = None,
     offset: int = 0,
     auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
         mlrun.api.api.deps.authenticate_request

mlrun/api/api/endpoints/frontend_spec.py

@@ -73,7 +73,7 @@ def get_frontend_spec(
         function_deployment_target_image_template=function_deployment_target_image_template,
         function_deployment_target_image_name_prefix_template=function_target_image_name_prefix_template,
         function_deployment_target_image_registries_to_enforce_prefix=registries_to_enforce_prefix,
-        function_deployment_mlrun_command=_resolve_function_deployment_mlrun_command(),
+        function_deployment_mlrun_requirement=mlrun.api.utils.builder.resolve_mlrun_install_command_version(),
         auto_mount_type=config.storage.auto_mount_type,
         auto_mount_params=config.get_storage_auto_mount_params(),
         default_artifact_path=config.artifact_path,
@@ -87,15 +87,6 @@ def get_frontend_spec(
     )
 
 
-def _resolve_function_deployment_mlrun_command():
-    # TODO: When UI adds a requirements section, mlrun should be specified there instead of the commands section i.e.
-    #  frontend spec will contain only the mlrun_version_specifier instead of the full command
-    mlrun_version_specifier = (
-        mlrun.api.utils.builder.resolve_mlrun_install_command_version()
-    )
-    return f'python -m pip install "{mlrun_version_specifier}"'
-
-
 def _resolve_jobs_dashboard_url(session: str) -> typing.Optional[str]:
     iguazio_client = mlrun.api.utils.clients.iguazio.Client()
     grafana_service_url = iguazio_client.try_get_grafana_service_url(session)

mlrun/api/api/endpoints/functions.py

@@ -47,7 +47,7 @@ import mlrun.common.model_monitoring
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas
 from mlrun.api.api import deps
-from mlrun.api.api.utils import log_and_raise, log_path
+from mlrun.api.api.utils import get_run_db_instance, log_and_raise, log_path
 from mlrun.api.crud.secrets import Secrets, SecretsClientType
 from mlrun.api.utils.builder import build_runtime
 from mlrun.api.utils.singletons.scheduler import get_scheduler
@@ -267,7 +267,7 @@ async def build_function(
     except ValueError:
         log_and_raise(HTTPStatus.BAD_REQUEST.value, reason="bad JSON body")
 
-    logger.info(f"build_function:\n{data}")
+    logger.info("Building function", data=data)
     function = data.get("function")
     project = function.get("metadata", {}).get("project", mlrun.mlconf.default_project)
     function_name = function.get("metadata", {}).get("name")
@@ -706,6 +706,10 @@ def _build_function(
             reason=f"runtime error: {err_to_str(err)}",
         )
     try:
+        # connect to run db
+        run_db = get_run_db_instance(db_session)
+        fn.set_db_connection(run_db)
+
         is_nuclio_runtime = fn.kind in RuntimeKinds.nuclio_runtimes()
 
         # Enrich runtime with project defaults
@@ -836,26 +840,32 @@ def _start_function(
     client_version: str = None,
     client_python_version: str = None,
 ):
+    db_session = mlrun.api.db.session.create_session()
     try:
-        mlrun.api.api.utils.apply_enrichment_and_validation_on_function(
-            function,
-            auth_info,
-        )
+        try:
+            run_db = get_run_db_instance(db_session)
+            function.set_db_connection(run_db)
+            mlrun.api.api.utils.apply_enrichment_and_validation_on_function(
+                function,
+                auth_info,
+            )
 
-        mlrun.api.crud.Functions().start_function(
-            function, client_version, client_python_version
-        )
-        logger.info("Fn:\n %s", function.to_yaml())
+            mlrun.api.crud.Functions().start_function(
+                function, client_version, client_python_version
+            )
+            logger.info("Fn:\n %s", function.to_yaml())
 
-    except mlrun.errors.MLRunBadRequestError:
-        raise
+        except mlrun.errors.MLRunBadRequestError:
+            raise
 
-    except Exception as err:
-        logger.error(traceback.format_exc())
-        log_and_raise(
-            HTTPStatus.BAD_REQUEST.value,
-            reason=f"Runtime error: {err_to_str(err)}",
-        )
+        except Exception as err:
+            logger.error(traceback.format_exc())
+            log_and_raise(
+                HTTPStatus.BAD_REQUEST.value,
+                reason=f"Runtime error: {err_to_str(err)}",
+            )
+    finally:
+        mlrun.api.db.session.close_session(db_session)
 
 
 async def _get_function_status(data, auth_info: mlrun.common.schemas.AuthInfo):

mlrun/api/api/endpoints/hub.py

@@ -81,13 +81,9 @@ async def list_sources(
         auth_info,
     )
 
-    hub_sources = await run_in_threadpool(
-        mlrun.api.utils.singletons.db.get_db().list_hub_sources, db_session
-    )
-
     return await run_in_threadpool(
-        mlrun.api.crud.Hub().filter_hub_sources,
-        hub_sources,
+        mlrun.api.crud.Hub().list_hub_sources,
+        db_session,
         item_name,
         tag,
         version,

mlrun/api/api/endpoints/pipelines.py

@@ -129,9 +129,11 @@ async def get_pipeline(
     auth_info: mlrun.common.schemas.AuthInfo = Depends(
         mlrun.api.api.deps.authenticate_request
     ),
+    db_session: Session = Depends(deps.get_db_session),
 ):
     pipeline = await run_in_threadpool(
         mlrun.api.crud.Pipelines().get_pipeline,
+        db_session,
         run_id,
         project,
         namespace,
@@ -143,7 +145,7 @@ async def get_pipeline(
         # legacy flow in which we first get the pipeline, resolve the project out of it, and only then query permissions
         # we don't use the return value from this function since the user may have asked for a different format than
         # summary which is the one used inside
-        await _get_pipeline_without_project(auth_info, run_id, namespace)
+        await _get_pipeline_without_project(db_session, auth_info, run_id, namespace)
     else:
         await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
             mlrun.common.schemas.AuthorizationResourceTypes.pipeline,
@@ -156,6 +158,7 @@ async def get_pipeline(
 
 
 async def _get_pipeline_without_project(
+    db_session: Session,
     auth_info: mlrun.common.schemas.AuthInfo,
     run_id: str,
     namespace: str,
@@ -167,6 +170,7 @@ async def _get_pipeline_without_project(
     """
     run = await run_in_threadpool(
         mlrun.api.crud.Pipelines().get_pipeline,
+        db_session,
         run_id,
         namespace=namespace,
         # minimal format that includes the project

mlrun/api/api/endpoints/projects.py

@@ -388,6 +388,7 @@ async def load_project(
         mlrun.api.crud.WorkflowRunners().create_runner,
         run_name=f"load-{name}",
         project=name,
+        db_session=db_session,
         auth_info=auth_info,
         image=mlrun.mlconf.default_base_image,
     )

mlrun/api/api/endpoints/workflows.py

@@ -141,6 +141,7 @@ async def submit_workflow(
             workflow_spec.name
         ),
         project=project.metadata.name,
+        db_session=db_session,
         auth_info=auth_info,
         image=workflow_spec.image
         or project.spec.default_image

mlrun/api/api/utils.py

@@ -36,6 +36,8 @@ import mlrun.common.schemas
 import mlrun.errors
 import mlrun.runtimes.pod
 import mlrun.utils.helpers
+from mlrun.api.db.sqldb.db import SQLDB
+from mlrun.api.rundb.sqldb import SQLRunDB
 from mlrun.api.utils.singletons.db import get_db
 from mlrun.api.utils.singletons.logs_dir import get_logs_dir
 from mlrun.api.utils.singletons.scheduler import get_scheduler
@@ -128,6 +130,20 @@ def get_secrets(
     }
 
 
+def get_run_db_instance(
+    db_session: Session,
+):
+    # TODO: getting the run db should be done seamlessly by the run db factory and not require this logic to
+    #  inject the session
+    db = get_db()
+    if isinstance(db, SQLDB):
+        run_db = SQLRunDB(db.dsn, db_session)
+    else:
+        run_db = db.db
+    run_db.connect()
+    return run_db
+
+
 def parse_submit_run_body(data):
     task = data.get("task")
     function_dict = data.get("function")
@@ -867,6 +883,8 @@ def submit_run_sync(
     try:
         fn, task = _generate_function_and_task_from_submit_run_body(db_session, data)
 
+        run_db = get_run_db_instance(db_session)
+        fn.set_db_connection(run_db)
         logger.info("Submitting run", function=fn.to_dict(), task=task)
         schedule = data.get("schedule")
         if schedule:

mlrun/api/crud/client_spec.py

@@ -104,6 +104,9 @@ class ClientSpec(
             model_endpoint_monitoring_store_type=self._get_config_value_if_not_default(
                 "model_endpoint_monitoring.store_type"
             ),
+            model_endpoint_monitoring_endpoint_store_connection=self._get_config_value_if_not_default(
+                "model_endpoint_monitoring.endpoint_store_connection"
+            ),
             packagers=self._get_config_value_if_not_default("packagers"),
         )
 

mlrun/api/crud/datastore_profiles.py

@@ -37,7 +37,7 @@ class DatastoreProfiles(
     def _store_secret(self, project, profile_name, profile_secret_json):
         if not self._in_k8s():
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "MLRun is not configured with k8s, hub source credentials cannot be stored securely"
+                "MLRun is not configured with k8s, datastore profile credentials cannot be stored securely"
             )
 
         adjusted_secret = {
@@ -56,7 +56,7 @@ class DatastoreProfiles(
     def _delete_secret(self, project, profile_name):
         if not self._in_k8s():
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "MLRun is not configured with k8s, hub source credentials cannot be stored securely"
+                "MLRun is not configured with k8s, datastore profile credentials cannot be deleted"
             )
 
         adjusted_secret = dsp.generate_secret_key(profile_name, project)