mlrun 1.5.0rc1__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

mlrun/__init__.py

@@ -36,14 +36,7 @@ from .errors import MLRunInvalidArgumentError, MLRunNotFoundError
 from .execution import MLClientCtx
 from .model import RunObject, RunTemplate, new_task
 from .package import ArtifactType, DefaultPackager, Packager, handler
-from .platforms import (
-    VolumeMount,
-    auto_mount,
-    mount_v3io,
-    mount_v3io_extended,
-    mount_v3io_legacy,
-    v3io_cred,
-)
+from .platforms import VolumeMount, auto_mount, mount_v3io, v3io_cred
 from .projects import (
     ProjectMetadata,
     build_function,
@@ -66,7 +59,6 @@ from .run import (
     import_function,
     new_function,
     run_local,
-    run_pipeline,
     wait_for_pipeline_completion,
 )
 from .runtimes import new_model_server
@@ -91,9 +83,7 @@ if "IGZ_NAMESPACE_DOMAIN" in environ:
 def set_environment(
     api_path: str = None,
     artifact_path: str = "",
-    project: str = "",
     access_key: str = None,
-    user_project=False,
     username: str = None,
     env_file: str = None,
     mock_functions: str = None,
@@ -109,18 +99,14 @@ def set_environment(
     example::
 
         from os import path
-        project_name, artifact_path = set_environment(project='my-project')
+        project_name, artifact_path = set_environment()
         set_environment("http://localhost:8080", artifact_path="./")
         set_environment(env_file="mlrun.env")
         set_environment("<remote-service-url>", access_key="xyz", username="joe")
 
     :param api_path:       location/url of mlrun api service
     :param artifact_path:  path/url for storing experiment artifacts
-    :param project:        default project name (deprecated in 1.3.0 and will be removed in 1.5.0) - use project
-                           APIs such as `get_or_create_project`, `load_project` to configure the active project
     :param access_key:     set the remote cluster access key (V3IO_ACCESS_KEY)
-    :param user_project:   add the current user name to the provided project name (making it unique per user)
-                           (deprecated in 1.3.0 and will be removed in 1.5.0)
     :param username:       name of the user to authenticate
     :param env_file:       path/url to .env file (holding MLRun config and other env vars), see: set_env_from_file()
     :param mock_functions: set to True to create local/mock functions instead of real containers,
@@ -129,14 +115,6 @@ def set_environment(
         default project name
         actual artifact path/url, can be used to create subpaths per task or group of artifacts
     """
-    if user_project or project:
-        warnings.warn(
-            "'user_project' and 'project' are deprecated in 1.3.0, and will be removed in 1.5.0, use project "
-            "APIs such as 'get_or_create_project', 'load_project' to configure the active project.",
-            # TODO: Remove in 1.5.0
-            FutureWarning,
-        )
-
     if env_file:
         set_env_from_file(env_file)
 
@@ -160,17 +138,6 @@ def set_environment(
     if api_path:
         environ["MLRUN_DBPATH"] = mlconf.dbpath
 
-    project = _add_username_to_project_name_if_needed(project, user_project)
-    if project:
-        ProjectMetadata.validate_project_name(project)
-
-    mlconf.default_project = project or mlconf.default_project
-    # We want to ensure the project exists, and verify we're authorized to work on it
-    # if it doesn't exist this will create it (and obviously if we created it, we're authorized to work on it)
-    # if it does exist - this will get it, which will fail if we're not authorized to work on it
-    if project:
-        get_or_create_project(mlconf.default_project, "./")
-
     if not mlconf.artifact_path and not artifact_path:
         raise ValueError(
             "default artifact_path was not configured, please specify a valid artifact_path"

mlrun/__main__.py

@@ -17,7 +17,6 @@ import json
 import pathlib
 import socket
 import traceback
-import warnings
 from ast import literal_eval
 from base64 import b64decode, b64encode
 from os import environ, path, remove
@@ -592,12 +591,6 @@ def build(
     default="",
     help="path/url of function yaml or function " "yaml or db://<project>/<name>[:tag]",
 )
-@click.option(
-    "--dashboard",
-    "-d",
-    default="",
-    help="Deprecated. Keep empty to allow auto-detect by MLRun API",
-)
 @click.option("--project", "-p", default="", help="project name")
 @click.option("--model", "-m", multiple=True, help="model name and path (name=path)")
 @click.option("--kind", "-k", default=None, help="runtime sub kind")
@@ -616,7 +609,6 @@ def deploy(
     spec,
     source,
     func_url,
-    dashboard,
     project,
     model,
     tag,
@@ -677,16 +669,8 @@ def deploy(
             function.set_env(k, v)
     function.verbose = verbose
 
-    if dashboard:
-        warnings.warn(
-            "'--dashboard' is deprecated in 1.3.0, and will be removed in 1.5.0, "
-            "Keep '--dashboard' value empty to allow auto-detection by MLRun API.",
-            # TODO: Remove in 1.5.0
-            FutureWarning,
-        )
-
     try:
-        addr = function.deploy(dashboard=dashboard, project=project, tag=tag)
+        addr = function.deploy(project=project, tag=tag)
     except Exception as err:
         print(f"deploy error: {err_to_str(err)}")
         exit(1)
@@ -1005,12 +989,6 @@ def logs(uid, project, offset, db, watch):
 @click.option(
     "--env-file", default="", help="path to .env file to load config/variables from"
 )
-# TODO: Remove --ensure-project in 1.5.0
-@click.option(
-    "--ensure-project",
-    is_flag=True,
-    help="ensure the project exists, if not, create project",
-)
 @click.option(
     "--save/--no-save",
     default=True,
@@ -1025,13 +1003,6 @@ def logs(uid, project, offset, db, watch):
     "https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron."
     "For using the pre-defined workflow's schedule, set --schedule 'true'",
 )
-# TODO: Remove in 1.5.0
-@click.option(
-    "--overwrite-schedule",
-    "-os",
-    is_flag=True,
-    help="Overwrite a schedule when submitting a new one with the same name.",
-)
 @click.option(
     "--save-secrets",
     is_flag=True,
@@ -1069,10 +1040,8 @@ def project(
     local,
     env_file,
     timeout,
-    ensure_project,
     schedule,
     notifications,
-    overwrite_schedule,
     save_secrets,
     save,
 ):
@@ -1080,13 +1049,6 @@ def project(
     if env_file:
         mlrun.set_env_from_file(env_file)
 
-    if ensure_project:
-        warnings.warn(
-            "'ensure_project' is deprecated and will be removed in 1.5.0, use 'save' (True by default) instead. ",
-            # TODO: Remove this in 1.5.0
-            FutureWarning,
-        )
-
     if db:
         mlconf.dbpath = db
 
@@ -1169,7 +1131,6 @@ def project(
                 local=local,
                 schedule=schedule,
                 timeout=timeout,
-                overwrite=overwrite_schedule,
             )
         except Exception as err:
             print(traceback.format_exc())

mlrun/api/api/api.py

@@ -41,6 +41,7 @@ from mlrun.api.api.endpoints import (
     secrets,
     submit,
     tags,
+    workflows,
 )
 
 api_router = APIRouter(dependencies=[Depends(mlrun.api.api.deps.verify_api_state)])
@@ -145,3 +146,8 @@ api_router.include_router(
     tags=["internal"],
     dependencies=[Depends(mlrun.api.api.deps.authenticate_request)],
 )
+api_router.include_router(
+    workflows.router,
+    tags=["workflows"],
+    dependencies=[Depends(mlrun.api.api.deps.authenticate_request)],
+)

mlrun/api/api/endpoints/feature_store.py

@@ -408,10 +408,6 @@ async def ingest_feature_set(
             mlrun.common.schemas.AuthorizationAction.read,
             auth_info,
         )
-    # Need to override the default rundb since we're in the server.
-    # this is done so further down the flow when running the function created for ingestion we won't access the httpdb
-    # but rather "understand" that we are running on server side and call the DB.
-    await run_in_threadpool(feature_set._override_run_db, db_session)
 
     if ingest_parameters.targets:
         data_targets = [

mlrun/api/api/endpoints/files.py

@@ -31,7 +31,13 @@ from mlrun.utils import logger
 router = fastapi.APIRouter()
 
 
-@router.get("/files")
+# TODO: Remove in 1.7.0
+@router.get(
+    "/files",
+    deprecated=True,
+    description="'/files' and '/filestat' will be removed in 1.7.0, "
+    "use /projects/{project}/files instead.",
+)
 def get_files(
     schema: str = "",
     objpath: str = fastapi.Query("", alias="path"),
@@ -73,7 +79,13 @@ async def get_files_with_project_secrets(
     )
 
 
-@router.get("/filestat")
+# TODO: Remove in 1.7.0
+@router.get(
+    "/filestat",
+    deprecated=True,
+    description="'/files' and '/filestat' will be removed in 1.7.0, "
+    "use /projects/{project}/filestat instead.",
+)
 def get_filestat(
     schema: str = "",
     path: str = "",

mlrun/api/api/endpoints/functions.py

@@ -706,9 +706,14 @@ def _build_function(
             reason=f"runtime error: {err_to_str(err)}",
         )
     try:
+        # connect to run db
         run_db = get_run_db_instance(db_session)
         fn.set_db_connection(run_db)
-        mlrun.api.launcher.ServerSideLauncher.enrich_runtime(runtime=fn)
+
+        # enrich runtime with project defaults
+        launcher = mlrun.api.launcher.ServerSideLauncher()
+        launcher.enrich_runtime(runtime=fn)
+
         fn.save(versioned=False)
         if fn.kind in RuntimeKinds.nuclio_runtimes():
             mlrun.api.api.utils.apply_enrichment_and_validation_on_function(

mlrun/api/api/endpoints/logs.py

@@ -21,10 +21,17 @@ import mlrun.api.crud
 import mlrun.api.utils.auth.verifier
 import mlrun.common.schemas
 
-router = fastapi.APIRouter(prefix="/log/{project}")
+router = fastapi.APIRouter()
 
 
-@router.post("/{uid}")
+# TODO: remove /log/{project}/{uid} in 1.7.0
+@router.post(
+    "/log/{project}/{uid}",
+    deprecated=True,
+    description="/log/{project}/{uid} is deprecated in 1.5.0 and will be removed in 1.7.0, "
+    "use /projects/{project}/logs/{uid} instead",
+)
+@router.post("/projects/{project}/logs/{uid}")
 async def store_log(
     request: fastapi.Request,
     project: str,
@@ -52,7 +59,14 @@ async def store_log(
     return {}
 
 
-@router.get("/{uid}")
+# TODO: remove /log/{project}/{uid} in 1.7.0
+@router.get(
+    "/log/{project}/{uid}",
+    deprecated=True,
+    description="/log/{project}/{uid} is deprecated in 1.5.0 and will be removed in 1.7.0, "
+    "use /projects/{project}/logs/{uid} instead",
+)
+@router.get("/projects/{project}/logs/{uid}")
 async def get_log(
     project: str,
     uid: str,

mlrun/api/api/endpoints/pipelines.py

@@ -129,11 +129,9 @@ async def get_pipeline(
     auth_info: mlrun.common.schemas.AuthInfo = Depends(
         mlrun.api.api.deps.authenticate_request
     ),
-    db_session: Session = Depends(deps.get_db_session),
 ):
     pipeline = await run_in_threadpool(
         mlrun.api.crud.Pipelines().get_pipeline,
-        db_session,
         run_id,
         project,
         namespace,
@@ -145,7 +143,7 @@ async def get_pipeline(
         # legacy flow in which we first get the pipeline, resolve the project out of it, and only then query permissions
         # we don't use the return value from this function since the user may have asked for a different format than
         # summary which is the one used inside
-        await _get_pipeline_without_project(db_session, auth_info, run_id, namespace)
+        await _get_pipeline_without_project(auth_info, run_id, namespace)
     else:
         await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
             mlrun.common.schemas.AuthorizationResourceTypes.pipeline,
@@ -158,7 +156,6 @@ async def get_pipeline(
 
 
 async def _get_pipeline_without_project(
-    db_session: Session,
     auth_info: mlrun.common.schemas.AuthInfo,
     run_id: str,
     namespace: str,
@@ -170,7 +167,6 @@ async def _get_pipeline_without_project(
     """
     run = await run_in_threadpool(
         mlrun.api.crud.Pipelines().get_pipeline,
-        db_session,
         run_id,
         namespace=namespace,
         # minimal format that includes the project

mlrun/api/api/endpoints/projects.py

@@ -20,6 +20,7 @@ import sqlalchemy.orm
 from fastapi.concurrency import run_in_threadpool
 
 import mlrun.api.api.deps
+import mlrun.api.crud
 import mlrun.api.utils.auth.verifier
 import mlrun.api.utils.clients.chief
 import mlrun.common.schemas
@@ -323,6 +324,93 @@ async def get_project_summary(
     return project_summary
 
 
+@router.post("/projects/{name}/load")
+async def load_project(
+    name: str,
+    url: str,
+    secrets: mlrun.common.schemas.SecretsData = None,
+    auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
+        mlrun.api.api.deps.authenticate_request
+    ),
+    db_session: sqlalchemy.orm.Session = fastapi.Depends(
+        mlrun.api.api.deps.get_db_session
+    ),
+):
+    """
+    Loading a project remotely from a given source.
+
+    :param name:                project name
+    :param url:                 git or tar.gz or .zip sources archive path e.g.:
+                                git://github.com/mlrun/demo-xgb-project.git
+                                http://mysite/archived-project.zip
+                                The git project should include the project yaml file.
+    :param secrets:             Secrets to store in project in order to load it from the provided url.
+                                For more information see :py:func:`mlrun.load_project` function.
+    :param auth_info:           auth info of the request
+    :param db_session:          session that manages the current dialog with the database
+
+    :returns: a Run object of the load project function
+    """
+
+    project = mlrun.common.schemas.Project(
+        metadata=mlrun.common.schemas.ProjectMetadata(name=name),
+        spec=mlrun.common.schemas.ProjectSpec(source=url),
+    )
+
+    # We must create the project before we run the remote load_project function because
+    # we want this function will be running under the project itself instead of the default project.
+    project, _ = await fastapi.concurrency.run_in_threadpool(
+        get_project_member().create_project,
+        db_session=db_session,
+        project=project,
+        projects_role=auth_info.projects_role,
+        leader_session=auth_info.session,
+    )
+
+    # Storing secrets in project
+    if secrets is not None:
+        await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
+            mlrun.common.schemas.AuthorizationResourceTypes.secret,
+            project.metadata.name,
+            secrets.provider,
+            mlrun.common.schemas.AuthorizationAction.create,
+            auth_info,
+        )
+
+        await run_in_threadpool(
+            mlrun.api.crud.Secrets().store_project_secrets,
+            project.metadata.name,
+            secrets,
+        )
+
+    # Creating the auxiliary function for loading the project:
+    load_project_runner = await fastapi.concurrency.run_in_threadpool(
+        mlrun.api.crud.WorkflowRunners().create_runner,
+        run_name=f"load-{name}",
+        project=name,
+        db_session=db_session,
+        auth_info=auth_info,
+        image=mlrun.mlconf.default_base_image,
+    )
+
+    logger.debug(
+        "Saved function for loading project",
+        project_name=name,
+        function_name=load_project_runner.metadata.name,
+        kind=load_project_runner.kind,
+        source=project.spec.source,
+    )
+
+    run = await fastapi.concurrency.run_in_threadpool(
+        mlrun.api.crud.WorkflowRunners().run,
+        runner=load_project_runner,
+        project=project,
+        workflow_request=None,
+        load_only=True,
+    )
+    return {"data": run.to_dict()}
+
+
 def _is_request_from_leader(
     projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole],
 ) -> bool:

mlrun/api/api/endpoints/runs.py

@@ -31,7 +31,14 @@ from mlrun.utils.helpers import datetime_from_iso
 router = APIRouter()
 
 
-@router.post("/run/{project}/{uid}")
+# TODO: remove /run/{project}/{uid} in 1.7.0
+@router.post(
+    "/run/{project}/{uid}",
+    deprecated=True,
+    description="/run/{project}/{uid} is deprecated in 1.5.0 and will be removed in 1.7.0, "
+    "use /projects/{project}/runs/{uid} instead",
+)
+@router.post("/projects/{project}/runs/{uid}")
 async def store_run(
     request: Request,
     project: str,
@@ -70,7 +77,14 @@ async def store_run(
     return {}
 
 
-@router.patch("/run/{project}/{uid}")
+# TODO: remove /run/{project}/{uid} in 1.7.0
+@router.patch(
+    "/run/{project}/{uid}",
+    deprecated=True,
+    description="/run/{project}/{uid} is deprecated in 1.5.0 and will be removed in 1.7.0, "
+    "use /projects/{project}/runs/{uid} instead",
+)
+@router.patch("/projects/{project}/runs/{uid}")
 async def update_run(
     request: Request,
     project: str,
@@ -103,7 +117,14 @@ async def update_run(
     return {}
 
 
-@router.get("/run/{project}/{uid}")
+# TODO: remove /run/{project}/{uid} in 1.7.0
+@router.get(
+    "/run/{project}/{uid}",
+    deprecated=True,
+    description="/run/{project}/{uid} is deprecated in 1.5.0 and will be removed in 1.7.0, "
+    "use /projects/{project}/runs/{uid} instead",
+)
+@router.get("/projects/{project}/runs/{uid}")
 async def get_run(
     project: str,
     uid: str,
@@ -126,7 +147,14 @@ async def get_run(
     }
 
 
-@router.delete("/run/{project}/{uid}")
+# TODO: remove /run/{project}/{uid} in 1.7.0
+@router.delete(
+    "/run/{project}/{uid}",
+    deprecated=True,
+    description="/run/{project}/{uid} is deprecated in 1.5.0 and will be removed in 1.7.0, "
+    "use /projects/{project}/runs/{uid} instead",
+)
+@router.delete("/projects/{project}/runs/{uid}")
 async def delete_run(
     project: str,
     uid: str,
@@ -151,7 +179,14 @@ async def delete_run(
     return {}
 
 
-@router.get("/runs")
+# TODO: remove /runs in 1.7.0
+@router.get(
+    "/runs",
+    deprecated=True,
+    description="/runs is deprecated in 1.5.0 and will be removed in 1.7.0, "
+    "use /projects/{project}/runs/{uid} instead",
+)
+@router.get("/projects/{project}/runs")
 async def list_runs(
     project: str = None,
     name: str = None,
@@ -222,7 +257,14 @@ async def list_runs(
     }
 
 
-@router.delete("/runs")
+# TODO: remove /runs in 1.7.0
+@router.delete(
+    "/runs",
+    deprecated=True,
+    description="/runs is deprecated in 1.5.0 and will be removed in 1.7.0, "
+    "use /projects/{project}/runs/{uid} instead",
+)
+@router.delete("/projects/{project}/runs")
 async def delete_runs(
     project: str = None,
     name: str = None,