fractal-server 2.16.5__py3-none-any.whl → 2.17.0a0__py3-none-any.whl

fractal_server/app/routes/api/v2/submit.py

@@ -10,6 +10,7 @@ from fastapi import Request
 from fastapi import status
 from sqlmodel import select
 
+from ...aux.validate_user_profile import validate_user_profile
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import clean_app_job_list_v2
@@ -29,6 +30,7 @@ from fractal_server.app.routes.aux.validate_user_settings import (
 from fractal_server.app.schemas.v2 import JobCreateV2
 from fractal_server.app.schemas.v2 import JobReadV2
 from fractal_server.app.schemas.v2 import JobStatusTypeV2
+from fractal_server.app.schemas.v2 import ResourceType
 from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.runner.set_start_and_last_task_index import (
@@ -122,10 +124,14 @@ async def apply_workflow(
         )
         used_task_group_ids.add(task.taskgroupv2_id)
 
+    # Get validated resource and profile
+    resource, profile = await validate_user_profile(
+        user=user,
+        db=db,
+    )
     # Validate user settings
-    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
     user_settings = await validate_user_settings(
-        user=user, backend=FRACTAL_RUNNER_BACKEND, db=db
+        user=user, backend=resource.type, db=db
     )
     # Check that no other job with the same dataset_id is SUBMITTED
     stm = (
@@ -157,11 +163,11 @@ async def apply_workflow(
             job_create.slurm_account = user_settings.slurm_accounts[0]
 
     # User appropriate FractalSSH object
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+    if resource.type == ResourceType.SLURM_SSH:
         ssh_config = dict(
-            user=user_settings.ssh_username,
-            host=user_settings.ssh_host,
-            key_path=user_settings.ssh_private_key_path,
+            user=profile.username,
+            host=resource.host,
+            key_path=profile.ssh_key_path,
         )
         fractal_ssh_list = request.app.state.fractal_ssh_list
         try:
@@ -193,7 +199,7 @@ async def apply_workflow(
             workflow.model_dump_json(exclude={"task_list"})
         ),
         project_dump=json.loads(
-            project.model_dump_json(exclude={"user_list"})
+            project.model_dump_json(exclude={"user_list", "resource_id"})
         ),
         **job_create.model_dump(),
     )
@@ -214,26 +220,28 @@ async def apply_workflow(
 
     # Define server-side job directory
     timestamp_string = job.start_timestamp.strftime("%Y%m%d_%H%M%S")
-    WORKFLOW_DIR_LOCAL = settings.FRACTAL_RUNNER_WORKING_BASE_DIR / (
+    WORKFLOW_DIR_LOCAL = Path(resource.jobs_local_dir) / (
         f"proj_v2_{project_id:07d}_wf_{workflow_id:07d}_job_{job.id:07d}"
         f"_{timestamp_string}"
     )
 
-    cache_dir = (
-        Path(user_settings.project_dir) / ".fractal_cache"
-        if user_settings.project_dir is not None
-        else None
-    )
-
     # Define user-side job directory
-    if FRACTAL_RUNNER_BACKEND == "local":
-        WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
-    elif FRACTAL_RUNNER_BACKEND == "slurm":
-        WORKFLOW_DIR_REMOTE = cache_dir / WORKFLOW_DIR_LOCAL.name
-    elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-        WORKFLOW_DIR_REMOTE = (
-            Path(user_settings.ssh_jobs_dir) / WORKFLOW_DIR_LOCAL.name
-        )
+    match resource.type:
+        case ResourceType.LOCAL:
+            WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
+            cache_dir = None
+        case ResourceType.SLURM_SUDO:
+            cache_dir = (
+                Path(user_settings.project_dir) / ".fractal_cache"
+                if user_settings.project_dir is not None
+                else None
+            )
+            WORKFLOW_DIR_REMOTE = cache_dir / WORKFLOW_DIR_LOCAL.name
+        case ResourceType.SLURM_SSH:
+            WORKFLOW_DIR_REMOTE = (
+                Path(profile.jobs_remote_dir) / WORKFLOW_DIR_LOCAL.name
+            )
+            cache_dir = None
 
     # Update job folders in the db
     job.working_dir = WORKFLOW_DIR_LOCAL.as_posix()
@@ -250,11 +258,11 @@ async def apply_workflow(
         dataset_id=dataset.id,
         job_id=job.id,
         user_id=user.id,
-        user_settings=user_settings,
         worker_init=job.worker_init,
-        slurm_user=user_settings.slurm_user,
         user_cache_dir=cache_dir.as_posix() if cache_dir else None,
         fractal_ssh=fractal_ssh,
+        resource=resource,
+        profile=profile,
     )
     request.app.state.jobsV2.append(job.id)
     logger.info(

fractal_server/app/routes/api/v2/task.py

@@ -9,6 +9,7 @@ from sqlmodel import func
 from sqlmodel import or_
 from sqlmodel import select
 
+from ._aux_functions import _get_resource_and_profile_ids
 from ._aux_functions_tasks import _get_task_full_access
 from ._aux_functions_tasks import _get_task_read_access
 from ._aux_functions_tasks import _get_valid_user_group_id
@@ -187,9 +188,13 @@ async def create_task(
         user_group_id=user_group_id,
         version=db_task.version,
     )
+    resource_id, _ = await _get_resource_and_profile_ids(
+        user_id=user.id, db=db
+    )
     db_task_group = TaskGroupV2(
         user_id=user.id,
         user_group_id=user_group_id,
+        resource_id=resource_id,
         active=True,
         task_list=[db_task],
         origin=TaskGroupV2OriginEnum.OTHER,

fractal_server/app/routes/api/v2/task_collection.py

@@ -14,10 +14,8 @@ from pydantic import BaseModel
 from pydantic import model_validator
 from pydantic import ValidationError
 
-from .....config import get_settings
 from .....logger import reset_logger_handlers
 from .....logger import set_logger
-from .....syringe import Inject
 from ....db import AsyncSession
 from ....db import get_async_db
 from ....models.v2 import TaskGroupV2
@@ -26,7 +24,8 @@ from ....schemas.v2 import TaskCollectPipV2
 from ....schemas.v2 import TaskGroupActivityStatusV2
 from ....schemas.v2 import TaskGroupActivityV2Read
 from ....schemas.v2 import TaskGroupCreateV2Strict
-from ...aux.validate_user_settings import validate_user_settings
+from ...aux.validate_user_profile import validate_user_profile
+from ._aux_functions import _get_resource_and_profile_ids
 from ._aux_functions_task_lifecycle import get_package_version_from_pypi
 from ._aux_functions_tasks import _get_valid_user_group_id
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
@@ -35,11 +34,11 @@ from ._aux_functions_tasks import _verify_non_duplication_user_constraint
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.routes.auth import current_active_verified_user
+from fractal_server.app.schemas.v2 import ResourceType
 from fractal_server.app.schemas.v2 import (
     TaskGroupActivityActionV2,
 )
 from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
-from fractal_server.ssh._fabric import SSHConfig
 from fractal_server.tasks.v2.local.collect import (
     collect_local,
 )
@@ -47,7 +46,7 @@ from fractal_server.tasks.v2.ssh import collect_ssh
 from fractal_server.tasks.v2.utils_package_names import _parse_wheel_filename
 from fractal_server.tasks.v2.utils_package_names import normalize_package_name
 from fractal_server.tasks.v2.utils_python_interpreter import (
-    get_python_interpreter_v2,
+    get_python_interpreter,
 )
 
 
@@ -168,12 +167,19 @@ async def collect_tasks_pip(
     """
     Task-collection endpoint
     """
-    # Get settings
-    settings = Inject(get_settings)
 
+    # Get validated resource and profile
+    resource, profile = await validate_user_profile(
+        user=user,
+        db=db,
+    )
     # Get some validated request data
     task_collect = request_data.task_collect
 
+    resource_id, _ = await _get_resource_and_profile_ids(
+        user_id=user.id, db=db
+    )
+
     # Initialize task-group attributes
     task_group_attrs = dict(
         user_id=user.id,
@@ -182,14 +188,15 @@ async def collect_tasks_pip(
 
     # Set/check python version
     if task_collect.python_version is None:
-        task_group_attrs[
-            "python_version"
-        ] = settings.FRACTAL_TASKS_PYTHON_DEFAULT_VERSION
+        task_group_attrs["python_version"] = resource.tasks_python_config[
+            "default_version"
+        ]
     else:
         task_group_attrs["python_version"] = task_collect.python_version
     try:
-        get_python_interpreter_v2(
-            python_version=task_group_attrs["python_version"]
+        get_python_interpreter(
+            python_version=task_group_attrs["python_version"],
+            resource=resource,
         )
     except ValueError:
         raise HTTPException(
@@ -259,16 +266,11 @@ async def collect_tasks_pip(
     # Set user_group_id
     task_group_attrs["user_group_id"] = user_group_id
 
-    # Validate user settings (backend-specific)
-    user_settings = await validate_user_settings(
-        user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
-    )
-
     # Set path and venv_path
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-        base_tasks_path = user_settings.ssh_tasks_dir
+    if resource.type == ResourceType.SLURM_SSH:
+        base_tasks_path = profile.tasks_remote_dir
     else:
-        base_tasks_path = settings.FRACTAL_TASKS_DIR.as_posix()
+        base_tasks_path = resource.tasks_local_dir
     task_group_path = (
         Path(base_tasks_path)
         / str(user.id)
@@ -309,7 +311,7 @@ async def collect_tasks_pip(
 
     # On-disk checks
 
-    if settings.FRACTAL_RUNNER_BACKEND != "slurm_ssh":
+    if resource.type != ResourceType.SLURM_SSH:
         # Verify that folder does not exist (for local collection)
         if Path(task_group_path).exists():
             raise HTTPException(
@@ -318,7 +320,7 @@ async def collect_tasks_pip(
             )
 
     # Create TaskGroupV2 object
-    task_group = TaskGroupV2(**task_group_attrs)
+    task_group = TaskGroupV2(**task_group_attrs, resource_id=resource_id)
     db.add(task_group)
     await db.commit()
     await db.refresh(task_group)
@@ -340,33 +342,20 @@ async def collect_tasks_pip(
 
     # END of SSH/non-SSH common part
 
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-        # SSH task collection
-        # Use appropriate FractalSSH object
-        ssh_config = SSHConfig(
-            user=user_settings.ssh_username,
-            host=user_settings.ssh_host,
-            key_path=user_settings.ssh_private_key_path,
-        )
-
-        background_tasks.add_task(
-            collect_ssh,
-            task_group_id=task_group.id,
-            task_group_activity_id=task_group_activity.id,
-            ssh_config=ssh_config,
-            tasks_base_dir=user_settings.ssh_tasks_dir,
-            wheel_file=wheel_file,
-        )
-
+    if resource.type == ResourceType.SLURM_SSH:
+        collect_function = collect_ssh
     else:
-        # Local task collection
+        collect_function = collect_local
+
+    background_tasks.add_task(
+        collect_function,
+        task_group_id=task_group.id,
+        task_group_activity_id=task_group_activity.id,
+        wheel_file=wheel_file,
+        resource=resource,
+        profile=profile,
+    )
 
-        background_tasks.add_task(
-            collect_local,
-            task_group_id=task_group.id,
-            task_group_activity_id=task_group_activity.id,
-            wheel_file=wheel_file,
-        )
     logger.debug(
         "Task-collection endpoint: start background collection "
         "and return task_group_activity"

fractal_server/app/routes/api/v2/task_collection_custom.py

@@ -9,6 +9,8 @@ from fastapi import HTTPException
 from fastapi import status
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from ...aux.validate_user_profile import validate_user_profile
+from ._aux_functions import _get_resource_and_profile_ids
 from ._aux_functions_tasks import _get_valid_user_group_id
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
 from ._aux_functions_tasks import _verify_non_duplication_user_constraint
@@ -16,15 +18,14 @@ from fractal_server.app.db import get_async_db
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.routes.auth import current_active_verified_user
+from fractal_server.app.schemas.v2 import ResourceType
 from fractal_server.app.schemas.v2 import TaskCollectCustomV2
 from fractal_server.app.schemas.v2 import TaskCreateV2
 from fractal_server.app.schemas.v2 import TaskGroupCreateV2
 from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
 from fractal_server.app.schemas.v2 import TaskReadV2
-from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.string_tools import validate_cmd
-from fractal_server.syringe import Inject
 from fractal_server.tasks.v2.utils_background import (
     prepare_tasks_metadata,
 )
@@ -47,7 +48,8 @@ async def collect_task_custom(
     user: UserOAuth = Depends(current_active_verified_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[TaskReadV2]:
-    settings = Inject(get_settings)
+    # Get validated resource and profile
+    resource, profile = await validate_user_profile(user=user, db=db)
 
     # Validate query parameters related to user-group ownership
     user_group_id = await _get_valid_user_group_id(
@@ -57,7 +59,7 @@ async def collect_task_custom(
         db=db,
     )
 
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+    if resource.type == ResourceType.SLURM_SSH:
         if task_collect.package_root is None:
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
@@ -143,6 +145,10 @@ async def collect_task_custom(
         package_version=task_collect.version,
     )
 
+    resource_id, _ = await _get_resource_and_profile_ids(
+        user_id=user.id, db=db
+    )
+
     # Prepare task-group attributes
     task_group_attrs = dict(
         origin=TaskGroupV2OriginEnum.OTHER,
@@ -167,7 +173,7 @@ async def collect_task_custom(
         db=db,
     )
 
-    task_group = TaskGroupV2(**task_group_attrs)
+    task_group = TaskGroupV2(**task_group_attrs, resource_id=resource_id)
     db.add(task_group)
     await db.commit()
     await db.refresh(task_group)

fractal_server/app/routes/api/v2/task_collection_pixi.py

@@ -10,6 +10,7 @@ from fastapi import Response
 from fastapi import status
 from fastapi import UploadFile
 
+from ._aux_functions import _get_resource_and_profile_ids
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
 from fractal_server.app.models import UserOAuth
@@ -28,18 +29,16 @@ from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _verify_non_duplication_user_constraint,
 )
 from fractal_server.app.routes.auth import current_active_verified_user
-from fractal_server.app.routes.aux.validate_user_settings import (
-    validate_user_settings,
+from fractal_server.app.routes.aux.validate_user_profile import (
+    validate_user_profile,
 )
 from fractal_server.app.schemas.v2 import FractalUploadedFile
+from fractal_server.app.schemas.v2 import ResourceType
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
 from fractal_server.app.schemas.v2.task_group import TaskGroupV2OriginEnum
-from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
-from fractal_server.ssh._fabric import SSHConfig
-from fractal_server.syringe import Inject
 from fractal_server.tasks.v2.local import collect_local_pixi
 from fractal_server.tasks.v2.ssh import collect_ssh_pixi
 from fractal_server.tasks.v2.utils_package_names import normalize_package_name
@@ -89,23 +88,26 @@ async def collect_task_pixi(
     user: UserOAuth = Depends(current_active_verified_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> TaskGroupActivityV2Read:
-    settings = Inject(get_settings)
+    # Get validated resource and profile
+    resource, profile = await validate_user_profile(user=user, db=db)
+
     # Check if Pixi is available
-    if settings.pixi is None:
+    if not resource.tasks_pixi_config:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
             detail="Pixi task collection is not available.",
         )
     # Check if provided Pixi version is available. Use default if not provided
     if pixi_version is None:
-        pixi_version = settings.pixi.default_version
+        pixi_version = resource.tasks_pixi_config["default_version"]
     else:
-        if pixi_version not in settings.pixi.versions:
+        if pixi_version not in resource.tasks_pixi_config["versions"]:
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
                 detail=(
-                    f"Pixi version {pixi_version} is not available. Available "
-                    f"versions: {list(settings.pixi.versions.keys())}"
+                    f"Pixi version '{pixi_version}' is not available. "
+                    "Available versions: "
+                    f"{list(resource.tasks_pixi_config['versions'].keys())}"
                 ),
             )
 
@@ -123,21 +125,22 @@ async def collect_task_pixi(
         db=db,
     )
 
-    user_settings = await validate_user_settings(
-        user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
-    )
-
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-        base_tasks_path = user_settings.ssh_tasks_dir
+    if resource.type == ResourceType.SLURM_SSH:
+        base_tasks_path = profile.tasks_remote_dir
     else:
-        base_tasks_path = settings.FRACTAL_TASKS_DIR.as_posix()
+        base_tasks_path = resource.tasks_local_dir
     task_group_path = (
         Path(base_tasks_path) / str(user.id) / pkg_name / version
     ).as_posix()
 
+    resource_id, _ = await _get_resource_and_profile_ids(
+        user_id=user.id, db=db
+    )
+
     task_group_attrs = dict(
         user_id=user.id,
         user_group_id=user_group_id,
+        resource_id=resource_id,
         origin=TaskGroupV2OriginEnum.PIXI,
         pixi_version=pixi_version,
         pkg_name=pkg_name,
@@ -162,7 +165,7 @@ async def collect_task_pixi(
         db=db,
     )
 
-    if settings.FRACTAL_RUNNER_BACKEND != "slurm_ssh":
+    if resource.type != ResourceType.SLURM_SSH:
         if Path(task_group_path).exists():
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
@@ -187,28 +190,19 @@ async def collect_task_pixi(
     await db.commit()
     await db.refresh(task_group_activity)
 
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-        ssh_config = SSHConfig(
-            user=user_settings.ssh_username,
-            host=user_settings.ssh_host,
-            key_path=user_settings.ssh_private_key_path,
-        )
-
-        background_tasks.add_task(
-            collect_ssh_pixi,
-            task_group_id=task_group.id,
-            task_group_activity_id=task_group_activity.id,
-            ssh_config=ssh_config,
-            tasks_base_dir=user_settings.ssh_tasks_dir,
-            tar_gz_file=tar_gz_file,
-        )
+    if resource.type == ResourceType.SLURM_SSH:
+        collect_function = collect_ssh_pixi
     else:
-        background_tasks.add_task(
-            collect_local_pixi,
-            task_group_id=task_group.id,
-            task_group_activity_id=task_group_activity.id,
-            tar_gz_file=tar_gz_file,
-        )
+        collect_function = collect_local_pixi
+
+    background_tasks.add_task(
+        collect_function,
+        task_group_id=task_group.id,
+        task_group_activity_id=task_group_activity.id,
+        tar_gz_file=tar_gz_file,
+        resource=resource,
+        profile=profile,
+    )
     logger.info(
         "Task-collection endpoint: start background collection "
         "and return task_group_activity. "