fractal-server 2.5.1__py3-none-any.whl → 2.6.0__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.5.1"
+__VERSION__ = "2.6.0"

fractal_server/__main__.py

@@ -43,9 +43,17 @@ openapi_parser.add_argument(
 )
 
 # fractalctl set-db
-subparsers.add_parser(
+set_db_parser = subparsers.add_parser(
     "set-db",
-    description="Initialise the database and apply schema migrations",
+    description=(
+        "Initialise/upgrade database schemas and create first group&user."
+    ),
+)
+set_db_parser.add_argument(
+    "--skip-init-data",
+    action="store_true",
+    help="If set, do not try creating first group and user.",
+    default=False,
 )
 
 # fractalctl update-db-data
@@ -66,27 +74,34 @@ def save_openapi(dest="openapi.json"):
         json.dump(openapi_schema, f)
 
 
-def set_db():
+def set_db(skip_init_data: bool = False):
     """
-    Set-up / Upgrade database schema
+    Upgrade database schema *and* create first group/user
 
     Call alembic to upgrade to the latest migration.
-
     Ref: https://stackoverflow.com/a/56683030/283972
+
+    Arguments:
+        skip_init_data: If `True`, skip creation of first group and user.
     """
-    import alembic.config
-    from pathlib import Path
-    import fractal_server
     from fractal_server.app.security import _create_first_user
     from fractal_server.app.security import _create_first_group
     from fractal_server.syringe import Inject
     from fractal_server.config import get_settings
 
+    import alembic.config
+    from pathlib import Path
+    import fractal_server
+
     alembic_ini = Path(fractal_server.__file__).parent / "alembic.ini"
     alembic_args = ["-c", alembic_ini.as_posix(), "upgrade", "head"]
     print(f"START: Run alembic.config, with argv={alembic_args}")
     alembic.config.main(argv=alembic_args)
     print("END: alembic.config")
+
+    if skip_init_data:
+        return
+
     # Insert default group
     print()
     _create_first_group()
@@ -179,7 +194,7 @@ def run():
     if args.cmd == "openapi":
         save_openapi(dest=args.openapi_file)
     elif args.cmd == "set-db":
-        set_db()
+        set_db(skip_init_data=args.skip_init_data)
     elif args.cmd == "update-db-data":
         update_db_data()
     elif args.cmd == "start":

fractal_server/app/models/__init__.py

@@ -7,5 +7,6 @@ from .linkusergroup import LinkUserGroup  # noqa: F401
 from .linkuserproject import LinkUserProject  # noqa: F401
 from .linkuserproject import LinkUserProjectV2  # noqa: F401
 from .security import *  # noqa
+from .user_settings import UserSettings  # noqa
 from .v1 import *  # noqa
 from .v2 import *  # noqa

fractal_server/app/models/security.py

@@ -20,6 +20,7 @@ from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
+from .user_settings import UserSettings
 from fractal_server.utils import get_timestamp
 
 
@@ -104,6 +105,13 @@ class UserOAuth(SQLModel, table=True):
         sa_relationship_kwargs={"lazy": "joined", "cascade": "all, delete"},
     )
 
+    user_settings_id: Optional[int] = Field(
+        foreign_key="user_settings.id", default=None
+    )
+    settings: Optional[UserSettings] = Relationship(
+        sa_relationship_kwargs=dict(lazy="selectin", cascade="all, delete")
+    )
+
     class Config:
         orm_mode = True
 

fractal_server/app/models/user_settings.py

@@ -0,0 +1,38 @@
+from typing import Optional
+
+from sqlalchemy import Column
+from sqlalchemy.types import JSON
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+
+class UserSettings(SQLModel, table=True):
+    """
+    Comprehensive list of user settings.
+
+    Attributes:
+        id: ID of database object
+        slurm_accounts:
+            List of SLURM accounts, to be used upon Fractal job submission.
+        ssh_host: SSH-reachable host where a SLURM client is available.
+        ssh_username: User on `ssh_host`.
+        ssh_private_key_path: Path of private SSH key for `ssh_username`.
+        ssh_tasks_dir: Task-venvs base folder on `ssh_host`.
+        ssh_jobs_dir: Jobs base folder on `ssh_host`.
+        slurm_user: Local user, to be impersonated via `sudo -u`
+        cache_dir: Folder where `slurm_user` can write.
+    """
+
+    __tablename__ = "user_settings"
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    slurm_accounts: list[str] = Field(
+        sa_column=Column(JSON, server_default="[]", nullable=False)
+    )
+    ssh_host: Optional[str] = None
+    ssh_username: Optional[str] = None
+    ssh_private_key_path: Optional[str] = None
+    ssh_tasks_dir: Optional[str] = None
+    ssh_jobs_dir: Optional[str] = None
+    slurm_user: Optional[str] = None
+    cache_dir: Optional[str] = None

fractal_server/app/routes/api/v1/_aux_functions.py

@@ -22,6 +22,7 @@ from ....models.v1 import Task
 from ....models.v1 import Workflow
 from ....models.v1 import WorkflowTask
 from ....schemas.v1 import JobStatusTypeV1
+from ...aux.validate_user_settings import verify_user_has_settings
 from fractal_server.app.models import UserOAuth
 
 
@@ -367,7 +368,11 @@ async def _get_task_check_owner(
                 ),
             )
         else:
-            owner = user.username or user.slurm_user
+            if user.username:
+                owner = user.username
+            else:
+                verify_user_has_settings(user)
+                owner = user.settings.slurm_user
             if owner != task.owner:
                 raise HTTPException(
                     status_code=status.HTTP_403_FORBIDDEN,

fractal_server/app/routes/api/v1/project.py

@@ -34,6 +34,7 @@ from ....schemas.v1 import JobStatusTypeV1
 from ....schemas.v1 import ProjectCreateV1
 from ....schemas.v1 import ProjectReadV1
 from ....schemas.v1 import ProjectUpdateV1
+from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions import _check_project_exists
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_project_check_owner
@@ -321,25 +322,11 @@ async def apply_workflow(
             ),
         )
 
-    # If backend is SLURM, check that the user has required attributes
-    backend = settings.FRACTAL_RUNNER_BACKEND
-    if backend == "slurm":
-        if not user.slurm_user:
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=(
-                    f"FRACTAL_RUNNER_BACKEND={backend}, "
-                    f"but {user.slurm_user=}."
-                ),
-            )
-        if not user.cache_dir:
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=(
-                    f"FRACTAL_RUNNER_BACKEND={backend}, "
-                    f"but {user.cache_dir=}."
-                ),
-            )
+    # Validate user settings
+    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+    user_settings = await validate_user_settings(
+        user=user, backend=FRACTAL_RUNNER_BACKEND, db=db
+    )
 
     # Check that datasets have the right number of resources
     if not input_dataset.resource_list:
@@ -386,7 +373,7 @@ async def apply_workflow(
         )
 
     if apply_workflow.slurm_account is not None:
-        if apply_workflow.slurm_account not in user.slurm_accounts:
+        if apply_workflow.slurm_account not in user_settings.slurm_accounts:
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                 detail=(
@@ -395,8 +382,8 @@ async def apply_workflow(
                 ),
             )
     else:
-        if len(user.slurm_accounts) > 0:
-            apply_workflow.slurm_account = user.slurm_accounts[0]
+        if len(user_settings.slurm_accounts) > 0:
+            apply_workflow.slurm_account = user_settings.slurm_accounts[0]
 
     # Add new ApplyWorkflow object to DB
     job = ApplyWorkflow(
@@ -480,8 +467,8 @@ async def apply_workflow(
         output_dataset_id=output_dataset.id,
         job_id=job.id,
         worker_init=apply_workflow.worker_init,
-        slurm_user=user.slurm_user,
-        user_cache_dir=user.cache_dir,
+        slurm_user=user_settings.slurm_user,
+        user_cache_dir=user_settings.cache_dir,
     )
     request.app.state.jobsV1.append(job.id)
     logger.info(

fractal_server/app/routes/api/v1/task.py

@@ -17,6 +17,7 @@ from ....models.v2 import TaskV2
 from ....schemas.v1 import TaskCreateV1
 from ....schemas.v1 import TaskReadV1
 from ....schemas.v1 import TaskUpdateV1
+from ...aux.validate_user_settings import verify_user_has_settings
 from ._aux_functions import _get_task_check_owner
 from ._aux_functions import _raise_if_v1_is_read_only
 from fractal_server.app.models import UserOAuth
@@ -126,16 +127,18 @@ async def create_task(
     # Set task.owner attribute
     if user.username:
         owner = user.username
-    elif user.slurm_user:
-        owner = user.slurm_user
     else:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(
-                "Cannot add a new task because current user does not "
-                "have `username` or `slurm_user` attributes."
-            ),
-        )
+        verify_user_has_settings(user)
+        if user.settings.slurm_user:
+            owner = user.settings.slurm_user
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Cannot add a new task because current user does not "
+                    "have `username` or `slurm_user` attributes."
+                ),
+            )
 
     # Prepend owner to task.source
     task.source = f"{owner}:{task.source}"

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -21,6 +21,7 @@ from ....models.v2 import TaskV2
 from ....models.v2 import WorkflowTaskV2
 from ....models.v2 import WorkflowV2
 from ....schemas.v2 import JobStatusTypeV2
+from ...aux.validate_user_settings import verify_user_has_settings
 from fractal_server.app.models import UserOAuth
 from fractal_server.images import Filters
 
@@ -362,7 +363,11 @@ async def _get_task_check_owner(
                 ),
             )
         else:
-            owner = user.username or user.slurm_user
+            if user.username:
+                owner = user.username
+            else:
+                verify_user_has_settings(user)
+                owner = user.settings.slurm_user
             if owner != task.owner:
                 raise HTTPException(
                     status_code=status.HTTP_403_FORBIDDEN,

fractal_server/app/routes/api/v2/submit.py

@@ -27,6 +27,7 @@ from ....runner.v2 import submit_workflow
 from ....schemas.v2 import JobCreateV2
 from ....schemas.v2 import JobReadV2
 from ....schemas.v2 import JobStatusTypeV2
+from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import clean_app_job_list_v2
@@ -109,19 +110,11 @@ async def apply_workflow(
             ),
         )
 
-    # If backend is SLURM, check that the user has required attributes
+    # Validate user settings
     FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
-    if FRACTAL_RUNNER_BACKEND == "slurm":
-        if not user.slurm_user:
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=f"{FRACTAL_RUNNER_BACKEND=}, but {user.slurm_user=}.",
-            )
-        if not user.cache_dir:
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=f"{FRACTAL_RUNNER_BACKEND=}, but {user.cache_dir=}.",
-            )
+    user_settings = await validate_user_settings(
+        user=user, backend=FRACTAL_RUNNER_BACKEND, db=db
+    )
 
     # Check that no other job with the same dataset_id is SUBMITTED
     stm = (
@@ -140,7 +133,7 @@ async def apply_workflow(
         )
 
     if job_create.slurm_account is not None:
-        if job_create.slurm_account not in user.slurm_accounts:
+        if job_create.slurm_account not in user_settings.slurm_accounts:
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                 detail=(
@@ -149,8 +142,8 @@ async def apply_workflow(
                 ),
             )
     else:
-        if len(user.slurm_accounts) > 0:
-            job_create.slurm_account = user.slurm_accounts[0]
+        if len(user_settings.slurm_accounts) > 0:
+            job_create.slurm_account = user_settings.slurm_accounts[0]
 
     # Add new Job object to DB
     job = JobV2(
@@ -224,12 +217,11 @@ async def apply_workflow(
         WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
     elif FRACTAL_RUNNER_BACKEND == "slurm":
         WORKFLOW_DIR_REMOTE = (
-            Path(user.cache_dir) / f"{WORKFLOW_DIR_LOCAL.name}"
+            Path(user_settings.cache_dir) / f"{WORKFLOW_DIR_LOCAL.name}"
         )
     elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
         WORKFLOW_DIR_REMOTE = (
-            Path(settings.FRACTAL_SLURM_SSH_WORKING_BASE_DIR)
-            / f"{WORKFLOW_DIR_LOCAL.name}"
+            Path(user_settings.ssh_jobs_dir) / f"{WORKFLOW_DIR_LOCAL.name}"
         )
 
     # Update job folders in the db
@@ -238,15 +230,31 @@ async def apply_workflow(
     await db.merge(job)
     await db.commit()
 
+    # User appropriate FractalSSH object
+    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+        ssh_credentials = dict(
+            user=user_settings.ssh_username,
+            host=user_settings.ssh_host,
+            key_path=user_settings.ssh_private_key_path,
+        )
+        fractal_ssh_list = request.app.state.fractal_ssh_list
+        fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
+    else:
+        fractal_ssh = None
+
+    # Expunge user settings from db, to use in background task
+    db.expunge(user_settings)
+
     background_tasks.add_task(
         submit_workflow,
         workflow_id=workflow.id,
         dataset_id=dataset.id,
         job_id=job.id,
+        user_settings=user_settings,
         worker_init=job.worker_init,
-        slurm_user=user.slurm_user,
-        user_cache_dir=user.cache_dir,
-        fractal_ssh=request.app.state.fractal_ssh,
+        slurm_user=user_settings.slurm_user,
+        user_cache_dir=user_settings.cache_dir,
+        fractal_ssh=fractal_ssh,
     )
     request.app.state.jobsV2.append(job.id)
     logger.info(

fractal_server/app/routes/api/v2/task.py

@@ -18,6 +18,7 @@ from ....models.v2 import WorkflowV2
 from ....schemas.v2 import TaskCreateV2
 from ....schemas.v2 import TaskReadV2
 from ....schemas.v2 import TaskUpdateV2
+from ...aux.validate_user_settings import verify_user_has_settings
 from ._aux_functions import _get_task_check_owner
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
@@ -150,16 +151,18 @@ async def create_task(
     # Set task.owner attribute
     if user.username:
         owner = user.username
-    elif user.slurm_user:
-        owner = user.slurm_user
     else:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(
-                "Cannot add a new task because current user does not "
-                "have `username` or `slurm_user` attributes."
-            ),
-        )
+        verify_user_has_settings(user)
+        if user.settings.slurm_user:
+            owner = user.settings.slurm_user
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Cannot add a new task because current user does not "
+                    "have `username` or `slurm_user` attributes."
+                ),
+            )
 
     # Prepend owner to task.source
     task.source = f"{owner}:{task.source}"

fractal_server/app/routes/api/v2/task_collection.py

@@ -25,6 +25,7 @@ from ....schemas.v2 import CollectionStateReadV2
 from ....schemas.v2 import CollectionStatusV2
 from ....schemas.v2 import TaskCollectPipV2
 from ....schemas.v2 import TaskReadV2
+from ...aux.validate_user_settings import validate_user_settings
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth import current_active_verified_user
@@ -41,7 +42,6 @@ from fractal_server.tasks.v2.endpoint_operations import download_package
 from fractal_server.tasks.v2.endpoint_operations import inspect_package
 from fractal_server.tasks.v2.utils import get_python_interpreter_v2
 
-
 router = APIRouter()
 
 logger = set_logger(__name__)
@@ -107,6 +107,11 @@ async def collect_tasks_pip(
             detail=f"Invalid task-collection object. Original error: {e}",
         )
 
+    # Validate user settings (backend-specific)
+    user_settings = await validate_user_settings(
+        user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
+    )
+
     # END of SSH/non-SSH common part
 
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
@@ -124,11 +129,21 @@ async def collect_tasks_pip(
         db.add(state)
         await db.commit()
 
+        # User appropriate FractalSSH object
+        ssh_credentials = dict(
+            user=user_settings.ssh_username,
+            host=user_settings.ssh_host,
+            key_path=user_settings.ssh_private_key_path,
+        )
+        fractal_ssh_list = request.app.state.fractal_ssh_list
+        fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
+
         background_tasks.add_task(
             background_collect_pip_ssh,
             state.id,
             task_pkg,
-            request.app.state.fractal_ssh,
+            fractal_ssh,
+            user_settings.ssh_tasks_dir,
         )
 
         response.status_code = status.HTTP_201_CREATED

fractal_server/app/routes/api/v2/task_collection_custom.py

@@ -18,6 +18,7 @@ from ....models.v2 import TaskV2
 from ....schemas.v2 import TaskCollectCustomV2
 from ....schemas.v2 import TaskCreateV2
 from ....schemas.v2 import TaskReadV2
+from ...aux.validate_user_settings import verify_user_has_settings
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_verified_user
 from fractal_server.string_tools import validate_cmd
@@ -112,7 +113,11 @@ async def collect_task_custom(
         package_root = Path(task_collect.package_root)
 
     # Set task.owner attribute
-    owner = user.username or user.slurm_user
+    if user.username:
+        owner = user.username
+    else:
+        verify_user_has_settings(user)
+        owner = user.settings.slurm_user
     if owner is None:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,

fractal_server/app/routes/auth/_aux_auth.py

@@ -3,11 +3,11 @@ from fastapi import status
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlmodel import select
 
-from ...models.linkusergroup import LinkUserGroup
-from ...models.security import UserGroup
-from ...models.security import UserOAuth
-from ...schemas.user import UserRead
-from ...schemas.user_group import UserGroupRead
+from fractal_server.app.models.linkusergroup import LinkUserGroup
+from fractal_server.app.models.security import UserGroup
+from fractal_server.app.models.security import UserOAuth
+from fractal_server.app.schemas.user import UserRead
+from fractal_server.app.schemas.user_group import UserGroupRead
 
 
 async def _get_single_user_with_group_names(

fractal_server/app/routes/auth/current_user.py

@@ -11,8 +11,12 @@ from ...db import get_async_db
 from ...schemas.user import UserRead
 from ...schemas.user import UserUpdate
 from ...schemas.user import UserUpdateStrict
+from ..aux.validate_user_settings import verify_user_has_settings
 from ._aux_auth import _get_single_user_with_group_names
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models import UserSettings
+from fractal_server.app.schemas import UserSettingsReadStrict
+from fractal_server.app.schemas import UserSettingsUpdateStrict
 from fractal_server.app.security import get_user_manager
 from fractal_server.app.security import UserManager
 
@@ -62,3 +66,40 @@ async def patch_current_user(
         patched_user, db
     )
     return patched_user_with_groups
+
+
+@router_current_user.get(
+    "/current-user/settings/", response_model=UserSettingsReadStrict
+)
+async def get_current_user_settings(
+    current_user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> UserSettingsReadStrict:
+
+    verify_user_has_settings(current_user)
+    user_settings = await db.get(UserSettings, current_user.user_settings_id)
+    return user_settings
+
+
+@router_current_user.patch(
+    "/current-user/settings/", response_model=UserSettingsReadStrict
+)
+async def patch_current_user_settings(
+    settings_update: UserSettingsUpdateStrict,
+    current_user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> UserSettingsReadStrict:
+
+    verify_user_has_settings(current_user)
+    current_user_settings = await db.get(
+        UserSettings, current_user.user_settings_id
+    )
+
+    for k, v in settings_update.dict(exclude_unset=True).items():
+        setattr(current_user_settings, k, v)
+
+    db.add(current_user_settings)
+    await db.commit()
+    await db.refresh(current_user_settings)
+
+    return current_user_settings

fractal_server/app/routes/auth/users.py

@@ -19,11 +19,15 @@ from ...db import get_async_db
 from ...schemas.user import UserRead
 from ...schemas.user import UserUpdate
 from ...schemas.user import UserUpdateWithNewGroupIds
+from ..aux.validate_user_settings import verify_user_has_settings
 from ._aux_auth import _get_single_user_with_group_ids
 from fractal_server.app.models import LinkUserGroup
 from fractal_server.app.models import UserGroup
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models import UserSettings
 from fractal_server.app.routes.auth._aux_auth import _user_or_404
+from fractal_server.app.schemas import UserSettingsRead
+from fractal_server.app.schemas import UserSettingsUpdate
 from fractal_server.app.security import get_user_manager
 from fractal_server.app.security import UserManager
 from fractal_server.logger import set_logger
@@ -196,3 +200,41 @@ async def list_users(
         )
 
     return user_list
+
+
+@router_users.get(
+    "/users/{user_id}/settings/", response_model=UserSettingsRead
+)
+async def get_user_settings(
+    user_id: int,
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> UserSettingsRead:
+
+    user = await _user_or_404(user_id=user_id, db=db)
+    verify_user_has_settings(user)
+    user_settings = await db.get(UserSettings, user.user_settings_id)
+    return user_settings
+
+
+@router_users.patch(
+    "/users/{user_id}/settings/", response_model=UserSettingsRead
+)
+async def patch_user_settings(
+    user_id: int,
+    settings_update: UserSettingsUpdate,
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> UserSettingsRead:
+    user = await _user_or_404(user_id=user_id, db=db)
+    verify_user_has_settings(user)
+    user_settings = await db.get(UserSettings, user.user_settings_id)
+
+    for k, v in settings_update.dict(exclude_unset=True).items():
+        setattr(user_settings, k, v)
+
+    db.add(user_settings)
+    await db.commit()
+    await db.refresh(user_settings)
+
+    return user_settings