fractal-server 2.6.4__py3-none-any.whl → 2.7.0__py3-none-any.whl

fractal_server/app/routes/auth/group.py

@@ -4,6 +4,7 @@ Definition of `/auth/group/` routes
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
+from fastapi import Response
 from fastapi import status
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -12,14 +13,19 @@ from sqlmodel import func
 from sqlmodel import select
 
 from . import current_active_superuser
-from ...db import get_async_db
-from ...schemas.user_group import UserGroupCreate
-from ...schemas.user_group import UserGroupRead
-from ...schemas.user_group import UserGroupUpdate
-from ._aux_auth import _get_single_group_with_user_ids
+from ._aux_auth import _get_single_usergroup_with_user_ids
+from ._aux_auth import _usergroup_or_404
+from fractal_server.app.db import get_async_db
 from fractal_server.app.models import LinkUserGroup
 from fractal_server.app.models import UserGroup
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models import UserSettings
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.schemas.user_group import UserGroupCreate
+from fractal_server.app.schemas.user_group import UserGroupRead
+from fractal_server.app.schemas.user_group import UserGroupUpdate
+from fractal_server.app.schemas.user_settings import UserSettingsUpdate
+from fractal_server.app.security import FRACTAL_DEFAULT_GROUP_NAME
 from fractal_server.logger import set_logger
 
 logger = set_logger(__name__)
@@ -70,7 +76,7 @@ async def get_single_user_group(
     user: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
 ) -> UserGroupRead:
-    group = await _get_single_group_with_user_ids(group_id=group_id, db=db)
+    group = await _get_single_usergroup_with_user_ids(group_id=group_id, db=db)
     return group
 
 
@@ -118,12 +124,7 @@ async def update_single_group(
     db: AsyncSession = Depends(get_async_db),
 ) -> UserGroupRead:
 
-    group = await db.get(UserGroup, group_id)
-    if group is None:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=f"UserGroup {group_id} not found.",
-        )
+    group = await _usergroup_or_404(group_id, db)
 
     # Check that all required users exist
     # Note: The reason for introducing `col` is as in
@@ -167,25 +168,74 @@ async def update_single_group(
         db.add(group)
         await db.commit()
 
-    updated_group = await _get_single_group_with_user_ids(
+    updated_group = await _get_single_usergroup_with_user_ids(
         group_id=group_id, db=db
     )
 
     return updated_group
 
 
-@router_group.delete(
-    "/group/{group_id}/", status_code=status.HTTP_405_METHOD_NOT_ALLOWED
-)
+@router_group.delete("/group/{group_id}/", status_code=204)
 async def delete_single_group(
     group_id: int,
     user: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
-) -> UserGroupRead:
-    raise HTTPException(
-        status_code=status.HTTP_405_METHOD_NOT_ALLOWED,
-        detail=(
-            "Deleting a user group is not allowed, as it may restrict "
-            "previously-granted access.",
-        ),
+) -> Response:
+
+    group = await _usergroup_or_404(group_id, db)
+
+    if group.name == FRACTAL_DEFAULT_GROUP_NAME:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot delete default UserGroup "
+                f"'{FRACTAL_DEFAULT_GROUP_NAME}'."
+            ),
+        )
+
+    # Cascade operations
+
+    res = await db.execute(
+        select(LinkUserGroup).where(LinkUserGroup.group_id == group_id)
     )
+    for link in res.scalars().all():
+        await db.delete(link)
+
+    res = await db.execute(
+        select(TaskGroupV2).where(TaskGroupV2.user_group_id == group_id)
+    )
+    for task_group in res.scalars().all():
+        task_group.user_group_id = None
+        db.add(task_group)
+
+    # Delete
+
+    await db.delete(group)
+    await db.commit()
+
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router_group.patch("/group/{group_id}/user-settings/", status_code=200)
+async def patch_user_settings_bulk(
+    group_id: int,
+    settings_update: UserSettingsUpdate,
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+):
+    await _usergroup_or_404(group_id, db)
+    res = await db.execute(
+        select(UserSettings)
+        .join(UserOAuth)
+        .where(LinkUserGroup.user_id == UserOAuth.id)
+        .where(LinkUserGroup.group_id == group_id)
+    )
+    settings_list = res.scalars().all()
+    update = settings_update.dict(exclude_unset=True)
+    for settings in settings_list:
+        for k, v in update.items():
+            setattr(settings, k, v)
+        db.add(settings)
+    await db.commit()
+
+    return Response(status_code=status.HTTP_200_OK)

fractal_server/app/routes/auth/router.py

@@ -2,7 +2,6 @@ from fastapi import APIRouter
 
 from .current_user import router_current_user
 from .group import router_group
-from .group_names import router_group_names
 from .login import router_login
 from .oauth import router_oauth
 from .register import router_register
@@ -13,7 +12,6 @@ router_auth = APIRouter()
 router_auth.include_router(router_register)
 router_auth.include_router(router_current_user)
 router_auth.include_router(router_login)
-router_auth.include_router(router_group_names)
 router_auth.include_router(router_users)
 router_auth.include_router(router_group)
 router_auth.include_router(router_oauth)

fractal_server/app/routes/auth/users.py

@@ -20,7 +20,7 @@ from ...schemas.user import UserRead
 from ...schemas.user import UserUpdate
 from ...schemas.user import UserUpdateWithNewGroupIds
 from ..aux.validate_user_settings import verify_user_has_settings
-from ._aux_auth import _get_single_user_with_group_ids
+from ._aux_auth import _get_single_user_with_groups
 from fractal_server.app.models import LinkUserGroup
 from fractal_server.app.models import UserGroup
 from fractal_server.app.models import UserOAuth
@@ -41,13 +41,14 @@ logger = set_logger(__name__)
 @router_users.get("/users/{user_id}/", response_model=UserRead)
 async def get_user(
     user_id: int,
-    group_ids: bool = True,
+    group_ids_names: bool = True,
     superuser: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
 ) -> UserRead:
     user = await _user_or_404(user_id, db)
-    if group_ids:
-        user = await _get_single_user_with_group_ids(user, db)
+    if group_ids_names:
+        user_with_groups = await _get_single_user_with_groups(user, db)
+        return user_with_groups
     return user
 
 
@@ -163,12 +164,12 @@ async def patch_user(
         # Nothing to do, just continue
         patched_user = user_to_patch
 
-    # Enrich user object with `group_ids` attribute
-    patched_user_with_group_ids = await _get_single_user_with_group_ids(
+    # Enrich user object with `group_ids_names` attribute
+    patched_user_with_groups = await _get_single_user_with_groups(
         patched_user, db
     )
 
-    return patched_user_with_group_ids
+    return patched_user_with_groups
 
 
 @router_users.get("/users/", response_model=list[UserRead])

fractal_server/app/runner/executors/slurm/ssh/executor.py

@@ -861,7 +861,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
 
         # Transfer archive
         t_0_put = time.perf_counter()
-        self.fractal_ssh.put(
+        self.fractal_ssh.send_file(
             local=tarfile_path_local,
             remote=tarfile_path_remote,
         )
@@ -1055,55 +1055,59 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         Arguments:
             jobid: ID of the SLURM job
         """
-
-        # Loop over all job_ids, and fetch future and job objects
-        futures: list[Future] = []
-        jobs: list[SlurmJob] = []
-        with self.jobs_lock:
-            for job_id in job_ids:
-                future, job = self.jobs.pop(job_id)
-                futures.append(future)
-                jobs.append(job)
-            if not self.jobs:
-                self.jobs_empty_cond.notify_all()
-
-        # Fetch subfolder from remote host
+        # Handle all uncaught exceptions in this broad try/except block
         try:
-            self._get_subfolder_sftp(jobs=jobs)
-        except NoValidConnectionsError as e:
-            logger.error("NoValidConnectionError")
-            logger.error(f"{str(e)=}")
-            logger.error(f"{e.errors=}")
-            for err in e.errors:
-                logger.error(f"{str(err)}")
-
-            raise e
-
-        # First round of checking whether all output files exist
-        missing_out_paths = []
-        for job in jobs:
-            for ind_out_path, out_path in enumerate(
-                job.output_pickle_files_local
-            ):
-                if not out_path.exists():
-                    missing_out_paths.append(out_path)
-        num_missing = len(missing_out_paths)
-        if num_missing > 0:
-            # Output pickle files may be missing e.g. because of some slow
-            # filesystem operation; wait some time before re-trying
-            settings = Inject(get_settings)
-            sleep_time = settings.FRACTAL_SLURM_ERROR_HANDLING_INTERVAL
             logger.info(
-                f"{num_missing} output pickle files are missing; "
-                f"sleep {sleep_time} seconds."
+                f"[FractalSlurmSSHExecutor._completion] START, for {job_ids=}."
             )
-            for missing_file in missing_out_paths:
-                logger.debug(f"Missing output pickle file: {missing_file}")
-            time.sleep(sleep_time)
 
-        # Handle all jobs
-        for ind_job, job_id in enumerate(job_ids):
+            # Loop over all job_ids, and fetch future and job objects
+            futures: list[Future] = []
+            jobs: list[SlurmJob] = []
+            with self.jobs_lock:
+                for job_id in job_ids:
+                    future, job = self.jobs.pop(job_id)
+                    futures.append(future)
+                    jobs.append(job)
+                if not self.jobs:
+                    self.jobs_empty_cond.notify_all()
+
+            # Fetch subfolder from remote host
             try:
+                self._get_subfolder_sftp(jobs=jobs)
+            except NoValidConnectionsError as e:
+                logger.error("NoValidConnectionError")
+                logger.error(f"{str(e)=}")
+                logger.error(f"{e.errors=}")
+                for err in e.errors:
+                    logger.error(f"{str(err)}")
+
+                raise e
+
+            # First round of checking whether all output files exist
+            missing_out_paths = []
+            for job in jobs:
+                for ind_out_path, out_path in enumerate(
+                    job.output_pickle_files_local
+                ):
+                    if not out_path.exists():
+                        missing_out_paths.append(out_path)
+            num_missing = len(missing_out_paths)
+            if num_missing > 0:
+                # Output pickle files may be missing e.g. because of some slow
+                # filesystem operation; wait some time before re-trying
+                settings = Inject(get_settings)
+                sleep_time = settings.FRACTAL_SLURM_ERROR_HANDLING_INTERVAL
+                logger.info(
+                    f"{num_missing} output pickle files are missing; "
+                    f"sleep {sleep_time} seconds."
+                )
+                for missing_file in missing_out_paths:
+                    logger.debug(f"Missing output pickle file: {missing_file}")
+                time.sleep(sleep_time)
+
+            # Handle all jobs
+            for ind_job, job_id in enumerate(job_ids):
                 # Retrieve job and future objects
                 job = jobs[ind_job]
                 future = futures[ind_job]
@@ -1128,6 +1132,11 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                                 remaining_futures=remaining_futures,
                                 remaining_job_ids=remaining_job_ids,
                             )
+                            logger.info(
+                                "[FractalSlurmSSHExecutor._completion] END, "
+                                f"for {job_ids=}, with JobExecutionError due "
+                                f"to missing {out_path.as_posix()}."
+                            )
                             return
                         except InvalidStateError:
                             logger.warning(
@@ -1141,6 +1150,12 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                                 remaining_futures=remaining_futures,
                                 remaining_job_ids=remaining_job_ids,
                             )
+                            logger.info(
+                                "[FractalSlurmSSHExecutor._completion] END, "
+                                f"for {job_ids=}, with JobExecutionError/"
+                                "InvalidStateError due to "
+                                f"missing {out_path.as_posix()}."
+                            )
                             return
 
                     # Read the task output
@@ -1217,16 +1232,22 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                 else:
                     future.set_result(outputs)
 
-            except Exception as e:
+        except Exception as e:
+            logger.warning(
+                "[FractalSlurmSSHExecutor._completion] "
+                f"An exception took place: {str(e)}."
+            )
+            for future in futures:
                 try:
+                    logger.info(f"Set exception for {future=}")
                     future.set_exception(e)
-                    return
                 except InvalidStateError:
-                    logger.warning(
-                        f"Future {future} (SLURM job ID: {job_id}) was already"
-                        " cancelled, exit from"
-                        " FractalSlurmSSHExecutor._completion."
-                    )
+                    logger.info(f"Future {future} was already cancelled.")
+            logger.info(
+                f"[FractalSlurmSSHExecutor._completion] END, for {job_ids=}, "
+                "from within exception handling."
+            )
+            return
 
     def _get_subfolder_sftp(self, jobs: list[SlurmJob]) -> None:
         """
@@ -1255,16 +1276,9 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             self.workflow_dir_remote / f"{subfolder_name}.tar.gz"
         ).as_posix()
 
-        # Remove local tarfile - FIXME SSH: is this needed?
-        logger.warning(f"In principle I just removed {tarfile_path_local}")
-        logger.warning(f"{Path(tarfile_path_local).exists()=}")
-
-        # Remove remote tarfile - FIXME SSH: is this needed?
-        # rm_command = f"rm {tarfile_path_remote}"
-        # _run_command_over_ssh(cmd=rm_command, fractal_ssh=self.fractal_ssh)
-        logger.warning(f"Unlink {tarfile_path_remote=} - START")
-        self.fractal_ssh.sftp().unlink(tarfile_path_remote)
-        logger.warning(f"Unlink {tarfile_path_remote=} - STOP")
+        # Remove remote tarfile
+        rm_command = f"rm {tarfile_path_remote}"
+        self.fractal_ssh.run_command(cmd=rm_command)
 
         # Create remote tarfile
         tar_command = (
@@ -1278,7 +1292,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
 
         # Fetch tarfile
         t_0_get = time.perf_counter()
-        self.fractal_ssh.get(
+        self.fractal_ssh.fetch_file(
             remote=tarfile_path_remote,
             local=tarfile_path_local,
         )
@@ -1291,6 +1305,11 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         # Extract tarfile locally
         extract_archive(Path(tarfile_path_local))
 
+        # Remove local tarfile
+        if Path(tarfile_path_local).exists():
+            logger.warning(f"Remove existing file {tarfile_path_local}.")
+            Path(tarfile_path_local).unlink()
+
         t_1 = time.perf_counter()
         logger.info("[_get_subfolder_sftp] End - " f"elapsed: {t_1-t_0:.3f} s")
 

fractal_server/app/runner/v2/__init__.py

@@ -177,11 +177,13 @@ async def submit_workflow(
             return
 
         try:
-
             # Create WORKFLOW_DIR_LOCAL
-            original_umask = os.umask(0)
-            WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
-            os.umask(original_umask)
+            if FRACTAL_RUNNER_BACKEND == "slurm":
+                original_umask = os.umask(0)
+                WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
+                os.umask(original_umask)
+            else:
+                WORKFLOW_DIR_LOCAL.mkdir(parents=True)
 
             # Define and create WORKFLOW_DIR_REMOTE
             if FRACTAL_RUNNER_BACKEND == "local":
@@ -214,15 +216,19 @@ async def submit_workflow(
                     order=order,
                     task_name=task_name,
                 )
-                original_umask = os.umask(0)
-                (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir(mode=0o755)
-                os.umask(original_umask)
                 if FRACTAL_RUNNER_BACKEND == "slurm":
+                    # Create local subfolder (with 755) and remote one
+                    # (via `sudo -u`)
+                    original_umask = os.umask(0)
+                    (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir(mode=0o755)
+                    os.umask(original_umask)
                     _mkdir_as_user(
                         folder=str(WORKFLOW_DIR_REMOTE / subfolder_name),
                         user=slurm_user,
                     )
                 else:
+                    # Create local subfolder (with standard permission set)
+                    (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir()
                     logger.info("Skip remote-subfolder creation")
         except Exception as e:
             error_type = type(e).__name__

fractal_server/app/runner/v2/task_interface.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from pydantic import BaseModel
+from pydantic import Extra
 from pydantic import Field
 from pydantic import validator
 
@@ -9,9 +10,7 @@ from fractal_server.images import Filters
 from fractal_server.urls import normalize_url
 
 
-class TaskOutput(BaseModel):
-    class Config:
-        extra = "forbid"
+class TaskOutput(BaseModel, extra=Extra.forbid):
 
     image_list_updates: list[SingleImageTaskOutput] = Field(
         default_factory=list
@@ -43,9 +42,7 @@ class TaskOutput(BaseModel):
         return [normalize_url(zarr_url) for zarr_url in v]
 
 
-class InitArgsModel(BaseModel):
-    class Config:
-        extra = "forbid"
+class InitArgsModel(BaseModel, extra=Extra.forbid):
 
     zarr_url: str
     init_args: dict[str, Any] = Field(default_factory=dict)
@@ -55,8 +52,6 @@ class InitArgsModel(BaseModel):
         return normalize_url(v)
 
 
-class InitTaskOutput(BaseModel):
-    class Config:
-        extra = "forbid"
+class InitTaskOutput(BaseModel, extra=Extra.forbid):
 
     parallelization_list: list[InitArgsModel] = Field(default_factory=list)

fractal_server/app/schemas/user.py

@@ -41,8 +41,7 @@ class UserRead(schemas.BaseUser[int]):
     """
 
     username: Optional[str]
-    group_names: Optional[list[str]] = None
-    group_ids: Optional[list[int]] = None
+    group_ids_names: Optional[list[tuple[int, str]]] = None
     oauth_accounts: list[OAuthAccountRead]
 
 

fractal_server/app/schemas/v2/__init__.py

@@ -20,20 +20,27 @@ from .project import ProjectUpdateV2  # noqa F401
 from .task import TaskCreateV2  # noqa F401
 from .task import TaskExportV2  # noqa F401
 from .task import TaskImportV2  # noqa F401
+from .task import TaskImportV2Legacy  # noqa F401
 from .task import TaskReadV2  # noqa F401
 from .task import TaskUpdateV2  # noqa F401
 from .task_collection import CollectionStateReadV2  # noqa F401
 from .task_collection import CollectionStatusV2  # noqa F401
 from .task_collection import TaskCollectCustomV2  # noqa F401
 from .task_collection import TaskCollectPipV2  # noqa F401
+from .task_group import TaskGroupCreateV2  # noqa F401
+from .task_group import TaskGroupReadV2  # noqa F401
+from .task_group import TaskGroupUpdateV2  # noqa F401
+from .task_group import TaskGroupV2OriginEnum  # noqa F401
 from .workflow import WorkflowCreateV2  # noqa F401
 from .workflow import WorkflowExportV2  # noqa F401
 from .workflow import WorkflowImportV2  # noqa F401
 from .workflow import WorkflowReadV2  # noqa F401
+from .workflow import WorkflowReadV2WithWarnings  # noqa F401
 from .workflow import WorkflowUpdateV2  # noqa F401
 from .workflowtask import WorkflowTaskCreateV2  # noqa F401
 from .workflowtask import WorkflowTaskExportV2  # noqa F401
 from .workflowtask import WorkflowTaskImportV2  # noqa F401
 from .workflowtask import WorkflowTaskReadV2  # noqa F401
+from .workflowtask import WorkflowTaskReadV2WithWarning  # noqa F401
 from .workflowtask import WorkflowTaskStatusTypeV2  # noqa F401
 from .workflowtask import WorkflowTaskUpdateV2  # noqa F401

fractal_server/app/schemas/v2/dataset.py

@@ -66,9 +66,7 @@ class DatasetReadV2(BaseModel):
     )
 
 
-class DatasetUpdateV2(BaseModel):
-    class Config:
-        extra = "forbid"
+class DatasetUpdateV2(BaseModel, extra=Extra.forbid):
 
     name: Optional[str]
     zarr_dir: Optional[str]
@@ -84,7 +82,7 @@ class DatasetUpdateV2(BaseModel):
     _name = validator("name", allow_reuse=True)(valstr("name"))
 
 
-class DatasetImportV2(BaseModel):
+class DatasetImportV2(BaseModel, extra=Extra.forbid):
     """
     Class for `Dataset` import.
 
@@ -95,9 +93,6 @@ class DatasetImportV2(BaseModel):
         filters:
     """
 
-    class Config:
-        extra = "forbid"
-
     name: str
     zarr_dir: str
     images: list[SingleImage] = Field(default_factory=[])

fractal_server/app/schemas/v2/dumps.py

@@ -30,8 +30,7 @@ class TaskDumpV2(BaseModel):
 
     command_non_parallel: Optional[str]
     command_parallel: Optional[str]
-    source: str
-    owner: Optional[str]
+    source: Optional[str] = None
     version: Optional[str]
 
     input_types: dict[str, bool]

fractal_server/app/schemas/v2/job.py

@@ -109,6 +109,6 @@ class JobReadV2(BaseModel):
     )
 
 
-class JobUpdateV2(BaseModel):
+class JobUpdateV2(BaseModel, extra=Extra.forbid):
 
     status: JobStatusTypeV2

fractal_server/app/schemas/v2/manifest.py

@@ -7,6 +7,8 @@ from pydantic import HttpUrl
 from pydantic import root_validator
 from pydantic import validator
 
+from .._validators import valstr
+
 
 class TaskManifestV2(BaseModel):
     """
@@ -50,6 +52,10 @@ class TaskManifestV2(BaseModel):
     docs_info: Optional[str] = None
     docs_link: Optional[HttpUrl] = None
 
+    category: Optional[str] = None
+    modality: Optional[str] = None
+    tags: list[str] = Field(default_factory=list)
+
     @root_validator
     def validate_executable_args_meta(cls, values):
 
@@ -128,7 +134,8 @@ class ManifestV2(BaseModel):
     manifest_version: str
     task_list: list[TaskManifestV2]
     has_args_schemas: bool = False
-    args_schema_version: Optional[str]
+    args_schema_version: Optional[str] = None
+    authors: Optional[str] = None
 
     @root_validator()
     def _check_args_schemas_are_present(cls, values):
@@ -152,8 +159,25 @@ class ManifestV2(BaseModel):
                         )
         return values
 
+    @root_validator()
+    def _unique_task_names(cls, values):
+        task_list = values["task_list"]
+        task_list_names = [t.name for t in task_list]
+        if len(set(task_list_names)) != len(task_list_names):
+            raise ValueError(
+                (
+                    "Task names in manifest must be unique.\n",
+                    f"Given: {task_list_names}.",
+                )
+            )
+        return values
+
     @validator("manifest_version")
     def manifest_version_2(cls, value):
         if value != "2":
             raise ValueError(f"Wrong manifest version (given {value})")
         return value
+
+    _authors = validator("authors", allow_reuse=True)(
+        valstr("authors", accept_none=True)
+    )

fractal_server/app/schemas/v2/project.py

@@ -27,7 +27,7 @@ class ProjectReadV2(BaseModel):
     )
 
 
-class ProjectUpdateV2(BaseModel):
+class ProjectUpdateV2(BaseModel, extra=Extra.forbid):
 
     name: Optional[str]
     # Validators