fractal-server 2.14.0a2__py3-none-any.whl → 2.14.0a4__py3-none-any.whl

fractal_server/app/routes/api/v2/status_legacy.py

@@ -0,0 +1,168 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import status
+
+from .....logger import set_logger
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v2 import JobV2
+from ....schemas.v2.status import StatusReadV2
+from ....schemas.v2.workflowtask import WorkflowTaskStatusTypeV2
+from ._aux_functions import _get_dataset_check_owner
+from ._aux_functions import _get_submitted_jobs_statement
+from ._aux_functions import _get_workflow_check_owner
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.routes.auth import current_active_user
+
+router = APIRouter()
+
+logger = set_logger(__name__)
+
+
+@router.get(
+    "/project/{project_id}/status-legacy/",
+    response_model=StatusReadV2,
+)
+async def get_workflowtask_status(
+    project_id: int,
+    dataset_id: int,
+    workflow_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[StatusReadV2]:
+    """
+    Extract the status of all `WorkflowTaskV2` of a given `WorkflowV2` that ran
+    on a given `DatasetV2`.
+
+    *NOTE*: the current endpoint is not guaranteed to provide consistent
+    results if the workflow task list is modified in a non-trivial way
+    (that is, by adding intermediate tasks, removing tasks, or changing their
+    order). See fractal-server GitHub issues: 793, 1083.
+    """
+    # Get the dataset DB entry
+    output = await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+    dataset = output["dataset"]
+
+    # Get the workflow DB entry
+    workflow = await _get_workflow_check_owner(
+        project_id=project_id,
+        workflow_id=workflow_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    # Check whether there exists a submitted job associated to this
+    # workflow/dataset pair. If it does exist, it will be used later.
+    # If there are multiple jobs, raise an error.
+    stm = _get_submitted_jobs_statement()
+    stm = stm.where(JobV2.dataset_id == dataset_id)
+    stm = stm.where(JobV2.workflow_id == workflow_id)
+    res = await db.execute(stm)
+    running_jobs = res.scalars().all()
+    if len(running_jobs) == 0:
+        running_job = None
+    elif len(running_jobs) == 1:
+        running_job = running_jobs[0]
+    else:
+        string_ids = str([job.id for job in running_jobs])[1:-1]
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Cannot get WorkflowTaskV2 statuses as DatasetV2 {dataset.id}"
+                f" is linked to multiple active jobs: {string_ids}."
+            ),
+        )
+
+    # Initialize empty dictionary for WorkflowTaskV2 status
+    workflow_tasks_status_dict: dict = {}
+
+    # Lowest priority: read status from DB, which corresponds to jobs that are
+    # not running
+    history = dataset.history
+    for history_item in history:
+        wftask_id = history_item["workflowtask"]["id"]
+        wftask_status = history_item["status"]
+        workflow_tasks_status_dict[wftask_id] = wftask_status
+
+    if running_job is None:
+        # If no job is running, the chronological-last history item is also the
+        # positional-last workflow task to be included in the response.
+        if len(history) > 0:
+            last_valid_wftask_id = history[-1]["workflowtask"]["id"]
+        else:
+            last_valid_wftask_id = None
+    else:
+        # If a job is running, then gather more up-to-date information
+
+        # Mid priority: Set all WorkflowTask's that are part of the running job
+        # as "submitted"
+        start = running_job.first_task_index
+        end = running_job.last_task_index + 1
+
+        running_job_wftasks = workflow.task_list[start:end]
+        running_job_statuses = [
+            workflow_tasks_status_dict.get(wft.id, None)
+            for wft in running_job_wftasks
+        ]
+        try:
+            first_submitted_index = running_job_statuses.index(
+                WorkflowTaskStatusTypeV2.SUBMITTED
+            )
+        except ValueError:
+            logger.warning(
+                f"Job {running_job.id} is submitted but its task list does "
+                f"not contain a {WorkflowTaskStatusTypeV2.SUBMITTED} task."
+            )
+            first_submitted_index = 0
+
+        for wftask in running_job_wftasks[first_submitted_index:]:
+            workflow_tasks_status_dict[
+                wftask.id
+            ] = WorkflowTaskStatusTypeV2.SUBMITTED
+
+        # The last workflow task that is included in the submitted job is also
+        # the positional-last workflow task to be included in the response.
+        try:
+            last_valid_wftask_id = workflow.task_list[end - 1].id
+        except IndexError as e:
+            logger.warning(
+                f"Handled IndexError in `get_workflowtask_status` ({str(e)})."
+            )
+            logger.warning(
+                "Additional information: "
+                f"{running_job.first_task_index=}; "
+                f"{running_job.last_task_index=}; "
+                f"{len(workflow.task_list)=}; "
+                f"{dataset_id=}; "
+                f"{workflow_id=}."
+            )
+            last_valid_wftask_id = None
+            logger.warning(f"Now setting {last_valid_wftask_id=}.")
+
+    # Based on previously-gathered information, clean up the response body
+    clean_workflow_tasks_status_dict = {}
+    for wf_task in workflow.task_list:
+        wf_task_status = workflow_tasks_status_dict.get(wf_task.id)
+        if wf_task_status is None:
+            # If a wftask ID was not found, ignore it and continue
+            continue
+        clean_workflow_tasks_status_dict[str(wf_task.id)] = wf_task_status
+        if wf_task_status == WorkflowTaskStatusTypeV2.FAILED:
+            # Starting from the beginning of `workflow.task_list`, stop the
+            # first time that you hit a failed job
+            break
+        if wf_task.id == last_valid_wftask_id:
+            # Starting from the beginning of `workflow.task_list`, stop the
+            # first time that you hit `last_valid_wftask_id``
+            break
+
+    response_body = StatusReadV2(status=clean_workflow_tasks_status_dict)
+    return response_body

fractal_server/app/routes/api/v2/workflow.py

@@ -7,13 +7,10 @@ from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
 from pydantic import BaseModel
-from sqlmodel import delete
 from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
-from ....models.v2 import HistoryItemV2
-from ....models.v2 import ImageStatus
 from ....models.v2 import JobV2
 from ....models.v2 import ProjectV2
 from ....models.v2 import WorkflowV2
@@ -228,26 +225,14 @@ async def delete_workflow(
             ),
         )
 
-    # Cascade operation: set foreign-keys to null for jobs and history items
-    # which are in relationship with the current workflow.
+    # Cascade operation: set foreign-keys to null for jobs which are in
+    # relationship with the current workflow.
     stm = select(JobV2).where(JobV2.workflow_id == workflow_id)
     res = await db.execute(stm)
     jobs = res.scalars().all()
     for job in jobs:
         job.workflow_id = None
 
-    wft_ids = [wft.id for wft in workflow.task_list]
-    stm = select(HistoryItemV2).where(
-        HistoryItemV2.workflowtask_id.in_(wft_ids)
-    )
-    res = await db.execute(stm)
-    history_items = res.scalars().all()
-    for history_item in history_items:
-        history_item.workflowtask_id = None
-
-    stm = delete(ImageStatus).where(ImageStatus.workflowtask_id.in_(wft_ids))
-    await db.execute(stm)
-
     # Delete workflow
     await db.delete(workflow)
     await db.commit()

fractal_server/app/routes/api/v2/workflowtask.py

@@ -6,8 +6,6 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
-from sqlmodel import delete
-from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
@@ -17,8 +15,6 @@ from ._aux_functions import _workflow_insert_task
 from ._aux_functions_tasks import _check_type_filters_compatibility
 from ._aux_functions_tasks import _get_task_read_access
 from fractal_server.app.models import UserOAuth
-from fractal_server.app.models.v2 import HistoryItemV2
-from fractal_server.app.models.v2 import ImageStatus
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.schemas.v2 import WorkflowTaskCreateV2
@@ -39,80 +35,73 @@ async def replace_workflowtask(
     workflow_id: int,
     workflow_task_id: int,
     task_id: int,
-    replace: Optional[WorkflowTaskReplaceV2] = None,
+    replace: WorkflowTaskReplaceV2,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> WorkflowTaskReadV2:
 
-    old_workflow_task, workflow = await _get_workflow_task_check_owner(
+    # Get objects from database
+    old_wftask, workflow = await _get_workflow_task_check_owner(
         project_id=project_id,
         workflow_id=workflow_id,
         workflow_task_id=workflow_task_id,
         user_id=user.id,
         db=db,
     )
-
     new_task = await _get_task_read_access(
-        task_id=task_id, user_id=user.id, db=db, require_active=True
+        task_id=task_id,
+        user_id=user.id,
+        db=db,
+        require_active=True,
     )
 
-    if new_task.type != old_workflow_task.task.type:
+    # Preliminary checks
+    if old_wftask.task_type != new_task.type:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=(
-                f"Cannot replace a Task '{old_workflow_task.task.type}' with a"
-                f"  Task '{new_task.type}'."
+                "Cannot change task type from "
+                f"{old_wftask.task_type} to {new_task.type}."
             ),
         )
-
+    if replace.args_non_parallel is not None and new_task.type == "parallel":
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Cannot set 'args_non_parallel' for parallel task.",
+        )
+    if replace.args_parallel is not None and new_task.type == "non_parallel":
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Cannot set 'args_parallel' for non-parallel task.",
+        )
     _check_type_filters_compatibility(
         task_input_types=new_task.input_types,
-        wftask_type_filters=old_workflow_task.type_filters,
+        wftask_type_filters=old_wftask.type_filters,
     )
 
-    _args_non_parallel = old_workflow_task.args_non_parallel
-    _args_parallel = old_workflow_task.args_parallel
-    if replace is not None:
-        if replace.args_non_parallel is not None:
-            if new_task.type == "parallel":
-                raise HTTPException(
-                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                    detail=(
-                        "Cannot set 'args_non_parallel' "
-                        "when Task is 'parallel'."
-                    ),
-                )
-            else:
-                _args_non_parallel = replace.args_non_parallel
-
-        if replace.args_parallel is not None:
-            if new_task.type == "non_parallel":
-                raise HTTPException(
-                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                    detail=(
-                        "Cannot set 'args_parallel' "
-                        "when Task is 'non_parallel'."
-                    ),
-                )
-            else:
-                _args_parallel = replace.args_parallel
+    # Task arguments
+    if replace.args_non_parallel is None:
+        _args_non_parallel = old_wftask.args_non_parallel
+    else:
+        _args_non_parallel = replace.args_non_parallel
+    if replace.args_parallel is None:
+        _args_parallel = old_wftask.args_parallel
+    else:
+        _args_parallel = replace.args_parallel
 
     # If user's changes to `meta_non_parallel` are compatible with new task,
     # keep them; else, get `meta_non_parallel` from new task
     if (
-        old_workflow_task.meta_non_parallel
-        != old_workflow_task.task.meta_non_parallel
-    ) and (
-        old_workflow_task.task.meta_non_parallel == new_task.meta_non_parallel
-    ):
-        _meta_non_parallel = old_workflow_task.meta_non_parallel
+        old_wftask.meta_non_parallel != old_wftask.task.meta_non_parallel
+    ) and (old_wftask.task.meta_non_parallel == new_task.meta_non_parallel):
+        _meta_non_parallel = old_wftask.meta_non_parallel
     else:
         _meta_non_parallel = new_task.meta_non_parallel
     # Same for `meta_parallel`
-    if (
-        old_workflow_task.meta_parallel != old_workflow_task.task.meta_parallel
-    ) and (old_workflow_task.task.meta_parallel == new_task.meta_parallel):
-        _meta_parallel = old_workflow_task.meta_parallel
+    if (old_wftask.meta_parallel != old_wftask.task.meta_parallel) and (
+        old_wftask.task.meta_parallel == new_task.meta_parallel
+    ):
+        _meta_parallel = old_wftask.meta_parallel
     else:
         _meta_parallel = new_task.meta_parallel
 
@@ -121,7 +110,7 @@ async def replace_workflowtask(
         task_type=new_task.type,
         task=new_task,
         # old-task values
-        type_filters=old_workflow_task.type_filters,
+        type_filters=old_wftask.type_filters,
         # possibly new values
         args_non_parallel=_args_non_parallel,
         args_parallel=_args_parallel,
@@ -129,8 +118,8 @@ async def replace_workflowtask(
         meta_parallel=_meta_parallel,
     )
 
-    workflow_task_order = old_workflow_task.order
-    workflow.task_list.remove(old_workflow_task)
+    workflow_task_order = old_wftask.order
+    workflow.task_list.remove(old_wftask)
     workflow.task_list.insert(workflow_task_order, new_workflow_task)
     await db.commit()
     await db.refresh(new_workflow_task)
@@ -205,8 +194,6 @@ async def create_workflowtask(
         db=db,
     )
 
-    await db.close()
-
     return wftask_db
 
 
@@ -337,23 +324,6 @@ async def delete_workflowtask(
         db=db,
     )
 
-    # Cascade operations:
-    # * set foreign-keys to null for history items which are in relationship
-    #   with the current workflowtask;
-    # * delete ImageStatus in relationship with the current workflowtask.
-    stm = select(HistoryItemV2).where(
-        HistoryItemV2.workflowtask_id == db_workflow_task.id
-    )
-    res = await db.execute(stm)
-    history_items = res.scalars().all()
-    for history_item in history_items:
-        history_item.workflowtask_id = None
-
-    stm = delete(ImageStatus).where(
-        ImageStatus.workflowtask_id == db_workflow_task.id
-    )
-    await db.execute(stm)
-
     # Delete WorkflowTask
     await db.delete(db_workflow_task)
     await db.commit()

fractal_server/app/routes/auth/oauth.py

@@ -27,20 +27,22 @@ for client_config in settings.OAUTH_CLIENTS_CONFIG:
         from httpx_oauth.clients.google import GoogleOAuth2
 
         client = GoogleOAuth2(
-            client_config.CLIENT_ID, client_config.CLIENT_SECRET
+            client_config.CLIENT_ID,
+            client_config.CLIENT_SECRET.get_secret_value(),
         )
     elif client_name == "github":
         from httpx_oauth.clients.github import GitHubOAuth2
 
         client = GitHubOAuth2(
-            client_config.CLIENT_ID, client_config.CLIENT_SECRET
+            client_config.CLIENT_ID,
+            client_config.CLIENT_SECRET.get_secret_value(),
         )
     else:
         from httpx_oauth.clients.openid import OpenID
 
         client = OpenID(
             client_config.CLIENT_ID,
-            client_config.CLIENT_SECRET,
+            client_config.CLIENT_SECRET.get_secret_value(),
             client_config.OIDC_CONFIGURATION_ENDPOINT,
         )
 

fractal_server/app/runner/executors/base_runner.py

@@ -106,7 +106,8 @@ class BaseRunner(object):
                 )
             if _COMPONENT_KEY_ not in single_kwargs.keys():
                 raise ValueError(
-                    f"No '{_COMPONENT_KEY_}' key in in {list(single_kwargs.keys())}"
+                    f"No '{_COMPONENT_KEY_}' key "
+                    f"in {list(single_kwargs.keys())}"
                 )
         if not in_compound_task:
             zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]

fractal_server/app/runner/executors/local/_submit_setup.py

@@ -21,18 +21,10 @@ def _local_submit_setup(
     FIXME
 
     Arguments:
-        wftask:
-            WorkflowTask for which the configuration is to be assembled
-        workflow_dir_local:
-            Not used in this function.
-        workflow_dir_remote:
-            Not used in this function.
-
-    Returns:
-        submit_setup_dict:
-            A dictionary that will be passed on to
-            `FractalThreadPoolExecutor.submit` and
-            `FractalThreadPoolExecutor.map`, so as to set extra options.
+        wftask: WorkflowTask for which the configuration is to be assembled
+        root_dir_local:
+        root_dir_rempote: Not used in this function.
+        which_type: Whether it is a parallel or non-parallel task.
     """
 
     local_backend_config = get_local_backend_config(
@@ -43,7 +35,7 @@ def _local_submit_setup(
     # Get TaskFiles object
     task_files = TaskFiles(
         root_dir_local=root_dir_local,
-        root_dir_remote=root_dir_remote,
+        root_dir_remote=root_dir_local,
         task_order=wftask.order,
         task_name=wftask.task.name,
     )

fractal_server/app/runner/executors/local/runner.py

@@ -1,3 +1,4 @@
+from concurrent.futures import Future
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Any
@@ -5,15 +6,12 @@ from typing import Optional
 
 from ._local_config import get_default_local_backend_config
 from ._local_config import LocalBackendConfig
-from fractal_server.app.history import HistoryItemImageStatus
-from fractal_server.app.history import update_all_images
-from fractal_server.app.history import update_single_image
-from fractal_server.app.history import update_single_image_logfile
 from fractal_server.app.runner.components import _COMPONENT_KEY_
 from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.task_files import TaskFiles
 from fractal_server.logger import set_logger
 
+
 logger = set_logger(__name__)
 
 
@@ -51,10 +49,8 @@ class LocalRunner(BaseRunner):
         self,
         func: callable,
         parameters: dict[str, Any],
-        history_item_id: int,
         task_files: TaskFiles,
-        in_compound_task: bool = False,
-        **kwargs,
+        local_backend_config: Optional[LocalBackendConfig] = None,
     ) -> tuple[Any, Exception]:
         logger.debug("[submit] START")
 
@@ -68,27 +64,17 @@ class LocalRunner(BaseRunner):
         self.validate_submit_parameters(parameters)
         workdir_local = current_task_files.wftask_subfolder_local
         workdir_local.mkdir()
+
         # SUBMISSION PHASE
         future = self.executor.submit(func, parameters=parameters)
 
         # RETRIEVAL PHASE
         try:
             result = future.result()
-            if not in_compound_task:
-                update_all_images(
-                    history_item_id=history_item_id,
-                    status=HistoryItemImageStatus.DONE,
-                    logfile=current_task_files.log_file_local,
-                )
             logger.debug(f"[submit] END {result=}")
             return result, None
         except Exception as e:
             exception = e
-            update_all_images(
-                history_item_id=history_item_id,
-                status=HistoryItemImageStatus.FAILED,
-                logfile=current_task_files.log_file_local,
-            )
             logger.debug(f"[submit] END {exception=}")
             return None, exception
 
@@ -96,11 +82,9 @@ class LocalRunner(BaseRunner):
         self,
         func: callable,
         list_parameters: list[dict],
-        history_item_id: int,
         task_files: TaskFiles,
         in_compound_task: bool = False,
         local_backend_config: Optional[LocalBackendConfig] = None,
-        **kwargs,
     ):
         logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
 
@@ -126,13 +110,12 @@ class LocalRunner(BaseRunner):
         original_task_files = task_files
 
         # Execute tasks, in chunks of size `parallel_tasks_per_job`
-        results = {}
-        exceptions = {}
+        results: dict[int, Any] = {}
+        exceptions: dict[int, BaseException] = {}
         for ind_chunk in range(0, n_elements, parallel_tasks_per_job):
             list_parameters_chunk = list_parameters[
                 ind_chunk : ind_chunk + parallel_tasks_per_job
             ]
-            from concurrent.futures import Future
 
             active_futures: dict[int, Future] = {}
             active_task_files: dict[int, TaskFiles] = {}
@@ -156,45 +139,17 @@ class LocalRunner(BaseRunner):
                 ]
                 for positional_index, fut in finished_futures:
                     active_futures.pop(positional_index)
-                    current_task_files = active_task_files.pop(
-                        positional_index
-                    )
+                    # current_task_files = active_task_files.pop(
+                    #     positional_index
+                    # )
                     zarr_url = list_parameters[positional_index]["zarr_url"]
-                    if not in_compound_task:
-                        update_single_image_logfile(
-                            history_item_id=history_item_id,
-                            zarr_url=zarr_url,
-                            logfile=current_task_files.log_file_local,
-                        )
                     try:
                         results[positional_index] = fut.result()
                         print(f"Mark {zarr_url=} as done, {kwargs}")
-                        if not in_compound_task:
-                            update_single_image(
-                                history_item_id=history_item_id,
-                                zarr_url=zarr_url,
-                                status=HistoryItemImageStatus.DONE,
-                            )
                     except Exception as e:
                         print(f"Mark {zarr_url=} as failed, {kwargs} - {e}")
                         exceptions[positional_index] = e
-                        if not in_compound_task:
-                            update_single_image(
-                                history_item_id=history_item_id,
-                                zarr_url=zarr_url,
-                                status=HistoryItemImageStatus.FAILED,
-                            )
-        if in_compound_task:
-            if exceptions == {}:
-                update_all_images(
-                    history_item_id=history_item_id,
-                    status=HistoryItemImageStatus.DONE,
-                )
-            else:
-                update_all_images(
-                    history_item_id=history_item_id,
-                    status=HistoryItemImageStatus.FAILED,
-                )
+
         logger.debug(f"[multisubmit] END, {results=}, {exceptions=}")
 
         return results, exceptions

fractal_server/app/runner/executors/slurm_common/_slurm_config.py

@@ -213,7 +213,7 @@ class SlurmConfig(BaseModel):
     expected file content are defined in
     [`SlurmConfigFile`](./#fractal_server.app.runner._slurm._slurm_config.SlurmConfigFile)).
 
-    Part of the attributes map directly to some of the SLURM attribues (see
+    Part of the attributes map directly to some of the SLURM attributes (see
     https://slurm.schedmd.com/sbatch.html), e.g. `partition`. Other attributes
     are metaparameters which are needed in fractal-server to combine multiple
     tasks in the same SLURM job (e.g. `parallel_tasks_per_job` or

fractal_server/app/runner/executors/slurm_common/get_slurm_config.py

@@ -19,7 +19,7 @@ def get_slurm_config(
     Prepare a `SlurmConfig` configuration object
 
     The argument `which_type` determines whether we use `wftask.meta_parallel`
-    or `wftask.meta_non_parallel`. In the following descritpion, let us assume
+    or `wftask.meta_non_parallel`. In the following description, let us assume
     that `which_type="parallel"`.
 
     The sources for `SlurmConfig` attributes, in increasing priority order, are

fractal_server/app/runner/executors/slurm_common/remote.py

@@ -134,7 +134,7 @@ def worker(
         _extra_import_paths = extra_import_paths.split(":")
         sys.path[:0] = _extra_import_paths
 
-    # Execute the job and catpure exceptions
+    # Execute the job and capture exceptions
     try:
         with open(in_fname, "rb") as f:
             indata = f.read()