fractal-server 1.4.10__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/app/routes/api/v2/workflowtask.py

@@ -0,0 +1,269 @@
+from copy import deepcopy
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v1 import Task
+from ....models.v2 import TaskV2
+from ....schemas.v2 import WorkflowTaskCreateV2
+from ....schemas.v2 import WorkflowTaskReadV2
+from ....schemas.v2 import WorkflowTaskUpdateV2
+from ....security import current_active_user
+from ....security import User
+from ._aux_functions import _get_workflow_check_owner
+from ._aux_functions import _get_workflow_task_check_owner
+from ._aux_functions import _workflow_insert_task
+
+router = APIRouter()
+
+
+@router.post(
+    "/project/{project_id}/workflow/{workflow_id}/wftask/",
+    response_model=WorkflowTaskReadV2,
+    status_code=status.HTTP_201_CREATED,
+)
+async def create_workflowtask(
+    project_id: int,
+    workflow_id: int,
+    task_id: int,
+    new_task: WorkflowTaskCreateV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[WorkflowTaskReadV2]:
+    """
+    Add a WorkflowTask to a Workflow
+    """
+
+    workflow = await _get_workflow_check_owner(
+        project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
+    )
+
+    if new_task.is_legacy_task is True:
+        task = await db.get(Task, task_id)
+        if not task.is_v2_compatible:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=f"Task {task_id} is not V2-compatible.",
+            )
+    else:
+        task = await db.get(TaskV2, task_id)
+
+    if not task:
+        if new_task.is_legacy_task:
+            error = f"Task {task_id} not found."
+        else:
+            error = f"TaskV2 {task_id} not found."
+
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=error
+        )
+
+    if new_task.is_legacy_task is True or task.type == "parallel":
+        if (
+            new_task.meta_non_parallel is not None
+            or new_task.args_non_parallel is not None
+        ):
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Cannot set `WorkflowTaskV2.meta_non_parallel` or "
+                    "`WorkflowTask.args_non_parallel` if the associated Task "
+                    "is `parallel` (or legacy)."
+                ),
+            )
+    elif task.type == "non_parallel":
+        if (
+            new_task.meta_parallel is not None
+            or new_task.args_parallel is not None
+        ):
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Cannot set `WorkflowTaskV2.meta_parallel` or "
+                    "`WorkflowTask.args_parallel` if the associated Task "
+                    "is `non_parallel`."
+                ),
+            )
+
+    async with db:
+        workflow_task = await _workflow_insert_task(
+            workflow_id=workflow.id,
+            is_legacy_task=new_task.is_legacy_task,
+            task_id=task_id,
+            order=new_task.order,
+            meta_non_parallel=new_task.meta_non_parallel,
+            meta_parallel=new_task.meta_parallel,
+            args_non_parallel=new_task.args_non_parallel,
+            args_parallel=new_task.args_parallel,
+            input_filters=new_task.input_filters,
+            db=db,
+        )
+
+    await db.close()
+
+    return workflow_task
+
+
+@router.get(
+    "/project/{project_id}/workflow/{workflow_id}/wftask/{workflow_task_id}/",
+    response_model=WorkflowTaskReadV2,
+)
+async def read_workflowtask(
+    project_id: int,
+    workflow_id: int,
+    workflow_task_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+):
+    workflow_task, _ = await _get_workflow_task_check_owner(
+        project_id=project_id,
+        workflow_task_id=workflow_task_id,
+        workflow_id=workflow_id,
+        user_id=user.id,
+        db=db,
+    )
+    return workflow_task
+
+
+@router.patch(
+    "/project/{project_id}/workflow/{workflow_id}/wftask/{workflow_task_id}/",
+    response_model=WorkflowTaskReadV2,
+)
+async def update_workflowtask(
+    project_id: int,
+    workflow_id: int,
+    workflow_task_id: int,
+    workflow_task_update: WorkflowTaskUpdateV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[WorkflowTaskReadV2]:
+    """
+    Edit a WorkflowTask of a Workflow
+    """
+
+    db_wf_task, db_workflow = await _get_workflow_task_check_owner(
+        project_id=project_id,
+        workflow_task_id=workflow_task_id,
+        workflow_id=workflow_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    if db_wf_task.task_type == "parallel" and (
+        workflow_task_update.args_non_parallel is not None
+        or workflow_task_update.meta_non_parallel is not None
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot patch `WorkflowTaskV2.args_non_parallel` or "
+                "`WorkflowTask.meta_non_parallel` if the associated Task is "
+                "parallel."
+            ),
+        )
+    elif db_wf_task.task_type == "non_parallel" and (
+        workflow_task_update.args_parallel is not None
+        or workflow_task_update.meta_parallel is not None
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot patch `WorkflowTaskV2.args_parallel` or "
+                "`WorkflowTask.meta_parallel` if the associated Task is "
+                "non parallel."
+            ),
+        )
+
+    for key, value in workflow_task_update.dict(exclude_unset=True).items():
+        if key == "args_parallel":
+            # Get default arguments via a Task property method
+            if db_wf_task.is_legacy_task:
+                default_args = (
+                    db_wf_task.task_legacy.default_args_from_args_schema
+                )
+            else:
+                default_args = (
+                    db_wf_task.task.default_args_parallel_from_args_schema
+                )
+            # Override default_args with args value items
+            actual_args = deepcopy(default_args)
+            if value is not None:
+                for k, v in value.items():
+                    actual_args[k] = v
+            if not actual_args:
+                actual_args = None
+            setattr(db_wf_task, key, actual_args)
+        elif key == "args_non_parallel":
+            # Get default arguments via a Task property method
+            default_args = deepcopy(
+                db_wf_task.task.default_args_non_parallel_from_args_schema
+            )
+            # Override default_args with args value items
+            actual_args = default_args.copy()
+            if value is not None:
+                for k, v in value.items():
+                    actual_args[k] = v
+            if not actual_args:
+                actual_args = None
+            setattr(db_wf_task, key, actual_args)
+        elif key == "meta_parallel":
+            current_meta_parallel = deepcopy(db_wf_task.meta_parallel) or {}
+            current_meta_parallel.update(value)
+            setattr(db_wf_task, key, current_meta_parallel)
+        elif key == "meta_non_parallel":
+            current_meta_non_parallel = (
+                deepcopy(db_wf_task.meta_non_parallel) or {}
+            )
+            current_meta_non_parallel.update(value)
+            setattr(db_wf_task, key, current_meta_non_parallel)
+        # FIXME handle `input_filters`
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=f"patch_workflow_task endpoint cannot set {key=}",
+            )
+
+    await db.commit()
+    await db.refresh(db_wf_task)
+    await db.close()
+
+    return db_wf_task
+
+
+@router.delete(
+    "/project/{project_id}/workflow/{workflow_id}/wftask/{workflow_task_id}/",
+    status_code=status.HTTP_204_NO_CONTENT,
+)
+async def delete_workflowtask(
+    project_id: int,
+    workflow_id: int,
+    workflow_task_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+    """
+    Delete a WorkflowTask of a Workflow
+    """
+
+    db_workflow_task, db_workflow = await _get_workflow_task_check_owner(
+        project_id=project_id,
+        workflow_task_id=workflow_task_id,
+        workflow_id=workflow_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    await db.delete(db_workflow_task)
+    await db.commit()
+
+    await db.refresh(db_workflow)
+    db_workflow.task_list.reorder()
+    await db.commit()
+
+    return Response(status_code=status.HTTP_204_NO_CONTENT)

fractal_server/app/routes/aux/_job.py

@@ -4,7 +4,7 @@ from zipfile import ZIP_DEFLATED
 from zipfile import ZipFile
 
 from ...models import ApplyWorkflow
-from ...runner._common import SHUTDOWN_FILENAME
+from ...runner.filenames import SHUTDOWN_FILENAME
 
 
 def _write_shutdown_file(*, job: ApplyWorkflow):

fractal_server/app/runner/async_wrap.py

@@ -0,0 +1,27 @@
+import asyncio
+from functools import partial
+from functools import wraps
+from typing import Callable
+
+
+def async_wrap(func: Callable) -> Callable:
+    """
+    Wrap a synchronous callable in an async task
+
+    Ref: [issue #140](https://github.com/fractal-analytics-platform/fractal-server/issues/140)
+    and [this StackOverflow answer](https://stackoverflow.com/q/43241221/19085332).
+
+    Returns:
+        async_wrapper:
+            A factory that allows wrapping a blocking callable within a
+            coroutine.
+    """  # noqa: E501
+
+    @wraps(func)
+    async def async_wrapper(*args, loop=None, executor=None, **kwargs):
+        if loop is None:
+            loop = asyncio.get_event_loop()
+        pfunc = partial(func, *args, **kwargs)
+        return await loop.run_in_executor(executor, pfunc)
+
+    return async_wrapper

fractal_server/app/runner/exceptions.py

@@ -0,0 +1,129 @@
+import os
+from typing import Optional
+
+
+class TaskExecutionError(RuntimeError):
+    """
+    Forwards errors occurred during the execution of a task
+
+    This error wraps and forwards errors occurred during the execution of
+    tasks, when the exit code is larger than 0 (i.e. the error took place
+    within the task). This error also adds information that is useful to track
+    down and debug the failing task within a workflow.
+
+    Attributes:
+        workflow_task_id:
+            ID of the workflow task that failed.
+        workflow_task_order:
+            Order of the task within the workflow.
+        task_name:
+            Human readable name of the failing task.
+    """
+
+    workflow_task_id: Optional[int] = None
+    workflow_task_order: Optional[int] = None
+    task_name: Optional[str] = None
+
+    def __init__(
+        self,
+        *args,
+        workflow_task_id: Optional[int] = None,
+        workflow_task_order: Optional[int] = None,
+        task_name: Optional[str] = None,
+    ):
+        super().__init__(*args)
+        self.workflow_task_id = workflow_task_id
+        self.workflow_task_order = workflow_task_order
+        self.task_name = task_name
+
+
+class JobExecutionError(RuntimeError):
+    """
+    Forwards errors in the execution of a task that are due to external factors
+
+    This error wraps and forwards errors occurred during the execution of
+    tasks, but related to external factors like:
+
+    1. A negative exit code (e.g. because the task received a TERM or KILL
+       signal);
+    2. An error on the executor side (e.g. the SLURM executor could not
+       find the pickled file with task output).
+
+    This error also adds information that is useful to track down and debug the
+    failing task within a workflow.
+
+    Attributes:
+        info:
+            A free field for additional information
+        cmd_file:
+            Path to the file of the command that was executed (e.g. a SLURM
+            submission script).
+        stdout_file:
+            Path to the file with the command stdout
+        stderr_file:
+            Path to the file with the command stderr
+    """
+
+    cmd_file: Optional[str] = None
+    stdout_file: Optional[str] = None
+    stderr_file: Optional[str] = None
+    info: Optional[str] = None
+
+    def __init__(
+        self,
+        *args,
+        cmd_file: Optional[str] = None,
+        stdout_file: Optional[str] = None,
+        stderr_file: Optional[str] = None,
+        info: Optional[str] = None,
+    ):
+        super().__init__(*args)
+        self.cmd_file = cmd_file
+        self.stdout_file = stdout_file
+        self.stderr_file = stderr_file
+        self.info = info
+
+    def _read_file(self, filepath: str) -> str:
+        """
+        Return the content of a text file, and handle the cases where it is
+        empty or missing
+        """
+        if os.path.exists(filepath):
+            with open(filepath, "r") as f:
+                content = f.read()
+                if content:
+                    return f"Content of {filepath}:\n{content}"
+                else:
+                    return f"File {filepath} is empty\n"
+        else:
+            return f"File {filepath} is missing\n"
+
+    def assemble_error(self) -> str:
+        """
+        Read the files that are specified in attributes, and combine them in an
+        error message.
+        """
+        if self.cmd_file:
+            content = self._read_file(self.cmd_file)
+            cmd_content = f"COMMAND:\n{content}\n\n"
+        else:
+            cmd_content = ""
+        if self.stdout_file:
+            content = self._read_file(self.stdout_file)
+            out_content = f"STDOUT:\n{content}\n\n"
+        else:
+            out_content = ""
+        if self.stderr_file:
+            content = self._read_file(self.stderr_file)
+            err_content = f"STDERR:\n{content}\n\n"
+        else:
+            err_content = ""
+
+        content = f"{cmd_content}{out_content}{err_content}"
+        if self.info:
+            content = f"{content}ADDITIONAL INFO:\n{self.info}\n\n"
+
+        if not content:
+            content = str(self)
+        message = f"JobExecutionError\n\n{content}"
+        return message

fractal_server/app/runner/executors/local/__init__.py

@@ -0,0 +1,3 @@
+from .executor import FractalThreadPoolExecutor
+
+__all__ = ["FractalThreadPoolExecutor"]

fractal_server/app/runner/{_local → executors/local}/executor.py

@@ -18,8 +18,8 @@ from typing import Iterable
 from typing import Optional
 from typing import Sequence
 
-from ._local_config import get_default_local_backend_config
-from ._local_config import LocalBackendConfig
+from ...v1._local._local_config import get_default_local_backend_config
+from ...v1._local._local_config import LocalBackendConfig
 
 
 class FractalThreadPoolExecutor(ThreadPoolExecutor):

fractal_server/app/runner/executors/slurm/__init__.py

@@ -0,0 +1,3 @@
+from .executor import SlurmExecutor
+
+__all__ = ["SlurmExecutor"]

fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py

@@ -14,7 +14,7 @@ Submodule to determine the number of total/parallel tasks per SLURM job.
 import math
 from typing import Optional
 
-from ....logger import set_logger
+from .....logger import set_logger
 
 logger = set_logger(__name__)
 

fractal_server/app/runner/{_slurm → executors/slurm}/_check_jobs_status.py

@@ -2,7 +2,7 @@ from subprocess import run  # nosec
 
 from cfut.slurm import STATES_FINISHED
 
-from ....logger import set_logger
+from .....logger import set_logger
 
 
 logger = set_logger(__name__)

fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py

@@ -7,7 +7,7 @@ from typing import Optional
 
 from cfut import FileWaitThread
 
-from ....logger import set_logger
+from .....logger import set_logger
 from ._check_jobs_status import _jobs_finished
 
 logger = set_logger(__name__)

fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py

@@ -22,10 +22,9 @@ from pydantic import Extra
 from pydantic import Field
 from pydantic.error_wrappers import ValidationError
 
-from ....config import get_settings
-from ....logger import set_logger
-from ....syringe import Inject
-from ...models import WorkflowTask
+from .....config import get_settings
+from .....logger import set_logger
+from .....syringe import Inject
 
 logger = set_logger(__name__)
 
@@ -459,151 +458,3 @@ def get_default_slurm_config():
         target_num_jobs=2,
         max_num_jobs=4,
     )
-
-
-def get_slurm_config(
-    wftask: WorkflowTask,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
-    config_path: Optional[Path] = None,
-) -> SlurmConfig:
-    """
-    Prepare a `SlurmConfig` configuration object
-
-    The sources for `SlurmConfig` attributes, in increasing priority order, are
-
-    1. The general content of the Fractal SLURM configuration file.
-    2. The GPU-specific content of the Fractal SLURM configuration file, if
-        appropriate.
-    3. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
-       `Workflow.insert_task`, also includes `wftask.task.meta`);
-
-    Note: `wftask.meta` may be `None`.
-
-    Arguments:
-        wftask:
-            WorkflowTask for which the SLURM configuration is is to be
-            prepared.
-        workflow_dir:
-            Server-owned directory to store all task-execution-related relevant
-            files (inputs, outputs, errors, and all meta files related to the
-            job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
-        config_path:
-            Path of aFractal  SLURM configuration file; if `None`, use
-            `FRACTAL_SLURM_CONFIG_FILE` variable from settings.
-
-    Returns:
-        slurm_config:
-            The SlurmConfig object
-    """
-
-    logger.debug(
-        "[get_slurm_config] WorkflowTask meta attribute: {wftask.meta=}"
-    )
-
-    # Incorporate slurm_env.default_slurm_config
-    slurm_env = load_slurm_config_file(config_path=config_path)
-    slurm_dict = slurm_env.default_slurm_config.dict(
-        exclude_unset=True, exclude={"mem"}
-    )
-    if slurm_env.default_slurm_config.mem:
-        slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
-
-    # Incorporate slurm_env.batching_config
-    for key, value in slurm_env.batching_config.dict().items():
-        slurm_dict[key] = value
-
-    # Incorporate slurm_env.user_local_exports
-    slurm_dict["user_local_exports"] = slurm_env.user_local_exports
-
-    logger.debug(
-        "[get_slurm_config] Fractal SLURM configuration file: "
-        f"{slurm_env.dict()=}"
-    )
-
-    # GPU-related options
-    # Notes about priority:
-    # 1. This block of definitions takes priority over other definitions from
-    #    slurm_env which are not under the `needs_gpu` subgroup
-    # 2. This block of definitions has lower priority than whatever comes next
-    #    (i.e. from WorkflowTask.meta).
-    if wftask.meta is not None:
-        needs_gpu = wftask.meta.get("needs_gpu", False)
-    else:
-        needs_gpu = False
-    logger.debug(f"[get_slurm_config] {needs_gpu=}")
-    if needs_gpu:
-        for key, value in slurm_env.gpu_slurm_config.dict(
-            exclude_unset=True, exclude={"mem"}
-        ).items():
-            slurm_dict[key] = value
-        if slurm_env.gpu_slurm_config.mem:
-            slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
-
-    # Number of CPUs per task, for multithreading
-    if wftask.meta is not None and "cpus_per_task" in wftask.meta:
-        cpus_per_task = int(wftask.meta["cpus_per_task"])
-        slurm_dict["cpus_per_task"] = cpus_per_task
-
-    # Required memory per task, in MB
-    if wftask.meta is not None and "mem" in wftask.meta:
-        raw_mem = wftask.meta["mem"]
-        mem_per_task_MB = _parse_mem_value(raw_mem)
-        slurm_dict["mem_per_task_MB"] = mem_per_task_MB
-
-    # Job name
-    job_name = wftask.task.name.replace(" ", "_")
-    slurm_dict["job_name"] = job_name
-
-    # Optional SLURM arguments and extra lines
-    if wftask.meta is not None:
-        account = wftask.meta.get("account", None)
-        if account is not None:
-            error_msg = (
-                f"Invalid {account=} property in WorkflowTask `meta` "
-                "attribute.\n"
-                "SLURM account must be set in the request body of the "
-                "apply-workflow endpoint, or by modifying the user properties."
-            )
-            logger.error(error_msg)
-            raise SlurmConfigError(error_msg)
-        for key in ["time", "gres", "constraint"]:
-            value = wftask.meta.get(key, None)
-            if value:
-                slurm_dict[key] = value
-    if wftask.meta is not None:
-        extra_lines = wftask.meta.get("extra_lines", [])
-    else:
-        extra_lines = []
-    extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
-    if len(set(extra_lines)) != len(extra_lines):
-        logger.debug(
-            "[get_slurm_config] Removing repeated elements "
-            f"from {extra_lines=}."
-        )
-        extra_lines = list(set(extra_lines))
-    slurm_dict["extra_lines"] = extra_lines
-
-    # Job-batching parameters (if None, they will be determined heuristically)
-    if wftask.meta is not None:
-        tasks_per_job = wftask.meta.get("tasks_per_job", None)
-        parallel_tasks_per_job = wftask.meta.get(
-            "parallel_tasks_per_job", None
-        )
-    else:
-        tasks_per_job = None
-        parallel_tasks_per_job = None
-    slurm_dict["tasks_per_job"] = tasks_per_job
-    slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
-
-    # Put everything together
-    logger.debug(
-        "[get_slurm_config] Now create a SlurmConfig object based "
-        f"on {slurm_dict=}"
-    )
-    slurm_config = SlurmConfig(**slurm_dict)
-
-    return slurm_config

fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py

@@ -19,7 +19,7 @@ import shlex
 import subprocess  # nosec
 from typing import Optional
 
-from ....logger import set_logger
+from .....logger import set_logger
 
 logger = set_logger(__name__)