fractal-server 2.10.5__py3-none-any.whl → 2.11.0__py3-none-any.whl

fractal_server/app/routes/api/v2/workflowtask.py

@@ -12,6 +12,7 @@ from ....db import get_async_db
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import _get_workflow_task_check_owner
 from ._aux_functions import _workflow_insert_task
+from ._aux_functions_tasks import _check_type_filters_compatibility
 from ._aux_functions_tasks import _get_task_read_access
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import WorkflowTaskV2
@@ -47,24 +48,29 @@ async def replace_workflowtask(
         db=db,
     )
 
-    task = await _get_task_read_access(
+    new_task = await _get_task_read_access(
         task_id=task_id, user_id=user.id, db=db, require_active=True
     )
 
-    if task.type != old_workflow_task.task.type:
+    if new_task.type != old_workflow_task.task.type:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=(
                 f"Cannot replace a Task '{old_workflow_task.task.type}' with a"
-                f"  Task '{task.type}'."
+                f"  Task '{new_task.type}'."
             ),
         )
 
+    _check_type_filters_compatibility(
+        task_input_types=new_task.input_types,
+        wftask_type_filters=old_workflow_task.type_filters,
+    )
+
     _args_non_parallel = old_workflow_task.args_non_parallel
     _args_parallel = old_workflow_task.args_parallel
     if replace is not None:
         if replace.args_non_parallel is not None:
-            if task.type == "parallel":
+            if new_task.type == "parallel":
                 raise HTTPException(
                     status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                     detail=(
@@ -76,7 +82,7 @@ async def replace_workflowtask(
                 _args_non_parallel = replace.args_non_parallel
 
         if replace.args_parallel is not None:
-            if task.type == "non_parallel":
+            if new_task.type == "non_parallel":
                 raise HTTPException(
                     status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                     detail=(
@@ -92,24 +98,26 @@ async def replace_workflowtask(
     if (
         old_workflow_task.meta_non_parallel
         != old_workflow_task.task.meta_non_parallel
-    ) and (old_workflow_task.task.meta_non_parallel == task.meta_non_parallel):
+    ) and (
+        old_workflow_task.task.meta_non_parallel == new_task.meta_non_parallel
+    ):
         _meta_non_parallel = old_workflow_task.meta_non_parallel
     else:
-        _meta_non_parallel = task.meta_non_parallel
+        _meta_non_parallel = new_task.meta_non_parallel
     # Same for `meta_parallel`
     if (
         old_workflow_task.meta_parallel != old_workflow_task.task.meta_parallel
-    ) and (old_workflow_task.task.meta_parallel == task.meta_parallel):
+    ) and (old_workflow_task.task.meta_parallel == new_task.meta_parallel):
         _meta_parallel = old_workflow_task.meta_parallel
     else:
-        _meta_parallel = task.meta_parallel
+        _meta_parallel = new_task.meta_parallel
 
     new_workflow_task = WorkflowTaskV2(
-        task_id=task.id,
-        task_type=task.type,
-        task=task,
+        task_id=new_task.id,
+        task_type=new_task.type,
+        task=new_task,
         # old-task values
-        input_filters=old_workflow_task.input_filters,
+        type_filters=old_workflow_task.type_filters,
         # possibly new values
         args_non_parallel=_args_non_parallel,
         args_parallel=_args_parallel,
@@ -134,7 +142,7 @@ async def create_workflowtask(
     project_id: int,
     workflow_id: int,
     task_id: int,
-    new_task: WorkflowTaskCreateV2,
+    wftask: WorkflowTaskCreateV2,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> Optional[WorkflowTaskReadV2]:
@@ -152,8 +160,8 @@ async def create_workflowtask(
 
     if task.type == "parallel":
         if (
-            new_task.meta_non_parallel is not None
-            or new_task.args_non_parallel is not None
+            wftask.meta_non_parallel is not None
+            or wftask.args_non_parallel is not None
         ):
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -165,8 +173,8 @@ async def create_workflowtask(
             )
     elif task.type == "non_parallel":
         if (
-            new_task.meta_parallel is not None
-            or new_task.args_parallel is not None
+            wftask.meta_parallel is not None
+            or wftask.args_parallel is not None
         ):
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -176,20 +184,26 @@ async def create_workflowtask(
                     "is `non_parallel`."
                 ),
             )
-    workflow_task = await _workflow_insert_task(
+
+    _check_type_filters_compatibility(
+        task_input_types=task.input_types,
+        wftask_type_filters=wftask.type_filters,
+    )
+
+    wftask_db = await _workflow_insert_task(
         workflow_id=workflow.id,
         task_id=task_id,
-        meta_non_parallel=new_task.meta_non_parallel,
-        meta_parallel=new_task.meta_parallel,
-        args_non_parallel=new_task.args_non_parallel,
-        args_parallel=new_task.args_parallel,
-        input_filters=new_task.input_filters,
+        meta_non_parallel=wftask.meta_non_parallel,
+        meta_parallel=wftask.meta_parallel,
+        args_non_parallel=wftask.args_non_parallel,
+        args_parallel=wftask.args_parallel,
+        type_filters=wftask.type_filters,
         db=db,
     )
 
     await db.close()
 
-    return workflow_task
+    return wftask_db
 
 
 @router.get(
@@ -236,6 +250,11 @@ async def update_workflowtask(
         user_id=user.id,
         db=db,
     )
+    if workflow_task_update.type_filters is not None:
+        _check_type_filters_compatibility(
+            task_input_types=db_wf_task.task.input_types,
+            wftask_type_filters=workflow_task_update.type_filters,
+        )
 
     if db_wf_task.task_type == "parallel" and (
         workflow_task_update.args_non_parallel is not None
@@ -274,7 +293,7 @@ async def update_workflowtask(
             if not actual_args:
                 actual_args = None
             setattr(db_wf_task, key, actual_args)
-        elif key in ["meta_parallel", "meta_non_parallel", "input_filters"]:
+        elif key in ["meta_parallel", "meta_non_parallel", "type_filters"]:
             setattr(db_wf_task, key, value)
         else:
             raise HTTPException(

fractal_server/app/runner/{async_wrap.py → async_wrap_v1.py}

@@ -4,7 +4,7 @@ from functools import wraps
 from typing import Callable
 
 
-def async_wrap(func: Callable) -> Callable:
+def async_wrap_v1(func: Callable) -> Callable:
     """
     Wrap a synchronous callable in an async task
 

fractal_server/app/runner/executors/slurm/_slurm_config.py

@@ -423,7 +423,7 @@ def _parse_mem_value(raw_mem: Union[str, int]) -> int:
     )
 
     # Handle integer argument
-    if isinstance(raw_mem, int):
+    if type(raw_mem) is int:
         return raw_mem
 
     # Handle string argument

fractal_server/app/runner/executors/slurm/ssh/executor.py

@@ -966,8 +966,8 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                 # Retrieve job and future objects
                 job = jobs[ind_job]
                 future = futures[ind_job]
-                remaining_job_ids = job_ids[ind_job + 1 :]  # noqa: E203
-                remaining_futures = futures[ind_job + 1 :]  # noqa: E203
+                remaining_job_ids = job_ids[ind_job + 1 :]
+                remaining_futures = futures[ind_job + 1 :]
 
                 outputs = []
 

fractal_server/app/runner/filenames.py

@@ -1,6 +1,4 @@
-HISTORY_FILENAME = "history.json"
-FILTERS_FILENAME = "filters.json"
-IMAGES_FILENAME = "images.json"
-METADATA_FILENAME = "metadata.json"
+HISTORY_FILENAME_V1 = "history.json"
+METADATA_FILENAME_V1 = "metadata.json"
 SHUTDOWN_FILENAME = "shutdown"
 WORKFLOW_LOG_FILENAME = "workflow.log"

fractal_server/app/runner/v1/_common.py

@@ -28,8 +28,8 @@ from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
 from .common import TaskParameters
 from .common import write_args_file
-from fractal_server.app.runner.filenames import HISTORY_FILENAME
-from fractal_server.app.runner.filenames import METADATA_FILENAME
+from fractal_server.app.runner.filenames import HISTORY_FILENAME_V1
+from fractal_server.app.runner.filenames import METADATA_FILENAME_V1
 from fractal_server.app.runner.task_files import get_task_file_paths
 from fractal_server.string_tools import validate_cmd
 
@@ -610,11 +610,11 @@ def execute_tasks(
         )
 
         # Write most recent metadata to METADATA_FILENAME
-        with open(workflow_dir_local / METADATA_FILENAME, "w") as f:
+        with open(workflow_dir_local / METADATA_FILENAME_V1, "w") as f:
             json.dump(current_task_pars.metadata, f, indent=2)
 
         # Write most recent metadata to HISTORY_FILENAME
-        with open(workflow_dir_local / HISTORY_FILENAME, "w") as f:
+        with open(workflow_dir_local / HISTORY_FILENAME_V1, "w") as f:
             json.dump(current_task_pars.history, f, indent=2)
 
     return current_task_pars

fractal_server/app/runner/v1/_local/__init__.py

@@ -24,7 +24,7 @@ from typing import Any
 from typing import Optional
 
 from ....models.v1 import Workflow
-from ...async_wrap import async_wrap
+from ...async_wrap_v1 import async_wrap_v1
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from .._common import execute_tasks
 from ..common import TaskParameters
@@ -172,7 +172,7 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    output_dataset_metadata_history = await async_wrap(_process_workflow)(
+    output_dataset_metadata_history = await async_wrap_v1(_process_workflow)(
         workflow=workflow,
         input_paths=input_paths,
         output_path=output_path,

fractal_server/app/runner/v1/_slurm/__init__.py

@@ -21,7 +21,7 @@ from typing import Any
 from typing import Optional
 from typing import Union
 
-from ...async_wrap import async_wrap
+from ...async_wrap_v1 import async_wrap_v1
 from ...executors.slurm.sudo.executor import FractalSlurmExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from .._common import execute_tasks
@@ -145,7 +145,7 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    output_dataset_metadata_history = await async_wrap(_process_workflow)(
+    output_dataset_metadata_history = await async_wrap_v1(_process_workflow)(
         workflow=workflow,
         input_paths=input_paths,
         output_path=output_path,

fractal_server/app/runner/v1/handle_failed_job.py

@@ -24,8 +24,8 @@ from ...models.v1 import Dataset
 from ...models.v1 import Workflow
 from ...models.v1 import WorkflowTask
 from ...schemas.v1 import WorkflowTaskStatusTypeV1
-from ..filenames import HISTORY_FILENAME
-from ..filenames import METADATA_FILENAME
+from ..filenames import HISTORY_FILENAME_V1
+from ..filenames import METADATA_FILENAME_V1
 
 
 def assemble_history_failed_job(
@@ -64,7 +64,7 @@ def assemble_history_failed_job(
     new_history = output_dataset.history
 
     # Part 2: Extend history based on tmp_metadata_file
-    tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME
+    tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME_V1
     try:
         with tmp_history_file.open("r") as f:
             tmp_file_history = json.load(f)
@@ -129,7 +129,7 @@ def assemble_meta_failed_job(
     """
 
     new_meta = deepcopy(output_dataset.meta)
-    metadata_file = Path(job.working_dir) / METADATA_FILENAME
+    metadata_file = Path(job.working_dir) / METADATA_FILENAME_V1
     try:
         with metadata_file.open("r") as f:
             metadata_update = json.load(f)

fractal_server/app/runner/v2/__init__.py

@@ -11,7 +11,6 @@ from pathlib import Path
 from typing import Optional
 
 from sqlalchemy.orm import Session as DBSyncSession
-from sqlalchemy.orm.attributes import flag_modified
 
 from ....config import get_settings
 from ....logger import get_logger
@@ -24,7 +23,6 @@ from ....zip_tools import _zip_folder_to_file_and_remove
 from ...db import DB
 from ...models.v2 import DatasetV2
 from ...models.v2 import JobV2
-from ...models.v2 import WorkflowTaskV2
 from ...models.v2 import WorkflowV2
 from ...schemas.v2 import JobStatusTypeV2
 from ..exceptions import JobExecutionError
@@ -38,12 +36,11 @@ from ._local_experimental import (
 )
 from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
 from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
-from .handle_failed_job import assemble_filters_failed_job
-from .handle_failed_job import assemble_history_failed_job
-from .handle_failed_job import assemble_images_failed_job
+from .handle_failed_job import mark_last_wftask_as_failed
 from fractal_server import __VERSION__
 from fractal_server.app.models import UserSettings
 
+
 _backends = {}
 _backends["local"] = local_process_workflow
 _backends["slurm"] = slurm_sudo_process_workflow
@@ -72,7 +69,7 @@ def fail_job(
     return
 
 
-async def submit_workflow(
+def submit_workflow(
     *,
     workflow_id: int,
     dataset_id: int,
@@ -115,7 +112,6 @@ async def submit_workflow(
     logger = set_logger(logger_name=logger_name)
 
     with next(DB.get_sync_db()) as db_sync:
-
         try:
             job: Optional[JobV2] = db_sync.get(JobV2, job_id)
             dataset: Optional[DatasetV2] = db_sync.get(DatasetV2, dataset_id)
@@ -322,7 +318,7 @@ async def submit_workflow(
         db_sync = next(DB.get_sync_db())
         db_sync.close()
 
-        new_dataset_attributes = await process_workflow(
+        process_workflow(
             workflow=workflow,
             dataset=dataset,
             workflow_dir_local=WORKFLOW_DIR_LOCAL,
@@ -331,6 +327,7 @@ async def submit_workflow(
             worker_init=worker_init,
             first_task_index=job.first_task_index,
             last_task_index=job.last_task_index,
+            job_attribute_filters=job.attribute_filters,
             **backend_specific_kwargs,
         )
 
@@ -340,14 +337,6 @@ async def submit_workflow(
         )
         logger.debug(f'END workflow "{workflow.name}"')
 
-        # Update dataset attributes, in case of successful execution
-        dataset.history.extend(new_dataset_attributes["history"])
-        dataset.filters = new_dataset_attributes["filters"]
-        dataset.images = new_dataset_attributes["images"]
-        for attribute_name in ["filters", "history", "images"]:
-            flag_modified(dataset, attribute_name)
-        db_sync.merge(dataset)
-
         # Update job DB entry
         job.status = JobStatusTypeV2.DONE
         job.end_timestamp = get_timestamp()
@@ -358,28 +347,13 @@ async def submit_workflow(
         db_sync.commit()
 
     except TaskExecutionError as e:
-
         logger.debug(f'FAILED workflow "{workflow.name}", TaskExecutionError.')
         logger.info(f'Workflow "{workflow.name}" failed (TaskExecutionError).')
 
-        # Read dataset attributes produced by the last successful task, and
-        # update the DB dataset accordingly
-        failed_wftask = db_sync.get(WorkflowTaskV2, e.workflow_task_id)
-        dataset.history = assemble_history_failed_job(
-            job,
-            dataset,
-            workflow,
+        mark_last_wftask_as_failed(
+            dataset_id=dataset_id,
             logger_name=logger_name,
-            failed_wftask=failed_wftask,
         )
-        latest_filters = assemble_filters_failed_job(job)
-        if latest_filters is not None:
-            dataset.filters = latest_filters
-        latest_images = assemble_images_failed_job(job)
-        if latest_images is not None:
-            dataset.images = latest_images
-        db_sync.merge(dataset)
-
         exception_args_string = "\n".join(e.args)
         log_msg = (
             f"TASK ERROR: "
@@ -390,26 +364,12 @@ async def submit_workflow(
         fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
 
     except JobExecutionError as e:
-
         logger.debug(f'FAILED workflow "{workflow.name}", JobExecutionError.')
         logger.info(f'Workflow "{workflow.name}" failed (JobExecutionError).')
-
-        # Read dataset attributes produced by the last successful task, and
-        # update the DB dataset accordingly
-        dataset.history = assemble_history_failed_job(
-            job,
-            dataset,
-            workflow,
+        mark_last_wftask_as_failed(
+            dataset_id=dataset_id,
             logger_name=logger_name,
         )
-        latest_filters = assemble_filters_failed_job(job)
-        if latest_filters is not None:
-            dataset.filters = latest_filters
-        latest_images = assemble_images_failed_job(job)
-        if latest_images is not None:
-            dataset.images = latest_images
-        db_sync.merge(dataset)
-
         fail_job(
             db=db_sync,
             job=job,
@@ -421,27 +381,13 @@ async def submit_workflow(
         )
 
     except Exception:
-
         logger.debug(f'FAILED workflow "{workflow.name}", unknown error.')
         logger.info(f'Workflow "{workflow.name}" failed (unkwnon error).')
-
-        current_traceback = traceback.format_exc()
-
-        # Read dataset attributes produced by the last successful task, and
-        # update the DB dataset accordingly
-        dataset.history = assemble_history_failed_job(
-            job,
-            dataset,
-            workflow,
+        mark_last_wftask_as_failed(
+            dataset_id=dataset_id,
             logger_name=logger_name,
         )
-        latest_filters = assemble_filters_failed_job(job)
-        if latest_filters is not None:
-            dataset.filters = latest_filters
-        latest_images = assemble_images_failed_job(job)
-        if latest_images is not None:
-            dataset.images = latest_images
-        db_sync.merge(dataset)
+        current_traceback = traceback.format_exc()
         fail_job(
             db=db_sync,
             job=job,

fractal_server/app/runner/v2/_local/__init__.py

@@ -24,44 +24,14 @@ from typing import Optional
 
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
-from ...async_wrap import async_wrap
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _local_submit_setup
 from .executor import FractalThreadPoolExecutor
+from fractal_server.images.models import AttributeFiltersType
 
 
-def _process_workflow(
-    *,
-    workflow: WorkflowV2,
-    dataset: DatasetV2,
-    logger_name: str,
-    workflow_dir_local: Path,
-    first_task_index: int,
-    last_task_index: int,
-) -> dict:
-    """
-    Internal processing routine
-
-    Schedules the workflow using a `FractalThreadPoolExecutor`.
-    """
-
-    with FractalThreadPoolExecutor() as executor:
-        new_dataset_attributes = execute_tasks_v2(
-            wf_task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)  # noqa
-            ],  # noqa
-            dataset=dataset,
-            executor=executor,
-            workflow_dir_local=workflow_dir_local,
-            workflow_dir_remote=workflow_dir_local,
-            logger_name=logger_name,
-            submit_setup_call=_local_submit_setup,
-        )
-    return new_dataset_attributes
-
-
-async def process_workflow(
+def process_workflow(
     *,
     workflow: WorkflowV2,
     dataset: DatasetV2,
@@ -70,12 +40,13 @@ async def process_workflow(
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
     logger_name: str,
+    job_attribute_filters: AttributeFiltersType,
     # Slurm-specific
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     worker_init: Optional[str] = None,
-) -> dict:
+) -> None:
     """
     Run a workflow
 
@@ -127,11 +98,6 @@ async def process_workflow(
                             (positive exit codes).
         JobExecutionError: wrapper for errors raised by the tasks' executors
                            (negative exit codes).
-
-    Returns:
-        output_dataset_metadata:
-            The updated metadata for the dataset, as returned by the last task
-            of the workflow
     """
 
     if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
@@ -148,12 +114,16 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    new_dataset_attributes = await async_wrap(_process_workflow)(
-        workflow=workflow,
-        dataset=dataset,
-        logger_name=logger_name,
-        workflow_dir_local=workflow_dir_local,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
-    )
-    return new_dataset_attributes
+    with FractalThreadPoolExecutor() as executor:
+        execute_tasks_v2(
+            wf_task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)
+            ],
+            dataset=dataset,
+            executor=executor,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_local,
+            logger_name=logger_name,
+            submit_setup_call=_local_submit_setup,
+            job_attribute_filters=job_attribute_filters,
+        )