fractal-server 2.11.0a8__tar.gz → 2.11.0a9__tar.gz

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.11.0a8
+Version: 2.11.0a9
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.11.0a9/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.11.0a9"

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/fractal_server/app/routes/api/v2/workflow.py

@@ -1,3 +1,4 @@
+from copy import copy
 from typing import Optional
 
 from fastapi import APIRouter
@@ -5,6 +6,7 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from pydantic import BaseModel
 from sqlmodel import select
 
 from ....db import AsyncSession
@@ -12,12 +14,16 @@ from ....db import get_async_db
 from ....models.v2 import JobV2
 from ....models.v2 import ProjectV2
 from ....models.v2 import WorkflowV2
+from ....runner.set_start_and_last_task_index import (
+    set_start_and_last_task_index,
+)
 from ....schemas.v2 import WorkflowCreateV2
 from ....schemas.v2 import WorkflowExportV2
 from ....schemas.v2 import WorkflowReadV2
 from ....schemas.v2 import WorkflowReadV2WithWarnings
 from ....schemas.v2 import WorkflowUpdateV2
 from ._aux_functions import _check_workflow_exists
+from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
@@ -25,6 +31,7 @@ from ._aux_functions_tasks import _add_warnings_to_workflow_tasks
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.routes.auth import current_active_user
+from fractal_server.images.tools import merge_type_filters
 
 router = APIRouter()
 
@@ -284,3 +291,91 @@ async def get_user_workflows(
     res = await db.execute(stm)
     workflow_list = res.scalars().all()
     return workflow_list
+
+
+class TypeFiltersFlow(BaseModel):
+    dataset_filters: list[dict[str, bool]]
+    input_filters: list[dict[str, bool]]
+    output_filters: list[dict[str, bool]]
+
+
+@router.get(
+    "/project/{project_id}/workflow/{workflow_id}/type-filters-flow/",
+    response_model=TypeFiltersFlow,
+)
+async def get_workflow_type_filters(
+    project_id: int,
+    workflow_id: int,
+    dataset_id: Optional[int] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[WorkflowReadV2WithWarnings]:
+    """
+    Get info on an existing workflow
+    """
+
+    workflow = await _get_workflow_check_owner(
+        project_id=project_id,
+        workflow_id=workflow_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    if len(workflow.task_list) == 0:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Workflow has no tasks.",
+        )
+
+    if dataset_id is None:
+        dataset_type_filters = {}
+    else:
+        res = await _get_dataset_check_owner(
+            project_id=project_id,
+            dataset_id=dataset_id,
+            user_id=user.id,
+            db=db,
+        )
+        dataset = res["dataset"]
+        dataset_type_filters = dataset.type_filters
+
+    num_tasks = len(workflow.task_list)
+    try:
+        first_task_index, last_task_index = set_start_and_last_task_index(
+            num_tasks,
+            first_task_index=first_task_index,
+            last_task_index=last_task_index,
+        )
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Invalid first/last task index.\nOriginal error: {str(e)}",
+        )
+
+    list_dataset_filters = [copy(dataset_type_filters)]
+    list_filters_in = []
+    list_filters_out = []
+    for wftask in workflow.task_list[first_task_index : last_task_index + 1]:
+
+        input_type_filters = copy(dataset_type_filters)
+        patch = merge_type_filters(
+            wftask_type_filters=wftask.type_filters,
+            task_input_types=wftask.task.input_types,
+        )
+        input_type_filters.update(patch)
+        list_filters_in.append(copy(input_type_filters))
+
+        output_type_filters = wftask.task.output_types
+        list_filters_out.append(output_type_filters)
+
+        dataset_type_filters.update(wftask.task.output_types)
+        list_dataset_filters.append(copy(dataset_type_filters))
+
+    response_body = dict(
+        dataset_filters=list_dataset_filters,
+        input_filters=list_filters_in,
+        output_filters=list_filters_out,
+    )
+    return response_body

fractal_server-2.11.0a8/fractal_server/app/runner/async_wrap.py → fractal_server-2.11.0a9/fractal_server/app/runner/async_wrap_v1.py

@@ -4,7 +4,7 @@ from functools import wraps
 from typing import Callable
 
 
-def async_wrap(func: Callable) -> Callable:
+def async_wrap_v1(func: Callable) -> Callable:
     """
     Wrap a synchronous callable in an async task
 

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/fractal_server/app/runner/v1/_local/__init__.py

@@ -24,7 +24,7 @@ from typing import Any
 from typing import Optional
 
 from ....models.v1 import Workflow
-from ...async_wrap import async_wrap
+from ...async_wrap_v1 import async_wrap_v1
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from .._common import execute_tasks
 from ..common import TaskParameters
@@ -172,7 +172,7 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    output_dataset_metadata_history = await async_wrap(_process_workflow)(
+    output_dataset_metadata_history = await async_wrap_v1(_process_workflow)(
         workflow=workflow,
         input_paths=input_paths,
         output_path=output_path,

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/fractal_server/app/runner/v1/_slurm/__init__.py

@@ -21,7 +21,7 @@ from typing import Any
 from typing import Optional
 from typing import Union
 
-from ...async_wrap import async_wrap
+from ...async_wrap_v1 import async_wrap_v1
 from ...executors.slurm.sudo.executor import FractalSlurmExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from .._common import execute_tasks
@@ -145,7 +145,7 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    output_dataset_metadata_history = await async_wrap(_process_workflow)(
+    output_dataset_metadata_history = await async_wrap_v1(_process_workflow)(
         workflow=workflow,
         input_paths=input_paths,
         output_path=output_path,

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/fractal_server/app/runner/v2/__init__.py

@@ -69,7 +69,7 @@ def fail_job(
     return
 
 
-async def submit_workflow(
+def submit_workflow(
     *,
     workflow_id: int,
     dataset_id: int,
@@ -318,7 +318,7 @@ async def submit_workflow(
         db_sync = next(DB.get_sync_db())
         db_sync.close()
 
-        await process_workflow(
+        process_workflow(
             workflow=workflow,
             dataset=dataset,
             workflow_dir_local=WORKFLOW_DIR_LOCAL,

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/fractal_server/app/runner/v2/_local/__init__.py

@@ -24,7 +24,6 @@ from typing import Optional
 
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
-from ...async_wrap import async_wrap
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _local_submit_setup
@@ -32,35 +31,7 @@ from .executor import FractalThreadPoolExecutor
 from fractal_server.images.models import AttributeFiltersType
 
 
-def _process_workflow(
-    *,
-    workflow: WorkflowV2,
-    dataset: DatasetV2,
-    logger_name: str,
-    workflow_dir_local: Path,
-    first_task_index: int,
-    last_task_index: int,
-    job_attribute_filters: AttributeFiltersType,
-) -> None:
-    """
-    Run the workflow using a `FractalThreadPoolExecutor`.
-    """
-    with FractalThreadPoolExecutor() as executor:
-        execute_tasks_v2(
-            wf_task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)
-            ],
-            dataset=dataset,
-            executor=executor,
-            workflow_dir_local=workflow_dir_local,
-            workflow_dir_remote=workflow_dir_local,
-            logger_name=logger_name,
-            submit_setup_call=_local_submit_setup,
-            job_attribute_filters=job_attribute_filters,
-        )
-
-
-async def process_workflow(
+def process_workflow(
     *,
     workflow: WorkflowV2,
     dataset: DatasetV2,
@@ -143,12 +114,16 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    await async_wrap(_process_workflow)(
-        workflow=workflow,
-        dataset=dataset,
-        logger_name=logger_name,
-        workflow_dir_local=workflow_dir_local,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
-        job_attribute_filters=job_attribute_filters,
-    )
+    with FractalThreadPoolExecutor() as executor:
+        execute_tasks_v2(
+            wf_task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)
+            ],
+            dataset=dataset,
+            executor=executor,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_local,
+            logger_name=logger_name,
+            submit_setup_call=_local_submit_setup,
+            job_attribute_filters=job_attribute_filters,
+        )

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/fractal_server/app/runner/v2/_local_experimental/__init__.py

@@ -4,7 +4,6 @@ from typing import Optional
 
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
-from ...async_wrap import async_wrap
 from ...exceptions import JobExecutionError
 from ...filenames import SHUTDOWN_FILENAME
 from ...set_start_and_last_task_index import set_start_and_last_task_index
@@ -14,45 +13,7 @@ from .executor import FractalProcessPoolExecutor
 from fractal_server.images.models import AttributeFiltersType
 
 
-def _process_workflow(
-    *,
-    workflow: WorkflowV2,
-    dataset: DatasetV2,
-    logger_name: str,
-    workflow_dir_local: Path,
-    first_task_index: int,
-    last_task_index: int,
-    job_attribute_filters: AttributeFiltersType,
-) -> None:
-    """
-    Run the workflow using a `FractalProcessPoolExecutor`.
-    """
-    with FractalProcessPoolExecutor(
-        shutdown_file=workflow_dir_local / SHUTDOWN_FILENAME
-    ) as executor:
-        try:
-            execute_tasks_v2(
-                wf_task_list=workflow.task_list[
-                    first_task_index : (last_task_index + 1)
-                ],
-                dataset=dataset,
-                executor=executor,
-                workflow_dir_local=workflow_dir_local,
-                workflow_dir_remote=workflow_dir_local,
-                logger_name=logger_name,
-                submit_setup_call=_local_submit_setup,
-                job_attribute_filters=job_attribute_filters,
-            )
-        except BrokenProcessPool as e:
-            raise JobExecutionError(
-                info=(
-                    "Job failed with BrokenProcessPool error, likely due to "
-                    f"an executor shutdown.\nOriginal error:\n{e.args[0]}"
-                )
-            )
-
-
-async def process_workflow(
+def process_workflow(
     *,
     workflow: WorkflowV2,
     dataset: DatasetV2,
@@ -135,12 +96,26 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    await async_wrap(_process_workflow)(
-        workflow=workflow,
-        dataset=dataset,
-        logger_name=logger_name,
-        workflow_dir_local=workflow_dir_local,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
-        job_attribute_filters=job_attribute_filters,
-    )
+    with FractalProcessPoolExecutor(
+        shutdown_file=workflow_dir_local / SHUTDOWN_FILENAME
+    ) as executor:
+        try:
+            execute_tasks_v2(
+                wf_task_list=workflow.task_list[
+                    first_task_index : (last_task_index + 1)
+                ],
+                dataset=dataset,
+                executor=executor,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_local,
+                logger_name=logger_name,
+                submit_setup_call=_local_submit_setup,
+                job_attribute_filters=job_attribute_filters,
+            )
+        except BrokenProcessPool as e:
+            raise JobExecutionError(
+                info=(
+                    "Job failed with BrokenProcessPool error, likely due to "
+                    f"an executor shutdown.\nOriginal error:\n{e.args[0]}"
+                )
+            )

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/fractal_server/app/runner/v2/_slurm_ssh/__init__.py

@@ -18,12 +18,10 @@ Executor objects.
 """
 from pathlib import Path
 from typing import Optional
-from typing import Union
 
 from .....ssh._fabric import FractalSSH
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
-from ...async_wrap import async_wrap
 from ...exceptions import JobExecutionError
 from ...executors.slurm.ssh.executor import FractalSlurmSSHExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
@@ -35,27 +33,35 @@ from fractal_server.logger import set_logger
 logger = set_logger(__name__)
 
 
-def _process_workflow(
+def process_workflow(
     *,
     workflow: WorkflowV2,
     dataset: DatasetV2,
-    logger_name: str,
     workflow_dir_local: Path,
-    workflow_dir_remote: Path,
-    first_task_index: int,
-    last_task_index: int,
-    fractal_ssh: FractalSSH,
-    worker_init: Optional[Union[str, list[str]]] = None,
+    workflow_dir_remote: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
     job_attribute_filters: AttributeFiltersType,
+    fractal_ssh: FractalSSH,
+    worker_init: Optional[str] = None,
+    # Not used
+    user_cache_dir: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
 ) -> None:
     """
-    Run the workflow using a `FractalSlurmSSHExecutor`.
-
-    This function initialises the a FractalSlurmExecutor, setting logging,
-    workflow working dir and user to impersonate. It then schedules the
-    workflow tasks and returns the new dataset attributes
+    Process workflow (SLURM backend public interface)
     """
 
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
     if isinstance(worker_init, str):
         worker_init = worker_init.split("\n")
 
@@ -89,46 +95,3 @@ def _process_workflow(
             submit_setup_call=_slurm_submit_setup,
             job_attribute_filters=job_attribute_filters,
         )
-
-
-async def process_workflow(
-    *,
-    workflow: WorkflowV2,
-    dataset: DatasetV2,
-    workflow_dir_local: Path,
-    workflow_dir_remote: Optional[Path] = None,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
-    logger_name: str,
-    job_attribute_filters: AttributeFiltersType,
-    fractal_ssh: FractalSSH,
-    worker_init: Optional[str] = None,
-    # Not used
-    user_cache_dir: Optional[str] = None,
-    slurm_user: Optional[str] = None,
-    slurm_account: Optional[str] = None,
-) -> None:
-    """
-    Process workflow (SLURM backend public interface)
-    """
-
-    # Set values of first_task_index and last_task_index
-    num_tasks = len(workflow.task_list)
-    first_task_index, last_task_index = set_start_and_last_task_index(
-        num_tasks,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
-    )
-
-    await async_wrap(_process_workflow)(
-        workflow=workflow,
-        dataset=dataset,
-        logger_name=logger_name,
-        workflow_dir_local=workflow_dir_local,
-        workflow_dir_remote=workflow_dir_remote,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
-        worker_init=worker_init,
-        fractal_ssh=fractal_ssh,
-        job_attribute_filters=job_attribute_filters,
-    )

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/fractal_server/app/runner/v2/_slurm_sudo/__init__.py

@@ -18,11 +18,9 @@ Executor objects.
 """
 from pathlib import Path
 from typing import Optional
-from typing import Union
 
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
-from ...async_wrap import async_wrap
 from ...executors.slurm.sudo.executor import FractalSlurmExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
@@ -30,29 +28,34 @@ from ._submit_setup import _slurm_submit_setup
 from fractal_server.images.models import AttributeFiltersType
 
 
-def _process_workflow(
+def process_workflow(
     *,
     workflow: WorkflowV2,
     dataset: DatasetV2,
-    logger_name: str,
     workflow_dir_local: Path,
-    workflow_dir_remote: Path,
-    first_task_index: int,
-    last_task_index: int,
+    workflow_dir_remote: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    job_attribute_filters: AttributeFiltersType,
+    # Slurm-specific
+    user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
-    user_cache_dir: str,
-    worker_init: Optional[Union[str, list[str]]] = None,
-    job_attribute_filters: AttributeFiltersType,
+    worker_init: Optional[str] = None,
 ) -> None:
     """
-    Run the workflow using a `FractalSlurmExecutor`.
-
-    This function initialises the a FractalSlurmExecutor, setting logging,
-    workflow working dir and user to impersonate. It then schedules the
-    workflow tasks and returns the new dataset attributes
+    Process workflow (SLURM backend public interface).
     """
 
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
     if not slurm_user:
         raise RuntimeError(
             "slurm_user argument is required, for slurm backend"
@@ -83,46 +86,3 @@ def _process_workflow(
             submit_setup_call=_slurm_submit_setup,
             job_attribute_filters=job_attribute_filters,
         )
-
-
-async def process_workflow(
-    *,
-    workflow: WorkflowV2,
-    dataset: DatasetV2,
-    workflow_dir_local: Path,
-    workflow_dir_remote: Optional[Path] = None,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
-    logger_name: str,
-    job_attribute_filters: AttributeFiltersType,
-    # Slurm-specific
-    user_cache_dir: Optional[str] = None,
-    slurm_user: Optional[str] = None,
-    slurm_account: Optional[str] = None,
-    worker_init: Optional[str] = None,
-) -> None:
-    """
-    Process workflow (SLURM backend public interface).
-    """
-
-    # Set values of first_task_index and last_task_index
-    num_tasks = len(workflow.task_list)
-    first_task_index, last_task_index = set_start_and_last_task_index(
-        num_tasks,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
-    )
-    await async_wrap(_process_workflow)(
-        workflow=workflow,
-        dataset=dataset,
-        logger_name=logger_name,
-        workflow_dir_local=workflow_dir_local,
-        workflow_dir_remote=workflow_dir_remote,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
-        user_cache_dir=user_cache_dir,
-        slurm_user=slurm_user,
-        slurm_account=slurm_account,
-        worker_init=worker_init,
-        job_attribute_filters=job_attribute_filters,
-    )

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/fractal_server/data_migrations/2_11_0.py

@@ -78,6 +78,7 @@ def fix_db():
                 ds.history[i]["workflowtask"]["type_filters"] = h[
                     "workflowtask"
                 ]["input_filters"]["types"]
+                ds.history[i]["workflowtask"].pop("input_filters")
             flag_modified(ds, "history")
             DatasetReadV2(
                 **ds.model_dump(),
@@ -155,8 +156,6 @@ def fix_db():
                     f"{job.dataset_id=}, "
                     f"{job.workflow_id=}."
                 )
-                # FIXME
-                pass
             job.dataset_dump.pop("filters")
             flag_modified(job, "dataset_dump")
             JobReadV2(**job.model_dump())

{fractal_server-2.11.0a8 → fractal_server-2.11.0a9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.11.0a8"
+version = "2.11.0a9"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -95,7 +95,7 @@ filterwarnings = [
 ]
 
 [tool.bumpver]
-current_version = "2.11.0a8"
+current_version = "2.11.0a9"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.11.0a8/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.11.0a8"