fractal-server 1.4.10__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

fractal_server/app/runner/{_slurm → executors/slurm}/executor.py

@@ -29,14 +29,14 @@ import cloudpickle
 from cfut import SlurmExecutor
 from cfut.util import random_string
 
-from ....config import get_settings
-from ....logger import set_logger
-from ....syringe import Inject
-from .._common import get_task_file_paths
-from .._common import SHUTDOWN_FILENAME
-from .._common import TaskFiles
-from ..common import JobExecutionError
-from ..common import TaskExecutionError
+from .....config import get_settings
+from .....logger import set_logger
+from .....syringe import Inject
+from ...exceptions import JobExecutionError
+from ...exceptions import TaskExecutionError
+from ...filenames import SHUTDOWN_FILENAME
+from ...task_files import get_task_file_paths
+from ...task_files import TaskFiles
 from ._batching import heuristics
 from ._executor_wait_thread import FractalSlurmWaitThread
 from ._slurm_config import get_default_slurm_config
@@ -46,6 +46,7 @@ from ._subprocess_run_as_user import _glob_as_user_strict
 from ._subprocess_run_as_user import _path_exists_as_user
 from ._subprocess_run_as_user import _run_command_as_user
 from fractal_server import __VERSION__
+from fractal_server.app.runner.components import _COMPONENT_KEY_
 
 
 logger = set_logger(__name__)
@@ -544,7 +545,7 @@ class FractalSlurmExecutor(SlurmExecutor):
         single_task_submission: bool = False,
         args: Optional[Sequence[Any]] = None,
         kwargs: Optional[dict] = None,
-        components: list[Any] = None,
+        components: Optional[list[Any]] = None,
     ) -> Future:
         """
         Submit a multi-task job to the pool, where each task is handled via the
@@ -580,6 +581,10 @@ class FractalSlurmExecutor(SlurmExecutor):
 
         # Define slurm-job-related files
         if single_task_submission:
+            if components is not None:
+                raise ValueError(
+                    f"{single_task_submission=} but components is not None"
+                )
             job = SlurmJob(
                 slurm_file_prefix=slurm_file_prefix,
                 num_tasks_tot=1,
@@ -603,15 +608,23 @@ class FractalSlurmExecutor(SlurmExecutor):
                 num_tasks_tot=num_tasks_tot,
                 slurm_config=slurm_config,
             )
-            job.wftask_file_prefixes = tuple(
-                get_task_file_paths(
-                    workflow_dir=task_files.workflow_dir,
-                    workflow_dir_user=task_files.workflow_dir_user,
-                    task_order=task_files.task_order,
-                    component=component,
-                ).file_prefix
-                for component in components
-            )
+
+            _prefixes = []
+            for component in components:
+                if isinstance(component, dict):
+                    # This is needed for V2
+                    actual_component = component.get(_COMPONENT_KEY_, None)
+                else:
+                    actual_component = component
+                _prefixes.append(
+                    get_task_file_paths(
+                        workflow_dir=task_files.workflow_dir,
+                        workflow_dir_user=task_files.workflow_dir_user,
+                        task_order=task_files.task_order,
+                        component=actual_component,
+                    ).file_prefix
+                )
+            job.wftask_file_prefixes = tuple(_prefixes)
 
         # Define I/O pickle file names/paths
         job.input_pickle_files = tuple(
@@ -1001,7 +1014,7 @@ class FractalSlurmExecutor(SlurmExecutor):
             cmdlines.append(
                 (
                     f"{python_worker_interpreter}"
-                    " -m fractal_server.app.runner._slurm.remote "
+                    " -m fractal_server.app.runner.executors.slurm.remote "
                     f"--input-file {input_pickle_file} "
                     f"--output-file {output_pickle_file}"
                 )

fractal_server/app/runner/filenames.py

@@ -0,0 +1,6 @@
+HISTORY_FILENAME = "history.json"
+FILTERS_FILENAME = "filters.json"
+IMAGES_FILENAME = "images.json"
+METADATA_FILENAME = "metadata.json"
+SHUTDOWN_FILENAME = "shutdown"
+WORKFLOW_LOG_FILENAME = "workflow.log"

fractal_server/app/runner/set_start_and_last_task_index.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+
+def set_start_and_last_task_index(
+    num_tasks: int,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+) -> tuple[int, int]:
+    """
+    Handle `first_task_index` and `last_task_index`, by setting defaults and
+    validating values.
+
+    num_tasks:
+        Total number of tasks in a workflow task list
+    first_task_index:
+        Positional index of the first task to execute
+    last_task_index:
+        Positional index of the last task to execute
+    """
+    # Set default values
+    if first_task_index is None:
+        first_task_index = 0
+    if last_task_index is None:
+        last_task_index = num_tasks - 1
+
+    # Perform checks
+    if first_task_index < 0:
+        raise ValueError(f"{first_task_index=} cannot be negative")
+    if last_task_index < 0:
+        raise ValueError(f"{last_task_index=} cannot be negative")
+    if last_task_index > num_tasks - 1:
+        raise ValueError(
+            f"{last_task_index=} cannot be larger than {(num_tasks-1)=}"
+        )
+    if first_task_index > last_task_index:
+        raise ValueError(
+            f"{first_task_index=} cannot be larger than {last_task_index=}"
+        )
+    return (first_task_index, last_task_index)

fractal_server/app/runner/task_files.py

@@ -0,0 +1,103 @@
+from pathlib import Path
+from typing import Optional
+
+
+def sanitize_component(value: str) -> str:
+    """
+    Remove {" ", "/", "."} form a string, e.g. going from
+    'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
+    """
+    return value.replace(" ", "_").replace("/", "_").replace(".", "_")
+
+
+class TaskFiles:
+    """
+    Group all file paths pertaining to a task
+
+    Attributes:
+        workflow_dir:
+            Server-owned directory to store all task-execution-related relevant
+            files (inputs, outputs, errors, and all meta files related to the
+            job execution). Note: users cannot write directly to this folder.
+        workflow_dir_user:
+            User-side directory with the same scope as `workflow_dir`, and
+            where a user can write.
+        task_order:
+            Positional order of the task within a workflow.
+        component:
+            Specific component to run the task for (relevant for tasks that
+            will be executed in parallel over many components).
+        file_prefix:
+            Prefix for all task-related files.
+        args:
+            Path for input json file.
+        metadiff:
+            Path for output json file with metadata update.
+        out:
+            Path for task-execution stdout.
+        err:
+            Path for task-execution stderr.
+    """
+
+    workflow_dir: Path
+    workflow_dir_user: Path
+    task_order: Optional[int] = None
+    component: Optional[str] = None  # FIXME: this is actually for V1 only
+
+    file_prefix: str
+    args: Path
+    out: Path
+    err: Path
+    log: Path
+    metadiff: Path
+
+    def __init__(
+        self,
+        workflow_dir: Path,
+        workflow_dir_user: Path,
+        task_order: Optional[int] = None,
+        component: Optional[str] = None,
+    ):
+        self.workflow_dir = workflow_dir
+        self.workflow_dir_user = workflow_dir_user
+        self.task_order = task_order
+        self.component = component
+
+        if self.component is not None:
+            component_safe = sanitize_component(str(self.component))
+            component_safe = f"_par_{component_safe}"
+        else:
+            component_safe = ""
+
+        if self.task_order is not None:
+            order = str(self.task_order)
+        else:
+            order = "task"
+        self.file_prefix = f"{order}{component_safe}"
+        self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
+        self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
+        self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
+        self.log = self.workflow_dir_user / f"{self.file_prefix}.log"
+        self.metadiff = (
+            self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
+        )
+
+
+def get_task_file_paths(
+    workflow_dir: Path,
+    workflow_dir_user: Path,
+    task_order: Optional[int] = None,
+    component: Optional[str] = None,
+) -> TaskFiles:
+    """
+    Return the corrisponding TaskFiles object
+
+    This function is mainly used as a cache to avoid instantiating needless
+    objects.
+    """
+    return TaskFiles(
+        workflow_dir=workflow_dir,
+        workflow_dir_user=workflow_dir_user,
+        task_order=task_order,
+        component=component,
+    )

fractal_server/app/runner/{__init__.py → v1/__init__.py}

@@ -22,26 +22,26 @@ import traceback
 from pathlib import Path
 from typing import Optional
 
-from ... import __VERSION__
-from ...config import get_settings
-from ...logger import set_logger
-from ...syringe import Inject
-from ...utils import get_timestamp
-from ..db import DB
-from ..models import ApplyWorkflow
-from ..models import Dataset
-from ..models import Workflow
-from ..models import WorkflowTask
-from ..schemas import JobStatusType
-from ._common import WORKFLOW_LOG_FILENAME
+from ....logger import set_logger
+from ....syringe import Inject
+from ....utils import get_timestamp
+from ...db import DB
+from ...models import ApplyWorkflow
+from ...models import Dataset
+from ...models import Workflow
+from ...models import WorkflowTask
+from ...schemas.v1 import JobStatusTypeV1
+from ..exceptions import JobExecutionError
+from ..exceptions import TaskExecutionError
+from ..filenames import WORKFLOW_LOG_FILENAME
 from ._local import process_workflow as local_process_workflow
 from ._slurm import process_workflow as slurm_process_workflow
 from .common import close_job_logger
-from .common import JobExecutionError
-from .common import TaskExecutionError
 from .common import validate_workflow_compatibility  # noqa: F401
 from .handle_failed_job import assemble_history_failed_job
 from .handle_failed_job import assemble_meta_failed_job
+from fractal_server import __VERSION__
+from fractal_server.config import get_settings
 
 
 _backends = {}
@@ -122,7 +122,7 @@ async def submit_workflow(
                 log_msg += (
                     f"Cannot fetch workflow {workflow_id} from database\n"
                 )
-            job.status = JobStatusType.FAILED
+            job.status = JobStatusTypeV1.FAILED
             job.end_timestamp = get_timestamp()
             job.log = log_msg
             db_sync.merge(job)
@@ -158,7 +158,9 @@ async def submit_workflow(
             WORKFLOW_DIR_USER = WORKFLOW_DIR
         elif FRACTAL_RUNNER_BACKEND == "slurm":
 
-            from ._slurm._subprocess_run_as_user import _mkdir_as_user
+            from ..executors.slurm._subprocess_run_as_user import (
+                _mkdir_as_user,
+            )
 
             WORKFLOW_DIR_USER = (
                 Path(user_cache_dir) / f"{WORKFLOW_DIR.name}"
@@ -259,7 +261,7 @@ async def submit_workflow(
         db_sync.merge(output_dataset)
 
         # Update job DB entry
-        job.status = JobStatusType.DONE
+        job.status = JobStatusTypeV1.DONE
         job.end_timestamp = get_timestamp()
         with log_file_path.open("r") as f:
             logs = f.read()
@@ -289,7 +291,7 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusType.FAILED
+        job.status = JobStatusTypeV1.FAILED
         job.end_timestamp = get_timestamp()
 
         exception_args_string = "\n".join(e.args)
@@ -322,7 +324,7 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusType.FAILED
+        job.status = JobStatusTypeV1.FAILED
         job.end_timestamp = get_timestamp()
         error = e.assemble_error()
         job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
@@ -351,7 +353,7 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusType.FAILED
+        job.status = JobStatusTypeV1.FAILED
         job.end_timestamp = get_timestamp()
         job.log = (
             f"UNKNOWN ERROR in Fractal job {job.id}\n"

fractal_server/app/runner/{_common.py → v1/_common.py}

@@ -11,7 +11,6 @@ import subprocess  # nosec
 import traceback
 from concurrent.futures import Executor
 from copy import deepcopy
-from functools import lru_cache
 from functools import partial
 from pathlib import Path
 from shlex import split as shlex_split
@@ -19,22 +18,19 @@ from typing import Any
 from typing import Callable
 from typing import Optional
 
-from ...config import get_settings
-from ...logger import get_logger
-from ...syringe import Inject
-from ..models import Task
-from ..models import WorkflowTask
-from ..schemas import WorkflowTaskStatusType
-from .common import JobExecutionError
-from .common import TaskExecutionError
+from ....config import get_settings
+from ....logger import get_logger
+from ....syringe import Inject
+from ...models import Task
+from ...models import WorkflowTask
+from ...schemas.v1 import WorkflowTaskStatusTypeV1
+from ..exceptions import JobExecutionError
+from ..exceptions import TaskExecutionError
 from .common import TaskParameters
 from .common import write_args_file
-
-
-HISTORY_FILENAME = "history.json"
-METADATA_FILENAME = "metadata.json"
-SHUTDOWN_FILENAME = "shutdown"
-WORKFLOW_LOG_FILENAME = "workflow.log"
+from fractal_server.app.runner.filenames import HISTORY_FILENAME
+from fractal_server.app.runner.filenames import METADATA_FILENAME
+from fractal_server.app.runner.task_files import get_task_file_paths
 
 
 def no_op_submit_setup_call(
@@ -42,7 +38,6 @@ def no_op_submit_setup_call(
     wftask: WorkflowTask,
     workflow_dir: Path,
     workflow_dir_user: Path,
-    task_pars: TaskParameters,
 ) -> dict:
     """
     Default (no-operation) interface of submit_setup_call.
@@ -50,14 +45,6 @@ def no_op_submit_setup_call(
     return {}
 
 
-def sanitize_component(value: str) -> str:
-    """
-    Remove {" ", "/", "."} form a string, e.g. going from
-    'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
-    """
-    return value.replace(" ", "_").replace("/", "_").replace(".", "_")
-
-
 def _task_needs_image_list(_task: Task) -> bool:
     """
     Whether a task requires `metadata["image"]` in its `args.json` file.
@@ -78,98 +65,6 @@ def _task_needs_image_list(_task: Task) -> bool:
         return False
 
 
-class TaskFiles:
-    """
-    Group all file paths pertaining to a task
-
-    Attributes:
-        workflow_dir:
-            Server-owned directory to store all task-execution-related relevant
-            files (inputs, outputs, errors, and all meta files related to the
-            job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
-        task_order:
-            Positional order of the task within a workflow.
-        component:
-            Specific component to run the task for (relevant for tasks that
-            will be executed in parallel over many components).
-        file_prefix:
-            Prefix for all task-related files.
-        args:
-            Path for input json file.
-        metadiff:
-            Path for output json file with metadata update.
-        out:
-            Path for task-execution stdout.
-        err:
-            Path for task-execution stderr.
-    """
-
-    workflow_dir: Path
-    workflow_dir_user: Path
-    task_order: Optional[int] = None
-    component: Optional[str] = None
-
-    file_prefix: str
-    args: Path
-    out: Path
-    err: Path
-    metadiff: Path
-
-    def __init__(
-        self,
-        workflow_dir: Path,
-        workflow_dir_user: Path,
-        task_order: Optional[int] = None,
-        component: Optional[str] = None,
-    ):
-        self.workflow_dir = workflow_dir
-        self.workflow_dir_user = workflow_dir_user
-        self.task_order = task_order
-        self.component = component
-
-        if self.component is not None:
-            component_safe = sanitize_component(str(self.component))
-            component_safe = f"_par_{component_safe}"
-        else:
-            component_safe = ""
-
-        if self.task_order is not None:
-            order = str(self.task_order)
-        else:
-            order = "task"
-        self.file_prefix = f"{order}{component_safe}"
-        self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
-        self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
-        self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
-        self.metadiff = (
-            self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
-        )
-
-
-@lru_cache()
-def get_task_file_paths(
-    workflow_dir: Path,
-    workflow_dir_user: Path,
-    task_order: Optional[int] = None,
-    component: Optional[str] = None,
-) -> TaskFiles:
-    """
-    Return the corrisponding TaskFiles object
-
-    This function is mainly used as a cache to avoid instantiating needless
-    objects.
-    """
-    return TaskFiles(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
-        task_order=task_order,
-        component=component,
-    )
-
-
 def _call_command_wrapper(cmd: str, stdout: Path, stderr: Path) -> None:
     """
     Call a command and write its stdout and stderr to files
@@ -331,7 +226,7 @@ def call_single_task(
     wftask_dump["task"] = wftask.task.model_dump()
     new_history_item = dict(
         workflowtask=wftask_dump,
-        status=WorkflowTaskStatusType.DONE,
+        status=WorkflowTaskStatusTypeV1.DONE,
         parallelization=None,
     )
     updated_history = task_pars.history.copy()
@@ -529,7 +424,6 @@ def call_parallel_task(
     try:
         extra_setup = submit_setup_call(
             wftask=wftask,
-            task_pars=task_pars_depend,
             workflow_dir=workflow_dir,
             workflow_dir_user=workflow_dir_user,
         )
@@ -592,7 +486,7 @@ def call_parallel_task(
     wftask_dump["task"] = wftask.task.model_dump()
     new_history_item = dict(
         workflowtask=wftask_dump,
-        status=WorkflowTaskStatusType.DONE,
+        status=WorkflowTaskStatusTypeV1.DONE,
         parallelization=dict(
             parallelization_level=wftask.parallelization_level,
             component_list=component_list,
@@ -681,7 +575,6 @@ def execute_tasks(
             try:
                 extra_setup = submit_setup_call(
                     wftask=this_wftask,
-                    task_pars=current_task_pars,
                     workflow_dir=workflow_dir,
                     workflow_dir_user=workflow_dir_user,
                 )

fractal_server/app/runner/{_local → v1/_local}/__init__.py

@@ -23,11 +23,11 @@ from pathlib import Path
 from typing import Any
 from typing import Optional
 
-from ...models import Workflow
-from .._common import execute_tasks
-from ..common import async_wrap
-from ..common import set_start_and_last_task_index
-from ..common import TaskParameters
+from ....models import Workflow  # FIXME: this is v1 specific
+from ...async_wrap import async_wrap
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from .._common import execute_tasks  # FIXME: this is v1 specific
+from ..common import TaskParameters  # FIXME: this is v1 specific
 from ._submit_setup import _local_submit_setup
 from .executor import FractalThreadPoolExecutor
 

fractal_server/app/runner/{_local → v1/_local}/_local_config.py

@@ -19,9 +19,9 @@ from pydantic import BaseModel
 from pydantic import Extra
 from pydantic.error_wrappers import ValidationError
 
-from ....config import get_settings
-from ....syringe import Inject
-from ...models import WorkflowTask
+from .....config import get_settings
+from .....syringe import Inject
+from ....models.v1 import WorkflowTask
 
 
 class LocalBackendConfigError(ValueError):
@@ -63,15 +63,14 @@ def get_local_backend_config(
     The sources for `parallel_tasks_per_job` attributes, starting from the
     highest-priority one, are
 
-    1. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
-       `Workflow.insert_task`, also includes `wftask.task.meta`);
+    1. Properties in `wftask.meta`;
     2. The general content of the local-backend configuration file;
     3. The default value (`None`).
 
     Arguments:
         wftask:
-            WorkflowTask for which the backend configuration is is to be
-            prepared.
+            WorkflowTask (V1) for which the backend configuration should
+            be prepared.
         config_path:
             Path of local-backend configuration file; if `None`, use
             `FRACTAL_LOCAL_CONFIG_FILE` variable from settings.

fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py

@@ -14,8 +14,7 @@ Submodule to define _local_submit_setup
 from pathlib import Path
 from typing import Optional
 
-from ...models import WorkflowTask
-from ..common import TaskParameters
+from ....models.v1 import WorkflowTask
 from ._local_config import get_local_backend_config
 
 
@@ -24,7 +23,6 @@ def _local_submit_setup(
     wftask: WorkflowTask,
     workflow_dir: Optional[Path] = None,
     workflow_dir_user: Optional[Path] = None,
-    task_pars: Optional[TaskParameters] = None,
 ) -> dict[str, object]:
     """
     Collect WorfklowTask-specific configuration parameters from different
@@ -33,8 +31,6 @@ def _local_submit_setup(
     Arguments:
         wftask:
             WorkflowTask for which the configuration is to be assembled
-        task_pars:
-            Not used in this function.
         workflow_dir:
             Not used in this function.
         workflow_dir_user: