fractal-server 1.4.9__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py

@@ -22,10 +22,9 @@ from pydantic import Extra
 from pydantic import Field
 from pydantic.error_wrappers import ValidationError
 
-from ....config import get_settings
-from ....logger import set_logger
-from ....syringe import Inject
-from ...models import WorkflowTask
+from .....config import get_settings
+from .....logger import set_logger
+from .....syringe import Inject
 
 logger = set_logger(__name__)
 
@@ -459,151 +458,3 @@ def get_default_slurm_config():
         target_num_jobs=2,
         max_num_jobs=4,
     )
-
-
-def get_slurm_config(
-    wftask: WorkflowTask,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
-    config_path: Optional[Path] = None,
-) -> SlurmConfig:
-    """
-    Prepare a `SlurmConfig` configuration object
-
-    The sources for `SlurmConfig` attributes, in increasing priority order, are
-
-    1. The general content of the Fractal SLURM configuration file.
-    2. The GPU-specific content of the Fractal SLURM configuration file, if
-        appropriate.
-    3. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
-       `Workflow.insert_task`, also includes `wftask.task.meta`);
-
-    Note: `wftask.meta` may be `None`.
-
-    Arguments:
-        wftask:
-            WorkflowTask for which the SLURM configuration is is to be
-            prepared.
-        workflow_dir:
-            Server-owned directory to store all task-execution-related relevant
-            files (inputs, outputs, errors, and all meta files related to the
-            job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
-        config_path:
-            Path of aFractal  SLURM configuration file; if `None`, use
-            `FRACTAL_SLURM_CONFIG_FILE` variable from settings.
-
-    Returns:
-        slurm_config:
-            The SlurmConfig object
-    """
-
-    logger.debug(
-        "[get_slurm_config] WorkflowTask meta attribute: {wftask.meta=}"
-    )
-
-    # Incorporate slurm_env.default_slurm_config
-    slurm_env = load_slurm_config_file(config_path=config_path)
-    slurm_dict = slurm_env.default_slurm_config.dict(
-        exclude_unset=True, exclude={"mem"}
-    )
-    if slurm_env.default_slurm_config.mem:
-        slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
-
-    # Incorporate slurm_env.batching_config
-    for key, value in slurm_env.batching_config.dict().items():
-        slurm_dict[key] = value
-
-    # Incorporate slurm_env.user_local_exports
-    slurm_dict["user_local_exports"] = slurm_env.user_local_exports
-
-    logger.debug(
-        "[get_slurm_config] Fractal SLURM configuration file: "
-        f"{slurm_env.dict()=}"
-    )
-
-    # GPU-related options
-    # Notes about priority:
-    # 1. This block of definitions takes priority over other definitions from
-    #    slurm_env which are not under the `needs_gpu` subgroup
-    # 2. This block of definitions has lower priority than whatever comes next
-    #    (i.e. from WorkflowTask.meta).
-    if wftask.meta is not None:
-        needs_gpu = wftask.meta.get("needs_gpu", False)
-    else:
-        needs_gpu = False
-    logger.debug(f"[get_slurm_config] {needs_gpu=}")
-    if needs_gpu:
-        for key, value in slurm_env.gpu_slurm_config.dict(
-            exclude_unset=True, exclude={"mem"}
-        ).items():
-            slurm_dict[key] = value
-        if slurm_env.gpu_slurm_config.mem:
-            slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
-
-    # Number of CPUs per task, for multithreading
-    if wftask.meta is not None and "cpus_per_task" in wftask.meta:
-        cpus_per_task = int(wftask.meta["cpus_per_task"])
-        slurm_dict["cpus_per_task"] = cpus_per_task
-
-    # Required memory per task, in MB
-    if wftask.meta is not None and "mem" in wftask.meta:
-        raw_mem = wftask.meta["mem"]
-        mem_per_task_MB = _parse_mem_value(raw_mem)
-        slurm_dict["mem_per_task_MB"] = mem_per_task_MB
-
-    # Job name
-    job_name = wftask.task.name.replace(" ", "_")
-    slurm_dict["job_name"] = job_name
-
-    # Optional SLURM arguments and extra lines
-    if wftask.meta is not None:
-        account = wftask.meta.get("account", None)
-        if account is not None:
-            error_msg = (
-                f"Invalid {account=} property in WorkflowTask `meta` "
-                "attribute.\n"
-                "SLURM account must be set in the request body of the "
-                "apply-workflow endpoint, or by modifying the user properties."
-            )
-            logger.error(error_msg)
-            raise SlurmConfigError(error_msg)
-        for key in ["time", "gres", "constraint"]:
-            value = wftask.meta.get(key, None)
-            if value:
-                slurm_dict[key] = value
-    if wftask.meta is not None:
-        extra_lines = wftask.meta.get("extra_lines", [])
-    else:
-        extra_lines = []
-    extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
-    if len(set(extra_lines)) != len(extra_lines):
-        logger.debug(
-            "[get_slurm_config] Removing repeated elements "
-            f"from {extra_lines=}."
-        )
-        extra_lines = list(set(extra_lines))
-    slurm_dict["extra_lines"] = extra_lines
-
-    # Job-batching parameters (if None, they will be determined heuristically)
-    if wftask.meta is not None:
-        tasks_per_job = wftask.meta.get("tasks_per_job", None)
-        parallel_tasks_per_job = wftask.meta.get(
-            "parallel_tasks_per_job", None
-        )
-    else:
-        tasks_per_job = None
-        parallel_tasks_per_job = None
-    slurm_dict["tasks_per_job"] = tasks_per_job
-    slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
-
-    # Put everything together
-    logger.debug(
-        "[get_slurm_config] Now create a SlurmConfig object based "
-        f"on {slurm_dict=}"
-    )
-    slurm_config = SlurmConfig(**slurm_dict)
-
-    return slurm_config

fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py

@@ -19,7 +19,7 @@ import shlex
 import subprocess  # nosec
 from typing import Optional
 
-from ....logger import set_logger
+from .....logger import set_logger
 
 logger = set_logger(__name__)
 

fractal_server/app/runner/{_slurm → executors/slurm}/executor.py

@@ -29,14 +29,14 @@ import cloudpickle
 from cfut import SlurmExecutor
 from cfut.util import random_string
 
-from ....config import get_settings
-from ....logger import set_logger
-from ....syringe import Inject
-from .._common import get_task_file_paths
-from .._common import SHUTDOWN_FILENAME
-from .._common import TaskFiles
-from ..common import JobExecutionError
-from ..common import TaskExecutionError
+from .....config import get_settings
+from .....logger import set_logger
+from .....syringe import Inject
+from ...exceptions import JobExecutionError
+from ...exceptions import TaskExecutionError
+from ...filenames import SHUTDOWN_FILENAME
+from ...task_files import get_task_file_paths
+from ...task_files import TaskFiles
 from ._batching import heuristics
 from ._executor_wait_thread import FractalSlurmWaitThread
 from ._slurm_config import get_default_slurm_config
@@ -1001,7 +1001,7 @@ class FractalSlurmExecutor(SlurmExecutor):
             cmdlines.append(
                 (
                     f"{python_worker_interpreter}"
-                    " -m fractal_server.app.runner._slurm.remote "
+                    " -m fractal_server.app.runner.executors.slurm.remote "
                     f"--input-file {input_pickle_file} "
                     f"--output-file {output_pickle_file}"
                 )

fractal_server/app/runner/filenames.py

@@ -0,0 +1,6 @@
+HISTORY_FILENAME = "history.json"
+FILTERS_FILENAME = "filters.json"
+IMAGES_FILENAME = "images.json"
+METADATA_FILENAME = "metadata.json"
+SHUTDOWN_FILENAME = "shutdown"
+WORKFLOW_LOG_FILENAME = "workflow.log"

fractal_server/app/runner/set_start_and_last_task_index.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+
+def set_start_and_last_task_index(
+    num_tasks: int,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+) -> tuple[int, int]:
+    """
+    Handle `first_task_index` and `last_task_index`, by setting defaults and
+    validating values.
+
+    num_tasks:
+        Total number of tasks in a workflow task list
+    first_task_index:
+        Positional index of the first task to execute
+    last_task_index:
+        Positional index of the last task to execute
+    """
+    # Set default values
+    if first_task_index is None:
+        first_task_index = 0
+    if last_task_index is None:
+        last_task_index = num_tasks - 1
+
+    # Perform checks
+    if first_task_index < 0:
+        raise ValueError(f"{first_task_index=} cannot be negative")
+    if last_task_index < 0:
+        raise ValueError(f"{last_task_index=} cannot be negative")
+    if last_task_index > num_tasks - 1:
+        raise ValueError(
+            f"{last_task_index=} cannot be larger than {(num_tasks-1)=}"
+        )
+    if first_task_index > last_task_index:
+        raise ValueError(
+            f"{first_task_index=} cannot be larger than {last_task_index=}"
+        )
+    return (first_task_index, last_task_index)

fractal_server/app/runner/task_files.py

@@ -0,0 +1,105 @@
+from functools import lru_cache
+from pathlib import Path
+from typing import Optional
+
+
+def sanitize_component(value: str) -> str:
+    """
+    Remove {" ", "/", "."} form a string, e.g. going from
+    'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
+    """
+    return value.replace(" ", "_").replace("/", "_").replace(".", "_")
+
+
+class TaskFiles:
+    """
+    Group all file paths pertaining to a task
+
+    Attributes:
+        workflow_dir:
+            Server-owned directory to store all task-execution-related relevant
+            files (inputs, outputs, errors, and all meta files related to the
+            job execution). Note: users cannot write directly to this folder.
+        workflow_dir_user:
+            User-side directory with the same scope as `workflow_dir`, and
+            where a user can write.
+        task_order:
+            Positional order of the task within a workflow.
+        component:
+            Specific component to run the task for (relevant for tasks that
+            will be executed in parallel over many components).
+        file_prefix:
+            Prefix for all task-related files.
+        args:
+            Path for input json file.
+        metadiff:
+            Path for output json file with metadata update.
+        out:
+            Path for task-execution stdout.
+        err:
+            Path for task-execution stderr.
+    """
+
+    workflow_dir: Path
+    workflow_dir_user: Path
+    task_order: Optional[int] = None
+    component: Optional[str] = None  # FIXME: this is actually for V1 only
+
+    file_prefix: str
+    args: Path
+    out: Path
+    err: Path
+    log: Path
+    metadiff: Path
+
+    def __init__(
+        self,
+        workflow_dir: Path,
+        workflow_dir_user: Path,
+        task_order: Optional[int] = None,
+        component: Optional[str] = None,
+    ):
+        self.workflow_dir = workflow_dir
+        self.workflow_dir_user = workflow_dir_user
+        self.task_order = task_order
+        self.component = component
+
+        if self.component is not None:
+            component_safe = sanitize_component(str(self.component))
+            component_safe = f"_par_{component_safe}"
+        else:
+            component_safe = ""
+
+        if self.task_order is not None:
+            order = str(self.task_order)
+        else:
+            order = "task"
+        self.file_prefix = f"{order}{component_safe}"
+        self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
+        self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
+        self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
+        self.log = self.workflow_dir_user / f"{self.file_prefix}.log"
+        self.metadiff = (
+            self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
+        )
+
+
+@lru_cache()
+def get_task_file_paths(
+    workflow_dir: Path,
+    workflow_dir_user: Path,
+    task_order: Optional[int] = None,
+    component: Optional[str] = None,
+) -> TaskFiles:
+    """
+    Return the corrisponding TaskFiles object
+
+    This function is mainly used as a cache to avoid instantiating needless
+    objects.
+    """
+    return TaskFiles(
+        workflow_dir=workflow_dir,
+        workflow_dir_user=workflow_dir_user,
+        task_order=task_order,
+        component=component,
+    )

fractal_server/app/runner/{__init__.py → v1/__init__.py}

@@ -22,52 +22,31 @@ import traceback
 from pathlib import Path
 from typing import Optional
 
-from ... import __VERSION__
-from ...config import get_settings
-from ...logger import set_logger
-from ...syringe import Inject
-from ...utils import get_timestamp
-from ..db import DB
-from ..models import ApplyWorkflow
-from ..models import Dataset
-from ..models import Workflow
-from ..models import WorkflowTask
-from ..schemas import JobStatusType
-from ._common import WORKFLOW_LOG_FILENAME
-from ._local import process_workflow as local_process_workflow
+from ....logger import set_logger
+from ....syringe import Inject
+from ....utils import get_timestamp
+from ...db import DB
+from ...models import ApplyWorkflow
+from ...models import Dataset
+from ...models import Workflow
+from ...models import WorkflowTask
+from ...schemas.v1 import JobStatusTypeV1
+from ..exceptions import JobExecutionError
+from ..exceptions import TaskExecutionError
+from ..filenames import WORKFLOW_LOG_FILENAME
+from ..v1._local import process_workflow as local_process_workflow
+from ..v1._slurm import process_workflow as slurm_process_workflow
 from .common import close_job_logger
-from .common import JobExecutionError
-from .common import TaskExecutionError
 from .common import validate_workflow_compatibility  # noqa: F401
 from .handle_failed_job import assemble_history_failed_job
 from .handle_failed_job import assemble_meta_failed_job
+from fractal_server import __VERSION__
+from fractal_server.config import get_settings
 
 
 _backends = {}
-_backend_errors: dict[str, Exception] = {}
 _backends["local"] = local_process_workflow
-
-try:
-    from ._slurm import process_workflow as slurm_process_workflow
-
-    _backends["slurm"] = slurm_process_workflow
-except ModuleNotFoundError as e:
-    _backend_errors["slurm"] = e
-
-
-def get_process_workflow():
-    settings = Inject(get_settings)
-    try:
-        process_workflow = _backends[settings.FRACTAL_RUNNER_BACKEND]
-    except KeyError:
-        raise _backend_errors.get(
-            settings.FRACTAL_RUNNER_BACKEND,
-            RuntimeError(
-                "Unknown error during collection of backend "
-                f"`{settings.FRACTAL_RUNNER_BACKEND}`"
-            ),
-        )
-    return process_workflow
+_backends["slurm"] = slurm_process_workflow
 
 
 async def submit_workflow(
@@ -107,6 +86,17 @@ async def submit_workflow(
             The username to impersonate for the workflow execution, for the
             slurm backend.
     """
+
+    # Declare runner backend and set `process_workflow` function
+    settings = Inject(get_settings)
+    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+    if FRACTAL_RUNNER_BACKEND == "local":
+        process_workflow = local_process_workflow
+    elif FRACTAL_RUNNER_BACKEND == "slurm":
+        process_workflow = slurm_process_workflow
+    else:
+        raise RuntimeError(f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}")
+
     with next(DB.get_sync_db()) as db_sync:
 
         job: ApplyWorkflow = db_sync.get(ApplyWorkflow, job_id)
@@ -132,7 +122,7 @@ async def submit_workflow(
                 log_msg += (
                     f"Cannot fetch workflow {workflow_id} from database\n"
                 )
-            job.status = JobStatusType.FAILED
+            job.status = JobStatusTypeV1.FAILED
             job.end_timestamp = get_timestamp()
             job.log = log_msg
             db_sync.merge(job)
@@ -140,11 +130,6 @@ async def submit_workflow(
             db_sync.close()
             return
 
-        # Select backend
-        settings = Inject(get_settings)
-        FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
-        process_workflow = get_process_workflow()
-
         # Prepare some of process_workflow arguments
         input_paths = input_dataset.paths
         output_path = output_dataset.paths[0]
@@ -173,7 +158,9 @@ async def submit_workflow(
             WORKFLOW_DIR_USER = WORKFLOW_DIR
         elif FRACTAL_RUNNER_BACKEND == "slurm":
 
-            from ._slurm._subprocess_run_as_user import _mkdir_as_user
+            from ..executors.slurm._subprocess_run_as_user import (
+                _mkdir_as_user,
+            )
 
             WORKFLOW_DIR_USER = (
                 Path(user_cache_dir) / f"{WORKFLOW_DIR.name}"
@@ -274,7 +261,7 @@ async def submit_workflow(
         db_sync.merge(output_dataset)
 
         # Update job DB entry
-        job.status = JobStatusType.DONE
+        job.status = JobStatusTypeV1.DONE
         job.end_timestamp = get_timestamp()
         with log_file_path.open("r") as f:
             logs = f.read()
@@ -304,7 +291,7 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusType.FAILED
+        job.status = JobStatusTypeV1.FAILED
         job.end_timestamp = get_timestamp()
 
         exception_args_string = "\n".join(e.args)
@@ -337,7 +324,7 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusType.FAILED
+        job.status = JobStatusTypeV1.FAILED
         job.end_timestamp = get_timestamp()
         error = e.assemble_error()
         job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
@@ -366,7 +353,7 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusType.FAILED
+        job.status = JobStatusTypeV1.FAILED
         job.end_timestamp = get_timestamp()
         job.log = (
             f"UNKNOWN ERROR in Fractal job {job.id}\n"