fractal-server 1.4.10__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/app/runner/v2/runner_functions_low_level.py

@@ -0,0 +1,134 @@
+import json
+import logging
+import shutil
+import subprocess  # nosec
+from pathlib import Path
+from shlex import split as shlex_split
+from typing import Any
+from typing import Optional
+
+from ..exceptions import JobExecutionError
+from ..exceptions import TaskExecutionError
+from .components import _COMPONENT_KEY_
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.app.runner.task_files import get_task_file_paths
+
+
+def _call_command_wrapper(cmd: str, log_path: Path) -> None:
+    """
+    Call a command and write its stdout and stderr to files
+
+    Raises:
+        TaskExecutionError: If the `subprocess.run` call returns a positive
+                            exit code
+        JobExecutionError:  If the `subprocess.run` call returns a negative
+                            exit code (e.g. due to the subprocess receiving a
+                            TERM or KILL signal)
+    """
+
+    # Verify that task command is executable
+    if shutil.which(shlex_split(cmd)[0]) is None:
+        msg = (
+            f'Command "{shlex_split(cmd)[0]}" is not valid. '
+            "Hint: make sure that it is executable."
+        )
+        raise TaskExecutionError(msg)
+
+    fp_log = open(log_path, "w")
+    try:
+        result = subprocess.run(  # nosec
+            shlex_split(cmd),
+            stderr=fp_log,
+            stdout=fp_log,
+        )
+    except Exception as e:
+        raise e
+    finally:
+        fp_log.close()
+
+    if result.returncode > 0:
+        with log_path.open("r") as fp_stderr:
+            err = fp_stderr.read()
+        raise TaskExecutionError(err)
+    elif result.returncode < 0:
+        raise JobExecutionError(
+            info=f"Task failed with returncode={result.returncode}"
+        )
+
+
+def run_single_task(
+    args: dict[str, Any],
+    command: str,
+    wftask: WorkflowTaskV2,
+    workflow_dir: Path,
+    workflow_dir_user: Optional[Path] = None,
+    logger_name: Optional[str] = None,
+    is_task_v1: bool = False,
+) -> dict[str, Any]:
+    """
+    Runs within an executor.
+    """
+
+    logger = logging.getLogger(logger_name)
+    logger.debug(f"Now start running {command=}")
+
+    if not workflow_dir_user:
+        workflow_dir_user = workflow_dir
+
+    component = args.pop(_COMPONENT_KEY_, None)
+    if component is None:
+        task_files = get_task_file_paths(
+            workflow_dir=workflow_dir,
+            workflow_dir_user=workflow_dir_user,
+            task_order=wftask.order,
+        )
+    else:
+        task_files = get_task_file_paths(
+            workflow_dir=workflow_dir,
+            workflow_dir_user=workflow_dir_user,
+            task_order=wftask.order,
+            component=component,
+        )
+
+    # Write arguments to args.json file
+    with task_files.args.open("w") as f:
+        json.dump(args, f, indent=2)
+
+    # Assemble full command
+    if is_task_v1:
+        full_command = (
+            f"{command} "
+            f"--json {task_files.args.as_posix()} "
+            f"--metadata-out {task_files.metadiff.as_posix()}"
+        )
+    else:
+        full_command = (
+            f"{command} "
+            f"--args-json {task_files.args.as_posix()} "
+            f"--out-json {task_files.metadiff.as_posix()}"
+        )
+
+    try:
+        _call_command_wrapper(
+            full_command,
+            log_path=task_files.log,
+        )
+    except TaskExecutionError as e:
+        e.workflow_task_order = wftask.order
+        e.workflow_task_id = wftask.id
+        e.task_name = wftask.task.name
+        raise e
+
+    try:
+        with task_files.metadiff.open("r") as f:
+            out_meta = json.load(f)
+    except FileNotFoundError as e:
+        logger.debug(
+            "Task did not produce output metadata. "
+            f"Original FileNotFoundError: {str(e)}"
+        )
+        out_meta = None
+
+    if out_meta == {}:
+        return None
+    return out_meta

fractal_server/app/runner/v2/task_interface.py

@@ -0,0 +1,43 @@
+from typing import Any
+
+from pydantic import BaseModel
+from pydantic import Field
+
+from ....images import SingleImage
+from fractal_server.images import Filters
+
+
+class TaskOutput(BaseModel):
+    class Config:
+        extra = "forbid"
+
+    image_list_updates: list[SingleImage] = Field(default_factory=list)
+    image_list_removals: list[str] = Field(default_factory=list)
+    filters: Filters = Field(default_factory=Filters)
+
+    def check_paths_are_unique(self) -> None:
+        paths = [img.path for img in self.image_list_updates]
+        paths.extend(self.image_list_removals)
+        if len(paths) != len(set(paths)):
+            duplicates = [path for path in set(paths) if paths.count(path) > 1]
+            msg = (
+                "TaskOutput image-list updates/removals has non-unique paths:"
+            )
+            for duplicate in duplicates:
+                msg = f"{msg}\n{duplicate}"
+            raise ValueError(msg)
+
+
+class InitArgsModel(BaseModel):
+    class Config:
+        extra = "forbid"
+
+    path: str
+    init_args: dict[str, Any] = Field(default_factory=dict)
+
+
+class InitTaskOutput(BaseModel):
+    class Config:
+        extra = "forbid"
+
+    parallelization_list: list[InitArgsModel] = Field(default_factory=list)

fractal_server/app/runner/v2/v1_compat.py

@@ -0,0 +1,21 @@
+from copy import deepcopy
+from pathlib import Path
+from typing import Any
+
+
+def convert_v2_args_into_v1(kwargs_v2: dict[str, Any]) -> dict[str, Any]:
+
+    kwargs_v1 = deepcopy(kwargs_v2)
+
+    path = kwargs_v2.pop("path")
+    input_path = Path(path).parents[3].as_posix()
+    component = path.replace(input_path, "").lstrip("/")
+
+    kwargs_v1 = dict(
+        input_paths=[input_path],
+        output_path=input_path,
+        metadata={},
+        component=component,
+    )
+
+    return kwargs_v1

fractal_server/app/schemas/__init__.py

@@ -1,42 +1,4 @@
-"""
-Schemas for API request/response bodies
-"""
-from .applyworkflow import ApplyWorkflowCreate  # noqa: F401
-from .applyworkflow import ApplyWorkflowRead  # noqa: F401
-from .applyworkflow import ApplyWorkflowUpdate  # noqa: F401
-from .applyworkflow import JobStatusType  # noqa: F401
-from .dataset import DatasetCreate  # noqa: F401
-from .dataset import DatasetRead  # noqa: F401
-from .dataset import DatasetStatusRead  # noqa: F401
-from .dataset import DatasetUpdate  # noqa: F401
-from .dataset import ResourceCreate  # noqa: F401
-from .dataset import ResourceRead  # noqa: F401
-from .dataset import ResourceUpdate  # noqa: F401
-from .manifest import ManifestV1  # noqa: F401
-from .manifest import TaskManifestV1  # noqa: F401
-from .project import ProjectCreate  # noqa: F401
-from .project import ProjectRead  # noqa: F401
-from .project import ProjectUpdate  # noqa: F401
-from .state import _StateBase  # noqa: F401
-from .state import StateRead  # noqa: F401
-from .task import TaskCreate  # noqa: F401
-from .task import TaskImport  # noqa: F401
-from .task import TaskRead  # noqa: F401
-from .task import TaskUpdate  # noqa: F401
-from .task_collection import TaskCollectPip  # noqa: F401
-from .task_collection import TaskCollectStatus  # noqa: F401
-from .user import UserCreate  # noqa: F401
-from .user import UserRead  # noqa: F401
-from .user import UserUpdate  # noqa: F401
-from .user import UserUpdateStrict  # noqa: F401
-from .workflow import WorkflowCreate  # noqa: F401
-from .workflow import WorkflowExport  # noqa: F401
-from .workflow import WorkflowImport  # noqa: F401
-from .workflow import WorkflowRead  # noqa: F401
-from .workflow import WorkflowTaskCreate  # noqa: F401
-from .workflow import WorkflowTaskExport  # noqa: F401
-from .workflow import WorkflowTaskImport  # noqa: F401
-from .workflow import WorkflowTaskRead  # noqa: F401
-from .workflow import WorkflowTaskStatusType  # noqa: F401
-from .workflow import WorkflowTaskUpdate  # noqa: F401
-from .workflow import WorkflowUpdate  # noqa: F401
+from .state import *  # noqa: F401, F403
+from .user import *  # noqa: F401, F403
+from .v1 import *  # noqa: F401, F403
+from .v2 import *  # noqa: F401, F403

fractal_server/app/schemas/v1/__init__.py

@@ -0,0 +1,42 @@
+"""
+Schemas for API request/response bodies
+"""
+from ..state import _StateBase  # noqa: F401
+from ..state import StateRead  # noqa: F401
+from ..user import UserCreate  # noqa: F401
+from ..user import UserRead  # noqa: F401
+from ..user import UserUpdate  # noqa: F401
+from ..user import UserUpdateStrict  # noqa: F401
+from .applyworkflow import ApplyWorkflowCreateV1  # noqa: F401
+from .applyworkflow import ApplyWorkflowReadV1  # noqa: F401
+from .applyworkflow import ApplyWorkflowUpdateV1  # noqa: F401
+from .applyworkflow import JobStatusTypeV1  # noqa: F401
+from .dataset import DatasetCreateV1  # noqa: F401
+from .dataset import DatasetReadV1  # noqa: F401
+from .dataset import DatasetStatusReadV1  # noqa: F401
+from .dataset import DatasetUpdateV1  # noqa: F401
+from .dataset import ResourceCreateV1  # noqa: F401
+from .dataset import ResourceReadV1  # noqa: F401
+from .dataset import ResourceUpdateV1  # noqa: F401
+from .manifest import ManifestV1  # noqa: F401
+from .manifest import TaskManifestV1  # noqa: F401
+from .project import ProjectCreateV1  # noqa: F401
+from .project import ProjectReadV1  # noqa: F401
+from .project import ProjectUpdateV1  # noqa: F401
+from .task import TaskCreateV1  # noqa: F401
+from .task import TaskImportV1  # noqa: F401
+from .task import TaskReadV1  # noqa: F401
+from .task import TaskUpdateV1  # noqa: F401
+from .task_collection import TaskCollectPipV1  # noqa: F401
+from .task_collection import TaskCollectStatusV1  # noqa: F401
+from .workflow import WorkflowCreateV1  # noqa: F401
+from .workflow import WorkflowExportV1  # noqa: F401
+from .workflow import WorkflowImportV1  # noqa: F401
+from .workflow import WorkflowReadV1  # noqa: F401
+from .workflow import WorkflowTaskCreateV1  # noqa: F401
+from .workflow import WorkflowTaskExportV1  # noqa: F401
+from .workflow import WorkflowTaskImportV1  # noqa: F401
+from .workflow import WorkflowTaskReadV1  # noqa: F401
+from .workflow import WorkflowTaskStatusTypeV1  # noqa: F401
+from .workflow import WorkflowTaskUpdateV1  # noqa: F401
+from .workflow import WorkflowUpdateV1  # noqa: F401

fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py}

@@ -6,21 +6,21 @@ from pydantic import BaseModel
 from pydantic import validator
 from pydantic.types import StrictStr
 
-from ._validators import valstr
-from ._validators import valutc
-from .dumps import DatasetDump
-from .dumps import ProjectDump
-from .dumps import WorkflowDump
+from .._validators import valstr
+from .._validators import valutc
+from .dumps import DatasetDumpV1
+from .dumps import ProjectDumpV1
+from .dumps import WorkflowDumpV1
 
 
 __all__ = (
-    "_ApplyWorkflowBase",
-    "ApplyWorkflowCreate",
-    "ApplyWorkflowRead",
+    "_ApplyWorkflowBaseV1",
+    "ApplyWorkflowCreateV1",
+    "ApplyWorkflowReadV1",
 )
 
 
-class JobStatusType(str, Enum):
+class JobStatusTypeV1(str, Enum):
     """
     Define the available job statuses
 
@@ -41,7 +41,7 @@ class JobStatusType(str, Enum):
     FAILED = "failed"
 
 
-class _ApplyWorkflowBase(BaseModel):
+class _ApplyWorkflowBaseV1(BaseModel):
     """
     Base class for `ApplyWorkflow`.
 
@@ -52,7 +52,7 @@ class _ApplyWorkflowBase(BaseModel):
     worker_init: Optional[str]
 
 
-class ApplyWorkflowCreate(_ApplyWorkflowBase):
+class ApplyWorkflowCreateV1(_ApplyWorkflowBaseV1):
     """
     Class for `ApplyWorkflow` creation.
 
@@ -104,7 +104,7 @@ class ApplyWorkflowCreate(_ApplyWorkflowBase):
         return v
 
 
-class ApplyWorkflowRead(_ApplyWorkflowBase):
+class ApplyWorkflowReadV1(_ApplyWorkflowBaseV1):
     """
     Class for `ApplyWorkflow` read from database.
 
@@ -132,15 +132,15 @@ class ApplyWorkflowRead(_ApplyWorkflowBase):
 
     id: int
     project_id: Optional[int]
-    project_dump: ProjectDump
+    project_dump: ProjectDumpV1
     user_email: str
     slurm_account: Optional[str]
     workflow_id: Optional[int]
-    workflow_dump: WorkflowDump
+    workflow_dump: WorkflowDumpV1
     input_dataset_id: Optional[int]
-    input_dataset_dump: DatasetDump
+    input_dataset_dump: DatasetDumpV1
     output_dataset_id: Optional[int]
-    output_dataset_dump: DatasetDump
+    output_dataset_dump: DatasetDumpV1
     start_timestamp: datetime
     end_timestamp: Optional[datetime]
     status: str
@@ -158,7 +158,7 @@ class ApplyWorkflowRead(_ApplyWorkflowBase):
     )
 
 
-class ApplyWorkflowUpdate(BaseModel):
+class ApplyWorkflowUpdateV1(BaseModel):
     """
     Class for updating a job status.
 
@@ -166,4 +166,4 @@ class ApplyWorkflowUpdate(BaseModel):
         status: New job status.
     """
 
-    status: JobStatusType
+    status: JobStatusTypeV1

fractal_server/app/schemas/{dataset.py → v1/dataset.py}

@@ -6,25 +6,25 @@ from pydantic import BaseModel
 from pydantic import Field
 from pydantic import validator
 
-from ._validators import val_absolute_path
-from ._validators import valstr
-from ._validators import valutc
-from .dumps import WorkflowTaskDump
-from .project import ProjectRead
-from .workflow import WorkflowTaskStatusType
+from .._validators import val_absolute_path
+from .._validators import valstr
+from .._validators import valutc
+from .dumps import WorkflowTaskDumpV1
+from .project import ProjectReadV1
+from .workflow import WorkflowTaskStatusTypeV1
 
 __all__ = (
-    "DatasetUpdate",
-    "DatasetCreate",
-    "DatasetRead",
-    "ResourceCreate",
-    "ResourceRead",
-    "ResourceUpdate",
-    "DatasetStatusRead",
+    "DatasetUpdateV1",
+    "DatasetCreateV1",
+    "DatasetReadV1",
+    "ResourceCreateV1",
+    "ResourceReadV1",
+    "ResourceUpdateV1",
+    "DatasetStatusReadV1",
 )
 
 
-class _ResourceBase(BaseModel):
+class _ResourceBaseV1(BaseModel):
     """
     Base class for `Resource`.
 
@@ -35,7 +35,7 @@ class _ResourceBase(BaseModel):
     path: str
 
 
-class ResourceCreate(_ResourceBase):
+class ResourceCreateV1(_ResourceBaseV1):
     """
     Class for `Resource` creation.
     """
@@ -44,7 +44,7 @@ class ResourceCreate(_ResourceBase):
     _path = validator("path", allow_reuse=True)(val_absolute_path("path"))
 
 
-class ResourceUpdate(_ResourceBase):
+class ResourceUpdateV1(_ResourceBaseV1):
     """
     Class for `Resource` update.
     """
@@ -53,7 +53,7 @@ class ResourceUpdate(_ResourceBase):
     _path = validator("path", allow_reuse=True)(val_absolute_path("path"))
 
 
-class ResourceRead(_ResourceBase):
+class ResourceReadV1(_ResourceBaseV1):
     """
     Class for `Resource` read from database.
 
@@ -66,7 +66,7 @@ class ResourceRead(_ResourceBase):
     dataset_id: int
 
 
-class _DatasetHistoryItem(BaseModel):
+class _DatasetHistoryItemV1(BaseModel):
     """
     Class for an item of `Dataset.history`.
 
@@ -77,12 +77,12 @@ class _DatasetHistoryItem(BaseModel):
             and `component_list`.
     """
 
-    workflowtask: WorkflowTaskDump
-    status: WorkflowTaskStatusType
+    workflowtask: WorkflowTaskDumpV1
+    status: WorkflowTaskStatusTypeV1
     parallelization: Optional[dict]
 
 
-class _DatasetBase(BaseModel):
+class _DatasetBaseV1(BaseModel):
     """
     Base class for `Dataset`.
 
@@ -97,11 +97,11 @@ class _DatasetBase(BaseModel):
     name: str
     type: Optional[str]
     meta: dict[str, Any] = Field(default={})
-    history: list[_DatasetHistoryItem] = Field(default=[])
+    history: list[_DatasetHistoryItemV1] = Field(default=[])
     read_only: bool = False
 
 
-class DatasetUpdate(_DatasetBase):
+class DatasetUpdateV1(_DatasetBaseV1):
     """
     Class for `Dataset` update.
 
@@ -114,7 +114,7 @@ class DatasetUpdate(_DatasetBase):
 
     name: Optional[str]
     meta: Optional[dict[str, Any]] = None
-    history: Optional[list[_DatasetHistoryItem]] = None
+    history: Optional[list[_DatasetHistoryItemV1]] = None
     read_only: Optional[bool]
 
     # Validators
@@ -122,7 +122,7 @@ class DatasetUpdate(_DatasetBase):
     _type = validator("type", allow_reuse=True)(valstr("type"))
 
 
-class DatasetCreate(_DatasetBase):
+class DatasetCreateV1(_DatasetBaseV1):
     """
     Class for `Dataset` creation.
     """
@@ -132,7 +132,7 @@ class DatasetCreate(_DatasetBase):
     _type = validator("type", allow_reuse=True)(valstr("type"))
 
 
-class DatasetRead(_DatasetBase):
+class DatasetReadV1(_DatasetBaseV1):
     """
     Class for `Dataset` read from database.
 
@@ -145,10 +145,10 @@ class DatasetRead(_DatasetBase):
     """
 
     id: int
-    resource_list: list[ResourceRead]
+    resource_list: list[ResourceReadV1]
     project_id: int
     read_only: bool
-    project: ProjectRead
+    project: ProjectReadV1
     timestamp_created: datetime
 
     _timestamp_created = validator("timestamp_created", allow_reuse=True)(
@@ -156,7 +156,7 @@ class DatasetRead(_DatasetBase):
     )
 
 
-class DatasetStatusRead(BaseModel):
+class DatasetStatusReadV1(BaseModel):
     """
     Response type for the
     `/project/{project_id}/dataset/{dataset_id}/status/` endpoint
@@ -165,6 +165,6 @@ class DatasetStatusRead(BaseModel):
     status: Optional[
         dict[
             int,
-            WorkflowTaskStatusType,
+            WorkflowTaskStatusTypeV1,
         ]
     ] = None

fractal_server/app/schemas/{dumps.py → v1/dumps.py}

@@ -14,7 +14,7 @@ from pydantic import BaseModel
 from pydantic import Extra
 
 
-class ProjectDump(BaseModel, extra=Extra.forbid):
+class ProjectDumpV1(BaseModel, extra=Extra.forbid):
 
     id: int
     name: str
@@ -22,7 +22,7 @@ class ProjectDump(BaseModel, extra=Extra.forbid):
     timestamp_created: str
 
 
-class TaskDump(BaseModel):
+class TaskDumpV1(BaseModel):
     id: int
     source: str
     name: str
@@ -33,32 +33,32 @@ class TaskDump(BaseModel):
     version: Optional[str]
 
 
-class WorkflowTaskDump(BaseModel):
+class WorkflowTaskDumpV1(BaseModel):
     id: int
     order: Optional[int]
     workflow_id: int
     task_id: int
-    task: TaskDump
+    task: TaskDumpV1
 
 
-class WorkflowDump(BaseModel):
+class WorkflowDumpV1(BaseModel):
     id: int
     name: str
     project_id: int
     timestamp_created: str
 
 
-class ResourceDump(BaseModel):
+class ResourceDumpV1(BaseModel):
     id: int
     path: str
     dataset_id: int
 
 
-class DatasetDump(BaseModel):
+class DatasetDumpV1(BaseModel):
     id: int
     name: str
     type: Optional[str]
     read_only: bool
-    resource_list: list[ResourceDump]
+    resource_list: list[ResourceDumpV1]
     project_id: int
     timestamp_created: str

fractal_server/app/schemas/{manifest.py → v1/manifest.py}

@@ -12,7 +12,7 @@ from pydantic import validator
 __all__ = ("TaskManifestV1", "ManifestV1")
 
 
-class _TaskManifestBase(BaseModel):
+class _TaskManifestBaseV1(BaseModel):
     """
     Base class for `TaskManifestV1`.
 
@@ -54,10 +54,10 @@ class _TaskManifestBase(BaseModel):
     docs_link: Optional[HttpUrl]
 
 
-TaskManifestType = TypeVar("TaskManifestType", bound=_TaskManifestBase)
+TaskManifestType = TypeVar("TaskManifestType", bound=_TaskManifestBaseV1)
 
 
-class _ManifestBase(BaseModel):
+class _ManifestBaseV1(BaseModel):
     """
     Base class for `ManifestV1`.
 
@@ -102,7 +102,7 @@ class _ManifestBase(BaseModel):
         return values
 
 
-class TaskManifestV1(_TaskManifestBase):
+class TaskManifestV1(_TaskManifestBaseV1):
     """
     Task manifest schema version 1.
     """
@@ -110,7 +110,7 @@ class TaskManifestV1(_TaskManifestBase):
     pass
 
 
-class ManifestV1(_ManifestBase):
+class ManifestV1(_ManifestBaseV1):
     """
     Manifest schema version 1.
 

fractal_server/app/schemas/{project.py → v1/project.py}

@@ -4,18 +4,18 @@ from typing import Optional
 from pydantic import BaseModel
 from pydantic import validator
 
-from ._validators import valstr
-from ._validators import valutc
+from .._validators import valstr
+from .._validators import valutc
 
 
 __all__ = (
-    "ProjectCreate",
-    "ProjectRead",
-    "ProjectUpdate",
+    "ProjectCreateV1",
+    "ProjectReadV1",
+    "ProjectUpdateV1",
 )
 
 
-class _ProjectBase(BaseModel):
+class _ProjectBaseV1(BaseModel):
     """
     Base class for `Project`.
 
@@ -28,7 +28,7 @@ class _ProjectBase(BaseModel):
     read_only: bool = False
 
 
-class ProjectCreate(_ProjectBase):
+class ProjectCreateV1(_ProjectBaseV1):
     """
     Class for `Project` creation.
     """
@@ -37,7 +37,7 @@ class ProjectCreate(_ProjectBase):
     _name = validator("name", allow_reuse=True)(valstr("name"))
 
 
-class ProjectRead(_ProjectBase):
+class ProjectReadV1(_ProjectBaseV1):
     """
     Class for `Project` read from database.
 
@@ -55,7 +55,7 @@ class ProjectRead(_ProjectBase):
     )
 
 
-class ProjectUpdate(_ProjectBase):
+class ProjectUpdateV1(_ProjectBaseV1):
     """
     Class for `Project` update.