fractal-server 2.11.1__py3-none-any.whl → 2.12.0__py3-none-any.whl

fractal_server/app/runner/v1/handle_failed_job.py

@@ -1,141 +0,0 @@
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
-#
-# Original authors:
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-# Marco Franzon <marco.franzon@exact-lab.it>
-#
-# This file is part of Fractal and was originally developed by eXact lab S.r.l.
-# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
-# Institute for Biomedical Research and Pelkmans Lab from the University of
-# Zurich.
-"""
-Helper functions to handle Dataset history.
-"""
-import json
-import logging
-from copy import deepcopy
-from pathlib import Path
-from typing import Any
-from typing import Optional
-
-from ...models.v1 import ApplyWorkflow
-from ...models.v1 import Dataset
-from ...models.v1 import Workflow
-from ...models.v1 import WorkflowTask
-from ...schemas.v1 import WorkflowTaskStatusTypeV1
-from ..filenames import HISTORY_FILENAME_V1
-from ..filenames import METADATA_FILENAME_V1
-
-
-def assemble_history_failed_job(
-    job: ApplyWorkflow,
-    output_dataset: Dataset,
-    workflow: Workflow,
-    logger: logging.Logger,
-    failed_wftask: Optional[WorkflowTask] = None,
-) -> list[dict[str, Any]]:
-    """
-    Assemble `history` after a workflow-execution job fails.
-
-    Args:
-        job:
-            The failed `ApplyWorkflow` object.
-        output_dataset:
-            The `output_dataset` associated to `job`.
-        workflow:
-            The `workflow` associated to `job`.
-        logger: A logger instance.
-        failed_wftask:
-            If set, append it to `history` during step 3; if `None`, infer
-            it by comparing the job task list and the one in
-            `tmp_metadata_file`.
-
-    Returns:
-        The new value of `history`, to be merged into
-        `output_dataset.meta`.
-    """
-
-    # The final value of the history attribute should include up to three
-    # parts, coming from: the database, the temporary file, the failed-task
-    # information.
-
-    # Part 1: Read exising history from DB
-    new_history = output_dataset.history
-
-    # Part 2: Extend history based on tmp_metadata_file
-    tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME_V1
-    try:
-        with tmp_history_file.open("r") as f:
-            tmp_file_history = json.load(f)
-            new_history.extend(tmp_file_history)
-    except FileNotFoundError:
-        tmp_file_history = []
-
-    # Part 3/A: Identify failed task, if needed
-    if failed_wftask is None:
-        job_wftasks = workflow.task_list[
-            job.first_task_index : (job.last_task_index + 1)  # noqa
-        ]
-        tmp_file_wftasks = [
-            history_item["workflowtask"] for history_item in tmp_file_history
-        ]
-        if len(job_wftasks) <= len(tmp_file_wftasks):
-            n_tasks_job = len(job_wftasks)
-            n_tasks_tmp = len(tmp_file_wftasks)
-            logger.error(
-                "Cannot identify the failed task based on job task list "
-                f"(length {n_tasks_job}) and temporary-file task list "
-                f"(length {n_tasks_tmp})."
-            )
-            logger.error("Failed task not appended to history.")
-        else:
-            failed_wftask = job_wftasks[len(tmp_file_wftasks)]
-
-    # Part 3/B: Append failed task to history
-    if failed_wftask is not None:
-        failed_wftask_dump = failed_wftask.model_dump(exclude={"task"})
-        failed_wftask_dump["task"] = failed_wftask.task.model_dump()
-        new_history_item = dict(
-            workflowtask=failed_wftask_dump,
-            status=WorkflowTaskStatusTypeV1.FAILED,
-            parallelization=dict(
-                parallelization_level=failed_wftask.parallelization_level,
-            ),
-        )
-        new_history.append(new_history_item)
-
-    return new_history
-
-
-def assemble_meta_failed_job(
-    job: ApplyWorkflow,
-    output_dataset: Dataset,
-) -> dict[str, Any]:
-    """
-    Assemble `Dataset.meta` (history excluded) for a failed workflow-execution.
-
-    Assemble new value of `output_dataset.meta` based on the last successful
-    task, i.e. based on the content of the temporary `METADATA_FILENAME` file.
-
-    Args:
-        job:
-            The failed `ApplyWorkflow` object.
-        output_dataset:
-            The `output_dataset` associated to `job`.
-
-    Returns:
-        The new value of `output_dataset.meta`, apart from its `history` key.
-    """
-
-    new_meta = deepcopy(output_dataset.meta)
-    metadata_file = Path(job.working_dir) / METADATA_FILENAME_V1
-    try:
-        with metadata_file.open("r") as f:
-            metadata_update = json.load(f)
-        for key, value in metadata_update.items():
-            new_meta[key] = value
-    except FileNotFoundError:
-        pass
-
-    return new_meta

fractal_server/app/schemas/v1/__init__.py

@@ -1,37 +0,0 @@
-"""
-Schemas for API request/response bodies
-"""
-from .applyworkflow import ApplyWorkflowCreateV1  # noqa: F401
-from .applyworkflow import ApplyWorkflowReadV1  # noqa: F401
-from .applyworkflow import ApplyWorkflowUpdateV1  # noqa: F401
-from .applyworkflow import JobStatusTypeV1  # noqa: F401
-from .dataset import DatasetCreateV1  # noqa: F401
-from .dataset import DatasetReadV1  # noqa: F401
-from .dataset import DatasetStatusReadV1  # noqa: F401
-from .dataset import DatasetUpdateV1  # noqa: F401
-from .dataset import ResourceCreateV1  # noqa: F401
-from .dataset import ResourceReadV1  # noqa: F401
-from .dataset import ResourceUpdateV1  # noqa: F401
-from .manifest import ManifestV1  # noqa: F401
-from .manifest import TaskManifestV1  # noqa: F401
-from .project import ProjectCreateV1  # noqa: F401
-from .project import ProjectReadV1  # noqa: F401
-from .project import ProjectUpdateV1  # noqa: F401
-from .state import StateRead  # noqa: F401
-from .task import TaskCreateV1  # noqa: F401
-from .task import TaskImportV1  # noqa: F401
-from .task import TaskReadV1  # noqa: F401
-from .task import TaskUpdateV1  # noqa: F401
-from .task_collection import TaskCollectPipV1  # noqa: F401
-from .task_collection import TaskCollectStatusV1  # noqa: F401
-from .workflow import WorkflowCreateV1  # noqa: F401
-from .workflow import WorkflowExportV1  # noqa: F401
-from .workflow import WorkflowImportV1  # noqa: F401
-from .workflow import WorkflowReadV1  # noqa: F401
-from .workflow import WorkflowTaskCreateV1  # noqa: F401
-from .workflow import WorkflowTaskExportV1  # noqa: F401
-from .workflow import WorkflowTaskImportV1  # noqa: F401
-from .workflow import WorkflowTaskReadV1  # noqa: F401
-from .workflow import WorkflowTaskStatusTypeV1  # noqa: F401
-from .workflow import WorkflowTaskUpdateV1  # noqa: F401
-from .workflow import WorkflowUpdateV1  # noqa: F401

fractal_server/app/schemas/v1/applyworkflow.py

@@ -1,161 +0,0 @@
-from datetime import datetime
-from enum import Enum
-from typing import Optional
-
-from pydantic import BaseModel
-from pydantic import validator
-from pydantic.types import StrictStr
-
-from .._validators import valstr
-from .dumps import DatasetDumpV1
-from .dumps import ProjectDumpV1
-from .dumps import WorkflowDumpV1
-
-
-__all__ = (
-    "_ApplyWorkflowBaseV1",
-    "ApplyWorkflowCreateV1",
-    "ApplyWorkflowReadV1",
-)
-
-
-class JobStatusTypeV1(str, Enum):
-    """
-    Define the available job statuses
-
-    Attributes:
-        SUBMITTED:
-            The job was created. This does not guarantee that it was also
-            submitted to an executor (e.g. other errors could have prevented
-            this), nor that it is actually running (e.g. SLURM jobs could be
-            still in the queue).
-        DONE:
-            The job successfully reached its end.
-        FAILED:
-            The workflow terminated with an error.
-    """
-
-    SUBMITTED = "submitted"
-    DONE = "done"
-    FAILED = "failed"
-
-
-class _ApplyWorkflowBaseV1(BaseModel):
-    """
-    Base class for `ApplyWorkflow`.
-
-    Attributes:
-        worker_init:
-    """
-
-    worker_init: Optional[str]
-
-
-class ApplyWorkflowCreateV1(_ApplyWorkflowBaseV1):
-    """
-    Class for `ApplyWorkflow` creation.
-
-    Attributes:
-        first_task_index:
-        last_task_index:
-        slurm_account:
-    """
-
-    first_task_index: Optional[int] = None
-    last_task_index: Optional[int] = None
-    slurm_account: Optional[StrictStr] = None
-
-    # Validators
-    _worker_init = validator("worker_init", allow_reuse=True)(
-        valstr("worker_init")
-    )
-
-    @validator("first_task_index", always=True)
-    def first_task_index_non_negative(cls, v, values):
-        """
-        Check that `first_task_index` is non-negative.
-        """
-        if v is not None and v < 0:
-            raise ValueError(
-                f"first_task_index cannot be negative (given: {v})"
-            )
-        return v
-
-    @validator("last_task_index", always=True)
-    def first_last_task_indices(cls, v, values):
-        """
-        Check that `last_task_index` is non-negative, and that it is not
-        smaller than `first_task_index`.
-        """
-        if v is not None and v < 0:
-            raise ValueError(
-                f"last_task_index cannot be negative (given: {v})"
-            )
-
-        first_task_index = values.get("first_task_index")
-        last_task_index = v
-        if first_task_index is not None and last_task_index is not None:
-            if first_task_index > last_task_index:
-                raise ValueError(
-                    f"{first_task_index=} cannot be larger than "
-                    f"{last_task_index=}"
-                )
-        return v
-
-
-class ApplyWorkflowReadV1(_ApplyWorkflowBaseV1):
-    """
-    Class for `ApplyWorkflow` read from database.
-
-    Attributes:
-        id:
-        project_id:
-        project_dump:
-        user_email:
-        slurm_account:
-        workflow_id:
-        workflow_dump:
-        input_dataset_id:
-        input_dataset_dump:
-        output_dataset_id:
-        output_dataset_dump:
-        start_timestamp:
-        end_timestamp:
-        status:
-        log:
-        working_dir:
-        working_dir_user:
-        first_task_index:
-        last_task_index:
-    """
-
-    id: int
-    project_id: Optional[int]
-    project_dump: ProjectDumpV1
-    user_email: str
-    slurm_account: Optional[str]
-    workflow_id: Optional[int]
-    workflow_dump: WorkflowDumpV1
-    input_dataset_id: Optional[int]
-    input_dataset_dump: DatasetDumpV1
-    output_dataset_id: Optional[int]
-    output_dataset_dump: DatasetDumpV1
-    start_timestamp: datetime
-    end_timestamp: Optional[datetime]
-    status: str
-    log: Optional[str]
-    working_dir: Optional[str]
-    working_dir_user: Optional[str]
-    first_task_index: Optional[int]
-    last_task_index: Optional[int]
-
-
-class ApplyWorkflowUpdateV1(BaseModel):
-    """
-    Class for updating a job status.
-
-    Attributes:
-        status: New job status.
-    """
-
-    status: JobStatusTypeV1

fractal_server/app/schemas/v1/dataset.py

@@ -1,165 +0,0 @@
-from datetime import datetime
-from typing import Any
-from typing import Optional
-
-from pydantic import BaseModel
-from pydantic import Field
-from pydantic import validator
-
-from .._validators import val_absolute_path
-from .._validators import valstr
-from .dumps import WorkflowTaskDumpV1
-from .project import ProjectReadV1
-from .workflow import WorkflowTaskStatusTypeV1
-
-__all__ = (
-    "DatasetUpdateV1",
-    "DatasetCreateV1",
-    "DatasetReadV1",
-    "ResourceCreateV1",
-    "ResourceReadV1",
-    "ResourceUpdateV1",
-    "DatasetStatusReadV1",
-)
-
-
-class _ResourceBaseV1(BaseModel):
-    """
-    Base class for `Resource`.
-
-    Attributes:
-        path:
-    """
-
-    path: str
-
-
-class ResourceCreateV1(_ResourceBaseV1):
-    """
-    Class for `Resource` creation.
-    """
-
-    # Validators
-    _path = validator("path", allow_reuse=True)(val_absolute_path("path"))
-
-
-class ResourceUpdateV1(_ResourceBaseV1):
-    """
-    Class for `Resource` update.
-    """
-
-    # Validators
-    _path = validator("path", allow_reuse=True)(val_absolute_path("path"))
-
-
-class ResourceReadV1(_ResourceBaseV1):
-    """
-    Class for `Resource` read from database.
-
-    Attributes:
-        id:
-        dataset_id:
-    """
-
-    id: int
-    dataset_id: int
-
-
-class _DatasetHistoryItemV1(BaseModel):
-    """
-    Class for an item of `Dataset.history`.
-
-    Attributes:
-        workflowtask:
-        status:
-        parallelization: If provided, it includes keys `parallelization_level`
-            and `component_list`.
-    """
-
-    workflowtask: WorkflowTaskDumpV1
-    status: WorkflowTaskStatusTypeV1
-    parallelization: Optional[dict]
-
-
-class _DatasetBaseV1(BaseModel):
-    """
-    Base class for `Dataset`.
-
-    Attributes:
-        name:
-        type:
-        meta:
-        history:
-        read_only:
-    """
-
-    name: str
-    type: Optional[str]
-    meta: dict[str, Any] = Field(default={})
-    history: list[_DatasetHistoryItemV1] = Field(default=[])
-    read_only: bool = False
-
-
-class DatasetUpdateV1(_DatasetBaseV1):
-    """
-    Class for `Dataset` update.
-
-    Attributes:
-        name:
-        meta:
-        history:
-        read_only:
-    """
-
-    name: Optional[str]
-    meta: Optional[dict[str, Any]] = None
-    history: Optional[list[_DatasetHistoryItemV1]] = None
-    read_only: Optional[bool]
-
-    # Validators
-    _name = validator("name", allow_reuse=True)(valstr("name"))
-    _type = validator("type", allow_reuse=True)(valstr("type"))
-
-
-class DatasetCreateV1(_DatasetBaseV1):
-    """
-    Class for `Dataset` creation.
-    """
-
-    # Validators
-    _name = validator("name", allow_reuse=True)(valstr("name"))
-    _type = validator("type", allow_reuse=True)(valstr("type"))
-
-
-class DatasetReadV1(_DatasetBaseV1):
-    """
-    Class for `Dataset` read from database.
-
-    Attributes:
-        id:
-        read_only:
-        resource_list:
-        project_id:
-        project:
-    """
-
-    id: int
-    resource_list: list[ResourceReadV1]
-    project_id: int
-    read_only: bool
-    project: ProjectReadV1
-    timestamp_created: datetime
-
-
-class DatasetStatusReadV1(BaseModel):
-    """
-    Response type for the
-    `/project/{project_id}/dataset/{dataset_id}/status/` endpoint
-    """
-
-    status: Optional[
-        dict[
-            int,
-            WorkflowTaskStatusTypeV1,
-        ]
-    ] = None

fractal_server/app/schemas/v1/dumps.py

@@ -1,64 +0,0 @@
-"""
-
-Dump models differ from their Read counterpart in that:
-* They are directly JSON-able, without any additional encoder.
-* They may only include a subset of the Read attributes.
-
-These models are used in at least two situations:
-1. In the "*_dump" attributes of ApplyWorkflow models;
-2. In the `_DatasetHistoryItem.workflowtask` model, to trim its size.
-"""
-from typing import Optional
-
-from pydantic import BaseModel
-from pydantic import Extra
-
-
-class ProjectDumpV1(BaseModel, extra=Extra.forbid):
-
-    id: int
-    name: str
-    read_only: bool
-    timestamp_created: str
-
-
-class TaskDumpV1(BaseModel):
-    id: int
-    source: str
-    name: str
-    command: str
-    input_type: str
-    output_type: str
-    owner: Optional[str]
-    version: Optional[str]
-
-
-class WorkflowTaskDumpV1(BaseModel):
-    id: int
-    order: Optional[int]
-    workflow_id: int
-    task_id: int
-    task: TaskDumpV1
-
-
-class WorkflowDumpV1(BaseModel):
-    id: int
-    name: str
-    project_id: int
-    timestamp_created: str
-
-
-class ResourceDumpV1(BaseModel):
-    id: int
-    path: str
-    dataset_id: int
-
-
-class DatasetDumpV1(BaseModel):
-    id: int
-    name: str
-    type: Optional[str]
-    read_only: bool
-    resource_list: list[ResourceDumpV1]
-    project_id: int
-    timestamp_created: str

fractal_server/app/schemas/v1/manifest.py

@@ -1,126 +0,0 @@
-from typing import Any
-from typing import Optional
-from typing import TypeVar
-
-from pydantic import BaseModel
-from pydantic import Field
-from pydantic import HttpUrl
-from pydantic import root_validator
-from pydantic import validator
-
-
-__all__ = ("TaskManifestV1", "ManifestV1")
-
-
-class _TaskManifestBaseV1(BaseModel):
-    """
-    Base class for `TaskManifestV1`.
-
-    Represents a task within a manfest
-
-    Attributes:
-        name:
-            The task name
-        executable:
-            Path to the executable relative to the package root
-
-            Note: by package root we mean "as it will be installed". If a
-            package `Pkg` installs in the folder `pkg` the executable
-            `pkg/executable.py`, this attribute must contain only
-            `executable.py`.
-        input_type:
-            The input type accepted by the task
-        output_type:
-            The output type returned by the task
-        meta:
-            Additional information about the package, such as hash of the
-            executable, specific runtime requirements (e.g., need_gpu=True),
-            etc.
-        args_schema:
-            JSON Schema for task arguments
-        docs_info:
-            Additional information about the Task, coming from the docstring.
-        docs_link:
-            Link to Task docs.
-    """
-
-    name: str
-    executable: str
-    input_type: str
-    output_type: str
-    meta: Optional[dict[str, Any]] = Field(default_factory=dict)
-    args_schema: Optional[dict[str, Any]]
-    docs_info: Optional[str]
-    docs_link: Optional[HttpUrl]
-
-
-TaskManifestType = TypeVar("TaskManifestType", bound=_TaskManifestBaseV1)
-
-
-class _ManifestBaseV1(BaseModel):
-    """
-    Base class for `ManifestV1`.
-
-    Packages containing tasks are required to include a special file
-    `__FRACTAL_MANIFEST__.json` in order to be discovered and used by Fractal.
-
-    This model class and the model classes it depends on provide the base
-    schema to read, write and validate manifests.
-
-    Attributes:
-        manifest_version:
-            A version string that provides indication for compatibility between
-            manifests as the schema evolves. This is for instance used by
-            Fractal to determine which subclass of the present base class needs
-            be used to read and validate the input.
-        task_list : list[TaskManifestType]
-            The list of tasks, represented as specified by subclasses of the
-            _TaskManifestBase (a.k.a. TaskManifestType)
-        has_args_schemas:
-            `True` if the manifest incldues JSON Schemas for the arguments of
-            each task.
-        args_schema_version:
-            Label of how `args_schema`s were generated (e.g. `pydantic_v1`).
-    """
-
-    manifest_version: str
-    task_list: list[TaskManifestType]
-    has_args_schemas: bool = False
-    args_schema_version: Optional[str]
-
-    @root_validator()
-    def _check_args_schemas_are_present(cls, values):
-        has_args_schemas = values["has_args_schemas"]
-        task_list = values["task_list"]
-        if has_args_schemas:
-            for task in task_list:
-                if task.args_schema is None:
-                    raise ValueError(
-                        f'has_args_schemas={has_args_schemas} but task "'
-                        f'{task.name}" has args_schema={task.args_schema}.'
-                    )
-        return values
-
-
-class TaskManifestV1(_TaskManifestBaseV1):
-    """
-    Task manifest schema version 1.
-    """
-
-    pass
-
-
-class ManifestV1(_ManifestBaseV1):
-    """
-    Manifest schema version 1.
-
-    Attributes:
-        task_list:
-    """
-
-    task_list: list[TaskManifestV1]
-
-    @validator("manifest_version")
-    def manifest_version_1(cls, value):
-        if value != "1":
-            raise ValueError(f"Wrong manifest version (given {value})")