fractal-server 1.3.11__py3-none-any.whl → 1.3.12__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "1.3.11"
+__VERSION__ = "1.3.12"

fractal_server/app/api/v1/dataset.py

@@ -14,12 +14,12 @@ from ...db import AsyncSession
 from ...db import get_db
 from ...models import ApplyWorkflow
 from ...models import Dataset
-from ...models import DatasetStatusRead
 from ...models import JobStatusType
 from ...models import Resource
-from ...runner._common import METADATA_FILENAME
+from ...runner._common import HISTORY_FILENAME
 from ...schemas import DatasetCreate
 from ...schemas import DatasetRead
+from ...schemas import DatasetStatusRead
 from ...schemas import DatasetUpdate
 from ...schemas import ResourceCreate
 from ...schemas import ResourceRead
@@ -100,6 +100,13 @@ async def update_dataset(
     """
     Edit a dataset associated to the current project
     """
+
+    if dataset_update.history is not None:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Cannot modify dataset history.",
+        )
+
     output = await _get_dataset_check_owner(
         project_id=project_id,
         dataset_id=dataset_id,
@@ -350,7 +357,7 @@ async def export_history_as_workflow(
     # means that the history in the DB is up-to-date.
 
     # Read history from DB
-    history = dataset.meta.get("history", [])
+    history = dataset.history
 
     # Construct reproducible workflow
     task_list = []
@@ -432,7 +439,7 @@ async def get_workflowtask_status(
 
     # Lowest priority: read status from DB, which corresponds to jobs that are
     # not running
-    history = dataset.meta.get("history", [])
+    history = dataset.history
     for history_item in history:
         wftask_id = history_item["workflowtask"]["id"]
         wftask_status = history_item["status"]
@@ -457,10 +464,10 @@ async def get_workflowtask_status(
         # Highest priority: Read status updates coming from the running-job
         # temporary file. Note: this file only contains information on
         # WorkflowTask's that ran through successfully
-        tmp_file = Path(running_job.working_dir) / METADATA_FILENAME
+        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME
         try:
             with tmp_file.open("r") as f:
-                history = json.load(f).get("history", [])
+                history = json.load(f)
         except FileNotFoundError:
             history = []
         for history_item in history:

fractal_server/app/api/v1/job.py

@@ -1,4 +1,3 @@
-import json
 from io import BytesIO
 from pathlib import Path
 from typing import Optional
@@ -17,7 +16,6 @@ from ....syringe import Inject
 from ...db import AsyncSession
 from ...db import get_db
 from ...models import ApplyWorkflow
-from ...runner._common import METADATA_FILENAME
 from ...runner._common import SHUTDOWN_FILENAME
 from ...schemas import ApplyWorkflowRead
 from ...security import current_active_user
@@ -38,7 +36,7 @@ async def read_job(
     job_id: int,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_db),
-) -> Optional[ApplyWorkflow]:
+) -> Optional[ApplyWorkflowRead]:
     """
     Return info on an existing job
     """
@@ -51,19 +49,8 @@ async def read_job(
     )
     job = output["job"]
 
-    job_read = ApplyWorkflowRead(**job.dict())
-
-    # FIXME: this operation is not reading from the DB, but from file
-    try:
-        metadata_file = Path(job_read.working_dir) / METADATA_FILENAME
-        with metadata_file.open("r") as f:
-            metadata = json.load(f)
-        job_read.history = metadata["HISTORY_LEGACY"]
-    except (KeyError, FileNotFoundError):
-        pass
-
     await db.close()
-    return job_read
+    return job
 
 
 @router.get(

fractal_server/app/api/v1/project.py

@@ -18,7 +18,6 @@ from ...db import DBSyncSession
 from ...db import get_db
 from ...db import get_sync_db
 from ...models import ApplyWorkflow
-from ...models import Dataset
 from ...models import JobStatusType
 from ...models import LinkUserProject
 from ...models import Project
@@ -76,7 +75,6 @@ async def create_project(
     )
 
     db_project = Project.from_orm(project)
-    db_project.dataset_list.append(Dataset(name=project.default_dataset_name))
     db_project.user_list.append(user)
     try:
         db.add(db_project)

fractal_server/app/models/dataset.py

@@ -1,7 +1,6 @@
 from typing import Any
 from typing import Optional
 
-from pydantic import BaseModel
 from sqlalchemy import Column
 from sqlalchemy.types import JSON
 from sqlmodel import Field
@@ -10,7 +9,6 @@ from sqlmodel import SQLModel
 
 from ..schemas.dataset import _DatasetBase
 from ..schemas.dataset import _ResourceBase
-from .workflow import WorkflowTaskStatusType
 
 
 class Resource(_ResourceBase, SQLModel, table=True):
@@ -18,20 +16,6 @@ class Resource(_ResourceBase, SQLModel, table=True):
     dataset_id: int = Field(foreign_key="dataset.id")
 
 
-class DatasetStatusRead(BaseModel):
-    """
-    Response type for the
-    `/project/{project_id}/dataset/{dataset_id}/status/` endpoint
-    """
-
-    status: Optional[
-        dict[
-            int,
-            WorkflowTaskStatusType,
-        ]
-    ] = None
-
-
 class Dataset(_DatasetBase, SQLModel, table=True):
     """
     Represent a dataset
@@ -43,7 +27,8 @@ class Dataset(_DatasetBase, SQLModel, table=True):
             ID of the project the workflow belongs to.
         meta:
             Metadata of the Dataset
-
+        history:
+            History of the Dataset
         resource_list:
             (Mapper attribute)
 
@@ -58,6 +43,9 @@ class Dataset(_DatasetBase, SQLModel, table=True):
         }
     )
     meta: dict[str, Any] = Field(sa_column=Column(JSON), default={})
+    history: list[dict[str, Any]] = Field(
+        sa_column=Column(JSON, server_default="[]", nullable=False)
+    )
 
     class Config:
         arbitrary_types_allowed = True

fractal_server/app/models/workflow.py

@@ -1,4 +1,3 @@
-from enum import Enum
 from typing import Any
 from typing import Optional
 from typing import Union
@@ -175,21 +174,3 @@ class Workflow(_WorkflowBase, SQLModel, table=True):
     @property
     def output_type(self):
         return self.task_list[-1].task.output_type
-
-
-class WorkflowTaskStatusType(str, Enum):
-    """
-    Define the available values for the status of a `WorkflowTask`.
-
-    This kind of status is constructed in the
-    `api/v1/project/{project_id}/dataset/{dataset_id}/status` endpoint.
-
-    Attributes:
-        SUBMITTED: The `WorkflowTask` is part of a running job.
-        DONE: The most-recent execution of this `WorkflowTask` was successful.
-        FAILED: The most-recent execution of this `WorkflowTask` failed.
-    """
-
-    SUBMITTED = "submitted"
-    DONE = "done"
-    FAILED = "failed"

fractal_server/app/runner/__init__.py

@@ -4,6 +4,7 @@
 # Original authors:
 # Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
 # Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
 #
 # This file is part of Fractal and was originally developed by eXact lab S.r.l.
 # <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
@@ -225,11 +226,12 @@ async def submit_workflow(
         db_sync = next(DB.get_sync_db())
         db_sync.close()
 
-        output_dataset_meta = await process_workflow(
+        output_dataset_meta_hist = await process_workflow(
             workflow=workflow,
             input_paths=input_paths,
             output_path=output_path,
             input_metadata=input_dataset.meta,
+            input_history=input_dataset.history,
             slurm_user=slurm_user,
             user_cache_dir=user_cache_dir,
             workflow_dir=WORKFLOW_DIR,
@@ -249,15 +251,9 @@ async def submit_workflow(
         # Replace output_dataset.meta with output_dataset_meta, while handling
         # the history property in a special way (i.e. appending to and
         # existing entry rather than replacing it)
-        new_meta = {}
-        for key, value in output_dataset_meta.items():
-            if key != "history":
-                # For non-history keys, replace with new value
-                new_meta[key] = value
-            else:
-                # For history key, append to existing entry
-                new_meta[key] = output_dataset.meta.get(key, []) + value
-        output_dataset.meta = new_meta
+        output_dataset.history = output_dataset_meta_hist.pop("history")
+        output_dataset.meta = output_dataset_meta_hist.pop("metadata")
+
         db_sync.merge(output_dataset)
 
         # Update job DB entry
@@ -277,19 +273,18 @@ async def submit_workflow(
 
         # Assemble output_dataset.meta based on the last successful task, i.e.
         # based on METADATA_FILENAME
-        new_meta = assemble_meta_failed_job(job, output_dataset)
+        output_dataset.meta = assemble_meta_failed_job(job, output_dataset)
 
         # Assemble new history and assign it to output_dataset.meta
         failed_wftask = db_sync.get(WorkflowTask, e.workflow_task_id)
-        new_history = assemble_history_failed_job(
+        output_dataset.history = assemble_history_failed_job(
             job,
             output_dataset,
             workflow,
             logger,
             failed_wftask=failed_wftask,
         )
-        new_meta["history"] = new_history
-        output_dataset.meta = new_meta
+
         db_sync.merge(output_dataset)
 
         job.status = JobStatusType.FAILED
@@ -313,17 +308,16 @@ async def submit_workflow(
 
         # Assemble output_dataset.meta based on the last successful task, i.e.
         # based on METADATA_FILENAME
-        new_meta = assemble_meta_failed_job(job, output_dataset)
+        output_dataset.meta = assemble_meta_failed_job(job, output_dataset)
 
         # Assemble new history and assign it to output_dataset.meta
-        new_history = assemble_history_failed_job(
+        output_dataset.history = assemble_history_failed_job(
             job,
             output_dataset,
             workflow,
             logger,
         )
-        new_meta["history"] = new_history
-        output_dataset.meta = new_meta
+
         db_sync.merge(output_dataset)
 
         job.status = JobStatusType.FAILED
@@ -343,17 +337,16 @@ async def submit_workflow(
 
         # Assemble output_dataset.meta based on the last successful task, i.e.
         # based on METADATA_FILENAME
-        new_meta = assemble_meta_failed_job(job, output_dataset)
+        output_dataset.meta = assemble_meta_failed_job(job, output_dataset)
 
         # Assemble new history and assign it to output_dataset.meta
-        new_history = assemble_history_failed_job(
+        output_dataset.history = assemble_history_failed_job(
             job,
             output_dataset,
             workflow,
             logger,
         )
-        new_meta["history"] = new_history
-        output_dataset.meta = new_meta
+
         db_sync.merge(output_dataset)
 
         job.status = JobStatusType.FAILED

fractal_server/app/runner/_common.py

@@ -20,13 +20,13 @@ from typing import Optional
 
 from ...logger import get_logger
 from ..models import WorkflowTask
-from ..models import WorkflowTaskStatusType
+from ..schemas import WorkflowTaskStatusType
 from .common import JobExecutionError
 from .common import TaskExecutionError
 from .common import TaskParameters
 from .common import write_args_file
 
-
+HISTORY_FILENAME = "history.json"
 METADATA_FILENAME = "metadata.json"
 SHUTDOWN_FILENAME = "shutdown"
 
@@ -259,7 +259,7 @@ def call_single_task(
 
     # write args file (by assembling task_pars and wftask.args)
     write_args_file(
-        task_pars.dict(),
+        task_pars.dict(exclude={"history"}),
         wftask.args or {},
         path=task_files.args,
     )
@@ -296,17 +296,12 @@ def call_single_task(
     if diff_metadata is None:
         diff_metadata = {}
 
+    # Prepare updated_metadata
     updated_metadata = task_pars.metadata.copy()
     updated_metadata.update(diff_metadata)
 
-    # Assemble a TaskParameter object
-    HISTORY_LEGACY = f"{wftask.task.name}"
-    try:
-        updated_metadata["HISTORY_LEGACY"].append(HISTORY_LEGACY)
-    except KeyError:
-        updated_metadata["HISTORY_LEGACY"] = [HISTORY_LEGACY]
-
-    # Update history
+    # Prepare updated_history (note: the expected type for history items is
+    # defined in `_DatasetHistoryItem`)
     wftask_dump = wftask.dict(exclude={"task"})
     wftask_dump["task"] = wftask.task.dict()
     new_history_item = dict(
@@ -314,15 +309,15 @@ def call_single_task(
         status=WorkflowTaskStatusType.DONE,
         parallelization=None,
     )
-    try:
-        updated_metadata["history"].append(new_history_item)
-    except KeyError:
-        updated_metadata["history"] = [new_history_item]
+    updated_history = task_pars.history.copy()
+    updated_history.append(new_history_item)
 
+    # Assemble a TaskParameter object
     out_task_parameters = TaskParameters(
         input_paths=[task_pars.output_path],
         output_path=task_pars.output_path,
         metadata=updated_metadata,
+        history=updated_history,
     )
 
     return out_task_parameters
@@ -391,7 +386,7 @@ def call_single_parallel_task(
 
     # write args file (by assembling task_pars, wftask.args and component)
     write_args_file(
-        task_pars.dict(),
+        task_pars.dict(exclude={"history"}),
         wftask.args or {},
         dict(component=component),
         path=task_files.args,
@@ -538,37 +533,31 @@ def call_parallel_task(
             "future releases."
         )
 
-    # Assemble parallel task metadiff files
+    # Prepare updated_metadata
     updated_metadata = task_pars_depend.metadata.copy()
     updated_metadata.update(aggregated_metadata_update)
 
-    # Assemble a TaskParameter object
-    HISTORY_LEGACY = f"{wftask.task.name}: {component_list}"
-    try:
-        updated_metadata["HISTORY_LEGACY"].append(HISTORY_LEGACY)
-    except KeyError:
-        updated_metadata["HISTORY_LEGACY"] = [HISTORY_LEGACY]
-
-    # Update history
+    # Prepare updated_history (note: the expected type for history items is
+    # defined in `_DatasetHistoryItem`)
     wftask_dump = wftask.dict(exclude={"task"})
     wftask_dump["task"] = wftask.task.dict()
     new_history_item = dict(
         workflowtask=wftask_dump,
-        status="done",
+        status=WorkflowTaskStatusType.DONE,
         parallelization=dict(
             parallelization_level=wftask.parallelization_level,
             component_list=component_list,
         ),
     )
-    try:
-        updated_metadata["history"].append(new_history_item)
-    except KeyError:
-        updated_metadata["history"] = [new_history_item]
+    updated_history = task_pars_depend.history.copy()
+    updated_history.append(new_history_item)
 
+    # Assemble a TaskParameter object
     out_task_parameters = TaskParameters(
         input_paths=[task_pars_depend.output_path],
         output_path=task_pars_depend.output_path,
         metadata=updated_metadata,
+        history=updated_history,
     )
 
     return out_task_parameters
@@ -676,4 +665,8 @@ def execute_tasks(
         with open(workflow_dir / METADATA_FILENAME, "w") as f:
             json.dump(current_task_pars.metadata, f, indent=2)
 
+        # Write most recent metadata to HISTORY_FILENAME
+        with open(workflow_dir / HISTORY_FILENAME, "w") as f:
+            json.dump(current_task_pars.history, f, indent=2)
+
     return current_task_pars

fractal_server/app/runner/_local/__init__.py

@@ -38,6 +38,7 @@ def _process_workflow(
     input_paths: list[Path],
     output_path: Path,
     input_metadata: dict[str, Any],
+    input_history: list[dict[str, Any]],
     logger_name: str,
     workflow_dir: Path,
     first_task_index: int,
@@ -62,14 +63,17 @@ def _process_workflow(
                 input_paths=input_paths,
                 output_path=output_path,
                 metadata=input_metadata,
+                history=input_history,
             ),
             workflow_dir=workflow_dir,
             workflow_dir_user=workflow_dir,
             logger_name=logger_name,
             submit_setup_call=_local_submit_setup,
         )
-    output_dataset_metadata = output_task_pars.metadata
-    return output_dataset_metadata
+    output_dataset_metadata_history = dict(
+        metadata=output_task_pars.metadata, history=output_task_pars.history
+    )
+    return output_dataset_metadata_history
 
 
 async def process_workflow(
@@ -78,6 +82,7 @@ async def process_workflow(
     input_paths: list[Path],
     output_path: Path,
     input_metadata: dict[str, Any],
+    input_history: list[dict[str, Any]],
     logger_name: str,
     workflow_dir: Path,
     workflow_dir_user: Optional[Path] = None,
@@ -160,14 +165,15 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    output_dataset_metadata = await async_wrap(_process_workflow)(
+    output_dataset_metadata_history = await async_wrap(_process_workflow)(
         workflow=workflow,
         input_paths=input_paths,
         output_path=output_path,
         input_metadata=input_metadata,
+        input_history=input_history,
         logger_name=logger_name,
         workflow_dir=workflow_dir,
         first_task_index=first_task_index,
         last_task_index=last_task_index,
     )
-    return output_dataset_metadata
+    return output_dataset_metadata_history

fractal_server/app/runner/_slurm/__init__.py

@@ -4,6 +4,7 @@
 # Original authors:
 # Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
 # Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
 #
 # This file is part of Fractal and was originally developed by eXact lab S.r.l.
 # <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
@@ -35,6 +36,7 @@ def _process_workflow(
     input_paths: list[Path],
     output_path: Path,
     input_metadata: dict[str, Any],
+    input_history: list[dict[str, Any]],
     logger_name: str,
     workflow_dir: Path,
     workflow_dir_user: Path,
@@ -83,14 +85,17 @@ def _process_workflow(
                 input_paths=input_paths,
                 output_path=output_path,
                 metadata=input_metadata,
+                history=input_history,
             ),
             workflow_dir=workflow_dir,
             workflow_dir_user=workflow_dir_user,
             submit_setup_call=_slurm_submit_setup,
             logger_name=logger_name,
         )
-    output_dataset_metadata = output_task_pars.metadata
-    return output_dataset_metadata
+    output_dataset_metadata_history = dict(
+        metadata=output_task_pars.metadata, history=output_task_pars.history
+    )
+    return output_dataset_metadata_history
 
 
 async def process_workflow(
@@ -99,6 +104,7 @@ async def process_workflow(
     input_paths: list[Path],
     output_path: Path,
     input_metadata: dict[str, Any],
+    input_history: list[dict[str, Any]],
     logger_name: str,
     workflow_dir: Path,
     workflow_dir_user: Optional[Path] = None,
@@ -122,11 +128,12 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    output_dataset_metadata = await async_wrap(_process_workflow)(
+    output_dataset_metadata_history = await async_wrap(_process_workflow)(
         workflow=workflow,
         input_paths=input_paths,
         output_path=output_path,
         input_metadata=input_metadata,
+        input_history=input_history,
         logger_name=logger_name,
         workflow_dir=workflow_dir,
         workflow_dir_user=workflow_dir_user,
@@ -136,4 +143,4 @@ async def process_workflow(
         first_task_index=first_task_index,
         last_task_index=last_task_index,
     )
-    return output_dataset_metadata
+    return output_dataset_metadata_history

fractal_server/app/runner/_slurm/_executor_wait_thread.py

@@ -108,6 +108,8 @@ class FractalSlurmWaitThread(FractalFileWaitThread):
     The function is copied from clusterfutures 0.5. Original Copyright: 2022
     Adrian Sampson, released under the MIT licence
 
+    **Note**: if `self.interval != 1` then this should be modified, but for
+    `clusterfutures` v0.5 `self.interval` is indeed equal to `1`.
 
     Changed from clusterfutures:
     * Rename `id_to_filename` to `id_to_filenames`

fractal_server/app/runner/common.py

@@ -176,11 +176,15 @@ class TaskParameters(BaseModel):
         metadata:
             Dataset metadata, as found in the input dataset or as updated by
             the previous task.
+        history:
+            Dataset history, as found in the input dataset or as updated by
+            the previous task.
     """
 
     input_paths: list[Path]
     output_path: Path
     metadata: dict[str, Any]
+    history: list[dict[str, Any]]
 
     class Config:
         arbitrary_types_allowed = True

fractal_server/app/runner/handle_failed_job.py

@@ -3,6 +3,7 @@
 #
 # Original authors:
 # Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
 #
 # This file is part of Fractal and was originally developed by eXact lab S.r.l.
 # <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
@@ -23,6 +24,7 @@ from ..models import Dataset
 from ..models import Workflow
 from ..models import WorkflowTask
 from ..models import WorkflowTaskStatusType
+from ._common import HISTORY_FILENAME
 from ._common import METADATA_FILENAME
 
 
@@ -59,14 +61,13 @@ def assemble_history_failed_job(
     # information.
 
     # Part 1: Read exising history from DB
-    new_history = output_dataset.meta.get("history", [])
+    new_history = output_dataset.history
 
     # Part 2: Extend history based on tmp_metadata_file
-    tmp_metadata_file = Path(job.working_dir) / METADATA_FILENAME
+    tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME
     try:
-        with tmp_metadata_file.open("r") as f:
-            tmp_file_meta = json.load(f)
-            tmp_file_history = tmp_file_meta.get("history", [])
+        with tmp_history_file.open("r") as f:
+            tmp_file_history = json.load(f)
             new_history.extend(tmp_file_history)
     except FileNotFoundError:
         tmp_file_history = []

fractal_server/app/schemas/__init__.py

@@ -5,6 +5,7 @@ from .applyworkflow import ApplyWorkflowCreate  # noqa: F401
 from .applyworkflow import ApplyWorkflowRead  # noqa: F401
 from .dataset import DatasetCreate  # noqa: F401
 from .dataset import DatasetRead  # noqa: F401
+from .dataset import DatasetStatusRead  # noqa: F401
 from .dataset import DatasetUpdate  # noqa: F401
 from .dataset import ResourceCreate  # noqa: F401
 from .dataset import ResourceRead  # noqa: F401
@@ -32,5 +33,6 @@ from .workflow import WorkflowTaskCreate  # noqa: F401
 from .workflow import WorkflowTaskExport  # noqa: F401
 from .workflow import WorkflowTaskImport  # noqa: F401
 from .workflow import WorkflowTaskRead  # noqa: F401
+from .workflow import WorkflowTaskStatusType  # noqa: F401
 from .workflow import WorkflowTaskUpdate  # noqa: F401
 from .workflow import WorkflowUpdate  # noqa: F401

fractal_server/app/schemas/applyworkflow.py

@@ -107,7 +107,6 @@ class ApplyWorkflowRead(_ApplyWorkflowBase):
     status: str
     log: Optional[str]
     workflow_dump: Optional[dict[str, Any]]
-    history: Optional[list[str]]
     working_dir: Optional[str]
     working_dir_user: Optional[str]
     first_task_index: Optional[int]

fractal_server/app/schemas/dataset.py

@@ -7,6 +7,8 @@ from pydantic import validator
 
 from ._validators import val_absolute_path
 from ._validators import valstr
+from .workflow import WorkflowTaskRead
+from .workflow import WorkflowTaskStatusType
 
 
 __all__ = (
@@ -16,6 +18,7 @@ __all__ = (
     "ResourceCreate",
     "ResourceRead",
     "ResourceUpdate",
+    "DatasetStatusRead",
 )
 
 
@@ -61,6 +64,22 @@ class ResourceRead(_ResourceBase):
     dataset_id: int
 
 
+class _DatasetHistoryItem(BaseModel):
+    """
+    Class for an item of `Dataset.history`.
+
+    Attributes:
+        workflowtask:
+        status:
+        parallelization: If provided, it includes keys `parallelization_level`
+            and `component_list`.
+    """
+
+    workflowtask: WorkflowTaskRead
+    status: WorkflowTaskStatusType
+    parallelization: Optional[dict]
+
+
 class _DatasetBase(BaseModel):
     """
     Base class for `Dataset`.
@@ -69,12 +88,14 @@ class _DatasetBase(BaseModel):
         name:
         type:
         meta:
+        history:
         read_only:
     """
 
     name: str
     type: Optional[str]
     meta: dict[str, Any] = Field(default={})
+    history: list[_DatasetHistoryItem] = Field(default=[])
     read_only: bool = False
 
 
@@ -85,11 +106,13 @@ class DatasetUpdate(_DatasetBase):
     Attributes:
         name:
         meta:
+        history:
         read_only:
     """
 
     name: Optional[str]
     meta: Optional[dict[str, Any]] = None
+    history: Optional[list[_DatasetHistoryItem]] = None
     read_only: Optional[bool]
 
     # Validators
@@ -122,3 +145,17 @@ class DatasetRead(_DatasetBase):
     resource_list: list[ResourceRead]
     project_id: int
     read_only: bool
+
+
+class DatasetStatusRead(BaseModel):
+    """
+    Response type for the
+    `/project/{project_id}/dataset/{dataset_id}/status/` endpoint
+    """
+
+    status: Optional[
+        dict[
+            int,
+            WorkflowTaskStatusType,
+        ]
+    ] = None

fractal_server/app/schemas/project.py

@@ -30,18 +30,10 @@ class _ProjectBase(BaseModel):
 class ProjectCreate(_ProjectBase):
     """
     Class for `Project` creation.
-
-    Attributes:
-        default_dataset_name:
     """
 
-    default_dataset_name: Optional[str] = "default"
-
     # Validators
     _name = validator("name", allow_reuse=True)(valstr("name"))
-    _default_dataset_name = validator(
-        "default_dataset_name", allow_reuse=True
-    )(valstr("default_dataset_name"))
 
 
 class ProjectRead(_ProjectBase):

fractal_server/app/schemas/workflow.py

@@ -1,3 +1,4 @@
+from enum import Enum
 from typing import Any
 from typing import Optional
 
@@ -22,6 +23,7 @@ __all__ = (
     "WorkflowTaskExport",
     "WorkflowTaskRead",
     "WorkflowTaskUpdate",
+    "WorkflowTaskStatusType",
 )
 
 
@@ -185,3 +187,22 @@ class WorkflowExport(_WorkflowBase):
     """
 
     task_list: list[WorkflowTaskExport]
+
+
+class WorkflowTaskStatusType(str, Enum):
+    """
+    Define the available values for the status of a `WorkflowTask`.
+
+    This model is used within the `Dataset.history` attribute, which is
+    constructed in the runner and then used in the API (e.g. in the
+    `api/v1/project/{project_id}/dataset/{dataset_id}/status` endpoint).
+
+    Attributes:
+        SUBMITTED: The `WorkflowTask` is part of a running job.
+        DONE: The most-recent execution of this `WorkflowTask` was successful.
+        FAILED: The most-recent execution of this `WorkflowTask` failed.
+    """
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"

fractal_server/config.py

@@ -419,4 +419,9 @@ class Settings(BaseSettings):
 
 
 def get_settings(settings=Settings()) -> Settings:
+    logging.debug("Fractal Settings:")
+    for key, value in settings.dict().items():
+        if any(s in key.upper() for s in ["PASSWORD", "SECRET"]):
+            value = "*****"
+        logging.debug(f"{key}: {value}")
     return settings

fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py

@@ -1,4 +1,4 @@
-"""Add task.args_schema and task.args_schema_version
+"""Add Task.args_schema and Task.args_schema_version
 
 Revision ID: 4c308bcaea2b
 Revises: 50a13d6138fd

fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py

@@ -1,4 +1,4 @@
-"""add docs_info and docs_link to Task model
+"""Add Task.docs_info and Task.docs_link
 
 Revision ID: 8f79bd162e35
 Revises: a7f4d6137b53

fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py

@@ -0,0 +1,36 @@
+"""Add Dataset.history
+
+Revision ID: 99ea79d9e5d2
+Revises: 8f79bd162e35
+Create Date: 2023-10-16 09:45:15.132185
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "99ea79d9e5d2"
+down_revision = "8f79bd162e35"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("dataset", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "history", sa.JSON(), server_default="[]", nullable=False
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("dataset", schema=None) as batch_op:
+        batch_op.drop_column("history")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py

@@ -1,4 +1,4 @@
-"""add workflow_dump to applyworkflow
+"""Add ApplyWorkflow.workflow_dump
 
 Revision ID: a7f4d6137b53
 Revises: 70e77f1c38b0

fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py

@@ -1,4 +1,4 @@
-"""Drop task.default_args columns
+"""Drop Task.default_args columns
 
 Revision ID: f384e1c0cf5d
 Revises: 4c308bcaea2b

fractal_server/tasks/collection.py

@@ -83,7 +83,7 @@ def get_absolute_venv_path(venv_path: Path) -> Path:
         package_path = venv_path
     else:
         settings = Inject(get_settings)
-        package_path = settings.FRACTAL_TASKS_DIR / venv_path  # type: ignore
+        package_path = settings.FRACTAL_TASKS_DIR / venv_path
     return package_path
 
 
@@ -214,7 +214,7 @@ def create_package_dir_pip(
             "with `version=None`."
         )
     package_dir = f"{task_pkg.package}{task_pkg.package_version}"
-    venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir  # type: ignore
+    venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir
     if create:
         venv_path.mkdir(exist_ok=False, parents=True)
     return venv_path

{fractal_server-1.3.11.dist-info → fractal_server-1.3.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 1.3.11
+Version: 1.3.12
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

{fractal_server-1.3.11.dist-info → fractal_server-1.3.12.dist-info}/RECORD

@@ -1,76 +1,77 @@
-fractal_server/__init__.py,sha256=VtL-dm6VymRHtb_5Z7kV4OrXXxjsGG8wHiz-7nekw2k,23
+fractal_server/__init__.py,sha256=qaJUIaR-bHaNPEu4hp4VMdOkjJr1Mo6mAbuEBNVfiF8,23
 fractal_server/__main__.py,sha256=znijcImbcEC4P26ICOhEJ9VY3_5vWdMwQcl-WP25sYA,2202
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/api/__init__.py,sha256=_g30kAzDmakaxQQYwSZwVbid1O-3zMzQqnSuQZOWI3U,1192
 fractal_server/app/api/v1/__init__.py,sha256=2HMymr1YkUk39V8iof7KENyLnre4ghouOSvNZ_kF1ec,24
 fractal_server/app/api/v1/_aux_functions.py,sha256=_-D-iTKvOfH1yufa4lIHvP88Sgic8cvKXq3GmDHC-lk,9659
-fractal_server/app/api/v1/dataset.py,sha256=G7mZdlkMO6c5Ig5-LuZuWah0oI8RKT8dpQthAed-j4Y,14353
-fractal_server/app/api/v1/job.py,sha256=IFxuxhbY6kX5JJQUbfP2L2zqLS3-Gd68qKHpVSBW2dI,4998
-fractal_server/app/api/v1/project.py,sha256=DQfxtySImmfhLAO0t7hcJTiAF6XlHC3erL3rUtNObiQ,10345
+fractal_server/app/api/v1/dataset.py,sha256=lJTuYoQpOFWyw_f5WhJK88JRKcUWzFiFmPvyUfrv2PM,14500
+fractal_server/app/api/v1/job.py,sha256=k1TxhmU_VMiHb6RxJUsiVaLWcH5GvtwsB44DsnmxNmc,4550
+fractal_server/app/api/v1/project.py,sha256=LlNkkKpJ7WtEcMOQ1KzGwxYqxScUex-wTlu1x1KL4rs,10236
 fractal_server/app/api/v1/task.py,sha256=0MJNhn5f8KZy4XBMUoJNhrk3E6GBQWcVfKoQzP5XXWw,5582
 fractal_server/app/api/v1/task_collection.py,sha256=mY1cSGepWvVz6IJCnFYA8iy4hU-8qsA1HbiQXZjg1OM,11697
 fractal_server/app/api/v1/workflow.py,sha256=A54m9PDD7MhT6s5-8kIrefEJ5uVxigxBohelT7XCLVo,9379
 fractal_server/app/api/v1/workflowtask.py,sha256=TIsCSBFImoRq0rz16ZVlFwTL-Qd9Uqywbq-DT4OxYh0,5421
 fractal_server/app/db/__init__.py,sha256=g2KS-KxWL6CheZqa8nsUI3_Hw5mGVz_geUWCNY102XM,3109
 fractal_server/app/models/__init__.py,sha256=RuxWH8fsmkTWsjLhYjrxSt-mvk74coCilAQlX2Q6OO0,353
-fractal_server/app/models/dataset.py,sha256=kTwr5OOWrpUEjz2wfVSwy9B9_eBDUBCLi0DoPitLzFc,1649
+fractal_server/app/models/dataset.py,sha256=TK2-tnEMfC3lAwSJMMJre-e2Ao6AbEqnhWTcVvcrJxo,1482
 fractal_server/app/models/job.py,sha256=eU1RIWg4C2n9BzjGdvOxqP-eueNNYdo6tpW2RBqZYrE,3753
 fractal_server/app/models/linkuserproject.py,sha256=RVtl25Q_N99uoVDE7wx0IN0SgFjc7Id5XbScsgrjv_E,309
 fractal_server/app/models/project.py,sha256=U3MvLVBestug5dCCw22VA-l8BeUnGoaNye5OPexsqoA,1191
 fractal_server/app/models/security.py,sha256=Dp54Hf7I72oo9PERdyR0_zStw2ppYlFVi5MhFWIE6Lw,2438
 fractal_server/app/models/state.py,sha256=0L4lcaaMyQE43Tk1DVYbvPFcsQ9OZDSzsEwJO-vLrKA,1111
 fractal_server/app/models/task.py,sha256=APndtea9A7EF7TtpVK8kWapBM01a6nk3FFCrQbbioI8,2632
-fractal_server/app/models/workflow.py,sha256=UtSVZ1dnlwDjmIZMKKpogSxSPqw54G-pC-hKWD-8Fbw,5761
+fractal_server/app/models/workflow.py,sha256=VlX-MNfqw3z-EVKMvwVR9HbnOFNHRnO-5sDYFQOSijQ,5191
 fractal_server/app/runner/.gitignore,sha256=ytzN_oyHWXrGU7iFAtoHSTUbM6Rn6kG0Zkddg0xZk6s,16
-fractal_server/app/runner/__init__.py,sha256=cUjtQCpu-zbAh_UvIjtrZP9ilLO5zezX3GKgEO4rLtA,13370
-fractal_server/app/runner/_common.py,sha256=aEZsG6_smDggoezGx-aU7eBXoEd5Tp-itjpPbdmEORM,22620
-fractal_server/app/runner/_local/__init__.py,sha256=G0mjpic_m-3UqZX-52nygodz-BudiErrXe6I0nwO2L8,6279
+fractal_server/app/runner/__init__.py,sha256=tdRkLDLZmfiMZbDpeDZiMqewIpfP58--IlERMVp33bI,13039
+fractal_server/app/runner/_common.py,sha256=XjyE8DZE6WECeFXI6i0vHVD6JywZQxkJgZrL-ep1USQ,22642
+fractal_server/app/runner/_local/__init__.py,sha256=mSJzpF6u6rgsSYO25szNVr2B296h7_iKD1eqS3o87Qo,6532
 fractal_server/app/runner/_local/_local_config.py,sha256=-oNTsjEUmytHlsYpWfw2CrPvSxDFeEhZSdQvI_wf3Mk,3245
 fractal_server/app/runner/_local/_submit_setup.py,sha256=cP4gjQ_3TFgqglscQacp9dB3aqeXup5tVqqWE7TZl9Q,1631
 fractal_server/app/runner/_local/executor.py,sha256=QrJlD77G6q4WohoJQO7XXbvi2RlCUsNvMnPDEZIoAqA,3620
 fractal_server/app/runner/_slurm/.gitignore,sha256=ytzN_oyHWXrGU7iFAtoHSTUbM6Rn6kG0Zkddg0xZk6s,16
-fractal_server/app/runner/_slurm/__init__.py,sha256=CH0pPbHV7kZ6soCxGLPUD0YoxDfsl7Iu-dFTQ4nLbRU,4396
+fractal_server/app/runner/_slurm/__init__.py,sha256=SjCzSjlf8_aVZ6jw1vUZZVV5GKp92_G7Ui_1RyL6dV0,4694
 fractal_server/app/runner/_slurm/_batching.py,sha256=KE4NrLXRHFZQSLW2vbUyu0X7TE7bTd2WCRrbYhXRTow,8840
-fractal_server/app/runner/_slurm/_executor_wait_thread.py,sha256=pmtCHeaPzNg7Q82E6KfCaUho0Bh7V1y6k1yNTYzz8Lk,4260
+fractal_server/app/runner/_slurm/_executor_wait_thread.py,sha256=ZGwquq2UHCr84f-b5gH14cmRFgJHB7pYwQSeRkIzxcA,4402
 fractal_server/app/runner/_slurm/_slurm_config.py,sha256=pPyduXQSOzPEy_fJxrE8NE9hzXDLZ3NmWAIhv2OMCH8,20562
 fractal_server/app/runner/_slurm/_submit_setup.py,sha256=JIPmZEqyLRByQ3SgqiyocQlsHjfm0wKCk7W-KRBGu_0,2930
 fractal_server/app/runner/_slurm/_subprocess_run_as_user.py,sha256=so0S9TRxHlVZoOTGbeZtklIErJatGbRRMXcD8F4jcv4,3862
 fractal_server/app/runner/_slurm/executor.py,sha256=ao5YuWtjsIfTYUucE1SvNS8a99Sgzm01RLLu1wLjP0Y,41983
 fractal_server/app/runner/_slurm/remote.py,sha256=wLziIsGdSMiO-jIXM8x77JRK82g_2hx0iBKTiMghuIo,5852
-fractal_server/app/runner/common.py,sha256=7y5X7V7JbZzYLRsiQCQHaT1AVpJ2ACed2i8db9NMVBI,9338
-fractal_server/app/runner/handle_failed_job.py,sha256=Bx14jMse_TfuJxRsinSfkxQl62knfOGBNriai4wyefw,4638
-fractal_server/app/schemas/__init__.py,sha256=ZkX3KT9lbxKBS92edxFsypH6nkjwbsLsNdbOGa3zchk,1692
+fractal_server/app/runner/common.py,sha256=nz0ZuIro0iwZm-OV-e-Y-PrtgKcLK0d7BrzebWyEWEk,9496
+fractal_server/app/runner/handle_failed_job.py,sha256=PKgJARHjXyv33sDsl7oTINdcTu7EwmFmIkp38RqAE3Q,4641
+fractal_server/app/schemas/__init__.py,sha256=Roc1gkp3jsafkgcsY8u5S6nshj9ER9fuAkthplrX4kI,1804
 fractal_server/app/schemas/_validators.py,sha256=7YEbgrnGRpzkLMfZzQNfczEmcNnO__SmVOaBHhzaiXE,1819
-fractal_server/app/schemas/applyworkflow.py,sha256=B9_pGOjp-yAZs7eT6hWOkFQOCtmh-_TrNr9ZS1eR948,2987
-fractal_server/app/schemas/dataset.py,sha256=i3TPN5ieT1pkLa5T68vRpuLTkRt3myXOOrvxr0JbG2I,2200
+fractal_server/app/schemas/applyworkflow.py,sha256=gGlVSfYZlVdWRhIhQ_0tzBKfuW1y1-ZeIS5UvjpXZTM,2954
+fractal_server/app/schemas/dataset.py,sha256=PPqGTsRQ5JEwkiM4NcjPYFckxnCdi_Zov-bWXDm1LUk,3092
 fractal_server/app/schemas/manifest.py,sha256=xxTd39dAXMK9Ox1y-p3gbyg0zd5udW99pV4JngCUGwM,3819
-fractal_server/app/schemas/project.py,sha256=fZmd5t9G4A-is_U5iuzA0wgjBdjFCVaxQtKB_yoZk-M,1273
+fractal_server/app/schemas/project.py,sha256=GoV1yUPVSJ7eFGXIBrYh_4FOVKYcBRzPbW7ImnBAg-4,1047
 fractal_server/app/schemas/state.py,sha256=CS8Rs5qF21TsnqmyzUHLqRaX1b61Oc6Yra6POYpYSQY,762
 fractal_server/app/schemas/task.py,sha256=2TBE5Ne9tO_-a2-Es0PRXMT8ZddSInTOPMor7u8-gx0,3671
 fractal_server/app/schemas/task_collection.py,sha256=mPk6E1LK2UvnHkhIQWHmTztsVT99iHZn-UZy7mGNjUk,2965
 fractal_server/app/schemas/user.py,sha256=zhB-2WfJ30hNcHaW2V126v5i7rHl66fX_SRmIWCQrjM,1587
-fractal_server/app/schemas/workflow.py,sha256=fNWroSxludGzWk6eBGpSxisydaqxE_JE8UNcmvjgQng,3543
+fractal_server/app/schemas/workflow.py,sha256=oFoO62JH5hfMJjKoicdpyC5hd2O9XgqoAm5RN9YjXAI,4238
 fractal_server/app/security/__init__.py,sha256=DCUIaIgzatnmtABAO4bR9jISVSoGowHlIQIHuV5oLUU,10880
-fractal_server/config.py,sha256=Iy_Qe64a9PtUSWOw1BW7THkak9ToxgmvmLVb1CtkK6M,13905
+fractal_server/config.py,sha256=UF9K_IfJU7-VPjCJbMDcdV11_DsmMxqq4ezTNBQASq8,14126
 fractal_server/logger.py,sha256=keri8i960WHT8Zz9Rm2MwfnrA2dw9TsrfCmojqtGDLs,4562
 fractal_server/main.py,sha256=9_T_cMqf0EfbfYwkYhKeU36v9PFi95BoydapKpmaTKc,5932
 fractal_server/migrations/README,sha256=4rQvyDfqodGhpJw74VYijRmgFP49ji5chyEemWGHsuw,59
 fractal_server/migrations/env.py,sha256=05EoWw0p43ojTNiz7UVG4lsl057B4ImSgXiHmiU-M80,2690
 fractal_server/migrations/script.py.mako,sha256=27QIow4iwANuPt6USRnqKXe8NX3POsWO0qT007P1HFU,550
-fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py,sha256=b2fdv2aEPjrFDZQdrxjxO07Ri10IOSGo3UBx76SdGNg,954
+fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py,sha256=-wHe-fOffmYeAm0JXVl_lxZ7hhDkaEVqxgxpHkb_uL8,954
 fractal_server/migrations/versions/50a13d6138fd_initial_schema.py,sha256=zwXegXs9J40eyCWi3w0c_iIBVJjXNn4VdVnQaT3KxDg,8770
 fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py,sha256=Q-DsMzG3IcUV2Ol1dhJWosDvKERamBE6QvA2zzS5zpQ,1632
-fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py,sha256=MLv90z2ZiJTAfKDbIUjVTRAQUOPuXmsa6PGUWwWG-Zo,1119
-fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py,sha256=rhdVOvUGukstA_XwdPrqbZSJLqsUVFLZP5zGVAvGnWU,870
-fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py,sha256=MLcJYkRIa4wH681kEi2afkbDhjpU13QJT4V6X_hhSQU,746
+fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py,sha256=6pgODDtyAxevZvAJBj9IJ41inhV1RpwbpZr_qfPPu1A,1115
+fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py,sha256=0im6TxDr53sKKcjiPgeH4ftVRGnRXZSh2lPbRQ1Ir9w,883
+fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py,sha256=ekDUML7ILpmdoqEclKbEUdyLi4uw9HSG_sTjG2hp_JE,867
+fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py,sha256=9BwqUS9Gf7UW_KjrzHbtViC880qhD452KAytkHWWZyk,746
 fractal_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/syringe.py,sha256=3qSMW3YaMKKnLdgnooAINOPxnCOxP7y2jeAQYB21Gdo,2786
 fractal_server/tasks/__init__.py,sha256=Wzuxf5EoH1v0fYzRpAZHG_S-Z9f6DmbIsuSvllBCGvc,72
-fractal_server/tasks/collection.py,sha256=fC8_4w19fA1MXhRGzy-Z-MwJvF-8w89ZAFIIpeKA2vE,17659
+fractal_server/tasks/collection.py,sha256=POKvQyS5G5ySybH0r0v21I_ZQ5AREe9kAqr_uFfGyaU,17627
 fractal_server/utils.py,sha256=b7WwFdcFZ8unyT65mloFToYuEDXpQoHRcmRNqrhd_dQ,2115
-fractal_server-1.3.11.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-1.3.11.dist-info/METADATA,sha256=kzxp9qiSqYoqNby5lIU0WA6EgXDi0-yaFaUxdwd4r6k,3836
-fractal_server-1.3.11.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-fractal_server-1.3.11.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-1.3.11.dist-info/RECORD,,
+fractal_server-1.3.12.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-1.3.12.dist-info/METADATA,sha256=Ak6e_L9Tth8H31dMN-PU3lbagRjYwMcD0YImfn4LhmI,3836
+fractal_server-1.3.12.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+fractal_server-1.3.12.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-1.3.12.dist-info/RECORD,,