fractal-server 2.14.0a8__py3-none-any.whl → 2.14.0a10__py3-none-any.whl

fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -6,6 +6,7 @@ import shlex
 import subprocess  # nosec
 import sys
 import time
+from copy import copy
 from pathlib import Path
 from typing import Any
 from typing import Optional
@@ -14,7 +15,9 @@ import cloudpickle
 from pydantic import BaseModel
 from pydantic import ConfigDict
 
-from ._check_jobs_status import get_finished_jobs
+from ..slurm_common._check_jobs_status import (
+    get_finished_jobs,
+)
 from ._subprocess_run_as_user import _mkdir_as_user
 from ._subprocess_run_as_user import _run_command_as_user
 from fractal_server import __VERSION__
@@ -132,6 +135,14 @@ class SlurmJob(BaseModel):
             self.workdir_remote / f"slurm-{self.label}-submit.sh"
         ).as_posix()
 
+    @property
+    def slurm_stdout(self) -> str:
+        return (self.workdir_remote / f"slurm-{self.label}.out").as_posix()
+
+    @property
+    def slurm_stderr(self) -> str:
+        return (self.workdir_remote / f"slurm-{self.label}.err").as_posix()
+
     @property
     def log_files_local(self) -> list[str]:
         return [task.task_files.log_file_local for task in self.tasks]
@@ -276,9 +287,6 @@ class RunnerSlurmSudo(BaseRunner):
         slurm_job: SlurmJob,
         slurm_config: SlurmConfig,
     ) -> str:
-        # if len(slurm_job.tasks) > 1:
-        #     raise NotImplementedError()
-
         # Prepare input pickle(s)
         versions = dict(
             python=sys.version_info[:3],
@@ -291,40 +299,76 @@ class RunnerSlurmSudo(BaseRunner):
             funcser = cloudpickle.dumps((versions, func, _args, _kwargs))
             with open(task.input_pickle_file_local, "wb") as f:
                 f.write(funcser)
-
         # Prepare commands to be included in SLURM submission script
-
-        preamble_lines = [
-            "#!/bin/bash",
-            "#SBATCH --partition=main",
-            "#SBATCH --ntasks=1",
-            "#SBATCH --cpus-per-task=1",
-            "#SBATCH --mem=10M",
-            f"#SBATCH --err={slurm_job.slurm_log_file_remote}",
-            f"#SBATCH --out={slurm_job.slurm_log_file_remote}",
-            f"#SBATCH -D {slurm_job.workdir_remote}",
-            "#SBATCH --job-name=test",
-            "\n",
-        ]
-
+        settings = Inject(get_settings)
+        python_worker_interpreter = (
+            settings.FRACTAL_SLURM_WORKER_PYTHON or sys.executable
+        )
         cmdlines = []
         for task in slurm_job.tasks:
-            cmd = (
-                f"{self.python_worker_interpreter}"
-                " -m fractal_server.app.runner.executors.slurm_common.remote "
-                f"--input-file {task.input_pickle_file_local} "
-                f"--output-file {task.output_pickle_file_remote}"
-            )
-            cmdlines.append("whoami")
+            input_pickle_file = task.input_pickle_file_local
+            output_pickle_file = task.output_pickle_file_remote
             cmdlines.append(
-                f"srun --ntasks=1 --cpus-per-task=1 --mem=10MB {cmd} &"
+                (
+                    f"{python_worker_interpreter}"
+                    " -m fractal_server.app.runner."
+                    "executors.slurm_common.remote "
+                    f"--input-file {input_pickle_file} "
+                    f"--output-file {output_pickle_file}"
+                )
             )
-        cmdlines.append("wait\n")
+
+        # ...
+        num_tasks_max_running = slurm_config.parallel_tasks_per_job
+        mem_per_task_MB = slurm_config.mem_per_task_MB
+
+        # Set ntasks
+        ntasks = min(len(cmdlines), num_tasks_max_running)
+        slurm_config.parallel_tasks_per_job = ntasks
+
+        # Prepare SLURM preamble based on SlurmConfig object
+        script_lines = slurm_config.to_sbatch_preamble(
+            remote_export_dir=self.user_cache_dir
+        )
+
+        # Extend SLURM preamble with variable which are not in SlurmConfig, and
+        # fix their order
+        script_lines.extend(
+            [
+                f"#SBATCH --err={slurm_job.slurm_stderr}",
+                f"#SBATCH --out={slurm_job.slurm_stdout}",
+                f"#SBATCH -D {slurm_job.workdir_remote}",
+            ]
+        )
+        script_lines = slurm_config.sort_script_lines(script_lines)
+        logger.debug(script_lines)
+
+        # Always print output of `uname -n` and `pwd`
+        script_lines.append(
+            '"Hostname: `uname -n`; current directory: `pwd`"\n'
+        )
+
+        # Complete script preamble
+        script_lines.append("\n")
+
+        # Include command lines
+        tmp_list_commands = copy(cmdlines)
+        while tmp_list_commands:
+            if tmp_list_commands:
+                cmd = tmp_list_commands.pop(0)  # take first element
+                script_lines.append(
+                    "srun --ntasks=1 --cpus-per-task=$SLURM_CPUS_PER_TASK "
+                    f"--mem={mem_per_task_MB}MB "
+                    f"{cmd} &"
+                )
+        script_lines.append("wait\n")
+
+        script = "\n".join(script_lines)
 
         # Write submission script
-        submission_script_contents = "\n".join(preamble_lines + cmdlines)
+        # submission_script_contents = "\n".join(preamble_lines + cmdlines)
         with open(slurm_job.slurm_submission_script_local, "w") as f:
-            f.write(submission_script_contents)
+            f.write(script)
 
         # Run sbatch
         pre_command = f"sudo --set-home --non-interactive -u {self.slurm_user}"
@@ -468,6 +512,8 @@ class RunnerSlurmSudo(BaseRunner):
                 )
             ],
         )  # TODO: replace with actual values (BASED ON TASKFILES)
+
+        slurm_config.parallel_tasks_per_job = 1
         self._submit_single_sbatch(
             func,
             slurm_job=slurm_job,

fractal_server/app/runner/v2/__init__.py

@@ -299,6 +299,7 @@ def submit_workflow(
             first_task_index=job.first_task_index,
             last_task_index=job.last_task_index,
             job_attribute_filters=job.attribute_filters,
+            job_type_filters=job.type_filters,
             **backend_specific_kwargs,
         )
 

fractal_server/app/runner/v2/_local.py

@@ -20,6 +20,7 @@ def process_workflow(
     last_task_index: Optional[int] = None,
     logger_name: str,
     job_attribute_filters: AttributeFiltersType,
+    job_type_filters: dict[str, bool],
     user_id: int,
     **kwargs,
 ) -> None:
@@ -80,5 +81,6 @@ def process_workflow(
             logger_name=logger_name,
             submit_setup_call=_local_submit_setup,
             job_attribute_filters=job_attribute_filters,
+            job_type_filters=job_type_filters,
             user_id=user_id,
         )

fractal_server/app/runner/v2/_slurm_ssh.py

@@ -43,6 +43,7 @@ def process_workflow(
     last_task_index: Optional[int] = None,
     logger_name: str,
     job_attribute_filters: AttributeFiltersType,
+    job_type_filters: dict[str, bool],
     fractal_ssh: FractalSSH,
     worker_init: Optional[str] = None,
     user_id: int,
@@ -92,5 +93,6 @@ def process_workflow(
             logger_name=logger_name,
             submit_setup_call=_slurm_submit_setup,
             job_attribute_filters=job_attribute_filters,
+            job_type_filters=job_type_filters,
             user_id=user_id,
         )

fractal_server/app/runner/v2/_slurm_sudo.py

@@ -38,6 +38,7 @@ def process_workflow(
     last_task_index: Optional[int] = None,
     logger_name: str,
     job_attribute_filters: AttributeFiltersType,
+    job_type_filters: dict[str, bool],
     user_id: int,
     # Slurm-specific
     user_cache_dir: Optional[str] = None,
@@ -84,5 +85,6 @@ def process_workflow(
             logger_name=logger_name,
             submit_setup_call=_slurm_submit_setup,
             job_attribute_filters=job_attribute_filters,
+            job_type_filters=job_type_filters,
             user_id=user_id,
         )

fractal_server/app/runner/v2/runner.py

@@ -42,6 +42,7 @@ def execute_tasks_v2(
     workflow_dir_remote: Optional[Path] = None,
     logger_name: Optional[str] = None,
     submit_setup_call: callable = no_op_submit_setup_call,
+    job_type_filters: dict[str, bool],
     job_attribute_filters: AttributeFiltersType,
 ) -> None:
     logger = logging.getLogger(logger_name)
@@ -56,7 +57,7 @@ def execute_tasks_v2(
     # Initialize local dataset attributes
     zarr_dir = dataset.zarr_dir
     tmp_images = deepcopy(dataset.images)
-    current_dataset_type_filters = deepcopy(dataset.type_filters)
+    current_dataset_type_filters = copy(job_type_filters)
 
     for wftask in wf_task_list:
         task = wftask.task
@@ -124,7 +125,7 @@ def execute_tasks_v2(
                 task=task,
                 workflow_dir_local=workflow_dir_local,
                 workflow_dir_remote=workflow_dir_remote,
-                executor=runner,
+                runner=runner,
                 submit_setup_call=submit_setup_call,
                 history_run_id=history_run_id,
                 dataset_id=dataset.id,
@@ -152,7 +153,7 @@ def execute_tasks_v2(
                 task=task,
                 workflow_dir_local=workflow_dir_local,
                 workflow_dir_remote=workflow_dir_remote,
-                executor=runner,
+                runner=runner,
                 submit_setup_call=submit_setup_call,
                 history_run_id=history_run_id,
                 dataset_id=dataset.id,
@@ -333,13 +334,10 @@ def execute_tasks_v2(
         current_dataset_type_filters.update(type_filters_from_task_manifest)
 
         with next(get_sync_db()) as db:
-            # Write current dataset attributes (history + filters) into the
-            # database.
+            # Write current dataset images into the database.
             db_dataset = db.get(DatasetV2, dataset.id)
-            db_dataset.type_filters = current_dataset_type_filters
             db_dataset.images = tmp_images
-            for attribute_name in ["type_filters", "images"]:
-                flag_modified(db_dataset, attribute_name)
+            flag_modified(db_dataset, "images")
             db.merge(db_dataset)
             db.commit()
             db.close()  # FIXME: why is this needed?

fractal_server/app/runner/v2/runner_functions.py

@@ -33,6 +33,8 @@ __all__ = [
     "run_v2_task_converter_compound",
 ]
 
+# FIXME: Review whether we need 5 functions or 3 are enough
+
 MAX_PARALLELIZATION_LIST_SIZE = 20_000
 
 
@@ -93,7 +95,7 @@ def run_v2_task_non_parallel(
     wftask: WorkflowTaskV2,
     workflow_dir_local: Path,
     workflow_dir_remote: Optional[Path] = None,
-    executor: BaseRunner,
+    runner: BaseRunner,
     submit_setup_call: callable = no_op_submit_setup_call,
     dataset_id: int,
     history_run_id: int,
@@ -148,7 +150,7 @@ def run_v2_task_non_parallel(
             ],
         )
 
-    result, exception = executor.submit(
+    result, exception = runner.submit(
         functools.partial(
             run_single_task,
             wftask=wftask,
@@ -274,7 +276,7 @@ def run_v2_task_parallel(
     images: list[dict[str, Any]],
     task: TaskV2,
     wftask: WorkflowTaskV2,
-    executor: BaseRunner,
+    runner: BaseRunner,
     workflow_dir_local: Path,
     workflow_dir_remote: Optional[Path] = None,
     submit_setup_call: callable = no_op_submit_setup_call,
@@ -334,7 +336,7 @@ def run_v2_task_parallel(
             db=db, list_upsert_objects=history_image_caches
         )
 
-    results, exceptions = executor.multisubmit(
+    results, exceptions = runner.multisubmit(
         functools.partial(
             run_single_task,
             wftask=wftask,
@@ -396,6 +398,9 @@ def run_v2_task_compound(
     dataset_id: int,
     history_run_id: int,
 ) -> tuple[TaskOutput, int, dict[int, BaseException]]:
+    # FIXME: Add task_files as a required argument, rather than a kwargs
+    # through executor_options_init
+
     executor_options_init = submit_setup_call(
         wftask=wftask,
         root_dir_local=workflow_dir_local,

fractal_server/app/schemas/v2/dataset.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from typing import Any
 from typing import Optional
 
 from pydantic import BaseModel
@@ -10,8 +9,6 @@ from pydantic import field_validator
 from pydantic import model_validator
 from pydantic.types import AwareDatetime
 
-from .._filter_validators import validate_attribute_filters
-from .._filter_validators import validate_type_filters
 from .._validators import cant_set_none
 from .._validators import NonEmptyString
 from .._validators import root_validate_dict_keys
@@ -28,7 +25,6 @@ class DatasetCreateV2(BaseModel):
 
     zarr_dir: Optional[str] = None
 
-    type_filters: dict[str, bool] = Field(default_factory=dict)
     attribute_filters: AttributeFiltersType = Field(default_factory=dict)
 
     # Validators
@@ -36,12 +32,6 @@ class DatasetCreateV2(BaseModel):
     _dict_keys = model_validator(mode="before")(
         classmethod(root_validate_dict_keys)
     )
-    _type_filters = field_validator("type_filters")(
-        classmethod(validate_type_filters)
-    )
-    _attribute_filters = field_validator("attribute_filters")(
-        classmethod(validate_attribute_filters)
-    )
 
     @field_validator("zarr_dir")
     @classmethod
@@ -61,8 +51,6 @@ class DatasetReadV2(BaseModel):
     timestamp_created: AwareDatetime
 
     zarr_dir: str
-    type_filters: dict[str, bool]
-    attribute_filters: AttributeFiltersType
 
     @field_serializer("timestamp_created")
     def serialize_datetime(v: datetime) -> str:
@@ -74,20 +62,12 @@ class DatasetUpdateV2(BaseModel):
 
     name: Optional[NonEmptyString] = None
     zarr_dir: Optional[str] = None
-    type_filters: Optional[dict[str, bool]] = None
-    attribute_filters: Optional[dict[str, list[Any]]] = None
 
     # Validators
 
     _dict_keys = model_validator(mode="before")(
         classmethod(root_validate_dict_keys)
     )
-    _type_filters = field_validator("type_filters")(
-        classmethod(validate_type_filters)
-    )
-    _attribute_filters = field_validator("attribute_filters")(
-        classmethod(validate_attribute_filters)
-    )
 
     @field_validator("name")
     @classmethod
@@ -106,63 +86,20 @@ class DatasetImportV2(BaseModel):
     """
     Class for `Dataset` import.
 
+    We are dropping `model_config = ConfigDict(extra="forbid")` so that any
+    kind of legacy filters can be included in the payload, and ignored in the
+    API.
+
     Attributes:
         name:
         zarr_dir:
         images:
-        filters:
-        type_filters:
-        attribute_filters:
     """
 
-    model_config = ConfigDict(extra="forbid")
-
     name: str
     zarr_dir: str
     images: list[SingleImage] = Field(default_factory=list)
 
-    filters: Optional[dict[str, Any]] = None
-    type_filters: dict[str, bool] = Field(default_factory=dict)
-    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
-
-    @model_validator(mode="before")
-    @classmethod
-    def update_legacy_filters(cls, values: dict):
-        """
-        Transform legacy filters (created with fractal-server<2.11.0)
-        into attribute/type filters
-        """
-        if values.get("filters") is not None:
-            if (
-                "type_filters" in values.keys()
-                or "attribute_filters" in values.keys()
-            ):
-                raise ValueError(
-                    "Cannot set filters both through the legacy field "
-                    "('filters') and the new ones ('type_filters' and/or "
-                    "'attribute_filters')."
-                )
-
-            else:
-                # Convert legacy filters.types into new type_filters
-                values["type_filters"] = values["filters"].get("types", {})
-                values["attribute_filters"] = {
-                    key: [value]
-                    for key, value in values["filters"]
-                    .get("attributes", {})
-                    .items()
-                }
-                values["filters"] = None
-
-        return values
-
-    _type_filters = field_validator("type_filters")(
-        classmethod(validate_type_filters)
-    )
-    _attribute_filters = field_validator("attribute_filters")(
-        classmethod(validate_attribute_filters)
-    )
-
     @field_validator("zarr_dir")
     @classmethod
     def normalize_zarr_dir(cls, v: str) -> str:
@@ -177,12 +114,8 @@ class DatasetExportV2(BaseModel):
         name:
         zarr_dir:
         images:
-        type_filters:
-        attribute_filters:
     """
 
     name: str
     zarr_dir: str
     images: list[SingleImage]
-    type_filters: dict[str, bool]
-    attribute_filters: AttributeFiltersType

fractal_server/app/schemas/v2/dumps.py

@@ -15,7 +15,6 @@ from pydantic import Field
 
 from .task import TaskTypeType
 from .task_group import TaskGroupV2OriginEnum
-from fractal_server.images.models import AttributeFiltersType
 
 
 class ProjectDumpV2(BaseModel):
@@ -65,15 +64,17 @@ class WorkflowDumpV2(BaseModel):
 
 
 class DatasetDumpV2(BaseModel):
-    model_config = ConfigDict(extra="forbid")
+    """
+    We do not include 'model_config = ConfigDict(extra="forbid")' because
+    legacy data may include 'type_filters' or 'attribute_filters' and we
+    want to avoid response-validation errors.
+    """
+
     id: int
     name: str
     project_id: int
     timestamp_created: str
-
     zarr_dir: str
-    type_filters: dict[str, bool]
-    attribute_filters: AttributeFiltersType
 
 
 class TaskGroupDumpV2(BaseModel):

fractal_server/app/schemas/v2/job.py

@@ -13,6 +13,7 @@ from pydantic.types import AwareDatetime
 from pydantic.types import StrictStr
 
 from .._filter_validators import validate_attribute_filters
+from .._filter_validators import validate_type_filters
 from .._validators import cant_set_none
 from .._validators import NonEmptyString
 from .._validators import root_validate_dict_keys
@@ -44,7 +45,6 @@ class JobStatusTypeV2(str, Enum):
 
 
 class JobCreateV2(BaseModel):
-
     model_config = ConfigDict(extra="forbid")
 
     first_task_index: Optional[int] = None
@@ -53,6 +53,7 @@ class JobCreateV2(BaseModel):
     worker_init: Optional[NonEmptyString] = None
 
     attribute_filters: AttributeFiltersType = Field(default_factory=dict)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
 
     # Validators
 
@@ -67,6 +68,9 @@ class JobCreateV2(BaseModel):
     _attribute_filters = field_validator("attribute_filters")(
         classmethod(validate_attribute_filters)
     )
+    _type_filters = field_validator("type_filters")(
+        classmethod(validate_type_filters)
+    )
 
     @field_validator("first_task_index")
     @classmethod
@@ -104,7 +108,6 @@ class JobCreateV2(BaseModel):
 
 
 class JobReadV2(BaseModel):
-
     id: int
     project_id: Optional[int] = None
     project_dump: ProjectDumpV2
@@ -124,6 +127,7 @@ class JobReadV2(BaseModel):
     last_task_index: Optional[int] = None
     worker_init: Optional[str] = None
     attribute_filters: AttributeFiltersType
+    type_filters: dict[str, bool]
 
     @field_serializer("start_timestamp")
     def serialize_datetime_start(v: datetime) -> str:
@@ -138,7 +142,6 @@ class JobReadV2(BaseModel):
 
 
 class JobUpdateV2(BaseModel):
-
     model_config = ConfigDict(extra="forbid")
 
     status: JobStatusTypeV2

fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py

@@ -0,0 +1,50 @@
+"""Drop dataset filters
+
+Revision ID: 47351f8c7ebc
+Revises: fbce16ff4e47
+Create Date: 2025-03-26 11:10:17.869028
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "47351f8c7ebc"
+down_revision = "fbce16ff4e47"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.drop_column("type_filters")
+        batch_op.drop_column("attribute_filters")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "attribute_filters",
+                postgresql.JSON(astext_type=sa.Text()),
+                server_default=sa.text("'{}'::json"),
+                autoincrement=False,
+                nullable=False,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "type_filters",
+                postgresql.JSON(astext_type=sa.Text()),
+                server_default=sa.text("'{}'::json"),
+                autoincrement=False,
+                nullable=False,
+            )
+        )
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/e81103413827_add_job_type_filters.py

@@ -0,0 +1,36 @@
+"""Add job.type_filters
+
+Revision ID: e81103413827
+Revises: 47351f8c7ebc
+Create Date: 2025-03-26 11:10:41.748248
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "e81103413827"
+down_revision = "47351f8c7ebc"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "type_filters", sa.JSON(), server_default="{}", nullable=False
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.drop_column("type_filters")
+
+    # ### end Alembic commands ###

{fractal_server-2.14.0a8.dist-info → fractal_server-2.14.0a10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a8
+Version: 2.14.0a10
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin