fractal-server 2.8.0__py3-none-any.whl → 2.9.0__py3-none-any.whl

fractal_server/app/runner/executors/slurm/sudo/executor.py

@@ -37,9 +37,11 @@ from ....exceptions import TaskExecutionError
 from ....filenames import SHUTDOWN_FILENAME
 from ....task_files import get_task_file_paths
 from ....task_files import TaskFiles
-from ...slurm._slurm_config import get_default_slurm_config
 from ...slurm._slurm_config import SlurmConfig
 from .._batching import heuristics
+from ..utils_executors import get_pickle_file_path
+from ..utils_executors import get_slurm_file_path
+from ..utils_executors import get_slurm_script_file_path
 from ._executor_wait_thread import FractalSlurmWaitThread
 from ._subprocess_run_as_user import _glob_as_user
 from ._subprocess_run_as_user import _glob_as_user_strict
@@ -305,72 +307,12 @@ class FractalSlurmExecutor(SlurmExecutor):
         with self.jobs_lock:
             self.map_jobid_to_slurm_files.pop(jobid)
 
-    def get_input_pickle_file_path(
-        self, *, arg: str, subfolder_name: str, prefix: Optional[str] = None
-    ) -> Path:
-
-        prefix = prefix or "cfut"
-        output = (
-            self.workflow_dir_local
-            / subfolder_name
-            / f"{prefix}_in_{arg}.pickle"
-        )
-        return output
-
-    def get_output_pickle_file_path(
-        self, *, arg: str, subfolder_name: str, prefix: Optional[str] = None
-    ) -> Path:
-        prefix = prefix or "cfut"
-        return (
-            self.workflow_dir_remote
-            / subfolder_name
-            / f"{prefix}_out_{arg}.pickle"
-        )
-
-    def get_slurm_script_file_path(
-        self, *, subfolder_name: str, prefix: Optional[str] = None
-    ) -> Path:
-        prefix = prefix or "_temp"
-        return (
-            self.workflow_dir_local
-            / subfolder_name
-            / f"{prefix}_slurm_submit.sbatch"
-        )
-
-    def get_slurm_stdout_file_path(
-        self,
-        *,
-        subfolder_name: str,
-        arg: str = "%j",
-        prefix: Optional[str] = None,
-    ) -> Path:
-        prefix = prefix or "slurmpy.stdout"
-        return (
-            self.workflow_dir_remote
-            / subfolder_name
-            / f"{prefix}_slurm_{arg}.out"
-        )
-
-    def get_slurm_stderr_file_path(
-        self,
-        *,
-        subfolder_name: str,
-        arg: str = "%j",
-        prefix: Optional[str] = None,
-    ) -> Path:
-        prefix = prefix or "slurmpy.stderr"
-        return (
-            self.workflow_dir_remote
-            / subfolder_name
-            / f"{prefix}_slurm_{arg}.err"
-        )
-
     def submit(
         self,
         fun: Callable[..., Any],
         *fun_args: Sequence[Any],
-        slurm_config: Optional[SlurmConfig] = None,
-        task_files: Optional[TaskFiles] = None,
+        slurm_config: SlurmConfig,
+        task_files: TaskFiles,
         **fun_kwargs: dict,
     ) -> Future:
         """
@@ -381,22 +323,14 @@ class FractalSlurmExecutor(SlurmExecutor):
             fun_args: Function positional arguments
             fun_kwargs: Function keyword arguments
             slurm_config:
-                A `SlurmConfig` object; if `None`, use
-                `get_default_slurm_config()`.
+                A `SlurmConfig` object.
             task_files:
-                A `TaskFiles` object; if `None`, use
-                `self.get_default_task_files()`.
+                A `TaskFiles` object.
 
         Returns:
             Future representing the execution of the current SLURM job.
         """
 
-        # Set defaults, if needed
-        if slurm_config is None:
-            slurm_config = get_default_slurm_config()
-        if task_files is None:
-            task_files = self.get_default_task_files()
-
         # Set slurm_file_prefix
         slurm_file_prefix = task_files.file_prefix
 
@@ -431,8 +365,8 @@ class FractalSlurmExecutor(SlurmExecutor):
         fn: Callable[..., Any],
         iterable: list[Sequence[Any]],
         *,
-        slurm_config: Optional[SlurmConfig] = None,
-        task_files: Optional[TaskFiles] = None,
+        slurm_config: SlurmConfig,
+        task_files: TaskFiles,
     ):
         """
         Return an iterator with the results of several execution of a function
@@ -455,11 +389,9 @@ class FractalSlurmExecutor(SlurmExecutor):
                 An iterable such that each element is the list of arguments to
                 be passed to `fn`, as in `fn(*args)`.
             slurm_config:
-                A `SlurmConfig` object; if `None`, use
-                `get_default_slurm_config()`.
+                A `SlurmConfig` object.
             task_files:
-                A `TaskFiles` object; if `None`, use
-                `self.get_default_task_files()`.
+                A `TaskFiles` object.
 
         """
 
@@ -479,12 +411,6 @@ class FractalSlurmExecutor(SlurmExecutor):
                 # self._exception
                 del fut
 
-        # Set defaults, if needed
-        if not slurm_config:
-            slurm_config = get_default_slurm_config()
-        if task_files is None:
-            task_files = self.get_default_task_files()
-
         # Include common_script_lines in extra_lines
         logger.debug(
             f"Adding {self.common_script_lines=} to "
@@ -700,39 +626,46 @@ class FractalSlurmExecutor(SlurmExecutor):
                 f"Missing folder {subfolder_path.as_posix()}."
             )
 
-        # Define I/O pickle file names/paths
         job.input_pickle_files = tuple(
-            self.get_input_pickle_file_path(
+            get_pickle_file_path(
                 arg=job.workerids[ind],
+                workflow_dir=self.workflow_dir_local,
                 subfolder_name=job.wftask_subfolder_name,
+                in_or_out="in",
                 prefix=job.wftask_file_prefixes[ind],
             )
             for ind in range(job.num_tasks_tot)
         )
         job.output_pickle_files = tuple(
-            self.get_output_pickle_file_path(
+            get_pickle_file_path(
                 arg=job.workerids[ind],
+                workflow_dir=self.workflow_dir_remote,
                 subfolder_name=job.wftask_subfolder_name,
+                in_or_out="out",
                 prefix=job.wftask_file_prefixes[ind],
             )
             for ind in range(job.num_tasks_tot)
         )
-
         # Define SLURM-job file names/paths
-        job.slurm_script = self.get_slurm_script_file_path(
+        job.slurm_script = get_slurm_script_file_path(
+            workflow_dir=self.workflow_dir_local,
             subfolder_name=job.wftask_subfolder_name,
             prefix=job.slurm_file_prefix,
         )
-        job.slurm_stdout = self.get_slurm_stdout_file_path(
+        job.slurm_stdout = get_slurm_file_path(
+            workflow_dir=self.workflow_dir_remote,
             subfolder_name=job.wftask_subfolder_name,
+            out_or_err="out",
             prefix=job.slurm_file_prefix,
         )
-        job.slurm_stderr = self.get_slurm_stderr_file_path(
+        job.slurm_stderr = get_slurm_file_path(
+            workflow_dir=self.workflow_dir_remote,
             subfolder_name=job.wftask_subfolder_name,
+            out_or_err="err",
             prefix=job.slurm_file_prefix,
         )
 
-        # Dump serialized versions+function+args+kwargs to pickle file
+        # Dump serialized versions+function+args+kwargs to pickle
         versions = dict(
             python=sys.version_info[:3],
             cloudpickle=cloudpickle.__version__,
@@ -824,7 +757,6 @@ class FractalSlurmExecutor(SlurmExecutor):
         """
         # Handle all uncaught exceptions in this broad try/except block
         try:
-
             # Retrieve job
             with self.jobs_lock:
                 try:
@@ -1039,7 +971,6 @@ class FractalSlurmExecutor(SlurmExecutor):
         )
 
         for prefix in prefixes:
-
             if prefix == job.slurm_file_prefix:
                 files_to_copy = _glob_as_user(
                     folder=str(self.workflow_dir_remote / subfolder_name),
@@ -1177,7 +1108,6 @@ class FractalSlurmExecutor(SlurmExecutor):
         slurm_err_path: str,
         slurm_config: SlurmConfig,
     ):
-
         num_tasks_max_running = slurm_config.parallel_tasks_per_job
         mem_per_task_MB = slurm_config.mem_per_task_MB
 
@@ -1229,19 +1159,6 @@ class FractalSlurmExecutor(SlurmExecutor):
         script = "\n".join(script_lines)
         return script
 
-    def get_default_task_files(self) -> TaskFiles:
-        """
-        This will be called when self.submit or self.map are called from
-        outside fractal-server, and then lack some optional arguments.
-        """
-        task_files = TaskFiles(
-            workflow_dir_local=self.workflow_dir_local,
-            workflow_dir_remote=self.workflow_dir_remote,
-            task_order=None,
-            task_name="name",
-        )
-        return task_files
-
     def shutdown(self, wait=True, *, cancel_futures=False):
         """
         Clean up all executor variables. Note that this function is executed on

fractal_server/app/runner/executors/slurm/utils_executors.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+
+def get_pickle_file_path(
+    *,
+    arg: str,
+    workflow_dir: Path,
+    subfolder_name: str,
+    in_or_out: Literal["in", "out"],
+    prefix: str,
+) -> Path:
+    if in_or_out in ["in", "out"]:
+        output = (
+            workflow_dir
+            / subfolder_name
+            / f"{prefix}_{in_or_out}_{arg}.pickle"
+        )
+        return output
+    else:
+        raise ValueError(
+            f"Missing or unexpected value in_or_out argument, {in_or_out=}"
+        )
+
+
+def get_slurm_script_file_path(
+    *, workflow_dir: Path, subfolder_name: str, prefix: Optional[str] = None
+) -> Path:
+    prefix = prefix or "_temp"
+    return workflow_dir / subfolder_name / f"{prefix}_slurm_submit.sbatch"
+
+
+def get_slurm_file_path(
+    *,
+    workflow_dir: Path,
+    subfolder_name: str,
+    arg: str = "%j",
+    out_or_err: Literal["out", "err"],
+    prefix: str,
+) -> Path:
+    if out_or_err == "out":
+        return (
+            workflow_dir
+            / subfolder_name
+            / f"{prefix}_slurm_{arg}.{out_or_err}"
+        )
+    elif out_or_err == "err":
+        return (
+            workflow_dir
+            / subfolder_name
+            / f"{prefix}_slurm_{arg}.{out_or_err}"
+        )
+    else:
+        raise ValueError(
+            "Missing or unexpected value out_or_err argument, "
+            f"{out_or_err=}"
+        )

fractal_server/app/runner/v2/_local_experimental/executor.py

@@ -2,6 +2,7 @@
 Custom version of Python
 [ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ProcessPoolExecutor)).
 """
+import multiprocessing as mp
 import threading
 import time
 from concurrent.futures import ProcessPoolExecutor
@@ -32,7 +33,7 @@ class FractalProcessPoolExecutor(ProcessPoolExecutor):
     def __init__(
         self, shutdown_file: Path, interval: float = 1.0, *args, **kwargs
     ):
-        super().__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs, mp_context=mp.get_context("spawn"))
         self.shutdown_file = Path(shutdown_file)
         self.interval = float(interval)
         logger.debug(

fractal_server/app/schemas/_validators.py

@@ -1,6 +1,4 @@
 import os
-from datetime import datetime
-from datetime import timezone
 from typing import Any
 from typing import Optional
 
@@ -74,7 +72,7 @@ def val_absolute_path(attribute: str, accept_none: bool = False):
     Check that a string attribute is an absolute path
     """
 
-    def val(string: Optional[str]) -> str:
+    def val(string: Optional[str]) -> Optional[str]:
         if string is None:
             if accept_none:
                 return string
@@ -103,16 +101,3 @@ def val_unique_list(attribute: str):
         return must_be_unique
 
     return val
-
-
-def valutc(attribute: str):
-    def val(timestamp: Optional[datetime]) -> Optional[datetime]:
-        """
-        Replacing `tzinfo` with `timezone.utc` is just required by SQLite data.
-        If using Postgres, this function leaves the datetime exactly as it is.
-        """
-        if timestamp is not None:
-            return timestamp.replace(tzinfo=timezone.utc)
-        return None
-
-    return val

fractal_server/app/schemas/user.py

@@ -3,6 +3,7 @@ from typing import Optional
 from fastapi_users import schemas
 from pydantic import BaseModel
 from pydantic import Extra
+from pydantic import Field
 from pydantic import validator
 
 from ._validators import val_unique_list
@@ -11,8 +12,8 @@ from ._validators import valstr
 __all__ = (
     "UserRead",
     "UserUpdate",
+    "UserUpdateGroups",
     "UserCreate",
-    "UserUpdateWithNewGroupIds",
 )
 
 
@@ -45,7 +46,7 @@ class UserRead(schemas.BaseUser[int]):
     oauth_accounts: list[OAuthAccountRead]
 
 
-class UserUpdate(schemas.BaseUserUpdate):
+class UserUpdate(schemas.BaseUserUpdate, extra=Extra.forbid):
     """
     Schema for `User` update.
 
@@ -82,14 +83,6 @@ class UserUpdateStrict(BaseModel, extra=Extra.forbid):
     pass
 
 
-class UserUpdateWithNewGroupIds(UserUpdate):
-    new_group_ids: Optional[list[int]] = None
-
-    _val_unique = validator("new_group_ids", allow_reuse=True)(
-        val_unique_list("new_group_ids")
-    )
-
-
 class UserCreate(schemas.BaseUserCreate):
     """
     Schema for `User` creation.
@@ -103,3 +96,16 @@ class UserCreate(schemas.BaseUserCreate):
     # Validators
 
     _username = validator("username", allow_reuse=True)(valstr("username"))
+
+
+class UserUpdateGroups(BaseModel, extra=Extra.forbid):
+    """
+    Schema for `POST /auth/users/{user_id}/set-groups/`
+
+    """
+
+    group_ids: list[int] = Field(min_items=1)
+
+    _group_ids = validator("group_ids", allow_reuse=True)(
+        val_unique_list("group_ids")
+    )

fractal_server/app/schemas/user_group.py

@@ -59,21 +59,10 @@ class UserGroupCreate(BaseModel, extra=Extra.forbid):
 class UserGroupUpdate(BaseModel, extra=Extra.forbid):
     """
     Schema for `UserGroup` update
-
-    NOTE: `new_user_ids` does not correspond to a column of the `UserGroup`
-    table, but it is rather used to create new `LinkUserGroup` rows.
-
-    Attributes:
-        new_user_ids: IDs of groups to be associated to user.
     """
 
-    new_user_ids: list[int] = Field(default_factory=list)
     viewer_paths: Optional[list[str]] = None
 
-    _val_unique = validator("new_user_ids", allow_reuse=True)(
-        val_unique_list("new_user_ids")
-    )
-
     @validator("viewer_paths")
     def viewer_paths_validator(cls, value):
         for i, path in enumerate(value):

fractal_server/app/schemas/v1/applyworkflow.py

@@ -7,7 +7,6 @@ from pydantic import validator
 from pydantic.types import StrictStr
 
 from .._validators import valstr
-from .._validators import valutc
 from .dumps import DatasetDumpV1
 from .dumps import ProjectDumpV1
 from .dumps import WorkflowDumpV1
@@ -150,13 +149,6 @@ class ApplyWorkflowReadV1(_ApplyWorkflowBaseV1):
     first_task_index: Optional[int]
     last_task_index: Optional[int]
 
-    _start_timestamp = validator("start_timestamp", allow_reuse=True)(
-        valutc("start_timestamp")
-    )
-    _end_timestamp = validator("end_timestamp", allow_reuse=True)(
-        valutc("end_timestamp")
-    )
-
 
 class ApplyWorkflowUpdateV1(BaseModel):
     """

fractal_server/app/schemas/v1/dataset.py

@@ -8,7 +8,6 @@ from pydantic import validator
 
 from .._validators import val_absolute_path
 from .._validators import valstr
-from .._validators import valutc
 from .dumps import WorkflowTaskDumpV1
 from .project import ProjectReadV1
 from .workflow import WorkflowTaskStatusTypeV1
@@ -151,10 +150,6 @@ class DatasetReadV1(_DatasetBaseV1):
     project: ProjectReadV1
     timestamp_created: datetime
 
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
-
 
 class DatasetStatusReadV1(BaseModel):
     """

fractal_server/app/schemas/v1/project.py

@@ -5,7 +5,6 @@ from pydantic import BaseModel
 from pydantic import validator
 
 from .._validators import valstr
-from .._validators import valutc
 
 
 __all__ = (
@@ -50,10 +49,6 @@ class ProjectReadV1(_ProjectBaseV1):
     id: int
     timestamp_created: datetime
 
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
-
 
 class ProjectUpdateV1(_ProjectBaseV1):
     """

fractal_server/app/schemas/v1/state.py

@@ -3,9 +3,6 @@ from typing import Any
 from typing import Optional
 
 from pydantic import BaseModel
-from pydantic import validator
-
-from fractal_server.app.schemas._validators import valutc
 
 
 class StateRead(BaseModel):
@@ -19,5 +16,3 @@ class StateRead(BaseModel):
     id: Optional[int]
     data: dict[str, Any]
     timestamp: datetime
-
-    _timestamp = validator("timestamp", allow_reuse=True)(valutc("timestamp"))

fractal_server/app/schemas/v1/workflow.py

@@ -8,7 +8,6 @@ from pydantic import validator
 
 from .._validators import valint
 from .._validators import valstr
-from .._validators import valutc
 from .project import ProjectReadV1
 from .task import TaskExportV1
 from .task import TaskImportV1
@@ -135,10 +134,6 @@ class WorkflowReadV1(_WorkflowBaseV1):
     project: ProjectReadV1
     timestamp_created: datetime
 
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
-
 
 class WorkflowCreateV1(_WorkflowBaseV1):
     """

fractal_server/app/schemas/v2/__init__.py

@@ -23,11 +23,13 @@ from .task import TaskImportV2  # noqa F401
 from .task import TaskImportV2Legacy  # noqa F401
 from .task import TaskReadV2  # noqa F401
 from .task import TaskUpdateV2  # noqa F401
-from .task_collection import CollectionStateReadV2  # noqa F401
-from .task_collection import CollectionStatusV2  # noqa F401
 from .task_collection import TaskCollectCustomV2  # noqa F401
 from .task_collection import TaskCollectPipV2  # noqa F401
+from .task_group import TaskGroupActivityActionV2  # noqa F401
+from .task_group import TaskGroupActivityStatusV2  # noqa F401
+from .task_group import TaskGroupActivityV2Read  # noqa F401
 from .task_group import TaskGroupCreateV2  # noqa F401
+from .task_group import TaskGroupCreateV2Strict  # noqa F401
 from .task_group import TaskGroupReadV2  # noqa F401
 from .task_group import TaskGroupUpdateV2  # noqa F401
 from .task_group import TaskGroupV2OriginEnum  # noqa F401

fractal_server/app/schemas/v2/dataset.py

@@ -7,7 +7,6 @@ from pydantic import Field
 from pydantic import validator
 
 from .._validators import valstr
-from .._validators import valutc
 from .dumps import WorkflowTaskDumpV2
 from .project import ProjectReadV2
 from .workflowtask import WorkflowTaskStatusTypeV2
@@ -39,7 +38,7 @@ class DatasetCreateV2(BaseModel, extra=Extra.forbid):
 
     # Validators
     @validator("zarr_dir")
-    def normalize_zarr_dir(cls, v: str) -> str:
+    def normalize_zarr_dir(cls, v: Optional[str]) -> Optional[str]:
         if v is not None:
             return normalize_url(v)
         return v
@@ -62,11 +61,6 @@ class DatasetReadV2(BaseModel):
     zarr_dir: str
     filters: Filters = Field(default_factory=Filters)
 
-    # Validators
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
-
 
 class DatasetUpdateV2(BaseModel, extra=Extra.forbid):
 

fractal_server/app/schemas/v2/job.py

@@ -8,7 +8,6 @@ from pydantic import validator
 from pydantic.types import StrictStr
 
 from .._validators import valstr
-from .._validators import valutc
 from .dumps import DatasetDumpV2
 from .dumps import ProjectDumpV2
 from .dumps import WorkflowDumpV2
@@ -101,13 +100,6 @@ class JobReadV2(BaseModel):
     last_task_index: Optional[int]
     worker_init: Optional[str]
 
-    _start_timestamp = validator("start_timestamp", allow_reuse=True)(
-        valutc("start_timestamp")
-    )
-    _end_timestamp = validator("end_timestamp", allow_reuse=True)(
-        valutc("end_timestamp")
-    )
-
 
 class JobUpdateV2(BaseModel, extra=Extra.forbid):
 

fractal_server/app/schemas/v2/project.py

@@ -6,7 +6,6 @@ from pydantic import Extra
 from pydantic import validator
 
 from .._validators import valstr
-from .._validators import valutc
 
 
 class ProjectCreateV2(BaseModel, extra=Extra.forbid):
@@ -21,10 +20,6 @@ class ProjectReadV2(BaseModel):
     id: int
     name: str
     timestamp_created: datetime
-    # Validators
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
 
 
 class ProjectUpdateV2(BaseModel, extra=Extra.forbid):