fractal-server 2.9.0a2__py3-none-any.whl → 2.9.0a3__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.9.0a2"
+__VERSION__ = "2.9.0a3"

fractal_server/app/routes/api/v2/task_collection.py

@@ -37,6 +37,7 @@ from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
 from fractal_server.tasks.v2.local.collect import (
     collect_local,
 )
+from fractal_server.tasks.v2.ssh import collect_ssh
 from fractal_server.tasks.v2.utils_package_names import _parse_wheel_filename
 from fractal_server.tasks.v2.utils_package_names import normalize_package_name
 from fractal_server.tasks.v2.utils_python_interpreter import (
@@ -63,10 +64,7 @@ async def collect_tasks_pip(
     db: AsyncSession = Depends(get_async_db),
 ) -> TaskGroupActivityV2Read:
     """
-    Task collection endpoint
-
-    Trigger the creation of a dedicated virtual environment, the installation
-    of a package and the collection of tasks as advertised in the manifest.
+    Task-collection endpoint
     """
 
     # Get settings
@@ -246,12 +244,7 @@ async def collect_tasks_pip(
 
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
         # SSH task collection
-
-        from fractal_server.tasks.v2.ssh.collect import (
-            collect_ssh,
-        )
-
-        # User appropriate FractalSSH object
+        # Use appropriate FractalSSH object
         ssh_credentials = dict(
             user=user_settings.ssh_username,
             host=user_settings.ssh_host,

fractal_server/app/routes/api/v2/task_group_lifecycle.py

@@ -2,9 +2,11 @@ from fastapi import APIRouter
 from fastapi import BackgroundTasks
 from fastapi import Depends
 from fastapi import HTTPException
+from fastapi import Request
 from fastapi import Response
 from fastapi import status
 
+from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions_task_lifecycle import check_no_ongoing_activity
 from ._aux_functions_tasks import _get_task_group_full_access
 from fractal_server.app.db import AsyncSession
@@ -22,6 +24,8 @@ from fractal_server.logger import set_logger
 from fractal_server.syringe import Inject
 from fractal_server.tasks.v2.local import deactivate_local
 from fractal_server.tasks.v2.local import reactivate_local
+from fractal_server.tasks.v2.ssh import deactivate_ssh
+from fractal_server.tasks.v2.ssh import reactivate_ssh
 from fractal_server.utils import get_timestamp
 
 router = APIRouter()
@@ -38,6 +42,7 @@ async def deactivate_task_group(
     task_group_id: int,
     background_tasks: BackgroundTasks,
     response: Response,
+    request: Request,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> TaskGroupReadV2:
@@ -103,10 +108,29 @@ async def deactivate_task_group(
     # Submit background task
     settings = Inject(get_settings)
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-        raise HTTPException(
-            status_code=status.HTTP_501_NOT_IMPLEMENTED,
-            detail="Not implemented (yet) for SSH.",
+
+        # Validate user settings (backend-specific)
+        user_settings = await validate_user_settings(
+            user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
+        )
+
+        # User appropriate FractalSSH object
+        ssh_credentials = dict(
+            user=user_settings.ssh_username,
+            host=user_settings.ssh_host,
+            key_path=user_settings.ssh_private_key_path,
+        )
+        fractal_ssh_list = request.app.state.fractal_ssh_list
+        fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
+
+        background_tasks.add_task(
+            deactivate_ssh,
+            task_group_id=task_group.id,
+            task_group_activity_id=task_group_activity.id,
+            fractal_ssh=fractal_ssh,
+            tasks_base_dir=user_settings.ssh_tasks_dir,
         )
+
     else:
         background_tasks.add_task(
             deactivate_local,
@@ -130,6 +154,7 @@ async def reactivate_task_group(
     task_group_id: int,
     background_tasks: BackgroundTasks,
     response: Response,
+    request: Request,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> TaskGroupReadV2:
@@ -203,10 +228,29 @@ async def reactivate_task_group(
     # Submit background task
     settings = Inject(get_settings)
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-        raise HTTPException(
-            status_code=status.HTTP_501_NOT_IMPLEMENTED,
-            detail="Not implemented (yet) for SSH.",
+
+        # Validate user settings (backend-specific)
+        user_settings = await validate_user_settings(
+            user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
+        )
+
+        # Use appropriate FractalSSH object
+        ssh_credentials = dict(
+            user=user_settings.ssh_username,
+            host=user_settings.ssh_host,
+            key_path=user_settings.ssh_private_key_path,
+        )
+        fractal_ssh_list = request.app.state.fractal_ssh_list
+        fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
+
+        background_tasks.add_task(
+            reactivate_ssh,
+            task_group_id=task_group.id,
+            task_group_activity_id=task_group_activity.id,
+            fractal_ssh=fractal_ssh,
+            tasks_base_dir=user_settings.ssh_tasks_dir,
         )
+
     else:
         background_tasks.add_task(
             reactivate_local,

fractal_server/app/schemas/_validators.py

@@ -1,6 +1,4 @@
 import os
-from datetime import datetime
-from datetime import timezone
 from typing import Any
 from typing import Optional
 
@@ -103,15 +101,3 @@ def val_unique_list(attribute: str):
         return must_be_unique
 
     return val
-
-
-def valutc(attribute: str):
-    def val(timestamp: Optional[datetime]) -> Optional[datetime]:
-        """
-        Replace `tzinfo` with `timezone.utc`.
-        """
-        if timestamp is not None:
-            return timestamp.replace(tzinfo=timezone.utc)
-        return None
-
-    return val

fractal_server/app/schemas/v1/applyworkflow.py

@@ -7,7 +7,6 @@ from pydantic import validator
 from pydantic.types import StrictStr
 
 from .._validators import valstr
-from .._validators import valutc
 from .dumps import DatasetDumpV1
 from .dumps import ProjectDumpV1
 from .dumps import WorkflowDumpV1
@@ -150,13 +149,6 @@ class ApplyWorkflowReadV1(_ApplyWorkflowBaseV1):
     first_task_index: Optional[int]
     last_task_index: Optional[int]
 
-    _start_timestamp = validator("start_timestamp", allow_reuse=True)(
-        valutc("start_timestamp")
-    )
-    _end_timestamp = validator("end_timestamp", allow_reuse=True)(
-        valutc("end_timestamp")
-    )
-
 
 class ApplyWorkflowUpdateV1(BaseModel):
     """

fractal_server/app/schemas/v1/dataset.py

@@ -8,7 +8,6 @@ from pydantic import validator
 
 from .._validators import val_absolute_path
 from .._validators import valstr
-from .._validators import valutc
 from .dumps import WorkflowTaskDumpV1
 from .project import ProjectReadV1
 from .workflow import WorkflowTaskStatusTypeV1
@@ -151,10 +150,6 @@ class DatasetReadV1(_DatasetBaseV1):
     project: ProjectReadV1
     timestamp_created: datetime
 
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
-
 
 class DatasetStatusReadV1(BaseModel):
     """

fractal_server/app/schemas/v1/project.py

@@ -5,7 +5,6 @@ from pydantic import BaseModel
 from pydantic import validator
 
 from .._validators import valstr
-from .._validators import valutc
 
 
 __all__ = (
@@ -50,10 +49,6 @@ class ProjectReadV1(_ProjectBaseV1):
     id: int
     timestamp_created: datetime
 
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
-
 
 class ProjectUpdateV1(_ProjectBaseV1):
     """

fractal_server/app/schemas/v1/state.py

@@ -3,9 +3,6 @@ from typing import Any
 from typing import Optional
 
 from pydantic import BaseModel
-from pydantic import validator
-
-from fractal_server.app.schemas._validators import valutc
 
 
 class StateRead(BaseModel):
@@ -19,5 +16,3 @@ class StateRead(BaseModel):
     id: Optional[int]
     data: dict[str, Any]
     timestamp: datetime
-
-    _timestamp = validator("timestamp", allow_reuse=True)(valutc("timestamp"))

fractal_server/app/schemas/v1/workflow.py

@@ -8,7 +8,6 @@ from pydantic import validator
 
 from .._validators import valint
 from .._validators import valstr
-from .._validators import valutc
 from .project import ProjectReadV1
 from .task import TaskExportV1
 from .task import TaskImportV1
@@ -135,10 +134,6 @@ class WorkflowReadV1(_WorkflowBaseV1):
     project: ProjectReadV1
     timestamp_created: datetime
 
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
-
 
 class WorkflowCreateV1(_WorkflowBaseV1):
     """

fractal_server/app/schemas/v2/dataset.py

@@ -7,7 +7,6 @@ from pydantic import Field
 from pydantic import validator
 
 from .._validators import valstr
-from .._validators import valutc
 from .dumps import WorkflowTaskDumpV2
 from .project import ProjectReadV2
 from .workflowtask import WorkflowTaskStatusTypeV2
@@ -62,11 +61,6 @@ class DatasetReadV2(BaseModel):
     zarr_dir: str
     filters: Filters = Field(default_factory=Filters)
 
-    # Validators
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
-
 
 class DatasetUpdateV2(BaseModel, extra=Extra.forbid):
 

fractal_server/app/schemas/v2/job.py

@@ -8,7 +8,6 @@ from pydantic import validator
 from pydantic.types import StrictStr
 
 from .._validators import valstr
-from .._validators import valutc
 from .dumps import DatasetDumpV2
 from .dumps import ProjectDumpV2
 from .dumps import WorkflowDumpV2
@@ -101,13 +100,6 @@ class JobReadV2(BaseModel):
     last_task_index: Optional[int]
     worker_init: Optional[str]
 
-    _start_timestamp = validator("start_timestamp", allow_reuse=True)(
-        valutc("start_timestamp")
-    )
-    _end_timestamp = validator("end_timestamp", allow_reuse=True)(
-        valutc("end_timestamp")
-    )
-
 
 class JobUpdateV2(BaseModel, extra=Extra.forbid):
 

fractal_server/app/schemas/v2/project.py

@@ -6,7 +6,6 @@ from pydantic import Extra
 from pydantic import validator
 
 from .._validators import valstr
-from .._validators import valutc
 
 
 class ProjectCreateV2(BaseModel, extra=Extra.forbid):
@@ -21,10 +20,6 @@ class ProjectReadV2(BaseModel):
     id: int
     name: str
     timestamp_created: datetime
-    # Validators
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
 
 
 class ProjectUpdateV2(BaseModel, extra=Extra.forbid):

fractal_server/app/schemas/v2/workflow.py

@@ -6,7 +6,6 @@ from pydantic import Extra
 from pydantic import validator
 
 from .._validators import valstr
-from .._validators import valutc
 from .project import ProjectReadV2
 from .workflowtask import WorkflowTaskExportV2
 from .workflowtask import WorkflowTaskImportV2
@@ -31,10 +30,6 @@ class WorkflowReadV2(BaseModel):
     project: ProjectReadV2
     timestamp_created: datetime
 
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
-
 
 class WorkflowReadV2WithWarnings(WorkflowReadV2):
     task_list: list[WorkflowTaskReadV2WithWarning]

fractal_server/ssh/_fabric.py

@@ -321,8 +321,10 @@ class FractalSSH(object):
                     f"{prefix} END   running '{cmd}' over SSH, "
                     f"elapsed {t_1-t_0:.3f}"
                 )
-                self.logger.debug(f"STDOUT: {res.stdout}")
-                self.logger.debug(f"STDERR: {res.stderr}")
+                self.logger.debug("STDOUT:")
+                self.logger.debug(res.stdout)
+                self.logger.debug("STDERR:")
+                self.logger.debug(res.stderr)
                 return res.stdout
             except NoValidConnectionsError as e:
                 # Case 2: Command fails with a connection error

fractal_server/tasks/v2/local/{utils_local.py → _utils.py}

@@ -1,5 +1,6 @@
 from pathlib import Path
 
+from fractal_server.app.schemas.v2 import TaskCreateV2
 from fractal_server.logger import get_logger
 from fractal_server.tasks.v2.utils_templates import customize_template
 from fractal_server.utils import execute_command_sync
@@ -43,3 +44,27 @@ def _customize_and_run_template(
     stdout = execute_command_sync(command=cmd, logger_name=logger_name)
     logger.debug(f"_customize_and_run_template {template_filename} - END")
     return stdout
+
+
+def check_task_files_exist(task_list: list[TaskCreateV2]) -> None:
+    """
+    Check that the modules listed in task commands point to existing files.
+
+    Args:
+        task_list:
+    """
+    for _task in task_list:
+        if _task.command_non_parallel is not None:
+            _task_path = _task.command_non_parallel.split()[1]
+            if not Path(_task_path).exists():
+                raise FileNotFoundError(
+                    f"Task `{_task.name}` has `command_non_parallel` "
+                    f"pointing to missing file `{_task_path}`."
+                )
+        if _task.command_parallel is not None:
+            _task_path = _task.command_parallel.split()[1]
+            if not Path(_task_path).exists():
+                raise FileNotFoundError(
+                    f"Task `{_task.name}` has `command_parallel` "
+                    f"pointing to missing file `{_task_path}`."
+                )

fractal_server/tasks/v2/local/collect.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from ..utils_database import create_db_tasks_and_update_task_group
-from .utils_local import _customize_and_run_template
+from ._utils import _customize_and_run_template
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2
@@ -16,9 +16,9 @@ from fractal_server.app.schemas.v2.manifest import ManifestV2
 from fractal_server.logger import get_logger
 from fractal_server.logger import set_logger
 from fractal_server.tasks.utils import get_log_path
+from fractal_server.tasks.v2.local._utils import check_task_files_exist
 from fractal_server.tasks.v2.utils_background import _prepare_tasks_metadata
 from fractal_server.tasks.v2.utils_background import add_commit_refresh
-from fractal_server.tasks.v2.utils_background import check_task_files_exist
 from fractal_server.tasks.v2.utils_background import fail_and_cleanup
 from fractal_server.tasks.v2.utils_background import get_current_log
 from fractal_server.tasks.v2.utils_package_names import compare_package_names

fractal_server/tasks/v2/local/deactivate.py

@@ -7,7 +7,7 @@ from tempfile import TemporaryDirectory
 from ..utils_background import add_commit_refresh
 from ..utils_background import fail_and_cleanup
 from ..utils_templates import get_collection_replacements
-from .utils_local import _customize_and_run_template
+from ._utils import _customize_and_run_template
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2

fractal_server/tasks/v2/local/reactivate.py

@@ -7,7 +7,7 @@ from tempfile import TemporaryDirectory
 from ..utils_background import add_commit_refresh
 from ..utils_background import fail_and_cleanup
 from ..utils_templates import get_collection_replacements
-from .utils_local import _customize_and_run_template
+from ._utils import _customize_and_run_template
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2

fractal_server/tasks/v2/ssh/__init__.py

@@ -0,0 +1,3 @@
+from .collect import collect_ssh  # noqa
+from .deactivate import deactivate_ssh  # noqa
+from .reactivate import reactivate_ssh  # noqa

fractal_server/tasks/v2/ssh/_utils.py

@@ -0,0 +1,87 @@
+import os
+from pathlib import Path
+
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.logger import get_logger
+from fractal_server.ssh._fabric import FractalSSH
+from fractal_server.tasks.v2.utils_templates import customize_template
+
+
+def _customize_and_run_template(
+    *,
+    template_filename: str,
+    replacements: list[tuple[str, str]],
+    script_dir_local: str,
+    prefix: str,
+    fractal_ssh: FractalSSH,
+    script_dir_remote: str,
+    logger_name: str,
+) -> str:
+    """
+    Customize one of the template bash scripts, transfer it to the remote host
+    via SFTP and then run it via SSH.
+
+    Args:
+        template_filename: Filename of the template file (ends with ".sh").
+        replacements: Dictionary of replacements.
+        script_dir: Local folder where the script will be placed.
+        prefix: Prefix for the script filename.
+        fractal_ssh: FractalSSH object
+        script_dir_remote: Remote scripts directory
+    """
+    logger = get_logger(logger_name=logger_name)
+    logger.debug(f"_customize_and_run_template {template_filename} - START")
+    # Prepare name and path of script
+    if not template_filename.endswith(".sh"):
+        raise ValueError(
+            f"Invalid {template_filename=} (it must end with '.sh')."
+        )
+    script_filename = f"{prefix}_{template_filename}"
+    script_path_local = (Path(script_dir_local) / script_filename).as_posix()
+
+    customize_template(
+        template_name=template_filename,
+        replacements=replacements,
+        script_path=script_path_local,
+    )
+
+    # Transfer script to remote host
+    script_path_remote = os.path.join(
+        script_dir_remote,
+        script_filename,
+    )
+    logger.debug(f"Now transfer {script_path_local=} over SSH.")
+    fractal_ssh.send_file(
+        local=script_path_local,
+        remote=script_path_remote,
+    )
+
+    # Execute script remotely
+    cmd = f"bash {script_path_remote}"
+    logger.debug(f"Now run '{cmd}' over SSH.")
+    stdout = fractal_ssh.run_command(cmd=cmd)
+
+    logger.debug(f"_customize_and_run_template {template_filename} - END")
+    return stdout
+
+
+def _copy_wheel_file_ssh(
+    *, task_group: TaskGroupV2, fractal_ssh: FractalSSH, logger_name: str
+) -> str:
+    """
+    Handle the situation where `task_group.wheel_path` is not part of
+    `task_group.path`, by copying `wheel_path` into `path`.
+
+    Returns:
+        The new `wheel_path`.
+    """
+    logger = get_logger(logger_name=logger_name)
+    source = task_group.wheel_path
+    dest = (
+        Path(task_group.path) / Path(task_group.wheel_path).name
+    ).as_posix()
+    cmd = f"cp {source} {dest}"
+    logger.debug(f"[_copy_wheel_file] START {source=} {dest=}")
+    fractal_ssh.run_command(cmd=cmd)
+    logger.debug(f"[_copy_wheel_file] END {source=} {dest=}")
+    return dest

fractal_server/tasks/v2/ssh/collect.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import time
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -13,16 +12,16 @@ from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
 from fractal_server.app.schemas.v2.manifest import ManifestV2
-from fractal_server.logger import get_logger
 from fractal_server.logger import set_logger
 from fractal_server.ssh._fabric import FractalSSH
+from fractal_server.tasks.v2.ssh._utils import _copy_wheel_file_ssh
+from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
 from fractal_server.tasks.v2.utils_background import add_commit_refresh
 from fractal_server.tasks.v2.utils_background import get_current_log
 from fractal_server.tasks.v2.utils_package_names import compare_package_names
 from fractal_server.tasks.v2.utils_python_interpreter import (
     get_python_interpreter_v2,
 )
-from fractal_server.tasks.v2.utils_templates import customize_template
 from fractal_server.tasks.v2.utils_templates import get_collection_replacements
 from fractal_server.tasks.v2.utils_templates import (
     parse_script_pip_show_stdout,
@@ -33,81 +32,6 @@ from fractal_server.utils import get_timestamp
 LOGGER_NAME = __name__
 
 
-def _customize_and_run_template(
-    *,
-    template_filename: str,
-    replacements: list[tuple[str, str]],
-    script_dir_local: str,
-    prefix: str,
-    fractal_ssh: FractalSSH,
-    script_dir_remote: str,
-    logger_name: str,
-) -> str:
-    """
-    Customize one of the template bash scripts, transfer it to the remote host
-    via SFTP and then run it via SSH.
-
-    Args:
-        template_filename: Filename of the template file (ends with ".sh").
-        replacements: Dictionary of replacements.
-        script_dir: Local folder where the script will be placed.
-        prefix: Prefix for the script filename.
-        fractal_ssh: FractalSSH object
-        script_dir_remote: Remote scripts directory
-    """
-    logger = get_logger(logger_name=logger_name)
-    logger.debug(f"_customize_and_run_template {template_filename} - START")
-
-    # Prepare name and path of script
-    if not template_filename.endswith(".sh"):
-        raise ValueError(
-            f"Invalid {template_filename=} (it must end with '.sh')."
-        )
-    script_filename = f"{prefix}_{template_filename}"
-    script_path_local = Path(script_dir_local) / script_filename
-
-    customize_template(
-        template_name=template_filename,
-        replacements=replacements,
-        script_path=script_path_local,
-    )
-
-    # Transfer script to remote host
-    script_path_remote = os.path.join(
-        script_dir_remote,
-        script_filename,
-    )
-    logger.debug(f"Now transfer {script_path_local=} over SSH.")
-    fractal_ssh.send_file(
-        local=script_path_local,
-        remote=script_path_remote,
-    )
-
-    # Execute script remotely
-    cmd = f"bash {script_path_remote}"
-    logger.debug(f"Now run '{cmd}' over SSH.")
-    stdout = fractal_ssh.run_command(cmd=cmd)
-    logger.debug(f"Standard output of '{cmd}':\n{stdout}")
-
-    logger.debug(f"_customize_and_run_template {template_filename} - END")
-    return stdout
-
-
-def _copy_wheel_file_ssh(
-    task_group: TaskGroupV2, fractal_ssh: FractalSSH
-) -> str:
-    logger = get_logger(LOGGER_NAME)
-    source = task_group.wheel_path
-    dest = (
-        Path(task_group.path) / Path(task_group.wheel_path).name
-    ).as_posix()
-    cmd = f"cp {source} {dest}"
-    logger.debug(f"[_copy_wheel_file] START {source=} {dest=}")
-    fractal_ssh.run_command(cmd=cmd)
-    logger.debug(f"[_copy_wheel_file] END {source=} {dest=}")
-    return dest
-
-
 def collect_ssh(
     *,
     task_group_id: int,
@@ -232,6 +156,7 @@ def collect_ssh(
                     new_wheel_path = _copy_wheel_file_ssh(
                         task_group=task_group,
                         fractal_ssh=fractal_ssh,
+                        logger_name=LOGGER_NAME,
                     )
                     task_group.wheel_path = new_wheel_path
                     task_group = add_commit_refresh(obj=task_group, db=db)

fractal_server/tasks/v2/ssh/deactivate.py

@@ -1,2 +1,243 @@
-def deactivate_ssh():
-    pass
+import logging
+import time
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from ..utils_background import add_commit_refresh
+from ..utils_background import fail_and_cleanup
+from ..utils_templates import get_collection_replacements
+from ._utils import _copy_wheel_file_ssh
+from ._utils import _customize_and_run_template
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models.v2 import TaskGroupActivityV2
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
+from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
+from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
+from fractal_server.logger import set_logger
+from fractal_server.ssh._fabric import FractalSSH
+from fractal_server.tasks.utils import get_log_path
+from fractal_server.tasks.v2.utils_background import get_current_log
+from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
+from fractal_server.utils import get_timestamp
+
+LOGGER_NAME = __name__
+
+
+def deactivate_ssh(
+    *,
+    task_group_activity_id: int,
+    task_group_id: int,
+    fractal_ssh: FractalSSH,
+    tasks_base_dir: str,
+) -> None:
+    """
+    Deactivate a task group venv.
+
+    This function is run as a background task, therefore exceptions must be
+    handled.
+
+    Arguments:
+        task_group_id:
+        task_group_activity_id:
+        fractal_ssh:
+        tasks_base_dir:
+            Only used as a `safe_root` in `remove_dir`, and typically set to
+            `user_settings.ssh_tasks_dir`.
+    """
+
+    with TemporaryDirectory() as tmpdir:
+        log_file_path = get_log_path(Path(tmpdir))
+        logger = set_logger(
+            logger_name=LOGGER_NAME,
+            log_file_path=log_file_path,
+        )
+
+        with next(get_sync_db()) as db:
+
+            # Get main objects from db
+            activity = db.get(TaskGroupActivityV2, task_group_activity_id)
+            task_group = db.get(TaskGroupV2, task_group_id)
+            if activity is None or task_group is None:
+                # Use `logging` directly
+                logging.error(
+                    "Cannot find database rows with "
+                    f"{task_group_id=} and {task_group_activity_id=}:\n"
+                    f"{task_group=}\n{activity=}. Exit."
+                )
+                return
+
+            # Log some info
+            logger.debug("START")
+            for key, value in task_group.model_dump().items():
+                logger.debug(f"task_group.{key}: {value}")
+
+            # Check that SSH connection works
+            try:
+                fractal_ssh.check_connection()
+            except Exception as e:
+                logger.error("Cannot establish SSH connection.")
+                fail_and_cleanup(
+                    task_group=task_group,
+                    task_group_activity=activity,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=e,
+                    db=db,
+                )
+                return
+
+            # Check that the (local) task_group venv_path does exist
+            if not fractal_ssh.remote_exists(task_group.venv_path):
+                error_msg = f"{task_group.venv_path} does not exist."
+                logger.error(error_msg)
+                fail_and_cleanup(
+                    task_group=task_group,
+                    task_group_activity=activity,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=FileNotFoundError(error_msg),
+                    db=db,
+                )
+                return
+
+            try:
+
+                activity.status = TaskGroupActivityStatusV2.ONGOING
+                activity = add_commit_refresh(obj=activity, db=db)
+
+                if task_group.pip_freeze is None:
+                    logger.warning(
+                        "Recreate pip-freeze information, since "
+                        f"{task_group.pip_freeze=}. NOTE: this should only "
+                        "happen for task groups created before 2.9.0."
+                    )
+
+                    # Prepare replacements for templates
+                    replacements = get_collection_replacements(
+                        task_group=task_group,
+                        python_bin="/not/applicable",
+                    )
+
+                    # Prepare arguments for `_customize_and_run_template`
+                    script_dir_remote = (
+                        Path(task_group.path) / SCRIPTS_SUBFOLDER
+                    ).as_posix()
+                    common_args = dict(
+                        replacements=replacements,
+                        script_dir_local=(
+                            Path(tmpdir) / SCRIPTS_SUBFOLDER
+                        ).as_posix(),
+                        script_dir_remote=script_dir_remote,
+                        prefix=(
+                            f"{int(time.time())}_"
+                            f"{TaskGroupActivityActionV2.DEACTIVATE}"
+                        ),
+                        fractal_ssh=fractal_ssh,
+                        logger_name=LOGGER_NAME,
+                    )
+
+                    # Run `pip freeze`
+                    pip_freeze_stdout = _customize_and_run_template(
+                        template_filename="3_pip_freeze.sh",
+                        **common_args,
+                    )
+
+                    # Update pip-freeze data
+                    logger.info("Add pip freeze stdout to TaskGroupV2 - start")
+                    activity.log = get_current_log(log_file_path)
+                    activity = add_commit_refresh(obj=activity, db=db)
+                    task_group.pip_freeze = pip_freeze_stdout
+                    task_group = add_commit_refresh(obj=task_group, db=db)
+                    logger.info("Add pip freeze stdout to TaskGroupV2 - end")
+
+                # Handle some specific cases for wheel-file case
+                if task_group.origin == TaskGroupV2OriginEnum.WHEELFILE:
+
+                    logger.info(
+                        f"Handle specific cases for {task_group.origin=}."
+                    )
+
+                    # Blocking situation: `wheel_path` is not set or points
+                    # to a missing path
+                    if (
+                        task_group.wheel_path is None
+                        or not fractal_ssh.remote_exists(task_group.wheel_path)
+                    ):
+                        error_msg = (
+                            "Invalid wheel path for task group with "
+                            f"{task_group_id=}. {task_group.wheel_path=} is "
+                            "unset or does not exist."
+                        )
+                        logger.error(error_msg)
+                        fail_and_cleanup(
+                            task_group=task_group,
+                            task_group_activity=activity,
+                            logger_name=LOGGER_NAME,
+                            log_file_path=log_file_path,
+                            exception=FileNotFoundError(error_msg),
+                            db=db,
+                        )
+                        return
+
+                    # Recoverable situation: `wheel_path` was not yet copied
+                    # over to the correct server-side folder
+                    wheel_path_parent_dir = Path(task_group.wheel_path).parent
+                    if wheel_path_parent_dir != Path(task_group.path):
+                        logger.warning(
+                            f"{wheel_path_parent_dir.as_posix()} differs from "
+                            f"{task_group.path}. NOTE: this should only "
+                            "happen for task groups created before 2.9.0."
+                        )
+
+                        if task_group.wheel_path not in task_group.pip_freeze:
+                            raise ValueError(
+                                f"Cannot find {task_group.wheel_path=} in "
+                                "pip-freeze data. Exit."
+                            )
+
+                        logger.info(
+                            f"Now copy wheel file into {task_group.path}."
+                        )
+                        new_wheel_path = _copy_wheel_file_ssh(
+                            task_group=task_group,
+                            fractal_ssh=fractal_ssh,
+                            logger_name=LOGGER_NAME,
+                        )
+                        logger.info(f"Copied wheel file to {new_wheel_path}.")
+
+                        task_group.wheel_path = new_wheel_path
+                        new_pip_freeze = task_group.pip_freeze.replace(
+                            task_group.wheel_path,
+                            new_wheel_path,
+                        )
+                        task_group.pip_freeze = new_pip_freeze
+                        task_group = add_commit_refresh(obj=task_group, db=db)
+                        logger.info(
+                            "Updated `wheel_path` and `pip_freeze` "
+                            "task-group attributes."
+                        )
+
+                # We now have all required information for reactivating the
+                # virtual environment at a later point
+                logger.info(f"Now removing {task_group.venv_path}.")
+                fractal_ssh.remove_folder(
+                    folder=task_group.venv_path,
+                    safe_root=tasks_base_dir,
+                )
+                logger.info(f"All good, {task_group.venv_path} removed.")
+                activity.status = TaskGroupActivityStatusV2.OK
+                activity.log = get_current_log(log_file_path)
+                activity.timestamp_ended = get_timestamp()
+                activity = add_commit_refresh(obj=activity, db=db)
+
+            except Exception as e:
+                fail_and_cleanup(
+                    task_group=task_group,
+                    task_group_activity=activity,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=e,
+                    db=db,
+                )
+        return

fractal_server/tasks/v2/ssh/reactivate.py

@@ -1,2 +1,199 @@
-def reactivate_ssh():
-    pass
+import logging
+import time
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from ..utils_background import add_commit_refresh
+from ..utils_background import fail_and_cleanup
+from ..utils_templates import get_collection_replacements
+from ._utils import _customize_and_run_template
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models.v2 import TaskGroupActivityV2
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
+from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
+from fractal_server.logger import set_logger
+from fractal_server.ssh._fabric import FractalSSH
+from fractal_server.tasks.utils import get_log_path
+from fractal_server.tasks.v2.utils_background import get_current_log
+from fractal_server.tasks.v2.utils_python_interpreter import (
+    get_python_interpreter_v2,
+)
+from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
+from fractal_server.utils import get_timestamp
+
+LOGGER_NAME = __name__
+
+
+def reactivate_ssh(
+    *,
+    task_group_activity_id: int,
+    task_group_id: int,
+    fractal_ssh: FractalSSH,
+    tasks_base_dir: str,
+) -> None:
+    """
+    Reactivate a task group venv.
+
+    This function is run as a background task, therefore exceptions must be
+    handled.
+
+    Arguments:
+        task_group_id:
+        task_group_activity_id:
+        fractal_ssh:
+        tasks_base_dir:
+            Only used as a `safe_root` in `remove_dir`, and typically set to
+            `user_settings.ssh_tasks_dir`.
+    """
+
+    with TemporaryDirectory() as tmpdir:
+        log_file_path = get_log_path(Path(tmpdir))
+        logger = set_logger(
+            logger_name=LOGGER_NAME,
+            log_file_path=log_file_path,
+        )
+
+        with next(get_sync_db()) as db:
+
+            # Get main objects from db
+            activity = db.get(TaskGroupActivityV2, task_group_activity_id)
+            task_group = db.get(TaskGroupV2, task_group_id)
+            if activity is None or task_group is None:
+                # Use `logging` directly
+                logging.error(
+                    "Cannot find database rows with "
+                    f"{task_group_id=} and {task_group_activity_id=}:\n"
+                    f"{task_group=}\n{activity=}. Exit."
+                )
+                return
+
+            # Log some info
+            logger.debug("START")
+            for key, value in task_group.model_dump().items():
+                logger.debug(f"task_group.{key}: {value}")
+
+            # Check that SSH connection works
+            try:
+                fractal_ssh.check_connection()
+            except Exception as e:
+                logger.error("Cannot establish SSH connection.")
+                fail_and_cleanup(
+                    task_group=task_group,
+                    task_group_activity=activity,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=e,
+                    db=db,
+                )
+                return
+
+            # Check that the (remote) task_group venv_path does not exist
+            if fractal_ssh.remote_exists(task_group.venv_path):
+                error_msg = f"{task_group.venv_path} already exists."
+                logger.error(error_msg)
+                fail_and_cleanup(
+                    task_group=task_group,
+                    task_group_activity=activity,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=FileExistsError(error_msg),
+                    db=db,
+                )
+                return
+
+            try:
+                activity.status = TaskGroupActivityStatusV2.ONGOING
+                activity = add_commit_refresh(obj=activity, db=db)
+
+                # Prepare replacements for templates
+                replacements = get_collection_replacements(
+                    task_group=task_group,
+                    python_bin=get_python_interpreter_v2(
+                        python_version=task_group.python_version
+                    ),
+                )
+
+                # Prepare replacements for templates
+                pip_freeze_file_local = f"{tmpdir}/pip_freeze.txt"
+                pip_freeze_file_remote = (
+                    Path(task_group.path) / "_tmp_pip_freeze.txt"
+                ).as_posix()
+                with open(pip_freeze_file_local, "w") as f:
+                    f.write(task_group.pip_freeze)
+                fractal_ssh.send_file(
+                    local=pip_freeze_file_local, remote=pip_freeze_file_remote
+                )
+                replacements.append(
+                    ("__PIP_FREEZE_FILE__", pip_freeze_file_remote)
+                )
+
+                # Prepare common arguments for `_customize_and_run_template``
+                script_dir_remote = (
+                    Path(task_group.path) / SCRIPTS_SUBFOLDER
+                ).as_posix()
+                common_args = dict(
+                    replacements=replacements,
+                    script_dir_local=(
+                        Path(tmpdir) / SCRIPTS_SUBFOLDER
+                    ).as_posix(),
+                    script_dir_remote=script_dir_remote,
+                    prefix=(
+                        f"{int(time.time())}_"
+                        f"{TaskGroupActivityActionV2.REACTIVATE}"
+                    ),
+                    fractal_ssh=fractal_ssh,
+                    logger_name=LOGGER_NAME,
+                )
+
+                # Create remote directory for scripts
+                fractal_ssh.mkdir(folder=script_dir_remote)
+
+                logger.debug("start - create venv")
+                _customize_and_run_template(
+                    template_filename="1_create_venv.sh",
+                    **common_args,
+                )
+                logger.debug("end - create venv")
+                activity.log = get_current_log(log_file_path)
+                activity.timestamp_ended = get_timestamp()
+                activity = add_commit_refresh(obj=activity, db=db)
+
+                logger.debug("start - install from pip freeze")
+                _customize_and_run_template(
+                    template_filename="6_pip_install_from_freeze.sh",
+                    **common_args,
+                )
+                logger.debug("end - install from pip freeze")
+                activity.log = get_current_log(log_file_path)
+                activity.status = TaskGroupActivityStatusV2.OK
+                activity.timestamp_ended = get_timestamp()
+                activity = add_commit_refresh(obj=activity, db=db)
+                task_group.active = True
+                task_group = add_commit_refresh(obj=task_group, db=db)
+                logger.debug("END")
+
+            except Exception as reactivate_e:
+                # Delete corrupted venv_path
+                try:
+                    logger.info(f"Now delete folder {task_group.venv_path}")
+                    fractal_ssh.remove_folder(
+                        folder=task_group.venv_path,
+                        safe_root=tasks_base_dir,
+                    )
+                    logger.info(f"Deleted folder {task_group.venv_path}")
+                except Exception as rm_e:
+                    logger.error(
+                        "Removing folder failed.\n"
+                        f"Original error:\n{str(rm_e)}"
+                    )
+
+                fail_and_cleanup(
+                    task_group=task_group,
+                    task_group_activity=activity,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=reactivate_e,
+                    db=db,
+                )
+        return

fractal_server/tasks/v2/utils_background.py

@@ -119,30 +119,6 @@ def _prepare_tasks_metadata(
     return task_list
 
 
-def check_task_files_exist(task_list: list[TaskCreateV2]) -> None:
-    """
-    Check that the modules listed in task commands point to existing files.
-
-    Args:
-        task_list:
-    """
-    for _task in task_list:
-        if _task.command_non_parallel is not None:
-            _task_path = _task.command_non_parallel.split()[1]
-            if not Path(_task_path).exists():
-                raise FileNotFoundError(
-                    f"Task `{_task.name}` has `command_non_parallel` "
-                    f"pointing to missing file `{_task_path}`."
-                )
-        if _task.command_parallel is not None:
-            _task_path = _task.command_parallel.split()[1]
-            if not Path(_task_path).exists():
-                raise FileNotFoundError(
-                    f"Task `{_task.name}` has `command_parallel` "
-                    f"pointing to missing file `{_task_path}`."
-                )
-
-
 def get_current_log(logger_file_path: str) -> str:
     with open(logger_file_path, "r") as f:
         return f.read()

fractal_server/zip_tools.py

@@ -122,9 +122,26 @@ def _zip_folder_to_file_and_remove(folder: str) -> None:
     3. Checks if the folder can be safely deleted using the
         `_folder_can_be_deleted` function. If so, deletes the original folder.
     """
-    _create_zip(folder, f"{folder}_tmp.zip")
-    shutil.move(f"{folder}_tmp.zip", f"{folder}.zip")
+
+    tmp_zipfile = f"{folder}_tmp.zip"
+    zipfile = f"{folder}.zip"
+
+    try:
+        logger.info(f"Start creating temporary zip file at '{tmp_zipfile}'.")
+        _create_zip(folder, tmp_zipfile)
+        logger.info("Zip file created.")
+    except Exception as e:
+        logger.error(
+            f"Error while creating temporary zip file. Original error: '{e}'."
+        )
+        Path(tmp_zipfile).unlink(missing_ok=True)
+        return
+
+    logger.info(f"Moving temporary zip file to {zipfile}.")
+    shutil.move(tmp_zipfile, zipfile)
+    logger.info("Zip file moved.")
+
     if _folder_can_be_deleted(folder):
-        logger.info(f"Deleting folder '{folder}'...")
+        logger.info(f"Removing folder '{folder}'.")
         shutil.rmtree(folder)
-        logger.info(f"Deletion of folder '{folder}' completed.")
+        logger.info("Folder removed.")

{fractal_server-2.9.0a2.dist-info → fractal_server-2.9.0a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.9.0a2
+Version: 2.9.0a3
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

{fractal_server-2.9.0a2.dist-info → fractal_server-2.9.0a3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=d0Y48obcx61ZaWA7ZS1o1kdZRZ6sJMhUfuKDOT5xPKU,24
+fractal_server/__init__.py,sha256=8LdiYD77VYwRKrVoavHTgYHkexYzgn-F5OpNKKf-WMU,24
 fractal_server/__main__.py,sha256=dEkCfzLLQrIlxsGC-HBfoR-RBMWnJDgNrxYTyzmE9c0,6146
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -52,10 +52,10 @@ fractal_server/app/routes/api/v2/project.py,sha256=eWYFJ7F2ZYQcpi-_n-rhPF-Q4gJhz
 fractal_server/app/routes/api/v2/status.py,sha256=6N9DSZ4iFqbZImorWfEAPoyoFUgEruo4Hweqo0x0xXU,6435
 fractal_server/app/routes/api/v2/submit.py,sha256=jqYix7X6dUJn8w6RWasu1lOFf7T_CcXQlpVY38njE24,8688
 fractal_server/app/routes/api/v2/task.py,sha256=K0ik33t7vL8BAK5S7fqyJDNdRK4stGqb_73bSa8tvPE,7159
-fractal_server/app/routes/api/v2/task_collection.py,sha256=-DVhultvdI3Jh8Jq8W5np6Lnkh5oisjbKCwxFmwddmo,9820
+fractal_server/app/routes/api/v2/task_collection.py,sha256=_K7c-83jpmaLz1gXM_MbndskkmV6oo_ivg0mWP6hfvs,9621
 fractal_server/app/routes/api/v2/task_collection_custom.py,sha256=cctW61-C2QYF2KXluS15lLhZJS_kt30Ca6UGLFO32z0,6207
 fractal_server/app/routes/api/v2/task_group.py,sha256=Ove7Vr3p8GKtskHANAZh8TWwOw8dfbFCN2UQFv6DhqM,8136
-fractal_server/app/routes/api/v2/task_group_lifecycle.py,sha256=UODAv1kk8qgT7wvcVKK6eRs-fEBw3T1pPwmT6VkkAn4,7404
+fractal_server/app/routes/api/v2/task_group_lifecycle.py,sha256=SPUB-QHvADaJm3zyuPIyCg-TqbHujwiwr7RauvYGXJ0,8972
 fractal_server/app/routes/api/v2/workflow.py,sha256=vjCNRzMHaAB4YWbAEWGlELHXDN4GjtE26IkIiB15RGM,8682
 fractal_server/app/routes/api/v2/workflow_import.py,sha256=WJST1AZypvOTGUrjhomYVh4R2ow8RoGpuwzNiq81Pzc,10971
 fractal_server/app/routes/api/v2/workflowtask.py,sha256=ciHTwXXFiFnMF7ZpJ3Xs0q6YfuZrFvIjqndlzAEdZpo,6969
@@ -134,31 +134,31 @@ fractal_server/app/runner/v2/runner_functions_low_level.py,sha256=1fWvQ6YZUUnDhO
 fractal_server/app/runner/v2/task_interface.py,sha256=hT3p-bRGsLNAR_dNv_PYFoqzIF_EQtSsGwl38j1haYA,1824
 fractal_server/app/runner/versions.py,sha256=dSaPRWqmFPHjg20kTCHmi_dmGNcCETflDtDLronNanU,852
 fractal_server/app/schemas/__init__.py,sha256=stURAU_t3AOBaH0HSUbV-GKhlPKngnnIMoqWc3orFyI,135
-fractal_server/app/schemas/_validators.py,sha256=Ft-Ueol_rFwGjEebKVK2bUELm7w7bFG8MOpTamlY8Fs,3503
+fractal_server/app/schemas/_validators.py,sha256=T5EswIJAJRvawfzqWtPcN2INAfiBXyE4m0iwQm4ht-0,3149
 fractal_server/app/schemas/user.py,sha256=aUD8YAcfYTEO06TEUoTx4heVrXFiX7E2Mb8D2--4FsA,2130
 fractal_server/app/schemas/user_group.py,sha256=YwJvYgj-PI66LWy38CEd_FIZPsBV1_2N5zJPGFcFvBw,2143
 fractal_server/app/schemas/user_settings.py,sha256=TalISeEfCrtN8LgqbLx1Q8ZPoeiZnbksg5NYAVzkIqY,3527
 fractal_server/app/schemas/v1/__init__.py,sha256=CrBGgBhoemCvmZ70ZUchM-jfVAICnoa7AjZBAtL2UB0,1852
-fractal_server/app/schemas/v1/applyworkflow.py,sha256=uuIh7fHlHEL4yLqL-dePI6-nfCsqgBYATmht7w_KITw,4302
-fractal_server/app/schemas/v1/dataset.py,sha256=n71lNUO3JLy2K3IM9BZM2Fk1EnKQOTU7pm2s2rJ1FGY,3444
+fractal_server/app/schemas/v1/applyworkflow.py,sha256=dYArxQAOBdUIEXX_Ejz8b9fBhEYu1nMm6b_Z6_P6TgA,4052
+fractal_server/app/schemas/v1/dataset.py,sha256=DWFCxZjApcKt2M6UJMK0tmejXwUT09vjUULf2D7Y-f0,3293
 fractal_server/app/schemas/v1/dumps.py,sha256=67VXnyLh_0Ufo7rPM2jZ9P9rk0CnYcVAkilx_cLX6sg,1274
 fractal_server/app/schemas/v1/manifest.py,sha256=Yht7guhs0Pcl2U0RMOCbI_UHBZ9YO_YU0H8hxACx3TY,3829
-fractal_server/app/schemas/v1/project.py,sha256=TO2TjI4m9FO-A9IB9lUCld7E4Ld0k4MacLcyA9j6Qi4,1218
-fractal_server/app/schemas/v1/state.py,sha256=GYeOE_1PtDOgu5W4t_3gw3DBHXH2aCGzINRs2dp8_h4,472
+fractal_server/app/schemas/v1/project.py,sha256=Zxd-AguQQG9z2CfJ_sJh5SB9WcHPFbWpLgP_AhjOyZs,1067
+fractal_server/app/schemas/v1/state.py,sha256=tBXzp_qW2TNNNPBo-AWEaffEU-1GkMBtUoaMgiN_EL0,302
 fractal_server/app/schemas/v1/task.py,sha256=7BxOZ_qoRQ8n3YbQpDvB7VMcxB5fSYQmR5RLIWhuJ5U,3704
 fractal_server/app/schemas/v1/task_collection.py,sha256=uvq9bcMaGD_qHsh7YtcpoSAkVAbw12eY4DocIO3MKOg,3057
-fractal_server/app/schemas/v1/workflow.py,sha256=tuOs5E5Q_ozA8if7YPZ07cQjzqB_QMkBS4u92qo4Ro0,4618
+fractal_server/app/schemas/v1/workflow.py,sha256=oRKamLSuAgrTcv3gMMxGcotDloLL2c3NNgPA39UEmmM,4467
 fractal_server/app/schemas/v2/__init__.py,sha256=dzDsAzLLMgyQBa3b64n71K1u83tbBn9_Oloaqqv1tjA,2444
-fractal_server/app/schemas/v2/dataset.py,sha256=Jipcj9LiOOipAeM2Ew113wuNQ6CrbC1nf3KwnNApBco,2638
+fractal_server/app/schemas/v2/dataset.py,sha256=zRlcO0wDZahTW1PINdVEuARZ7GZUuEqqop7UdE3-5do,2470
 fractal_server/app/schemas/v2/dumps.py,sha256=s6dg-pHZFui6t2Ktm0SMxjKDN-v-ZqBHz9iTsBQF3eU,1712
-fractal_server/app/schemas/v2/job.py,sha256=oYSLYkQ0HL83QyjEGIaggtZ117FndzFlONMKWd9sTXM,3270
+fractal_server/app/schemas/v2/job.py,sha256=42V-bFfMvysRplwTKGsL_WshAVsWSM6yjFqypxwrY3k,3020
 fractal_server/app/schemas/v2/manifest.py,sha256=Uqtd7DbyOkf9bxBOKkU7Sv7nToBIFGUcfjY7rd5iO7c,6981
-fractal_server/app/schemas/v2/project.py,sha256=UXEA0UUUe0bFFOVLLmVtvDFLBO5vmD1JVI7EeTIcwDo,756
+fractal_server/app/schemas/v2/project.py,sha256=ABv9LSLVCq1QYthEhBvZOTn_4DFEC-7cH28tFGFdM7I,589
 fractal_server/app/schemas/v2/status.py,sha256=SQaUpQkjFq5c5k5J4rOjNhuQaDOEg8lksPhkKmPU5VU,332
 fractal_server/app/schemas/v2/task.py,sha256=FFAbYwDlqowB8gVMdjFVPVHvAM0T89PYLixUth49xfQ,6870
 fractal_server/app/schemas/v2/task_collection.py,sha256=yHpCRxoj6tKqCiQfUjaTj8SfCn1ChD_P6okfEOzyUDE,6518
 fractal_server/app/schemas/v2/task_group.py,sha256=e4NwFuOmiO0afoZLVsI_XHIpD_o_QWDpzlI7ZoNAYwo,3014
-fractal_server/app/schemas/v2/workflow.py,sha256=HSNQSrBRdoBzh8Igr76FUWCAWvVzykrqmUv1vGv-8og,2026
+fractal_server/app/schemas/v2/workflow.py,sha256=-KWvXnbHBFA3pj5n7mfSyLKJQSqkJmoziIEe7mpLl3M,1875
 fractal_server/app/schemas/v2/workflowtask.py,sha256=vDdMktYbHeYBgB5OuWSv6wRPRXWqvetkeqQ7IC5YtfA,5751
 fractal_server/app/security/__init__.py,sha256=8Xd4GxumZgvxEH1Vli3ULehwdesEPiaAbtffJvAEgNo,12509
 fractal_server/app/user_settings.py,sha256=aZgQ3i0JkHfgwLGW1ee6Gzr1ae3IioFfJKKSsSS8Svk,1312
@@ -202,7 +202,7 @@ fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py
 fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py,sha256=9BwqUS9Gf7UW_KjrzHbtViC880qhD452KAytkHWWZyk,746
 fractal_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/ssh/__init__.py,sha256=sVUmzxf7_DuXG1xoLQ1_00fo5NPhi2LJipSmU5EAkPs,124
-fractal_server/ssh/_fabric.py,sha256=Nwsc5uU2BcOE7t1AFFpytKUsZHfbQxB0uKF5HD-dycA,21647
+fractal_server/ssh/_fabric.py,sha256=O1Dxl6xlg9pvqdKyKqy18mYQlslJoJHapXtDMCzZcBA,21711
 fractal_server/string_tools.py,sha256=XtMNsr5R7GmgzmFi68zkKMedHs8vjGoVMMCXqWhIk9k,2568
 fractal_server/syringe.py,sha256=3qSMW3YaMKKnLdgnooAINOPxnCOxP7y2jeAQYB21Gdo,2786
 fractal_server/tasks/__init__.py,sha256=kadmVUoIghl8s190_Tt-8f-WBqMi8u8oU4Pvw39NHE8,23
@@ -215,30 +215,31 @@ fractal_server/tasks/v1/get_collection_data.py,sha256=5C22jp356rCH5IIC0J57wOu-DC
 fractal_server/tasks/v1/utils.py,sha256=HYFyNAyZofmf--mVgdwGC5TJpGShIWIDaS01yRr4HxM,1771
 fractal_server/tasks/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/tasks/v2/local/__init__.py,sha256=9RVItnS7OyLsJOuJjWMCicaky4ASUPQEYD4SzDs0hOE,141
-fractal_server/tasks/v2/local/collect.py,sha256=kTgMDiOX9qVDncmrgieOfYE71jqFwBpDgO1cbD5ZQVQ,12199
-fractal_server/tasks/v2/local/deactivate.py,sha256=PyJm3PoQrYOBCm0F6p6xD7Gx9Tfnn_YZlXY-j3Fi6pM,8975
-fractal_server/tasks/v2/local/reactivate.py,sha256=eTSrEoZ54_6ExviozxRTgjNsrESG9PbQW-TiGqNMJcA,6117
-fractal_server/tasks/v2/local/utils_local.py,sha256=JHHiS_SvPLSezxGWSWslMhLEWmMp9PepnzePXV6r0e8,1521
-fractal_server/tasks/v2/ssh/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/tasks/v2/ssh/collect.py,sha256=j50euINjxGS4S5-aYTIWwV34xydKvT6W5P9Vxv5TMuk,15263
-fractal_server/tasks/v2/ssh/deactivate.py,sha256=fwBCtj-8I_8s8Zd-nyQX7YaTjoPOK1pn07Tznvjbv8Y,31
-fractal_server/tasks/v2/ssh/reactivate.py,sha256=0pqK-g5uyykCkLsjYDVr8QTEBrIB31XQXvi0MQfqv-w,31
+fractal_server/tasks/v2/local/_utils.py,sha256=EvhmVwYjqaNyDCUMEsTWYOUXLgEwR1xr6bu32apCEI8,2491
+fractal_server/tasks/v2/local/collect.py,sha256=BbXSgxExPUxFxcmBs3ejwWzRae-sQgfbk3zZkAQg77Y,12190
+fractal_server/tasks/v2/local/deactivate.py,sha256=Tjooode2oXfaSLiJz7iSu-qC1OkeIJtOQeUvUo4K4Ts,8970
+fractal_server/tasks/v2/local/reactivate.py,sha256=R3rArAzUpMGf6xa3dGVwwXHW9WVDi5ia28AFisZsqNc,6112
+fractal_server/tasks/v2/ssh/__init__.py,sha256=aSQbVi6Ummt9QzcSLWNmSqYjfdxrn9ROmqgH6bDpI7k,135
+fractal_server/tasks/v2/ssh/_utils.py,sha256=2E-F_862zM6FZA-im-E8t8kjptWRIhBj1IDHC6QD1H8,2818
+fractal_server/tasks/v2/ssh/collect.py,sha256=U8f6-dgtAI9Pr-gWPeDMeYdOkxYSqfsrYcSPDYEqz4I,12942
+fractal_server/tasks/v2/ssh/deactivate.py,sha256=YNJEcUPNuQ7H3I9EtWn84es2AY9Dnij388w6PQra4wc,10186
+fractal_server/tasks/v2/ssh/reactivate.py,sha256=brIa3XuoD7g9m9vIB7CSgog6KC9w8e45phLT-vSSXlI,7754
 fractal_server/tasks/v2/templates/1_create_venv.sh,sha256=PK0jdHKtQpda1zULebBaVPORt4t6V17wa4N1ohcj5ac,548
 fractal_server/tasks/v2/templates/2_pip_install.sh,sha256=RDDfbFnGOK3aRuHyXqDOUNCGullzAr0zS7BFqG1CJeE,1720
 fractal_server/tasks/v2/templates/3_pip_freeze.sh,sha256=JldREScEBI4cD_qjfX4UK7V4aI-FnX9ZvVNxgpSOBFc,168
 fractal_server/tasks/v2/templates/4_pip_show.sh,sha256=84NGHlg6JIbrQktgGKyfGsggPFzy6RBJuOmIpPUhsrw,1747
 fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh,sha256=q-6ZUvA6w6FDVEoSd9O63LaJ9tKZc7qAFH72SGPrd_k,284
 fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh,sha256=n9C8w76YraLbeTe7NhuLzvAQiJCm_akL3Mc3EMfxrHo,1007
-fractal_server/tasks/v2/utils_background.py,sha256=nAnel85xWIWPJjlv1XtRBJ59zoSjufvLobU8GlPRWUI,5155
+fractal_server/tasks/v2/utils_background.py,sha256=tikXhggqxdU7EnKdx2co3UwinlDazEjfOPQOXtO58zs,4240
 fractal_server/tasks/v2/utils_database.py,sha256=6r56yyFPnEBrXl6ncmO6D76znzISQCFZqCYcD-Ummd4,1213
 fractal_server/tasks/v2/utils_package_names.py,sha256=RDg__xrvQs4ieeVzmVdMcEh95vGQYrv9Hfal-5EDBM8,2393
 fractal_server/tasks/v2/utils_python_interpreter.py,sha256=-EWh3Y3VqHLDOWUO_wG_wknqmGqKAD0O2KTLhNjrZaI,948
 fractal_server/tasks/v2/utils_templates.py,sha256=C5WLuY3uGG2s53OEL-__H35-fmSlguwZx836BPFHBpE,2732
 fractal_server/urls.py,sha256=5o_qq7PzKKbwq12NHSQZDmDitn5RAOeQ4xufu-2v9Zk,448
 fractal_server/utils.py,sha256=utvmBx8K9I8hRWFquxna2pBaOqe0JifDL_NVPmihEJI,3525
-fractal_server/zip_tools.py,sha256=xYpzBshysD2nmxkD5WLYqMzPYUcCRM3kYy-7n9bJL-U,4426
-fractal_server-2.9.0a2.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.9.0a2.dist-info/METADATA,sha256=Qk7nGzAcl_hZRCJp8pdCV2PQACAWwqOzTCH7nFW__pw,4585
-fractal_server-2.9.0a2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-fractal_server-2.9.0a2.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.9.0a2.dist-info/RECORD,,
+fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
+fractal_server-2.9.0a3.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.9.0a3.dist-info/METADATA,sha256=W92NgqXaqh1eO8X_o9XPN8chROYn0ZyszQCjkBo_BIg,4585
+fractal_server-2.9.0a3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fractal_server-2.9.0a3.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.9.0a3.dist-info/RECORD,,