fractal-server 2.5.0a1__tar.gz → 2.5.2__tar.gz

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.5.0a1
+Version: 2.5.2
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.5.2/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.5.2"

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/routes/api/v2/submit.py

@@ -238,6 +238,18 @@ async def apply_workflow(
     await db.merge(job)
     await db.commit()
 
+    # User appropriate FractalSSH object
+    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+        ssh_credentials = dict(
+            user=settings.FRACTAL_SLURM_SSH_USER,
+            host=settings.FRACTAL_SLURM_SSH_HOST,
+            key_path=settings.FRACTAL_SLURM_SSH_PRIVATE_KEY_PATH,
+        )
+        fractal_ssh_list = request.app.state.fractal_ssh_list
+        fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
+    else:
+        fractal_ssh = None
+
     background_tasks.add_task(
         submit_workflow,
         workflow_id=workflow.id,
@@ -246,7 +258,7 @@ async def apply_workflow(
         worker_init=job.worker_init,
         slurm_user=user.slurm_user,
         user_cache_dir=user.cache_dir,
-        fractal_ssh=request.app.state.fractal_ssh,
+        fractal_ssh=fractal_ssh,
     )
     request.app.state.jobsV2.append(job.id)
     logger.info(

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/routes/api/v2/task_collection.py

@@ -124,11 +124,20 @@ async def collect_tasks_pip(
         db.add(state)
         await db.commit()
 
+        # User appropriate FractalSSH object
+        ssh_credentials = dict(
+            user=settings.FRACTAL_SLURM_SSH_USER,
+            host=settings.FRACTAL_SLURM_SSH_HOST,
+            key_path=settings.FRACTAL_SLURM_SSH_PRIVATE_KEY_PATH,
+        )
+        fractal_ssh_list = request.app.state.fractal_ssh_list
+        fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
+
         background_tasks.add_task(
             background_collect_pip_ssh,
             state.id,
             task_pkg,
-            request.app.state.fractal_ssh,
+            fractal_ssh,
         )
 
         response.status_code = status.HTTP_201_CREATED

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/routes/api/v2/task_collection_custom.py

@@ -20,12 +20,12 @@ from ....schemas.v2 import TaskCreateV2
 from ....schemas.v2 import TaskReadV2
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_verified_user
+from fractal_server.string_tools import validate_cmd
 from fractal_server.tasks.v2.background_operations import _insert_tasks
 from fractal_server.tasks.v2.background_operations import (
     _prepare_tasks_metadata,
 )
 
-
 router = APIRouter()
 
 logger = set_logger(__name__)
@@ -74,6 +74,7 @@ async def collect_task_custom(
         package_name_underscore = task_collect.package_name.replace("-", "_")
         # Note that python_command is then used as part of a subprocess.run
         # statement: be careful with mixing `'` and `"`.
+        validate_cmd(package_name_underscore)
         python_command = (
             "import importlib.util; "
             "from pathlib import Path; "

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/runner/compress_folder.py

@@ -48,7 +48,7 @@ def create_tar_archive(
         "."
     )
     logger.debug(f"cmd tar:\n{cmd_tar}")
-    run_subprocess(cmd=cmd_tar, logger_name=logger_name)
+    run_subprocess(cmd=cmd_tar, logger_name=logger_name, allow_char="*")
 
 
 def remove_temp_subfolder(subfolder_path_tmp_copy: Path, logger_name: str):

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/runner/executors/slurm/ssh/executor.py

@@ -163,17 +163,34 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         settings = Inject(get_settings)
         self.python_remote = settings.FRACTAL_SLURM_WORKER_PYTHON
         if self.python_remote is None:
+            self._stop_and_join_wait_thread()
             raise ValueError("FRACTAL_SLURM_WORKER_PYTHON is not set. Exit.")
 
         # Initialize connection and perform handshake
         self.fractal_ssh = fractal_ssh
         logger.warning(self.fractal_ssh)
-        self.handshake()
+        try:
+            self.handshake()
+        except Exception as e:
+            logger.warning(
+                "Stop/join waiting thread and then "
+                f"re-raise original error {str(e)}"
+            )
+            self._stop_and_join_wait_thread()
+            raise e
 
         # Set/validate parameters for SLURM submission scripts
         self.slurm_account = slurm_account
         self.common_script_lines = common_script_lines or []
-        self._validate_common_script_lines()
+        try:
+            self._validate_common_script_lines()
+        except Exception as e:
+            logger.warning(
+                "Stop/join waiting thread and then "
+                f"re-raise original error {str(e)}"
+            )
+            self._stop_and_join_wait_thread()
+            raise e
 
         # Set/initialize some more options
         self.keep_pickle_files = keep_pickle_files
@@ -1385,6 +1402,10 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             self.fractal_ssh.run_command(cmd=scancel_command)
         logger.debug("Executor shutdown: end")
 
+    def _stop_and_join_wait_thread(self):
+        self.wait_thread.stop()
+        self.wait_thread.join()
+
     def __exit__(self, *args, **kwargs):
         """
         See
@@ -1393,8 +1414,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         logger.debug(
             "[FractalSlurmSSHExecutor.__exit__] Stop and join `wait_thread`"
         )
-        self.wait_thread.stop()
-        self.wait_thread.join()
+        self._stop_and_join_wait_thread()
         logger.debug("[FractalSlurmSSHExecutor.__exit__] End")
 
     def run_squeue(self, job_ids):

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/runner/executors/slurm/sudo/_subprocess_run_as_user.py

@@ -20,6 +20,7 @@ import subprocess  # nosec
 from typing import Optional
 
 from ......logger import set_logger
+from fractal_server.string_tools import validate_cmd
 
 logger = set_logger(__name__)
 
@@ -47,6 +48,7 @@ def _run_command_as_user(
     Returns:
         res: The return value from `subprocess.run`.
     """
+    validate_cmd(cmd)
     logger.debug(f'[_run_command_as_user] {user=}, cmd="{cmd}"')
     if user:
         new_cmd = f"sudo --set-home --non-interactive -u {user} {cmd}"

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/runner/executors/slurm/sudo/executor.py

@@ -47,6 +47,7 @@ from ._subprocess_run_as_user import _path_exists_as_user
 from ._subprocess_run_as_user import _run_command_as_user
 from fractal_server import __VERSION__
 from fractal_server.app.runner.components import _COMPONENT_KEY_
+from fractal_server.string_tools import validate_cmd
 
 
 logger = set_logger(__name__)
@@ -65,6 +66,7 @@ def _subprocess_run_or_raise(full_command: str) -> Optional[CompletedProcess]:
     Returns:
         The actual `CompletedProcess` output of `subprocess.run`.
     """
+    validate_cmd(full_command)
     try:
         output = subprocess.run(  # nosec
             shlex.split(full_command),
@@ -257,6 +259,7 @@ class FractalSlurmExecutor(SlurmExecutor):
                 for line in self.common_script_lines
                 if line.startswith("#SBATCH --account=")
             )
+            self._stop_and_join_wait_thread()
             raise RuntimeError(
                 "Invalid line in `FractalSlurmExecutor.common_script_lines`: "
                 f"'{invalid_line}'.\n"
@@ -1266,6 +1269,7 @@ class FractalSlurmExecutor(SlurmExecutor):
             pre_command = f"sudo --non-interactive -u {self.slurm_user}"
             submit_command = f"scancel {scancel_string}"
             full_command = f"{pre_command} {submit_command}"
+            validate_cmd(full_command)
             logger.debug(f"Now execute `{full_command}`")
             try:
                 subprocess.run(  # nosec
@@ -1284,6 +1288,10 @@ class FractalSlurmExecutor(SlurmExecutor):
 
         logger.debug("Executor shutdown: end")
 
+    def _stop_and_join_wait_thread(self):
+        self.wait_thread.stop()
+        self.wait_thread.join()
+
     def __exit__(self, *args, **kwargs):
         """
         See
@@ -1292,6 +1300,5 @@ class FractalSlurmExecutor(SlurmExecutor):
         logger.debug(
             "[FractalSlurmExecutor.__exit__] Stop and join `wait_thread`"
         )
-        self.wait_thread.stop()
-        self.wait_thread.join()
+        self._stop_and_join_wait_thread()
         logger.debug("[FractalSlurmExecutor.__exit__] End")

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/runner/run_subprocess.py

@@ -3,11 +3,15 @@ import subprocess  # nosec
 from typing import Optional
 
 from fractal_server.logger import get_logger
+from fractal_server.string_tools import validate_cmd
 
 
 def run_subprocess(
-    cmd: str, logger_name: Optional[str] = None
+    cmd: str,
+    allow_char: Optional[str] = None,
+    logger_name: Optional[str] = None,
 ) -> subprocess.CompletedProcess:
+    validate_cmd(cmd, allow_char=allow_char)
     logger = get_logger(logger_name)
     try:
         res = subprocess.run(  # nosec

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/runner/v1/_common.py

@@ -31,6 +31,7 @@ from .common import write_args_file
 from fractal_server.app.runner.filenames import HISTORY_FILENAME
 from fractal_server.app.runner.filenames import METADATA_FILENAME
 from fractal_server.app.runner.task_files import get_task_file_paths
+from fractal_server.string_tools import validate_cmd
 
 
 def no_op_submit_setup_call(
@@ -77,6 +78,7 @@ def _call_command_wrapper(cmd: str, stdout: Path, stderr: Path) -> None:
                             TERM or KILL signal)
     """
 
+    validate_cmd(cmd)
     # Verify that task command is executable
     if shutil.which(shlex_split(cmd)[0]) is None:
         msg = (

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/runner/v2/runner_functions_low_level.py

@@ -12,6 +12,7 @@ from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.runner.task_files import get_task_file_paths
+from fractal_server.string_tools import validate_cmd
 
 
 def _call_command_wrapper(cmd: str, log_path: Path) -> None:
@@ -25,6 +26,10 @@ def _call_command_wrapper(cmd: str, log_path: Path) -> None:
                             exit code (e.g. due to the subprocess receiving a
                             TERM or KILL signal)
     """
+    try:
+        validate_cmd(cmd)
+    except ValueError as e:
+        raise TaskExecutionError(f"Invalid command. Original error: {str(e)}")
 
     # Verify that task command is executable
     if shutil.which(shlex_split(cmd)[0]) is None:

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/schemas/user.py

@@ -10,7 +10,7 @@ from pydantic.types import StrictStr
 from ._validators import val_absolute_path
 from ._validators import val_unique_list
 from ._validators import valstr
-
+from fractal_server.string_tools import validate_cmd
 
 __all__ = (
     "UserRead",
@@ -77,14 +77,16 @@ class UserUpdate(schemas.BaseUserUpdate):
         valstr("slurm_user")
     )
     _username = validator("username", allow_reuse=True)(valstr("username"))
-    _cache_dir = validator("cache_dir", allow_reuse=True)(
-        val_absolute_path("cache_dir")
-    )
 
     _slurm_accounts = validator("slurm_accounts", allow_reuse=True)(
         val_unique_list("slurm_accounts")
     )
 
+    @validator("cache_dir")
+    def cache_dir_validator(cls, value):
+        validate_cmd(value)
+        return val_absolute_path("cache_dir")(value)
+
     @validator(
         "is_active",
         "is_verified",
@@ -115,9 +117,10 @@ class UserUpdateStrict(BaseModel, extra=Extra.forbid):
         val_unique_list("slurm_accounts")
     )
 
-    _cache_dir = validator("cache_dir", allow_reuse=True)(
-        val_absolute_path("cache_dir")
-    )
+    @validator("cache_dir")
+    def cache_dir_validator(cls, value):
+        validate_cmd(value)
+        return val_absolute_path("cache_dir")(value)
 
 
 class UserUpdateWithNewGroupIds(UserUpdate):
@@ -157,6 +160,8 @@ class UserCreate(schemas.BaseUserCreate):
         valstr("slurm_user")
     )
     _username = validator("username", allow_reuse=True)(valstr("username"))
-    _cache_dir = validator("cache_dir", allow_reuse=True)(
-        val_absolute_path("cache_dir")
-    )
+
+    @validator("cache_dir")
+    def cache_dir_validator(cls, value):
+        validate_cmd(value)
+        return val_absolute_path("cache_dir")(value)

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/schemas/v2/dumps.py

@@ -39,14 +39,23 @@ class TaskDumpV2(BaseModel):
 
 
 class WorkflowTaskDumpV2(BaseModel):
+    """
+    Before v2.5.0, WorkflowTaskV2 could have `task_id=task=None` and
+    non-`None` `task_legacy_id` and `task_legacy`. Since these objects
+    may still exist in the database after version updates, we are setting
+    `task_id` and `task` to `Optional` to avoid response-validation errors
+    for the endpoints that GET datasets.
+    Ref issue #1783.
+    """
+
     id: int
     workflow_id: int
     order: Optional[int]
 
     input_filters: Filters
 
-    task_id: int
-    task: TaskDumpV2
+    task_id: Optional[int]
+    task: Optional[TaskDumpV2]
 
 
 class WorkflowDumpV2(BaseModel, extra=Extra.forbid):

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/app/schemas/v2/task.py

@@ -11,6 +11,7 @@ from pydantic import validator
 
 from .._validators import valdictkeys
 from .._validators import valstr
+from fractal_server.string_tools import validate_cmd
 
 
 class TaskCreateV2(BaseModel, extra=Extra.forbid):
@@ -43,6 +44,11 @@ class TaskCreateV2(BaseModel, extra=Extra.forbid):
                 "Task must have at least one valid command "
                 "(parallel and/or non_parallel)"
             )
+        if command_parallel is not None:
+            validate_cmd(command_parallel)
+        if command_non_parallel is not None:
+            validate_cmd(command_non_parallel)
+
         return values
 
     _name = validator("name", allow_reuse=True)(valstr("name"))

{fractal_server-2.5.0a1 → fractal_server-2.5.2}/fractal_server/main.py

@@ -92,32 +92,34 @@ async def lifespan(app: FastAPI):
     settings = Inject(get_settings)
 
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-        from fractal_server.ssh._fabric import get_ssh_connection
-        from fractal_server.ssh._fabric import FractalSSH
 
-        connection = get_ssh_connection()
-        app.state.fractal_ssh = FractalSSH(connection=connection)
+        from fractal_server.ssh._fabric import FractalSSHList
+
+        app.state.fractal_ssh_list = FractalSSHList()
+
         logger.info(
-            f"Created SSH connection "
-            f"({app.state.fractal_ssh.is_connected=})."
+            "Added empty FractalSSHList to app.state "
+            f"(id={id(app.state.fractal_ssh_list)})."
         )
     else:
-        app.state.fractal_ssh = None
+        app.state.fractal_ssh_list = None
 
     config_uvicorn_loggers()
     logger.info("End application startup")
     reset_logger_handlers(logger)
+
     yield
+
     logger = get_logger("fractal_server.lifespan")
     logger.info("Start application shutdown")
 
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
         logger.info(
-            f"Closing SSH connection "
-            f"(current: {app.state.fractal_ssh.is_connected=})."
+            "Close FractalSSH connections "
+            f"(current size: {app.state.fractal_ssh_list.size})."
         )
 
-        app.state.fractal_ssh.close()
+        app.state.fractal_ssh_list.close_all()
 
     logger.info(
         f"Current worker with pid {os.getpid()} is shutting down. "