fractal-server 2.13.1__py3-none-any.whl → 2.14.0a1__py3-none-any.whl

fractal_server/app/runner/executors/{slurm/ssh → slurm_ssh}/executor.py

@@ -15,22 +15,21 @@ from typing import Sequence
 
 import cloudpickle
 
-from ....filenames import SHUTDOWN_FILENAME
-from ....task_files import get_task_file_paths
-from ....task_files import TaskFiles
-from ....versions import get_versions
-from ..._job_states import STATES_FINISHED
-from ...slurm._slurm_config import SlurmConfig
-from .._batching import heuristics
-from ..utils_executors import get_pickle_file_path
-from ..utils_executors import get_slurm_file_path
-from ..utils_executors import get_slurm_script_file_path
+from ...filenames import SHUTDOWN_FILENAME
+from ...task_files import TaskFiles
+from ...versions import get_versions
+from ..slurm_common._batching import heuristics
+from ..slurm_common._job_states import STATES_FINISHED
+from ..slurm_common._slurm_config import SlurmConfig
+from ..slurm_common.utils_executors import get_pickle_file_path
+from ..slurm_common.utils_executors import get_slurm_file_path
+from ..slurm_common.utils_executors import get_slurm_script_file_path
 from ._executor_wait_thread import FractalSlurmSSHWaitThread
 from fractal_server.app.runner.components import _COMPONENT_KEY_
 from fractal_server.app.runner.compress_folder import compress_folder
 from fractal_server.app.runner.exceptions import JobExecutionError
 from fractal_server.app.runner.exceptions import TaskExecutionError
-from fractal_server.app.runner.executors.slurm.ssh._slurm_job import SlurmJob
+from fractal_server.app.runner.executors.slurm_ssh._slurm_job import SlurmJob
 from fractal_server.app.runner.extract_archive import extract_archive
 from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
@@ -533,9 +532,9 @@ class FractalSlurmSSHExecutor(Executor):
                 except AttributeError:
                     actual_component = str(component)
 
-                _task_file_paths = get_task_file_paths(
-                    workflow_dir_local=task_files.workflow_dir_local,
-                    workflow_dir_remote=task_files.workflow_dir_remote,
+                _task_file_paths = TaskFiles(
+                    root_dir_local=task_files.workflow_dir_local,
+                    root_dir_remote=task_files.workflow_dir_remote,
                     task_name=task_files.task_name,
                     task_order=task_files.task_order,
                     component=actual_component,

fractal_server/app/runner/executors/{slurm/sudo → slurm_sudo}/_check_jobs_status.py

@@ -1,18 +1,20 @@
-from subprocess import run  # nosec
+import subprocess  # nosec
 
-from ......logger import set_logger
-from ..._job_states import STATES_FINISHED
+from fractal_server.app.runner.executors.slurm_common._job_states import (
+    STATES_FINISHED,
+)
+from fractal_server.logger import set_logger
 
 
 logger = set_logger(__name__)
 
 
-def run_squeue(job_ids):
-    res = run(  # nosec
+def run_squeue(job_ids: list[str]) -> subprocess.CompletedProcess:
+    res = subprocess.run(  # nosec
         [
             "squeue",
             "--noheader",
-            "--format=%i %T",
+            "--format='%i %T'",
             "--jobs",
             ",".join([str(j) for j in job_ids]),
             "--states=all",
@@ -23,14 +25,14 @@ def run_squeue(job_ids):
     )
     if res.returncode != 0:
         logger.warning(
-            f"squeue command with {job_ids}"
-            f" failed with:\n{res.stderr=}\n{res.stdout=}"
+            f"squeue command with {job_ids} failed with:"
+            f"\n{res.stderr=}\n{res.stdout=}"
         )
 
     return res
 
 
-def _jobs_finished(job_ids) -> set[str]:
+def get_finished_jobs(job_ids: list[str]) -> set[str]:
     """
     Check which ones of the given Slurm jobs already finished
 

fractal_server/app/runner/executors/{slurm/sudo → slurm_sudo}/_executor_wait_thread.py

@@ -5,9 +5,9 @@ import traceback
 from itertools import count
 from typing import Optional
 
-from ......logger import set_logger
-from ._check_jobs_status import _jobs_finished
+from ._check_jobs_status import get_finished_jobs
 from fractal_server.app.runner.exceptions import JobExecutionError
+from fractal_server.logger import set_logger
 
 logger = set_logger(__name__)
 
@@ -115,7 +115,7 @@ class FractalSlurmSudoWaitThread(threading.Thread):
         self.check_shutdown(i)
         if i % (self.slurm_poll_interval // self.interval) == 0:
             try:
-                finished_jobs = _jobs_finished(self.waiting.values())
+                finished_jobs = get_finished_jobs(self.waiting.values())
             except Exception:
                 # Don't abandon completion checking if jobs_finished errors
                 traceback.print_exc()

fractal_server/app/runner/executors/{slurm/sudo → slurm_sudo}/_subprocess_run_as_user.py

@@ -19,7 +19,7 @@ import shlex
 import subprocess  # nosec
 from typing import Optional
 
-from ......logger import set_logger
+from fractal_server.logger import set_logger
 from fractal_server.string_tools import validate_cmd
 
 logger = set_logger(__name__)
@@ -65,10 +65,7 @@ def _run_command_as_user(
 
     if check and not res.returncode == 0:
         raise RuntimeError(
-            f"{cmd=}\n\n"
-            f"{res.returncode=}\n\n"
-            f"{res.stdout=}\n\n"
-            f"{res.stderr=}\n"
+            f"{cmd=}\n\n{res.returncode=}\n\n{res.stdout=}\n\n{res.stderr=}\n"
         )
 
     return res
@@ -93,69 +90,6 @@ def _mkdir_as_user(*, folder: str, user: str) -> None:
     _run_command_as_user(cmd=cmd, user=user, check=True)
 
 
-def _glob_as_user(
-    *, folder: str, user: str, startswith: Optional[str] = None
-) -> list[str]:
-    """
-    Run `ls` in a folder (as a user) and filter results
-
-    Execute `ls` on a folder (impersonating a user, if `user` is not `None`)
-    and select results that start with `startswith` (if not `None`).
-
-    Arguments:
-        folder: Absolute path to the folder
-        user: If not `None`, the user to be impersonated via `sudo -u`
-        startswith: If not `None`, this is used to filter output of `ls`.
-    """
-
-    res = _run_command_as_user(cmd=f"ls {folder}", user=user, check=True)
-    output = res.stdout.split()
-    if startswith:
-        output = [f for f in output if f.startswith(startswith)]
-    return output
-
-
-def _glob_as_user_strict(
-    *,
-    folder: str,
-    user: str,
-    startswith: str,
-) -> list[str]:
-    """
-    Run `ls` in a folder (as a user) and filter results
-
-    Execute `ls` on a folder (impersonating a user, if `user` is not `None`)
-    and select results that comply with a set of rules. They all start with
-    `startswith` (if not `None`), and they match one of the known filename
-    patterns. See details in
-    https://github.com/fractal-analytics-platform/fractal-server/issues/1240
-
-
-    Arguments:
-        folder: Absolute path to the folder
-        user: If not `None`, the user to be impersonated via `sudo -u`
-        startswith: If not `None`, this is used to filter output of `ls`.
-    """
-
-    res = _run_command_as_user(cmd=f"ls {folder}", user=user, check=True)
-    output = res.stdout.split()
-
-    new_output = []
-    known_filenames = [
-        f"{startswith}{suffix}"
-        for suffix in [".args.json", ".metadiff.json", ".err", ".out", ".log"]
-    ]
-    for filename in output:
-        if filename in known_filenames:
-            new_output.append(filename)
-        elif filename.startswith(f"{startswith}_out_") and filename.endswith(
-            ".pickle"
-        ):
-            new_output.append(filename)
-
-    return new_output
-
-
 def _path_exists_as_user(*, path: str, user: Optional[str] = None) -> bool:
     """
     Impersonate a user and check if `path` exists via `ls`