fractal-server 2.2.0a0__py3-none-any.whl → 2.3.0a0__py3-none-any.whl

fractal_server/app/runner/executors/slurm/sudo/__init__.py

@@ -0,0 +1,3 @@
+from .executor import SlurmExecutor
+
+__all__ = ["SlurmExecutor"]

fractal_server/app/runner/executors/slurm/{_check_jobs_status.py → sudo/_check_jobs_status.py}

@@ -2,7 +2,7 @@ from subprocess import run  # nosec
 
 from cfut.slurm import STATES_FINISHED
 
-from .....logger import set_logger
+from ......logger import set_logger
 
 
 logger = set_logger(__name__)

fractal_server/app/runner/executors/slurm/{_executor_wait_thread.py → sudo/_executor_wait_thread.py}

@@ -7,7 +7,7 @@ from typing import Optional
 
 from cfut import FileWaitThread
 
-from .....logger import set_logger
+from ......logger import set_logger
 from ._check_jobs_status import _jobs_finished
 
 logger = set_logger(__name__)

fractal_server/app/runner/executors/slurm/{_subprocess_run_as_user.py → sudo/_subprocess_run_as_user.py}

@@ -19,7 +19,7 @@ import shlex
 import subprocess  # nosec
 from typing import Optional
 
-from .....logger import set_logger
+from ......logger import set_logger
 
 logger = set_logger(__name__)
 

fractal_server/app/runner/executors/slurm/{executor.py → sudo/executor.py}

@@ -29,18 +29,18 @@ import cloudpickle
 from cfut import SlurmExecutor
 from cfut.util import random_string
 
-from .....config import get_settings
-from .....logger import set_logger
-from .....syringe import Inject
-from ...exceptions import JobExecutionError
-from ...exceptions import TaskExecutionError
-from ...filenames import SHUTDOWN_FILENAME
-from ...task_files import get_task_file_paths
-from ...task_files import TaskFiles
-from ._batching import heuristics
+from ......config import get_settings
+from ......logger import set_logger
+from ......syringe import Inject
+from ....exceptions import JobExecutionError
+from ....exceptions import TaskExecutionError
+from ....filenames import SHUTDOWN_FILENAME
+from ....task_files import get_task_file_paths
+from ....task_files import TaskFiles
+from ...slurm._slurm_config import get_default_slurm_config
+from ...slurm._slurm_config import SlurmConfig
+from .._batching import heuristics
 from ._executor_wait_thread import FractalSlurmWaitThread
-from ._slurm_config import get_default_slurm_config
-from ._slurm_config import SlurmConfig
 from ._subprocess_run_as_user import _glob_as_user
 from ._subprocess_run_as_user import _glob_as_user_strict
 from ._subprocess_run_as_user import _path_exists_as_user
@@ -1180,7 +1180,7 @@ class FractalSlurmExecutor(SlurmExecutor):
 
         # Prepare SLURM preamble based on SlurmConfig object
         script_lines = slurm_config.to_sbatch_preamble(
-            user_cache_dir=self.user_cache_dir
+            remote_export_dir=self.user_cache_dir
         )
 
         # Extend SLURM preamble with variable which are not in SlurmConfig, and

fractal_server/app/runner/extract_archive.py

@@ -0,0 +1,38 @@
+import sys
+import tarfile
+from pathlib import Path
+
+
+def _remove_suffix(*, string: str, suffix: str) -> str:
+    if string.endswith(suffix):
+        return string[: -len(suffix)]
+    else:
+        raise ValueError(f"Cannot remove {suffix=} from {string=}.")
+
+
+if __name__ == "__main__":
+    help_msg = (
+        "Expected use:\n"
+        "python -m fractal_server.app.runner.extract_archive "
+        "path/to/archive.tar.gz"
+    )
+
+    if len(sys.argv[1:]) != 1:
+        raise ValueError(
+            f"Invalid argument.\n{help_msg}\nProvided: {sys.argv=}"
+        )
+    elif not sys.argv[1].endswith(".tar.gz"):
+        raise ValueError(
+            f"Invalid argument.\n{help_msg}\nProvided: {sys.argv=}"
+        )
+
+    tarfile_path = Path(sys.argv[1])
+
+    print(f"[extract_archive.py] {tarfile_path=}")
+
+    job_folder = tarfile_path.parent
+    subfolder_name = _remove_suffix(string=tarfile_path.name, suffix=".tar.gz")
+    with tarfile.open(tarfile_path) as tar:
+        tar.extractall(path=Path(job_folder, subfolder_name).as_posix())
+
+    print(f"[extract_archive.py] {tarfile_path=}")

fractal_server/app/runner/v1/__init__.py

@@ -22,6 +22,10 @@ import traceback
 from pathlib import Path
 from typing import Optional
 
+from sqlalchemy.orm import Session as DBSyncSession
+
+from ....logger import get_logger
+from ....logger import reset_logger_handlers
 from ....logger import set_logger
 from ....syringe import Inject
 from ....utils import get_timestamp
@@ -33,7 +37,7 @@ from ...models.v1 import WorkflowTask
 from ...schemas.v1 import JobStatusTypeV1
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
-from ..executors.slurm._subprocess_run_as_user import (
+from ..executors.slurm.sudo._subprocess_run_as_user import (
     _mkdir_as_user,
 )
 from ..filenames import WORKFLOW_LOG_FILENAME
@@ -53,6 +57,27 @@ _backends["local"] = local_process_workflow
 _backends["slurm"] = slurm_process_workflow
 
 
+def fail_job(
+    *,
+    db: DBSyncSession,
+    job: ApplyWorkflow,
+    log_msg: str,
+    logger_name: str,
+    emit_log: bool = False,
+) -> None:
+    logger = get_logger(logger_name=logger_name)
+    if emit_log:
+        logger.error(log_msg)
+    reset_logger_handlers(logger)
+    job.status = JobStatusTypeV1.FAILED
+    job.end_timestamp = get_timestamp()
+    job.log = log_msg
+    db.merge(job)
+    db.commit()
+    db.close()
+    return
+
+
 async def submit_workflow(
     *,
     workflow_id: int,
@@ -91,21 +116,41 @@ async def submit_workflow(
             slurm backend.
     """
 
-    # Declare runner backend and set `process_workflow` function
-    settings = Inject(get_settings)
-    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
-    if FRACTAL_RUNNER_BACKEND == "local":
-        process_workflow = local_process_workflow
-    elif FRACTAL_RUNNER_BACKEND == "slurm":
-        process_workflow = slurm_process_workflow
-    else:
-        raise RuntimeError(f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}")
+    logger_name = f"WF{workflow_id}_job{job_id}"
+    logger = set_logger(logger_name=logger_name)
 
     with next(DB.get_sync_db()) as db_sync:
 
         job: ApplyWorkflow = db_sync.get(ApplyWorkflow, job_id)
         if not job:
-            raise ValueError(f"Cannot fetch job {job_id} from database")
+            logger.error(f"ApplyWorkflow {job_id} does not exist")
+            return
+
+        settings = Inject(get_settings)
+        FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+        if FRACTAL_RUNNER_BACKEND == "local":
+            process_workflow = local_process_workflow
+        elif FRACTAL_RUNNER_BACKEND == "slurm":
+            process_workflow = slurm_process_workflow
+        else:
+
+            if FRACTAL_RUNNER_BACKEND == "local_experimental":
+                log_msg = (
+                    f"{FRACTAL_RUNNER_BACKEND=} is not available for v1 jobs."
+                )
+            else:
+                log_msg = f"Invalid {FRACTAL_RUNNER_BACKEND=}"
+
+            fail_job(
+                job=job,
+                db=db_sync,
+                log_msg=log_msg,
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
+
+        # Declare runner backend and set `process_workflow` function
 
         input_dataset: Dataset = db_sync.get(Dataset, input_dataset_id)
         output_dataset: Dataset = db_sync.get(Dataset, output_dataset_id)
@@ -126,12 +171,9 @@ async def submit_workflow(
                 log_msg += (
                     f"Cannot fetch workflow {workflow_id} from database\n"
                 )
-            job.status = JobStatusTypeV1.FAILED
-            job.end_timestamp = get_timestamp()
-            job.log = log_msg
-            db_sync.merge(job)
-            db_sync.commit()
-            db_sync.close()
+            fail_job(
+                db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name
+            )
             return
 
         # Prepare some of process_workflow arguments
@@ -147,9 +189,14 @@ async def submit_workflow(
         )
 
         if WORKFLOW_DIR_LOCAL.exists():
-            raise RuntimeError(
-                f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists."
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists.",
+                logger_name=logger_name,
+                emit_log=True,
             )
+            return
 
         # Create WORKFLOW_DIR
         original_umask = os.umask(0)
@@ -202,7 +249,6 @@ async def submit_workflow(
         db_sync.refresh(workflow)
 
         # Write logs
-        logger_name = f"WF{workflow_id}_job{job_id}"
         log_file_path = WORKFLOW_DIR_LOCAL / WORKFLOW_LOG_FILENAME
         logger = set_logger(
             logger_name=logger_name,
@@ -302,19 +348,14 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusTypeV1.FAILED
-        job.end_timestamp = get_timestamp()
-
         exception_args_string = "\n".join(e.args)
-        job.log = (
+        log_msg = (
             f"TASK ERROR: "
             f"Task name: {e.task_name}, "
             f"position in Workflow: {e.workflow_task_order}\n"
             f"TRACEBACK:\n{exception_args_string}"
         )
-        db_sync.merge(job)
-        close_job_logger(logger)
-        db_sync.commit()
+        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
 
     except JobExecutionError as e:
 
@@ -334,14 +375,13 @@ async def submit_workflow(
         )
 
         db_sync.merge(output_dataset)
-
-        job.status = JobStatusTypeV1.FAILED
-        job.end_timestamp = get_timestamp()
         error = e.assemble_error()
-        job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
-        db_sync.merge(job)
-        close_job_logger(logger)
-        db_sync.commit()
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}",
+            logger_name=logger_name,
+        )
 
     except Exception:
 
@@ -364,14 +404,12 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusTypeV1.FAILED
-        job.end_timestamp = get_timestamp()
-        job.log = (
+        log_msg = (
             f"UNKNOWN ERROR in Fractal job {job.id}\n"
             f"TRACEBACK:\n{current_traceback}"
         )
-        db_sync.merge(job)
-        close_job_logger(logger)
-        db_sync.commit()
+        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
+
     finally:
         db_sync.close()
+        reset_logger_handlers(logger)

fractal_server/app/runner/v1/_slurm/__init__.py

@@ -22,7 +22,7 @@ from typing import Optional
 from typing import Union
 
 from ...async_wrap import async_wrap
-from ...executors.slurm.executor import FractalSlurmExecutor
+from ...executors.slurm.sudo.executor import FractalSlurmExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from .._common import execute_tasks
 from ..common import TaskParameters