hpc-runner 0.2.2__py3-none-any.whl → 0.3.1__py3-none-any.whl

hpc_runner/schedulers/local/scheduler.py

@@ -9,15 +9,17 @@ from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from hpc_runner.core.config import get_config
 from hpc_runner.core.exceptions import AccountingNotAvailable, JobNotFoundError
 from hpc_runner.core.job_info import JobInfo
-from hpc_runner.core.result import ArrayJobResult, JobResult, JobStatus
+from hpc_runner.core.result import JobResult, JobStatus
 from hpc_runner.schedulers.base import BaseScheduler
 from hpc_runner.templates import render_template
 
 if TYPE_CHECKING:
     from hpc_runner.core.job import Job
     from hpc_runner.core.job_array import JobArray
+    from hpc_runner.core.result import ArrayJobResult
 
 
 class LocalScheduler(BaseScheduler):
@@ -26,145 +28,89 @@ class LocalScheduler(BaseScheduler):
     name = "local"
 
     _job_counter: int = 0
-    _processes: dict[str, subprocess.Popen] = {}  # type: ignore[type-arg]
-    _exit_codes: dict[str, int] = {}
-    _output_paths: dict[str, dict[str, Path]] = {}
 
-    def submit(
-        self, job: "Job", interactive: bool = False, keep_script: bool = False
-    ) -> JobResult:
+    def __init__(self) -> None:
+        """Initialize local scheduler with config-driven settings."""
+        config = get_config()
+        local_config = config.get_scheduler_config("local")
+
+        self.purge_modules = local_config.get("purge_modules", True)
+        self.silent_modules = local_config.get("silent_modules", False)
+        self.module_init_script = local_config.get("module_init_script", "")
+
+        self._processes: dict[str, subprocess.Popen[bytes]] = {}
+        self._exit_codes: dict[str, int] = {}
+        self._output_paths: dict[str, dict[str, Path]] = {}
+        self._script_paths: dict[str, Path] = {}
+
+    def submit(self, job: Job, interactive: bool = False, keep_script: bool = False) -> JobResult:
         """Run job as local subprocess."""
         LocalScheduler._job_counter += 1
         job_id = f"local_{LocalScheduler._job_counter}_{datetime.now().strftime('%Y%m%d%H%M%S')}"
 
-        # Set up environment with modules (modules not actually loaded locally)
+        # Set up environment
         env = os.environ.copy() if job.inherit_env else {}
+        if job.env_vars:
+            env.update(job.env_vars)
 
-        # Generate and write script
-        script = self.generate_script(job)
-        script_path = Path(tempfile.gettempdir()) / f".hpc_local_{job_id}.sh"
-        script_path.write_text(script)
-        script_path.chmod(0o755)
-
-        workdir = Path(job.workdir) if job.workdir else Path.cwd()
+        workdir = Path(job.workdir).resolve() if job.workdir else Path.cwd()
 
-        # Determine output paths
-        stdout_file = job.stdout or f"{job.name}.{job_id}.out"
-        stdout_path = workdir / stdout_file
-        if job.merge_output:
-            stderr_path = stdout_path  # Merge stderr into stdout
-        else:
-            stderr_file = job.stderr or f"{job.name}.{job_id}.err"
-            stderr_path = workdir / stderr_file
+        # Resolve output paths for template-based redirection
+        stdout_path: Path | None = None
+        stderr_path: Path | None = None
 
-        # Store output paths
-        LocalScheduler._output_paths[job_id] = {
-            "stdout": stdout_path,
-            "stderr": stderr_path,
-        }
-
-        if interactive:
-            # Blocking execution
-            with open(stdout_path, "w") as stdout_f:
-                if job.merge_output:
-                    result = subprocess.run(
-                        [str(script_path)],
-                        cwd=workdir,
-                        env=env,
-                        stdout=stdout_f,
-                        stderr=subprocess.STDOUT,
-                    )
-                else:
-                    with open(stderr_path, "w") as stderr_f:
-                        result = subprocess.run(
-                            [str(script_path)],
-                            cwd=workdir,
-                            env=env,
-                            stdout=stdout_f,
-                            stderr=stderr_f,
-                        )
-            LocalScheduler._exit_codes[job_id] = result.returncode
-            script_path.unlink(missing_ok=True)
-        else:
-            # Background execution
-            stdout_f = open(stdout_path, "w")
+        if job.stdout is not None or job.stderr is not None:
+            stdout_file = job.stdout or f"{job.name}.{job_id}.out"
+            stdout_path = workdir / stdout_file
             if job.merge_output:
-                proc = subprocess.Popen(
-                    [str(script_path)],
-                    cwd=workdir,
-                    env=env,
-                    stdout=stdout_f,
-                    stderr=subprocess.STDOUT,
-                )
+                stderr_path = None
             else:
-                stderr_f = open(stderr_path, "w")
-                proc = subprocess.Popen(
-                    [str(script_path)],
-                    cwd=workdir,
-                    env=env,
-                    stdout=stdout_f,
-                    stderr=stderr_f,
-                )
-            LocalScheduler._processes[job_id] = proc
-            # Store script path for cleanup
-            proc._script_path = script_path  # type: ignore[attr-defined]
-            proc._stdout_file = stdout_f  # type: ignore[attr-defined]
-            if not job.merge_output:
-                proc._stderr_file = stderr_f  # type: ignore[attr-defined]
-
-        return JobResult(job_id=job_id, scheduler=self, job=job)
-
-    def submit_array(self, array: "JobArray") -> ArrayJobResult:
-        """Simulate array job by submitting multiple jobs."""
-        # For local scheduler, we just run one job
-        # and return an ArrayJobResult pointing to it
-        LocalScheduler._job_counter += 1
-        base_job_id = f"local_array_{LocalScheduler._job_counter}"
-
-        # Run jobs sequentially (or could be parallel)
-        for idx in array.indices:
-            # Set array index environment variable
-            os.environ["HPC_ARRAY_TASK_ID"] = str(idx)
-            os.environ["SGE_TASK_ID"] = str(idx)  # SGE compat
-            os.environ["SLURM_ARRAY_TASK_ID"] = str(idx)  # Slurm compat
-
-            # Create a job ID for this task
-            task_job_id = f"{base_job_id}.{idx}"
-            self._submit_array_task(array.job, task_job_id, idx)
-
-        return ArrayJobResult(base_job_id=base_job_id, scheduler=self, array=array)
-
-    def _submit_array_task(self, job: "Job", job_id: str, index: int) -> None:
-        """Submit a single array task."""
-        env = os.environ.copy() if job.inherit_env else {}
-        env["HPC_ARRAY_TASK_ID"] = str(index)
-
-        script = self.generate_script(job)
+                stderr_file = job.stderr or f"{job.name}.{job_id}.err"
+                stderr_path = workdir / stderr_file
+
+            self._output_paths[job_id] = {
+                "stdout": stdout_path,
+                "stderr": stderr_path if stderr_path else stdout_path,
+            }
+
+        # Generate and write script (template handles output redirection)
+        script = self.generate_script(
+            job,
+            stdout_path=stdout_path,
+            stderr_path=stderr_path,
+        )
         script_path = Path(tempfile.gettempdir()) / f".hpc_local_{job_id}.sh"
         script_path.write_text(script)
         script_path.chmod(0o755)
 
-        workdir = Path(job.workdir) if job.workdir else Path.cwd()
-        stdout_path = workdir / f"{job.name}.{job_id}.out"
+        if interactive:
+            result = subprocess.run(
+                [str(script_path)],
+                cwd=workdir,
+                env=env,
+            )
+            self._exit_codes[job_id] = result.returncode
+            if not keep_script:
+                script_path.unlink(missing_ok=True)
+        else:
+            proc = subprocess.Popen(
+                [str(script_path)],
+                cwd=workdir,
+                env=env,
+            )
+            self._processes[job_id] = proc
+            self._script_paths[job_id] = script_path
 
-        LocalScheduler._output_paths[job_id] = {"stdout": stdout_path, "stderr": stdout_path}
+        return JobResult(job_id=job_id, scheduler=self, job=job)
 
-        stdout_f = open(stdout_path, "w")
-        proc = subprocess.Popen(
-            [str(script_path)],
-            cwd=workdir,
-            env=env,
-            stdout=stdout_f,
-            stderr=subprocess.STDOUT,
-        )
-        LocalScheduler._processes[job_id] = proc
-        proc._script_path = script_path  # type: ignore[attr-defined]
-        proc._stdout_file = stdout_f  # type: ignore[attr-defined]
+    def submit_array(self, array: JobArray) -> ArrayJobResult:
+        """Array jobs are not supported by the local scheduler."""
+        raise NotImplementedError("Array jobs are not supported by the local scheduler")
 
     def cancel(self, job_id: str) -> bool:
         """Cancel a local job."""
-        if job_id in LocalScheduler._processes:
-            proc = LocalScheduler._processes[job_id]
+        if job_id in self._processes:
+            proc = self._processes[job_id]
             proc.terminate()
             proc.wait()
             self._cleanup_process(job_id)
@@ -173,76 +119,80 @@ class LocalScheduler(BaseScheduler):
 
     def get_status(self, job_id: str) -> JobStatus:
         """Get job status."""
-        if job_id in LocalScheduler._exit_codes:
-            # Already completed
-            return JobStatus.COMPLETED if LocalScheduler._exit_codes[job_id] == 0 else JobStatus.FAILED
+        if job_id in self._exit_codes:
+            if self._exit_codes[job_id] == 0:
+                return JobStatus.COMPLETED
+            return JobStatus.FAILED
 
-        if job_id not in LocalScheduler._processes:
+        if job_id not in self._processes:
             return JobStatus.UNKNOWN
 
-        proc = LocalScheduler._processes[job_id]
+        proc = self._processes[job_id]
         poll = proc.poll()
 
         if poll is None:
             return JobStatus.RUNNING
 
         # Process completed
-        LocalScheduler._exit_codes[job_id] = poll
+        self._exit_codes[job_id] = poll
         self._cleanup_process(job_id)
 
         return JobStatus.COMPLETED if poll == 0 else JobStatus.FAILED
 
     def _cleanup_process(self, job_id: str) -> None:
         """Clean up process resources."""
-        if job_id in LocalScheduler._processes:
-            proc = LocalScheduler._processes[job_id]
-            # Close file handles
-            if hasattr(proc, "_stdout_file"):
-                proc._stdout_file.close()  # type: ignore[attr-defined]
-            if hasattr(proc, "_stderr_file"):
-                proc._stderr_file.close()  # type: ignore[attr-defined]
-            # Remove script
-            if hasattr(proc, "_script_path"):
-                proc._script_path.unlink(missing_ok=True)  # type: ignore[attr-defined]
-            del LocalScheduler._processes[job_id]
+        self._processes.pop(job_id, None)
+        script_path = self._script_paths.pop(job_id, None)
+        if script_path:
+            script_path.unlink(missing_ok=True)
 
     def get_exit_code(self, job_id: str) -> int | None:
         """Get exit code."""
-        # First check if we have a cached exit code
-        if job_id in LocalScheduler._exit_codes:
-            return LocalScheduler._exit_codes[job_id]
+        if job_id in self._exit_codes:
+            return self._exit_codes[job_id]
 
-        # Check if process is done
-        if job_id in LocalScheduler._processes:
-            proc = LocalScheduler._processes[job_id]
+        if job_id in self._processes:
+            proc = self._processes[job_id]
             poll = proc.poll()
             if poll is not None:
-                LocalScheduler._exit_codes[job_id] = poll
+                self._exit_codes[job_id] = poll
                 return poll
 
         return None
 
     def get_output_path(self, job_id: str, stream: str) -> Path | None:
         """Get output file path."""
-        if job_id in LocalScheduler._output_paths:
-            return LocalScheduler._output_paths[job_id].get(stream)
+        if job_id in self._output_paths:
+            return self._output_paths[job_id].get(stream)
         return None
 
-    def generate_script(self, job: "Job", array_range: str | None = None) -> str:
+    def generate_script(
+        self,
+        job: Job,
+        array_range: str | None = None,
+        *,
+        stdout_path: Path | None = None,
+        stderr_path: Path | None = None,
+    ) -> str:
         """Generate local execution script."""
         return render_template(
             "local/templates/job.sh.j2",
             job=job,
             scheduler=self,
+            stdout_path=stdout_path,
+            stderr_path=stderr_path,
+            merge_output=job.merge_output,
         )
 
-    def build_submit_command(self, job: "Job") -> list[str]:
+    def build_submit_command(self, job: Job) -> list[str]:
         """Build command - for local, just bash."""
-        return ["bash", "-c", job.command if isinstance(job.command, str) else " ".join(job.command)]
+        cmd = job.command if isinstance(job.command, str) else " ".join(job.command)
+        return ["bash", "-c", cmd]
 
-    def build_interactive_command(self, job: "Job") -> list[str]:
+    def build_interactive_command(self, job: Job) -> list[str]:
         """Build interactive command - for local, just bash."""
-        return ["bash", "-c", job.command if isinstance(job.command, str) else " ".join(job.command)]
+        cmd = job.command if isinstance(job.command, str) else " ".join(job.command)
+        return ["bash", "-c", cmd]
 
     # -------------------------------------------------------------------------
     # TUI Monitor API (stubs for local scheduler)
@@ -254,31 +204,26 @@ class LocalScheduler(BaseScheduler):
         status: set[JobStatus] | None = None,
         queue: str | None = None,
     ) -> list[JobInfo]:
-        """List active local jobs.
-
-        The local scheduler tracks running processes in memory.
-        """
+        """List active local jobs."""
         jobs: list[JobInfo] = []
         current_user = os.environ.get("USER", "unknown")
 
-        for job_id, proc in LocalScheduler._processes.items():
+        for job_id, proc in self._processes.items():
             poll = proc.poll()
-            if poll is None:  # Still running
+            if poll is None:
                 job_status = JobStatus.RUNNING
             else:
-                continue  # Skip completed jobs
+                continue
 
-            # Apply filters
             if user is not None and user != current_user:
                 continue
             if status is not None and job_status not in status:
                 continue
-            # queue filter doesn't apply to local scheduler
 
             jobs.append(
                 JobInfo(
                     job_id=job_id,
-                    name=job_id,  # Local scheduler doesn't track job names
+                    name=job_id,
                     user=current_user,
                     status=job_status,
                     queue="local",
@@ -307,22 +252,20 @@ class LocalScheduler(BaseScheduler):
         )
 
     def has_accounting(self) -> bool:
-        """Check if job accounting is available.
-
-        Local scheduler does not have persistent accounting.
-        """
+        """Check if job accounting is available."""
         return False
 
     def get_job_details(self, job_id: str) -> tuple[JobInfo, dict[str, object]]:
         """Get details for a local job."""
         current_user = os.environ.get("USER", "unknown")
 
-        # Check running processes
-        if job_id in LocalScheduler._processes:
-            proc = LocalScheduler._processes[job_id]
+        if job_id in self._processes:
+            proc = self._processes[job_id]
             poll = proc.poll()
-            status = JobStatus.RUNNING if poll is None else (
-                JobStatus.COMPLETED if poll == 0 else JobStatus.FAILED
+            status = (
+                JobStatus.RUNNING
+                if poll is None
+                else (JobStatus.COMPLETED if poll == 0 else JobStatus.FAILED)
             )
             job_info = JobInfo(
                 job_id=job_id,
@@ -331,14 +274,13 @@ class LocalScheduler(BaseScheduler):
                 status=status,
                 queue="local",
                 exit_code=poll if poll is not None else None,
-                stdout_path=LocalScheduler._output_paths.get(job_id, {}).get("stdout"),
-                stderr_path=LocalScheduler._output_paths.get(job_id, {}).get("stderr"),
+                stdout_path=self._output_paths.get(job_id, {}).get("stdout"),
+                stderr_path=self._output_paths.get(job_id, {}).get("stderr"),
             )
             return job_info, {}
 
-        # Check completed jobs with cached exit codes
-        if job_id in LocalScheduler._exit_codes:
-            exit_code = LocalScheduler._exit_codes[job_id]
+        if job_id in self._exit_codes:
+            exit_code = self._exit_codes[job_id]
             job_info = JobInfo(
                 job_id=job_id,
                 name=job_id,
@@ -346,8 +288,8 @@ class LocalScheduler(BaseScheduler):
                 status=JobStatus.COMPLETED if exit_code == 0 else JobStatus.FAILED,
                 queue="local",
                 exit_code=exit_code,
-                stdout_path=LocalScheduler._output_paths.get(job_id, {}).get("stdout"),
-                stderr_path=LocalScheduler._output_paths.get(job_id, {}).get("stderr"),
+                stdout_path=self._output_paths.get(job_id, {}).get("stdout"),
+                stderr_path=self._output_paths.get(job_id, {}).get("stderr"),
             )
             return job_info, {}
 

hpc_runner/schedulers/local/templates/job.sh.j2

@@ -5,20 +5,44 @@
 # Exit on error
 set -e
 
+# Module system initialization
+{% if scheduler.module_init_script %}
+. {{ scheduler.module_init_script }}
+{% else %}
+if [ -f /etc/profile.d/modules.sh ]; then
+    . /etc/profile.d/modules.sh
+elif [ -f /usr/share/Modules/init/bash ]; then
+    . /usr/share/Modules/init/bash
+elif [ -f /etc/modules/init/bash ]; then
+    . /etc/modules/init/bash
+fi
+{% endif %}
+
+{% if scheduler.purge_modules %}
+# Purge modules for clean environment
+module purge{% if scheduler.silent_modules %} -s{% endif %}
+
+{% endif %}
 {% if job.modules_path %}
-# Additional module paths (simulated for local)
+# Additional module paths
 {% for path in job.modules_path %}
-# module use {{ path }}
+module use {{ path }}
 {% endfor %}
 {% endif %}
 
 {% if job.modules %}
-# Modules (simulated for local - not actually loaded)
+# Load modules
 {% for mod in job.modules %}
-# module load {{ mod }}
+module load {{ mod }}{% if scheduler.silent_modules %} -s{% endif %}
+
 {% endfor %}
 {% endif %}
 
+{% if job.venv %}
+# Activate virtual environment
+source {{ job.venv }}/bin/activate
+{% endif %}
+
 {% if job.env_prepend %}
 # Prepend to environment variables
 {% for key, value in job.env_prepend.items() %}
@@ -40,9 +64,16 @@ export {{ key }}="{{ value }}"
 {% endfor %}
 {% endif %}
 
-{% if job.workdir %}
-# Change to working directory
-cd {{ job.workdir }}
+{% if stdout_path and merge_output %}
+# Redirect stdout and stderr to file
+exec > {{ stdout_path }} 2>&1
+{% elif stdout_path %}
+# Redirect stdout to file
+exec > {{ stdout_path }}
+{% endif %}
+{% if stderr_path and not merge_output %}
+# Redirect stderr to file
+exec 2> {{ stderr_path }}
 {% endif %}
 
 # Execute command

hpc_runner/schedulers/sge/args.py

@@ -7,7 +7,7 @@ both as a script directive (#$ ...) and as command-line arguments.
 from hpc_runner.core.descriptors import SchedulerArg
 
 
-class SGEArg(SchedulerArg):
+class SGEArg(SchedulerArg[str]):
     """Base class for SGE arguments.
 
     SGE uses:
@@ -15,12 +15,12 @@ class SGEArg(SchedulerArg):
     - CLI args: -flag value
     """
 
-    def to_args(self, value) -> list[str]:
+    def to_args(self, value: str | None) -> list[str]:
         if value is None:
             return []
         return [f"-{self.flag}", str(value)]
 
-    def to_directive(self, value) -> str | None:
+    def to_directive(self, value: str | None) -> str | None:
         if value is None:
             return None
         return f"#$ -{self.flag} {value}"
@@ -34,42 +34,42 @@ class SGEArg(SchedulerArg):
 class SGEJobNameArg(SGEArg):
     """Job name: -N name"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("N", doc="Job name")
 
 
 class SGEQueueArg(SGEArg):
     """Queue selection: -q queue_name"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("q", doc="Queue/partition name")
 
 
 class SGEOutputArg(SGEArg):
     """Stdout path: -o path"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("o", doc="Stdout file path")
 
 
 class SGEErrorArg(SGEArg):
     """Stderr path: -e path"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("e", doc="Stderr file path")
 
 
 class SGEPriorityArg(SGEArg):
     """Job priority: -p priority"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("p", doc="Job priority (-1023 to 1024)")
 
 
 class SGEShellArg(SGEArg):
     """Shell selection: -S /path/to/shell"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("S", doc="Shell path")
 
 
@@ -81,7 +81,7 @@ class SGEShellArg(SGEArg):
 class SGECwdArg(SchedulerArg[bool]):
     """Use current working directory: -cwd"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("cwd", doc="Execute in current working directory")
 
     def to_args(self, value: bool | None) -> list[str]:
@@ -94,7 +94,7 @@ class SGECwdArg(SchedulerArg[bool]):
 class SGEInheritEnvArg(SchedulerArg[bool]):
     """Inherit environment: -V"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("V", doc="Inherit environment variables")
 
     def to_args(self, value: bool | None) -> list[str]:
@@ -107,7 +107,7 @@ class SGEInheritEnvArg(SchedulerArg[bool]):
 class SGEMergeOutputArg(SchedulerArg[bool]):
     """Merge stdout and stderr: -j y"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("j", doc="Join stdout and stderr")
 
     def to_args(self, value: bool | None) -> list[str]:
@@ -196,7 +196,7 @@ class SGEArrayArg(SchedulerArg[str]):
     Range formats: 1-100, 1-100:10, 1,2,3,4
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("t", doc="Array job range")
 
     def to_args(self, value: str | None) -> list[str]:
@@ -218,7 +218,7 @@ class SGEArrayArg(SchedulerArg[str]):
 class SGEHoldArg(SchedulerArg[str]):
     """Job dependency: -hold_jid job_id[,job_id,...]"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__("hold_jid", doc="Hold until jobs complete")
 
     def to_args(self, value: str | None) -> list[str]:

hpc_runner/schedulers/sge/parser.py

@@ -1,8 +1,8 @@
 """SGE output parsing utilities."""
 
 import re
-from datetime import datetime
 import xml.etree.ElementTree as ET
+from datetime import datetime
 from typing import Any
 
 from hpc_runner.core.result import JobStatus
@@ -131,9 +131,9 @@ def parse_qstat_plain(output: str) -> dict[str, Any]:
     """Parse plain qstat output.
 
     Format:
-    job-ID  prior   name       user         state submit/start at     queue                          slots ja-task-ID
-    --------------------------------------------------------------------------------
-    12345   0.55500 myjob      user         r     01/01/2024 10:00:00 all.q@node1                    1
+    job-ID  prior   name       user         state submit/start at     queue  slots ja-task-ID
+    --------------------------------------------------------------------------
+    12345   0.55500 myjob      user         r     01/01/2024 10:00:00 all.q@node1  1
     """
     jobs: dict[str, Any] = {}