hpc-runner 0.2.2__tar.gz → 0.3.0__tar.gz

{hpc_runner-0.2.2 → hpc_runner-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hpc-runner
-Version: 0.2.2
+Version: 0.3.0
 Summary: Unified HPC job submission across multiple schedulers
 Project-URL: Homepage, https://github.com/sjalloq/hpc-runner
 Project-URL: Repository, https://github.com/sjalloq/hpc-runner

{hpc_runner-0.2.2 → hpc_runner-0.3.0}/src/hpc_runner/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.2.2'
-__version_tuple__ = version_tuple = (0, 2, 2)
+__version__ = version = '0.3.0'
+__version_tuple__ = version_tuple = (0, 3, 0)
 
 __commit_id__ = commit_id = None

{hpc_runner-0.2.2 → hpc_runner-0.3.0}/src/hpc_runner/schedulers/local/scheduler.py

@@ -9,6 +9,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from hpc_runner.core.config import get_config
 from hpc_runner.core.exceptions import AccountingNotAvailable, JobNotFoundError
 from hpc_runner.core.job_info import JobInfo
 from hpc_runner.core.result import ArrayJobResult, JobResult, JobStatus
@@ -26,6 +27,14 @@ class LocalScheduler(BaseScheduler):
     name = "local"
 
     _job_counter: int = 0
+
+    def __init__(self) -> None:
+        """Initialize local scheduler with config-driven settings."""
+        config = get_config()
+        local_config = config.get_scheduler_config("local")
+
+        self.module_init_script = local_config.get("module_init_script", "")
+        
     _processes: dict[str, subprocess.Popen] = {}  # type: ignore[type-arg]
     _exit_codes: dict[str, int] = {}
     _output_paths: dict[str, dict[str, Path]] = {}
@@ -37,8 +46,9 @@ class LocalScheduler(BaseScheduler):
         LocalScheduler._job_counter += 1
         job_id = f"local_{LocalScheduler._job_counter}_{datetime.now().strftime('%Y%m%d%H%M%S')}"
 
-        # Set up environment with modules (modules not actually loaded locally)
+        # Set up environment
         env = os.environ.copy() if job.inherit_env else {}
+        env.update(job.env_vars)
 
         # Generate and write script
         script = self.generate_script(job)
@@ -48,69 +58,87 @@ class LocalScheduler(BaseScheduler):
 
         workdir = Path(job.workdir) if job.workdir else Path.cwd()
 
-        # Determine output paths
-        stdout_file = job.stdout or f"{job.name}.{job_id}.out"
-        stdout_path = workdir / stdout_file
-        if job.merge_output:
-            stderr_path = stdout_path  # Merge stderr into stdout
-        else:
-            stderr_file = job.stderr or f"{job.name}.{job_id}.err"
-            stderr_path = workdir / stderr_file
+        # Check if output should pass through to console (no redirection)
+        passthrough = job.stdout is None and job.stderr is None
+
+        if not passthrough:
+            # Determine output paths
+            stdout_file = job.stdout or f"{job.name}.{job_id}.out"
+            stdout_path = workdir / stdout_file
+            if job.merge_output:
+                stderr_path = stdout_path  # Merge stderr into stdout
+            else:
+                stderr_file = job.stderr or f"{job.name}.{job_id}.err"
+                stderr_path = workdir / stderr_file
 
-        # Store output paths
-        LocalScheduler._output_paths[job_id] = {
-            "stdout": stdout_path,
-            "stderr": stderr_path,
-        }
+            # Store output paths
+            LocalScheduler._output_paths[job_id] = {
+                "stdout": stdout_path,
+                "stderr": stderr_path,
+            }
 
         if interactive:
             # Blocking execution
-            with open(stdout_path, "w") as stdout_f:
-                if job.merge_output:
-                    result = subprocess.run(
-                        [str(script_path)],
-                        cwd=workdir,
-                        env=env,
-                        stdout=stdout_f,
-                        stderr=subprocess.STDOUT,
-                    )
-                else:
-                    with open(stderr_path, "w") as stderr_f:
+            if passthrough:
+                result = subprocess.run(
+                    [str(script_path)],
+                    cwd=workdir,
+                    env=env,
+                )
+            else:
+                with open(stdout_path, "w") as stdout_f:
+                    if job.merge_output:
                         result = subprocess.run(
                             [str(script_path)],
                             cwd=workdir,
                             env=env,
                             stdout=stdout_f,
-                            stderr=stderr_f,
+                            stderr=subprocess.STDOUT,
                         )
+                    else:
+                        with open(stderr_path, "w") as stderr_f:
+                            result = subprocess.run(
+                                [str(script_path)],
+                                cwd=workdir,
+                                env=env,
+                                stdout=stdout_f,
+                                stderr=stderr_f,
+                            )
             LocalScheduler._exit_codes[job_id] = result.returncode
             script_path.unlink(missing_ok=True)
         else:
             # Background execution
-            stdout_f = open(stdout_path, "w")
-            if job.merge_output:
+            if passthrough:
                 proc = subprocess.Popen(
                     [str(script_path)],
                     cwd=workdir,
                     env=env,
-                    stdout=stdout_f,
-                    stderr=subprocess.STDOUT,
                 )
             else:
-                stderr_f = open(stderr_path, "w")
-                proc = subprocess.Popen(
-                    [str(script_path)],
-                    cwd=workdir,
-                    env=env,
-                    stdout=stdout_f,
-                    stderr=stderr_f,
-                )
+                stdout_f = open(stdout_path, "w")
+                if job.merge_output:
+                    proc = subprocess.Popen(
+                        [str(script_path)],
+                        cwd=workdir,
+                        env=env,
+                        stdout=stdout_f,
+                        stderr=subprocess.STDOUT,
+                    )
+                else:
+                    stderr_f = open(stderr_path, "w")
+                    proc = subprocess.Popen(
+                        [str(script_path)],
+                        cwd=workdir,
+                        env=env,
+                        stdout=stdout_f,
+                        stderr=stderr_f,
+                    )
+                proc._stdout_file = stdout_f  # type: ignore[attr-defined]
+                if not job.merge_output:
+                    proc._stderr_file = stderr_f  # type: ignore[attr-defined]
             LocalScheduler._processes[job_id] = proc
             # Store script path for cleanup
             proc._script_path = script_path  # type: ignore[attr-defined]
-            proc._stdout_file = stdout_f  # type: ignore[attr-defined]
-            if not job.merge_output:
-                proc._stderr_file = stderr_f  # type: ignore[attr-defined]
 
         return JobResult(job_id=job_id, scheduler=self, job=job)
 
@@ -137,6 +165,7 @@ class LocalScheduler(BaseScheduler):
     def _submit_array_task(self, job: "Job", job_id: str, index: int) -> None:
         """Submit a single array task."""
         env = os.environ.copy() if job.inherit_env else {}
+        env.update(job.env_vars)
         env["HPC_ARRAY_TASK_ID"] = str(index)
 
         script = self.generate_script(job)

{hpc_runner-0.2.2 → hpc_runner-0.3.0}/src/hpc_runner/schedulers/local/templates/job.sh.j2

@@ -5,20 +5,38 @@
 # Exit on error
 set -e
 
+# Module system initialization
+{% if scheduler.module_init_script %}
+. {{ scheduler.module_init_script }}
+{% else %}
+if [ -f /etc/profile.d/modules.sh ]; then
+    . /etc/profile.d/modules.sh
+elif [ -f /usr/share/Modules/init/bash ]; then
+    . /usr/share/Modules/init/bash
+elif [ -f /etc/modules/init/bash ]; then
+    . /etc/modules/init/bash
+fi
+{% endif %}
+
 {% if job.modules_path %}
-# Additional module paths (simulated for local)
+# Additional module paths
 {% for path in job.modules_path %}
-# module use {{ path }}
+module use {{ path }}
 {% endfor %}
 {% endif %}
 
 {% if job.modules %}
-# Modules (simulated for local - not actually loaded)
+# Load modules
 {% for mod in job.modules %}
-# module load {{ mod }}
+module load {{ mod }}
 {% endfor %}
 {% endif %}
 
+{% if job.venv %}
+# Activate virtual environment
+source {{ job.venv }}/bin/activate
+{% endif %}
+
 {% if job.env_prepend %}
 # Prepend to environment variables
 {% for key, value in job.env_prepend.items() %}

{hpc_runner-0.2.2 → hpc_runner-0.3.0}/tests/test_schedulers/test_local.py

@@ -98,7 +98,7 @@ class TestLocalScheduler:
     def test_output_path(self, temp_dir):
         """Test output file creation."""
         scheduler = LocalScheduler()
-        job = Job(command="echo hello", workdir=temp_dir)
+        job = Job(command="echo hello", workdir=temp_dir, stdout="out.log")
 
         result = scheduler.submit(job, interactive=True)
 
@@ -113,6 +113,7 @@ class TestLocalScheduler:
         job = Job(
             command="echo stdout; echo stderr >&2",
             workdir=temp_dir,
+            stdout="merged.log",
         )
 
         result = scheduler.submit(job, interactive=True)