hpc-runner 0.2.2__tar.gz → 0.3.1__tar.gz

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/.github/workflows/ci.yml

@@ -7,6 +7,30 @@ on:
     branches: [main]
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Ruff lint
+        run: ruff check src/hpc_runner
+
+      - name: Ruff format
+        run: ruff format --check src/hpc_runner
+
+      - name: Mypy
+        run: mypy src/hpc_runner
+
   test:
     runs-on: ubuntu-latest
     strategy:

hpc_runner-0.3.1/.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.19.1
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - "rich-click>=1.7"
+          - "jinja2>=3.0"
+          - "textual>=6.11"
+        pass_filenames: false
+        args: [src/hpc_runner]

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/CLAUDE.md

@@ -2,6 +2,46 @@
 
 This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
 
+## Project Overview
+
+Provides a front end for submitting jobs to an HPC cluster using SGE, Slurm, LSF, etc.  The aim is to enable two things:
+
+1. Abstract away the intricacies of scheduler CLI args and allow the user to submit jobs based on tool name or job type.
+2. Use Environment Modules to construct a clean environment for a job and avoid the 'works for me' problem.
+
+### Job/Type Abstraction
+
+Usage model:
+
+```bash
+# Launch an extern on the cluster via SGE:
+qsub -q <queue> -N <job_name> -cwd -V -l <resources> xterm
+
+# Launch an xterm on the cluster via hpc-runner:
+hpc run xterm
+
+# Or launch a script that uses a particular tool using a job type:
+hpc run --type xcelium run_sim.sh
+```
+
+Configuration file:
+
+* use a TOML based config file to define defaults for jobs
+* define tool flows and specify all scheduler arguments
+    * tool flows are detected via the first argument passed to 'hpc run'
+* definine job types for flows that don't pass the tool
+    * when using makefiles or scripts, the runner can't extract the tool name from the command line
+
+### Consistent Environment
+
+One of the common pitfalls of HPC flows is that what works for one user doesn't always work for another.  Using
+Environment Modules means that common tools flows use a fixed tool version for all users.
+
+To accomplish this, the scheduler must purge all modules as part of its setup script and load any modules defined 
+for the flow.  Each tool or type defined in the configuration file must also define the set of module files that must
+be loaded.
+
+
 ## Build & Development Commands
 
 ```bash

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hpc-runner
-Version: 0.2.2
+Version: 0.3.1
 Summary: Unified HPC job submission across multiple schedulers
 Project-URL: Homepage, https://github.com/sjalloq/hpc-runner
 Project-URL: Repository, https://github.com/sjalloq/hpc-runner
@@ -28,22 +28,24 @@ Provides-Extra: all
 Requires-Dist: build; extra == 'all'
 Requires-Dist: furo>=2024.0.0; extra == 'all'
 Requires-Dist: hatch-vcs; extra == 'all'
-Requires-Dist: mypy; extra == 'all'
+Requires-Dist: mypy>=1.19; extra == 'all'
+Requires-Dist: pre-commit; extra == 'all'
 Requires-Dist: pytest-asyncio; extra == 'all'
 Requires-Dist: pytest-cov; extra == 'all'
 Requires-Dist: pytest>=7.0; extra == 'all'
-Requires-Dist: ruff; extra == 'all'
+Requires-Dist: ruff>=0.15; extra == 'all'
 Requires-Dist: sphinx>=7.0; extra == 'all'
 Requires-Dist: twine; extra == 'all'
 Provides-Extra: dev
 Requires-Dist: build; extra == 'dev'
 Requires-Dist: furo>=2024.0.0; extra == 'dev'
 Requires-Dist: hatch-vcs; extra == 'dev'
-Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: mypy>=1.19; extra == 'dev'
+Requires-Dist: pre-commit; extra == 'dev'
 Requires-Dist: pytest-asyncio; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'
 Requires-Dist: pytest>=7.0; extra == 'dev'
-Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: ruff>=0.15; extra == 'dev'
 Requires-Dist: sphinx>=7.0; extra == 'dev'
 Requires-Dist: twine; extra == 'dev'
 Description-Content-Type: text/markdown

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/defaults/config.toml

@@ -17,7 +17,7 @@ time_resource = "h_rt"
 merge_output = true
 
 # Module handling
-purge_modules = false       # Run 'module purge' before loading modules
+purge_modules = true        # Run 'module purge' before loading modules
 silent_modules = false      # Use -s flag for silent module operations
 module_init_script = ""     # Site-specific module init (empty = auto-detect)
 
@@ -25,6 +25,12 @@ module_init_script = ""     # Site-specific module init (empty = auto-detect)
 expand_makeflags = true     # Expand $NSLOTS in MAKEFLAGS
 unset_vars = []             # Environment variables to unset (e.g., ["https_proxy"])
 
+# Local scheduler settings
+[schedulers.local]
+purge_modules = true
+silent_modules = false
+module_init_script = ""
+
 # Slurm-specific settings (for future use)
 [schedulers.slurm]
 # Default Slurm settings

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/docs/source/configuration.rst

@@ -100,7 +100,7 @@ Save as ``hpc-runner.toml`` (or ``~/.config/hpc-runner/config.toml``):
    time_resource = "h_rt"
 
    merge_output = true
-   purge_modules = false
+   purge_modules = true
    silent_modules = false
    module_init_script = ""
    expand_makeflags = true

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/pyproject.toml

@@ -39,13 +39,14 @@ dev = [
     "pytest>=7.0",
     "pytest-cov",
     "pytest-asyncio",
-    "mypy",
-    "ruff",
+    "mypy>=1.19",
+    "ruff>=0.15",
     "sphinx>=7.0",
     "furo>=2024.0.0",
     "build",
     "twine",
     "hatch-vcs",
+    "pre-commit",
 ]
 all = [
     "hpc-runner[dev]",

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.2.2'
-__version_tuple__ = version_tuple = (0, 2, 2)
+__version__ = version = '0.3.1'
+__version_tuple__ = version_tuple = (0, 3, 1)
 
 __commit_id__ = commit_id = None

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/cli/config.py

@@ -62,7 +62,7 @@ def init(ctx: Context, global_config: bool) -> None:
             return
 
     # Write default config
-    default_config = '''# hpc-runner configuration
+    default_config = """# hpc-runner configuration
 #
 # This file is safe to commit to a project repo (for shared defaults).
 # For a per-user config, run: hpc config init --global
@@ -92,7 +92,7 @@ merge_output = true
 # [types.gpu]
 # queue = "gpu"
 # resources = [{name = "gpu", value = 1}]
-'''
+"""
 
     config_path.write_text(default_config)
     console.print(f"[green]Created {config_path}[/green]")

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/cli/main.py

@@ -1,6 +1,8 @@
 """Main CLI entry point using rich-click."""
 
+from collections.abc import Callable
 from pathlib import Path
+from typing import TypeVar
 
 import rich_click as click
 from rich.console import Console
@@ -11,6 +13,7 @@ click.rich_click.SHOW_ARGUMENTS = True
 # Global console for Rich output
 console = Console()
 
+
 # Context object to pass state between commands
 class Context:
     def __init__(self) -> None:
@@ -18,7 +21,9 @@ class Context:
         self.scheduler: str | None = None
         self.verbose: bool = False
 
-pass_context = click.make_pass_decorator(Context, ensure=True)
+
+F = TypeVar("F", bound=Callable[..., object])
+pass_context: Callable[[F], F] = click.make_pass_decorator(Context, ensure=True)  # type: ignore[assignment]
 
 
 @click.group(context_settings={"help_option_names": ["-h", "--help"]})
@@ -68,8 +73,8 @@ from hpc_runner.cli.status import status  # noqa: E402
 
 cli.add_command(run)
 cli.add_command(status)
-cli.add_command(cancel)
-cli.add_command(config_cmd, name="config")
+cli.add_command(cancel)  # type: ignore[has-type]
+cli.add_command(config_cmd, name="config")  # type: ignore[has-type]
 cli.add_command(monitor)
 
 

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/cli/run.py

@@ -11,6 +11,7 @@ from hpc_runner.cli.main import Context, pass_context
 
 if TYPE_CHECKING:
     from hpc_runner.core.job import Job
+    from hpc_runner.schedulers.base import BaseScheduler
 
 console = Console()
 
@@ -34,10 +35,15 @@ console = Console()
 @click.option("--job-type", "job_type", help="Job type from config")
 @click.option("--module", "modules", multiple=True, help="Modules to load (repeatable)")
 @click.option("--stderr", help="Separate stderr file (default: merged)")
-@click.option("--output", help="Stdout file path pattern")
+@click.option("--stdout", "stdout", help="Stdout file path pattern")
 @click.option("--array", help="Array job specification (e.g., 1-100)")
 @click.option("--depend", help="Job dependency specification")
-@click.option("--inherit-env/--no-inherit-env", "inherit_env", default=True, help="Inherit environment variables")
+@click.option(
+    "--inherit-env/--no-inherit-env",
+    "inherit_env",
+    default=True,
+    help="Inherit environment variables",
+)
 @click.option("--interactive", is_flag=True, help="Run interactively (srun/qrsh)")
 @click.option("--local", is_flag=True, help="Run locally (no scheduler)")
 @click.option("--dry-run", "dry_run", is_flag=True, help="Show what would be submitted")
@@ -58,7 +64,7 @@ def run(
     job_type: str | None,
     modules: tuple[str, ...],
     stderr: str | None,
-    output: str | None,
+    stdout: str | None,
     array: str | None,
     depend: str | None,
     inherit_env: bool,
@@ -128,8 +134,8 @@ def run(
         job.modules = list(modules)
     if stderr:
         job.stderr = stderr
-    if output:
-        job.stdout = output
+    if stdout:
+        job.stdout = stdout
     if depend:
         job.dependency = depend
 
@@ -233,7 +239,10 @@ def _parse_args(args: tuple[str, ...]) -> tuple[list[str], list[str]]:
 
 
 def _show_dry_run(
-    job: "Job", scheduler, scheduler_args: list[str], interactive: bool = False
+    job: "Job",
+    scheduler: "BaseScheduler",
+    scheduler_args: list[str],
+    interactive: bool = False,
 ) -> None:
     """Display what would be submitted."""
     mode = "interactive" if interactive else "batch"
@@ -252,15 +261,21 @@ def _show_dry_run(
         console.print(f"\n[bold]Scheduler passthrough args:[/bold] {' '.join(scheduler_args)}")
 
     console.print("\n[bold]Generated script:[/bold]")
-    if interactive and hasattr(scheduler, "_generate_interactive_script"):
-        script = scheduler._generate_interactive_script(job, "/tmp/example_script.sh")
+    if interactive:
+        script = scheduler.generate_interactive_script(job, "/tmp/example_script.sh")
     else:
         script = scheduler.generate_script(job)
     syntax = Syntax(script, "bash", theme="monokai", line_numbers=True)
     console.print(syntax)
 
 
-def _handle_array_job(job, array_spec: str, scheduler, dry_run: bool, verbose: bool) -> None:
+def _handle_array_job(
+    job: "Job",
+    array_spec: str,
+    scheduler: "BaseScheduler",
+    dry_run: bool,
+    verbose: bool,
+) -> None:
     """Handle array job submission."""
     from hpc_runner.core.job_array import JobArray
 

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/cli/status.py

@@ -1,6 +1,5 @@
 """Status command - check job status."""
 
-
 import rich_click as click
 from rich.console import Console
 from rich.table import Table

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/cli/submit.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 import sys
 from typing import Final
 
-
 _GLOBAL_FLAGS: Final[set[str]] = {"--config", "--scheduler", "--verbose", "-h", "--help"}
 
 
@@ -69,4 +68,3 @@ def main() -> None:
 
     global_opts, rest = _split_global_flags(argv)
     cli.main(args=[*global_opts, "run", *rest], prog_name="submit")
-

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/core/config.py

@@ -10,7 +10,7 @@ from typing import Any
 if sys.version_info >= (3, 11):
     import tomllib
 else:
-    import tomli as tomllib
+    import tomli as tomllib  # type: ignore[import-not-found]
 
 
 @dataclass
@@ -68,7 +68,13 @@ def _merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
             if value and value[0] == "-":
                 result[key] = value[1:]
             else:
-                result[key] = list(set(result[key] + value))
+                seen: set[Any] = set()
+                merged: list[Any] = []
+                for item in result[key] + value:
+                    if item not in seen:
+                        seen.add(item)
+                        merged.append(item)
+                result[key] = merged
         else:
             result[key] = value
     return result

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/core/descriptors.py

@@ -1,7 +1,7 @@
 """Descriptor pattern for job attributes and scheduler arguments."""
 
 from abc import ABC, abstractmethod
-from typing import Any, Generic, TypeVar
+from typing import Any, Generic, TypeVar, overload
 
 T = TypeVar("T")
 
@@ -34,12 +34,18 @@ class JobAttribute(Generic[T]):
     def __init__(self, name: str, *, default: T | None = None):
         self.public_name = name
         self.default = default
-        self._private_name: str | None = None
+        self._private_name: str = f"_{name}"
 
     def __set_name__(self, owner: type, name: str) -> None:
         self._private_name = f"_{name}"
 
-    def __get__(self, obj: Any, objtype: type | None = None) -> T | "JobAttribute[T]":
+    @overload
+    def __get__(self, obj: None, objtype: type) -> "JobAttribute[T]": ...
+
+    @overload
+    def __get__(self, obj: Any, objtype: type | None = None) -> T | None: ...
+
+    def __get__(self, obj: Any, objtype: type | None = None) -> "T | None | JobAttribute[T]":
         if obj is None:
             return self
         return getattr(obj, self._private_name, self.default)

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/core/job.py

@@ -3,7 +3,8 @@
 from __future__ import annotations
 
 import os
-from typing import TYPE_CHECKING, Any, Iterator
+from collections.abc import Iterator
+from typing import TYPE_CHECKING, Any
 
 from hpc_runner.core.descriptors import JobAttribute
 from hpc_runner.core.resources import ResourceSet
@@ -182,7 +183,7 @@ class Job:
     # Submission API
     # =========================================================================
 
-    def submit(self, scheduler: "BaseScheduler | None" = None) -> "JobResult":
+    def submit(self, scheduler: BaseScheduler | None = None) -> JobResult:
         """Submit the job to a scheduler.
 
         This is the primary programmatic API for job submission.
@@ -216,7 +217,7 @@ class Job:
         tool_or_type: str,
         command: str | None = None,
         **overrides: Any,
-    ) -> "Job":
+    ) -> Job:
         """Create a job from configuration.
 
         Looks up job settings from the config file by tool name or job type,
@@ -309,9 +310,9 @@ class Job:
 
     def after(
         self,
-        *jobs: "JobResult",
+        *jobs: JobResult,
         type: str = "afterok",
-    ) -> "Job":
+    ) -> Job:
         """Add job dependencies.
 
         Args:

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/core/job_array.py

@@ -2,8 +2,9 @@
 
 from __future__ import annotations
 
+from collections.abc import Iterator
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Iterator
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from hpc_runner.core.job import Job

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/core/resources.py

@@ -1,5 +1,6 @@
 """Resource abstraction for job resource requests."""
 
+from collections.abc import Iterator
 from dataclasses import dataclass, field
 
 
@@ -39,7 +40,7 @@ class ResourceSet:
                 return r
         return None
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[Resource]:
         return iter(self.resources)
 
     def __len__(self) -> int:

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/schedulers/__init__.py

@@ -18,7 +18,7 @@ _SCHEDULERS: dict[str, str] = {
 }
 
 
-def get_scheduler(name: str | None = None) -> "BaseScheduler":
+def get_scheduler(name: str | None = None) -> BaseScheduler:
     """Get scheduler instance.
 
     Args:
@@ -39,7 +39,7 @@ def get_scheduler(name: str | None = None) -> "BaseScheduler":
     module = importlib.import_module(module_path)
     scheduler_class = getattr(module, class_name)
 
-    return scheduler_class()
+    return scheduler_class()  # type: ignore[no-any-return]
 
 
 def register_scheduler(name: str, import_path: str) -> None:

{hpc_runner-0.2.2 → hpc_runner-0.3.1}/src/hpc_runner/schedulers/base.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from datetime import datetime
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from hpc_runner.core.descriptors import SchedulerArg
 
@@ -35,13 +35,13 @@ class BaseScheduler(ABC):
     name: str = ""
 
     # Subclasses populate this in __init__ with config-driven values
-    ARG_RENDERERS: dict[str, SchedulerArg] = {}
+    ARG_RENDERERS: dict[str, SchedulerArg[Any]] = {}
 
     # =========================================================================
     # Rendering Protocol
     # =========================================================================
 
-    def render_directives(self, job: "Job") -> list[str]:
+    def render_directives(self, job: Job) -> list[str]:
         """Render job attributes as script directives.
 
         Iterates over job's renderable attributes and uses ARG_RENDERERS
@@ -66,7 +66,7 @@ class BaseScheduler(ABC):
 
         return directives
 
-    def render_args(self, job: "Job") -> list[str]:
+    def render_args(self, job: Job) -> list[str]:
         """Render job attributes as command-line arguments.
 
         Iterates over job's renderable attributes and uses ARG_RENDERERS
@@ -94,9 +94,7 @@ class BaseScheduler(ABC):
     # =========================================================================
 
     @abstractmethod
-    def submit(
-        self, job: "Job", interactive: bool = False, keep_script: bool = False
-    ) -> "JobResult":
+    def submit(self, job: Job, interactive: bool = False, keep_script: bool = False) -> JobResult:
         """Submit a job to the scheduler.
 
         Args:
@@ -106,7 +104,7 @@ class BaseScheduler(ABC):
         """
 
     @abstractmethod
-    def submit_array(self, array: "JobArray") -> "ArrayJobResult":
+    def submit_array(self, array: JobArray) -> ArrayJobResult:
         """Submit an array job."""
 
     @abstractmethod
@@ -114,7 +112,7 @@ class BaseScheduler(ABC):
         """Cancel a job."""
 
     @abstractmethod
-    def get_status(self, job_id: str) -> "JobStatus":
+    def get_status(self, job_id: str) -> JobStatus:
         """Get job status."""
 
     @abstractmethod
@@ -122,21 +120,37 @@ class BaseScheduler(ABC):
         """Get job exit code."""
 
     @abstractmethod
-    def generate_script(self, job: "Job", array_range: str | None = None) -> str:
+    def generate_script(self, job: Job, array_range: str | None = None) -> str:
         """Generate submission script."""
 
     @abstractmethod
-    def build_submit_command(self, job: "Job") -> list[str]:
+    def build_submit_command(self, job: Job) -> list[str]:
         """Build submission command line."""
 
     @abstractmethod
-    def build_interactive_command(self, job: "Job") -> list[str]:
+    def build_interactive_command(self, job: Job) -> list[str]:
         """Build interactive execution command."""
 
     # =========================================================================
     # Optional Methods - Override if scheduler supports these
     # =========================================================================
 
+    def generate_interactive_script(self, job: Job, script_path: str) -> str:
+        """Generate wrapper script for interactive jobs.
+
+        By default, falls back to the standard batch script.
+        Override in subclasses that need a different template for
+        interactive sessions (e.g. SGE uses qrsh with no #$ directives).
+
+        Args:
+            job: Job to generate script for.
+            script_path: Path where the script will be written.
+
+        Returns:
+            Script content as a string.
+        """
+        return self.generate_script(job)
+
     def get_output_path(self, job_id: str, stream: str) -> Path | None:
         """Get path to output file.
 
@@ -149,16 +163,16 @@ class BaseScheduler(ABC):
         """
         return None
 
-    def get_scheduler_args(self, job: "Job") -> list[str]:
+    def get_scheduler_args(self, job: Job) -> list[str]:
         """Get scheduler-specific raw args from job."""
         return getattr(job, f"{self.name}_args", [])
 
     def list_active_jobs(
         self,
         user: str | None = None,
-        status: set["JobStatus"] | None = None,
+        status: set[JobStatus] | None = None,
         queue: str | None = None,
-    ) -> list["JobInfo"]:
+    ) -> list[JobInfo]:
         """List active jobs. Override in subclass."""
         return []
 
@@ -170,7 +184,7 @@ class BaseScheduler(ABC):
         exit_code: int | None = None,
         queue: str | None = None,
         limit: int = 100,
-    ) -> list["JobInfo"]:
+    ) -> list[JobInfo]:
         """List completed jobs from accounting. Override in subclass."""
         return []
 
@@ -178,7 +192,7 @@ class BaseScheduler(ABC):
         """Check if job accounting/history is available."""
         return False
 
-    def get_job_details(self, job_id: str) -> tuple["JobInfo", dict[str, object]]:
+    def get_job_details(self, job_id: str) -> tuple[JobInfo, dict[str, object]]:
         """Get detailed information for a single job.
 
         Args: