hpc-runner 0.1.0__py3-none-any.whl

hpc_runner/__init__.py

@@ -0,0 +1,57 @@
+"""hpc-runner: HPC job submission across multiple schedulers."""
+
+try:
+    from hpc_runner._version import __version__
+except ImportError:
+    __version__ = "0.0.0.dev0"
+
+from hpc_runner.core.config import HPCConfig, get_config, load_config, reload_config
+from hpc_runner.core.exceptions import (
+    ConfigError,
+    ConfigNotFoundError,
+    HPCToolsError,
+    JobNotFoundError,
+    SchedulerError,
+    SubmissionError,
+    ValidationError,
+)
+from hpc_runner.core.job import Job
+from hpc_runner.core.job_array import JobArray
+from hpc_runner.core.resources import Resource, ResourceSet
+from hpc_runner.core.result import ArrayJobResult, JobResult, JobStatus
+from hpc_runner.schedulers import get_scheduler, list_schedulers, register_scheduler
+from hpc_runner.workflow import DependencyType, Pipeline, PipelineJob
+
+__all__ = [
+    # Version
+    "__version__",
+    # Core
+    "Job",
+    "JobArray",
+    "JobResult",
+    "ArrayJobResult",
+    "JobStatus",
+    "Resource",
+    "ResourceSet",
+    # Config
+    "load_config",
+    "get_config",
+    "reload_config",
+    "HPCConfig",
+    # Schedulers
+    "get_scheduler",
+    "register_scheduler",
+    "list_schedulers",
+    # Workflow
+    "Pipeline",
+    "PipelineJob",
+    "DependencyType",
+    # Exceptions
+    "HPCToolsError",
+    "SchedulerError",
+    "SubmissionError",
+    "JobNotFoundError",
+    "ConfigError",
+    "ConfigNotFoundError",
+    "ValidationError",
+]

hpc_runner/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

hpc_runner/cli/__init__.py

@@ -0,0 +1 @@
+"""CLI for hpc-tools."""

hpc_runner/cli/cancel.py

@@ -0,0 +1,38 @@
+"""Cancel command - cancel running jobs."""
+
+import rich_click as click
+from rich.console import Console
+
+from hpc_runner.cli.main import Context, pass_context
+
+console = Console()
+
+
+@click.command()
+@click.argument("job_id")
+@click.option("--force", "-f", is_flag=True, help="Force cancel without confirmation")
+@pass_context
+def cancel(
+    ctx: Context,
+    job_id: str,
+    force: bool,
+) -> None:
+    """Cancel a job.
+
+    JOB_ID is the job ID to cancel.
+    """
+    from hpc_runner.schedulers import get_scheduler
+
+    scheduler = get_scheduler(ctx.scheduler)
+
+    if not force:
+        if not click.confirm(f"Cancel job {job_id}?"):
+            console.print("[yellow]Cancelled[/yellow]")
+            return
+
+    success = scheduler.cancel(job_id)
+
+    if success:
+        console.print(f"[green]Job {job_id} cancelled[/green]")
+    else:
+        console.print(f"[red]Failed to cancel job {job_id}[/red]")

hpc_runner/cli/config.py

@@ -0,0 +1,109 @@
+"""Config command - manage configuration."""
+
+from pathlib import Path
+
+import rich_click as click
+from rich.console import Console
+from rich.syntax import Syntax
+
+from hpc_runner.cli.main import Context, pass_context
+
+console = Console()
+
+
+@click.group()
+def config_cmd() -> None:
+    """Manage configuration."""
+    pass
+
+
+@config_cmd.command("show")
+@pass_context
+def show(ctx: Context) -> None:
+    """Show current configuration."""
+    from hpc_runner.core.config import find_config_file, load_config
+
+    config_path = ctx.config_path or find_config_file()
+
+    if config_path is None:
+        console.print("[yellow]No configuration file found[/yellow]")
+        console.print("Using default settings")
+        console.print("\nSearch locations:")
+        console.print("  1. ./hpc-tools.toml")
+        console.print("  2. ./pyproject.toml [tool.hpc-tools]")
+        console.print("  3. <git root>/hpc-tools.toml")
+        console.print("  4. ~/.config/hpc-tools/config.toml")
+        return
+
+    console.print(f"[bold]Config file:[/bold] {config_path}")
+    console.print()
+
+    # Read and display the config file
+    content = config_path.read_text()
+    syntax = Syntax(content, "toml", theme="monokai", line_numbers=True)
+    console.print(syntax)
+
+
+@config_cmd.command("init")
+@click.option("--global", "-g", "global_config", is_flag=True, help="Create global config")
+@pass_context
+def init(ctx: Context, global_config: bool) -> None:
+    """Create a new configuration file."""
+    if global_config:
+        config_dir = Path.home() / ".config" / "hpc-tools"
+        config_dir.mkdir(parents=True, exist_ok=True)
+        config_path = config_dir / "config.toml"
+    else:
+        config_path = Path.cwd() / "hpc-tools.toml"
+
+    if config_path.exists():
+        if not click.confirm(f"{config_path} already exists. Overwrite?"):
+            console.print("[yellow]Cancelled[/yellow]")
+            return
+
+    # Write default config
+    default_config = '''# hpc-tools configuration
+
+[defaults]
+# Default job settings
+cpu = 1
+mem = "4G"
+time = "1:00:00"
+# queue = "batch"
+
+# Modules to always load
+modules = []
+
+[schedulers.sge]
+# SGE-specific settings
+parallel_environment = "smp"
+memory_resource = "mem_free"
+time_resource = "h_rt"
+merge_output = true
+
+# Tool-specific configurations
+# [tools.python]
+# modules = ["python/3.11"]
+
+# Job type configurations
+# [types.gpu]
+# queue = "gpu"
+# resources = [{name = "gpu", value = 1}]
+'''
+
+    config_path.write_text(default_config)
+    console.print(f"[green]Created {config_path}[/green]")
+
+
+@config_cmd.command("path")
+@pass_context
+def path(ctx: Context) -> None:
+    """Show path to active configuration file."""
+    from hpc_runner.core.config import find_config_file
+
+    config_path = ctx.config_path or find_config_file()
+
+    if config_path:
+        console.print(str(config_path))
+    else:
+        console.print("[yellow]No configuration file found[/yellow]")

hpc_runner/cli/main.py

@@ -0,0 +1,72 @@
+"""Main CLI entry point using rich-click."""
+
+from pathlib import Path
+from typing import Optional
+
+import rich_click as click
+from rich.console import Console
+
+# Configure rich-click
+click.rich_click.SHOW_ARGUMENTS = True
+
+# Global console for Rich output
+console = Console()
+
+# Context object to pass state between commands
+class Context:
+    def __init__(self) -> None:
+        self.config_path: Optional[Path] = None
+        self.scheduler: Optional[str] = None
+        self.verbose: bool = False
+
+pass_context = click.make_pass_decorator(Context, ensure=True)
+
+
+@click.group()
+@click.option(
+    "--config", "-c",
+    type=click.Path(exists=True, path_type=Path),
+    help="Path to configuration file",
+)
+@click.option(
+    "--scheduler", "-s",
+    type=str,
+    help="Force scheduler (sge, slurm, pbs, local)",
+)
+@click.option(
+    "--verbose", "-v",
+    is_flag=True,
+    help="Enable verbose output",
+)
+@click.version_option(package_name="hpc-runner")
+@pass_context
+def cli(ctx: Context, config: Optional[Path], scheduler: Optional[str], verbose: bool) -> None:
+    """HPC job submission tool.
+
+    Submit and manage jobs across different HPC schedulers (SGE, Slurm, PBS)
+    with a unified interface.
+    """
+    ctx.config_path = config
+    ctx.scheduler = scheduler
+    ctx.verbose = verbose
+
+
+# Import and register subcommands
+from hpc_runner.cli.run import run
+from hpc_runner.cli.status import status
+from hpc_runner.cli.cancel import cancel
+from hpc_runner.cli.config import config_cmd
+
+cli.add_command(run)
+cli.add_command(status)
+cli.add_command(cancel)
+cli.add_command(config_cmd, name="config")
+
+
+def main() -> None:
+    """Entry point for console script."""
+    cli()
+
+
+if __name__ == "__main__":
+    main()

hpc_runner/cli/run.py

@@ -0,0 +1,136 @@
+"""Run command - submit jobs to the scheduler."""
+
+from typing import Optional, Tuple
+
+import rich_click as click
+from rich.console import Console
+from rich.panel import Panel
+from rich.syntax import Syntax
+
+from hpc_runner.cli.main import Context, pass_context
+
+console = Console()
+
+
+@click.command()
+@click.argument("command", nargs=-1, required=True)
+@click.option("--name", "-N", help="Job name")
+@click.option("--cpu", "-c", type=int, help="Number of CPUs")
+@click.option("--mem", "-m", help="Memory requirement (e.g., 16G)")
+@click.option("--time", "-t", help="Time limit (e.g., 4:00:00)")
+@click.option("--queue", "-q", help="Queue/partition name")
+@click.option("--interactive", "-I", is_flag=True, help="Run interactively (blocking)")
+@click.option("--local", "-L", is_flag=True, help="Run locally (no scheduler)")
+@click.option("--type", "-T", "job_type", help="Job type from config")
+@click.option("--module", "-M", multiple=True, help="Modules to load (can be repeated)")
+@click.option("--raw", "-R", multiple=True, help="Raw scheduler args (can be repeated)")
+@click.option("--dry-run", "-n", is_flag=True, help="Show what would be submitted")
+@click.option("--stderr", "-e", help="Separate stderr file (default: merged with stdout)")
+@pass_context
+def run(
+    ctx: Context,
+    command: Tuple[str, ...],
+    name: Optional[str],
+    cpu: Optional[int],
+    mem: Optional[str],
+    time: Optional[str],
+    queue: Optional[str],
+    interactive: bool,
+    local: bool,
+    job_type: Optional[str],
+    module: Tuple[str, ...],
+    raw: Tuple[str, ...],
+    dry_run: bool,
+    stderr: Optional[str],
+) -> None:
+    """Submit a job to the scheduler.
+
+    COMMAND is the command to execute. Use quotes for complex commands:
+
+        hpc run "make -j8 all"
+
+        hpc run python script.py --arg value
+    """
+    from hpc_runner.core.job import Job
+    from hpc_runner.schedulers import get_scheduler
+
+    # Get scheduler
+    scheduler_name = "local" if local else ctx.scheduler
+    scheduler = get_scheduler(scheduler_name)
+
+    # Build command string
+    cmd_str = " ".join(command)
+
+    # Create job from config or parameters
+    if job_type:
+        job = Job.from_config(job_type, command=cmd_str)
+    else:
+        job = Job(command=cmd_str)
+
+    # Override with CLI arguments
+    if name:
+        job.name = name
+    if cpu:
+        job.cpu = cpu
+    if mem:
+        job.mem = mem
+    if time:
+        job.time = time
+    if queue:
+        job.queue = queue
+    if module:
+        job.modules = list(module)
+    if raw:
+        job.raw_args = list(raw)
+    if stderr:
+        job.stderr = stderr
+
+    if dry_run:
+        _show_dry_run(job, scheduler)
+        return
+
+    # Submit
+    result = scheduler.submit(job, interactive=interactive)
+
+    if interactive:
+        if result.returncode == 0:
+            console.print(f"[green]Job completed successfully[/green]")
+        else:
+            console.print(f"[red]Job failed with exit code: {result.returncode}[/red]")
+    else:
+        console.print(f"Submitted job [bold cyan]{result.job_id}[/bold cyan]")
+        if ctx.verbose:
+            console.print(f"  Scheduler: {scheduler.name}")
+            console.print(f"  Job name: {job.name}")
+            console.print(f"  Command: {job.command}")
+
+
+def _show_dry_run(job: "Job", scheduler: "BaseScheduler") -> None:
+    """Display what would be submitted."""
+    from hpc_runner.schedulers.base import BaseScheduler
+
+    console.print(Panel.fit("[bold]Dry Run[/bold]", border_style="yellow"))
+    console.print(f"[bold]Scheduler:[/bold] {scheduler.name}")
+    console.print(f"[bold]Job Name:[/bold] {job.name}")
+    console.print(f"[bold]Command:[/bold] {job.command}")
+
+    if job.cpu:
+        console.print(f"[bold]CPU:[/bold] {job.cpu}")
+    if job.mem:
+        console.print(f"[bold]Memory:[/bold] {job.mem}")
+    if job.time:
+        console.print(f"[bold]Time:[/bold] {job.time}")
+    if job.queue:
+        console.print(f"[bold]Queue:[/bold] {job.queue}")
+    if job.modules:
+        console.print(f"[bold]Modules:[/bold] {', '.join(job.modules)}")
+    if job.merge_output:
+        console.print(f"[bold]Output:[/bold] merged (stdout only)")
+    else:
+        console.print(f"[bold]Stderr:[/bold] {job.stderr}")
+
+    console.print()
+    console.print("[bold]Generated Script:[/bold]")
+    script = scheduler.generate_script(job)
+    syntax = Syntax(script, "bash", theme="monokai", line_numbers=True)
+    console.print(syntax)

hpc_runner/cli/status.py

@@ -0,0 +1,65 @@
+"""Status command - check job status."""
+
+from typing import Optional
+
+import rich_click as click
+from rich.console import Console
+from rich.table import Table
+
+from hpc_runner.cli.main import Context, pass_context
+
+console = Console()
+
+
+@click.command()
+@click.argument("job_id", required=False)
+@click.option("--all", "-a", "all_users", is_flag=True, help="Show all users' jobs")
+@click.option("--watch", "-w", is_flag=True, help="Watch mode (refresh periodically)")
+@pass_context
+def status(
+    ctx: Context,
+    job_id: Optional[str],
+    all_users: bool,
+    watch: bool,
+) -> None:
+    """Check job status.
+
+    If JOB_ID is provided, show status of that specific job.
+    Otherwise, list all your jobs.
+    """
+    from hpc_runner.schedulers import get_scheduler
+
+    scheduler = get_scheduler(ctx.scheduler)
+
+    if job_id:
+        # Show specific job status
+        status = scheduler.get_status(job_id)
+        exit_code = scheduler.get_exit_code(job_id)
+
+        table = Table(title=f"Job {job_id}")
+        table.add_column("Property", style="cyan")
+        table.add_column("Value")
+
+        table.add_row("Status", _status_style(status.name))
+        if exit_code is not None:
+            table.add_row("Exit Code", str(exit_code))
+
+        console.print(table)
+    else:
+        # List all jobs (not implemented for all schedulers)
+        console.print("[yellow]Listing all jobs requires scheduler-specific implementation[/yellow]")
+        console.print("Use 'hpc status <job_id>' to check a specific job")
+
+
+def _status_style(status: str) -> str:
+    """Apply color to status string."""
+    colors = {
+        "PENDING": "[yellow]PENDING[/yellow]",
+        "RUNNING": "[blue]RUNNING[/blue]",
+        "COMPLETED": "[green]COMPLETED[/green]",
+        "FAILED": "[red]FAILED[/red]",
+        "CANCELLED": "[magenta]CANCELLED[/magenta]",
+        "TIMEOUT": "[red]TIMEOUT[/red]",
+        "UNKNOWN": "[dim]UNKNOWN[/dim]",
+    }
+    return colors.get(status, status)

hpc_runner/core/__init__.py

@@ -0,0 +1 @@
+"""Core models and abstractions for hpc-tools."""

hpc_runner/core/config.py

@@ -0,0 +1,177 @@
+"""Configuration loading and management."""
+
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+
+@dataclass
+class HPCConfig:
+    """Loaded configuration."""
+
+    defaults: dict[str, Any] = field(default_factory=dict)
+    tools: dict[str, dict[str, Any]] = field(default_factory=dict)
+    types: dict[str, dict[str, Any]] = field(default_factory=dict)
+    schedulers: dict[str, dict[str, Any]] = field(default_factory=dict)
+
+    _source_path: Path | None = field(default=None, repr=False)
+
+    def get_job_config(self, tool_or_type: str) -> dict[str, Any]:
+        """Get merged configuration for a tool or type.
+
+        Lookup order:
+        1. Check types[tool_or_type]
+        2. Check tools[tool_or_type]
+        3. Fall back to defaults
+        """
+        config = self.defaults.copy()
+
+        if tool_or_type in self.types:
+            config = _merge(config, self.types[tool_or_type])
+        elif tool_or_type in self.tools:
+            config = _merge(config, self.tools[tool_or_type])
+
+        return config
+
+    def get_tool_config(self, command: str) -> dict[str, Any]:
+        """Get configuration matching a command.
+
+        Extracts tool name from command and looks up config.
+        """
+        # Extract tool name (first word, strip path)
+        tool = command.split()[0]
+        tool = Path(tool).name
+
+        return self.get_job_config(tool)
+
+    def get_scheduler_config(self, scheduler: str) -> dict[str, Any]:
+        """Get scheduler-specific configuration."""
+        return self.schedulers.get(scheduler, {})
+
+
+def _merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+    """Deep merge with override taking precedence."""
+    result = base.copy()
+    for key, value in override.items():
+        if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+            result[key] = _merge(result[key], value)
+        elif key in result and isinstance(result[key], list) and isinstance(value, list):
+            # Check for list reset marker
+            if value and value[0] == "-":
+                result[key] = value[1:]
+            else:
+                result[key] = list(set(result[key] + value))
+        else:
+            result[key] = value
+    return result
+
+
+def find_config_file() -> Path | None:
+    """Find configuration file in priority order.
+
+    Search order:
+    1. ./hpc-tools.toml (current directory)
+    2. ./pyproject.toml [tool.hpc-tools] section
+    3. Git repository root hpc-tools.toml
+    4. ~/.config/hpc-tools/config.toml
+    5. Package defaults
+    """
+    # Current directory
+    cwd = Path.cwd()
+    if (cwd / "hpc-tools.toml").exists():
+        return cwd / "hpc-tools.toml"
+
+    if (cwd / "pyproject.toml").exists():
+        try:
+            with open(cwd / "pyproject.toml", "rb") as f:
+                pyproject = tomllib.load(f)
+            if "tool" in pyproject and "hpc-tools" in pyproject["tool"]:
+                return cwd / "pyproject.toml"
+        except Exception:
+            pass
+
+    # Git root
+    git_root = _find_git_root(cwd)
+    if git_root and (git_root / "hpc-tools.toml").exists():
+        return git_root / "hpc-tools.toml"
+
+    # User config
+    user_config = Path.home() / ".config" / "hpc-tools" / "config.toml"
+    if user_config.exists():
+        return user_config
+
+    # Package defaults
+    package_defaults = Path(__file__).parent.parent.parent.parent / "defaults" / "config.toml"
+    if package_defaults.exists():
+        return package_defaults
+
+    return None
+
+
+def _find_git_root(start: Path) -> Path | None:
+    """Find git repository root."""
+    current = start.resolve()
+    while current != current.parent:
+        if (current / ".git").exists():
+            return current
+        current = current.parent
+    return None
+
+
+def load_config(path: Path | str | None = None) -> HPCConfig:
+    """Load configuration from file.
+
+    Args:
+        path: Explicit config path or None to auto-discover
+    """
+    if path is None:
+        path = find_config_file()
+
+    if path is None:
+        return HPCConfig()  # Empty config, use defaults
+
+    path = Path(path)
+
+    with open(path, "rb") as f:
+        data = tomllib.load(f)
+
+    # Handle pyproject.toml
+    if path.name == "pyproject.toml":
+        data = data.get("tool", {}).get("hpc-tools", {})
+
+    config = HPCConfig(
+        defaults=data.get("defaults", {}),
+        tools=data.get("tools", {}),
+        types=data.get("types", {}),
+        schedulers=data.get("schedulers", {}),
+    )
+    config._source_path = path
+
+    return config
+
+
+# Global config cache
+_cached_config: HPCConfig | None = None
+
+
+def get_config() -> HPCConfig:
+    """Get the global configuration (cached)."""
+    global _cached_config
+    if _cached_config is None:
+        _cached_config = load_config()
+    return _cached_config
+
+
+def reload_config(path: Path | str | None = None) -> HPCConfig:
+    """Reload configuration (clears cache)."""
+    global _cached_config
+    _cached_config = load_config(path)
+    return _cached_config