hpc-runner 0.2.0__py3-none-any.whl

hpc_runner/core/job.py

@@ -0,0 +1,328 @@
+"""Job model - pure data container with no scheduler knowledge."""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING, Any, Iterator
+
+from hpc_runner.core.descriptors import JobAttribute
+from hpc_runner.core.resources import ResourceSet
+
+if TYPE_CHECKING:
+    from hpc_runner.core.result import JobResult
+    from hpc_runner.schedulers.base import BaseScheduler
+
+
+class Job:
+    """HPC job specification.
+
+    This is a pure data container. It has no knowledge of any specific
+    scheduler's syntax. Rendering to scheduler-specific formats is handled
+    by the scheduler classes.
+
+    Attributes are defined using JobAttribute descriptors, which enables:
+    - Clean attribute access: job.name, job.cpu, etc.
+    - Iteration over set attributes via iter_attributes()
+    - Class-level introspection: Job.name returns the descriptor
+
+    Example:
+        job = Job(
+            command="python train.py",
+            name="training_run",
+            cpu=4,
+            mem="16G",
+            time="4:00:00",
+        )
+
+        # Direct access
+        print(job.name)  # "training_run"
+        print(job.cpu)   # 4
+
+        # Iterate over set attributes
+        for attr, value in job.iter_attributes():
+            print(f"{attr}={value}")
+    """
+
+    # =========================================================================
+    # Attribute Descriptors
+    # =========================================================================
+
+    # Job identification
+    name = JobAttribute[str]("name")
+
+    # Resource requests
+    cpu = JobAttribute[int]("cpu")
+    mem = JobAttribute[str]("mem")
+    time = JobAttribute[str]("time")
+
+    # Scheduling
+    queue = JobAttribute[str]("queue")
+    priority = JobAttribute[int]("priority")
+
+    # MPI/Multi-node jobs (primarily Slurm, but kept for compatibility)
+    nodes = JobAttribute[int]("nodes")
+    tasks = JobAttribute[int]("tasks")
+
+    # Output handling
+    stdout = JobAttribute[str]("stdout")
+    stderr = JobAttribute[str]("stderr")
+
+    # Environment
+    inherit_env = JobAttribute[bool]("inherit_env", default=True)
+    workdir = JobAttribute[str]("workdir")
+    shell = JobAttribute[str]("shell", default="/bin/bash")
+    venv = JobAttribute[str]("venv")  # Virtual environment path
+
+    # Working directory behavior
+    use_cwd = JobAttribute[bool]("use_cwd", default=True)
+
+    # Note: 'dependency' is NOT a descriptor - it's handled specially by schedulers
+    # because it involves both string form (CLI) and programmatic form (Job.after())
+
+    # =========================================================================
+    # Attribute Registry - Order matters for directive generation
+    # =========================================================================
+
+    RENDERABLE_ATTRIBUTES: list[str] = [
+        "shell",
+        "use_cwd",
+        "inherit_env",
+        "name",
+        "cpu",
+        "mem",
+        "time",
+        "queue",
+        "priority",
+        "nodes",
+        "tasks",
+        "stdout",
+        "stderr",
+    ]
+
+    # =========================================================================
+    # Initialization
+    # =========================================================================
+
+    def __init__(
+        self,
+        command: str | list[str],
+        *,
+        name: str | None = None,
+        cpu: int | None = None,
+        mem: str | None = None,
+        time: str | None = None,
+        queue: str | None = None,
+        priority: int | None = None,
+        nodes: int | None = None,
+        tasks: int | None = None,
+        stdout: str | None = None,
+        stderr: str | None = None,
+        inherit_env: bool = True,
+        workdir: str | None = None,
+        shell: str = "/bin/bash",
+        use_cwd: bool = True,
+        venv: str | None = None,
+        env_vars: dict[str, str] | None = None,
+        modules: list[str] | None = None,
+        modules_path: list[str] | None = None,
+        resources: ResourceSet | None = None,
+        raw_args: list[str] | None = None,
+        sge_args: list[str] | None = None,
+        slurm_args: list[str] | None = None,
+        pbs_args: list[str] | None = None,
+        dependency: str | None = None,
+    ):
+        # Command handling
+        if isinstance(command, list):
+            self.command = " ".join(command)
+        else:
+            self.command = command
+
+        # Set descriptor-based attributes
+        self.name = name or self._generate_name()
+        self.cpu = cpu
+        self.mem = mem
+        self.time = time
+        self.queue = queue
+        self.priority = priority
+        self.nodes = nodes
+        self.tasks = tasks
+        self.stdout = stdout
+        self.stderr = stderr
+        self.inherit_env = inherit_env
+        self.workdir = workdir
+        self.shell = shell
+        self.use_cwd = use_cwd
+
+        # Virtual environment - auto-capture from VIRTUAL_ENV if not specified
+        if venv is None:
+            venv = os.environ.get("VIRTUAL_ENV")
+        self.venv = venv
+
+        # Non-descriptor attributes
+        self.env_vars: dict[str, str] = env_vars or {}
+        self.modules: list[str] = modules or []
+        self.modules_path: list[str] = modules_path or []
+        self.resources: ResourceSet = resources or ResourceSet()
+        self.raw_args: list[str] = raw_args or []
+        self.sge_args: list[str] = sge_args or []
+        self.slurm_args: list[str] = slurm_args or []
+        self.pbs_args: list[str] = pbs_args or []
+        self.dependency: str | None = dependency
+
+        # Programmatic dependencies (from .after() method)
+        self.dependencies: list[JobResult] = []
+        self.dependency_type: str = "afterok"
+
+    # =========================================================================
+    # Submission API
+    # =========================================================================
+
+    def submit(self, scheduler: "BaseScheduler | None" = None) -> "JobResult":
+        """Submit the job to a scheduler.
+
+        This is the primary programmatic API for job submission.
+
+        Args:
+            scheduler: Scheduler to use. If None, auto-detects based on
+                       environment (checks HPC_SCHEDULER env var, then
+                       probes for SGE_ROOT, sbatch, etc.)
+
+        Returns:
+            JobResult with job ID and methods to check status, get output, etc.
+
+        Example:
+            job = Job("python train.py", cpu=4, mem="16G")
+            result = job.submit()
+            print(f"Submitted: {result.job_id}")
+
+            # Wait for completion
+            result.wait()
+            print(f"Exit code: {result.exit_code}")
+        """
+        from hpc_runner.schedulers import get_scheduler
+
+        if scheduler is None:
+            scheduler = get_scheduler()
+        return scheduler.submit(self)
+
+    @classmethod
+    def from_config(
+        cls,
+        tool_or_type: str,
+        command: str | None = None,
+        **overrides: Any,
+    ) -> "Job":
+        """Create a job from configuration.
+
+        Looks up job settings from the config file by tool name or job type,
+        then applies any overrides.
+
+        Args:
+            tool_or_type: Tool name (e.g., "python", "make") or job type
+                          (e.g., "interactive", "gpu") from config
+            command: Command to run. If None, uses command from config.
+            **overrides: Override any job parameters
+
+        Returns:
+            Job configured according to config file + overrides
+
+        Example:
+            # Config file has [types.gpu] with queue="gpu", resources=[{gpu=1}]
+            job = Job.from_config("gpu", command="python train.py")
+        """
+        from hpc_runner.core.config import get_config
+
+        config = get_config()
+        job_config = config.get_job_config(tool_or_type)
+
+        if command is not None:
+            job_config["command"] = command
+        job_config.update(overrides)
+
+        # Handle resources specially
+        if "resources" in job_config and isinstance(job_config["resources"], list):
+            resource_set = ResourceSet()
+            for r in job_config["resources"]:
+                resource_set.add(r["name"], r["value"])
+            job_config["resources"] = resource_set
+
+        return cls(**job_config)
+
+    # =========================================================================
+    # Attribute Iteration
+    # =========================================================================
+
+    def iter_attributes(self) -> Iterator[tuple[str, Any]]:
+        """Iterate over renderable attributes that have been set.
+
+        Yields:
+            Tuples of (attribute_name, value) for attributes that are not None
+            and not equal to their default "skip" values.
+
+        Note:
+            The iteration order follows RENDERABLE_ATTRIBUTES, which is
+            designed to produce sensible directive ordering.
+        """
+        for attr_name in self.RENDERABLE_ATTRIBUTES:
+            value = getattr(self, attr_name)
+
+            # Skip None values
+            if value is None:
+                continue
+
+            # Skip False for boolean attributes (they're opt-in)
+            # Exception: use_cwd and inherit_env default True, so False means explicit opt-out
+            descriptor = getattr(self.__class__, attr_name)
+            if isinstance(value, bool) and value is False and descriptor.default is not True:
+                continue
+
+            yield attr_name, value
+
+    # =========================================================================
+    # Properties
+    # =========================================================================
+
+    @property
+    def merge_output(self) -> bool:
+        """Whether to merge stderr into stdout."""
+        return self.stderr is None
+
+    # =========================================================================
+    # Helper Methods
+    # =========================================================================
+
+    def _generate_name(self) -> str:
+        """Generate a job name from username and command."""
+        user = os.environ.get("USER", "user")
+        # Extract first meaningful word from command
+        cmd_parts = self.command.split()
+        for part in cmd_parts:
+            if "=" not in part:
+                cmd_name = part.split("/")[-1]  # Handle paths
+                return f"{user}_{cmd_name}"
+        return f"{user}_job"
+
+    def after(
+        self,
+        *jobs: "JobResult",
+        type: str = "afterok",
+    ) -> "Job":
+        """Add job dependencies.
+
+        Args:
+            *jobs: JobResult objects this job depends on
+            type: Dependency type (afterok, afterany, afternotok)
+
+        Returns:
+            Self for method chaining
+        """
+        self.dependencies.extend(jobs)
+        self.dependency_type = type
+        return self
+
+    def __repr__(self) -> str:
+        attrs = []
+        for attr, value in self.iter_attributes():
+            attrs.append(f"{attr}={value!r}")
+        return f"Job(command={self.command!r}, {', '.join(attrs)})"

hpc_runner/core/job_array.py

@@ -0,0 +1,58 @@
+"""Job array support for batch processing."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Iterator
+
+if TYPE_CHECKING:
+    from hpc_runner.core.job import Job
+    from hpc_runner.core.result import ArrayJobResult
+    from hpc_runner.schedulers.base import BaseScheduler
+
+
+@dataclass
+class JobArray:
+    """Represents an array job.
+
+    Attributes:
+        job: Base job specification
+        start: Array start index
+        end: Array end index
+        step: Array step (default 1)
+        max_concurrent: Max simultaneous tasks (throttling)
+    """
+
+    job: Job
+    start: int = 1
+    end: int = 1
+    step: int = 1
+    max_concurrent: int | None = None
+
+    @property
+    def range_str(self) -> str:
+        """Format as scheduler range string."""
+        s = f"{self.start}-{self.end}"
+        if self.step != 1:
+            s += f":{self.step}"
+        if self.max_concurrent:
+            s += f"%{self.max_concurrent}"
+        return s
+
+    @property
+    def indices(self) -> Iterator[int]:
+        """Iterate over array indices."""
+        return iter(range(self.start, self.end + 1, self.step))
+
+    @property
+    def count(self) -> int:
+        """Number of array tasks."""
+        return len(range(self.start, self.end + 1, self.step))
+
+    def submit(self, scheduler: BaseScheduler | None = None) -> ArrayJobResult:
+        """Submit the array job."""
+        from hpc_runner.schedulers import get_scheduler
+
+        if scheduler is None:
+            scheduler = get_scheduler()
+        return scheduler.submit_array(self)

hpc_runner/core/job_info.py

@@ -0,0 +1,104 @@
+"""Job information types for TUI display."""
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from .result import JobStatus
+
+
+@dataclass
+class JobInfo:
+    """Unified job information for TUI display.
+
+    This dataclass provides a scheduler-agnostic view of job information
+    suitable for display in the monitor TUI. All fields except job_id,
+    name, user, and status are optional to handle varying levels of
+    information availability across schedulers.
+    """
+
+    job_id: str
+    name: str
+    user: str
+    status: JobStatus
+
+    # Queue/partition info
+    queue: str | None = None
+
+    # Timing information
+    submit_time: datetime | None = None
+    start_time: datetime | None = None
+    end_time: datetime | None = None
+    runtime: timedelta | None = None
+
+    # Resource requests/usage
+    cpu: int | None = None
+    memory: str | None = None  # e.g., "16G", "4096M"
+    gpu: int | None = None
+
+    # Completion info (None for active jobs)
+    exit_code: int | None = None
+
+    # Output file paths
+    stdout_path: Path | None = None
+    stderr_path: Path | None = None
+
+    # Extended info
+    node: str | None = None
+    dependencies: list[str] | None = None
+    array_task_id: int | None = None
+
+    @property
+    def is_active(self) -> bool:
+        """Check if job is still active (not yet completed)."""
+        return self.status in (
+            JobStatus.PENDING,
+            JobStatus.RUNNING,
+            JobStatus.UNKNOWN,
+        )
+
+    @property
+    def is_complete(self) -> bool:
+        """Check if job has finished (success or failure)."""
+        return self.status in (
+            JobStatus.COMPLETED,
+            JobStatus.FAILED,
+            JobStatus.CANCELLED,
+            JobStatus.TIMEOUT,
+        )
+
+    @property
+    def runtime_display(self) -> str:
+        """Format runtime for display (e.g., '2h 15m')."""
+        if self.runtime is None:
+            return "—"
+
+        total_seconds = int(self.runtime.total_seconds())
+        if total_seconds < 60:
+            return f"{total_seconds}s"
+
+        minutes = total_seconds // 60
+        if minutes < 60:
+            return f"{minutes}m"
+
+        hours = minutes // 60
+        remaining_minutes = minutes % 60
+        if hours < 24:
+            return f"{hours}h {remaining_minutes}m"
+
+        days = hours // 24
+        remaining_hours = hours % 24
+        return f"{days}d {remaining_hours}h"
+
+    @property
+    def resources_display(self) -> str:
+        """Format resources for display (e.g., '4/16G')."""
+        parts = []
+        if self.cpu is not None:
+            parts.append(str(self.cpu))
+        if self.memory is not None:
+            parts.append(self.memory)
+        if self.gpu is not None:
+            parts.append(f"{self.gpu}GPU")
+
+        return "/".join(parts) if parts else "—"

hpc_runner/core/resources.py

@@ -0,0 +1,49 @@
+"""Resource abstraction for job resource requests."""
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Resource:
+    """A scheduler resource request.
+
+    Examples:
+        Resource("gpu", 2)              # 2 GPUs
+        Resource("xilinx", 1)           # 1 Xilinx license
+        Resource("mem", "16G")          # Memory
+    """
+
+    name: str
+    value: int | str
+
+    # Scheduler-specific mappings (populated by scheduler)
+    _sge_resource: str | None = field(default=None, repr=False)
+    _slurm_gres: str | None = field(default=None, repr=False)
+
+
+@dataclass
+class ResourceSet:
+    """Collection of resources for a job."""
+
+    resources: list[Resource] = field(default_factory=list)
+
+    def add(self, name: str, value: int | str) -> "ResourceSet":
+        """Add a resource to the set."""
+        self.resources.append(Resource(name, value))
+        return self
+
+    def get(self, name: str) -> Resource | None:
+        """Get a resource by name."""
+        for r in self.resources:
+            if r.name == name:
+                return r
+        return None
+
+    def __iter__(self):
+        return iter(self.resources)
+
+    def __len__(self) -> int:
+        return len(self.resources)
+
+    def __bool__(self) -> bool:
+        return bool(self.resources)

hpc_runner/core/result.py

@@ -0,0 +1,161 @@
+"""Job result and status types."""
+
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from hpc_runner.core.job import Job
+    from hpc_runner.core.job_array import JobArray
+    from hpc_runner.schedulers.base import BaseScheduler
+
+
+class JobStatus(Enum):
+    """Unified job status across schedulers."""
+
+    PENDING = auto()  # Waiting in queue
+    RUNNING = auto()  # Currently executing
+    COMPLETED = auto()  # Finished successfully
+    FAILED = auto()  # Finished with error
+    CANCELLED = auto()  # User cancelled
+    TIMEOUT = auto()  # Hit time limit
+    UNKNOWN = auto()  # Cannot determine
+
+
+@dataclass
+class JobResult:
+    """Result of a submitted job.
+
+    Provides methods to query status, wait for completion,
+    and access output.
+    """
+
+    job_id: str
+    scheduler: "BaseScheduler"
+    job: "Job"
+
+    _cached_status: JobStatus | None = field(default=None, repr=False)
+    _exit_code: int | None = field(default=None, repr=False)  # For interactive jobs
+
+    @property
+    def status(self) -> JobStatus:
+        """Get current job status (queries scheduler)."""
+        return self.scheduler.get_status(self.job_id)
+
+    @property
+    def is_complete(self) -> bool:
+        """Check if job has finished (success or failure)."""
+        return self.status in (
+            JobStatus.COMPLETED,
+            JobStatus.FAILED,
+            JobStatus.CANCELLED,
+            JobStatus.TIMEOUT,
+        )
+
+    @property
+    def returncode(self) -> int | None:
+        """Get exit code (None if not complete)."""
+        # For interactive jobs, use cached exit code
+        if self._exit_code is not None:
+            return self._exit_code
+        if not self.is_complete:
+            return None
+        return self.scheduler.get_exit_code(self.job_id)
+
+    def wait(self, poll_interval: float = 5.0, timeout: float | None = None) -> JobStatus:
+        """Block until job completes.
+
+        Args:
+            poll_interval: Seconds between status checks
+            timeout: Max seconds to wait (None = forever)
+
+        Returns:
+            Final job status
+        """
+        import time
+
+        start = time.time()
+        while not self.is_complete:
+            if timeout and (time.time() - start) > timeout:
+                raise TimeoutError(f"Job {self.job_id} did not complete within {timeout}s")
+            time.sleep(poll_interval)
+        return self.status
+
+    def cancel(self) -> bool:
+        """Cancel the job."""
+        return self.scheduler.cancel(self.job_id)
+
+    def stdout_path(self) -> Path | None:
+        """Get path to stdout file."""
+        return self.scheduler.get_output_path(self.job_id, "stdout")
+
+    def stderr_path(self) -> Path | None:
+        """Get path to stderr file."""
+        return self.scheduler.get_output_path(self.job_id, "stderr")
+
+    def read_stdout(self, tail: int | None = None) -> str:
+        """Read stdout content."""
+        path = self.stdout_path()
+        if not path or not path.exists():
+            return ""
+        content = path.read_text()
+        if tail:
+            lines = content.splitlines()
+            content = "\n".join(lines[-tail:])
+        return content
+
+    def read_stderr(self, tail: int | None = None) -> str:
+        """Read stderr content."""
+        path = self.stderr_path()
+        if not path or not path.exists():
+            return ""
+        content = path.read_text()
+        if tail:
+            lines = content.splitlines()
+            content = "\n".join(lines[-tail:])
+        return content
+
+
+@dataclass
+class ArrayJobResult:
+    """Result of a submitted array job."""
+
+    base_job_id: str
+    scheduler: "BaseScheduler"
+    array: "JobArray"
+
+    def task_id(self, index: int) -> str:
+        """Get job ID for specific array task."""
+        return f"{self.base_job_id}.{index}"
+
+    def task_status(self, index: int) -> JobStatus:
+        """Get status of specific array task."""
+        return self.scheduler.get_status(self.task_id(index))
+
+    def wait(self, poll_interval: float = 5.0) -> dict[int, JobStatus]:
+        """Wait for all array tasks to complete."""
+        import time
+
+        results: dict[int, JobStatus] = {}
+        pending = set(self.array.indices)
+
+        while pending:
+            for idx in list(pending):
+                status = self.task_status(idx)
+                if status in (
+                    JobStatus.COMPLETED,
+                    JobStatus.FAILED,
+                    JobStatus.CANCELLED,
+                    JobStatus.TIMEOUT,
+                ):
+                    results[idx] = status
+                    pending.remove(idx)
+            if pending:
+                time.sleep(poll_interval)
+
+        return results
+
+    def cancel(self) -> bool:
+        """Cancel all array tasks."""
+        return self.scheduler.cancel(self.base_job_id)