hpc-runner 0.2.0__py3-none-any.whl

hpc_runner/schedulers/local/templates/job.sh.j2

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Generated by hpc-tools (local scheduler)
+
+# Exit on error
+set -e
+
+{% if job.modules_path %}
+# Additional module paths (simulated for local)
+{% for path in job.modules_path %}
+# module use {{ path }}
+{% endfor %}
+{% endif %}
+
+{% if job.modules %}
+# Modules (simulated for local - not actually loaded)
+{% for mod in job.modules %}
+# module load {{ mod }}
+{% endfor %}
+{% endif %}
+
+{% if job.workdir %}
+# Change to working directory
+cd {{ job.workdir }}
+{% endif %}
+
+# Execute command
+{{ job.command }}
+exit $?

hpc_runner/schedulers/sge/__init__.py

@@ -0,0 +1,5 @@
+"""SGE (Sun Grid Engine) scheduler implementation."""
+
+from hpc_runner.schedulers.sge.scheduler import SGEScheduler
+
+__all__ = ["SGEScheduler"]

hpc_runner/schedulers/sge/args.py

@@ -0,0 +1,232 @@
+"""SGE-specific argument renderers.
+
+Each class knows how to render a single job attribute to SGE syntax,
+both as a script directive (#$ ...) and as command-line arguments.
+"""
+
+from hpc_runner.core.descriptors import SchedulerArg
+
+
+class SGEArg(SchedulerArg):
+    """Base class for SGE arguments.
+
+    SGE uses:
+    - Directives: #$ -flag value
+    - CLI args: -flag value
+    """
+
+    def to_args(self, value) -> list[str]:
+        if value is None:
+            return []
+        return [f"-{self.flag}", str(value)]
+
+    def to_directive(self, value) -> str | None:
+        if value is None:
+            return None
+        return f"#$ -{self.flag} {value}"
+
+
+# =============================================================================
+# Simple Flag Arguments
+# =============================================================================
+
+
+class SGEJobNameArg(SGEArg):
+    """Job name: -N name"""
+
+    def __init__(self):
+        super().__init__("N", doc="Job name")
+
+
+class SGEQueueArg(SGEArg):
+    """Queue selection: -q queue_name"""
+
+    def __init__(self):
+        super().__init__("q", doc="Queue/partition name")
+
+
+class SGEOutputArg(SGEArg):
+    """Stdout path: -o path"""
+
+    def __init__(self):
+        super().__init__("o", doc="Stdout file path")
+
+
+class SGEErrorArg(SGEArg):
+    """Stderr path: -e path"""
+
+    def __init__(self):
+        super().__init__("e", doc="Stderr file path")
+
+
+class SGEPriorityArg(SGEArg):
+    """Job priority: -p priority"""
+
+    def __init__(self):
+        super().__init__("p", doc="Job priority (-1023 to 1024)")
+
+
+class SGEShellArg(SGEArg):
+    """Shell selection: -S /path/to/shell"""
+
+    def __init__(self):
+        super().__init__("S", doc="Shell path")
+
+
+# =============================================================================
+# Boolean Flag Arguments (no value, just presence)
+# =============================================================================
+
+
+class SGECwdArg(SchedulerArg[bool]):
+    """Use current working directory: -cwd"""
+
+    def __init__(self):
+        super().__init__("cwd", doc="Execute in current working directory")
+
+    def to_args(self, value: bool | None) -> list[str]:
+        return ["-cwd"] if value else []
+
+    def to_directive(self, value: bool | None) -> str | None:
+        return "#$ -cwd" if value else None
+
+
+class SGEInheritEnvArg(SchedulerArg[bool]):
+    """Inherit environment: -V"""
+
+    def __init__(self):
+        super().__init__("V", doc="Inherit environment variables")
+
+    def to_args(self, value: bool | None) -> list[str]:
+        return ["-V"] if value else []
+
+    def to_directive(self, value: bool | None) -> str | None:
+        return "#$ -V" if value else None
+
+
+class SGEMergeOutputArg(SchedulerArg[bool]):
+    """Merge stdout and stderr: -j y"""
+
+    def __init__(self):
+        super().__init__("j", doc="Join stdout and stderr")
+
+    def to_args(self, value: bool | None) -> list[str]:
+        return ["-j", "y"] if value else []
+
+    def to_directive(self, value: bool | None) -> str | None:
+        return "#$ -j y" if value else None
+
+
+# =============================================================================
+# Resource Arguments (configurable resource names)
+# =============================================================================
+
+
+class SGECpuArg(SchedulerArg[int]):
+    """Parallel environment slots: -pe <pe_name> <slots>
+
+    The PE name is configurable per-cluster (e.g., 'smp', 'mpi', 'orte').
+    """
+
+    def __init__(self, pe_name: str = "smp"):
+        super().__init__("pe", doc=f"Parallel environment ({pe_name})")
+        self.pe_name = pe_name
+
+    def to_args(self, value: int | None) -> list[str]:
+        if value is None:
+            return []
+        return ["-pe", self.pe_name, str(value)]
+
+    def to_directive(self, value: int | None) -> str | None:
+        if value is None:
+            return None
+        return f"#$ -pe {self.pe_name} {value}"
+
+
+class SGEMemArg(SchedulerArg[str]):
+    """Memory request: -l <resource>=<value>
+
+    The resource name is configurable (e.g., 'mem_free', 'h_vmem', 'mem').
+    """
+
+    def __init__(self, resource_name: str = "mem_free"):
+        super().__init__("l", doc=f"Memory ({resource_name})")
+        self.resource_name = resource_name
+
+    def to_args(self, value: str | None) -> list[str]:
+        if value is None:
+            return []
+        return ["-l", f"{self.resource_name}={value}"]
+
+    def to_directive(self, value: str | None) -> str | None:
+        if value is None:
+            return None
+        return f"#$ -l {self.resource_name}={value}"
+
+
+class SGETimeArg(SchedulerArg[str]):
+    """Time limit: -l <resource>=<HH:MM:SS>
+
+    The resource name is configurable (e.g., 'h_rt', 's_rt').
+    """
+
+    def __init__(self, resource_name: str = "h_rt"):
+        super().__init__("l", doc=f"Time limit ({resource_name})")
+        self.resource_name = resource_name
+
+    def to_args(self, value: str | None) -> list[str]:
+        if value is None:
+            return []
+        return ["-l", f"{self.resource_name}={value}"]
+
+    def to_directive(self, value: str | None) -> str | None:
+        if value is None:
+            return None
+        return f"#$ -l {self.resource_name}={value}"
+
+
+# =============================================================================
+# Array Job Arguments
+# =============================================================================
+
+
+class SGEArrayArg(SchedulerArg[str]):
+    """Array job range: -t range
+
+    Range formats: 1-100, 1-100:10, 1,2,3,4
+    """
+
+    def __init__(self):
+        super().__init__("t", doc="Array job range")
+
+    def to_args(self, value: str | None) -> list[str]:
+        if value is None:
+            return []
+        return ["-t", value]
+
+    def to_directive(self, value: str | None) -> str | None:
+        if value is None:
+            return None
+        return f"#$ -t {value}"
+
+
+# =============================================================================
+# Dependency Arguments
+# =============================================================================
+
+
+class SGEHoldArg(SchedulerArg[str]):
+    """Job dependency: -hold_jid job_id[,job_id,...]"""
+
+    def __init__(self):
+        super().__init__("hold_jid", doc="Hold until jobs complete")
+
+    def to_args(self, value: str | None) -> list[str]:
+        if value is None:
+            return []
+        return ["-hold_jid", value]
+
+    def to_directive(self, value: str | None) -> str | None:
+        if value is None:
+            return None
+        return f"#$ -hold_jid {value}"

hpc_runner/schedulers/sge/parser.py

@@ -0,0 +1,287 @@
+"""SGE output parsing utilities."""
+
+import re
+from datetime import datetime
+import xml.etree.ElementTree as ET
+from typing import Any
+
+from hpc_runner.core.result import JobStatus
+
+
+def parse_qstat_xml(xml_output: str) -> dict[str, Any]:
+    """Parse qstat -xml output.
+
+    Returns dict with job_id -> job_info mappings.
+    """
+    jobs: dict[str, Any] = {}
+
+    try:
+        root = ET.fromstring(xml_output)
+        _strip_namespaces(root)
+
+        # Parse queue_info (running jobs)
+        for job_list in root.findall(".//job_list"):
+            job_info = _parse_job_element(job_list)
+            if job_info:
+                jobs[job_info["job_id"]] = job_info
+
+        # Parse job_info (pending jobs)
+        for job_list in root.findall(".//job_info/job_list"):
+            job_info = _parse_job_element(job_list)
+            if job_info:
+                jobs[job_info["job_id"]] = job_info
+
+    except ET.ParseError:
+        pass
+
+    return jobs
+
+
+def _parse_job_element(elem: ET.Element) -> dict[str, Any] | None:
+    """Parse a single job_list element.
+
+    SGE XML elements include:
+    - JB_job_number: Job ID
+    - JB_name: Job name
+    - JB_owner: Username
+    - state: Job state (r, qw, hqw, etc.)
+    - queue_name: Queue@host (for running jobs)
+    - hard_req_queue: Requested queue (for pending jobs)
+    - slots: Number of slots/CPUs
+    - JB_submission_time: Submission timestamp (epoch)
+    - JAT_start_time: Start timestamp (epoch, running jobs only)
+    - tasks: Array task ID (for array jobs)
+    """
+    job_id_elem = elem.find("JB_job_number")
+    if job_id_elem is None or job_id_elem.text is None:
+        return None
+
+    job_info: dict[str, Any] = {
+        "job_id": job_id_elem.text,
+    }
+
+    # Job name
+    name_elem = elem.find("JB_name")
+    if name_elem is not None and name_elem.text:
+        job_info["name"] = name_elem.text
+
+    # Owner/user
+    owner_elem = elem.find("JB_owner")
+    if owner_elem is not None and owner_elem.text:
+        job_info["user"] = owner_elem.text
+
+    # State
+    state_elem = elem.find("state")
+    if state_elem is not None and state_elem.text:
+        job_info["state"] = state_elem.text
+
+    # Queue - running jobs have queue_name, pending may have hard_req_queue
+    queue_elem = elem.find("queue_name")
+    if queue_elem is not None and queue_elem.text:
+        # Format is usually "queue@host", extract queue and host separately
+        queue_full = queue_elem.text
+        if "@" in queue_full:
+            queue_name, host = queue_full.split("@", 1)
+            job_info["queue"] = queue_name
+            job_info["node"] = host
+        else:
+            job_info["queue"] = queue_full
+    else:
+        # Check for requested queue (pending jobs)
+        hard_queue = elem.find("hard_req_queue")
+        if hard_queue is not None and hard_queue.text:
+            job_info["queue"] = hard_queue.text
+
+    # Slots (CPU count)
+    slots_elem = elem.find("slots")
+    if slots_elem is not None and slots_elem.text:
+        job_info["slots"] = int(slots_elem.text)
+
+    # Submission time (epoch seconds)
+    submit_text = elem.findtext(".//JB_submission_time")
+    if submit_text:
+        try:
+            job_info["submit_time"] = int(submit_text)
+        except ValueError:
+            pass
+
+    # Start time (epoch seconds, only for running jobs)
+    start_text = elem.findtext(".//JAT_start_time")
+    if start_text:
+        start_epoch = _parse_sge_timestamp(start_text)
+        if start_epoch is not None:
+            job_info["start_time"] = start_epoch
+
+    # Array task ID
+    tasks_elem = elem.find("tasks")
+    if tasks_elem is not None and tasks_elem.text:
+        job_info["array_task_id"] = tasks_elem.text
+
+    return job_info
+
+
+def _strip_namespaces(root: ET.Element) -> None:
+    """Strip XML namespaces so ElementTree finds simple tag names."""
+    for elem in root.iter():
+        if isinstance(elem.tag, str) and "}" in elem.tag:
+            elem.tag = elem.tag.split("}", 1)[1]
+
+
+def parse_qstat_plain(output: str) -> dict[str, Any]:
+    """Parse plain qstat output.
+
+    Format:
+    job-ID  prior   name       user         state submit/start at     queue                          slots ja-task-ID
+    --------------------------------------------------------------------------------
+    12345   0.55500 myjob      user         r     01/01/2024 10:00:00 all.q@node1                    1
+    """
+    jobs: dict[str, Any] = {}
+
+    lines = output.strip().split("\n")
+
+    # Skip header lines
+    data_started = False
+    for line in lines:
+        if line.startswith("-"):
+            data_started = True
+            continue
+        if not data_started:
+            continue
+
+        parts = line.split()
+        if len(parts) >= 5:
+            job_id = parts[0]
+            jobs[job_id] = {
+                "job_id": job_id,
+                "priority": parts[1],
+                "name": parts[2],
+                "user": parts[3],
+                "state": parts[4],
+            }
+
+            # Parse submit/start time (MM/DD/YYYY HH:MM:SS)
+            if len(parts) >= 7:
+                timestamp = _parse_qstat_datetime(parts[5], parts[6])
+                if timestamp is not None:
+                    if "r" in parts[4]:
+                        jobs[job_id]["start_time"] = timestamp
+                    else:
+                        jobs[job_id]["submit_time"] = timestamp
+
+            # Parse queue if present
+            if len(parts) >= 8:
+                jobs[job_id]["queue"] = parts[7]
+
+            # Parse slots if present
+            if len(parts) >= 9:
+                try:
+                    jobs[job_id]["slots"] = int(parts[8])
+                except ValueError:
+                    pass
+
+    return jobs
+
+
+def _parse_qstat_datetime(date_part: str, time_part: str) -> int | None:
+    """Parse qstat date/time into epoch seconds."""
+    try:
+        dt = datetime.strptime(f"{date_part} {time_part}", "%m/%d/%Y %H:%M:%S")
+    except ValueError:
+        return None
+    return int(dt.timestamp())
+
+
+def _parse_sge_timestamp(value: str) -> int | None:
+    """Parse SGE timestamps that may be epoch seconds or ISO 8601."""
+    if value.isdigit():
+        try:
+            return int(value)
+        except ValueError:
+            return None
+    try:
+        return int(datetime.strptime(value, "%Y-%m-%dT%H:%M:%S").timestamp())
+    except ValueError:
+        return None
+
+
+def parse_qacct_output(output: str) -> dict[str, Any]:
+    """Parse qacct output for job accounting info.
+
+    Format:
+    ==============================================================
+    qname        all.q
+    hostname     node1
+    group        users
+    owner        user
+    jobname      myjob
+    jobnumber    12345
+    ...
+    exit_status  0
+    """
+    info: dict[str, Any] = {}
+
+    for line in output.strip().split("\n"):
+        if line.startswith("="):
+            continue
+
+        parts = line.split(None, 1)
+        if len(parts) == 2:
+            key, value = parts
+            info[key] = value.strip()
+
+    return info
+
+
+def state_to_status(state: str) -> JobStatus:
+    """Convert SGE state code to JobStatus.
+
+    SGE states:
+    - qw: pending (queued, waiting)
+    - hqw: hold (on hold)
+    - r: running
+    - t: transferring
+    - Rr, Rt: restarted
+    - s, ts: suspended
+    - S, tS: queue suspended
+    - T, tT: threshold
+    - Eqw: error (waiting)
+    - dr: deleting (running)
+    - dt: deleting (transferring)
+    """
+    state = state.lower()
+
+    # Deleting or error states take precedence over other flags.
+    if "d" in state:
+        return JobStatus.CANCELLED
+    if "e" in state:
+        return JobStatus.FAILED
+
+    # Running or transferring states.
+    if "r" in state or "t" in state:
+        return JobStatus.RUNNING
+
+    # Queued, held, or suspended states.
+    if "q" in state or "h" in state or "s" in state:
+        return JobStatus.PENDING
+
+    return JobStatus.UNKNOWN
+
+
+def parse_qsub_output(output: str) -> str | None:
+    """Parse qsub output to extract job ID.
+
+    Expected format:
+    Your job 12345 ("jobname") has been submitted
+    Your job-array 12345.1-10:1 ("jobname") has been submitted
+    """
+    # Standard job
+    match = re.search(r"Your job (\d+)", output)
+    if match:
+        return match.group(1)
+
+    # Array job
+    match = re.search(r"Your job-array (\d+)", output)
+    if match:
+        return match.group(1)
+
+    return None