hpc-runner 0.1.1__py3-none-any.whl → 0.2.1__py3-none-any.whl

hpc_runner/schedulers/sge/args.py

@@ -1,165 +1,232 @@
-"""SGE-specific argument descriptors."""
+"""SGE-specific argument renderers.
+
+Each class knows how to render a single job attribute to SGE syntax,
+both as a script directive (#$ ...) and as command-line arguments.
+"""
 
 from hpc_runner.core.descriptors import SchedulerArg
 
 
 class SGEArg(SchedulerArg):
-    """Base SGE argument.
+    """Base class for SGE arguments.
 
-    SGE uses #$ -flag value format for directives.
+    SGE uses:
+    - Directives: #$ -flag value
+    - CLI args: -flag value
     """
 
     def to_args(self, value) -> list[str]:
         if value is None:
             return []
-        return [f"-{self.flag}", str(self.converter(value))]
+        return [f"-{self.flag}", str(value)]
 
     def to_directive(self, value) -> str | None:
         if value is None:
             return None
-        return f"#$ -{self.flag} {self.converter(value)}"
+        return f"#$ -{self.flag} {value}"
+
+
+# =============================================================================
+# Simple Flag Arguments
+# =============================================================================
+
+
+class SGEJobNameArg(SGEArg):
+    """Job name: -N name"""
+
+    def __init__(self):
+        super().__init__("N", doc="Job name")
+
+
+class SGEQueueArg(SGEArg):
+    """Queue selection: -q queue_name"""
+
+    def __init__(self):
+        super().__init__("q", doc="Queue/partition name")
+
+
+class SGEOutputArg(SGEArg):
+    """Stdout path: -o path"""
+
+    def __init__(self):
+        super().__init__("o", doc="Stdout file path")
+
+
+class SGEErrorArg(SGEArg):
+    """Stderr path: -e path"""
+
+    def __init__(self):
+        super().__init__("e", doc="Stderr file path")
 
 
-class SGECpuArg(SGEArg):
-    """CPU/slots argument using parallel environment.
+class SGEPriorityArg(SGEArg):
+    """Job priority: -p priority"""
 
-    Note: The PE name is configurable via config.
+    def __init__(self):
+        super().__init__("p", doc="Job priority (-1023 to 1024)")
+
+
+class SGEShellArg(SGEArg):
+    """Shell selection: -S /path/to/shell"""
+
+    def __init__(self):
+        super().__init__("S", doc="Shell path")
+
+
+# =============================================================================
+# Boolean Flag Arguments (no value, just presence)
+# =============================================================================
+
+
+class SGECwdArg(SchedulerArg[bool]):
+    """Use current working directory: -cwd"""
+
+    def __init__(self):
+        super().__init__("cwd", doc="Execute in current working directory")
+
+    def to_args(self, value: bool | None) -> list[str]:
+        return ["-cwd"] if value else []
+
+    def to_directive(self, value: bool | None) -> str | None:
+        return "#$ -cwd" if value else None
+
+
+class SGEInheritEnvArg(SchedulerArg[bool]):
+    """Inherit environment: -V"""
+
+    def __init__(self):
+        super().__init__("V", doc="Inherit environment variables")
+
+    def to_args(self, value: bool | None) -> list[str]:
+        return ["-V"] if value else []
+
+    def to_directive(self, value: bool | None) -> str | None:
+        return "#$ -V" if value else None
+
+
+class SGEMergeOutputArg(SchedulerArg[bool]):
+    """Merge stdout and stderr: -j y"""
+
+    def __init__(self):
+        super().__init__("j", doc="Join stdout and stderr")
+
+    def to_args(self, value: bool | None) -> list[str]:
+        return ["-j", "y"] if value else []
+
+    def to_directive(self, value: bool | None) -> str | None:
+        return "#$ -j y" if value else None
+
+
+# =============================================================================
+# Resource Arguments (configurable resource names)
+# =============================================================================
+
+
+class SGECpuArg(SchedulerArg[int]):
+    """Parallel environment slots: -pe <pe_name> <slots>
+
+    The PE name is configurable per-cluster (e.g., 'smp', 'mpi', 'orte').
     """
 
     def __init__(self, pe_name: str = "smp"):
-        super().__init__("pe", converter=lambda v: f"{pe_name} {v}", doc="Parallel environment")
+        super().__init__("pe", doc=f"Parallel environment ({pe_name})")
         self.pe_name = pe_name
 
-    def to_args(self, value, pe_name: str | None = None) -> list[str]:
+    def to_args(self, value: int | None) -> list[str]:
         if value is None:
             return []
-        pe = pe_name or self.pe_name
-        return ["-pe", f"{pe} {value}"]
+        return ["-pe", self.pe_name, str(value)]
 
-    def to_directive(self, value, pe_name: str | None = None) -> str | None:
+    def to_directive(self, value: int | None) -> str | None:
         if value is None:
             return None
-        pe = pe_name or self.pe_name
-        return f"#$ -pe {pe} {value}"
+        return f"#$ -pe {self.pe_name} {value}"
 
 
-class SGEMemArg(SGEArg):
-    """Memory argument.
+class SGEMemArg(SchedulerArg[str]):
+    """Memory request: -l <resource>=<value>
 
-    Uses -l resource=value format. Resource name is configurable.
+    The resource name is configurable (e.g., 'mem_free', 'h_vmem', 'mem').
     """
 
     def __init__(self, resource_name: str = "mem_free"):
-        super().__init__("l", doc="Memory requirement")
+        super().__init__("l", doc=f"Memory ({resource_name})")
         self.resource_name = resource_name
 
-    def to_args(self, value, resource_name: str | None = None) -> list[str]:
+    def to_args(self, value: str | None) -> list[str]:
         if value is None:
             return []
-        res = resource_name or self.resource_name
-        return ["-l", f"{res}={value}"]
+        return ["-l", f"{self.resource_name}={value}"]
 
-    def to_directive(self, value, resource_name: str | None = None) -> str | None:
+    def to_directive(self, value: str | None) -> str | None:
         if value is None:
             return None
-        res = resource_name or self.resource_name
-        return f"#$ -l {res}={value}"
+        return f"#$ -l {self.resource_name}={value}"
 
 
-class SGETimeArg(SGEArg):
-    """Time limit argument.
+class SGETimeArg(SchedulerArg[str]):
+    """Time limit: -l <resource>=<HH:MM:SS>
 
-    Uses -l h_rt=HH:MM:SS format. Resource name is configurable.
+    The resource name is configurable (e.g., 'h_rt', 's_rt').
     """
 
     def __init__(self, resource_name: str = "h_rt"):
-        super().__init__("l", doc="Hard runtime limit")
+        super().__init__("l", doc=f"Time limit ({resource_name})")
         self.resource_name = resource_name
 
-    def to_args(self, value, resource_name: str | None = None) -> list[str]:
+    def to_args(self, value: str | None) -> list[str]:
         if value is None:
             return []
-        res = resource_name or self.resource_name
-        return ["-l", f"{res}={value}"]
+        return ["-l", f"{self.resource_name}={value}"]
 
-    def to_directive(self, value, resource_name: str | None = None) -> str | None:
+    def to_directive(self, value: str | None) -> str | None:
         if value is None:
             return None
-        res = resource_name or self.resource_name
-        return f"#$ -l {res}={value}"
-
-
-class SGEQueueArg(SGEArg):
-    """Queue argument."""
-
-    def __init__(self):
-        super().__init__("q", doc="Queue name")
-
-
-class SGEJobNameArg(SGEArg):
-    """Job name argument."""
-
-    def __init__(self):
-        super().__init__("N", doc="Job name")
-
-
-class SGEOutputArg(SGEArg):
-    """Stdout path argument."""
-
-    def __init__(self):
-        super().__init__("o", doc="Stdout file path")
+        return f"#$ -l {self.resource_name}={value}"
 
 
-class SGEErrorArg(SGEArg):
-    """Stderr path argument."""
+# =============================================================================
+# Array Job Arguments
+# =============================================================================
 
-    def __init__(self):
-        super().__init__("e", doc="Stderr file path")
 
+class SGEArrayArg(SchedulerArg[str]):
+    """Array job range: -t range
 
-class SGEArrayArg(SGEArg):
-    """Array job argument."""
+    Range formats: 1-100, 1-100:10, 1,2,3,4
+    """
 
     def __init__(self):
-        super().__init__("t", doc="Array job range (e.g., 1-100, 1-100:10)")
+        super().__init__("t", doc="Array job range")
 
+    def to_args(self, value: str | None) -> list[str]:
+        if value is None:
+            return []
+        return ["-t", value]
 
-class SGEJoinOutputArg(SGEArg):
-    """Join stdout and stderr."""
-
-    def __init__(self):
-        super().__init__("j", doc="Join stdout and stderr")
+    def to_directive(self, value: str | None) -> str | None:
+        if value is None:
+            return None
+        return f"#$ -t {value}"
 
-    def to_args(self, value) -> list[str]:
-        if value:
-            return ["-j", "y"]
-        return []
 
-    def to_directive(self, value) -> str | None:
-        if value:
-            return "#$ -j y"
-        return None
+# =============================================================================
+# Dependency Arguments
+# =============================================================================
 
 
-class SGECwdArg(SGEArg):
-    """Use current working directory."""
+class SGEHoldArg(SchedulerArg[str]):
+    """Job dependency: -hold_jid job_id[,job_id,...]"""
 
     def __init__(self):
-        super().__init__("cwd", doc="Use current working directory")
+        super().__init__("hold_jid", doc="Hold until jobs complete")
 
-    def to_args(self, value) -> list[str]:
-        if value:
-            return ["-cwd"]
-        return []
-
-    def to_directive(self, value) -> str | None:
-        if value:
-            return "#$ -cwd"
-        return None
-
-
-class SGEShellArg(SGEArg):
-    """Shell to use for the job."""
+    def to_args(self, value: str | None) -> list[str]:
+        if value is None:
+            return []
+        return ["-hold_jid", value]
 
-    def __init__(self):
-        super().__init__("S", doc="Shell path")
+    def to_directive(self, value: str | None) -> str | None:
+        if value is None:
+            return None
+        return f"#$ -hold_jid {value}"

hpc_runner/schedulers/sge/parser.py

@@ -1,6 +1,7 @@
 """SGE output parsing utilities."""
 
 import re
+from datetime import datetime
 import xml.etree.ElementTree as ET
 from typing import Any
 
@@ -16,6 +17,7 @@ def parse_qstat_xml(xml_output: str) -> dict[str, Any]:
 
     try:
         root = ET.fromstring(xml_output)
+        _strip_namespaces(root)
 
         # Parse queue_info (running jobs)
         for job_list in root.findall(".//job_list"):
@@ -36,7 +38,20 @@ def parse_qstat_xml(xml_output: str) -> dict[str, Any]:
 
 
 def _parse_job_element(elem: ET.Element) -> dict[str, Any] | None:
-    """Parse a single job_list element."""
+    """Parse a single job_list element.
+
+    SGE XML elements include:
+    - JB_job_number: Job ID
+    - JB_name: Job name
+    - JB_owner: Username
+    - state: Job state (r, qw, hqw, etc.)
+    - queue_name: Queue@host (for running jobs)
+    - hard_req_queue: Requested queue (for pending jobs)
+    - slots: Number of slots/CPUs
+    - JB_submission_time: Submission timestamp (epoch)
+    - JAT_start_time: Start timestamp (epoch, running jobs only)
+    - tasks: Array task ID (for array jobs)
+    """
     job_id_elem = elem.find("JB_job_number")
     if job_id_elem is None or job_id_elem.text is None:
         return None
@@ -50,24 +65,68 @@ def _parse_job_element(elem: ET.Element) -> dict[str, Any] | None:
     if name_elem is not None and name_elem.text:
         job_info["name"] = name_elem.text
 
+    # Owner/user
+    owner_elem = elem.find("JB_owner")
+    if owner_elem is not None and owner_elem.text:
+        job_info["user"] = owner_elem.text
+
     # State
     state_elem = elem.find("state")
     if state_elem is not None and state_elem.text:
         job_info["state"] = state_elem.text
 
-    # Queue
+    # Queue - running jobs have queue_name, pending may have hard_req_queue
     queue_elem = elem.find("queue_name")
     if queue_elem is not None and queue_elem.text:
-        job_info["queue"] = queue_elem.text
-
-    # Slots
+        # Format is usually "queue@host", extract queue and host separately
+        queue_full = queue_elem.text
+        if "@" in queue_full:
+            queue_name, host = queue_full.split("@", 1)
+            job_info["queue"] = queue_name
+            job_info["node"] = host
+        else:
+            job_info["queue"] = queue_full
+    else:
+        # Check for requested queue (pending jobs)
+        hard_queue = elem.find("hard_req_queue")
+        if hard_queue is not None and hard_queue.text:
+            job_info["queue"] = hard_queue.text
+
+    # Slots (CPU count)
     slots_elem = elem.find("slots")
     if slots_elem is not None and slots_elem.text:
         job_info["slots"] = int(slots_elem.text)
 
+    # Submission time (epoch seconds)
+    submit_text = elem.findtext(".//JB_submission_time")
+    if submit_text:
+        try:
+            job_info["submit_time"] = int(submit_text)
+        except ValueError:
+            pass
+
+    # Start time (epoch seconds, only for running jobs)
+    start_text = elem.findtext(".//JAT_start_time")
+    if start_text:
+        start_epoch = _parse_sge_timestamp(start_text)
+        if start_epoch is not None:
+            job_info["start_time"] = start_epoch
+
+    # Array task ID
+    tasks_elem = elem.find("tasks")
+    if tasks_elem is not None and tasks_elem.text:
+        job_info["array_task_id"] = tasks_elem.text
+
     return job_info
 
 
+def _strip_namespaces(root: ET.Element) -> None:
+    """Strip XML namespaces so ElementTree finds simple tag names."""
+    for elem in root.iter():
+        if isinstance(elem.tag, str) and "}" in elem.tag:
+            elem.tag = elem.tag.split("}", 1)[1]
+
+
 def parse_qstat_plain(output: str) -> dict[str, Any]:
     """Parse plain qstat output.
 
@@ -100,6 +159,15 @@ def parse_qstat_plain(output: str) -> dict[str, Any]:
                 "state": parts[4],
             }
 
+            # Parse submit/start time (MM/DD/YYYY HH:MM:SS)
+            if len(parts) >= 7:
+                timestamp = _parse_qstat_datetime(parts[5], parts[6])
+                if timestamp is not None:
+                    if "r" in parts[4]:
+                        jobs[job_id]["start_time"] = timestamp
+                    else:
+                        jobs[job_id]["submit_time"] = timestamp
+
             # Parse queue if present
             if len(parts) >= 8:
                 jobs[job_id]["queue"] = parts[7]
@@ -114,6 +182,28 @@ def parse_qstat_plain(output: str) -> dict[str, Any]:
     return jobs
 
 
+def _parse_qstat_datetime(date_part: str, time_part: str) -> int | None:
+    """Parse qstat date/time into epoch seconds."""
+    try:
+        dt = datetime.strptime(f"{date_part} {time_part}", "%m/%d/%Y %H:%M:%S")
+    except ValueError:
+        return None
+    return int(dt.timestamp())
+
+
+def _parse_sge_timestamp(value: str) -> int | None:
+    """Parse SGE timestamps that may be epoch seconds or ISO 8601."""
+    if value.isdigit():
+        try:
+            return int(value)
+        except ValueError:
+            return None
+    try:
+        return int(datetime.strptime(value, "%Y-%m-%dT%H:%M:%S").timestamp())
+    except ValueError:
+        return None
+
+
 def parse_qacct_output(output: str) -> dict[str, Any]:
     """Parse qacct output for job accounting info.
 
@@ -160,16 +250,19 @@ def state_to_status(state: str) -> JobStatus:
     """
     state = state.lower()
 
-    if state in ("r", "t", "rr", "rt"):
+    # Deleting or error states take precedence over other flags.
+    if "d" in state:
+        return JobStatus.CANCELLED
+    if "e" in state:
+        return JobStatus.FAILED
+
+    # Running or transferring states.
+    if "r" in state or "t" in state:
         return JobStatus.RUNNING
-    elif state in ("qw", "hqw"):
+
+    # Queued, held, or suspended states.
+    if "q" in state or "h" in state or "s" in state:
         return JobStatus.PENDING
-    elif state in ("eqw",):
-        return JobStatus.FAILED
-    elif state in ("dr", "dt"):
-        return JobStatus.CANCELLED
-    elif state in ("s", "ts", "ss", "ts"):
-        return JobStatus.PENDING  # Suspended, treat as pending
 
     return JobStatus.UNKNOWN