hpc-runner 0.2.0__py3-none-any.whl

hpc_runner/workflow/pipeline.py

@@ -0,0 +1,180 @@
+"""Pipeline API for job workflows with dependencies."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from hpc_runner.core.job import Job
+from hpc_runner.workflow.dependency import DependencyType
+
+if TYPE_CHECKING:
+    from hpc_runner.core.result import JobResult
+    from hpc_runner.schedulers.base import BaseScheduler
+
+
+@dataclass
+class PipelineJob:
+    """A job within a pipeline."""
+
+    job: Job
+    name: str
+    depends_on: list[PipelineJob] = field(default_factory=list)
+    result: JobResult | None = None
+
+
+class Pipeline:
+    """Workflow pipeline with job dependencies.
+
+    Example:
+        with Pipeline("build-test") as p:
+            build = p.add("make build", name="build", cpu=4)
+            test = p.add("make test", name="test", depends_on=["build"])
+            package = p.add("make package", name="package", depends_on=["test"])
+
+        results = p.submit()
+        p.wait()
+    """
+
+    def __init__(self, name: str = "pipeline") -> None:
+        self.name = name
+        self.jobs: list[PipelineJob] = []
+        self._name_map: dict[str, PipelineJob] = {}
+        self._submitted = False
+
+    def add(
+        self,
+        command: str,
+        name: str | None = None,
+        depends_on: list[str | PipelineJob] | None = None,
+        **job_kwargs: Any,
+    ) -> PipelineJob:
+        """Add a job to the pipeline.
+
+        Args:
+            command: Command to execute
+            name: Job name (auto-generated if None)
+            depends_on: List of job names or PipelineJob objects
+            **job_kwargs: Additional Job parameters
+        """
+        if name is None:
+            name = f"step_{len(self.jobs) + 1}"
+
+        if name in self._name_map:
+            raise ValueError(f"Job name '{name}' already exists in pipeline")
+
+        job = Job(command=command, name=f"{self.name}_{name}", **job_kwargs)
+
+        dependencies: list[PipelineJob] = []
+        if depends_on:
+            for dep in depends_on:
+                if isinstance(dep, str):
+                    if dep not in self._name_map:
+                        raise ValueError(f"Unknown dependency: {dep}")
+                    dependencies.append(self._name_map[dep])
+                else:
+                    dependencies.append(dep)
+
+        pipeline_job = PipelineJob(job=job, name=name, depends_on=dependencies)
+        self.jobs.append(pipeline_job)
+        self._name_map[name] = pipeline_job
+
+        return pipeline_job
+
+    def submit(
+        self,
+        scheduler: BaseScheduler | None = None,
+        dependency_type: DependencyType = DependencyType.AFTEROK,
+    ) -> dict[str, JobResult]:
+        """Submit all jobs respecting dependencies.
+
+        Args:
+            scheduler: Scheduler to use (auto-detect if None)
+            dependency_type: Type of dependency to use
+
+        Returns:
+            Dict mapping job names to results
+        """
+        from hpc_runner.schedulers import get_scheduler
+
+        if self._submitted:
+            raise RuntimeError("Pipeline has already been submitted")
+
+        if scheduler is None:
+            scheduler = get_scheduler()
+
+        results: dict[str, JobResult] = {}
+
+        for pjob in self._topological_sort():
+            # Set up dependencies
+            if pjob.depends_on:
+                dep_results = [results[d.name] for d in pjob.depends_on]
+                pjob.job.dependencies = dep_results
+                pjob.job.dependency_type = str(dependency_type)
+
+            # Submit
+            result = scheduler.submit(pjob.job)
+            pjob.result = result
+            results[pjob.name] = result
+
+        self._submitted = True
+        return results
+
+    def _topological_sort(self) -> list[PipelineJob]:
+        """Sort jobs by dependency order (Kahn's algorithm)."""
+        # Build in-degree map
+        in_degree: dict[str, int] = {pj.name: 0 for pj in self.jobs}
+        for pj in self.jobs:
+            for dep in pj.depends_on:
+                in_degree[pj.name] += 1
+
+        # Find all jobs with no dependencies
+        queue = [pj for pj in self.jobs if in_degree[pj.name] == 0]
+        result: list[PipelineJob] = []
+
+        while queue:
+            pj = queue.pop(0)
+            result.append(pj)
+
+            # Reduce in-degree for dependent jobs
+            for other_pj in self.jobs:
+                if pj in other_pj.depends_on:
+                    in_degree[other_pj.name] -= 1
+                    if in_degree[other_pj.name] == 0:
+                        queue.append(other_pj)
+
+        if len(result) != len(self.jobs):
+            raise ValueError("Circular dependency detected in pipeline")
+
+        return result
+
+    def wait(self, poll_interval: float = 5.0) -> dict[str, JobResult]:
+        """Wait for all jobs to complete.
+
+        Returns:
+            Dict mapping job names to results
+        """
+        if not self._submitted:
+            raise RuntimeError("Pipeline has not been submitted")
+
+        for pjob in self.jobs:
+            if pjob.result:
+                pjob.result.wait(poll_interval=poll_interval)
+
+        return {pj.name: pj.result for pj in self.jobs if pj.result}
+
+    def get_job(self, name: str) -> PipelineJob | None:
+        """Get a job by name."""
+        return self._name_map.get(name)
+
+    def __enter__(self) -> Pipeline:
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        pass
+
+    def __len__(self) -> int:
+        return len(self.jobs)
+
+    def __iter__(self):
+        return iter(self.jobs)

hpc_runner-0.2.0.dist-info/METADATA

@@ -0,0 +1,285 @@
+Metadata-Version: 2.4
+Name: hpc-runner
+Version: 0.2.0
+Summary: Unified HPC job submission across multiple schedulers
+Project-URL: Homepage, https://github.com/sjalloq/hpc-runner
+Project-URL: Repository, https://github.com/sjalloq/hpc-runner
+Author: Shareef Jalloq
+License-Expression: MIT
+Keywords: cluster,hpc,job-submission,pbs,sge,slurm
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: System :: Clustering
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
+Requires-Dist: jinja2>=3.0
+Requires-Dist: rich-click>=1.7
+Requires-Dist: textual>=6.11
+Requires-Dist: tomli>=2.0; python_version < '3.11'
+Provides-Extra: all
+Requires-Dist: build; extra == 'all'
+Requires-Dist: hatch-vcs; extra == 'all'
+Requires-Dist: mypy; extra == 'all'
+Requires-Dist: pytest-cov; extra == 'all'
+Requires-Dist: pytest>=7.0; extra == 'all'
+Requires-Dist: ruff; extra == 'all'
+Requires-Dist: twine; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: hatch-vcs; extra == 'dev'
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# hpc-runner
+
+**Unified HPC job submission across multiple schedulers**
+
+Write your jobs once, run them on any cluster - SGE, Slurm, PBS, or locally for testing.
+
+## Features
+
+- **Unified CLI** - Same commands work across SGE, Slurm, PBS
+- **Python API** - Programmatic job submission with dependencies and pipelines
+- **Auto-detection** - Automatically finds your cluster's scheduler
+- **Interactive TUI** - Monitor jobs with a terminal dashboard
+- **Job Dependencies** - Chain jobs with afterok, afterany, afternotok
+- **Array Jobs** - Batch processing with throttling support
+- **Virtual Environment Handling** - Automatic venv activation on compute nodes
+- **Module Integration** - Load environment modules in job scripts
+- **Dry-run Mode** - Preview generated scripts before submission
+
+## Installation
+
+```bash
+pip install hpc-runner
+```
+
+Or with uv:
+
+```bash
+uv pip install hpc-runner
+```
+
+## Quick Start
+
+### CLI
+
+```bash
+# Basic job submission
+hpc run python train.py
+
+# With resources
+hpc run --cpu 4 --mem 16G --time 4:00:00 "python train.py"
+
+# GPU job
+hpc run --queue gpu --cpu 4 --mem 32G "python train.py --epochs 100"
+
+# Preview without submitting
+hpc run --dry-run --cpu 8 "make -j8"
+
+# Interactive session
+hpc run --interactive bash
+
+# Array job
+hpc run --array 1-100 "python process.py --task-id \$SGE_TASK_ID"
+
+# Wait for completion
+hpc run --wait python long_job.py
+```
+
+### Python API
+
+```python
+from hpc_runner import Job
+
+# Create and submit a job
+job = Job(
+    command="python train.py",
+    cpu=4,
+    mem="16G",
+    time="4:00:00",
+    queue="gpu",
+)
+result = job.submit()
+
+# Wait for completion
+status = result.wait()
+print(f"Exit code: {result.returncode}")
+
+# Read output
+print(result.read_stdout())
+```
+
+### Job Dependencies
+
+```python
+from hpc_runner import Job
+
+# First job
+preprocess = Job(command="python preprocess.py", cpu=8, mem="32G")
+result1 = preprocess.submit()
+
+# Second job runs after first succeeds
+train = Job(command="python train.py", cpu=4, mem="48G", queue="gpu")
+train.after(result1, type="afterok")
+result2 = train.submit()
+```
+
+### Pipelines
+
+```python
+from hpc_runner import Pipeline
+
+with Pipeline("ml_workflow") as p:
+    p.add("python preprocess.py", name="preprocess", cpu=8)
+    p.add("python train.py", name="train", depends_on=["preprocess"], queue="gpu")
+    p.add("python evaluate.py", name="evaluate", depends_on=["train"])
+
+results = p.submit()
+p.wait()
+```
+
+## Scheduler Support
+
+| Scheduler | Status | Notes |
+|-----------|--------|-------|
+| SGE | Fully implemented | qsub, qstat, qdel, qrsh |
+| Local | Fully implemented | Run as subprocess (for testing) |
+| Slurm | Planned | sbatch, squeue, scancel |
+| PBS | Planned | qsub, qstat, qdel |
+
+### Auto-detection Priority
+
+1. `HPC_SCHEDULER` environment variable
+2. SGE (`SGE_ROOT` or `qstat` available)
+3. Slurm (`sbatch` available)
+4. PBS (`qsub` with PBS)
+5. Local fallback
+
+## Configuration
+
+hpc-runner uses TOML configuration files. Location priority:
+
+1. `--config /path/to/config.toml`
+2. `./hpc-tools.toml`
+3. `./pyproject.toml` under `[tool.hpc-tools]`
+4. Git repository root `hpc-tools.toml`
+5. `~/.config/hpc-tools/config.toml`
+6. Package defaults
+
+### Example Configuration
+
+```toml
+[defaults]
+cpu = 1
+mem = "4G"
+time = "1:00:00"
+inherit_env = true
+
+[schedulers.sge]
+parallel_environment = "smp"
+memory_resource = "mem_free"
+purge_modules = true
+
+[types.gpu]
+queue = "gpu"
+resources = [{name = "gpu", value = 1}]
+
+[types.interactive]
+queue = "interactive"
+time = "8:00:00"
+```
+
+Use named job types:
+
+```bash
+hpc run --job-type gpu "python train.py"
+```
+
+## TUI Monitor
+
+Launch the interactive job monitor:
+
+```bash
+hpc monitor
+```
+
+Key bindings:
+- `q` - Quit
+- `r` - Refresh
+- `u` - Toggle user filter (my jobs / all)
+- `/` - Search
+- `Enter` - View job details
+- `Tab` - Switch tabs
+
+## CLI Reference
+
+```
+hpc run [OPTIONS] COMMAND
+
+Options:
+  --job-name TEXT       Job name
+  --cpu INTEGER         Number of CPUs
+  --mem TEXT            Memory (e.g., 16G, 4096M)
+  --time TEXT           Time limit (e.g., 4:00:00)
+  --queue TEXT          Queue/partition name
+  --directory PATH      Working directory
+  --module TEXT         Module to load (repeatable)
+  --array TEXT          Array spec (e.g., 1-100, 1-100%5)
+  --depend TEXT         Job dependencies
+  --inherit-env         Inherit environment (default: true)
+  --no-inherit-env      Don't inherit environment
+  --interactive         Run interactively (qrsh/srun)
+  --local               Run locally (no scheduler)
+  --dry-run             Show script without submitting
+  --wait                Wait for completion
+  --keep-script         Keep job script for debugging
+  -h, --help            Show help
+
+Other commands:
+  hpc status [JOB_ID]   Check job status
+  hpc cancel JOB_ID     Cancel a job
+  hpc monitor           Interactive TUI
+  hpc config show       Show active configuration
+```
+
+## Development
+
+```bash
+# Setup environment
+source sourceme
+source sourceme --clean  # Clean rebuild
+
+# Run tests
+pytest
+pytest -v
+pytest -k "test_job"
+
+# Type checking
+mypy src/hpc_runner
+
+# Linting
+ruff check src/hpc_runner
+ruff format src/hpc_runner
+```
+
+## Documentation
+
+- [Programmatic API Reference](docs/programmatic_api.md)
+- [TUI Styling Guide](docs/TEXTUAL_STYLING_COOKBOOK.md)
+
+## License
+
+MIT License - see LICENSE file for details.

hpc_runner-0.2.0.dist-info/RECORD

@@ -0,0 +1,56 @@
+hpc_runner/__init__.py,sha256=1xnewzCt8FVqseOG5PRmLbiMz0I8V8_TrTyIjW0Q8_A,1414
+hpc_runner/_version.py,sha256=Dg8AmJomLVpjKL6prJylOONZAPRtB86LOce7dorQS_A,704
+hpc_runner/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hpc_runner/cli/__init__.py,sha256=SkyeG7kzxbBGgO9fU96VvHjMjLaSdGI3KFVcqeUAYtg,25
+hpc_runner/cli/cancel.py,sha256=8qRkPjlSGs6OGs7-_ilwpUlWpGQe7ch8aB_rEWiiLAI,901
+hpc_runner/cli/config.py,sha256=dxNNOyuUyzuk9UUUaELp5WkhHrkZnXGi7zQkJdUl4YA,2920
+hpc_runner/cli/main.py,sha256=Did1C3P0Cz7VjuR_a4a0dgHD9u_an_kxaBLNcMWYHR4,2061
+hpc_runner/cli/monitor.py,sha256=aqr39IqwEUz_eczDLMw1dyfs_Gz6mtqKM2eMHI80f4g,698
+hpc_runner/cli/run.py,sha256=fGK0gwIiQdUl0i2aC5FnwSs9MmuKF6ZZFgg7pn2NitU,9646
+hpc_runner/cli/status.py,sha256=LMKvfIiR2U-r8kZOLw1L1-Xfli0mCtX-sSqI6_gy0i0,1957
+hpc_runner/core/__init__.py,sha256=C6R3f_AWv5KOfZ3p2iJiht2B2r1LC62atHWntz51o9c,643
+hpc_runner/core/config.py,sha256=UTbJnIveShGAS_YtS_tRT8H-_Jzq1qGidg3qSQZL6_Q,5184
+hpc_runner/core/descriptors.py,sha256=xaoK3-YbVUcBdeumaY4JZ9noq-d5fqglIwZQqsWvYzU,3307
+hpc_runner/core/exceptions.py,sha256=Z5JIzbpznCCx-wT4uZtHtwxstKZGIAxMubCJV64QzOc,889
+hpc_runner/core/job.py,sha256=N_7HpDzxJipbJU0nVHOoYuaVAGjEXfMzgxJ9ve54mok,11052
+hpc_runner/core/job_array.py,sha256=7VHoZHRzN5JvzWZ_e5pAMeTrNB7-OZw06A8548oK8_c,1595
+hpc_runner/core/job_info.py,sha256=QE4iHH3uqYp4KdvZmBtfFUUaZfmPLkrF6LtlK4RuSJM,2858
+hpc_runner/core/resources.py,sha256=ng1biVsXgGy6AWG50I50aC4LWB1pd60qFLpj5FjIpQc,1275
+hpc_runner/core/result.py,sha256=UGw6ZRmdGI4oVD5ENzrSK72Igx7QNo-OJX9hLVyldp8,4914
+hpc_runner/core/types.py,sha256=a6Yq7vhcpBwgA_hGZwGXiZgCy8wG87FIXBT3LN5qHw8,251
+hpc_runner/schedulers/__init__.py,sha256=v55_DYgiigiFjXJhI7_XFtvxHoyDFWOntWRB9-HWMrM,1623
+hpc_runner/schedulers/base.py,sha256=vhugT9lFrSZb8BMDO_x-h4nSF2oLg2nOvd347g3qfUY,6223
+hpc_runner/schedulers/detection.py,sha256=b5_qdyroMQsvIf_DYY_RDoje6ozKLA3RS-fWWNR9qUU,1467
+hpc_runner/schedulers/local/__init__.py,sha256=bM0RT5SBxs7TsGDUeUi--Z_vlVt9dgokJcftX92VE58,147
+hpc_runner/schedulers/local/scheduler.py,sha256=tTq_5uZEIYCjK3PmPjI_iBAPijwUlTmSaLNoy4_AB5c,13310
+hpc_runner/schedulers/local/templates/job.sh.j2,sha256=pv_R20o5G3LU-DIEIC7gpt7fYIZJmzkrIQEwmVkUWUI,522
+hpc_runner/schedulers/sge/__init__.py,sha256=aR_jJD-YA0HOYPaBwW_CEwi3PASXY2x9sOfmrj6-cjM,144
+hpc_runner/schedulers/sge/args.py,sha256=LBRj4UUDJ5jR8tEoCVF4ivjEKjqDI6LMvFsFzTgtOwk,6553
+hpc_runner/schedulers/sge/parser.py,sha256=P6bG7II2icqSrnzgxGanPITV12X1zMvWYV04L6RIH2c,8381
+hpc_runner/schedulers/sge/scheduler.py,sha256=-vz58MQzaJD97TqE4KfrnR3drLhQgHy8bpjpNwpHt2s,31998
+hpc_runner/schedulers/sge/templates/batch.sh.j2,sha256=au-Hr7o55J1SutLLlpcTBJhRvQYHM4d9bSHk95u3nv4,1696
+hpc_runner/schedulers/sge/templates/interactive.sh.j2,sha256=CY_hcmWoK8EuBNHmP7_lrfMf_i5BjZn8YY9Fi4ousZI,1622
+hpc_runner/templates/__init__.py,sha256=kiaaYBS4QXfGie83SS0rfmZtlbyRUkGkR_yw04oExNQ,137
+hpc_runner/templates/engine.py,sha256=XDW9C51rz9O2rcB3mpjVHQ0qkeG4386gXa8fwaDYsxs,1393
+hpc_runner/tui/__init__.py,sha256=xuet6Iz1ZL6Ys5h95DMcuiapZNo8QA-sFU3R0NflbgE,136
+hpc_runner/tui/app.py,sha256=rsiA2ViNGkfo6pWf2lvVRM3LVM1MunLB8CcNMLHRzso,16466
+hpc_runner/tui/snapshot.py,sha256=sAcxdGaKsFXQMByPip-o1YwFaU-MLf_jGwDlZvEYTxY,5194
+hpc_runner/tui/components/__init__.py,sha256=cS1VT2uhaWDq18-FvT9Hje_Uev7pI8WGFgWVNERfJMQ,303
+hpc_runner/tui/components/detail_panel.py,sha256=3sRxO8J6ceBUctKm-LSF_aujIa63TEdWZJt2v08ae28,6983
+hpc_runner/tui/components/filter_bar.py,sha256=v3YqUnieTzipJCaUHsVIcXc1VwTM3TorYBW-H6Ai5vo,4965
+hpc_runner/tui/components/filter_popup.py,sha256=_car5ic8SS32183XWvbvVX8q1paNoX2Un-Ohy3z2vwo,10428
+hpc_runner/tui/components/job_table.py,sha256=3gqYey0L9i2WInKl7Hknh31_E_PEPSKdFvVwSsG7uuY,9444
+hpc_runner/tui/providers/__init__.py,sha256=iLzT7HR2ETSf6MxDw64yVn99kiHqzEDOHqNU7XLAEME,100
+hpc_runner/tui/providers/jobs.py,sha256=YjhufHvBJu6QICIZLs5FhFBAcQMEyEhw-heSzYxYwvo,6340
+hpc_runner/tui/screens/__init__.py,sha256=EoyZZGr_fruzDuDXtsKXyY5QhHbEFVbm820eiRtciBc,221
+hpc_runner/tui/screens/confirm.py,sha256=ZBjrqcIk6f_5KGLzsSCfewZdyERnR0x2W2A34KYlkbI,2101
+hpc_runner/tui/screens/job_details.py,sha256=B7KiZunhfM7QW3xVvmEBQnOB5CQfxYpH9Td0JC6M29A,7488
+hpc_runner/tui/screens/log_viewer.py,sha256=II6B2Vj4KY-nx1wWWq_5K2gsmi6dSyQm-suaEzauEcM,5888
+hpc_runner/tui/styles/monitor.tcss,sha256=-XJ_mF02AlaEuX81N6tkYKhKYDcBKYTDKXejJO5efkI,11349
+hpc_runner/workflow/__init__.py,sha256=Y5h7wfnlWcav_f2USZaP6r5m7I5KFam8eEJGo4UqJ0w,221
+hpc_runner/workflow/dependency.py,sha256=PjhgCurnlfcKYO_arlUDLCKOpwqq324l09Sj0v-iwOU,639
+hpc_runner/workflow/pipeline.py,sha256=MYV95Kp-5m2YVxYDk_zIYUsyt31ctl7jnVA10F2OJgg,5540
+hpc_runner-0.2.0.dist-info/METADATA,sha256=5wCFmYvXMhg6Y2RQS66K3cUvjML64lRsulpgyRf_5O8,7015
+hpc_runner-0.2.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+hpc_runner-0.2.0.dist-info/entry_points.txt,sha256=Ou3LsalLkY9mNOrUwyCDN8Q4PekM1K5h52hAU6fADk8,48
+hpc_runner-0.2.0.dist-info/RECORD,,

hpc_runner-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

hpc_runner-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+hpc = hpc_runner.cli.main:cli