hpc-runner 0.1.0__tar.gz

hpc_runner-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Full history for hatch-vcs
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest -v

hpc_runner-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,47 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Full history for hatch-vcs
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install build dependencies
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Upload dist artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # Required for trusted publishing
+    steps:
+      - name: Download dist artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

hpc_runner-0.1.0/.gitignore

@@ -0,0 +1,66 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Ruff
+.ruff_cache/
+
+# Local development
+*.log
+.DS_Store
+
+# HPC job outputs (don't commit job logs)
+*.out
+*.err
+*.o[0-9]*
+*.e[0-9]*
+
+# Generated version file
+src/hpc_runner/_version.py

hpc_runner-0.1.0/CLAUDE.md

@@ -0,0 +1,65 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Build & Development Commands
+
+```bash
+# Setup virtual environment
+source sourceme           # Creates .venv with Python 3.11, installs deps
+source sourceme --clean   # Clean rebuild of .venv
+
+# Run tests
+pytest                    # All tests
+pytest -v                 # Verbose
+pytest tests/test_core/   # Specific directory
+pytest -k "test_job"      # By name pattern
+
+# Type checking and linting
+mypy src/hpc_runner
+ruff check src/hpc_runner
+ruff format src/hpc_runner
+
+# CLI usage
+hpc --help
+hpc run --dry-run "echo hello"
+hpc --scheduler sge run --cpu 4 --mem 8G "python script.py"
+```
+
+## Architecture
+
+**Package**: `hpc-runner` (PyPI) / `hpc_runner` (import) / `hpc` (CLI)
+
+### Core Abstractions (`src/hpc_runner/core/`)
+
+- **Job** - Central job model with command, resources (cpu/mem/time), modules, dependencies
+- **JobResult/ArrayJobResult** - Returned from submission, provides status polling and output access
+- **JobStatus** - Unified enum: PENDING, RUNNING, COMPLETED, FAILED, CANCELLED, TIMEOUT, UNKNOWN
+- **ResourceSet** - Collection of named resources (gpu, licenses, etc.)
+- **HPCConfig** - TOML-based config with hierarchy: ./hpc-tools.toml > pyproject.toml > git root > ~/.config > package defaults
+
+### Scheduler System (`src/hpc_runner/schedulers/`)
+
+- **BaseScheduler** - ABC defining submit(), cancel(), get_status(), generate_script()
+- **SGEScheduler** - Sun Grid Engine (qsub/qstat/qdel). All SGE-specific settings (PE name, resource names) are configurable via `[schedulers.sge]` config section
+- **LocalScheduler** - Runs jobs as local subprocesses (for testing without cluster)
+- **detection.py** - Auto-detects scheduler: HPC_SCHEDULER env > SGE_ROOT > sbatch > PBS > local fallback
+- **Registry** - `get_scheduler(name)` with lazy imports, `register_scheduler()` for custom schedulers
+
+### CLI (`src/hpc_runner/cli/`)
+
+Uses `rich-click` for styled output. Commands: `run`, `status`, `cancel`, `config`.
+
+### Workflow (`src/hpc_runner/workflow/`)
+
+**Pipeline** - Job dependency graphs with topological sorting. Jobs reference other jobs by name; dependencies are resolved at submit time.
+
+### Templates (`src/hpc_runner/templates/`)
+
+Jinja2 templates for job scripts. Each scheduler has its own template in `schedulers/{name}/templates/job.sh.j2`.
+
+## Key Design Decisions
+
+- **Merged output by default**: stderr goes to stdout unless `--stderr` specified
+- **Configurable SGE settings**: PE name, memory resource name, time resource name all come from config, not hardcoded
+- **Descriptor pattern**: Scheduler arguments use Python descriptors for type-safe flag/directive generation

hpc_runner-0.1.0/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: hpc-runner
+Version: 0.1.0
+Summary: Unified HPC job submission across multiple schedulers
+Project-URL: Homepage, https://github.com/shareefj/hpc-tools
+Project-URL: Repository, https://github.com/shareefj/hpc-tools
+Author: Shareef Jalloq
+License-Expression: MIT
+Keywords: cluster,hpc,job-submission,pbs,sge,slurm
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: System :: Clustering
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
+Requires-Dist: jinja2>=3.0
+Requires-Dist: rich-click>=1.7
+Requires-Dist: tomli>=2.0; python_version < '3.11'
+Provides-Extra: all
+Requires-Dist: build; extra == 'all'
+Requires-Dist: hatch-vcs; extra == 'all'
+Requires-Dist: mypy; extra == 'all'
+Requires-Dist: pytest-cov; extra == 'all'
+Requires-Dist: pytest>=7.0; extra == 'all'
+Requires-Dist: ruff; extra == 'all'
+Requires-Dist: twine; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: hatch-vcs; extra == 'dev'
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# hpc-tools
+
+A collection of tools aimed at abstracting the intricacies of HPC job schedulers from the user.  Having writen tools like this in every job I've had, I thought it about time to write one to rule them all.  With some help from my mate Claude.

hpc_runner-0.1.0/README.md

@@ -0,0 +1,3 @@
+# hpc-tools
+
+A collection of tools aimed at abstracting the intricacies of HPC job schedulers from the user.  Having writen tools like this in every job I've had, I thought it about time to write one to rule them all.  With some help from my mate Claude.