dpdispatcher 0.6.10__tar.gz → 0.6.12__tar.gz

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/.github/workflows/ci-docker.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out the repo
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
 
     - name: Log in to Docker Hub
       uses: docker/login-action@v3

dpdispatcher-0.6.12/.github/workflows/copilot-setup-steps.yml

@@ -0,0 +1,65 @@
+name: "Copilot Setup Steps"
+
+# Automatically run the setup steps when they are changed to allow for easy validation, and
+# allow manual testing through the repository's "Actions" tab
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/copilot-setup-steps.yml
+  pull_request:
+    paths:
+      - .github/workflows/copilot-setup-steps.yml
+
+jobs:
+  # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot.
+  copilot-setup-steps:
+    runs-on: ubuntu-latest
+
+    # Set the permissions to the lowest permissions possible needed for your steps.
+    # Copilot will be given its own token for its operations.
+    permissions:
+      # If you want to clone the repository as part of your setup steps, for example to install dependencies, you'll need the `contents: read` permission. If you don't clone the repository in your setup steps, Copilot will do this for you automatically after the steps complete.
+      contents: read
+
+    # You can define any steps you want, and they will run before the agent starts.
+    # If you do not check out your code, Copilot will do this for you.
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        run: |
+          # Install uv using pip to avoid network restrictions
+          python -m pip install --upgrade pip
+          python -m pip install uv
+
+      - name: Create virtual environment and install dependencies
+        run: |
+          uv venv .venv
+          source .venv/bin/activate
+          uv pip install .[test] coverage
+
+      - name: Install development tools
+        run: |
+          uv tool install pre-commit
+          uv tool install pyright
+
+      - name: Set up pre-commit hooks
+        run: |
+          source .venv/bin/activate
+          pre-commit install --install-hooks
+
+      - name: Verify installation
+        run: |
+          source .venv/bin/activate
+          python --version
+          uv --version
+          pre-commit --version
+          pyright --version
+          python -c "import dpdispatcher; print('DPDispatcher installed successfully')"

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/.github/workflows/machines.yml

@@ -1,8 +1,12 @@
 name: Test on different machines
 
-on:
-  - push
-  - pull_request
+'on':
+  push:
+    branches-ignore:
+      - 'copilot/**'
+      - 'dependabot/**'
+      - 'pre-commit-ci-update-config'
+  pull_request:
 
 jobs:
   build:
@@ -15,7 +19,7 @@ jobs:
           - ssh
           - ssh_rsync
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - run: ./ci/${{ matrix.machine }}.sh
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/.github/workflows/publish_conda.yml

@@ -9,7 +9,7 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: publish-to-conda
       uses: felix5572/conda-publish-action@v1.9
       with:

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/.github/workflows/pyright.yml

@@ -1,5 +1,9 @@
-on:
+'on':
   push:
+    branches-ignore:
+      - 'copilot/**'
+      - 'dependabot/**'
+      - 'pre-commit-ci-update-config'
   pull_request:
 name: Type checker
 jobs:
@@ -7,12 +11,12 @@ jobs:
     name: pyright
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
+    - uses: actions/checkout@v5
+    - uses: actions/setup-python@v6
       with:
         python-version: '3.11'
     - run: pip install uv
     - run: uv pip install --system -e .[cloudserver,gui]
     - uses: jakebailey/pyright-action@v2
       with:
-        version: 1.1.308
+        version: 1.1.404

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/.github/workflows/release.yml

@@ -1,4 +1,12 @@
-on: [push, pull_request]
+'on':
+  push:
+    branches-ignore:
+      - 'copilot/**'
+      - 'dependabot/**'
+      - 'pre-commit-ci-update-config'
+    tags:
+      - 'v*'
+  pull_request:
 name: Release to pypi
 jobs:
   release-to-pypi:
@@ -8,9 +16,9 @@ jobs:
         # IMPORTANT: this permission is mandatory for trusted publishing
         id-token: write
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Setup python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: 3.x
         architecture: x64

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/.github/workflows/test-bohrium.yml

@@ -1,7 +1,11 @@
 name: Test Bohrium
 
-on:
+'on':
   push:
+    branches-ignore:
+      - 'copilot/**'
+      - 'dependabot/**'
+      - 'pre-commit-ci-update-config'
   pull_request_target:
     types:
       - "labeled"
@@ -12,11 +16,11 @@ jobs:
     environment: bohrium
     if: github.repository_owner == 'deepmodeling' && (github.event.label.name == 'Test Bohrium' || github.event_name == 'push')
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
       with:
         ref: "${{ github.event.pull_request.merge_commit_sha }}"
     - name: Set up Python 3.12
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.12'
         cache: 'pip'

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/.github/workflows/test.yml

@@ -1,8 +1,12 @@
 name: Python package
 
-on:
-  - push
-  - pull_request
+'on':
+  push:
+    branches-ignore:
+      - 'copilot/**'
+      - 'dependabot/**'
+      - 'pre-commit-ci-update-config'
+  pull_request:
 
 jobs:
   test:
@@ -24,11 +28,11 @@ jobs:
             platform: macos-latest
         include:  # So run those legacy versions on Intel CPUs
           - python-version: "3.7"
-            platform: macos-13
+            platform: macos-15-intel
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
     - uses: astral-sh/setup-uv@v6

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/.gitignore

@@ -37,7 +37,7 @@ pydispatcher.egg-info
 */__pycache__
 *.swp
 .eggs
-.coverage
+.coverage*
 dbconfig.json
 .vscode/*
 .idea
@@ -46,3 +46,17 @@ dbconfig.json
 *.egg
 *.egg-info
 venv/*
+node_modules/
+# Test artifacts
+*_flag_if_job_task_fail
+*_job_id
+*_job_tag_finished
+*_task_tag_finished
+*.sub
+*.sub.run
+script_*.py
+# Job execution temporary files
+err
+log
+# JSON files with hash names (job state files)
+[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*.json

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
         exclude: "^tests/"
@@ -18,7 +18,7 @@ repos:
   # Python
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.9.3
+    rev: v0.12.8
     hooks:
       - id: ruff
         args: ["--fix"]

dpdispatcher-0.6.12/AGENTS.md

@@ -0,0 +1,172 @@
+# DPDispatcher Development Guide
+
+DPDispatcher is a Python package used to generate HPC (High-Performance Computing) scheduler systems (Slurm/PBS/LSF/Bohrium) jobs input scripts, submit them to HPC systems, and poke until they finish.
+
+Always reference these instructions first and fallback to search or bash commands only when you encounter unexpected information that does not match the info here.
+
+## Working Effectively
+
+### Bootstrap, Build, and Test
+
+- **Environment setup:** The Python environment, `uv`, `pre-commit`, and `pyright` are automatically set up by the Copilot environment. Simply activate the virtual environment:
+
+  ```bash
+  source .venv/bin/activate
+  ```
+
+- **Run the test suite:**
+
+  ```bash
+  python -m coverage run -p --source=./dpdispatcher -m unittest -v
+  python -m coverage combine
+  python -m coverage report
+  ```
+
+  - **TIMING: Tests take ~25 seconds. NEVER CANCEL - set timeout to 2+ minutes.**
+
+- **Run linting and formatting:**
+
+  ```bash
+  pre-commit run --all-files
+  ```
+
+  - **TIMING: Linting takes <5 seconds.**
+  - **Fallback:** If pre-commit fails with network issues, use `ruff check . && ruff format --check .`
+
+- **Run type checking:**
+
+  ```bash
+  pyright
+  ```
+
+  - **TIMING: Type checking takes ~2-3 seconds.**
+
+- **Build documentation:**
+
+  ```bash
+  uv pip install .[docs]
+  cd doc
+  make html
+  ```
+
+  - **TIMING: Documentation build takes ~14 seconds.**
+
+### CLI Usage
+
+- **Test basic CLI functionality:**
+
+  ```bash
+  dpdisp --help
+  dpdisp run examples/dpdisp_run.py
+  ```
+
+- **Run sample job dispatch script:**
+  ```bash
+  python examples/dpdisp_run.py
+  ```
+
+## Validation
+
+- **ALWAYS run the test suite after making code changes.** Tests execute quickly (~25 seconds) and should never be cancelled.
+- **ALWAYS run linting before committing:** `pre-commit run --all-files`
+- **ALWAYS run type checking:** `pyright` to catch type-related issues.
+- **Test CLI functionality:** Run `dpdisp --help` and test with example scripts to ensure the CLI works correctly.
+- **Build documentation:** Run `make html` in the `doc/` directory to verify documentation builds without errors.
+
+## Build and Development Workflow
+
+- **Dependencies:** The project uses `uv` as the package manager for fast dependency resolution.
+- **Build system:** Uses `setuptools` with `pyproject.toml` configuration.
+- **Python versions:** Supports Python 3.7+ (check `pyproject.toml` for current support matrix).
+- **Testing:** Uses `unittest` framework with coverage reporting.
+- **Linting:** Uses `pre-commit` with `ruff` for linting and formatting, plus other quality checks.
+- **Type checking:** Uses `pyright` for static type analysis (configured in `pyproject.toml`).
+- **Documentation:** Uses Sphinx with MyST parser for markdown support.
+
+## Key Components
+
+### Core Modules
+
+- **`dpdispatcher/submission.py`**: Main classes - `Submission`, `Job`, `Task`, `Resources`
+- **`dpdispatcher/machine.py`**: Machine configuration and dispatch logic
+- **`dpdispatcher/contexts/`**: Context managers for different execution environments (SSH, local, etc.)
+- **`dpdispatcher/machines/`**: HPC scheduler implementations (Slurm, PBS, LSF, Shell, etc.)
+
+### Configuration Files
+
+- **`examples/machine/`**: Example machine configurations (JSON format)
+- **`examples/resources/`**: Example resource specifications
+- **`examples/task/`**: Example task definitions
+- **`examples/dpdisp_run.py`**: Complete working example using PEP 723 script metadata
+
+### Important Directories
+
+- **`tests/`**: Comprehensive test suite including unit tests and integration tests
+- **`doc/`**: Sphinx documentation source files
+- **`ci/`**: CI/CD configuration for Docker-based testing
+
+## Common Tasks
+
+### Running Examples
+
+```bash
+# Example with lazy local execution (no HPC needed)
+dpdisp run examples/dpdisp_run.py
+
+# Example output files created:
+# - log: contains "hello world!" output
+# - Various job tracking files (automatically cleaned up)
+```
+
+### Testing with Different HPC Systems
+
+- **Local/Shell execution:** Use `LazyLocalContext` with `Shell` batch type (examples/machine/lazy_local.json)
+- **SSH remote execution:** Use `SSHContext` (requires SSH setup)
+- **HPC schedulers:** Use `Slurm`, `PBS`, `LSF` batch types (require actual HPC environment)
+
+### Debugging Job Execution
+
+- Check job logs in the work directory
+- Use `dpdisp submission` commands to inspect job states
+- Review machine and resource configurations in JSON files
+
+## Project Structure Reference
+
+```
+.
+├── README.md
+├── pyproject.toml           # Project configuration and dependencies
+├── dpdispatcher/            # Main package
+│   ├── __init__.py
+│   ├── submission.py        # Core submission/job/task classes
+│   ├── machine.py          # Machine configuration
+│   ├── contexts/           # Execution contexts (SSH, local, etc.)
+│   ├── machines/           # HPC scheduler implementations
+│   └── utils/              # Utility functions
+├── examples/               # Example configurations and scripts
+│   ├── machine/           # Example machine configs
+│   ├── resources/         # Example resource specs
+│   ├── task/             # Example task definitions
+│   └── dpdisp_run.py     # Complete working example
+├── tests/                # Comprehensive test suite
+├── doc/                  # Sphinx documentation
+└── ci/                   # CI/CD Docker configurations
+```
+
+## Development Notes
+
+- **Virtual environment is pre-configured** - The environment is automatically set up; simply activate with `source .venv/bin/activate`
+- **Always add type hints** - Include proper type annotations in all Python code for better maintainability
+- **Always use conventional commit format** - All commit messages and PR titles must follow the conventional commit specification (e.g., `feat:`, `fix:`, `docs:`, `refactor:`, `test:`, `chore:`)
+- **Test artifacts are gitignored** - Job execution creates temporary files that are automatically excluded
+- **Pre-commit hooks are ready** - Pre-commit is pre-installed and hooks are configured
+- **Multiple execution contexts** - Code supports local execution, SSH remote execution, and various HPC schedulers
+- **Extensive examples** - Use `examples/` directory as reference for proper configuration formats
+
+## Troubleshooting
+
+- **Virtual environment issues:** The virtual environment is pre-created; simply activate with `source .venv/bin/activate`
+- **Development tools:** `pre-commit` and `pyright` are pre-installed and ready to use
+- **Test failures:** Most tests run locally; some require specific HPC environments and will be skipped
+- **Documentation build warnings:** Some warnings about external inventory URLs are expected in sandboxed environments
+- **Pre-commit network issues:** If pre-commit fails with network timeouts, run `ruff check` and `ruff format` directly

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dpdispatcher
-Version: 0.6.10
+Version: 0.6.12
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/ci/ssh/docker-compose.yml

@@ -12,6 +12,17 @@ services:
       - "22"
     volumes:
       - ssh_config:/root/.ssh
+  jumphost:
+    image: takeyamajp/ubuntu-sshd:ubuntu22.04
+    build: .
+    container_name: jumphost
+    hostname: jumphost
+    environment:
+      ROOT_PASSWORD: dpdispatcher
+    expose:
+      - "22"
+    volumes:
+      - ssh_config:/root/.ssh
   test:
     image: python:3.10
     tty: true
@@ -25,6 +36,7 @@ services:
       - ../..:/dpdispatcher
     depends_on:
       - server
+      - jumphost
 
 volumes:
   ssh_config:

dpdispatcher-0.6.12/ci/ssh/start-ssh.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+docker compose up -d --no-build
+
+# Set up SSH keys on server
+docker exec server /bin/bash -c "ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N \"\" && cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"
+docker exec server /bin/bash -c "mkdir -p /dpdispatcher_working"
+docker exec server /bin/bash -c "mkdir -p /tmp/rsync_test"
+
+# Set up SSH keys on jumphost and configure it to access server
+docker exec jumphost /bin/bash -c "ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N \"\" && cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"
+
+# Copy keys between containers to enable jump host functionality
+# Get the public key from jumphost and add it to server's authorized_keys
+docker exec jumphost /bin/bash -c "cat /root/.ssh/id_rsa.pub" | docker exec -i server /bin/bash -c "cat >> /root/.ssh/authorized_keys"
+
+# Get the public key from test (which shares volume with server) and add it to jumphost authorized_keys
+docker exec test /bin/bash -c "cat /root/.ssh/id_rsa.pub" | docker exec -i jumphost /bin/bash -c "cat >> /root/.ssh/authorized_keys"
+
+# Configure SSH client settings for known hosts to avoid host key verification
+docker exec test /bin/bash -c "echo 'StrictHostKeyChecking no' >> /root/.ssh/config && echo 'UserKnownHostsFile /dev/null' >> /root/.ssh/config"
+docker exec jumphost /bin/bash -c "echo 'StrictHostKeyChecking no' >> /root/.ssh/config && echo 'UserKnownHostsFile /dev/null' >> /root/.ssh/config"
+docker exec server /bin/bash -c "echo 'StrictHostKeyChecking no' >> /root/.ssh/config && echo 'UserKnownHostsFile /dev/null' >> /root/.ssh/config"
+
+# Install rsync on all containers
+docker exec test /bin/bash -c "apt-get -y update && apt-get -y install rsync"
+docker exec jumphost /bin/bash -c "apt-get -y update && apt-get -y install rsync"
+docker exec server /bin/bash -c "apt-get -y update && apt-get -y install rsync"

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/doc/context.md

@@ -33,6 +33,28 @@ It's suggested to generate [SSH keys](https://help.ubuntu.com/community/SSH/Open
 
 Note that `SSH` context is [non-login](https://www.gnu.org/software/bash/manual/html_node/Bash-Startup-Files.html), so `bash_profile` files will not be executed outside the submission script.
 
+### SSH Jump Host (Bastion Server)
+
+For connecting to internal servers through a jump host (bastion server), SSH context supports jump host configuration. This allows connecting to internal servers that are not directly accessible from the internet.
+
+Specify the ProxyCommand directly using {dargs:argument}`proxy_command <machine[SSHContext]/remote_profile/proxy_command>`:
+
+```json
+{
+  "context_type": "SSHContext",
+  "remote_profile": {
+    "hostname": "internal-server.company.com",
+    "username": "user",
+    "key_filename": "/path/to/internal_key",
+    "proxy_command": "ssh -W internal-server.company.com:22 -i /path/to/jump_key jumpuser@bastion.company.com"
+  }
+}
+```
+
+**Note**: Unlike OpenSSH, Paramiko's ProxyCommand requires explicit hostname and port instead of `%h:%p` placeholders. The proxy command must specify the actual target hostname and port that matches the `hostname` and `port` in the remote profile.
+
+This configuration establishes the connection path: Local → Jump Host → Target Server.
+
 ## Bohrium
 
 {dargs:argument}`context_type <machine/context_type>`: `Bohrium`

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/dpdispatcher/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.6.10'
-__version_tuple__ = version_tuple = (0, 6, 10)
+__version__ = version = '0.6.12'
+__version_tuple__ = version_tuple = (0, 6, 12)
+
+__commit_id__ = commit_id = 'gfd51e420a'

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/dpdispatcher/contexts/dp_cloud_server_context.py

@@ -161,7 +161,9 @@ class BohriumContext(BaseContext):
         # return oss_task_zip
         # api.upload(self.oss_task_dir, zip_task_file)
 
-    def download(self, submission):
+    def download(
+        self, submission, check_exists=False, mark_failure=True, back_error=False
+    ):
         jobs = submission.belonging_jobs
         job_hashs = {}
         job_infos = {}

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/dpdispatcher/contexts/lazy_local_context.py

@@ -83,7 +83,7 @@ class LazyLocalContext(BaseContext):
 
     def upload(
         self,
-        jobs,
+        submission,
         # local_up_files,
         dereference=True,
     ):
@@ -91,7 +91,7 @@ class LazyLocalContext(BaseContext):
 
     def download(
         self,
-        jobs,
+        submission,
         # remote_down_files,
         check_exists=False,
         mark_failure=True,

{dpdispatcher-0.6.10 → dpdispatcher-0.6.12}/dpdispatcher/contexts/openapi_context.py

@@ -95,12 +95,13 @@ class OpenAPIContext(BaseContext):
             raise ValueError(
                 "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'"
             )
-        self.client = Bohrium(
+        self.client = Bohrium(  # type: ignore[reportPossiblyUnboundVariable]
             access_key=access_key, project_id=project_id, app_key=app_key
         )
-        self.storage = Tiefblue()
-        self.job = Job(client=self.client)
+        self.storage = Tiefblue()  # type: ignore[reportPossiblyUnboundVariable]
+        self.job = Job(client=self.client)  # type: ignore[reportPossiblyUnboundVariable]
         self.jgid = None
+        os.makedirs(DP_CLOUD_SERVER_HOME_DIR, exist_ok=True)
 
     @classmethod
     def load_from_dict(cls, context_dict):
@@ -205,7 +206,9 @@ class OpenAPIContext(BaseContext):
         # return oss_task_zip
         # api.upload(self.oss_task_dir, zip_task_file)
 
-    def download(self, submission):
+    def download(
+        self, submission, check_exists=False, mark_failure=True, back_error=False
+    ):
         jobs = submission.belonging_jobs
         job_hashs = {}
         job_infos = {}