dpdispatcher 0.6.11__tar.gz → 0.6.12__tar.gz

dpdispatcher-0.6.12/.github/workflows/copilot-setup-steps.yml

@@ -0,0 +1,65 @@
+name: "Copilot Setup Steps"
+
+# Automatically run the setup steps when they are changed to allow for easy validation, and
+# allow manual testing through the repository's "Actions" tab
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/copilot-setup-steps.yml
+  pull_request:
+    paths:
+      - .github/workflows/copilot-setup-steps.yml
+
+jobs:
+  # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot.
+  copilot-setup-steps:
+    runs-on: ubuntu-latest
+
+    # Set the permissions to the lowest permissions possible needed for your steps.
+    # Copilot will be given its own token for its operations.
+    permissions:
+      # If you want to clone the repository as part of your setup steps, for example to install dependencies, you'll need the `contents: read` permission. If you don't clone the repository in your setup steps, Copilot will do this for you automatically after the steps complete.
+      contents: read
+
+    # You can define any steps you want, and they will run before the agent starts.
+    # If you do not check out your code, Copilot will do this for you.
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        run: |
+          # Install uv using pip to avoid network restrictions
+          python -m pip install --upgrade pip
+          python -m pip install uv
+
+      - name: Create virtual environment and install dependencies
+        run: |
+          uv venv .venv
+          source .venv/bin/activate
+          uv pip install .[test] coverage
+
+      - name: Install development tools
+        run: |
+          uv tool install pre-commit
+          uv tool install pyright
+
+      - name: Set up pre-commit hooks
+        run: |
+          source .venv/bin/activate
+          pre-commit install --install-hooks
+
+      - name: Verify installation
+        run: |
+          source .venv/bin/activate
+          python --version
+          uv --version
+          pre-commit --version
+          pyright --version
+          python -c "import dpdispatcher; print('DPDispatcher installed successfully')"

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/.github/workflows/machines.yml

@@ -1,8 +1,12 @@
 name: Test on different machines
 
-on:
-  - push
-  - pull_request
+'on':
+  push:
+    branches-ignore:
+      - 'copilot/**'
+      - 'dependabot/**'
+      - 'pre-commit-ci-update-config'
+  pull_request:
 
 jobs:
   build:

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/.github/workflows/pyright.yml

@@ -1,5 +1,9 @@
-on:
+'on':
   push:
+    branches-ignore:
+      - 'copilot/**'
+      - 'dependabot/**'
+      - 'pre-commit-ci-update-config'
   pull_request:
 name: Type checker
 jobs:
@@ -8,11 +12,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v5
-    - uses: actions/setup-python@v5
+    - uses: actions/setup-python@v6
       with:
         python-version: '3.11'
     - run: pip install uv
     - run: uv pip install --system -e .[cloudserver,gui]
     - uses: jakebailey/pyright-action@v2
       with:
-        version: 1.1.308
+        version: 1.1.404

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/.github/workflows/release.yml

@@ -1,4 +1,12 @@
-on: [push, pull_request]
+'on':
+  push:
+    branches-ignore:
+      - 'copilot/**'
+      - 'dependabot/**'
+      - 'pre-commit-ci-update-config'
+    tags:
+      - 'v*'
+  pull_request:
 name: Release to pypi
 jobs:
   release-to-pypi:
@@ -10,7 +18,7 @@ jobs:
     steps:
     - uses: actions/checkout@v5
     - name: Setup python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: 3.x
         architecture: x64

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/.github/workflows/test-bohrium.yml

@@ -1,7 +1,11 @@
 name: Test Bohrium
 
-on:
+'on':
   push:
+    branches-ignore:
+      - 'copilot/**'
+      - 'dependabot/**'
+      - 'pre-commit-ci-update-config'
   pull_request_target:
     types:
       - "labeled"
@@ -16,7 +20,7 @@ jobs:
       with:
         ref: "${{ github.event.pull_request.merge_commit_sha }}"
     - name: Set up Python 3.12
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.12'
         cache: 'pip'

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/.github/workflows/test.yml

@@ -1,8 +1,12 @@
 name: Python package
 
-on:
-  - push
-  - pull_request
+'on':
+  push:
+    branches-ignore:
+      - 'copilot/**'
+      - 'dependabot/**'
+      - 'pre-commit-ci-update-config'
+  pull_request:
 
 jobs:
   test:
@@ -24,11 +28,11 @@ jobs:
             platform: macos-latest
         include:  # So run those legacy versions on Intel CPUs
           - python-version: "3.7"
-            platform: macos-13
+            platform: macos-15-intel
     steps:
     - uses: actions/checkout@v5
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
     - uses: astral-sh/setup-uv@v6

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/.gitignore

@@ -37,7 +37,7 @@ pydispatcher.egg-info
 */__pycache__
 *.swp
 .eggs
-.coverage
+.coverage*
 dbconfig.json
 .vscode/*
 .idea
@@ -46,3 +46,17 @@ dbconfig.json
 *.egg
 *.egg-info
 venv/*
+node_modules/
+# Test artifacts
+*_flag_if_job_task_fail
+*_job_id
+*_job_tag_finished
+*_task_tag_finished
+*.sub
+*.sub.run
+script_*.py
+# Job execution temporary files
+err
+log
+# JSON files with hash names (job state files)
+[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*.json

dpdispatcher-0.6.12/AGENTS.md

@@ -0,0 +1,172 @@
+# DPDispatcher Development Guide
+
+DPDispatcher is a Python package used to generate HPC (High-Performance Computing) scheduler systems (Slurm/PBS/LSF/Bohrium) jobs input scripts, submit them to HPC systems, and poke until they finish.
+
+Always reference these instructions first and fallback to search or bash commands only when you encounter unexpected information that does not match the info here.
+
+## Working Effectively
+
+### Bootstrap, Build, and Test
+
+- **Environment setup:** The Python environment, `uv`, `pre-commit`, and `pyright` are automatically set up by the Copilot environment. Simply activate the virtual environment:
+
+  ```bash
+  source .venv/bin/activate
+  ```
+
+- **Run the test suite:**
+
+  ```bash
+  python -m coverage run -p --source=./dpdispatcher -m unittest -v
+  python -m coverage combine
+  python -m coverage report
+  ```
+
+  - **TIMING: Tests take ~25 seconds. NEVER CANCEL - set timeout to 2+ minutes.**
+
+- **Run linting and formatting:**
+
+  ```bash
+  pre-commit run --all-files
+  ```
+
+  - **TIMING: Linting takes <5 seconds.**
+  - **Fallback:** If pre-commit fails with network issues, use `ruff check . && ruff format --check .`
+
+- **Run type checking:**
+
+  ```bash
+  pyright
+  ```
+
+  - **TIMING: Type checking takes ~2-3 seconds.**
+
+- **Build documentation:**
+
+  ```bash
+  uv pip install .[docs]
+  cd doc
+  make html
+  ```
+
+  - **TIMING: Documentation build takes ~14 seconds.**
+
+### CLI Usage
+
+- **Test basic CLI functionality:**
+
+  ```bash
+  dpdisp --help
+  dpdisp run examples/dpdisp_run.py
+  ```
+
+- **Run sample job dispatch script:**
+  ```bash
+  python examples/dpdisp_run.py
+  ```
+
+## Validation
+
+- **ALWAYS run the test suite after making code changes.** Tests execute quickly (~25 seconds) and should never be cancelled.
+- **ALWAYS run linting before committing:** `pre-commit run --all-files`
+- **ALWAYS run type checking:** `pyright` to catch type-related issues.
+- **Test CLI functionality:** Run `dpdisp --help` and test with example scripts to ensure the CLI works correctly.
+- **Build documentation:** Run `make html` in the `doc/` directory to verify documentation builds without errors.
+
+## Build and Development Workflow
+
+- **Dependencies:** The project uses `uv` as the package manager for fast dependency resolution.
+- **Build system:** Uses `setuptools` with `pyproject.toml` configuration.
+- **Python versions:** Supports Python 3.7+ (check `pyproject.toml` for current support matrix).
+- **Testing:** Uses `unittest` framework with coverage reporting.
+- **Linting:** Uses `pre-commit` with `ruff` for linting and formatting, plus other quality checks.
+- **Type checking:** Uses `pyright` for static type analysis (configured in `pyproject.toml`).
+- **Documentation:** Uses Sphinx with MyST parser for markdown support.
+
+## Key Components
+
+### Core Modules
+
+- **`dpdispatcher/submission.py`**: Main classes - `Submission`, `Job`, `Task`, `Resources`
+- **`dpdispatcher/machine.py`**: Machine configuration and dispatch logic
+- **`dpdispatcher/contexts/`**: Context managers for different execution environments (SSH, local, etc.)
+- **`dpdispatcher/machines/`**: HPC scheduler implementations (Slurm, PBS, LSF, Shell, etc.)
+
+### Configuration Files
+
+- **`examples/machine/`**: Example machine configurations (JSON format)
+- **`examples/resources/`**: Example resource specifications
+- **`examples/task/`**: Example task definitions
+- **`examples/dpdisp_run.py`**: Complete working example using PEP 723 script metadata
+
+### Important Directories
+
+- **`tests/`**: Comprehensive test suite including unit tests and integration tests
+- **`doc/`**: Sphinx documentation source files
+- **`ci/`**: CI/CD configuration for Docker-based testing
+
+## Common Tasks
+
+### Running Examples
+
+```bash
+# Example with lazy local execution (no HPC needed)
+dpdisp run examples/dpdisp_run.py
+
+# Example output files created:
+# - log: contains "hello world!" output
+# - Various job tracking files (automatically cleaned up)
+```
+
+### Testing with Different HPC Systems
+
+- **Local/Shell execution:** Use `LazyLocalContext` with `Shell` batch type (examples/machine/lazy_local.json)
+- **SSH remote execution:** Use `SSHContext` (requires SSH setup)
+- **HPC schedulers:** Use `Slurm`, `PBS`, `LSF` batch types (require actual HPC environment)
+
+### Debugging Job Execution
+
+- Check job logs in the work directory
+- Use `dpdisp submission` commands to inspect job states
+- Review machine and resource configurations in JSON files
+
+## Project Structure Reference
+
+```
+.
+├── README.md
+├── pyproject.toml           # Project configuration and dependencies
+├── dpdispatcher/            # Main package
+│   ├── __init__.py
+│   ├── submission.py        # Core submission/job/task classes
+│   ├── machine.py          # Machine configuration
+│   ├── contexts/           # Execution contexts (SSH, local, etc.)
+│   ├── machines/           # HPC scheduler implementations
+│   └── utils/              # Utility functions
+├── examples/               # Example configurations and scripts
+│   ├── machine/           # Example machine configs
+│   ├── resources/         # Example resource specs
+│   ├── task/             # Example task definitions
+│   └── dpdisp_run.py     # Complete working example
+├── tests/                # Comprehensive test suite
+├── doc/                  # Sphinx documentation
+└── ci/                   # CI/CD Docker configurations
+```
+
+## Development Notes
+
+- **Virtual environment is pre-configured** - The environment is automatically set up; simply activate with `source .venv/bin/activate`
+- **Always add type hints** - Include proper type annotations in all Python code for better maintainability
+- **Always use conventional commit format** - All commit messages and PR titles must follow the conventional commit specification (e.g., `feat:`, `fix:`, `docs:`, `refactor:`, `test:`, `chore:`)
+- **Test artifacts are gitignored** - Job execution creates temporary files that are automatically excluded
+- **Pre-commit hooks are ready** - Pre-commit is pre-installed and hooks are configured
+- **Multiple execution contexts** - Code supports local execution, SSH remote execution, and various HPC schedulers
+- **Extensive examples** - Use `examples/` directory as reference for proper configuration formats
+
+## Troubleshooting
+
+- **Virtual environment issues:** The virtual environment is pre-created; simply activate with `source .venv/bin/activate`
+- **Development tools:** `pre-commit` and `pyright` are pre-installed and ready to use
+- **Test failures:** Most tests run locally; some require specific HPC environments and will be skipped
+- **Documentation build warnings:** Some warnings about external inventory URLs are expected in sandboxed environments
+- **Pre-commit network issues:** If pre-commit fails with network timeouts, run `ruff check` and `ruff format` directly

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dpdispatcher
-Version: 0.6.11
+Version: 0.6.12
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/ci/ssh/docker-compose.yml

@@ -12,6 +12,17 @@ services:
       - "22"
     volumes:
       - ssh_config:/root/.ssh
+  jumphost:
+    image: takeyamajp/ubuntu-sshd:ubuntu22.04
+    build: .
+    container_name: jumphost
+    hostname: jumphost
+    environment:
+      ROOT_PASSWORD: dpdispatcher
+    expose:
+      - "22"
+    volumes:
+      - ssh_config:/root/.ssh
   test:
     image: python:3.10
     tty: true
@@ -25,6 +36,7 @@ services:
       - ../..:/dpdispatcher
     depends_on:
       - server
+      - jumphost
 
 volumes:
   ssh_config:

dpdispatcher-0.6.12/ci/ssh/start-ssh.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+docker compose up -d --no-build
+
+# Set up SSH keys on server
+docker exec server /bin/bash -c "ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N \"\" && cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"
+docker exec server /bin/bash -c "mkdir -p /dpdispatcher_working"
+docker exec server /bin/bash -c "mkdir -p /tmp/rsync_test"
+
+# Set up SSH keys on jumphost and configure it to access server
+docker exec jumphost /bin/bash -c "ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N \"\" && cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"
+
+# Copy keys between containers to enable jump host functionality
+# Get the public key from jumphost and add it to server's authorized_keys
+docker exec jumphost /bin/bash -c "cat /root/.ssh/id_rsa.pub" | docker exec -i server /bin/bash -c "cat >> /root/.ssh/authorized_keys"
+
+# Get the public key from test (which shares volume with server) and add it to jumphost authorized_keys
+docker exec test /bin/bash -c "cat /root/.ssh/id_rsa.pub" | docker exec -i jumphost /bin/bash -c "cat >> /root/.ssh/authorized_keys"
+
+# Configure SSH client settings for known hosts to avoid host key verification
+docker exec test /bin/bash -c "echo 'StrictHostKeyChecking no' >> /root/.ssh/config && echo 'UserKnownHostsFile /dev/null' >> /root/.ssh/config"
+docker exec jumphost /bin/bash -c "echo 'StrictHostKeyChecking no' >> /root/.ssh/config && echo 'UserKnownHostsFile /dev/null' >> /root/.ssh/config"
+docker exec server /bin/bash -c "echo 'StrictHostKeyChecking no' >> /root/.ssh/config && echo 'UserKnownHostsFile /dev/null' >> /root/.ssh/config"
+
+# Install rsync on all containers
+docker exec test /bin/bash -c "apt-get -y update && apt-get -y install rsync"
+docker exec jumphost /bin/bash -c "apt-get -y update && apt-get -y install rsync"
+docker exec server /bin/bash -c "apt-get -y update && apt-get -y install rsync"

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/doc/context.md

@@ -33,6 +33,28 @@ It's suggested to generate [SSH keys](https://help.ubuntu.com/community/SSH/Open
 
 Note that `SSH` context is [non-login](https://www.gnu.org/software/bash/manual/html_node/Bash-Startup-Files.html), so `bash_profile` files will not be executed outside the submission script.
 
+### SSH Jump Host (Bastion Server)
+
+For connecting to internal servers through a jump host (bastion server), SSH context supports jump host configuration. This allows connecting to internal servers that are not directly accessible from the internet.
+
+Specify the ProxyCommand directly using {dargs:argument}`proxy_command <machine[SSHContext]/remote_profile/proxy_command>`:
+
+```json
+{
+  "context_type": "SSHContext",
+  "remote_profile": {
+    "hostname": "internal-server.company.com",
+    "username": "user",
+    "key_filename": "/path/to/internal_key",
+    "proxy_command": "ssh -W internal-server.company.com:22 -i /path/to/jump_key jumpuser@bastion.company.com"
+  }
+}
+```
+
+**Note**: Unlike OpenSSH, Paramiko's ProxyCommand requires explicit hostname and port instead of `%h:%p` placeholders. The proxy command must specify the actual target hostname and port that matches the `hostname` and `port` in the remote profile.
+
+This configuration establishes the connection path: Local → Jump Host → Target Server.
+
 ## Bohrium
 
 {dargs:argument}`context_type <machine/context_type>`: `Bohrium`

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/dpdispatcher/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.6.11'
-__version_tuple__ = version_tuple = (0, 6, 11)
+__version__ = version = '0.6.12'
+__version_tuple__ = version_tuple = (0, 6, 12)
 
-__commit_id__ = commit_id = 'gce67803b9'
+__commit_id__ = commit_id = 'gfd51e420a'

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/dpdispatcher/contexts/dp_cloud_server_context.py

@@ -161,7 +161,9 @@ class BohriumContext(BaseContext):
         # return oss_task_zip
         # api.upload(self.oss_task_dir, zip_task_file)
 
-    def download(self, submission):
+    def download(
+        self, submission, check_exists=False, mark_failure=True, back_error=False
+    ):
         jobs = submission.belonging_jobs
         job_hashs = {}
         job_infos = {}

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/dpdispatcher/contexts/lazy_local_context.py

@@ -83,7 +83,7 @@ class LazyLocalContext(BaseContext):
 
     def upload(
         self,
-        jobs,
+        submission,
         # local_up_files,
         dereference=True,
     ):
@@ -91,7 +91,7 @@ class LazyLocalContext(BaseContext):
 
     def download(
         self,
-        jobs,
+        submission,
         # remote_down_files,
         check_exists=False,
         mark_failure=True,

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/dpdispatcher/contexts/openapi_context.py

@@ -95,11 +95,11 @@ class OpenAPIContext(BaseContext):
             raise ValueError(
                 "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'"
             )
-        self.client = Bohrium(
+        self.client = Bohrium(  # type: ignore[reportPossiblyUnboundVariable]
             access_key=access_key, project_id=project_id, app_key=app_key
         )
-        self.storage = Tiefblue()
-        self.job = Job(client=self.client)
+        self.storage = Tiefblue()  # type: ignore[reportPossiblyUnboundVariable]
+        self.job = Job(client=self.client)  # type: ignore[reportPossiblyUnboundVariable]
         self.jgid = None
         os.makedirs(DP_CLOUD_SERVER_HOME_DIR, exist_ok=True)
 
@@ -206,7 +206,9 @@ class OpenAPIContext(BaseContext):
         # return oss_task_zip
         # api.upload(self.oss_task_dir, zip_task_file)
 
-    def download(self, submission):
+    def download(
+        self, submission, check_exists=False, mark_failure=True, back_error=False
+    ):
         jobs = submission.belonging_jobs
         job_hashs = {}
         job_infos = {}

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/dpdispatcher/contexts/ssh_context.py

@@ -45,6 +45,7 @@ class SSHSession:
         tar_compress=True,
         look_for_keys=True,
         execute_command=None,
+        proxy_command=None,
     ):
         self.hostname = hostname
         self.username = username
@@ -58,6 +59,7 @@ class SSHSession:
         self.tar_compress = tar_compress
         self.look_for_keys = look_for_keys
         self.execute_command = execute_command
+        self.proxy_command = proxy_command
         self._keyboard_interactive_auth = False
         self._setup_ssh()
 
@@ -142,7 +144,12 @@ class SSHSession:
         # transport = self.ssh.get_transport()
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         sock.settimeout(self.timeout)
-        sock.connect((self.hostname, self.port))
+
+        # Use ProxyCommand if configured (either directly or via jump host parameters)
+        if self.proxy_command is not None:
+            sock = paramiko.ProxyCommand(self.proxy_command)
+        else:
+            sock.connect((self.hostname, self.port))
 
         # Make a Paramiko Transport object using the socket
         ts = paramiko.Transport(sock)
@@ -340,6 +347,9 @@ class SSHSession:
             "enable searching for discoverable private key files in ~/.ssh/"
         )
         doc_execute_command = "execute command after ssh connection is established."
+        doc_proxy_command = (
+            "ProxyCommand to use for SSH connection through intermediate servers."
+        )
         ssh_remote_profile_args = [
             Argument("hostname", str, optional=False, doc=doc_hostname),
             Argument("username", str, optional=False, doc=doc_username),
@@ -388,6 +398,13 @@ class SSHSession:
                 default=None,
                 doc=doc_execute_command,
             ),
+            Argument(
+                "proxy_command",
+                [str, type(None)],
+                optional=True,
+                default=None,
+                doc=doc_proxy_command,
+            ),
         ]
         ssh_remote_profile_format = Argument(
             "ssh_session", dict, ssh_remote_profile_args
@@ -396,23 +413,37 @@ class SSHSession:
 
     def put(self, from_f, to_f):
         if self.rsync_available:
+            # For rsync, we need to use %h:%p placeholders for target host/port
+            proxy_cmd_rsync = None
+            if self.proxy_command is not None:
+                proxy_cmd_rsync = self.proxy_command.replace(
+                    f"{self.hostname}:{self.port}", "%h:%p"
+                )
             return rsync(
                 from_f,
                 self.remote + ":" + to_f,
                 port=self.port,
                 key_filename=self.key_filename,
                 timeout=self.timeout,
+                proxy_command=proxy_cmd_rsync,
             )
         return self.sftp.put(from_f, to_f)
 
     def get(self, from_f, to_f):
         if self.rsync_available:
+            # For rsync, we need to use %h:%p placeholders for target host/port
+            proxy_cmd_rsync = None
+            if self.proxy_command is not None:
+                proxy_cmd_rsync = self.proxy_command.replace(
+                    f"{self.hostname}:{self.port}", "%h:%p"
+                )
             return rsync(
                 self.remote + ":" + from_f,
                 to_f,
                 port=self.port,
                 key_filename=self.key_filename,
                 timeout=self.timeout,
+                proxy_command=proxy_cmd_rsync,
             )
         return self.sftp.get(from_f, to_f)
 
@@ -825,8 +856,8 @@ class SSHContext(BaseContext):
         # print(pid)
         return {"stdin": stdin, "stdout": stdout, "stderr": stderr}
 
-    def check_finish(self, cmd_pipes):
-        return cmd_pipes["stdout"].channel.exit_status_ready()
+    def check_finish(self, proc):
+        return proc["stdout"].channel.exit_status_ready()
 
     def get_return(self, cmd_pipes):
         if not self.check_finish(cmd_pipes):
@@ -888,11 +919,11 @@ class SSHContext(BaseContext):
         # local tar
         if os.path.isfile(os.path.join(self.local_root, of)):
             os.remove(os.path.join(self.local_root, of))
-        with tarfile.open(
+        with tarfile.open(  # type: ignore[reportCallIssue, reportArgumentType]
             os.path.join(self.local_root, of),
-            tarfile_mode,
-            dereference=dereference,
-            **kwargs,
+            mode=tarfile_mode,  # type: ignore[reportArgumentType]
+            dereference=dereference,  # type: ignore[reportArgumentType]
+            **kwargs,  # type: ignore[reportArgumentType]
         ) as tar:
             # avoid compressing duplicated files or directories
             for ii in set(files):

{dpdispatcher-0.6.11 → dpdispatcher-0.6.12}/dpdispatcher/machine.py

@@ -227,7 +227,7 @@ class Machine(metaclass=ABCMeta):
         return if_recover
 
     @abstractmethod
-    def check_finish_tag(self, **kwargs):
+    def check_finish_tag(self, job):
         raise NotImplementedError(
             "abstract method check_finish_tag should be implemented by derived class"
         )