dpdispatcher 0.6.4__tar.gz → 0.6.6__tar.gz

dpdispatcher-0.6.6/.git_archival.txt

@@ -0,0 +1,3 @@
+node: $Format:%H$
+node-date: $Format:%cI$
+describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$

dpdispatcher-0.6.6/.gitattributes

@@ -0,0 +1 @@
+.git_archival.txt  export-subst

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/.github/workflows/ci-docker.yml

@@ -15,7 +15,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Log in to Docker Hub
-      uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/.github/workflows/machines.yml

@@ -17,3 +17,5 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: ./ci/${{ matrix.machine }}.sh
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/.github/workflows/pyright.yml

@@ -11,7 +11,8 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: '3.11'
-    - run: pip install -e .[cloudserver,gui]
-    - uses: jakebailey/pyright-action@v1
+    - run: pip install uv
+    - run: uv pip install --system -e .[cloudserver,gui]
+    - uses: jakebailey/pyright-action@v2
       with:
         version: 1.1.308

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/.github/workflows/test-bohrium.yml

@@ -20,7 +20,8 @@ jobs:
       with:
         python-version: '3.12'
         cache: 'pip'
-    - run: pip install .[bohrium] coverage
+    - run: pip install uv
+    - run: uv pip install --system .[bohrium] coverage
     - name: Test
       run: coverage run --source=./dpdispatcher -m unittest -v tests/test_run_submission_bohrium.py && coverage report
       env:
@@ -29,7 +30,9 @@ jobs:
         BOHRIUM_PASSWORD: ${{ secrets.BOHRIUM_PASSWORD }}
         BOHRIUM_PROJECT_ID: ${{ secrets.BOHRIUM_PROJECT_ID }}
         BOHRIUM_ACCESS_KEY: ${{ secrets.BOHRIUM_ACCESS_KEY }}
-    - uses: codecov/codecov-action@v3
+    - uses: codecov/codecov-action@v4
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   remove_label:
     permissions:
       contents: read

dpdispatcher-0.6.6/.github/workflows/test.yml

@@ -0,0 +1,56 @@
+name: Python package
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  test:
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      matrix:
+        python-version:
+          - 3.7
+          - 3.9
+          - '3.10'
+          - '3.11'
+          - '3.12'
+        platform:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+        exclude:  # Apple Silicon ARM64 does not support Python < v3.8
+          - python-version: "3.7"
+            platform: macos-latest
+        include:  # So run those legacy versions on Intel CPUs
+          - python-version: "3.7"
+            platform: macos-13
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      if: matrix.platform != 'windows-latest'
+    - run: powershell -c "irm https://astral.sh/uv/install.ps1 | iex"
+      if: matrix.platform == 'windows-latest'
+    - run: uv pip install --system .[test] coverage
+    - name: Test
+      run: |
+        python -m coverage run -p --source=./dpdispatcher -m unittest -v
+        python -m coverage run -p --source=./dpdispatcher -m dpdispatcher -h
+        python -m coverage combine
+        python -m coverage report
+    - uses: codecov/codecov-action@v4
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+  pass:
+    needs: [test]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+    - name: Decide whether the needed jobs succeeded or failed
+      uses: re-actors/alls-green@release/v1
+      with:
+        jobs: ${{ toJSON(needs) }}

dpdispatcher-0.6.6/.pre-commit-config.yaml

@@ -0,0 +1,44 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: "^tests/"
+      - id: end-of-file-fixer
+        exclude: "^tests/"
+      - id: check-yaml
+        exclude: "^conda/"
+      - id: check-json
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-toml
+  # Python
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.5.4
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
+  # numpydoc
+  - repo: https://github.com/Carreau/velin
+    rev: 0.0.12
+    hooks:
+      - id: velin
+        args: ["--write"]
+  # Python inside docs
+  - repo: https://github.com/asottile/blacken-docs
+    rev: 1.18.0
+    hooks:
+      - id: blacken-docs
+  # markdown, yaml
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v4.0.0-alpha.8
+    hooks:
+      - id: prettier
+        types_or: [markdown, yaml]
+        # workflow files cannot be modified by pre-commit.ci
+        exclude: ^(\.github/workflows|conda)

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/.readthedocs.yaml

@@ -10,15 +10,14 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.10"
-
+  jobs:
+    post_create_environment:
+      - pip install uv
+    post_install:
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH uv pip install .[docs]
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
-   configuration: doc/conf.py
+  configuration: doc/conf.py
 
 # If using Sphinx, optionally build your docs in additional formats such as PDF
 formats: all
-
-# Optionally declare the Python requirements required to build your docs
-python:
-   install:
-   - requirements: doc/requirements.txt

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.6.4
+Version: 0.6.6
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE
@@ -172,7 +172,7 @@ License:                    GNU LESSER GENERAL PUBLIC LICENSE
 Project-URL: Homepage, https://github.com/deepmodeling/dpdispatcher
 Project-URL: documentation, https://docs.deepmodeling.com/projects/dpdispatcher
 Project-URL: repository, https://github.com/deepmodeling/dpdispatcher
-Keywords: dispatcher,hpc,slurm,lsf,pbs,ssh
+Keywords: dispatcher,hpc,slurm,lsf,pbs,ssh,jh_unischeduler
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
@@ -191,6 +191,7 @@ Requires-Dist: requests
 Requires-Dist: tqdm>=4.9.0
 Requires-Dist: typing_extensions; python_version < "3.7"
 Requires-Dist: pyyaml
+Requires-Dist: tomli>=1.1.0; python_version < "3.11"
 Provides-Extra: docs
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: myst-parser; extra == "docs"
@@ -198,7 +199,7 @@ Requires-Dist: sphinx_rtd_theme>=1.0.0rc1; extra == "docs"
 Requires-Dist: numpydoc; extra == "docs"
 Requires-Dist: deepmodeling_sphinx>=0.1.1; extra == "docs"
 Requires-Dist: dargs>=0.3.1; extra == "docs"
-Requires-Dist: sphinx-argparse; extra == "docs"
+Requires-Dist: sphinx-argparse<0.5.0; extra == "docs"
 Provides-Extra: cloudserver
 Requires-Dist: oss2; extra == "cloudserver"
 Requires-Dist: tqdm; extra == "cloudserver"
@@ -250,4 +251,4 @@ See [Contributing Guide](CONTRIBUTING.md) to become a contributor! 🤓
 
 ## References
 
-DPDispatcher is derivated from the [DP-GEN](https://github.com/deepmodeling/dpgen) package. To mention DPDispatcher in a scholarly publication, please read Section 3.3 in the [DP-GEN paper](https://doi.org/10.1016/j.cpc.2020.107206).
+DPDispatcher is derived from the [DP-GEN](https://github.com/deepmodeling/dpgen) package. To mention DPDispatcher in a scholarly publication, please read Section 3.3 in the [DP-GEN paper](https://doi.org/10.1016/j.cpc.2020.107206).

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/README.md

@@ -36,4 +36,4 @@ See [Contributing Guide](CONTRIBUTING.md) to become a contributor! 🤓
 
 ## References
 
-DPDispatcher is derivated from the [DP-GEN](https://github.com/deepmodeling/dpgen) package. To mention DPDispatcher in a scholarly publication, please read Section 3.3 in the [DP-GEN paper](https://doi.org/10.1016/j.cpc.2020.107206).
+DPDispatcher is derived from the [DP-GEN](https://github.com/deepmodeling/dpgen) package. To mention DPDispatcher in a scholarly publication, please read Section 3.3 in the [DP-GEN paper](https://doi.org/10.1016/j.cpc.2020.107206).

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/ci/pbs/docker-compose.yml

@@ -1,7 +1,6 @@
 version: "2"
 
 services:
-
   master:
     image: daskdev/dask-jobqueue:pbs
     build: .

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/ci/pbs.sh

@@ -9,6 +9,6 @@ cd -
 docker exec pbs_master /bin/bash -c "chmod -R 777 /shared_space"
 docker exec pbs_master /bin/bash -c "chown -R pbsuser:pbsuser /home/pbsuser"
 
-docker exec pbs_master /bin/bash -c "cd /dpdispatcher && pip install .[test] coverage && chown -R pbsuser ."
+docker exec pbs_master /bin/bash -c "cd /dpdispatcher && pip install uv && uv pip install --system .[test] coverage && chown -R pbsuser ."
 docker exec -u pbsuser pbs_master /bin/bash -c "cd /dpdispatcher && coverage run --source=./dpdispatcher -m unittest -v && coverage report"
 docker exec -u pbsuser --env-file <(env | grep GITHUB) pbs_master /bin/bash -c "cd /dpdispatcher && curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov"

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/ci/slurm/docker-compose.yml

@@ -40,7 +40,7 @@ services:
     hostname: slurmctld
     environment:
       CI_SHARED_SPACE: /data
-      DPDISPATCHER_TEST : slurm
+      DPDISPATCHER_TEST: slurm
     volumes:
       - etc_munge:/etc/munge
       - etc_slurm:/etc/slurm

dpdispatcher-0.6.6/ci/slurm.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+cd ./ci/slurm
+docker-compose pull
+./start-slurm.sh
+cd -
+
+docker exec slurmctld /bin/bash -c "cd dpdispatcher && pip install uv && uv pip install --system .[test] coverage && coverage run --source=./dpdispatcher -m unittest -v && coverage report"
+docker exec --env-file <(env | grep -e GITHUB -e CODECOV) slurmctld /bin/bash -c "cd dpdispatcher && curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov"

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/ci/ssh/docker-compose.yml

@@ -19,7 +19,7 @@ services:
     container_name: test
     hostname: test
     environment:
-      DPDISPATCHER_TEST : ssh
+      DPDISPATCHER_TEST: ssh
     volumes:
       - ssh_config:/root/.ssh
       - ../..:/dpdispatcher

dpdispatcher-0.6.6/ci/ssh.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+cd ./ci/ssh
+docker-compose pull
+./start-ssh.sh
+cd -
+
+docker exec test /bin/bash -c "cd /dpdispatcher && pip install uv && uv pip install --system .[test] coverage && coverage run --source=./dpdispatcher -m unittest -v && coverage report"
+docker exec --env-file <(env | grep -e GITHUB -e CODECOV) test /bin/bash -c "cd /dpdispatcher && curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov"

dpdispatcher-0.6.6/ci/ssh_rsync.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+
+cd ./ci/ssh
+docker-compose pull
+./start-ssh.sh
+cd -
+
+# install rsync
+docker exec server /bin/bash -c "apt-get -y update && apt-get -y install rsync"
+docker exec test /bin/bash -c "apt-get -y update && apt-get -y install rsync"
+
+docker exec test /bin/bash -c "cd /dpdispatcher && pip install uv && uv pip install --system .[test] coverage && coverage run --source=./dpdispatcher -m unittest -v && coverage report"
+docker exec --env-file <(env | grep -e GITHUB -e CODECOV) test /bin/bash -c "cd /dpdispatcher && curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov"

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/doc/batch.md

@@ -1,85 +1,91 @@
-# Supported batch job systems
-
-Batch job system is a system to process batch jobs.
-One needs to set {dargs:argument}`batch_type <machine/batch_type>` to one of the following values:
-
-## Bash
-
-{dargs:argument}`batch_type <resources/batch_type>`: `Shell`
-
-When {dargs:argument}`batch_type <resources/batch_type>` is set to `Shell`, dpdispatcher will generate a bash script to process jobs.
-No extra packages are required for `Shell`.
-
-Due to lack of scheduling system, `Shell` runs all jobs at the same time.
-To avoid running multiple jobs at the same time, one could set {dargs:argument}`group_size <resources/group_size>` to `0` (means infinity) to generate only one job with multiple tasks.
-
-## Slurm
-
-{dargs:argument}`batch_type <resources/batch_type>`: `Slurm`, `SlurmJobArray`
-
-[Slurm](https://slurm.schedmd.com/) is a job scheduling system used by lots of HPCs.
-One needs to make sure slurm has been setup in the remote server and the related environment is activated.
-
-When `SlurmJobArray` is used, dpdispatcher submits Slurm jobs with [job arrays](https://slurm.schedmd.com/job_array.html).
-In this way, several dpdispatcher {class}`task <dpdispatcher.submission.Task>`s map to a Slurm job and a dpdispatcher {class}`job <dpdispatcher.submission.Job>` maps to a Slurm job array.
-Millions of Slurm jobs can be submitted quickly and Slurm can execute all Slurm jobs at the same time.
-One can use {dargs:argument}`group_size <resources/group_size>` and {dargs:argument}`slurm_job_size <resources[SlurmJobArray]/kwargs/slurm_job_size>` to control how many Slurm jobs are contained in a Slurm job array.
-
-## OpenPBS or PBSPro
-
-{dargs:argument}`batch_type <resources/batch_type>`: `PBS`
-
-[OpenPBS](https://www.openpbs.org/) is an open-source job scheduling of the Linux Foundation and [PBS Profession](https://www.altair.com/pbs-professional/) is its commercial solution.
-One needs to make sure OpenPBS has been setup in the remote server and the related environment is activated.
-
-Note that do not use `PBS` for Torque.
-
-## TORQUE
-
-{dargs:argument}`batch_type <resources/batch_type>`: `Torque`
-
-The [Terascale Open-source Resource and QUEue Manager (TORQUE)](https://adaptivecomputing.com/cherry-services/torque-resource-manager/) is a distributed resource manager based on standard OpenPBS.
-However, not all OpenPBS flags are still supported in TORQUE.
-One needs to make sure TORQUE has been setup in the remote server and the related environment is activated.
-
-## LSF
-
-{dargs:argument}`batch_type <resources/batch_type>`: `LSF`
-
-[IBM Spectrum LSF Suites](https://www.ibm.com/products/hpc-workload-management) is a comprehensive workload management solution used by HPCs.
-One needs to make sure LSF has been setup in the remote server and the related environment is activated.
-
-## Bohrium
-
-{dargs:argument}`batch_type <resources/batch_type>`: `Bohrium`
-
-Bohrium is the cloud platform for scientific computing.
-Read Bohrium documentation for details.
-
-## DistributedShell
-
-{dargs:argument}`batch_type <resources/batch_type>`: `DistributedShell`
-
-`DistributedShell` is used to submit yarn jobs.
-Read [Support DPDispatcher on Yarn](dpdispatcher_on_yarn.md) for details.
-
-## Fugaku
-
-{dargs:argument}`batch_type <resources/batch_type>`: `Fugaku`
-
-[Fujitsu cloud service](https://doc.cloud.global.fujitsu.com/lib/common/jp/hpc-user-manual/) is a job scheduling system used by Fujitsu's HPCs such as Fugaku, ITO and K computer. It should be noted that although the same job scheduling system is used, there are some differences in the details, Fagaku class cannot be directly used for other HPCs.
-
-Read Fujitsu cloud service documentation for details.
-
-
-## OpenAPI
-
-{dargs:argument}`batcy_type <resources/batch_type>`: `OpenAPI`
-OpenAPI is a new way to submit jobs to Bohrium. It using [AccessKey](https://bohrium.dp.tech/personal/setting) instead of username and password. Read Bohrium documentation for details.
-
-
-## SGE
-
-{dargs:argument}`batch_type <resources/batch_type>`: `SGE`
-
-The [Sun Grid Engine (SGE) scheduler](https://gridscheduler.sourceforge.net) is a batch-queueing system distributed resource management. The commands and flags of SGE share a lot similarity with PBS except when checking job status. Use this argument if one is submitting job to SGE based batch system.
+# Supported batch job systems
+
+Batch job system is a system to process batch jobs.
+One needs to set {dargs:argument}`batch_type <machine/batch_type>` to one of the following values:
+
+## Bash
+
+{dargs:argument}`batch_type <resources/batch_type>`: `Shell`
+
+When {dargs:argument}`batch_type <resources/batch_type>` is set to `Shell`, dpdispatcher will generate a bash script to process jobs.
+No extra packages are required for `Shell`.
+
+Due to lack of scheduling system, `Shell` runs all jobs at the same time.
+To avoid running multiple jobs at the same time, one could set {dargs:argument}`group_size <resources/group_size>` to `0` (means infinity) to generate only one job with multiple tasks.
+
+## Slurm
+
+{dargs:argument}`batch_type <resources/batch_type>`: `Slurm`, `SlurmJobArray`
+
+[Slurm](https://slurm.schedmd.com/) is a job scheduling system used by lots of HPCs.
+One needs to make sure slurm has been set up in the remote server and the related environment is activated.
+
+When `SlurmJobArray` is used, dpdispatcher submits Slurm jobs with [job arrays](https://slurm.schedmd.com/job_array.html).
+In this way, several dpdispatcher {class}`task <dpdispatcher.submission.Task>`s map to a Slurm job and a dpdispatcher {class}`job <dpdispatcher.submission.Job>` maps to a Slurm job array.
+Millions of Slurm jobs can be submitted quickly and Slurm can execute all Slurm jobs at the same time.
+One can use {dargs:argument}`group_size <resources/group_size>` and {dargs:argument}`slurm_job_size <resources[SlurmJobArray]/kwargs/slurm_job_size>` to control how many Slurm jobs are contained in a Slurm job array.
+
+## OpenPBS or PBSPro
+
+{dargs:argument}`batch_type <resources/batch_type>`: `PBS`
+
+[OpenPBS](https://www.openpbs.org/) is an open-source job scheduling of the Linux Foundation and [PBS Profession](https://www.altair.com/pbs-professional/) is its commercial solution.
+One needs to make sure OpenPBS has been set up in the remote server and the related environment is activated.
+
+Note that do not use `PBS` for Torque.
+
+## TORQUE
+
+{dargs:argument}`batch_type <resources/batch_type>`: `Torque`
+
+The [Terascale Open-source Resource and QUEue Manager (TORQUE)](https://adaptivecomputing.com/cherry-services/torque-resource-manager/) is a distributed resource manager based on standard OpenPBS.
+However, not all OpenPBS flags are still supported in TORQUE.
+One needs to make sure TORQUE has been set up in the remote server and the related environment is activated.
+
+## LSF
+
+{dargs:argument}`batch_type <resources/batch_type>`: `LSF`
+
+[IBM Spectrum LSF Suites](https://www.ibm.com/products/hpc-workload-management) is a comprehensive workload management solution used by HPCs.
+One needs to make sure LSF has been set up in the remote server and the related environment is activated.
+
+## JH UniScheduler
+
+{dargs:argument}`batch_type <resources/batch_type>`: `JH_UniScheduler`
+
+[JH UniScheduler](http://www.jhinno.com/m/custom_case_05.html) was developed by JHINNO company and uses "jsub" to submit tasks.
+Its overall architecture is similar to that of IBM's LSF. However, there are still some differences between them. One needs to
+make sure JH UniScheduler has been set up in the remote server and the related environment is activated.
+
+## Bohrium
+
+{dargs:argument}`batch_type <resources/batch_type>`: `Bohrium`
+
+Bohrium is the cloud platform for scientific computing.
+Read Bohrium documentation for details.
+
+## DistributedShell
+
+{dargs:argument}`batch_type <resources/batch_type>`: `DistributedShell`
+
+`DistributedShell` is used to submit yarn jobs.
+Read [Support DPDispatcher on Yarn](dpdispatcher_on_yarn.md) for details.
+
+## Fugaku
+
+{dargs:argument}`batch_type <resources/batch_type>`: `Fugaku`
+
+[Fujitsu cloud service](https://doc.cloud.global.fujitsu.com/lib/common/jp/hpc-user-manual/) is a job scheduling system used by Fujitsu's HPCs such as Fugaku, ITO and K computer. It should be noted that although the same job scheduling system is used, there are some differences in the details, Fagaku class cannot be directly used for other HPCs.
+
+Read Fujitsu cloud service documentation for details.
+
+## OpenAPI
+
+{dargs:argument}`batcy_type <resources/batch_type>`: `OpenAPI`
+OpenAPI is a new way to submit jobs to Bohrium. It is using [AccessKey](https://bohrium.dp.tech/personal/setting) instead of username and password. Read Bohrium documentation for details.
+
+## SGE
+
+{dargs:argument}`batch_type <resources/batch_type>`: `SGE`
+
+The [Sun Grid Engine (SGE) scheduler](https://gridscheduler.sourceforge.net) is a batch-queueing system distributed resource management. The commands and flags of SGE share a lot of similarity with PBS except when checking job status. Use this argument if one is submitting job to an SGE-based batch system.

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/doc/context.md

@@ -1,52 +1,55 @@
-# Supported contexts
-
-Context is the way to connect to the remote server.
-One needs to set {dargs:argument}`context_type <machine/context_type>` to one of the following values:
-
-## LazyLocal
-
-{dargs:argument}`context_type <machine/context_type>`: `LazyLocal`
-
-`LazyLocal` directly runs jobs in the local server and local directory.
-
-## Local
-
-{dargs:argument}`context_type <machine/context_type>`: `Local`
-
-`Local` runs jobs in the local server, but in a different directory.
-Files will be copied to the remote directory before jobs start and copied back after jobs finish.
-
-## SSH
-
-{dargs:argument}`context_type <machine/context_type>`: `SSH`
-
-`SSH` runs jobs in a remote server.
-Files will be copied to the remote directory via SSH channels before jobs start and copied back after jobs finish.
-To use SSH, one needs to provide necessary parameters in {dargs:argument}`remote_profile <machine[SSHContext]/remote_profile>`, such as {dargs:argument}`username <machine[SSHContext]/remote_profile/hostname>` and {dargs:argument}`hostname <username[SSHContext]/remote_profile/hostname>`.
-
-It's suggested to generate [SSH keys](https://help.ubuntu.com/community/SSH/OpenSSH/Keys) and transfer the public key to the remote server in advance, which is more secure than password authentication.
-
-Note that `SSH` context is [non-login](https://www.gnu.org/software/bash/manual/html_node/Bash-Startup-Files.html), so `bash_profile` files will not be executed.
-
-## Bohrium
-
-{dargs:argument}`context_type <machine/context_type>`: `Bohrium`
-
-Bohrium is the cloud platform for scientific computing.
-Read Bohrium documentation for details.
-To use Bohrium, one needs to provide necessary parameters in {dargs:argument}`remote_profile <machine[BohriumContext]/remote_profile>`.
-
-## HDFS
-
-{dargs:argument}`context_type <machine/context_type>`: `HDFS`
-
-The Hadoop Distributed File System (HDFS) is a distributed file system.
-Read [Support DPDispatcher on Yarn](dpdispatcher_on_yarn.md) for details.
-
-
-## OpenAPI
-
-{dargs:argument}`context_type <machine/context_type>`: `OpenAPI`
-
-OpenAPI is a new way to submit jobs to Bohrium. It using [AccessKey](https://bohrium.dp.tech/personal/setting) instead of username and password. Read Bohrium documentation for details.
+# Supported contexts
+
+Context is the way to connect to the remote server.
+One needs to set {dargs:argument}`context_type <machine/context_type>` to one of the following values:
+
+## LazyLocal
+
+{dargs:argument}`context_type <machine/context_type>`: `LazyLocal`
+
+`LazyLocal` directly runs jobs in the local server and local directory.
+
+Since [`bash -l`](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) is used in the shebang line of the submission scripts, the [login shell startup files](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) will be executed, potentially overriding the current environmental variables. Therefore, it's advisable to explicitly set the environmental variables using {dargs:argument}`envs <resources/envs>` or {dargs:argument}`source_list <resources/source_list>`.
+
+## Local
+
+{dargs:argument}`context_type <machine/context_type>`: `Local`
+
+`Local` runs jobs in the local server, but in a different directory.
+Files will be copied to the remote directory before jobs start and copied back after jobs finish.
+
+Since [`bash -l`](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) is used in the shebang line of the submission scripts, the [login shell startup files](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) will be executed, potentially overriding the current environmental variables. Therefore, it's advisable to explicitly set the environmental variables using {dargs:argument}`envs <resources/envs>` or {dargs:argument}`source_list <resources/source_list>`.
+
+## SSH
+
+{dargs:argument}`context_type <machine/context_type>`: `SSH`
+
+`SSH` runs jobs in a remote server.
+Files will be copied to the remote directory via SSH channels before jobs start and copied back after jobs finish.
+To use SSH, one needs to provide necessary parameters in {dargs:argument}`remote_profile <machine[SSHContext]/remote_profile>`, such as {dargs:argument}`username <machine[SSHContext]/remote_profile/hostname>` and {dargs:argument}`hostname <username[SSHContext]/remote_profile/hostname>`.
+
+It's suggested to generate [SSH keys](https://help.ubuntu.com/community/SSH/OpenSSH/Keys) and transfer the public key to the remote server in advance, which is more secure than password authentication.
+
+Note that `SSH` context is [non-login](https://www.gnu.org/software/bash/manual/html_node/Bash-Startup-Files.html), so `bash_profile` files will not be executed outside the submission script.
+
+## Bohrium
+
+{dargs:argument}`context_type <machine/context_type>`: `Bohrium`
+
+Bohrium is the cloud platform for scientific computing.
+Read Bohrium documentation for details.
+To use Bohrium, one needs to provide necessary parameters in {dargs:argument}`remote_profile <machine[BohriumContext]/remote_profile>`.
+
+## HDFS
+
+{dargs:argument}`context_type <machine/context_type>`: `HDFS`
+
+The Hadoop Distributed File System (HDFS) is a distributed file system.
+Read [Support DPDispatcher on Yarn](dpdispatcher_on_yarn.md) for details.
+
+## OpenAPI
+
+{dargs:argument}`context_type <machine/context_type>`: `OpenAPI`
+
+OpenAPI is a new way to submit jobs to Bohrium. It using [AccessKey](https://bohrium.dp.tech/personal/setting) instead of username and password. Read Bohrium documentation for details.
 To use OpenAPI, one needs to provide necessary parameters in {dargs:argument}`remote_profile <machine[OpenAPIContext]/remote_profile>`.

{dpdispatcher-0.6.4 → dpdispatcher-0.6.6}/doc/dpdispatcher_on_yarn.md

@@ -1,14 +1,19 @@
 # Support DPDispatcher on Yarn
+
 ## Background
+
 Currently, DPGen(or other DP softwares) supports for HPC systems like Slurm, PBS, LSF and cloud machines. In order to run DPGen jobs on ByteDance internal platform, we need to extend it to support yarn resources. Hadoop Ecosystem is a very commonly used platform to process the big data, and in the process of developing the new interface, we found it can be implemented by only using hadoop opensource components. So for the convenience of the masses, we decided to contribute the codes to opensource community.
 
 ## Design
+
 We use DistributedShell and HDFS to implement it. The control flow shows as follows:
 ![image](https://github.com/shazj99/dpdispatcher/blob/yarn/doc/dpgen_yarn.jpg?raw=true)
+
 - Use DistributedShell to submit yarn jobs. It contains generating shell script and submitting it to yarn queues.
 - Use HDFS to save input files and output results. For performance reasons, we choose to pack forward files to a tar.gz file and upload it to HDFS directory. Accordingly, the task will download the tar file before running and upload result tar file to HDFS after it has done.
 
 ## Implement
+
 We only need to add two Class which are HDFSContext and DistributedShell:
 
 ```
@@ -106,6 +111,7 @@ class DistributedShell(Machine):
 ```
 
 The following is an example of generated shell script. It will be executed in a yarn container:
+
 ```
 #!/bin/bash
 
@@ -146,6 +152,7 @@ hadoop fs -put -f uuid_download.tar.gz /root/uuid/sys-0001-0015
 ## mark the job has finished
 hadoop fs -touchz /root/uuid/uuid_tag_finished
 ```
+
 An example of machine.json is as follows, whose batch_type is `DistributedShell`，and context_type is `HDFSContext`:
 
 ```