dpdispatcher 0.5.11__tar.gz → 0.6.1__tar.gz

dpdispatcher-0.6.1/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/.github/workflows/ci-docker.yml

@@ -2,8 +2,8 @@ name: Build docker image and push to Docker Hub
 
 on:
   push:
-    branches:
-      - master
+    tags:
+      - 'v*'
 
 jobs:
   build-n-push:
@@ -12,21 +12,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out the repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Log in to Docker Hub
-      uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+      uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
 
     - name: Set up Docker Buildx
       id: buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
 
     - name: Build and push
       run: |
-        docker buildx build --platform linux/arm64,linux/amd64 -t dptechnology/dpdispatcher:latest --push .
+        docker buildx build --platform linux/arm64,linux/amd64 -t dptechnology/dpdispatcher:${{ github.ref_name }} -t dptechnology/dpdispatcher:latest --push .

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/.github/workflows/machines.yml

@@ -15,5 +15,5 @@ jobs:
           - ssh
           - ssh_rsync
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - run: ./ci/${{ matrix.machine }}.sh

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/.github/workflows/publish_conda.yml

@@ -9,7 +9,7 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: publish-to-conda
       uses: felix5572/conda-publish-action@v1.9
       with:

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/.github/workflows/pyright.yml

@@ -7,11 +7,11 @@ jobs:
     name: pyright
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: '3.11'
-    - run: pip install -e .[cloudserver]
+    - run: pip install -e .[cloudserver,gui]
     - uses: jakebailey/pyright-action@v1
       with:
         version: 1.1.308

dpdispatcher-0.6.1/.github/workflows/release.yml

@@ -0,0 +1,24 @@
+on: [push, pull_request]
+name: Release to pypi
+jobs:
+  release-to-pypi:
+    name: Release to pypi
+    runs-on: ubuntu-latest
+    permissions:
+        # IMPORTANT: this permission is mandatory for trusted publishing
+        id-token: write
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+        architecture: x64
+    - name: Install dependencies
+      run: python -m pip install build
+    - run: python -m build
+    - name: Publish a Python distribution to PyPI
+      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        verbose: true

dpdispatcher-0.6.1/.github/workflows/test-bohrium.yml

@@ -0,0 +1,44 @@
+name: Test Bohrium
+
+on:
+  push:
+  pull_request_target:
+    types:
+      - "labeled"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    environment: bohrium
+    if: github.repository_owner == 'deepmodeling' && (github.event.label.name == 'Test Bohrium' || github.event_name == 'push')
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: "${{ github.event.pull_request.merge_commit_sha }}"
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+        cache: 'pip'
+    - run: pip install .[bohrium] coverage
+    - name: Test
+      run: coverage run --source=./dpdispatcher -m unittest -v tests/test_run_submission_bohrium.py && coverage report
+      env:
+        DPDISPATCHER_TEST: bohrium
+        BOHRIUM_EMAIL: ${{ secrets.BOHRIUM_EMAIL }}
+        BOHRIUM_PASSWORD: ${{ secrets.BOHRIUM_PASSWORD }}
+        BOHRIUM_PROJECT_ID: ${{ secrets.BOHRIUM_PROJECT_ID }}
+        BOHRIUM_ACCESS_KEY: ${{ secrets.BOHRIUM_ACCESS_KEY }}
+    - uses: codecov/codecov-action@v3
+  remove_label:
+    permissions:
+      contents: read
+      pull-requests: write
+    # so one can re-trigger the workflow without manually removing the label
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'deepmodeling' && github.event.label.name == 'Test Bohrium'
+    steps:
+    - uses: actions-ecosystem/action-remove-labels@v1
+      with:
+        labels: Test Bohrium
+        number: ${{ github.event.pull_request.number }}

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/.github/workflows/test.yml

@@ -11,23 +11,27 @@ jobs:
       matrix:
         python-version:
           - 3.7
-          - 3.8
           - 3.9
           - '3.10'
           - '3.11'
+          - '3.12'
         platform:
           - ubuntu-latest
           - macos-latest
           - windows-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - run: pip install .[test] coverage
     - name: Test
-      run: coverage run --source=./dpdispatcher -m unittest -v && coverage report
+      run: |
+        coverage run -p --source=./dpdispatcher -m unittest -v
+        coverage run -p --source=./dpdispatcher -m dpdispatcher -h
+        coverage combine
+        coverage report
     - uses: codecov/codecov-action@v3
   pass:
     needs: [test]

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     -   id: trailing-whitespace
         exclude: "^tests/"
@@ -16,16 +16,13 @@ repos:
     -   id: check-symlinks
     -   id: check-toml
 # Python
--   repo: https://github.com/psf/black
-    rev: 23.7.0
-    hooks:
-    -   id: black-jupyter
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.0.285
+    rev: v0.1.5
     hooks:
     - id: ruff
       args: ["--fix"]
+    - id: ruff-format
 # numpydoc
 -   repo: https://github.com/Carreau/velin
     rev: 0.0.12

dpdispatcher-0.6.1/.readthedocs.yaml

@@ -0,0 +1,24 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: doc/conf.py
+
+# If using Sphinx, optionally build your docs in additional formats such as PDF
+formats: all
+
+# Optionally declare the Python requirements required to build your docs
+python:
+   install:
+   - requirements: doc/requirements.txt

dpdispatcher-0.6.1/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.12 AS compile-image
+
+RUN python -m venv /opt/venv
+# Make sure we use the virtualenv
+ENV PATH="/opt/venv/bin:$PATH"
+
+WORKDIR /data/dpdispatcher
+COPY ./ ./
+RUN pip install .[bohrium]
+
+FROM python:3.12 AS build-image
+COPY --from=compile-image /opt/venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+CMD ["/bin/bash"]

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.5.11
+Version: 0.6.1
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE
@@ -184,11 +184,33 @@ Classifier: Operating System :: Microsoft :: Windows
 Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: paramiko
+Requires-Dist: dargs>=0.4.1
+Requires-Dist: requests
+Requires-Dist: tqdm>=4.9.0
+Requires-Dist: typing_extensions; python_version < "3.7"
+Requires-Dist: pyyaml
 Provides-Extra: docs
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: myst-parser; extra == "docs"
+Requires-Dist: sphinx_rtd_theme>=1.0.0rc1; extra == "docs"
+Requires-Dist: numpydoc; extra == "docs"
+Requires-Dist: deepmodeling_sphinx>=0.1.1; extra == "docs"
+Requires-Dist: dargs>=0.3.1; extra == "docs"
+Requires-Dist: sphinx-argparse; extra == "docs"
 Provides-Extra: cloudserver
+Requires-Dist: oss2; extra == "cloudserver"
+Requires-Dist: tqdm; extra == "cloudserver"
+Requires-Dist: bohrium-sdk; extra == "cloudserver"
 Provides-Extra: bohrium
+Requires-Dist: oss2; extra == "bohrium"
+Requires-Dist: tqdm; extra == "bohrium"
+Requires-Dist: bohrium-sdk; extra == "bohrium"
+Provides-Extra: gui
+Requires-Dist: dpgui; extra == "gui"
 Provides-Extra: test
-License-File: LICENSE
+Requires-Dist: dpgui; extra == "test"
 
 # DPDispatcher
 

dpdispatcher-0.6.1/doc/cli.rst

@@ -0,0 +1,9 @@
+.. _cli:
+
+Command line interface
+======================
+
+.. argparse::
+   :module: dpdispatcher.dpdisp
+   :func: main_parser
+   :prog: dpdisp

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/doc/conf.py

@@ -38,6 +38,7 @@ extensions = [
     "sphinx.ext.intersphinx",
     "numpydoc",
     "sphinx.ext.autosummary",
+    "sphinxarg.ext",
 ]
 
 # Add any paths that contain templates here, relative to this directory.

dpdispatcher-0.6.1/doc/examples/template.md

@@ -0,0 +1,16 @@
+# Customizing the submission script header
+
+When submitting jobs to some clusters, such as the [Tiger Cluster](https://researchcomputing.princeton.edu/systems/tiger) at Princeton University, the Slurm header is quite different from the standard one. In this case, DPDispatcher allows users to customize the templates by setting {dargs:argument}`strategy/customized_script_header_template_file <resources/strategy/customized_script_header_template_file>` to a template file:
+
+```{literalinclude} ../../examples/resources/tiger.json
+:language: json
+:linenos:
+```
+
+`template.slurm` is the template file, where {meth}`str.format` is used to format the template with [Resources Parameters](resources):
+
+```{literalinclude} ../../examples/resources/template.slurm
+:linenos:
+```
+
+See [Python Format String Syntax](https://docs.python.org/3/library/string.html#formatstrings) for how to insert parameters inside the template.

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/doc/index.rst

@@ -22,13 +22,17 @@ DPDispatcher will monitor (poke) until these jobs finish and download the result
    machine
    resources
    task
+   cli
    api/api
 
 .. toctree::
    :caption: Examples
    :glob:
 
-   examples/*
+   examples/expanse
+   examples/g16
+   examples/shell
+   examples/template
 
 .. toctree::
    :caption: Project details

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/doc/machine.rst

@@ -1,7 +1,7 @@
 Machine parameters
 ======================================
 .. note::
-   One can load, modify, and export the input file by using our effective web-based tool `DP-GUI <https://deepmodeling.com/dpgui/input/dpdispatcher-machine>`_. All parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file.
+   One can load, modify, and export the input file by using our effective web-based tool `DP-GUI <https://deepmodeling.com/dpgui/input/dpdispatcher-machine>`_ online or hosted using the :ref:`command line interface <cli>` :code:`dpdisp gui`. All parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file.
 
 .. dargs::
    :module: dpdispatcher.arginfo

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/doc/resources.rst

@@ -1,7 +1,7 @@
 Resources parameters
 ======================================
 .. note::
-   One can load, modify, and export the input file by using our effective web-based tool `DP-GUI <https://deepmodeling.com/dpgui/input/dpdispatcher-resources>`_. All parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file for.
+   One can load, modify, and export the input file by using our effective web-based tool `DP-GUI <https://deepmodeling.com/dpgui/input/dpdispatcher-resources>`_ online or hosted using the :ref:`command line interface <cli>` :code:`dpdisp gui`. All parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file for.
 
 .. dargs::
    :module: dpdispatcher.arginfo

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/doc/task.rst

@@ -1,7 +1,7 @@
 Task parameters
 ======================================
 .. note::
-   One can load, modify, and export the input file by using our effective web-based tool `DP-GUI <https://deepmodeling.com/dpgui/input/dpdispatcher-task>`_. All parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file.
+   One can load, modify, and export the input file by using our effective web-based tool `DP-GUI <https://deepmodeling.com/dpgui/input/dpdispatcher-task>`_ online or hosted using the :ref:`command line interface <cli>` :code:`dpdisp gui`. All parameters below can be set in DP-GUI. By clicking "SAVE JSON", one can download the input file.
 
 .. dargs::
    :module: dpdispatcher.arginfo

dpdispatcher-0.6.1/dpdispatcher/__init__.py

@@ -0,0 +1,21 @@
+__author__ = "DeepModeling Team"
+__copyright__ = "Copyright 2019-2023, DeepModeling"
+__status__ = "Production"
+try:
+    from ._version import version as __version__
+except ImportError:
+    __version__ = "unknown"
+
+import dpdispatcher.contexts  # noqa: F401
+import dpdispatcher.machines  # noqa: F401
+from dpdispatcher.machine import Machine
+from dpdispatcher.submission import Job, Resources, Submission, Task
+
+__all__ = [
+    "__version__",
+    "Machine",
+    "Submission",
+    "Task",
+    "Job",
+    "Resources",
+]

dpdispatcher-0.6.1/dpdispatcher/__main__.py

@@ -0,0 +1,8 @@
+"""Package dp entry point."""
+
+from dpdispatcher.dpdisp import (
+    main,
+)
+
+if __name__ == "__main__":
+    main()

dpdispatcher-0.6.1/dpdispatcher/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.6.1'
+__version_tuple__ = version_tuple = (0, 6, 1)

{dpdispatcher-0.5.11 → dpdispatcher-0.6.1}/dpdispatcher/base_context.py

@@ -3,7 +3,7 @@ from typing import List, Tuple
 
 from dargs import Argument
 
-from dpdispatcher import dlog
+from dpdispatcher.dlog import dlog
 
 
 class BaseContext(metaclass=ABCMeta):

dpdispatcher-0.6.1/dpdispatcher/contexts/__init__.py

@@ -0,0 +1,11 @@
+"""Contexts."""
+import importlib
+from pathlib import Path
+
+PACKAGE_BASE = "dpdispatcher.contexts"
+NOT_LOADABLE = ("__init__.py",)
+
+for module_file in Path(__file__).parent.glob("*.py"):
+    if module_file.name not in NOT_LOADABLE:
+        module_name = f".{module_file.stem}"
+        importlib.import_module(module_name, PACKAGE_BASE)

{dpdispatcher-0.5.11/dpdispatcher → dpdispatcher-0.6.1/dpdispatcher/contexts}/dp_cloud_server_context.py

@@ -8,13 +8,16 @@ from typing import List
 import tqdm
 from dargs.dargs import Argument
 
-from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
-from dpdispatcher.dpcloudserver.config import ALI_STS_BUCKET_NAME, ALI_STS_ENDPOINT
+from dpdispatcher.dlog import dlog
 
 # from dpdispatcher.submission import Machine
 # from . import dlog
-from .dpcloudserver import Client, zip_file
+from dpdispatcher.utils.dpcloudserver import Client, zip_file
+from dpdispatcher.utils.dpcloudserver.config import (
+    ALI_STS_BUCKET_NAME,
+    ALI_STS_ENDPOINT,
+)
 
 # from zip_file import zip_files
 
@@ -113,6 +116,7 @@ class BohriumContext(BaseContext):
 
         upload_file_list = [
             job.script_file_name,
+            f"{job.script_file_name}.run",
         ]
         upload_file_list.extend(common_files)
 

{dpdispatcher-0.5.11/dpdispatcher → dpdispatcher-0.6.1/dpdispatcher/contexts}/hdfs_context.py

@@ -3,9 +3,9 @@ import shutil
 import tarfile
 from glob import glob
 
-from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
-from dpdispatcher.hdfs_cli import HDFS
+from dpdispatcher.dlog import dlog
+from dpdispatcher.utils.hdfs_cli import HDFS
 
 
 class HDFSContext(BaseContext):

{dpdispatcher-0.5.11/dpdispatcher → dpdispatcher-0.6.1/dpdispatcher/contexts}/local_context.py

@@ -1,12 +1,11 @@
-import hashlib
 import os
 import shutil
 import subprocess as sp
 from glob import glob
 from subprocess import TimeoutExpired
 
-from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
+from dpdispatcher.dlog import dlog
 
 
 class SPRetObj:
@@ -30,14 +29,6 @@ def _check_file_path(fname):
         os.makedirs(dirname, exist_ok=True)
 
 
-def _identical_files(fname0, fname1):
-    with open(fname0) as fp:
-        code0 = hashlib.sha1(fp.read().encode("utf-8")).hexdigest()
-    with open(fname1) as fp:
-        code1 = hashlib.sha1(fp.read().encode("utf-8")).hexdigest()
-    return code0 == code1
-
-
 class LocalContext(BaseContext):
     """Run jobs in the local server and remote directory.
 
@@ -152,9 +143,34 @@ class LocalContext(BaseContext):
         for ii in submission.belonging_tasks:
             local_job = os.path.join(self.local_root, ii.task_work_path)
             remote_job = os.path.join(self.remote_root, ii.task_work_path)
-            flist = ii.backward_files
+            flist = []
+            for kk in ii.backward_files:
+                abs_flist_r = glob(os.path.join(remote_job, kk))
+                abs_flist_l = glob(os.path.join(local_job, kk))
+                if not abs_flist_r and not abs_flist_l:
+                    if check_exists:
+                        if mark_failure:
+                            tag_file_path = os.path.join(
+                                self.local_root,
+                                ii.task_work_path,
+                                "tag_failure_download_%s" % kk,
+                            )
+                            with open(tag_file_path, "w") as fp:
+                                pass
+                        else:
+                            pass
+                    else:
+                        raise RuntimeError(
+                            "cannot find download file " + os.path.join(remote_job, kk)
+                        )
+                rel_flist = [
+                    os.path.relpath(ii, start=remote_job) for ii in abs_flist_r
+                ]
+                flist.extend(rel_flist)
             if back_error:
-                flist += glob(os.path.join(remote_job, "error*"))
+                abs_flist = glob(os.path.join(remote_job, "error*"))
+                rel_flist = [os.path.relpath(ii, start=remote_job) for ii in abs_flist]
+                flist.extend(rel_flist)
             for jj in flist:
                 rfile = os.path.join(remote_job, jj)
                 lfile = os.path.join(local_job, jj)
@@ -198,9 +214,30 @@ class LocalContext(BaseContext):
                     pass
         local_job = self.local_root
         remote_job = self.remote_root
-        flist = submission.backward_common_files
+        flist = []
+        for kk in submission.backward_common_files:
+            abs_flist_r = glob(os.path.join(remote_job, kk))
+            abs_flist_l = glob(os.path.join(local_job, kk))
+            if not abs_flist_r and not abs_flist_l:
+                if check_exists:
+                    if mark_failure:
+                        tag_file_path = os.path.join(
+                            self.local_root, "tag_failure_download_%s" % kk
+                        )
+                        with open(tag_file_path, "w") as fp:
+                            pass
+                    else:
+                        pass
+                else:
+                    raise RuntimeError(
+                        "cannot find download file " + os.path.join(remote_job, kk)
+                    )
+            rel_flist = [os.path.relpath(ii, start=remote_job) for ii in abs_flist_r]
+            flist.extend(rel_flist)
         if back_error:
-            flist += glob(os.path.join(remote_job, "error*"))
+            abs_flist = glob(os.path.join(remote_job, "error*"))
+            rel_flist = [os.path.relpath(ii, start=remote_job) for ii in abs_flist]
+            flist.extend(rel_flist)
         for jj in flist:
             rfile = os.path.join(remote_job, jj)
             lfile = os.path.join(local_job, jj)

{dpdispatcher-0.5.11/dpdispatcher → dpdispatcher-0.6.1/dpdispatcher/contexts}/openapi_context.py

@@ -14,9 +14,9 @@ except ModuleNotFoundError:
 else:
     found_bohriumsdk = True
 
-from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
+from dpdispatcher.utils.job_status import JobStatus
 
 DP_CLOUD_SERVER_HOME_DIR = os.path.join(
     os.path.expanduser("~"), ".dpdispatcher/", "dp_cloud_server/"
@@ -88,6 +88,7 @@ class OpenAPIContext(BaseContext):
 
         upload_file_list = [
             job.script_file_name,
+            f"{job.script_file_name}.run",
         ]
 
         upload_file_list.extend(common_files)