stepup-queue 1.0.4__tar.gz → 1.0.6__tar.gz

stepup_queue-1.0.6/MANIFEST.in

@@ -0,0 +1,16 @@
+# Exclude documentation and development files
+prune .github
+prune .vscode
+prune docs
+prune overrides
+exclude .editorconfig
+exclude .gitignore
+exclude .markdownlint-cli2.jsonc
+exclude .pre-commit-config.yaml
+exclude mkdocs.yaml
+
+# Exclude tests for now. (Could be useful later for conda package.)
+prune tests
+
+# Exclude common build artifacts and cache files
+global-exclude *.py[cod] __pycache__ *.so

{stepup_queue-1.0.4 → stepup_queue-1.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup-queue
-Version: 1.0.4
+Version: 1.0.6
 Summary: StepUp Queue integrates queued jobs into a StepUp workflow.
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later
@@ -19,11 +19,12 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Build Tools
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: stepup<4.0.0,>=3.0.0
+Requires-Dist: stepup<4.0.0,>=3.1.3
 Provides-Extra: dev
 Requires-Dist: psutil; extra == "dev"
 Requires-Dist: pytest; extra == "dev"

{stepup_queue-1.0.4 → stepup_queue-1.0.6}/pyproject.toml

@@ -23,11 +23,12 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Software Development :: Build Tools",
 ]
 dependencies = [
     # Ensure changes to these dependencies are reflected in .github/requirements-old.txt
-    "stepup>=3.0.0,<4.0.0",
+    "stepup>=3.1.3,<4.0.0",
 ]
 dynamic = ["version"]
 
@@ -88,6 +89,9 @@ ignore = [
     "TRY301",  # https://docs.astral.sh/ruff/rules/raise-within-try/
 ]
 
+[tool.ruff.lint.isort]
+known-first-party = ["stepup"]
+
 [tool.setuptools]
 packages = ["stepup.queue"]
 

{stepup_queue-1.0.4 → stepup_queue-1.0.6}/stepup/queue/actions.py

@@ -26,7 +26,6 @@ import shlex
 
 from path import Path
 
-from stepup.core.utils import string_to_bool
 from stepup.core.worker import WorkThread
 
 from .canceljobs import read_jobid_cluster
@@ -38,9 +37,13 @@ def sbatch(argstr: str, work_thread: WorkThread) -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("ext", nargs="?", default=".sh")
     parser.add_argument("--rc", default=None)
+    default_onchange = os.getenv("STEPUP_QUEUE_ONCHANGE", "raise")
+    parser.add_argument(
+        "--onchange", default=default_onchange, choices=["raise", "resubmit", "ignore"]
+    )
     args = parser.parse_args(shlex.split(argstr))
 
-    if string_to_bool(os.getenv("STEPUP_QUEUE_RESUBMIT_CHANGED_INPUTS", "0")):
+    if args.onchange == "resubmit":
         with contextlib.suppress(InpDigestError):
             return submit_once_and_wait(work_thread, args.ext, args.rc)
         # Cancel running job (if any), clean log and resubmit
@@ -48,4 +51,4 @@ def sbatch(argstr: str, work_thread: WorkThread) -> int:
         job_id, cluster = read_jobid_cluster(path_log)
         work_thread.runsh(f"scancel -M {cluster} {job_id}")
         path_log.remove_p()
-    return submit_once_and_wait(work_thread, args.ext, args.rc)
+    return submit_once_and_wait(work_thread, args.ext, args.rc, args.onchange != "ignore")

{stepup_queue-1.0.4 → stepup_queue-1.0.6}/stepup/queue/api.py

@@ -37,6 +37,7 @@ def sbatch(
     env: Collection[str] | str = (),
     out: Collection[str] | str = (),
     vol: Collection[str] | str = (),
+    onchange: str | None = None,
     optional: bool = False,
     pool: str | None = None,
     block: bool = False,
@@ -61,6 +62,9 @@ def sbatch(
 
     See `step()` documentation in StepUp Core for all optional arguments.
     and the return value.
+    Note that the `inp`, `out` and `vol` arguments are extended
+    with the files mentioned above and that any additional files you specify
+    are interpreted relative to the working directory.
 
     Parameters
     ----------
@@ -76,6 +80,9 @@ def sbatch(
         If multiple instructions are needed, put them in a file, e.g. `rc.sh`
         and pass it here as `source rc.sh`.
         In this case, you usually also want to include `rc.sh` in the `inp` list.
+    onchange
+        Policy when a the inputs of a previously submitted job have changed.
+        Must be one of `"raise"`, `"resubmit"` or `"ignore"`.
     """
     if ext == "":
         ext = ".sh"
@@ -88,6 +95,10 @@ def sbatch(
         action += f" {ext}"
     if rc is not None:
         action += f" --rc={shlex.quote(rc)}"
+    if onchange is not None:
+        if onchange not in ["raise", "resubmit", "ignore"]:
+            raise ValueError(f"Invalid onchange policy {onchange}.")
+        action += f" --onchange={onchange}"
     return step(
         action,
         inp=[f"slurmjob{ext}", *string_to_list(inp)],

{stepup_queue-1.0.4 → stepup_queue-1.0.6}/stepup/queue/sbatch.py

@@ -36,11 +36,14 @@ SCONTROL_FAILED = "The command `scontrol show job` failed!\n"
 DEBUG = string_to_bool(os.getenv("STEPUP_SBATCH_DEBUG", "0"))
 CACHE_TIMEOUT = int(os.getenv("STEPUP_SBATCH_CACHE_TIMEOUT", "30"))
 POLLING_INTERVAL = int(os.getenv("STEPUP_SBATCH_POLLING_INTERVAL", "10"))
-TIME_MARGIN = int(os.getenv("STEPUP_SBATCH_TIME_MARGIN", "5"))
+TIME_MARGIN = int(os.getenv("STEPUP_SBATCH_TIME_MARGIN", "15"))
 
 
 def submit_once_and_wait(
-    work_thread: WorkThread, job_ext: str, sbatch_rc: str | None = None
+    work_thread: WorkThread,
+    job_ext: str,
+    sbatch_rc: str | None = None,
+    validate_inp_digest: bool = True,
 ) -> int:
     """Submit a job and wait for it to complete. When called a second time, just wait.
 
@@ -53,6 +56,9 @@ def submit_once_and_wait(
     sbatch_rc
         A resource configuration needed before calling sbatch.
         This is executed in the same shell, right before calling sbatch.
+    validate_inp_digest
+        If False, the input digest is not checked.
+        This is useful when the job script is modified but the changes are harmless.
 
     Returns
     -------
@@ -63,7 +69,7 @@ def submit_once_and_wait(
     # Read previously logged steps
     path_log = Path("slurmjob.log")
     if path_log.is_file():
-        previous_lines = _read_log(path_log)
+        previous_lines = read_log(path_log, validate_inp_digest)
     else:
         previous_lines = []
         _init_log(path_log)
@@ -105,7 +111,7 @@ def submit_once_and_wait(
     return int(returncode)
 
 
-def _read_log(path_log: str) -> list[str]:
+def read_log(path_log: str, do_inp_digest: bool = True) -> list[str]:
     """Read lines from a previously created log file."""
     lines = []
     with open(path_log) as f:
@@ -114,9 +120,11 @@ def _read_log(path_log: str) -> list[str]:
         except StopIteration as exc:
             raise ValueError("Existing log file is empty.") from exc
         try:
-            check_log_inp_digest(next(f).strip())
+            inp_digest = next(f).strip()
         except StopIteration as exc:
-            raise ValueError("Existing log file is empty.") from exc
+            raise ValueError("Existing has no input digest.") from exc
+        if do_inp_digest:
+            check_log_inp_digest(inp_digest)
         for line in f:
             line = line.strip()
             lines.append(line)
@@ -177,7 +185,9 @@ def _read_or_poll_status(
         # Call scontrol and parse its response.
         rndsleep()
         status_time, status = get_status(work_thread, jobid, cluster)
-        if status != last_status:
+        # Log only if the status changed, and is not invalid or unlisted.
+        # These two statuses are (potentially) transient and should not be logged.
+        if status != last_status and status not in ["invalid", "unlisted"]:
             log_step(path_log, status)
     done = (status_time > submit_time + TIME_MARGIN) and (
         status not in ["PENDING", "CONFIGURING", "RUNNING", "invalid"]

{stepup_queue-1.0.4 → stepup_queue-1.0.6}/stepup_queue.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup-queue
-Version: 1.0.4
+Version: 1.0.6
 Summary: StepUp Queue integrates queued jobs into a StepUp workflow.
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later
@@ -19,11 +19,12 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Build Tools
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: stepup<4.0.0,>=3.0.0
+Requires-Dist: stepup<4.0.0,>=3.1.3
 Provides-Extra: dev
 Requires-Dist: psutil; extra == "dev"
 Requires-Dist: pytest; extra == "dev"

stepup_queue-1.0.6/stepup_queue.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+LICENSE
+MANIFEST.in
+README.md
+pyproject.toml
+stepup/queue/__init__.py
+stepup/queue/actions.py
+stepup/queue/api.py
+stepup/queue/canceljobs.py
+stepup/queue/sbatch.py
+stepup_queue.egg-info/PKG-INFO
+stepup_queue.egg-info/SOURCES.txt
+stepup_queue.egg-info/dependency_links.txt
+stepup_queue.egg-info/entry_points.txt
+stepup_queue.egg-info/requires.txt
+stepup_queue.egg-info/top_level.txt

{stepup_queue-1.0.4 → stepup_queue-1.0.6}/stepup_queue.egg-info/requires.txt

@@ -1,4 +1,4 @@
-stepup<4.0.0,>=3.0.0
+stepup<4.0.0,>=3.1.3
 
 [dev]
 psutil

stepup_queue-1.0.4/.editorconfig

@@ -1,18 +0,0 @@
-# EditorConfig is awesome: https://EditorConfig.org
-
-root = true
-
-[*]
-end_of_line = lf
-insert_final_newline = true
-charset = utf-8
-indent_style = space
-indent_size = 4
-max_line_length = 100
-
-[Makefile]
-indent_style = tab
-
-[{*.json,*.yml,*.yaml}]
-indent_style = space
-indent_size = 2

stepup_queue-1.0.4/.github/requirements-old.txt

@@ -1,2 +0,0 @@
-# Ensure changes to these dependencies are reflected in pyproject.toml
-stepup==3.0.0

stepup_queue-1.0.4/.github/scripts/extract-notes.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-# Usage: .github/scripts/extract-notes.sh OWNER/SLUG GITREF
-
-IFS='/'; read -ra REPOSITORY <<<"${1}"
-OWNER=${REPOSITORY[0]}
-SLUG=${REPOSITORY[1]}
-GITREF=${2}
-
-if [[ "${GITREF}" == refs/tags/* ]]; then
-    TAG="${GITREF#refs/tags/}"
-    VERSION="${TAG#v}"
-    MACRO_MESO=$(echo "${VERSION}" | cut -d. -f1,2)
-else
-    TAG="unreleased"
-    VERSION="Unreleased"
-    MACRO_MESO="dev"
-fi
-
-# Extract the release notes from the changelog
-sed -n "/## \[${VERSION}\]/, /## /{ /##/!p }" docs/changelog.md > notes.md
-
-# Add a link to the release notes
-URL="https://${OWNER}.github.io/${SLUG}/${MACRO_MESO}/changelog/#${TAG}"
-echo "See [docs/changelog/#${TAG}](${URL}) for more details." >> notes.md
-
-# Remove leading and trailing empty lines
-sed -e :a -e '/./,$!d;/^\n*$/{$d;N;};/\n$/ba' -i notes.md

stepup_queue-1.0.4/.github/workflows/mkdocs.yaml

@@ -1,72 +0,0 @@
-# This workflow assumes that you deploy the documentation from the gh-pages branch.
-
-name: mkdocs
-on:
-  push:
-    branches:
-      # Run on the main branch ...
-      - main
-    tags:
-      # ... and on release tags ...
-      - 'v*'
-    paths:
-      # ... and only if relevant files have changed.
-      - stepup/**
-      - docs/**
-      - mkdocs.yaml
-      - pyproject.toml
-      - .github/workflows/mkdocs.yaml
-  pull_request:
-  # Run tests on pull requests ...
-    paths:
-      # ... only if relevant files have changed.
-      - stepup/**
-      - docs/**
-      - mkdocs.yaml
-      - pyproject.toml
-      - .github/workflows/mkdocs.yaml
-
-permissions:
-  contents: write
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: pages
-  cancel-in-progress: false
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      # Get the source
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.x
-
-      # Prepare the environment for building docs.
-      - name: Setup Pages
-        id: pages
-        uses: actions/configure-pages@v5
-      - name: Install development version
-        run: pip install -e .[dev]
-
-      # Update the site using to the gh-pages branch with mike.
-      - name: Get the docs branch
-        run: git fetch --depth=1 origin gh-pages
-      - name: Configure Git user for documentation commit
-        run: |
-          git config --global user.name 'github-actions[bot]'
-          git config --global user.email 'github-actions[bot]@users.noreply.github.com'
-      - name: Build mkdocs
-        run: mike deploy dev
-      - name: Push gh-pages branch with development version of docs
-        if: github.ref == 'refs/heads/main'
-        run: git push origin gh-pages
-      - name: Deploy gh-pages branch with stable version of docs
-        if: startsWith(github.ref, 'refs/tags/v')
-        run: |
-          VERSION="${GITHUB_REF#refs/tags/v}"
-          MACRO_MESO=$(echo "${VERSION}" | cut -d. -f1,2)
-          mike deploy ${MACRO_MESO} -u stable -p

stepup_queue-1.0.4/.github/workflows/pytest.yaml

@@ -1,45 +0,0 @@
-name: pytest
-on:
-  push:
-    branches:
-      # Run tests for change on the main branch ...
-      - main
-    tags-ignore:
-      # ... but not for tags (avoids duplicate work) ...
-      - '**'
-    paths:
-      # ... and only if relevant files have changed.
-      - stepup/**
-      - tests/**
-      - pyproject.toml
-      - .github/workflows/pytest.yaml
-  pull_request:
-    # Run tests on pull requests ...
-    paths:
-      # ... only if relevant files have changed.
-      - stepup/**
-      - tests/**
-      - pyproject.toml
-      - .github/workflows/pytest.yaml
-
-jobs:
-  tests:
-    strategy:
-      matrix:
-        python-version: ['3.11', '3.13']
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install oldest versions of supported dependencies
-        if: ${{ matrix.python-version == '3.11'}}
-        run: pip install -r .github/requirements-old.txt
-      - name: Install development version
-        run: pip install -e .[dev]
-      - name: Run pytest
-        # Enfore strict checks instead of trying to repair fixable issues.
-        # RPC timeout is set shorter to speed up tests in case of failures.
-        run: STEPUP_DEBUG=1 STEPUP_SYNC_RPC_TIMEOUT=10 pytest -vv

stepup_queue-1.0.4/.github/workflows/release.yaml

@@ -1,145 +0,0 @@
-# This workflow is adapted from:
-# https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
-
-name: release
-on:
-  push:
-    branches:
-      - main
-    tags:
-      - 'v*'
-
-env:
-  # This is not critical
-  # It is used only for showing URLs in GitHub Actions web interface.
-  PYPI_NAME: stepup-queue
-  TITLE: StepUp Queue
-
-jobs:
-  build:
-    name: Build distribution
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.11'
-    - name: Install pypa/build
-      run: >-
-        python -m pip install build
-    - name: Build package
-      run: >-
-        python -m build
-    - name: Store the distribution packages
-      uses: actions/upload-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Run release notes script
-      run: >-
-        .github/scripts/extract-notes.sh
-        '${{ github.repository }}'
-        '${{ github.ref }}'
-    - name: Store the release notes
-      uses: actions/upload-artifact@v4
-      with:
-        name: release-notes
-        path: notes.md
-
-  publish-to-pypi:
-    name: Publish Python distribution to PyPI
-    if: startsWith(github.ref, 'refs/tags/v')  # only publish to PyPI on tag pushes
-    needs:
-    - build
-    runs-on: ubuntu-latest
-    environment:
-      name: pypi
-      url: https://pypi.org/p/${{ env.PYPI_NAME }}
-    permissions:
-      id-token: write
-
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-
-  github-release:
-    name: >-
-      Sign the Python distribution with Sigstore
-      and upload them to GitHub Release
-    needs:
-    - publish-to-pypi
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: write
-      id-token: write
-
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Download the release notes
-      uses: actions/download-artifact@v4
-      with:
-        name: release-notes
-    - name: Sign the dists with Sigstore
-      uses: sigstore/gh-action-sigstore-python@v3.0.0
-      with:
-        inputs: >-
-          ./dist/*.tar.gz
-          ./dist/*.whl
-    - name: Create GitHub Release
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-      run: >-
-        gh release create '${{ github.ref_name }}'
-        --repo '${{ github.repository }}'
-        --title '${{ env.TITLE }} ${{ github.ref_name }}'
-        --notes-file notes.md
-    - name: Upload artifact signatures to GitHub Release
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-      # Upload to GitHub Release using the `gh` CLI.
-      # `dist/` contains the built packages, and the
-      # sigstore-produced signatures and certificates.
-      run: >-
-        gh release upload
-        '${{ github.ref_name }}'
-        dist/**
-        --repo '${{ github.repository }}'
-
-  publish-to-testpypi:
-    name: Publish Python distribution to TestPyPI
-    if: ${{ github.ref == 'refs/heads/main' &&  github.repository_owner == 'reproducible-reporting'}}
-    needs:
-    - build
-    runs-on: ubuntu-latest
-
-    environment:
-      name: testpypi
-      url: https://test.pypi.org/p/${{ env.PYPI_NAME }}
-
-    permissions:
-      id-token: write
-
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Publish distribution to TestPyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        repository-url: https://test.pypi.org/legacy/

stepup_queue-1.0.4/.gitignore

@@ -1,29 +0,0 @@
-# Local dev environment
-.envrc
-.idea
-.local
-tmp
-site/
-venv/
-.venv/
-.ruff_cache
-.pytest_cache
-notes.md
-
-# Python and packaging
-__pycache__
-*.egg-info
-dist
-build
-_version.py
-
-# StepUp specifics
-docs/.stepup
-
-# Temporary files
-*.swp
-
-# NodeJS (for markdownlint)
-node_modules/
-package-lock.json
-package.json

stepup_queue-1.0.4/.markdownlint-cli2.jsonc

@@ -1,15 +0,0 @@
-{
-  "config": {
-    "MD013": {
-      "line_length": 100,
-      "heading_line_length": 100,
-      "code_block_line_length": 100
-    },
-    "MD007": { "indent": 4 },
-    "MD033": {
-      "allowed_elements": ["script"]
-    },
-    "MD046": false,
-    "MD052": false
-  }
-}

stepup_queue-1.0.4/.pre-commit-config.yaml

@@ -1,42 +0,0 @@
-repos:
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
-  hooks:
-    - id: check-added-large-files
-    - id: check-ast
-    - id: check-case-conflict
-    - id: check-executables-have-shebangs
-    - id: check-json
-    - id: check-merge-conflict
-    - id: check-symlinks
-    - id: check-toml
-    - id: check-vcs-permalinks
-    - id: debug-statements
-    - id: detect-private-key
-    - id: destroyed-symlinks
-    - id: end-of-file-fixer
-    - id: fix-byte-order-marker
-    - id: mixed-line-ending
-    - id: pretty-format-json
-      args: ["--autofix", "--no-sort-keys"]
-    - id: trailing-whitespace
-- repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: v1.5.5
-  hooks:
-    - id: remove-crlf
-- repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.4
-  hooks:
-    - id: ruff-format
-      exclude: "^docs/examples/slurm-basic/pass/slurmjob.py$"
-    - id: ruff
-      args: ["--fix", "--show-fixes"]
-- repo: https://github.com/DavidAnson/markdownlint-cli2
-  rev: v0.17.2
-  hooks:
-  - id: markdownlint-cli2
-    require_serial: true
-- repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.30.0
-  hooks:
-    - id: check-github-workflows