qstat-cache 3.0__tar.gz

qstat_cache-3.0/.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,95 @@
+name: Publish package to PyPI
+on: push
+
+jobs:
+  build:
+    name: 📦 Build distribution
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: 🐍 Publish package to PyPI
+    # Trigger this and signing step only when a tag is pushed
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/qstat-cache
+    permissions:
+      # IMPORTANT: mandatory for trusted publishing
+      id-token: write
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: >-
+      📝 Sign the distribution with Sigstore
+      and upload them to GitHub Release
+    needs:
+    - publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write  # IMPORTANT: mandatory for sigstore
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/gh-action-sigstore-python@v3.0.0
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+    - name: Create GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: >-
+        gh release create
+        "$GITHUB_REF_NAME"
+        --repo "$GITHUB_REPOSITORY"
+        --notes ""
+    - name: Upload artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      # Upload to GitHub Release using the `gh` CLI.
+      # `dist/` contains the built packages, and the
+      # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        "$GITHUB_REF_NAME" dist/**
+        --repo "$GITHUB_REPOSITORY"

qstat_cache-3.0/.gitignore

@@ -0,0 +1,10 @@
+data
+temp
+logs
+*.pyc
+__pycache__
+dist/
+*.egg-info
+build/
+*.swp
+*.sh

qstat_cache-3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 National Center for Atmospheric Research
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

qstat_cache-3.0/Makefile

@@ -0,0 +1,27 @@
+PREFIX ?= /usr/local
+VERSION := 3.0
+
+make install:
+	mkdir -p $(PREFIX)/bin $(PREFIX)/util $(PREFIX)/lib
+	sed 's|/src|/lib|' bin/qstat > $(PREFIX)/bin/qstat
+	sed 's|/src|/lib|' util/gen_data > $(PREFIX)/util/gen_data
+	cp -r src/qscache $(PREFIX)/lib/qscache
+	ln -s lib/qscache/cfg $(PREFIX)/cfg
+	chmod +x $(PREFIX)/bin/qstat $(PREFIX)/util/gen_data
+
+build:
+	python3 -m build
+
+# These commands can only be run successfully by package maintainers
+manual-upload: 
+	python3 -m twine upload dist/*
+
+test-upload:
+	python3 -m twine upload --repository testpypi dist/*
+
+#release: man
+#	git tag v$(VERSION)
+#	git push origin v$(VERSION)
+
+clean:
+	rm -rf dist build

qstat_cache-3.0/PKG-INFO

@@ -0,0 +1,190 @@
+Metadata-Version: 2.4
+Name: qstat-cache
+Version: 3.0
+Summary: A caching qstat that reduces load on the PBS server
+Author-email: Brian Vanderwende <vanderwb@ucar.edu>
+License-Expression: MIT
+Project-URL: homepage, https://github.com/NCAR/qstat-cache
+Project-URL: issues, https://github.com/NCAR/qstat-cache/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# qstat-cache
+A cached version of the PBS Pro qstat command that reduces load on the
+scheduler's database
+
+## Details
+Most users run the qstat command at reasonable intervals and things work well.
+However, with the advent of workflow managers more users are running qstat at
+frequencies much too high for current versions of PBS Pro to support well. This
+utility creates a simple text-based cache of common qstat output and provides a
+script to serve that data to users. If an option is not cached (e.g., -Q
+output), the query is sent to PBS's version of qstat for processing. Usage:
+
+```
+usage: qstat [-h] [-1] [-a] [-D DELIMITER] [-f] [-F {json,dsv}]
+             [--format FORMAT] [-H] [-J] [--noheader] [-n] [-s]
+             [--status STATUS] [-t] [-T] [-u USER] [-w] [-x]
+             [filters ...]
+
+This command provides a lightweight alternative to qstat. Data are queried and
+updated every minute from the PBS job scheduler. Options not listed here will
+be forwarded to the scheduler. Please use those options sparingly. Job IDs, if
+provided, should be numeric only and space delimited. If a destination is
+provided, it should be a valid execution queue on the chosen server. This
+cached version of qstat does not allow mixed queries from multiple servers -
+only one server may be specified per request.
+
+positional arguments:
+  filters          job IDs or queues
+
+options:
+  -h, --help       show this help message and exit
+  -1               display node or comment information on job line
+  -a               display all jobs (default unless -f specified)
+  -D DELIMITER     specify a delimiter if using -Fdsv (default = '|')
+  -f               display full output for a job
+  -F {json,dsv}    full output (-f) in custom format
+  --format FORMAT  column output in custom format (=help for more)
+  -H               all moved or finished jobs / specific job of any state
+  -J               only show information for jobs (or subjobs with -t)
+  --noheader       disable labels (no header)
+  -n               display a list of nodes at the end of the line
+  -s               display administrator comment on the next line
+  --status STATUS  filter jobs by specific single-character status code
+  -t               show information for both jobs and array subjobs
+  -T               displays estimated start time for queued jobs
+  -u USER          filter jobs by the submitting user
+  -w               use wide format output (120 columns)
+  -x               all job records in recent history
+```
+
+## Installation
+
+There are two methods for installing **qstat-cache** - using the included
+`Makefile` or with `pip`.
+
+### Makefile method
+
+1. Clone this repository on your system.
+2. Install at your desired path using `make install
+   PREFIX=/path/to/qstat-cache`.
+
+### pip method
+
+1. If desired, first create a virtual environment with `venv` or `conda/mamba`
+   and activate it.
+2. Now run `python3 -m pip install qstat-cache`.
+
+### Site setup
+
+In either case, the following steps are required to finish configuration.
+
+3. In `$PREFIX/lib/qscache/cfg` or `lib/python3.x/site-packages/qscache/cfg`,
+   copy the `site.cfg.example` file to `site.cfg` and customize settings as
+   described below. Alternatively, copy the example to `<system>.cfg` and then
+   set the environment variable `QSCACHE_SERVER=<system>`. The latter approach
+   allows you to cache multiple servers in a complex at the same time.
+4. Schedule the `util/gen_data.sh` script to run at regular intervals (typically
+   every minute via a cron job).
+7. Add the cached version of `qstat` to your (and your users') environment PATH.
+
+### site.cfg settings
+
+```
+[paths]
+# Temporary data path used by gen_data when creating
+# cached output from qstat (fast file system is best)
+Temp = ${install_dir}/temp/derecho
+
+# Path where cached data will be stored and accessed
+Data = ${install_dir}/data
+
+# Optional path for logging qstat invocations
+# If set, a log will be created for each user on each day
+#   that records calls to qstat along with arguments
+# If blank, logging will be disabled
+Logs = ${install_dir}/test/logs
+
+[cache]
+# The maximum wait time in seconds before the cache is
+# bypassed and the real qstat is called
+MaxWait = 20
+
+# The maximum allowed age in seconds of cache data. Beyond
+# this age we bypass the cache and call the true qstat
+MaxAge = 300
+
+# Delay in seconds to impose on qstat calls that bypass
+# the cache due to aged data. Increasing this value can help
+# the scheduler when under high load
+AgeDelay = 5
+
+# Specify the sub-minute frequency to generate data
+# in seconds
+Frequency = 60
+
+[pbs]
+# Specify the location of the actual qstat command
+Qstat = /opt/pbs/bin/qstat
+
+# Some sites may need to prefix calls to PBS with another
+# command (e.g., a sudo operation). Use this variable to
+# specify a prefix for PBS calls
+Prefix = sudo -u adminuser
+
+[servermap]
+# Mapping of long-form server names for peer-scheduling
+# user queries (use qstat -Bf to get server names)
+casper = casper-pbs
+derecho = desched1
+
+[privileges]
+# Enable privilege checking according to following user and
+# group settings. If false, all queries allowed.
+Active = True
+
+[priv.all]
+# Permit users and groups from these two lists respectively to
+# see "full" job output from other users excluding the user
+# environment contained in a job's Variable_List
+Users = vanderwb
+Groups = csgteam
+
+[priv.env]
+# Permit users and groups from these two lists to view all full
+# job output, including the user environment
+Users = vanderwb
+Groups =
+```
+
+### Example crontab
+
+Here is a sample crontab that will run the `gen_data.sh` script every minute
+(sub-minute scheduled is recommended and enabled via the site.cfg). The idea
+here is to run often enough that users and their workflows are satisfied, but
+not so often that we put our own load on PBS.
+
+Here, we use the `QSCACHE_SERVER` variable to specify a particular system in a
+multi-server complex.
+
+```
+#   Run qstat cache generation script every minute
+#       Added by Joe User on 4 Dec 2019
+* * * * * QSCACHE_SERVER=sitename /path/to/qstat_cache/util/gen_data.sh
+```
+
+## Debugging
+
+There are two environment variables you may set to assist in debugging. Setting
+`QSCACHE_DEBUG` will cause qstat to print the age of the cache, assuming it can
+be found.
+
+If you set `QSCACHE_BYPASS` to `true`, the cache will be bypassed regardless of
+which options are set, and the scheduler version of qstat will instead be
+called.

qstat_cache-3.0/README.md

@@ -0,0 +1,174 @@
+# qstat-cache
+A cached version of the PBS Pro qstat command that reduces load on the
+scheduler's database
+
+## Details
+Most users run the qstat command at reasonable intervals and things work well.
+However, with the advent of workflow managers more users are running qstat at
+frequencies much too high for current versions of PBS Pro to support well. This
+utility creates a simple text-based cache of common qstat output and provides a
+script to serve that data to users. If an option is not cached (e.g., -Q
+output), the query is sent to PBS's version of qstat for processing. Usage:
+
+```
+usage: qstat [-h] [-1] [-a] [-D DELIMITER] [-f] [-F {json,dsv}]
+             [--format FORMAT] [-H] [-J] [--noheader] [-n] [-s]
+             [--status STATUS] [-t] [-T] [-u USER] [-w] [-x]
+             [filters ...]
+
+This command provides a lightweight alternative to qstat. Data are queried and
+updated every minute from the PBS job scheduler. Options not listed here will
+be forwarded to the scheduler. Please use those options sparingly. Job IDs, if
+provided, should be numeric only and space delimited. If a destination is
+provided, it should be a valid execution queue on the chosen server. This
+cached version of qstat does not allow mixed queries from multiple servers -
+only one server may be specified per request.
+
+positional arguments:
+  filters          job IDs or queues
+
+options:
+  -h, --help       show this help message and exit
+  -1               display node or comment information on job line
+  -a               display all jobs (default unless -f specified)
+  -D DELIMITER     specify a delimiter if using -Fdsv (default = '|')
+  -f               display full output for a job
+  -F {json,dsv}    full output (-f) in custom format
+  --format FORMAT  column output in custom format (=help for more)
+  -H               all moved or finished jobs / specific job of any state
+  -J               only show information for jobs (or subjobs with -t)
+  --noheader       disable labels (no header)
+  -n               display a list of nodes at the end of the line
+  -s               display administrator comment on the next line
+  --status STATUS  filter jobs by specific single-character status code
+  -t               show information for both jobs and array subjobs
+  -T               displays estimated start time for queued jobs
+  -u USER          filter jobs by the submitting user
+  -w               use wide format output (120 columns)
+  -x               all job records in recent history
+```
+
+## Installation
+
+There are two methods for installing **qstat-cache** - using the included
+`Makefile` or with `pip`.
+
+### Makefile method
+
+1. Clone this repository on your system.
+2. Install at your desired path using `make install
+   PREFIX=/path/to/qstat-cache`.
+
+### pip method
+
+1. If desired, first create a virtual environment with `venv` or `conda/mamba`
+   and activate it.
+2. Now run `python3 -m pip install qstat-cache`.
+
+### Site setup
+
+In either case, the following steps are required to finish configuration.
+
+3. In `$PREFIX/lib/qscache/cfg` or `lib/python3.x/site-packages/qscache/cfg`,
+   copy the `site.cfg.example` file to `site.cfg` and customize settings as
+   described below. Alternatively, copy the example to `<system>.cfg` and then
+   set the environment variable `QSCACHE_SERVER=<system>`. The latter approach
+   allows you to cache multiple servers in a complex at the same time.
+4. Schedule the `util/gen_data.sh` script to run at regular intervals (typically
+   every minute via a cron job).
+7. Add the cached version of `qstat` to your (and your users') environment PATH.
+
+### site.cfg settings
+
+```
+[paths]
+# Temporary data path used by gen_data when creating
+# cached output from qstat (fast file system is best)
+Temp = ${install_dir}/temp/derecho
+
+# Path where cached data will be stored and accessed
+Data = ${install_dir}/data
+
+# Optional path for logging qstat invocations
+# If set, a log will be created for each user on each day
+#   that records calls to qstat along with arguments
+# If blank, logging will be disabled
+Logs = ${install_dir}/test/logs
+
+[cache]
+# The maximum wait time in seconds before the cache is
+# bypassed and the real qstat is called
+MaxWait = 20
+
+# The maximum allowed age in seconds of cache data. Beyond
+# this age we bypass the cache and call the true qstat
+MaxAge = 300
+
+# Delay in seconds to impose on qstat calls that bypass
+# the cache due to aged data. Increasing this value can help
+# the scheduler when under high load
+AgeDelay = 5
+
+# Specify the sub-minute frequency to generate data
+# in seconds
+Frequency = 60
+
+[pbs]
+# Specify the location of the actual qstat command
+Qstat = /opt/pbs/bin/qstat
+
+# Some sites may need to prefix calls to PBS with another
+# command (e.g., a sudo operation). Use this variable to
+# specify a prefix for PBS calls
+Prefix = sudo -u adminuser
+
+[servermap]
+# Mapping of long-form server names for peer-scheduling
+# user queries (use qstat -Bf to get server names)
+casper = casper-pbs
+derecho = desched1
+
+[privileges]
+# Enable privilege checking according to following user and
+# group settings. If false, all queries allowed.
+Active = True
+
+[priv.all]
+# Permit users and groups from these two lists respectively to
+# see "full" job output from other users excluding the user
+# environment contained in a job's Variable_List
+Users = vanderwb
+Groups = csgteam
+
+[priv.env]
+# Permit users and groups from these two lists to view all full
+# job output, including the user environment
+Users = vanderwb
+Groups =
+```
+
+### Example crontab
+
+Here is a sample crontab that will run the `gen_data.sh` script every minute
+(sub-minute scheduled is recommended and enabled via the site.cfg). The idea
+here is to run often enough that users and their workflows are satisfied, but
+not so often that we put our own load on PBS.
+
+Here, we use the `QSCACHE_SERVER` variable to specify a particular system in a
+multi-server complex.
+
+```
+#   Run qstat cache generation script every minute
+#       Added by Joe User on 4 Dec 2019
+* * * * * QSCACHE_SERVER=sitename /path/to/qstat_cache/util/gen_data.sh
+```
+
+## Debugging
+
+There are two environment variables you may set to assist in debugging. Setting
+`QSCACHE_DEBUG` will cause qstat to print the age of the cache, assuming it can
+be found.
+
+If you set `QSCACHE_BYPASS` to `true`, the cache will be bypassed regardless of
+which options are set, and the scheduler version of qstat will instead be
+called.

qstat_cache-3.0/bin/qstat

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+if __name__ == "__main__":
+    import os, sys
+    
+    my_root = os.path.dirname(os.path.realpath(__file__)).rsplit("/", 1)[0]
+    sys.path.insert(0, f"{my_root}/src")
+
+    from qscache import qscache
+    qscache.main()

qstat_cache-3.0/pyproject.toml

@@ -0,0 +1,34 @@
+# Setuptools version required for PEP 639
+[build-system]
+requires = [
+    "setuptools >= 77.0",
+    "setuptools-scm>=8",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "qstat-cache"
+dynamic = [ "version" ]
+authors = [
+    { name="Brian Vanderwende", email="vanderwb@ucar.edu" },
+]
+description = "A caching qstat that reduces load on the PBS server"
+readme = "README.md"
+requires-python = ">=3.6"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+    "Topic :: System :: Distributed Computing",
+]
+
+license = "MIT"
+license-files = ["LICENSE"]
+
+[project.urls]
+homepage = "https://github.com/NCAR/qstat-cache"
+issues = "https://github.com/NCAR/qstat-cache/issues"
+
+[project.scripts]
+qstat = "qscache.qscache:main"
+
+[tool.setuptools_scm]

qstat_cache-3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

qstat_cache-3.0/src/qscache/cfg/casper.cfg

@@ -0,0 +1,30 @@
+[paths]
+Temp = ${install_dir}/temp/casper
+Data = ${install_dir}/data
+#Logs = /glade/derecho/scratch/csgteam/logs/qstat
+
+[cache]
+MaxWait = 20
+MaxAge = 300
+AgeDelay = 5
+Frequency = 60
+
+[history]
+MaxAge = 600
+
+[pbs]
+Qstat = /opt/pbs/bin/qstat
+Prefix = sudo -u csgteam
+
+[servermap]
+casper = casper-pbs
+derecho = desched1
+
+[privileges]
+Active = True
+
+[priv.all]
+Users = *
+
+[priv.env]
+Groups = csg

qstat_cache-3.0/src/qscache/cfg/derecho.cfg

@@ -0,0 +1,30 @@
+[paths]
+Temp = ${install_dir}/temp/derecho
+Data = ${install_dir}/data
+#Logs = /glade/derecho/scratch/csgteam/logs/qstat
+
+[cache]
+MaxWait = 20
+MaxAge = 300
+AgeDelay = 5
+Frequency = 60
+
+[history]
+MaxAge = 600
+
+[pbs]
+Qstat = /opt/pbs/bin/qstat
+Prefix = sudo -u csgteam
+
+[servermap]
+casper = casper-pbs
+derecho = desched1
+
+[privileges]
+Active = True
+
+[priv.all]
+Users = *
+
+[priv.env]
+Groups = csg

qstat_cache-3.0/src/qscache/cfg/gust.cfg

@@ -0,0 +1,29 @@
+[paths]
+Temp = ${install_dir}/temp/gust
+Data = ${install_dir}/data
+Logs = /glade/gust/scratch/csgteam/logs/qstat
+
+[cache]
+MaxWait = 20
+MaxAge = 300
+AgeDelay = 5
+Frequency = 60
+
+[history]
+MaxAge = 600
+
+[pbs]
+Qstat = /opt/pbs/bin/qstat
+Prefix = sudo -u csgteam
+
+[servermap]
+gust = gusched1
+
+[privileges]
+Active = True
+
+[priv.all]
+Users = *
+
+[priv.env]
+Groups = csg