lsst-ctrl-execute 28.2025.500__tar.gz

lsst_ctrl_execute-28.2025.500/.github/workflows/build.yaml

@@ -0,0 +1,115 @@
+name: build_and_test
+
+on:
+  push:
+    branches:
+      - main
+      - "u/**"
+    tags:
+      - "*"
+  pull_request:
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Need to clone everything for the git tags.
+          fetch-depth: 0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          uv sync
+
+      - name: Run tests
+        run: >-
+          uv run pytest -r a -v -n 3
+          --cov=lsst.ctrl.execute --cov-report=xml --cov-report=term --cov-branch
+          --junitxml=junit.xml -o junit_family=legacy
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v4
+        with:
+          files: ./coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+  check-changes:
+    outputs:
+      skip: ${{ steps.check.outputs.skip }}
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Check if weekly changed
+        id: check
+        run: |
+          # Get SHA hashes for most recent weekly tags
+          weekly_sha=$(git tag -l 'w.*' | while read tag; do
+            git rev-list -n 1 "${tag}"
+          done)
+
+          echo "Weekly tag SHA ${weekly_sha}"
+          # Extract the current tag and its SHA
+          current_tag=${GITHUB_REF#refs/tags/}
+          echo "Current tag: ${current_tag}"
+          current_sha=$(git rev-list -1 "${current_tag}") || echo "no_value"
+          echo "Current sha: ${current_sha}"
+          # Count occurrences of the current SHA in the weekly SHA list
+          n=$(echo "${weekly_sha}" | grep -c "${current_sha}") || echo "0"
+          echo "Current tag ${current_tag} (${current_sha}) SHA found ${n} time(s)"
+
+          # Determine whether to skip the upload based on the count
+          if [ "${n}" -gt 1 ]; then
+            echo "Skip upload"
+            echo "skip=true" >> "${GITHUB_OUTPUT}"
+          else
+            echo "Enable upload"
+            echo "skip=false" >> "${GITHUB_OUTPUT}"
+          fi
+
+  pypi:
+    runs-on: ubuntu-latest
+    needs: [build_and_test, check-changes]
+    permissions:
+      id-token: write
+    if: "${{ ! startsWith(github.ref, 'refs/tags/w.') || needs.check-changes.outputs.skip == 'false' }}"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Need to clone everything to embed the version.
+          fetch-depth: 0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+          enable-cache: true
+          python-version: "3.11"
+
+      - name: Build and create distribution
+        run: |
+          uv build
+
+      - name: Upload
+        uses: pypa/gh-action-pypi-publish@release/v1

lsst_ctrl_execute-28.2025.500/.github/workflows/formatting.yaml

@@ -0,0 +1,11 @@
+name: Check Python formatting
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  call-workflow:
+    uses: lsst/rubin_workflows/.github/workflows/formatting.yaml@main

lsst_ctrl_execute-28.2025.500/.github/workflows/lint.yaml

@@ -0,0 +1,11 @@
+name: lint
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  call-workflow:
+    uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main

lsst_ctrl_execute-28.2025.500/.github/workflows/rebase_checker.yaml

@@ -0,0 +1,8 @@
+---
+name: Check that 'main' is not merged into the development branch
+
+on: pull_request
+
+jobs:
+  call-workflow:
+    uses: lsst/rubin_workflows/.github/workflows/rebase_checker.yaml@main

lsst_ctrl_execute-28.2025.500/.gitignore

@@ -0,0 +1,234 @@
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.env.*
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# PyPI configuration file
+.pypirc
+
+# Local Ignores
+
+# Do not commit the local version file
+**/version.py
+
+# VS Code
+.vscode/
+
+# Ruff
+# Ruff usually installs a .gitignore file in its directory.
+.ruff_cache/
+
+# Scons
+.sconsign.dblite
+.sconf_temp
+tests/.tests/
+bin/*
+python/*.dist-info
+config.log
+_build.*

lsst_ctrl_execute-28.2025.500/.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+        args:
+          - "--unsafe"
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 24.8.0
+    hooks:
+      - id: black
+        # It is recommended to specify the latest version of Python
+        # supported by your project here, or alternatively use
+        # pre-commit's default_language_version, see
+        # https://pre-commit.com/#top_level-default_language_version
+        language_version: python3.10
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (python)
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8

lsst_ctrl_execute-28.2025.500/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: lsst-ctrl-execute
+Version: 28.2025.500
+Summary: Utilities for executing and managing workloads.
+Project-URL: Homepage, https://github.com/lsst/ctrl_execute
+Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
+License: BSD 3-Clause License
+Keywords: lsst
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Astronomy
+Requires-Python: <3.13.0,>=3.11.0
+Requires-Dist: lsst-ctrl-bps-htcondor>=28.2024.5000; sys_platform == 'linux'
+Requires-Dist: lsst-pex-config
+Requires-Dist: lsst-resources
+Requires-Dist: lsst-utils

lsst_ctrl_execute-28.2025.500/SConstruct

@@ -0,0 +1,4 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+
+scripts.BasicSConstruct("ctrl_execute", disableCc=True, noCfgFile=True)

lsst_ctrl_execute-28.2025.500/etc/configs/gordon_config.py

@@ -0,0 +1,7 @@
+# flake8: noqa
+config.platform.localScratch = "$HOME/condor_logs"
+config.platform.defaultRoot = "/oasis/scratch/ux453102/temp_project/lsst"
+config.platform.dataDirectory = (
+    "/oasis/scratch/ux453102/temp_project/lsst/stripe82/dr7/runs"
+)
+config.platform.fileSystemDomain = "sdsc.edu"

lsst_ctrl_execute-28.2025.500/etc/configs/lsst_config.py

@@ -0,0 +1,5 @@
+# flake8: noqa
+config.platform.localScratch = "$HOME/condor_logs"
+config.platform.defaultRoot = "/lsst/DC3root"
+config.platform.dataDirectory = "/lsst7/stripe82/dr7/runs"
+config.platform.fileSystemDomain = "ncsa.illinois.edu"

lsst_ctrl_execute-28.2025.500/etc/scripts/generateDag.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python
+
+#
+# LSST Data Management System
+# Copyright 2008, 2009, 2010 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+import argparse
+import sys
+from shlex import split as cmd_split
+
+
+def _line_to_args(self, line):
+    for arg in cmd_split.split(line, comments=True, posix=True):
+        if not arg.strip():
+            continue
+        yield arg
+
+
+def makeArgumentParser(description, inRootsRequired=True, addRegistryOption=True):
+    parser = argparse.ArgumentParser(
+        description=description,
+        fromfile_prefix_chars="@",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=" \n" "ly.",
+    )
+    parser.convert_arg_line_to_args = _line_to_args
+
+    parser.add_argument(
+        "-s", "--source", dest="source", help="Source site for file transfer."
+    )
+
+    parser.add_argument("-w", "--workerdir", dest="workerdir", help="workers directory")
+
+    parser.add_argument("-t", "--template", dest="template", help="template file")
+
+    parser.add_argument("-p", "--prescript", dest="prescript", help="pre shell script")
+
+    parser.add_argument("-r", "--runid", dest="runid", help="runid of production")
+
+    parser.add_argument(
+        "-i", "--idsPerJob", dest="idsPerJob", help="number of ids to run per job"
+    )
+
+    return parser
+
+
+def writeVarsInfo(output, count, myDataTotal, visit, runid):
+    output.write("VARS A" + count + ' var1="' + myDataTotal + '" \n')
+    output.write("VARS A" + count + ' var2="' + count + '" \n')
+    output.write("VARS A" + count + ' visit="' + visit + '" \n')
+    output.write("VARS A" + count + ' runid="' + runid + '" \n')
+    output.write("VARS A" + count + ' workerid="' + count + '" \n')
+
+
+def writeMapInfo(output, count, newDataTotal, myDataTotal):
+    output.write(count + "  " + newDataTotal + "\n")
+    output.write(count + "  " + myDataTotal + "\n")
+
+
+def writeDagFile(
+    pipeline, templateFile, infile, workerdir, prescriptFile, runid, idsPerJob
+):
+    """
+    Write Condor Dag Submission files.
+    """
+
+    print("Writing DAG file ")
+    print("idsPerJob")
+    print(idsPerJob)
+
+    listSize = idsPerJob
+
+    outname = pipeline + ".diamond.dag"
+    mapname = pipeline + ".mapping"
+    configname = pipeline + ".config"
+
+    print(outname)
+
+    mapObj = open(mapname, "w")
+    outObj = open(outname, "w")
+    configObj = open(configname, "w")
+
+    configObj.write("DAGMAN_MAX_SUBMITS_PER_INTERVAL=1000\n")
+    configObj.write("DAGMAN_SUBMIT_DELAY=0\n")
+    configObj.write("DAGMAN_USER_LOG_SCAN_INTERVAL=5\n")
+
+    outObj.write("CONFIG %s\n" % configname)
+    outObj.write("JOB A " + workerdir + "/" + pipeline + ".pre\n")
+    outObj.write("JOB B " + workerdir + "/" + pipeline + ".post\n")
+    outObj.write(" \n")
+
+    print("prescriptFile = ", prescriptFile)
+    if prescriptFile is not None:
+        outObj.write("SCRIPT PRE A " + prescriptFile + "\n")
+
+    #
+    # note: we make multiple passes through the input file because it could be
+    # quite large
+    #
+
+    #
+    # A first pass through the Input File to define the individual Jobs
+    # Loop over input entries
+    #
+    fileObj = open(infile, "r")
+    count = 0
+    acount = 0
+    myDataTotal = ""
+    myDataList = []
+    newDataTotal = ""
+    newDataList = []
+    for aline in fileObj:
+        acount += 1
+        myData = aline.rstrip()
+
+        #
+        # Construct a string without spaces from the dataids for a job
+        # suitable for a unix file name
+        #
+        # Searching for a space detects
+        # extended input like :  visit=887136081 raft=2,2 sensor=0,1
+        # If there is no space, the dataid is something simple like
+        # a skytile id
+        newData = myData
+        visit = str(count // 100)
+
+        myDataList.append(myData)
+        newDataList.append(newData)
+
+        # For example:
+        # VARS A1 var1="visit=887136081 raft=2,2 sensor=0,1"
+        # VARS A1 var2="visit-887136081:raft-2_2:sensor-0_1"
+
+        if acount == listSize:
+            count += 1
+            outObj.write(
+                "JOB A" + str(count) + " " + workerdir + "/" + templateFile + "\n"
+            )
+            myDataTotal = " X ".join(myDataList)
+            newDataTotal = "_".join(newDataList)
+            writeVarsInfo(outObj, str(count), myDataTotal, visit, runid)
+            writeMapInfo(mapObj, str(count), newDataTotal, myDataTotal)
+            # PARENT A CHILD A1
+            # PARENT A1 CHILD B
+            outObj.write("PARENT A CHILD A" + str(count) + " \n")
+            outObj.write("PARENT A" + str(count) + " CHILD B \n")
+
+            acount = 0
+            myDataTotal = ""
+            newDataTotal = ""
+            myDataList = []
+            newDataList = []
+            outObj.write("\n")
+
+    # if acount != 0, then we have left over ids to deal with, and need
+    # to create one more worker to do so.
+    if acount != 0:
+        count += 1
+        outObj.write("JOB A" + str(count) + " " + workerdir + "/" + templateFile + "\n")
+        myDataTotal = " X ".join(myDataList)
+        newDataTotal = "_".join(newDataList)
+        writeVarsInfo(outObj, str(count), myDataTotal, visit, runid)
+        writeMapInfo(mapObj, str(count), newDataTotal, myDataTotal)
+        outObj.write("PARENT A CHILD A" + str(count) + " \n")
+        outObj.write("PARENT A" + str(count) + " CHILD B \n")
+        outObj.write("\n")
+
+    fileObj.close()
+    configObj.close()
+    outObj.close()
+    mapObj.close()
+
+
+def main():
+    print("Starting generateDag.py")
+    parser = makeArgumentParser(
+        description="generateDag.py write a Condor DAG for job submission"
+        "by reading input list and writing the attribute as an argument."
+    )
+    print("Created parser")
+    ns = parser.parse_args()
+    print("Parsed Arguments")
+    print(ns)
+    print(ns.idsPerJob)
+
+    pipeline = "Workflow"
+
+    writeDagFile(
+        pipeline,
+        ns.template,
+        ns.source,
+        ns.workerdir,
+        ns.prescript,
+        ns.runid,
+        int(ns.idsPerJob),
+    )
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()