lsst-ctrl-execute 28.2025.500__py3-none-any.whl

lsst/ctrl/execute/allocatorParser.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python
+
+#
+# LSST Data Management System
+# Copyright 2008-2016 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+import argparse
+
+
+class AllocatorParser:
+    """An argument parser for node allocation requests.
+
+    Parameters
+    ----------
+    basename : `str`
+        The name used to identify the running program
+    """
+
+    def __init__(self, basename):
+        """Construct an AllocatorParser
+        @param argv: list containing the command line arguments
+        @return: the parser options and remaining arguments
+        """
+
+        self.defaults = {}
+        self.args = self.parseArgs(basename)
+
+    def parseArgs(self, basename) -> argparse.Namespace:
+        """Parse command line, and test for required arguments
+
+        Parameters
+        ----------
+        argv: `list`
+            list of strings containing the command line arguments
+
+        Returns
+        -------
+        The parser options and remaining arguments
+        """
+
+        parser = argparse.ArgumentParser(prog=basename)
+        parser.add_argument(
+            "platform", type=str, default="s3df", help="node allocation platform"
+        )
+        parser.add_argument(
+            "--auto",
+            action="store_true",
+            dest="auto",
+            help="use automatic detection of jobs to determine glide-ins",
+        )
+        parser.add_argument(
+            "-n",
+            "--node-count",
+            action="store",
+            default=None,
+            dest="nodeCount",
+            help="number of glideins to submit; these are chunks of a node, size the number of cores/cpus",
+            type=int,
+            required=True,
+        )
+        parser.add_argument(
+            "-c",
+            "--cpus",
+            action="store",
+            default=16,
+            dest="cpus",
+            help="cores / cpus per glidein",
+            type=int,
+            required=False,
+        )
+        parser.add_argument(
+            "-a",
+            "--account",
+            action="store",
+            default="rubin:developers",
+            dest="account",
+            help="Slurm account for glidein job",
+            type=str,
+        )
+        parser.add_argument(
+            "--collector",
+            action="store",
+            default=None,
+            dest="collector",
+            help="machine name of nondefault htcondor collector",
+            type=str,
+            required=False,
+        )
+        parser.add_argument(
+            "--collector-port",
+            action="store",
+            default=9618,
+            dest="collectorport",
+            help="port used for nondefault htcondor collector",
+            type=int,
+            required=False,
+        )
+        parser.add_argument(
+            "-s",
+            "--qos",
+            action="store",
+            default=None,
+            dest="qos",
+            help="Slurm qos for glidein job",
+            type=str,
+        )
+        parser.add_argument(
+            "-m",
+            "--maximum-wall-clock",
+            action="store",
+            dest="maximumWallClock",
+            default=None,
+            help="maximum wall clock time; e.g., 3600, 10:00:00, 6-00:00:00, etc",
+            type=str,
+            required=True,
+        )
+        parser.add_argument(
+            "-q",
+            "--queue",
+            action="store",
+            dest="queue",
+            default="roma,milano",
+            help="queue / partition  name",
+        )
+        parser.add_argument(
+            "-O",
+            "--output-log",
+            action="store",
+            dest="outputLog",
+            default=None,
+            help="Output log filename; this option for PBS, unused with Slurm",
+        )
+        parser.add_argument(
+            "-E",
+            "--error-log",
+            action="store",
+            dest="errorLog",
+            default=None,
+            help="Error log filename; this option for PBS, unused with Slurm",
+        )
+        parser.add_argument(
+            "-g",
+            "--glidein-shutdown",
+            action="store",
+            dest="glideinShutdown",
+            type=int,
+            default=None,
+            help="glide-in inactivity shutdown time in seconds",
+        )
+        parser.add_argument(
+            "--openfiles",
+            action="store",
+            dest="openfiles",
+            type=int,
+            default=20480,
+            help="set the limit on number of open files (fd) per process",
+        )
+        parser.add_argument(
+            "-p",
+            "--pack",
+            action="store_true",
+            dest="packnodes",
+            help="encourage nodes to pack jobs rather than spread",
+        )
+        parser.add_argument(
+            "-v", "--verbose", action="store_true", dest="verbose", help="verbose"
+        )
+        parser.add_argument(
+            "-r",
+            "--reservation",
+            action="store",
+            dest="reservation",
+            default=None,
+            help="target a particular Slurm reservation",
+        )
+        parser.add_argument(
+            "-d",
+            "--dynamic",
+            const="__default__",
+            nargs="?",
+            action="store",
+            dest="dynamic",
+            type=str,
+            default="__default__",
+            help="configure to use dynamic/partitionable slot; legacy option: this is always enabled now",
+        )
+
+        self.args = parser.parse_args()
+
+        return self.args
+
+    def getArgs(self):
+        """Accessor method to get arguments left after standard parsed options
+        are initialized.
+
+        Returns
+        -------
+        args: `argparse.Namespace`
+            remaining command line arguments
+        """
+        return self.args
+
+    def getPlatform(self):
+        """Accessor method to retrieve the "platform" that was specified on
+        the command line.
+
+        Returns
+        -------
+        platform: `str`
+            the name of the "platform"
+        """
+        return self.args.platform

lsst/ctrl/execute/condorConfig.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+#
+# LSST Data Management System
+# Copyright 2008-2016 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+import lsst.pex.config as pexConfig
+
+
+class PlatformConfig(pexConfig.Config):
+    """Platform specific information"""
+
+    defaultRoot = pexConfig.Field(
+        doc="remote root for working directories", dtype=str, default=None
+    )
+    localScratch = pexConfig.Field(
+        doc="local Condor scratch directory", dtype=str, default=None
+    )
+    idsPerJob = pexConfig.Field(
+        doc="number of ids to work on per job", dtype=int, default=1
+    )
+    dataDirectory = pexConfig.Field(
+        doc="remote directory where date that jobs will use is kept",
+        dtype=str,
+        default=None,
+    )
+    fileSystemDomain = pexConfig.Field(
+        doc="network domain name of remote system", dtype=str, default=None
+    )
+    eupsPath = pexConfig.Field(
+        doc="location of remote EUPS stack", dtype=str, default=None
+    )
+    nodeSetRequired = pexConfig.Field(
+        doc="is the nodeset required", dtype=bool, default=False
+    )
+    scheduler = pexConfig.Field(doc="scheduler type", dtype=str, default=None)
+    manager = pexConfig.Field(doc="workflow manager", dtype=str, default=None)
+    setup_using = pexConfig.Field(doc="environment setup type", dtype=str, default=None)
+    manager_software_home = pexConfig.Field(
+        doc="location of workflow manager software", dtype=str, default=None
+    )
+
+
+class CondorConfig(pexConfig.Config):
+    """A pex_config file describing the platform specific information required
+    to fill out templates for running ctrl_orca jobs
+    """
+
+    platform = pexConfig.ConfigField("platform configuration", PlatformConfig)
+
+
+class FakeTypeMap(dict):
+    def __init__(self, configClass):
+        self.configClass = configClass
+
+    def __getitem__(self, k):
+        return self.setdefault(k, self.configClass)

lsst/ctrl/execute/condorInfoConfig.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+
+#
+# LSST Data Management System
+# Copyright 2008-2016 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+import lsst.pex.config as pexConfig
+from lsst.ctrl.execute.findPackageFile import find_package_file
+
+
+class FakeTypeMap(dict):
+    def __init__(self, configClass):
+        self.configClass = configClass
+
+    def __getitem__(self, k):
+        return self.setdefault(k, self.configClass)
+
+
+class UserInfoConfig(pexConfig.Config):
+    """User information"""
+
+    name = pexConfig.Field(doc="user login name", dtype=str, default=None)
+    home = pexConfig.Field(doc="user home directory", dtype=str, default=None)
+    scratch = pexConfig.Field(doc="user scratch directory", dtype=str, default=None)
+
+
+class UserConfig(pexConfig.Config):
+    """User specific information"""
+
+    user = pexConfig.ConfigField(doc="user", dtype=UserInfoConfig)
+
+
+class CondorInfoConfig(pexConfig.Config):
+    """A pex_config file describing the platform specific information about
+    remote user logins.
+    """
+
+    platform = pexConfig.ConfigChoiceField("platform info", FakeTypeMap(UserConfig))
+
+
+if __name__ == "__main__":
+    config = CondorInfoConfig()
+    filename = find_package_file("condor-info.py")
+    config.loadFromStream(filename.read())
+
+    for i in config.platform:
+        print(i)

lsst/ctrl/execute/envString.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+#
+# LSST Data Management System
+# Copyright 2008-2016 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+import os
+
+
+def resolve(input: str) -> str:
+    """Render a string with any `$`-prefixed words substituted with a matching
+    environment variable.
+
+    .. deprecated:: w.2025.05
+        `lsst.ctrl.execute.envString.resolve` is deprecated and no longer used
+        by any internal APIs because `lsst.resource.ResourcePath` handles
+        environment variable expansion.
+
+    Parameters
+    ----------
+    input : `str`
+        A string containing environment variables to resolve.
+
+    Raises
+    ------
+    RuntimeError
+        If the environment variable does not exist
+    """
+
+    if "$" in (retVal := os.path.expandvars(input)):
+        raise RuntimeError(f"couldn't resolve all environment variables in {retVal}")
+    return retVal

lsst/ctrl/execute/findPackageFile.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+
+#
+# LSST Data Management System
+# Copyright 2008-2016 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+import os
+import sys
+
+import lsst.utils
+from lsst.resources import ResourcePath
+
+
+def find_package_file(
+    filename: str, kind: str = "config", platform: str | None = None
+) -> ResourcePath:
+    """Find a package file from a set of candidate locations.
+
+    Parameters
+    ----------
+    filename : `str`
+        The unqualified name of a file to locate.
+
+    kind : `str`
+        The name of a subdirectory in which to look for the file within a
+        package location, relative to an ``etc/`` directory.
+
+    platform : `str` | `None`
+        The name of a platform plugin in which to look for the file, or `None`
+        if no platform plugin should be searched.
+
+    Returns
+    -------
+    `lsst.resources.ResourcePath`
+
+    Raises
+    ------
+    FileNotFoundError
+        If a requested file object cannot be located in the candidate hierarchy
+
+    Notes
+    -----
+    The candidate locations are, in descending order of preference:
+    - An ``.lsst`` directory in the user's home directory.
+    - An ``lsst`` directory in the user's ``$XDG_CONFIG_HOME`` directory
+    - An ``etc/{kind}`` directory in the EUPS stack environment for the
+      platform.
+    - An ``etc/{kind}`` directory in the current Python environment/venv shared
+      data directory.
+    - An ``etc/{kind}`` directory in an installed ``lsst.ctrl.platform.*``
+      package.
+    - An ``etc/{kind}`` directory in the ``lsst.ctrl.execute`` package.
+    """
+    # If the path, after expansion, is absolute, we don't need to go looking
+    # for it, it should be exactly where it is.
+    if (_filename := ResourcePath(filename, forceAbsolute=False)).isabs():
+        return _filename
+
+    home_dir = os.getenv("HOME", "~")
+    xdg_config_home = os.getenv("XDG_CONFIG_HOME", "~/.config")
+    try:
+        platform_pkg_dir = lsst.utils.getPackageDir(f"ctrl_platform_{platform}")
+    except (LookupError, ValueError):
+        platform_pkg_dir = None
+
+    file_candidates = [
+        ResourcePath(home_dir).join(".lsst").join(_filename),
+        ResourcePath(xdg_config_home).join("lsst").join(_filename),
+        (
+            ResourcePath(platform_pkg_dir).join("etc").join(kind).join(_filename)
+            if platform_pkg_dir
+            else None
+        ),
+        ResourcePath(sys.exec_prefix).join("etc").join(kind).join(_filename),
+        (
+            ResourcePath(
+                f"resource://lsst.ctrl.platform.{platform}/etc/{kind}/{_filename}"
+            )
+            if platform
+            else None
+        ),
+        ResourcePath(f"resource://lsst.ctrl.execute/etc/{kind}/{_filename}"),
+    ]
+    try:
+        found_file: ResourcePath = [
+            c for c in file_candidates if c is not None and c.exists()
+        ][0]
+    except IndexError:
+        raise FileNotFoundError(f"No file {filename} found in package file lookup")
+    return found_file

lsst/ctrl/execute/libexec/allocateNodes.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+
+#
+# LSST Data Management System
+# Copyright 2008-2016 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+import logging
+import sys
+from typing import Any
+
+from lsst.ctrl.execute.allocator import Allocator
+from lsst.ctrl.execute.allocatorParser import AllocatorParser
+from lsst.ctrl.execute.condorConfig import CondorConfig
+from lsst.ctrl.execute.findPackageFile import find_package_file
+from lsst.ctrl.execute.namedClassFactory import NamedClassFactory
+
+_LOG = logging.getLogger("lsst.ctrl.execute")
+
+
+def setup_logging(options: dict[str, Any] | None = None) -> None:
+    """Configure logger.
+
+    Parameters
+    ----------
+    options : dict[str, Any]
+       Logger settings. The key/value pairs it contains will be used to
+       override corresponding default settings.  If empty or None (default),
+       logger will be set up with default settings.
+    """
+    settings = {
+        "datefmt": "%Y-%m-%dT%H:%M:%S%z",
+        "format": "%(levelname)s %(asctime)s %(name)s - %(message)s",
+        "level": logging.INFO,
+        "stream": sys.stderr,
+    }
+    if options is not None:
+        settings |= options
+    logging.basicConfig(**settings)
+
+
+def main():
+    """Allocates Condor glide-in nodes a scheduler on a remote Node."""
+
+    p = AllocatorParser(sys.argv[0])
+
+    options = {}
+    if p.args.verbose:
+        options = {"level": logging.DEBUG}
+    setup_logging(options)
+
+    platform = p.getPlatform()
+
+    # load the CondorConfig file
+    execConfigName = find_package_file("execConfig.py", platform=platform)
+    configuration = CondorConfig()
+    configuration.loadFromStream(execConfigName.read())
+
+    # create the plugin class
+    schedulerName = configuration.platform.scheduler
+    schedulerClass = NamedClassFactory.createClass(
+        "lsst.ctrl.execute." + schedulerName + "Plugin"
+    )
+
+    # create the plugin
+    condor_info_file = find_package_file("condor-info.py", platform=platform)
+    scheduler: Allocator = schedulerClass(
+        platform, p.getArgs(), configuration, condor_info_file
+    )
+
+    # submit the request
+    scheduler.submit()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

lsst/ctrl/execute/libexec/dagIdInfo.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+#
+# LSST Data Management System
+# Copyright 2008-2016 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+import errno
+import os
+import re
+import sys
+
+
+# extracts a line from a DAG file to show which ids were processed for a
+# particular dag node
+def main():
+    if len(sys.argv) != 3:
+        print("usage:  %s dagNodeName filename" % os.path.basename(sys.argv[0]))
+        return errno.EINVAL
+
+    dagNode = sys.argv[1]
+    filename = sys.argv[2]
+
+    if not os.path.exists(filename):
+        print("file %s not found" % filename)
+        return errno.ENOENT
+
+    ex = r"VARS %s var1=\"(?P<idlist>.+?)\"" % dagNode
+    with open(filename) as file:
+        for line in file:
+            line = line.rstrip(" \n")
+
+            # look for the line with the dagnode name in it
+            # and extract everything after "var1", but not the quotes
+            values = re.search(ex, line)
+            if values is None:
+                continue
+            ids = values.groupdict()["idlist"]
+            print(ids)
+            break
+
+
+if __name__ == "__main__":
+    sys.exit(main())