lsst-ctrl-bps-parsl 29.2025.4800__tar.gz → 30.0.0__tar.gz

{lsst_ctrl_bps_parsl-29.2025.4800/python/lsst_ctrl_bps_parsl.egg-info → lsst_ctrl_bps_parsl-30.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-parsl
-Version: 29.2025.4800
+Version: 30.0.0
 Summary: Parsl-based plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/pyproject.toml

@@ -55,7 +55,7 @@ version = { attr = "lsst_versions.get_lsst_version" }
     package_dir = "python"
     filename = "doc/lsst.ctrl.bps.parsl/CHANGES.rst"
     directory = "doc/changes"
-    title_format = "lsst-ctrl-bps-parsl {version} {project_date}"
+    title_format = "lsst-ctrl-bps-parsl {version} ({project_date})"
     issue_format = "`{issue} <https://rubinobs.atlassian.net/browse/{issue}>`_"
 
     [[tool.towncrier.type]]

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/python/lsst/ctrl/bps/parsl/job.py

@@ -237,11 +237,18 @@ class ParslJob:
             ("request_cpus", "cores", None),
             ("request_disk", "disk", None),  # Both are MB
             ("request_walltime", "running_time_min", None),  # Both are minutes
+            ("priority", "priority", None),
         ):
             value = getattr(self.generic, bps_name)
-            if scale is not None:
+            if value is not None and scale is not None:
                 value *= scale
-            resources[parsl_name] = value
+            # Parsl's `HighThroughputExecutor` cannot have
+            # `priority=None`, but it can be omitted.  By contrast,
+            # `WorkQueueExecutor` needs to have the other resource
+            # requests provided, but `priority` can be omitted, so we
+            # need special handling for `priority`.
+            if (parsl_name == "priority" and value is not None) or parsl_name != "priority":
+                resources[parsl_name] = value
         return resources
 
     def get_future(
@@ -249,7 +256,7 @@ class ParslJob:
         app: BashApp,
         inputs: list[Future],
         command_prefix: str | None = None,
-        add_resources: bool = False,
+        resource_list: list | None = None,
     ) -> Future | None:
         """Get the parsl app future for the job.
 
@@ -265,10 +272,8 @@ class ParslJob:
         command_prefix : `str`, optional
             Bash commands to execute before the job command, e.g., for setting
             the environment.
-        add_resources : `bool`
-            Add resource specification when submitting the job? This is only
-            appropriate for the ``WorkQueue`` executor; other executors will
-            raise an exception.
+        resource_list : `list`, optional
+            List of resource specifications to pass to the Parsl executor.
 
         Returns
         -------
@@ -283,7 +288,11 @@ class ParslJob:
             command = self.evaluate_command_line(command)
             if command_prefix:
                 command = command_prefix + "\n" + command
-            resources = self.get_resources() if add_resources else {}
+            resources = (
+                {k: v for k, v in self.get_resources().items() if k in resource_list}
+                if resource_list is not None
+                else {}
+            )
 
             # Add a layer of indirection to which we can add a useful name.
             # This name is used by parsl for tracking workflow status.

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/python/lsst/ctrl/bps/parsl/site.py

@@ -56,16 +56,14 @@ class SiteConfig(ABC):
     ----------
     config : `BpsConfig`
         BPS configuration.
-    add_resources : `bool`
-        Add resource specification when submitting the job? This is only
-        appropriate for the ``WorkQueue`` executor; other executors will
-        raise an exception.
+    resource_list : `list`, optional
+        List of parsl resource specifications to pass to the executor.
     """
 
-    def __init__(self, config: BpsConfig, add_resources: bool = False):
+    def __init__(self, config: BpsConfig, resource_list: list = None):
         self.config = config
         self.site = self.get_site_subconfig(config)
-        self.add_resources = add_resources
+        self.resource_list = resource_list
 
     @staticmethod
     def get_site_subconfig(config: BpsConfig) -> BpsConfig:

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/python/lsst/ctrl/bps/parsl/sites/__init__.py

@@ -29,3 +29,4 @@ from .local import *
 from .slurm import *
 from .torque import *
 from .work_queue import *
+from .ccin2p3 import *

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/python/lsst/ctrl/bps/parsl/sites/ccin2p3.py

@@ -122,6 +122,8 @@ class Ccin2p3(SiteConfig):
     ]
 
     def __init__(self, *args, **kwargs):
+        # Have BPS-defined resource requests for each job passed to executor.
+        kwargs["resource_list"] = ["priority"]
         super().__init__(*args, **kwargs)
         self._account = get_bps_config_value(self.site, ".account", str, self.DEFAULT_ACCOUNT)
         self._scheduler_options = get_bps_config_value(

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/python/lsst/ctrl/bps/parsl/sites/local.py

@@ -47,6 +47,11 @@ class Local(SiteConfig):
     ``site.<computeSite>.cores`` (`int`).
     """
 
+    def __init__(self, *args, **kwargs):
+        # Have BPS-defined resource requests for each job passed to executor.
+        kwargs["resource_list"] = ["priority"]
+        super().__init__(*args, **kwargs)
+
     def get_executors(self) -> list[ParslExecutor]:
         """Get a list of executors to be used in processing.
 

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/python/lsst/ctrl/bps/parsl/sites/slurm.py

@@ -80,6 +80,11 @@ class Slurm(SiteConfig):
       script (each line usually starting with ``#SBATCH``).
     """
 
+    def __init__(self, *args, **kwargs):
+        # Have BPS-defined resource requests for each job passed to executor.
+        kwargs["resource_list"] = ["priority"]
+        super().__init__(*args, **kwargs)
+
     def make_executor(
         self,
         label: str,

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/python/lsst/ctrl/bps/parsl/sites/torque.py

@@ -75,6 +75,11 @@ class Torque(SiteConfig):
       script (each line usually starting with ``#PBS``).
     """
 
+    def __init__(self, *args, **kwargs):
+        # Have BPS-defined resource requests for each job passed to executor.
+        kwargs["resource_list"] = ["priority"]
+        super().__init__(*args, **kwargs)
+
     def make_executor(
         self,
         label: str,

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/python/lsst/ctrl/bps/parsl/sites/work_queue.py

@@ -59,7 +59,7 @@ class WorkQueue(SiteConfig):
         Parameters forwarded to base class constructor.
     **kwargs : `~typing.Any`
         Keyword arguments passed to base class constructor, augmented by
-        the ``add_resources`` argument.
+        the ``resource_list`` argument.
 
     Notes
     -----
@@ -79,7 +79,7 @@ class WorkQueue(SiteConfig):
 
     def __init__(self, *args, **kwargs):
         # Have BPS-defined resource requests for each job passed to work_queue.
-        kwargs["add_resources"] = True
+        kwargs["resource_list"] = ["memory", "cores", "disk", "running_time_min", "priority"]
         super().__init__(*args, **kwargs)
 
     def make_executor(

lsst_ctrl_bps_parsl-30.0.0/python/lsst/ctrl/bps/parsl/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "30.0.0"

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/python/lsst/ctrl/bps/parsl/workflow.py

@@ -278,7 +278,7 @@ class ParslWorkflow(BaseWmsWorkflow):
             self.apps[label],
             [ff for ff in inputs if ff is not None],
             self.command_prefix,
-            self.site.add_resources,
+            self.site.resource_list,
         )
 
     def load_dfk(self):

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0/python/lsst_ctrl_bps_parsl.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-parsl
-Version: 29.2025.4800
+Version: 30.0.0
 Summary: Parsl-based plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/python/lsst_ctrl_bps_parsl.egg-info/SOURCES.txt

@@ -30,4 +30,5 @@ python/lsst_ctrl_bps_parsl.egg-info/top_level.txt
 python/lsst_ctrl_bps_parsl.egg-info/zip-safe
 tests/test_config.py
 tests/test_import.py
-tests/test_job.py
+tests/test_job.py
+tests/test_sites.py

{lsst_ctrl_bps_parsl-29.2025.4800 → lsst_ctrl_bps_parsl-30.0.0}/tests/test_job.py

@@ -82,3 +82,28 @@ def testInitWithoutTemplate():
     parsl_job = ParslJob(gwjob, config, {})
     assert parsl_job.stdout == os.path.join(submit_path, "logs/job3.stdout")
     assert parsl_job.stderr == os.path.join(submit_path, "logs/job3.stderr")
+
+
+def testJobResourceLists():
+    """Test that ParslJob.get_resources() returns resource specifications
+    expected by Parsl Executors, including special handling needed for the
+    `priority` request.
+    """
+    # Test doesn't actually use directory
+    submit_path = os.path.join(TESTDIR, "without_template")
+    config = BpsConfig({"subDirTemplate": "", "submitPath": submit_path}, search_order=[])
+    gwjob = GenericWorkflowJob("job", label="label")
+
+    # If gwjob.priority is None, for Parsl's HighThroughputExecutor
+    # `"priority"` must be omitted, but WorkQueueExecutor requires
+    # `("cores", "disk", "memory")` to be provided.
+    gwjob.priority = None
+    parsl_job = ParslJob(gwjob, config, {})
+    assert "priority" not in parsl_job.get_resources()
+    for resource in ("cores", "disk", "memory"):
+        assert resource in parsl_job.get_resources()
+
+    # Check case where gwjob.priority is not None.
+    gwjob.priority = 1
+    for resource in ("cores", "disk", "memory", "priority"):
+        assert resource in parsl_job.get_resources()

lsst_ctrl_bps_parsl-30.0.0/tests/test_sites.py

@@ -0,0 +1,78 @@
+# This file is part of ctrl_bps_parsl.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org) and the LSST DESC (https://www.lsstdesc.org/).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+import platform
+
+from parsl.executors import HighThroughputExecutor, WorkQueueExecutor
+
+from lsst.ctrl.bps import BpsConfig
+from lsst.ctrl.bps.parsl.sites import Ccin2p3, Local, LocalSrunWorkQueue, Slurm, Torque
+
+
+def testSiteResourceLists():
+    """Test compute site resource lists."""
+    # Provide a minimal config that allows compute site classes to be created.
+    config = BpsConfig(
+        {
+            "submitPath": ".",
+            "operator": "operator",
+            "computeSite": "local",
+            "site": {
+                "local": {
+                    "class": "lsst.ctrl.bps.parsl.sites.Local",
+                    "cores": 1,
+                    "nodes": 1,
+                    "walltime": "00:01:00",
+                }
+            },
+        }
+    )
+
+    expected_resources = {
+        HighThroughputExecutor: {"priority"},
+        WorkQueueExecutor: {"memory", "cores", "disk", "running_time_min", "priority"},
+    }
+
+    site_classes = [Local, Slurm, Torque]
+    if platform.machine() in ("aarch64", "x86_64"):
+        # If running on a supported architecture, then add Ccin2p3 site class.
+        site_classes.append(Ccin2p3)
+
+    try:
+        # If the ndcctools module is available, then add WorkQueue
+        # subclasses to the site_classes list.
+        import ndcctools  # noqa F401
+
+        site_classes.append(LocalSrunWorkQueue)
+    except ImportError:
+        pass
+
+    for site_class in site_classes:
+        compute_site = site_class(config)
+        executor_type = type(compute_site.get_executors()[0])
+        assert expected_resources[executor_type] == set(compute_site.resource_list)

lsst_ctrl_bps_parsl-29.2025.4800/python/lsst/ctrl/bps/parsl/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.4800"