lsst-ctrl-bps-parsl 30.0.0rc2__py3-none-any.whl → 30.0.0rc3__py3-none-any.whl

lsst/ctrl/bps/parsl/job.py

@@ -237,11 +237,18 @@ class ParslJob:
             ("request_cpus", "cores", None),
             ("request_disk", "disk", None),  # Both are MB
             ("request_walltime", "running_time_min", None),  # Both are minutes
+            ("priority", "priority", None),
         ):
             value = getattr(self.generic, bps_name)
-            if scale is not None:
+            if value is not None and scale is not None:
                 value *= scale
-            resources[parsl_name] = value
+            # Parsl's `HighThroughputExecutor` cannot have
+            # `priority=None`, but it can be omitted.  By contrast,
+            # `WorkQueueExecutor` needs to have the other resource
+            # requests provided, but `priority` can be omitted, so we
+            # need special handling for `priority`.
+            if (parsl_name == "priority" and value is not None) or parsl_name != "priority":
+                resources[parsl_name] = value
         return resources
 
     def get_future(
@@ -249,7 +256,7 @@ class ParslJob:
         app: BashApp,
         inputs: list[Future],
         command_prefix: str | None = None,
-        add_resources: bool = False,
+        resource_list: list | None = None,
     ) -> Future | None:
         """Get the parsl app future for the job.
 
@@ -265,10 +272,8 @@ class ParslJob:
         command_prefix : `str`, optional
             Bash commands to execute before the job command, e.g., for setting
             the environment.
-        add_resources : `bool`
-            Add resource specification when submitting the job? This is only
-            appropriate for the ``WorkQueue`` executor; other executors will
-            raise an exception.
+        resource_list : `list`, optional
+            List of resource specifications to pass to the Parsl executor.
 
         Returns
         -------
@@ -283,7 +288,11 @@ class ParslJob:
             command = self.evaluate_command_line(command)
             if command_prefix:
                 command = command_prefix + "\n" + command
-            resources = self.get_resources() if add_resources else {}
+            resources = (
+                {k: v for k, v in self.get_resources().items() if k in resource_list}
+                if resource_list is not None
+                else {}
+            )
 
             # Add a layer of indirection to which we can add a useful name.
             # This name is used by parsl for tracking workflow status.

lsst/ctrl/bps/parsl/site.py

@@ -56,16 +56,14 @@ class SiteConfig(ABC):
     ----------
     config : `BpsConfig`
         BPS configuration.
-    add_resources : `bool`
-        Add resource specification when submitting the job? This is only
-        appropriate for the ``WorkQueue`` executor; other executors will
-        raise an exception.
+    resource_list : `list`, optional
+        List of parsl resource specifications to pass to the executor.
     """
 
-    def __init__(self, config: BpsConfig, add_resources: bool = False):
+    def __init__(self, config: BpsConfig, resource_list: list = None):
         self.config = config
         self.site = self.get_site_subconfig(config)
-        self.add_resources = add_resources
+        self.resource_list = resource_list
 
     @staticmethod
     def get_site_subconfig(config: BpsConfig) -> BpsConfig:

lsst/ctrl/bps/parsl/sites/__init__.py

@@ -29,3 +29,4 @@ from .local import *
 from .slurm import *
 from .torque import *
 from .work_queue import *
+from .ccin2p3 import *

lsst/ctrl/bps/parsl/sites/ccin2p3.py

@@ -122,6 +122,8 @@ class Ccin2p3(SiteConfig):
     ]
 
     def __init__(self, *args, **kwargs):
+        # Have BPS-defined resource requests for each job passed to executor.
+        kwargs["resource_list"] = ["priority"]
         super().__init__(*args, **kwargs)
         self._account = get_bps_config_value(self.site, ".account", str, self.DEFAULT_ACCOUNT)
         self._scheduler_options = get_bps_config_value(

lsst/ctrl/bps/parsl/sites/local.py

@@ -47,6 +47,11 @@ class Local(SiteConfig):
     ``site.<computeSite>.cores`` (`int`).
     """
 
+    def __init__(self, *args, **kwargs):
+        # Have BPS-defined resource requests for each job passed to executor.
+        kwargs["resource_list"] = ["priority"]
+        super().__init__(*args, **kwargs)
+
     def get_executors(self) -> list[ParslExecutor]:
         """Get a list of executors to be used in processing.
 

lsst/ctrl/bps/parsl/sites/slurm.py

@@ -80,6 +80,11 @@ class Slurm(SiteConfig):
       script (each line usually starting with ``#SBATCH``).
     """
 
+    def __init__(self, *args, **kwargs):
+        # Have BPS-defined resource requests for each job passed to executor.
+        kwargs["resource_list"] = ["priority"]
+        super().__init__(*args, **kwargs)
+
     def make_executor(
         self,
         label: str,

lsst/ctrl/bps/parsl/sites/torque.py

@@ -75,6 +75,11 @@ class Torque(SiteConfig):
       script (each line usually starting with ``#PBS``).
     """
 
+    def __init__(self, *args, **kwargs):
+        # Have BPS-defined resource requests for each job passed to executor.
+        kwargs["resource_list"] = ["priority"]
+        super().__init__(*args, **kwargs)
+
     def make_executor(
         self,
         label: str,

lsst/ctrl/bps/parsl/sites/work_queue.py

@@ -59,7 +59,7 @@ class WorkQueue(SiteConfig):
         Parameters forwarded to base class constructor.
     **kwargs : `~typing.Any`
         Keyword arguments passed to base class constructor, augmented by
-        the ``add_resources`` argument.
+        the ``resource_list`` argument.
 
     Notes
     -----
@@ -79,7 +79,7 @@ class WorkQueue(SiteConfig):
 
     def __init__(self, *args, **kwargs):
         # Have BPS-defined resource requests for each job passed to work_queue.
-        kwargs["add_resources"] = True
+        kwargs["resource_list"] = ["memory", "cores", "disk", "running_time_min", "priority"]
         super().__init__(*args, **kwargs)
 
     def make_executor(

lsst/ctrl/bps/parsl/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "30.0.0rc2"
+__version__ = "30.0.0rc3"

lsst/ctrl/bps/parsl/workflow.py

@@ -278,7 +278,7 @@ class ParslWorkflow(BaseWmsWorkflow):
             self.apps[label],
             [ff for ff in inputs if ff is not None],
             self.command_prefix,
-            self.site.add_resources,
+            self.site.resource_list,
         )
 
     def load_dfk(self):

{lsst_ctrl_bps_parsl-30.0.0rc2.dist-info → lsst_ctrl_bps_parsl-30.0.0rc3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-parsl
-Version: 30.0.0rc2
+Version: 30.0.0rc3
 Summary: Parsl-based plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

lsst_ctrl_bps_parsl-30.0.0rc3.dist-info/RECORD

@@ -0,0 +1,26 @@
+lsst/ctrl/bps/parsl/__init__.py,sha256=gU6EMvA--BVorVVMw_4i9-F_kran4V-25WzAeLJRsqE,1466
+lsst/ctrl/bps/parsl/configuration.py,sha256=Dq9hZW0fnmNHZoD_eAH3u2Dk0sNCZQ9NuSnFjTzFEi4,5731
+lsst/ctrl/bps/parsl/environment.py,sha256=xg2_WM-H77KOdn0M_Wi2sE7h7aqTSV1uxSiFHJmnu5w,2250
+lsst/ctrl/bps/parsl/job.py,sha256=WHkH4mhbnKhAGO_i8C612pRuUFzU0SJ8YghEoN920QI,12237
+lsst/ctrl/bps/parsl/service.py,sha256=mKM6C4mVTr3n0PsZWf5w0zc_o6qBiroQA1YfEgmde5s,3951
+lsst/ctrl/bps/parsl/site.py,sha256=1hFRB-475txuOR-jaztMglnjBxy9Ca2gZtI8Ptrvv2g,8614
+lsst/ctrl/bps/parsl/version.py,sha256=Tt7y1Knxuicvk026_H8m0GXmxEFM8XEWyfbDoHcAxbs,52
+lsst/ctrl/bps/parsl/workflow.py,sha256=9vtPa_bAX02aCZzoEG0bs2e0TWhNRm5RMiJ7BoCxBcE,11435
+lsst/ctrl/bps/parsl/sites/__init__.py,sha256=J9HpVUqqlIkve0tIEELPbicY81vz_liZKpBUrTA1D2Q,1464
+lsst/ctrl/bps/parsl/sites/ccin2p3.py,sha256=_H_vsq-WkO5QQRuTi8w2cv49ArPO7J-lxHu5c-exsgU,14722
+lsst/ctrl/bps/parsl/sites/local.py,sha256=s6NPuSbn_esI9rucQrpbg-y9R3nROBYVTk30KU99d7o,2831
+lsst/ctrl/bps/parsl/sites/nersc.py,sha256=obUiaHgHZj_23PIrYdihUnb8eoEJMs288wHy84jIzaQ,4549
+lsst/ctrl/bps/parsl/sites/princeton.py,sha256=F_dHTN5sm2NM2VjtJoWbtKfhkowBocIs5F7eSgjxc-0,5575
+lsst/ctrl/bps/parsl/sites/slac.py,sha256=IN04Nx0YvOl-GN33MB05zT0XG3IQ91rXri_EHz_dzTs,3893
+lsst/ctrl/bps/parsl/sites/slurm.py,sha256=5J0djEHyKZ0CXFNXY9zmL-m6e9VLcNc6Sd6mCwPYzgs,13031
+lsst/ctrl/bps/parsl/sites/torque.py,sha256=lNAh8-yZgWfsDRD7ryF0OwNV64tBv_7tCvs_xPHizNs,9830
+lsst/ctrl/bps/parsl/sites/work_queue.py,sha256=H6tjm8T5TTh1h5-vd7bR2yv-aFJc6hzEqYRnSnXMKwk,7095
+lsst_ctrl_bps_parsl-30.0.0rc3.dist-info/licenses/COPYRIGHT,sha256=M5ylfMNsihIJcPySQ5ZFjec4zROSZdlNN9oGHBhu7WM,181
+lsst_ctrl_bps_parsl-30.0.0rc3.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps_parsl-30.0.0rc3.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps_parsl-30.0.0rc3.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps_parsl-30.0.0rc3.dist-info/METADATA,sha256=dspZP-wIHizg8nU8XwvalcqXCyViA3tPi3be73NE7cg,2951
+lsst_ctrl_bps_parsl-30.0.0rc3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_ctrl_bps_parsl-30.0.0rc3.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps_parsl-30.0.0rc3.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps_parsl-30.0.0rc3.dist-info/RECORD,,

lsst_ctrl_bps_parsl-30.0.0rc2.dist-info/RECORD

@@ -1,26 +0,0 @@
-lsst/ctrl/bps/parsl/__init__.py,sha256=gU6EMvA--BVorVVMw_4i9-F_kran4V-25WzAeLJRsqE,1466
-lsst/ctrl/bps/parsl/configuration.py,sha256=Dq9hZW0fnmNHZoD_eAH3u2Dk0sNCZQ9NuSnFjTzFEi4,5731
-lsst/ctrl/bps/parsl/environment.py,sha256=xg2_WM-H77KOdn0M_Wi2sE7h7aqTSV1uxSiFHJmnu5w,2250
-lsst/ctrl/bps/parsl/job.py,sha256=O95H0_f-S92xgjNQeDBCzDtQkiXKLK8uzFcx6cm-AVA,11728
-lsst/ctrl/bps/parsl/service.py,sha256=mKM6C4mVTr3n0PsZWf5w0zc_o6qBiroQA1YfEgmde5s,3951
-lsst/ctrl/bps/parsl/site.py,sha256=L_wAF7qwMkcIChD0KTrIokZOD8E47ck-Qixf8L95sPk,8708
-lsst/ctrl/bps/parsl/version.py,sha256=CrAQNQjb1NNXqaNvScI53U2GvkmrnpRtyOiy0bIrcl4,52
-lsst/ctrl/bps/parsl/workflow.py,sha256=s6ayWoZ1CX2_mdFJvksiwUNChy9pfloxZ6JU87bfuLA,11435
-lsst/ctrl/bps/parsl/sites/__init__.py,sha256=w_IOW3_lBguS5k3U9R925wCENgUhu3407FSw7ExrKus,1441
-lsst/ctrl/bps/parsl/sites/ccin2p3.py,sha256=UD86cgS9ffJaz__zs4PQmHQqE7B4jhMWAerViCAt7Gk,14597
-lsst/ctrl/bps/parsl/sites/local.py,sha256=El8U6XfeYBbfHyg-7R3a1gtWX1IBLq_xZvKXyjiAQIk,2622
-lsst/ctrl/bps/parsl/sites/nersc.py,sha256=obUiaHgHZj_23PIrYdihUnb8eoEJMs288wHy84jIzaQ,4549
-lsst/ctrl/bps/parsl/sites/princeton.py,sha256=F_dHTN5sm2NM2VjtJoWbtKfhkowBocIs5F7eSgjxc-0,5575
-lsst/ctrl/bps/parsl/sites/slac.py,sha256=IN04Nx0YvOl-GN33MB05zT0XG3IQ91rXri_EHz_dzTs,3893
-lsst/ctrl/bps/parsl/sites/slurm.py,sha256=6aA4cJSS1nSUydKZ1o-4xKDydgOoictWBl9mJPhB3vY,12822
-lsst/ctrl/bps/parsl/sites/torque.py,sha256=gP8X-u3oH6P79kw2Fj_Mx3NqpVSQmqSHMT1tgxiU7Yc,9621
-lsst/ctrl/bps/parsl/sites/work_queue.py,sha256=QpeGjd7o0FpnXRrbrxTNSXcfkTHclBKv6vMPPRRgfd8,7040
-lsst_ctrl_bps_parsl-30.0.0rc2.dist-info/licenses/COPYRIGHT,sha256=M5ylfMNsihIJcPySQ5ZFjec4zROSZdlNN9oGHBhu7WM,181
-lsst_ctrl_bps_parsl-30.0.0rc2.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps_parsl-30.0.0rc2.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps_parsl-30.0.0rc2.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps_parsl-30.0.0rc2.dist-info/METADATA,sha256=JFQVegSzXO950hXyKBPDX0zou-0HUfn0F6OS4ZRlQWI,2951
-lsst_ctrl_bps_parsl-30.0.0rc2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_ctrl_bps_parsl-30.0.0rc2.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps_parsl-30.0.0rc2.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps_parsl-30.0.0rc2.dist-info/RECORD,,