lsst-ctrl-bps 29.2025.2400__py3-none-any.whl → 29.2025.2600__py3-none-any.whl

lsst/ctrl/bps/bps_config.py

@@ -44,6 +44,8 @@ from lsst.daf.butler import Config
 from lsst.resources import ResourcePath
 from lsst.utils import doImport
 
+from .bps_utils import bps_eval
+
 _LOG = logging.getLogger(__name__)
 
 # Using lsst.daf.butler.Config to resolve possible includes.
@@ -417,4 +419,63 @@ class BpsConfig(Config):
         if default != _NO_SEARCH_DEFAULT_VALUE:
             opt["default"] = default
 
+        # check for bpsEval
+        value = re.sub(
+            r"bpsEval\(([^,)]+), ([^)]+)\)", lambda m: str(bps_eval(m.group(1), m.group(2))), value
+        )
+        if "bpsEval" in value:
+            raise ValueError(f"Unparsable bpsEval in '{value}'")
+
         return value
+
+    def generate_config(self) -> None:
+        """Update config with values generated by bpsGenerateConfig
+        entries.
+        """
+        _LOG.debug("generate_config before: %s", self)
+        self._recursive_generate_config("", self)
+        _LOG.debug("generate_config after: %s", self)
+
+    def _recursive_generate_config(self, recursive_key: str, sub_config: Config) -> None:
+        """Update config with values generated by bpsGenerateConfig
+        entries.
+
+        Parameters
+        ----------
+        recursive_key : `str`
+            Corresponds to a new subconfig in which to search
+            and replace bpsGenerateConfig.
+
+        sub_config : `lsst.daf.butler.Config`
+            The nested config corresponding to the recursive_key.
+
+        Raises
+        ------
+        ValueError
+            If bpsGenerateConfig value isn't parseable.
+        ImportError
+            If problems importing bpsGenerateConfig's method.
+        """
+        _LOG.debug("recursive_key = '%s'", recursive_key)
+        genkey = "bpsGenerateConfig"  # to make it easier to change
+
+        # Save to avoid dictionary changed size during iteration error.
+        orig_keys = list(sub_config)
+        for key in orig_keys:
+            value = Config.__getitem__(sub_config, key)
+            _LOG.debug("key = %s, type(value) = %s", key, type(value))
+            if isinstance(value, Config):
+                self._recursive_generate_config(f"{recursive_key}.{key}", value)
+            elif key == genkey:
+                value = self.replace_vars(value, {"searchobj": sub_config})
+
+                m = re.match(r"(\S+)\((.+)\)", value)
+                if m:
+                    results = bps_eval(m.group(1), m.group(2))
+                    del sub_config[genkey]
+                    sub_config.update(results)
+                    if recursive_key:
+                        self[recursive_key] = sub_config
+                    _LOG.debug("After config = %s", self)
+                else:
+                    raise ValueError(f"Unparsable {genkey} value='{value}'")

lsst/ctrl/bps/bps_utils.py

@@ -31,6 +31,7 @@ __all__ = [
     "_dump_env_info",
     "_dump_pkg_info",
     "_make_id_link",
+    "bps_eval",
     "chdir",
     "create_count_summary",
     "create_job_quantum_graph_filename",
@@ -52,6 +53,7 @@ from typing import Any
 
 import yaml
 
+from lsst.utils import doImport
 from lsst.utils.packages import Packages
 
 _LOG = logging.getLogger(__name__)
@@ -353,3 +355,35 @@ def subset_dimension_values(
             f"{desc_what} missing dimensions ({', '.join(sorted(missing_dims))}) required for {desc_for}"
         )
     return dim_values
+
+
+def bps_eval(func: str, args: str) -> Any:
+    """Evaluate user provided expression/function.
+
+    Parameters
+    ----------
+    func : `str`
+        Importable string or built-in function name.
+    args : `str`
+        Parameters to pass to the function.
+
+    Returns
+    -------
+    results : `~typing.Any`
+        Results of running eval.
+
+    Raises
+    ------
+    ImportError
+        If problems importing.
+    """
+    if "." in func:
+        genfunc = doImport(func)  # noqa: F841
+        func_reference = "genfunc"
+    else:
+        func_reference = func
+    eval_str = f"{func_reference}({args})"
+    _LOG.debug("String passed to eval: '%s'", eval_str)
+    results = eval(eval_str)
+
+    return results

lsst/ctrl/bps/etc/bps_defaults.yaml

@@ -115,6 +115,7 @@ executionButler:
   whenCreate: "NEVER"
 
 finalJob:
+  retryUnlessExit: 2
   updateOutputChain: "--update-output-chain"
   whenSetup: "NEVER"
   whenRun: "ALWAYS"

lsst/ctrl/bps/initialize.py

@@ -130,6 +130,9 @@ def init_submission(
     else:
         _LOG.debug("Skipping submission checks.")
 
+    # Replace all bpsGenerateConfig
+    config.generate_config()
+
     # Make submit directory to contain all outputs.
     submit_path = mkdir(config["submitPath"])
     config[".bps_defined.submitPath"] = str(submit_path)

lsst/ctrl/bps/tests/config_test_utils.py

@@ -0,0 +1,114 @@
+# This file is part of ctrl_bps.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""BpsConfig-related utilities to support ctrl_bps testing."""
+
+__all__ = ["generate_config_1", "generate_config_2", "generate_config_all"]
+
+from typing import Any
+
+
+def generate_config_1(param1: int, param2: int = -1, param3: int = -2) -> dict[str, Any]:
+    """Return a dictionary for updating a config in unit tests.
+
+    Parameters
+    ----------
+    param1 : `int`
+        First param.
+    param2 : `int`, optional
+        Second param.  Defaults to -1.
+    param3 : `int`, optional
+        Third param.  Defaults to -2.
+
+    Returns
+    -------
+    results : `dict` [`str`, `~typing.Any`]
+        The mocked results.
+    """
+    results = {"gencfg_1": param1, "gencfg_2": param2, "gencfg_3": param3, "p4": 41}
+    return results
+
+
+def generate_config_2(param1: int, param2: int = -3, param3: int = -4) -> dict[str, Any]:
+    """Return a dictionary for updating a config in unit tests.
+
+    Parameters
+    ----------
+    param1 : `int`
+        First param.
+    param2 : `int`, optional
+        Second param.  Defaults to -3.
+    param3 : `int`, optional
+        Third param.  Defaults to -4.
+
+    Returns
+    -------
+    results : `dict` [`str`, `~typing.Any`]
+        The mocked results.
+    """
+    results = {"gencfg_4": param1, "gencfg_5": param2, "gencfg_6": param3, "p4": 42}
+    return results
+
+
+def generate_config_all(param1: str, param2: int = -5, param3: int = -6) -> dict[str, Any]:
+    """Return a dictionary for updating multiple sections in a config.
+
+    Parameters
+    ----------
+    param1 : `str`
+        First param.
+    param2 : `int`, optional
+        Second param.  Defaults to -2.
+    param3 : `int`, optional
+        Third param.  Defaults to -5.
+
+    Returns
+    -------
+    results : `dict` [`str`, `~typing.Any`]
+        The mocked results.
+    """
+    results = {
+        "genall_1": param1,
+        "pipetask": {"ptask1": {"genall_2": param2}},
+        "finalJob": {"genall_3": param3},
+    }
+    return results
+
+
+def generate_value_1(param1: int) -> str:
+    """Return a string for updating a config value in unit tests.
+
+    Parameters
+    ----------
+    param1 : `int`
+        First param.
+
+    Returns
+    -------
+    results : `str`
+        The mocked result.
+    """
+    return f"-c val2:{param1}"

lsst/ctrl/bps/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.2400"
+__version__ = "29.2025.2600"

{lsst_ctrl_bps-29.2025.2400.dist-info → lsst_ctrl_bps-29.2025.2600.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps
-Version: 29.2025.2400
+Version: 29.2025.2600
 Summary: Pluggable execution of workflow graphs from Rubin pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps-29.2025.2400.dist-info → lsst_ctrl_bps-29.2025.2600.dist-info}/RECORD

@@ -2,17 +2,17 @@ lsst/__init__.py,sha256=Gk14M2Z1rlkmKNnIVprTMAua6WiShDHrlt32AkqTfj0,1366
 lsst/ctrl/__init__.py,sha256=Gk14M2Z1rlkmKNnIVprTMAua6WiShDHrlt32AkqTfj0,1366
 lsst/ctrl/bps/__init__.py,sha256=qNei3h4EwqUnyyCsBUkYXcRNNlSvzUzbM6Zdkd4eklE,1663
 lsst/ctrl/bps/_exceptions.py,sha256=qNftFVE8yhiRD8yvACKEf5iHhi1o2eaa69ZCBvw6lDo,1974
-lsst/ctrl/bps/bps_config.py,sha256=Nw7TXbBY9VHry6u1erzLodyI72uiJgklTcYoDtuIns8,16103
+lsst/ctrl/bps/bps_config.py,sha256=4t1jyIBiw9leAySjkcksCaZ3CPJMOt8afxGxTEWRSyQ,18460
 lsst/ctrl/bps/bps_draw.py,sha256=Sl04jpbm6jB_5lBzfJprqnDes9t_aqQQhr0Ag5RSv-Y,1984
 lsst/ctrl/bps/bps_reports.py,sha256=j6HdAvkOBTSxSHmkx_vd04FvifToTaZVGpwfF-LIUu0,12487
-lsst/ctrl/bps/bps_utils.py,sha256=c7sB9DzuVI4-zdB5mpBPqFrON3ZqPRX7URzAbpXrVqs,11446
+lsst/ctrl/bps/bps_utils.py,sha256=-w0HSUXKw8jgJFl28yzwJuuqLqT_UOw12_XQBELUWrE,12206
 lsst/ctrl/bps/cancel.py,sha256=mAdBi-oUpepyo-1MCqx_I34dbm6cqT0VJu3d2-y9T2Y,3317
 lsst/ctrl/bps/clustered_quantum_graph.py,sha256=mBf8s_DlTzGCFq7aAKmD1cAXq6Cqr8QxjwpgXPOQcRc,18765
 lsst/ctrl/bps/constants.py,sha256=dsnsNMqwU6Xl_ln6nQ0PmxsghlbBY9mLJWf2rlzYayQ,1733
 lsst/ctrl/bps/construct.py,sha256=a5Gb_oinowb7m9rLsk0R_hz8GFu9WGCzDd_QWbsK0Mk,4944
 lsst/ctrl/bps/drivers.py,sha256=M_zWXvzK23MBBNvqdbJCnA_9DUCoFznCjWfwQw9BO6Q,22512
 lsst/ctrl/bps/generic_workflow.py,sha256=BwEFBudncziHTZaTTknOQroY_a-33mGf-mq9PzxH8G0,53720
-lsst/ctrl/bps/initialize.py,sha256=mTCWHLFudfu_Kk3QVbd1tnz76UdG822K9CNWuOOgUv0,7488
+lsst/ctrl/bps/initialize.py,sha256=DCQwB9ZSo_36Ncs8jMJwxKZcYCu46jjww6rzyiuMFKY,7554
 lsst/ctrl/bps/ping.py,sha256=orwTZUNFtlexMYFcNWW_48jaa7Jo1oK4_eb_HuC-p5E,2235
 lsst/ctrl/bps/pre_transform.py,sha256=mPUW1QuHwgkSgZGVK6xO4RN590z4g1zf3p4kNsx_yOg,10222
 lsst/ctrl/bps/prepare.py,sha256=Fa2OEQIo4Pa8R5WmRo0PvJgXWNjynRijATvu1x80qlw,3129
@@ -22,7 +22,7 @@ lsst/ctrl/bps/restart.py,sha256=yVwxeviLiehyIfPmwU-H3tJ9ou7OWZZcrNf8PMxjr8o,2298
 lsst/ctrl/bps/status.py,sha256=Lrt0cAqROv77B8UvYXGimCa4cDHBD1N0K2Xx7oS6fXk,3362
 lsst/ctrl/bps/submit.py,sha256=Ev-yhcoZwtBPIo5bRt_4XFJRtgQBd8JHUurEfn01HpU,2880
 lsst/ctrl/bps/transform.py,sha256=jGEA3YGIQIJiLv-4Qm00WPNW0EEKughX7lcoJRZhsKU,35600
-lsst/ctrl/bps/version.py,sha256=AyivLkwA4FcYO6eiFsfTA_EuqMhydgvzBupNZlQZL-E,55
+lsst/ctrl/bps/version.py,sha256=eKEngRNTsqzq5yv4faNqXQ7KB1ya5wGTLTxJ1qbWG50,55
 lsst/ctrl/bps/wms_service.py,sha256=l3T6i1MG72dhHY0KXMUlBjWUpCLOfaySs7o2W8oCwhs,18891
 lsst/ctrl/bps/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/ctrl/bps/cli/bps.py,sha256=CHfAL-U4mSi-FqeGKtmkX5nI5H9gkRre4XEWNVdeMRk,2559
@@ -32,15 +32,16 @@ lsst/ctrl/bps/cli/opt/__init__.py,sha256=6-bAftMGxt_uEKrQl0gtrg8dJLBNsPd_DWCApg_
 lsst/ctrl/bps/cli/opt/arguments.py,sha256=Q9nU1Ad1CRv0HgIM68bu6sqCfJT1om-ARZy7pVXGJ94,1534
 lsst/ctrl/bps/cli/opt/option_groups.py,sha256=xylZX9CmxrTg1FJQ0ziyWYF0Se6CvO6ydgAwUZFTkR4,2722
 lsst/ctrl/bps/cli/opt/options.py,sha256=pZQpjJ2Vrx6kYJGs5vVERBMFstPocwBXHffxkWkmavg,3088
-lsst/ctrl/bps/etc/bps_defaults.yaml,sha256=pEmaTJ-F7hHVo2rrqhSoxgLTdMuFRpm-sU7FhoX2eco,4807
+lsst/ctrl/bps/etc/bps_defaults.yaml,sha256=UwKAKWl09-Xjtl6kiz-AiXA9SDczeCvPGRXOPG3fzb8,4828
+lsst/ctrl/bps/tests/config_test_utils.py,sha256=WM8Vrigk4OO0TBoL1A73a6hLhf2a6-ACD20fROJ0U7A,3537
 lsst/ctrl/bps/tests/gw_test_utils.py,sha256=zVVQqzwSiQgPgk9TnqDzgR7uDnaTMeuBLYKA8vOp5RI,22452
-lsst_ctrl_bps-29.2025.2400.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
-lsst_ctrl_bps-29.2025.2400.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps-29.2025.2400.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps-29.2025.2400.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps-29.2025.2400.dist-info/METADATA,sha256=f2SkQsZUeoXJ2jUagkNzhvtrjTUaGoJBJK4PM5J3UNc,2190
-lsst_ctrl_bps-29.2025.2400.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_ctrl_bps-29.2025.2400.dist-info/entry_points.txt,sha256=d6FhN79h7s9frdBI7YkScsGEInwpGGub49pAjXWbIbI,51
-lsst_ctrl_bps-29.2025.2400.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps-29.2025.2400.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps-29.2025.2400.dist-info/RECORD,,
+lsst_ctrl_bps-29.2025.2600.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
+lsst_ctrl_bps-29.2025.2600.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps-29.2025.2600.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps-29.2025.2600.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps-29.2025.2600.dist-info/METADATA,sha256=2lLcDL1u06EDG3qahVfcgvJmJHZP_5KedjOEO4e34vI,2190
+lsst_ctrl_bps-29.2025.2600.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_ctrl_bps-29.2025.2600.dist-info/entry_points.txt,sha256=d6FhN79h7s9frdBI7YkScsGEInwpGGub49pAjXWbIbI,51
+lsst_ctrl_bps-29.2025.2600.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps-29.2025.2600.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps-29.2025.2600.dist-info/RECORD,,