lsst-ctrl-bps 29.2025.2500__tar.gz → 29.2025.2700__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {lsst_ctrl_bps-29.2025.2500/python/lsst_ctrl_bps.egg-info → lsst_ctrl_bps-29.2025.2700}/PKG-INFO +1 -1
  2. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/doc/lsst.ctrl.bps/quickstart.rst +43 -0
  3. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/bps_config.py +61 -0
  4. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/bps_utils.py +34 -0
  5. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/initialize.py +3 -0
  6. lsst_ctrl_bps-29.2025.2700/python/lsst/ctrl/bps/tests/config_test_utils.py +114 -0
  7. lsst_ctrl_bps-29.2025.2700/python/lsst/ctrl/bps/version.py +2 -0
  8. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700/python/lsst_ctrl_bps.egg-info}/PKG-INFO +1 -1
  9. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst_ctrl_bps.egg-info/SOURCES.txt +2 -0
  10. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_bps_utils.py +12 -1
  11. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_bpsconfig.py +133 -1
  12. lsst_ctrl_bps-29.2025.2700/tests/test_initialize.py +180 -0
  13. lsst_ctrl_bps-29.2025.2500/python/lsst/ctrl/bps/version.py +0 -2
  14. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/COPYRIGHT +0 -0
  15. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/LICENSE +0 -0
  16. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/MANIFEST.in +0 -0
  17. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/README.md +0 -0
  18. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/bsd_license.txt +0 -0
  19. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/doc/lsst.ctrl.bps/CHANGES.rst +0 -0
  20. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/doc/lsst.ctrl.bps/index.rst +0 -0
  21. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/gpl-v3.0.txt +0 -0
  22. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/pyproject.toml +0 -0
  23. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/__init__.py +0 -0
  24. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/__init__.py +0 -0
  25. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/__init__.py +0 -0
  26. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/_exceptions.py +0 -0
  27. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/bps_draw.py +0 -0
  28. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/bps_reports.py +0 -0
  29. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/cancel.py +0 -0
  30. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/cli/__init__.py +0 -0
  31. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/cli/bps.py +0 -0
  32. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/cli/cmd/__init__.py +0 -0
  33. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/cli/cmd/commands.py +0 -0
  34. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/cli/opt/__init__.py +0 -0
  35. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/cli/opt/arguments.py +0 -0
  36. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/cli/opt/option_groups.py +0 -0
  37. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/cli/opt/options.py +0 -0
  38. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/clustered_quantum_graph.py +0 -0
  39. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/constants.py +0 -0
  40. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/construct.py +0 -0
  41. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/drivers.py +0 -0
  42. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/etc/bps_defaults.yaml +0 -0
  43. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/generic_workflow.py +0 -0
  44. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/ping.py +0 -0
  45. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/pre_transform.py +0 -0
  46. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/prepare.py +0 -0
  47. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/quantum_clustering_funcs.py +0 -0
  48. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/report.py +0 -0
  49. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/restart.py +0 -0
  50. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/status.py +0 -0
  51. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/submit.py +0 -0
  52. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/tests/gw_test_utils.py +0 -0
  53. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/transform.py +0 -0
  54. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst/ctrl/bps/wms_service.py +0 -0
  55. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst_ctrl_bps.egg-info/dependency_links.txt +0 -0
  56. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst_ctrl_bps.egg-info/entry_points.txt +0 -0
  57. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst_ctrl_bps.egg-info/requires.txt +0 -0
  58. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst_ctrl_bps.egg-info/top_level.txt +0 -0
  59. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/python/lsst_ctrl_bps.egg-info/zip-safe +0 -0
  60. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/setup.cfg +0 -0
  61. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_cli_commands.py +0 -0
  62. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_clustered_quantum_graph.py +0 -0
  63. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_construct.py +0 -0
  64. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_drivers.py +0 -0
  65. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_generic_workflow.py +0 -0
  66. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_ping.py +0 -0
  67. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_pre_transform.py +0 -0
  68. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_quantum_clustering_funcs.py +0 -0
  69. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_report.py +0 -0
  70. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_status.py +0 -0
  71. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_transform.py +0 -0
  72. {lsst_ctrl_bps-29.2025.2500 → lsst_ctrl_bps-29.2025.2700}/tests/test_wms_service.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lsst-ctrl-bps
3
- Version: 29.2025.2500
3
+ Version: 29.2025.2700
4
4
  Summary: Pluggable execution of workflow graphs from Rubin pipelines.
5
5
  Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
6
6
  License: BSD 3-Clause License
@@ -1497,6 +1497,49 @@ invisible to the user. ``bps report`` will still show same labels and
1497
1497
  total counts as without ordering. ``cancel`` and ``restart`` will still
1498
1498
  work the same.
1499
1499
 
1500
+ .. _bps-config-generation:
1501
+
1502
+ Config Generation
1503
+ -----------------
1504
+
1505
+ In some rare use cases, the submit yaml depends upon what happened in
1506
+ previous runs (e.g., passing pipeline configuration values to the HiPS
1507
+ QuantumGraph generation command depending upon colors of generated outputs
1508
+ of previous run). One can wait until a run finishes, query the results,
1509
+ and then manually modify the submit yaml for the next run. To help make
1510
+ this easier to automate, two special mechanisms, ``bpsGenerateConfig`` and
1511
+ ``bpsEval`` have been added to ``bps``. While different syntax, both take
1512
+ two pieces of information. The first piece describes what to import and
1513
+ execute and the second the parameters to pass which typically will
1514
+ be config variables (e.g., "{butlerConfig}").
1515
+
1516
+ ``bpsGenerateConfig`` is a key/value pair where the function returns a
1517
+ Mapping to update the config. It can be used at the root level and can
1518
+ return nested dictionaries to replace values across multiple sections.
1519
+ It can also be used inside sections (e.g., inside a specific pipetask
1520
+ section). The function is not run when loading the config. Instead it
1521
+ is run during the initialization in the ``bps submit`` (before saving
1522
+ the config yaml to the submit directory and before running QuantumGraph
1523
+ generation). Example:
1524
+
1525
+ ..code::
1526
+
1527
+ bpsGenerateConfig: "lsst.my.package.my_func_1('{butlerConfig}', param3='{output}')"
1528
+
1529
+
1530
+ ``bpsEval`` is a placeholder in an submit yaml value. It is executed when
1531
+ the corresponding key is requested from the config. Its function needs
1532
+ to return a value whose string representation can replace ``bpsEval``. Example:
1533
+
1534
+ ..code::
1535
+
1536
+ extraQgraphOptions: "--dataset-query-constraint off bpsEval(lsst.my.package.my_func_2, '{butlerConfig}', '{output}')
1537
+
1538
+ .. warning::
1539
+
1540
+ Quotes must be placed around variables that return strings (e.g., '{butlerConfig}').
1541
+ Forgetting the quotes typically results in an invalid syntax error.
1542
+
1500
1543
  .. _bps-softlink:
1501
1544
 
1502
1545
  WMS-id softlink
@@ -44,6 +44,8 @@ from lsst.daf.butler import Config
44
44
  from lsst.resources import ResourcePath
45
45
  from lsst.utils import doImport
46
46
 
47
+ from .bps_utils import bps_eval
48
+
47
49
  _LOG = logging.getLogger(__name__)
48
50
 
49
51
  # Using lsst.daf.butler.Config to resolve possible includes.
@@ -417,4 +419,63 @@ class BpsConfig(Config):
417
419
  if default != _NO_SEARCH_DEFAULT_VALUE:
418
420
  opt["default"] = default
419
421
 
422
+ # check for bpsEval
423
+ value = re.sub(
424
+ r"bpsEval\(([^,)]+), ([^)]+)\)", lambda m: str(bps_eval(m.group(1), m.group(2))), value
425
+ )
426
+ if "bpsEval" in value:
427
+ raise ValueError(f"Unparsable bpsEval in '{value}'")
428
+
420
429
  return value
430
+
431
+ def generate_config(self) -> None:
432
+ """Update config with values generated by bpsGenerateConfig
433
+ entries.
434
+ """
435
+ _LOG.debug("generate_config before: %s", self)
436
+ self._recursive_generate_config("", self)
437
+ _LOG.debug("generate_config after: %s", self)
438
+
439
+ def _recursive_generate_config(self, recursive_key: str, sub_config: Config) -> None:
440
+ """Update config with values generated by bpsGenerateConfig
441
+ entries.
442
+
443
+ Parameters
444
+ ----------
445
+ recursive_key : `str`
446
+ Corresponds to a new subconfig in which to search
447
+ and replace bpsGenerateConfig.
448
+
449
+ sub_config : `lsst.daf.butler.Config`
450
+ The nested config corresponding to the recursive_key.
451
+
452
+ Raises
453
+ ------
454
+ ValueError
455
+ If bpsGenerateConfig value isn't parseable.
456
+ ImportError
457
+ If problems importing bpsGenerateConfig's method.
458
+ """
459
+ _LOG.debug("recursive_key = '%s'", recursive_key)
460
+ genkey = "bpsGenerateConfig" # to make it easier to change
461
+
462
+ # Save to avoid dictionary changed size during iteration error.
463
+ orig_keys = list(sub_config)
464
+ for key in orig_keys:
465
+ value = Config.__getitem__(sub_config, key)
466
+ _LOG.debug("key = %s, type(value) = %s", key, type(value))
467
+ if isinstance(value, Config):
468
+ self._recursive_generate_config(f"{recursive_key}.{key}", value)
469
+ elif key == genkey:
470
+ value = self.replace_vars(value, {"searchobj": sub_config})
471
+
472
+ m = re.match(r"(\S+)\((.+)\)", value)
473
+ if m:
474
+ results = bps_eval(m.group(1), m.group(2))
475
+ del sub_config[genkey]
476
+ sub_config.update(results)
477
+ if recursive_key:
478
+ self[recursive_key] = sub_config
479
+ _LOG.debug("After config = %s", self)
480
+ else:
481
+ raise ValueError(f"Unparsable {genkey} value='{value}'")
@@ -31,6 +31,7 @@ __all__ = [
31
31
  "_dump_env_info",
32
32
  "_dump_pkg_info",
33
33
  "_make_id_link",
34
+ "bps_eval",
34
35
  "chdir",
35
36
  "create_count_summary",
36
37
  "create_job_quantum_graph_filename",
@@ -52,6 +53,7 @@ from typing import Any
52
53
 
53
54
  import yaml
54
55
 
56
+ from lsst.utils import doImport
55
57
  from lsst.utils.packages import Packages
56
58
 
57
59
  _LOG = logging.getLogger(__name__)
@@ -353,3 +355,35 @@ def subset_dimension_values(
353
355
  f"{desc_what} missing dimensions ({', '.join(sorted(missing_dims))}) required for {desc_for}"
354
356
  )
355
357
  return dim_values
358
+
359
+
360
+ def bps_eval(func: str, args: str) -> Any:
361
+ """Evaluate user provided expression/function.
362
+
363
+ Parameters
364
+ ----------
365
+ func : `str`
366
+ Importable string or built-in function name.
367
+ args : `str`
368
+ Parameters to pass to the function.
369
+
370
+ Returns
371
+ -------
372
+ results : `~typing.Any`
373
+ Results of running eval.
374
+
375
+ Raises
376
+ ------
377
+ ImportError
378
+ If problems importing.
379
+ """
380
+ if "." in func:
381
+ genfunc = doImport(func) # noqa: F841
382
+ func_reference = "genfunc"
383
+ else:
384
+ func_reference = func
385
+ eval_str = f"{func_reference}({args})"
386
+ _LOG.debug("String passed to eval: '%s'", eval_str)
387
+ results = eval(eval_str)
388
+
389
+ return results
@@ -130,6 +130,9 @@ def init_submission(
130
130
  else:
131
131
  _LOG.debug("Skipping submission checks.")
132
132
 
133
+ # Replace all bpsGenerateConfig
134
+ config.generate_config()
135
+
133
136
  # Make submit directory to contain all outputs.
134
137
  submit_path = mkdir(config["submitPath"])
135
138
  config[".bps_defined.submitPath"] = str(submit_path)
@@ -0,0 +1,114 @@
1
+ # This file is part of ctrl_bps.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (https://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <https://www.gnu.org/licenses/>.
27
+ """BpsConfig-related utilities to support ctrl_bps testing."""
28
+
29
+ __all__ = ["generate_config_1", "generate_config_2", "generate_config_all"]
30
+
31
+ from typing import Any
32
+
33
+
34
+ def generate_config_1(param1: int, param2: int = -1, param3: int = -2) -> dict[str, Any]:
35
+ """Return a dictionary for updating a config in unit tests.
36
+
37
+ Parameters
38
+ ----------
39
+ param1 : `int`
40
+ First param.
41
+ param2 : `int`, optional
42
+ Second param. Defaults to -1.
43
+ param3 : `int`, optional
44
+ Third param. Defaults to -2.
45
+
46
+ Returns
47
+ -------
48
+ results : `dict` [`str`, `~typing.Any`]
49
+ The mocked results.
50
+ """
51
+ results = {"gencfg_1": param1, "gencfg_2": param2, "gencfg_3": param3, "p4": 41}
52
+ return results
53
+
54
+
55
+ def generate_config_2(param1: int, param2: int = -3, param3: int = -4) -> dict[str, Any]:
56
+ """Return a dictionary for updating a config in unit tests.
57
+
58
+ Parameters
59
+ ----------
60
+ param1 : `int`
61
+ First param.
62
+ param2 : `int`, optional
63
+ Second param. Defaults to -3.
64
+ param3 : `int`, optional
65
+ Third param. Defaults to -4.
66
+
67
+ Returns
68
+ -------
69
+ results : `dict` [`str`, `~typing.Any`]
70
+ The mocked results.
71
+ """
72
+ results = {"gencfg_4": param1, "gencfg_5": param2, "gencfg_6": param3, "p4": 42}
73
+ return results
74
+
75
+
76
+ def generate_config_all(param1: str, param2: int = -5, param3: int = -6) -> dict[str, Any]:
77
+ """Return a dictionary for updating multiple sections in a config.
78
+
79
+ Parameters
80
+ ----------
81
+ param1 : `str`
82
+ First param.
83
+ param2 : `int`, optional
84
+ Second param. Defaults to -2.
85
+ param3 : `int`, optional
86
+ Third param. Defaults to -5.
87
+
88
+ Returns
89
+ -------
90
+ results : `dict` [`str`, `~typing.Any`]
91
+ The mocked results.
92
+ """
93
+ results = {
94
+ "genall_1": param1,
95
+ "pipetask": {"ptask1": {"genall_2": param2}},
96
+ "finalJob": {"genall_3": param3},
97
+ }
98
+ return results
99
+
100
+
101
+ def generate_value_1(param1: int) -> str:
102
+ """Return a string for updating a config value in unit tests.
103
+
104
+ Parameters
105
+ ----------
106
+ param1 : `int`
107
+ First param.
108
+
109
+ Returns
110
+ -------
111
+ results : `str`
112
+ The mocked result.
113
+ """
114
+ return f"-c val2:{param1}"
@@ -0,0 +1,2 @@
1
+ __all__ = ["__version__"]
2
+ __version__ = "29.2025.2700"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lsst-ctrl-bps
3
- Version: 29.2025.2500
3
+ Version: 29.2025.2700
4
4
  Summary: Pluggable execution of workflow graphs from Rubin pipelines.
5
5
  Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
6
6
  License: BSD 3-Clause License
@@ -44,6 +44,7 @@ python/lsst/ctrl/bps/cli/opt/arguments.py
44
44
  python/lsst/ctrl/bps/cli/opt/option_groups.py
45
45
  python/lsst/ctrl/bps/cli/opt/options.py
46
46
  python/lsst/ctrl/bps/etc/bps_defaults.yaml
47
+ python/lsst/ctrl/bps/tests/config_test_utils.py
47
48
  python/lsst/ctrl/bps/tests/gw_test_utils.py
48
49
  python/lsst_ctrl_bps.egg-info/PKG-INFO
49
50
  python/lsst_ctrl_bps.egg-info/SOURCES.txt
@@ -59,6 +60,7 @@ tests/test_clustered_quantum_graph.py
59
60
  tests/test_construct.py
60
61
  tests/test_drivers.py
61
62
  tests/test_generic_workflow.py
63
+ tests/test_initialize.py
62
64
  tests/test_ping.py
63
65
  tests/test_pre_transform.py
64
66
  tests/test_quantum_clustering_funcs.py
@@ -31,7 +31,7 @@ import unittest
31
31
  from pathlib import Path
32
32
 
33
33
  from lsst.ctrl.bps import BpsConfig
34
- from lsst.ctrl.bps.bps_utils import _make_id_link, chdir, mkdir
34
+ from lsst.ctrl.bps.bps_utils import _make_id_link, bps_eval, chdir, mkdir
35
35
 
36
36
 
37
37
  class TestMkdir(unittest.TestCase):
@@ -224,5 +224,16 @@ class TestMakeIdLink(unittest.TestCase):
224
224
  self.assertFalse(link_path.is_symlink())
225
225
 
226
226
 
227
+ class TestBpsEval(unittest.TestCase):
228
+ """Test bps_eval function."""
229
+
230
+ def testBuiltIn(self):
231
+ """Test using a built-in function."""
232
+ with self.assertLogs("lsst.ctrl.bps.bps_utils", level=logging.DEBUG) as cm:
233
+ results = bps_eval("sum", "[1, 2]")
234
+ self.assertEqual(results, 3)
235
+ self.assertEqual(cm.records[-1].getMessage(), "String passed to eval: 'sum([1, 2])'")
236
+
237
+
227
238
  if __name__ == "__main__":
228
239
  unittest.main()
@@ -289,7 +289,6 @@ class TestBpsConfigSearch(unittest.TestCase):
289
289
  test_opt = {"expandEnvVars": True, "replaceEnvVars": True, "replaceVars": True}
290
290
  found, value = self.config.search("grault", opt=test_opt)
291
291
  self.assertEqual(found, True)
292
- self.assertEqual(found, True)
293
292
  self.assertEqual(value, "garply/waldo/002")
294
293
 
295
294
  def testRequired(self):
@@ -298,5 +297,138 @@ class TestBpsConfigSearch(unittest.TestCase):
298
297
  self.config.search("fred", opt={"required": True})
299
298
 
300
299
 
300
+ class TestBpsConfigGenerateConfig(unittest.TestCase):
301
+ """Test BpsConfig.generate_config and bpsEval methods."""
302
+
303
+ def setUp(self):
304
+ # Just to shorten string length in tests
305
+ self.test_prefix = "lsst.ctrl.bps.tests.config_test_utils"
306
+
307
+ filename = os.path.join(TESTDIR, "data/initialize_config.yaml")
308
+ self.config = BpsConfig(filename, BPS_SEARCH_ORDER, defaults={})
309
+
310
+ def testUnparsableValue(self):
311
+ config = BpsConfig(
312
+ {
313
+ "p1": 3,
314
+ "p3": 16,
315
+ },
316
+ BPS_SEARCH_ORDER,
317
+ )
318
+ # invalid function name
319
+ config["bpsGenerateConfig"] = "not a valid name({p1}, param3={p3})"
320
+ with self.assertRaisesRegex(ValueError, "Unparsable bpsGenerateConfig value='not a valid"):
321
+ config.generate_config()
322
+
323
+ def testMissingParen(self):
324
+ config = BpsConfig(
325
+ {
326
+ "p1": 3,
327
+ "p3": 16,
328
+ },
329
+ BPS_SEARCH_ORDER,
330
+ )
331
+ # invalid function name
332
+ config["bpsGenerateConfig"] = self.test_prefix + ".generate_config_1(1, param3=2"
333
+ with self.assertRaisesRegex(ValueError, "Unparsable bpsGenerateConfig value='"):
334
+ config.generate_config()
335
+
336
+ def testBadFunctionName(self):
337
+ config = BpsConfig(
338
+ {
339
+ "p1": 3,
340
+ "p3": 16,
341
+ },
342
+ BPS_SEARCH_ORDER,
343
+ )
344
+ # invalid function name
345
+ config["bpsGenerateConfig"] = self.test_prefix + ".notthere({p1}, param3={p3})"
346
+ with self.assertRaisesRegex(ImportError, "notthere"):
347
+ config.generate_config()
348
+
349
+ def testBadModuleName(self):
350
+ config = BpsConfig(
351
+ {
352
+ "p1": 3,
353
+ "p3": 16,
354
+ },
355
+ BPS_SEARCH_ORDER,
356
+ )
357
+ # invalid module name
358
+ config["bpsGenerateConfig"] = "lsst.ctrl.bps.notthere.generate_config(1, 2)"
359
+ with self.assertRaisesRegex(ImportError, "notthere"):
360
+ config.generate_config()
361
+
362
+ def testBadParamName(self):
363
+ config = BpsConfig(
364
+ {"p1": 3, "p3": 16, "bpsGenerateConfig": self.test_prefix + ".generate_config_1(1, param5=2)"},
365
+ BPS_SEARCH_ORDER,
366
+ )
367
+ with self.assertRaisesRegex(TypeError, "unexpected keyword argument 'param5'"):
368
+ config.generate_config()
369
+
370
+ def testExtraParam(self):
371
+ config = BpsConfig(
372
+ {
373
+ "p1": 3,
374
+ "p3": 16,
375
+ "bpsGenerateConfig": self.test_prefix + ".generate_config_1({p1}, 2, {p3}, 4)",
376
+ },
377
+ BPS_SEARCH_ORDER,
378
+ )
379
+ with self.assertRaisesRegex(TypeError, "positional arguments"):
380
+ config.generate_config()
381
+
382
+ def testWithSearchOrder(self):
383
+ # Check that bpsGenerateConfig is replaced in search sections
384
+ # (e.g., pipetask) And when replacing vars in subsections
385
+ # config ordering is used.
386
+ # Ditto for finalJob (which isn't a search section).
387
+ # Checking all in single function to ensure doesn't quit early.
388
+ self.config.generate_config()
389
+
390
+ filename = os.path.join(TESTDIR, "data/initialize_config_truth.yaml")
391
+ truth = BpsConfig(filename, BPS_SEARCH_ORDER, defaults={})
392
+
393
+ self.assertEqual(self.config, truth)
394
+
395
+ def testBpsEval(self):
396
+ """Test replacing bpsEval when need to import module."""
397
+ test_opt = {
398
+ "expandEnvVars": True,
399
+ "replaceEnvVars": True,
400
+ "replaceVars": True,
401
+ "curvals": {"curr_pipetask": "ptask1"},
402
+ }
403
+ found, value = self.config.search("genval1", opt=test_opt)
404
+ self.assertEqual(found, True)
405
+ self.assertEqual(value, "-c val1:0.1 -c val2:3")
406
+
407
+ def testBpsEvalBuiltin(self):
408
+ """Test replacing bpsEval with builtin function."""
409
+ test_opt = {
410
+ "expandEnvVars": True,
411
+ "replaceEnvVars": True,
412
+ "replaceVars": True,
413
+ "curvals": {"curr_pipetask": "ptask1"},
414
+ }
415
+ found, value = self.config.search("genval2", opt=test_opt)
416
+ self.assertEqual(found, True)
417
+ self.assertEqual(value, "-c val1:32")
418
+
419
+ def testBpsEvalInvalid(self):
420
+ """Test reporting not replacing bpsEval."""
421
+ test_opt = {
422
+ "expandEnvVars": True,
423
+ "replaceEnvVars": True,
424
+ "replaceVars": True,
425
+ "curvals": {"curr_pipetask": "ptask1"},
426
+ }
427
+
428
+ self.config["badkey1"] = "badval1 bpsEval('sum([1,2]') blah"
429
+ with self.assertRaisesRegex(ValueError, "Unparsable bpsEval in 'badval1 bpsEval"):
430
+ _ = self.config.search("badkey1", opt=test_opt)
431
+
432
+
301
433
  if __name__ == "__main__":
302
434
  unittest.main()
@@ -0,0 +1,180 @@
1
+ # This file is part of ctrl_bps.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (https://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <https://www.gnu.org/licenses/>.
27
+
28
+ """Unit tests for initialize.py"""
29
+
30
+ import logging
31
+ import os
32
+ import tempfile
33
+ import unittest
34
+ from pathlib import Path
35
+
36
+ from lsst.ctrl.bps import BpsConfig
37
+ from lsst.ctrl.bps.initialize import (
38
+ init_submission,
39
+ out_collection_validator,
40
+ output_run_validator,
41
+ submit_path_validator,
42
+ )
43
+
44
+ TESTDIR = os.path.abspath(os.path.dirname(__file__))
45
+
46
+
47
+ class TestOutCollectionValidator(unittest.TestCase):
48
+ """Checks that 'outCollection' is *not* specified in config.
49
+
50
+ Assumes BpsConfig tests cover whether it finds values.
51
+ """
52
+
53
+ def testSuccess(self):
54
+ config = BpsConfig({}, defaults={})
55
+ out_collection_validator(config)
56
+
57
+ def testFailure(self):
58
+ config = BpsConfig({"outCollection": "dummy_collection"}, defaults={})
59
+ with self.assertRaisesRegex(
60
+ KeyError, "'outCollection' is deprecated. Replace all references to it with 'outputRun'"
61
+ ):
62
+ out_collection_validator(config)
63
+
64
+
65
+ class TestOutputRunValidator(unittest.TestCase):
66
+ """Checks that 'outputRun' is specified in config.
67
+
68
+ Assumes BpsConfig tests cover whether it finds values.
69
+ """
70
+
71
+ def testSuccess(self):
72
+ config = BpsConfig({"outputRun": "dummy_run_value"}, defaults={})
73
+ output_run_validator(config)
74
+
75
+ def testFailure(self):
76
+ config = BpsConfig({}, defaults={})
77
+ with self.assertRaisesRegex(KeyError, "Must specify the output run collection using 'outputRun'"):
78
+ output_run_validator(config)
79
+
80
+
81
+ class TestSubmitPathValidator(unittest.TestCase):
82
+ """Check that 'submitPath' is specified in BPS config.
83
+
84
+ Assumes BpsConfig tests cover whether it finds values.
85
+ """
86
+
87
+ def testSuccess(self):
88
+ config = BpsConfig({"submitPath": "submit/dummy/path"}, defaults={})
89
+ submit_path_validator(config)
90
+
91
+ def testFailure(self):
92
+ config = BpsConfig({}, defaults={})
93
+ with self.assertRaisesRegex(
94
+ KeyError, "Must specify the submit-side run directory using 'submitPath'"
95
+ ):
96
+ submit_path_validator(config)
97
+
98
+
99
+ class TestInitSubmission(unittest.TestCase):
100
+ """Check init_submission_function."""
101
+
102
+ def setUp(self):
103
+ self.temp_dir = tempfile.TemporaryDirectory()
104
+
105
+ def tearDown(self):
106
+ self.temp_dir.cleanup()
107
+
108
+ def testBasicSuccess(self):
109
+ filename = os.path.join(TESTDIR, "data/initialize_config.yaml")
110
+ validators = [submit_path_validator, output_run_validator, out_collection_validator]
111
+ config = init_submission(
112
+ filename,
113
+ validators=validators,
114
+ compute_site="local",
115
+ runWmsSubmissionChecks=False,
116
+ wms_service="wms_test_utils.WmsServiceSuccess",
117
+ tempDir=self.temp_dir.name,
118
+ multVal=["a", "b", "c"],
119
+ )
120
+ self.assertIn(".bps_defined.timestamp", config)
121
+ self.assertIn(".bps_defined.operator", config)
122
+ self.assertIn(".bps_defined.uniqProcName", config)
123
+ self.assertIn(".bps_defined.submitPath", config)
124
+ self.assertEqual(config[".bps_cmdline.computeSite"], "local")
125
+ self.assertEqual(config[".bps_cmdline.multVal"], "a,b,c")
126
+ uniq_proc_name = config[".bps_defined.uniqProcName"]
127
+ submit_path = Path(config[".bps_defined.submitPath"]).resolve()
128
+ files = [f.name for f in submit_path.iterdir() if f.is_file()]
129
+ self.assertIn("initialize_config.yaml", files)
130
+ self.assertIn(f"{uniq_proc_name}_config.yaml", files)
131
+ self.assertIn(f"{uniq_proc_name}.env.info.yaml", files)
132
+ self.assertIn(f"{uniq_proc_name}.pkg.info.yaml", files)
133
+
134
+ # generate_config tested elsewhere so just
135
+ # check a couple values that shows it ran.
136
+ self.assertEqual(config[".genall_1"], "/repo/test")
137
+ self.assertEqual(config[".pipetask.ptask1.p3"], 32)
138
+ self.assertEqual(config[".finalJob.gencfg_4"], 9)
139
+
140
+ @unittest.mock.patch("lsst.ctrl.bps.initialize.BPS_DEFAULTS", {})
141
+ def testMissingWmsServiceClass(self):
142
+ filename = os.path.join(TESTDIR, "data/initialize_config.yaml")
143
+
144
+ with self.assertRaisesRegex(KeyError, "Missing wmsServiceClass in bps config. Aborting."):
145
+ _ = init_submission(filename, runWmsSubmissionChecks=True, tempDir=self.temp_dir.name)
146
+
147
+ @unittest.mock.patch("lsst.ctrl.bps.initialize.BPS_DEFAULTS", {})
148
+ def testSubmissionChecksNotImplemented(self):
149
+ filename = os.path.join(TESTDIR, "data/initialize_config.yaml")
150
+
151
+ with self.assertLogs("lsst.ctrl.bps.initialize", level=logging.DEBUG) as cm:
152
+ config = init_submission(
153
+ filename,
154
+ runWmsSubmissionChecks=True,
155
+ tempDir=self.temp_dir.name,
156
+ wms_service="wms_test_utils.WmsServiceSuccess",
157
+ )
158
+ output = " ".join(cm.output)
159
+ self.assertIn("run_submission_checks is not implemented in wms_test_utils.WmsServiceSuccess.", output)
160
+ self.assertIn(".bps_defined.timestamp", config)
161
+ uniq_proc_name = config[".bps_defined.uniqProcName"]
162
+ submit_path = Path(config[".bps_defined.submitPath"]).resolve()
163
+ files = [f.name for f in submit_path.iterdir() if f.is_file()]
164
+ self.assertIn("initialize_config.yaml", files)
165
+ self.assertIn(f"{uniq_proc_name}_config.yaml", files)
166
+ self.assertIn(f"{uniq_proc_name}.env.info.yaml", files)
167
+ self.assertIn(f"{uniq_proc_name}.pkg.info.yaml", files)
168
+
169
+ @unittest.mock.patch(
170
+ "lsst.ctrl.bps.initialize.BPS_DEFAULTS", {"operator": "testuser", "uniqProcName": "uniqval"}
171
+ )
172
+ def testAlreadySet(self):
173
+ """Test if operator and uniqProcName already set."""
174
+ filename = os.path.join(TESTDIR, "data/initialize_config.yaml")
175
+
176
+ config = init_submission(filename, runWmsSubmissionChecks=False, tempDir=self.temp_dir.name)
177
+ self.assertEqual(config["operator"], "testuser")
178
+ self.assertNotIn(".bps_defined.operator", config)
179
+ self.assertEqual(config["uniqProcName"], "uniqval")
180
+ self.assertNotIn(".bps_defined.uniqProcName", config)
@@ -1,2 +0,0 @@
1
- __all__ = ["__version__"]
2
- __version__ = "29.2025.2500"