lsst-ctrl-bps-parsl 30.0.1rc1__tar.gz → 30.2025.5000__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lsst_ctrl_bps_parsl-30.0.1rc1/python/lsst_ctrl_bps_parsl.egg-info → lsst_ctrl_bps_parsl-30.2025.5000}/PKG-INFO +3 -3
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/pyproject.toml +13 -5
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/configuration.py +11 -11
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/job.py +14 -16
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/site.py +6 -6
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/sites/ccin2p3.py +17 -18
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/sites/local.py +1 -11
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/sites/princeton.py +1 -1
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/sites/slac.py +1 -1
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/sites/slurm.py +8 -20
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/sites/torque.py +16 -54
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/sites/work_queue.py +12 -12
- lsst_ctrl_bps_parsl-30.2025.5000/python/lsst/ctrl/bps/parsl/version.py +2 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/workflow.py +9 -10
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000/python/lsst_ctrl_bps_parsl.egg-info}/PKG-INFO +3 -3
- lsst_ctrl_bps_parsl-30.0.1rc1/python/lsst/ctrl/bps/parsl/version.py +0 -2
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/COPYRIGHT +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/LICENSE +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/README.md +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/bsd_license.txt +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/gpl-v3.0.txt +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/__init__.py +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/environment.py +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/service.py +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/sites/__init__.py +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/sites/nersc.py +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst_ctrl_bps_parsl.egg-info/SOURCES.txt +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst_ctrl_bps_parsl.egg-info/dependency_links.txt +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst_ctrl_bps_parsl.egg-info/requires.txt +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst_ctrl_bps_parsl.egg-info/top_level.txt +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst_ctrl_bps_parsl.egg-info/zip-safe +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/setup.cfg +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/tests/test_config.py +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/tests/test_import.py +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/tests/test_job.py +0 -0
- {lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/tests/test_sites.py +0 -0
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-ctrl-bps-parsl
|
|
3
|
-
Version: 30.
|
|
3
|
+
Version: 30.2025.5000
|
|
4
4
|
Summary: Parsl-based plugin for lsst-ctrl-bps.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License-Expression: BSD-3-Clause OR GPL-3.0-or-later
|
|
7
7
|
Project-URL: Homepage, https://github.com/lsst/ctrl_bps_parsl
|
|
8
|
-
Project-URL: Source, https://github.com/lsst/ctrl_bps_parsl
|
|
9
8
|
Keywords: lsst
|
|
10
9
|
Classifier: Intended Audience :: Science/Research
|
|
11
10
|
Classifier: Operating System :: OS Independent
|
|
12
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
16
16
|
Classifier: Topic :: Scientific/Engineering :: Astronomy
|
|
17
|
-
Requires-Python: >=3.
|
|
17
|
+
Requires-Python: >=3.11.0
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: COPYRIGHT
|
|
20
20
|
License-File: LICENSE
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "lsst-ctrl-bps-parsl"
|
|
7
|
-
requires-python = ">=3.
|
|
7
|
+
requires-python = ">=3.11.0"
|
|
8
8
|
description = "Parsl-based plugin for lsst-ctrl-bps."
|
|
9
9
|
license = "BSD-3-Clause OR GPL-3.0-or-later"
|
|
10
10
|
license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
|
|
@@ -16,6 +16,7 @@ classifiers = [
|
|
|
16
16
|
"Intended Audience :: Science/Research",
|
|
17
17
|
"Operating System :: OS Independent",
|
|
18
18
|
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
19
20
|
"Programming Language :: Python :: 3.12",
|
|
20
21
|
"Programming Language :: Python :: 3.13",
|
|
21
22
|
"Programming Language :: Python :: 3.14",
|
|
@@ -30,7 +31,6 @@ dynamic = ["version"]
|
|
|
30
31
|
|
|
31
32
|
[project.urls]
|
|
32
33
|
"Homepage" = "https://github.com/lsst/ctrl_bps_parsl"
|
|
33
|
-
"Source" = "https://github.com/lsst/ctrl_bps_parsl"
|
|
34
34
|
|
|
35
35
|
[project.optional-dependencies]
|
|
36
36
|
test = [
|
|
@@ -55,7 +55,7 @@ version = { attr = "lsst_versions.get_lsst_version" }
|
|
|
55
55
|
package_dir = "python"
|
|
56
56
|
filename = "doc/lsst.ctrl.bps.parsl/CHANGES.rst"
|
|
57
57
|
directory = "doc/changes"
|
|
58
|
-
title_format = "lsst-ctrl-bps-parsl {version}
|
|
58
|
+
title_format = "lsst-ctrl-bps-parsl {version} {project_date}"
|
|
59
59
|
issue_format = "`{issue} <https://rubinobs.atlassian.net/browse/{issue}>`_"
|
|
60
60
|
|
|
61
61
|
[[tool.towncrier.type]]
|
|
@@ -88,6 +88,15 @@ version = { attr = "lsst_versions.get_lsst_version" }
|
|
|
88
88
|
name = "An API Removal or Deprecation"
|
|
89
89
|
showcontent = true
|
|
90
90
|
|
|
91
|
+
[tool.black]
|
|
92
|
+
line-length = 110
|
|
93
|
+
target-version = ["py311"]
|
|
94
|
+
|
|
95
|
+
[tool.isort]
|
|
96
|
+
profile = "black"
|
|
97
|
+
line_length = 110
|
|
98
|
+
known_first_party = ["lsst"]
|
|
99
|
+
|
|
91
100
|
[tool.lsst_versions]
|
|
92
101
|
write_to = "python/lsst/ctrl/bps/parsl/version.py"
|
|
93
102
|
|
|
@@ -106,7 +115,7 @@ add-ignore = ["D107", "D105", "D102", "D100", "D200", "D205", "D400", "D104"]
|
|
|
106
115
|
|
|
107
116
|
[tool.ruff]
|
|
108
117
|
line-length = 110
|
|
109
|
-
target-version = "
|
|
118
|
+
target-version = "py311"
|
|
110
119
|
exclude = [
|
|
111
120
|
"__init__.py",
|
|
112
121
|
]
|
|
@@ -171,7 +180,6 @@ checks = [
|
|
|
171
180
|
"RT01", # Unfortunately our @property trigger this.
|
|
172
181
|
"RT02", # Does not want named return value. DM style says we do.
|
|
173
182
|
"SS05", # pydocstyle is better at finding infinitive verb.
|
|
174
|
-
"RT03", # Temporarily disable for sphinx empty returns.
|
|
175
183
|
]
|
|
176
184
|
exclude = [
|
|
177
185
|
'^__init__$',
|
|
@@ -89,32 +89,32 @@ def get_bps_config_value(
|
|
|
89
89
|
|
|
90
90
|
Parameters
|
|
91
91
|
----------
|
|
92
|
-
config : `
|
|
92
|
+
config : `BpsConfig`
|
|
93
93
|
Configuration from which to retrieve value.
|
|
94
94
|
key : `str`
|
|
95
95
|
Key name.
|
|
96
96
|
dataType : `type`
|
|
97
97
|
We require that the returned value have this type.
|
|
98
|
-
default :
|
|
98
|
+
default : optional
|
|
99
99
|
Default value to be provided if ``key`` doesn't exist in the
|
|
100
100
|
``config``. A default value of `None` means that there is no default.
|
|
101
101
|
required : `bool`, optional
|
|
102
|
-
If
|
|
102
|
+
If ``True``, the returned value may come from the configuration or from
|
|
103
103
|
the default, but it may not be `None`.
|
|
104
104
|
|
|
105
105
|
Returns
|
|
106
106
|
-------
|
|
107
|
-
|
|
108
|
-
Value for ``key`` in the
|
|
109
|
-
|
|
107
|
+
value
|
|
108
|
+
Value for ``key`` in the `config`` if it exists, otherwise ``default``,
|
|
109
|
+
if provided.
|
|
110
110
|
|
|
111
111
|
Raises
|
|
112
112
|
------
|
|
113
113
|
KeyError
|
|
114
|
-
|
|
115
|
-
|
|
114
|
+
If ``key`` is not in ``config`` and no default is provided but a value
|
|
115
|
+
is ``required``.
|
|
116
116
|
RuntimeError
|
|
117
|
-
|
|
117
|
+
If the value is not set or is of the wrong type.
|
|
118
118
|
"""
|
|
119
119
|
options: dict[str, Any] = {"expandEnvVars": True, "replaceVars": True, "required": required}
|
|
120
120
|
if default is not None:
|
|
@@ -138,7 +138,7 @@ def get_workflow_name(config: BpsConfig) -> str:
|
|
|
138
138
|
|
|
139
139
|
Parameters
|
|
140
140
|
----------
|
|
141
|
-
config : `
|
|
141
|
+
config : `BpsConfig`
|
|
142
142
|
BPS configuration.
|
|
143
143
|
|
|
144
144
|
Returns
|
|
@@ -177,7 +177,7 @@ def set_parsl_logging(config: BpsConfig) -> int:
|
|
|
177
177
|
|
|
178
178
|
Parameters
|
|
179
179
|
----------
|
|
180
|
-
config : `
|
|
180
|
+
config : `BpsConfig`
|
|
181
181
|
BPS configuration.
|
|
182
182
|
|
|
183
183
|
Returns
|
{lsst_ctrl_bps_parsl-30.0.1rc1 → lsst_ctrl_bps_parsl-30.2025.5000}/python/lsst/ctrl/bps/parsl/job.py
RENAMED
|
@@ -35,7 +35,7 @@ from textwrap import dedent
|
|
|
35
35
|
from typing import Any
|
|
36
36
|
|
|
37
37
|
from parsl.app.bash import BashApp
|
|
38
|
-
from parsl.
|
|
38
|
+
from parsl.app.futures import Future
|
|
39
39
|
|
|
40
40
|
from lsst.ctrl.bps import BpsConfig, GenericWorkflow, GenericWorkflowJob
|
|
41
41
|
|
|
@@ -49,7 +49,7 @@ _file_regex = re.compile(r"<FILE:(\S+)>") # Regex for replacing <FILE:WHATEVER>
|
|
|
49
49
|
|
|
50
50
|
def run_command(
|
|
51
51
|
command_line: str,
|
|
52
|
-
inputs: Sequence[
|
|
52
|
+
inputs: Sequence[Future] = (),
|
|
53
53
|
stdout: str | None = None,
|
|
54
54
|
stderr: str | None = None,
|
|
55
55
|
parsl_resource_specification: dict[str, Any] | None = None,
|
|
@@ -58,14 +58,13 @@ def run_command(
|
|
|
58
58
|
|
|
59
59
|
This function exists to get information into parsl, through the ``inputs``,
|
|
60
60
|
``stdout`` and ``stderr`` parameters. It needs to be wrapped by a parsl
|
|
61
|
-
``bash_app`` decorator before use, after which it will return a
|
|
62
|
-
`parsl.dataflow.futures.AppFuture`.
|
|
61
|
+
``bash_app`` decorator before use, after which it will return a `Future`.
|
|
63
62
|
|
|
64
63
|
Parameters
|
|
65
64
|
----------
|
|
66
65
|
command_line : `str`
|
|
67
66
|
Command-line to have parsl run.
|
|
68
|
-
inputs :
|
|
67
|
+
inputs : list of `Future`
|
|
69
68
|
Other commands that must have run before this.
|
|
70
69
|
stdout, stderr : `str`, optional
|
|
71
70
|
Filenames for stdout and stderr.
|
|
@@ -85,7 +84,7 @@ def get_file_paths(workflow: GenericWorkflow, name: str) -> dict[str, str]:
|
|
|
85
84
|
|
|
86
85
|
Parameters
|
|
87
86
|
----------
|
|
88
|
-
workflow : `
|
|
87
|
+
workflow : `GenericWorkflow`
|
|
89
88
|
BPS workflow that knows the file paths.
|
|
90
89
|
name : `str`
|
|
91
90
|
Job name.
|
|
@@ -103,9 +102,9 @@ class ParslJob:
|
|
|
103
102
|
|
|
104
103
|
Parameters
|
|
105
104
|
----------
|
|
106
|
-
generic : `
|
|
105
|
+
generic : `GenericWorkflowJob`
|
|
107
106
|
BPS job information.
|
|
108
|
-
config : `
|
|
107
|
+
config : `BpsConfig`
|
|
109
108
|
BPS configuration.
|
|
110
109
|
file_paths : `dict` mapping `str` to `str`
|
|
111
110
|
File paths for job, indexed by symbolic name.
|
|
@@ -255,10 +254,10 @@ class ParslJob:
|
|
|
255
254
|
def get_future(
|
|
256
255
|
self,
|
|
257
256
|
app: BashApp,
|
|
258
|
-
inputs: list[
|
|
257
|
+
inputs: list[Future],
|
|
259
258
|
command_prefix: str | None = None,
|
|
260
259
|
resource_list: list | None = None,
|
|
261
|
-
) ->
|
|
260
|
+
) -> Future | None:
|
|
262
261
|
"""Get the parsl app future for the job.
|
|
263
262
|
|
|
264
263
|
This effectively queues the job for execution by a worker, subject to
|
|
@@ -266,9 +265,9 @@ class ParslJob:
|
|
|
266
265
|
|
|
267
266
|
Parameters
|
|
268
267
|
----------
|
|
269
|
-
app :
|
|
268
|
+
app : callable
|
|
270
269
|
A parsl bash_app decorator to use.
|
|
271
|
-
inputs :
|
|
270
|
+
inputs : list of `Future`
|
|
272
271
|
Dependencies to be satisfied before executing this job.
|
|
273
272
|
command_prefix : `str`, optional
|
|
274
273
|
Bash commands to execute before the job command, e.g., for setting
|
|
@@ -278,10 +277,9 @@ class ParslJob:
|
|
|
278
277
|
|
|
279
278
|
Returns
|
|
280
279
|
-------
|
|
281
|
-
future : `
|
|
282
|
-
A `
|
|
283
|
-
|
|
284
|
-
``run_local``).
|
|
280
|
+
future : `Future` or `None`
|
|
281
|
+
A `Future` object linked to the execution of the job, or `None` if
|
|
282
|
+
the job has already been done (e.g., by ``run_local``).
|
|
285
283
|
"""
|
|
286
284
|
if self.done:
|
|
287
285
|
return None # Nothing to do
|
|
@@ -54,7 +54,7 @@ class SiteConfig(ABC):
|
|
|
54
54
|
|
|
55
55
|
Parameters
|
|
56
56
|
----------
|
|
57
|
-
config : `
|
|
57
|
+
config : `BpsConfig`
|
|
58
58
|
BPS configuration.
|
|
59
59
|
resource_list : `list`, optional
|
|
60
60
|
List of parsl resource specifications to pass to the executor.
|
|
@@ -74,12 +74,12 @@ class SiteConfig(ABC):
|
|
|
74
74
|
|
|
75
75
|
Parameters
|
|
76
76
|
----------
|
|
77
|
-
config : `
|
|
77
|
+
config : `BpsConfig`
|
|
78
78
|
BPS configuration.
|
|
79
79
|
|
|
80
80
|
Returns
|
|
81
81
|
-------
|
|
82
|
-
site : `
|
|
82
|
+
site : `BpsConfig`
|
|
83
83
|
Site sub-configuration.
|
|
84
84
|
"""
|
|
85
85
|
computeSite = get_bps_config_value(config, "computeSite", str, required=True)
|
|
@@ -97,7 +97,7 @@ class SiteConfig(ABC):
|
|
|
97
97
|
|
|
98
98
|
Parameters
|
|
99
99
|
----------
|
|
100
|
-
config : `
|
|
100
|
+
config : `BpsConfig`
|
|
101
101
|
BPS configuration.
|
|
102
102
|
|
|
103
103
|
Returns
|
|
@@ -126,7 +126,7 @@ class SiteConfig(ABC):
|
|
|
126
126
|
|
|
127
127
|
Parameters
|
|
128
128
|
----------
|
|
129
|
-
job : `
|
|
129
|
+
job : `ParslJob`
|
|
130
130
|
Job to be executed.
|
|
131
131
|
|
|
132
132
|
Returns
|
|
@@ -185,7 +185,7 @@ class SiteConfig(ABC):
|
|
|
185
185
|
|
|
186
186
|
Returns
|
|
187
187
|
-------
|
|
188
|
-
monitor : `
|
|
188
|
+
monitor : `MonitoringHub` or `None`
|
|
189
189
|
Parsl monitor, or `None` for no monitor.
|
|
190
190
|
"""
|
|
191
191
|
if not get_bps_config_value(self.site, "monitorEnable", bool, False):
|
|
@@ -11,7 +11,7 @@ from ..configuration import get_bps_config_value
|
|
|
11
11
|
from ..site import SiteConfig
|
|
12
12
|
|
|
13
13
|
if TYPE_CHECKING:
|
|
14
|
-
from
|
|
14
|
+
from .job import ParslJob
|
|
15
15
|
|
|
16
16
|
__all__ = ("Ccin2p3",)
|
|
17
17
|
|
|
@@ -46,7 +46,7 @@ class Ccin2p3(SiteConfig):
|
|
|
46
46
|
If you do need to modify those defaults, you can overwrite them for
|
|
47
47
|
all job slots or for specific each job slots. Requirements specified
|
|
48
48
|
for a job slot take priority over those specified for all job slots
|
|
49
|
-
at the level of entry
|
|
49
|
+
at the level of entry '.site.ccin2p3:'.
|
|
50
50
|
|
|
51
51
|
This is an example of how to overwrite selected requirements in your BPS
|
|
52
52
|
submission file:
|
|
@@ -79,41 +79,40 @@ class Ccin2p3(SiteConfig):
|
|
|
79
79
|
- "--licenses=my_product"
|
|
80
80
|
- "--reservation=my_reservation"
|
|
81
81
|
|
|
82
|
-
At the level of entry
|
|
82
|
+
At the level of entry 'site.ccin2p3:' in the BPS submission file, the
|
|
83
83
|
following configuration parameters are accepted, which apply to all slot
|
|
84
84
|
sizes:
|
|
85
85
|
|
|
86
|
-
-
|
|
86
|
+
- `partition` (`str`): name of the one or more configured partitions. If
|
|
87
87
|
more than one, separate them with comma (',').
|
|
88
88
|
(Default: "lsst,htc")
|
|
89
|
-
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
Slurm for scheduling purposes.
|
|
89
|
+
- `walltime` (`str`): walltime to require for the job (Default: "72:00:00")
|
|
90
|
+
- `scheduler_options` (`list` [`str`] ): scheduler options to send to Slurm
|
|
91
|
+
for scheduling purposes.
|
|
93
92
|
(Default: "--licenses=sps")
|
|
94
93
|
|
|
95
94
|
In addition, as shown in the previous example, for each job slot (i.e.
|
|
96
95
|
"small", "medium", etc.) you can specify the requirements above as well as
|
|
97
96
|
the following:
|
|
98
97
|
|
|
99
|
-
-
|
|
100
|
-
|
|
101
|
-
-
|
|
102
|
-
(Defaults: 4 for "small", 10 for "medium", 50
|
|
98
|
+
- `max_blocks` (`int`): maximum number of Slurm jobs that your workflow can
|
|
99
|
+
simultaneously use.
|
|
100
|
+
- `memory` (`int`): required amount of memory for each job, in Gigabytes.
|
|
101
|
+
(Defaults: 4 for "small", 10 for "medium", 50 fo "large" and
|
|
103
102
|
150 for "xlarge").
|
|
104
103
|
|
|
105
104
|
Parameters
|
|
106
105
|
----------
|
|
107
|
-
*args
|
|
106
|
+
*args : optional
|
|
108
107
|
Arguments to initialize the super-class.
|
|
109
|
-
**kwargs
|
|
108
|
+
**kwargs : optional
|
|
110
109
|
Keyword arguments to initialize the super-class.
|
|
111
110
|
|
|
112
111
|
Returns
|
|
113
112
|
-------
|
|
114
|
-
Ccin2p3 : `
|
|
115
|
-
Concrete instance of a
|
|
116
|
-
|
|
113
|
+
Ccin2p3 : `SiteConfig`
|
|
114
|
+
Concrete instance of a `SiteConfig` specific for the CC-IN2P3 Slurm
|
|
115
|
+
farm.
|
|
117
116
|
"""
|
|
118
117
|
|
|
119
118
|
DEFAULT_ACCOUNT: str = "lsst"
|
|
@@ -317,7 +316,7 @@ class Ccin2p3(SiteConfig):
|
|
|
317
316
|
|
|
318
317
|
Parameters
|
|
319
318
|
----------
|
|
320
|
-
job : `
|
|
319
|
+
job : `ParslJob`
|
|
321
320
|
Job to be executed.
|
|
322
321
|
|
|
323
322
|
Returns
|
|
@@ -43,16 +43,6 @@ __all__ = ("Local",)
|
|
|
43
43
|
class Local(SiteConfig):
|
|
44
44
|
"""Configuration for running jobs on the local machine.
|
|
45
45
|
|
|
46
|
-
Parameters
|
|
47
|
-
----------
|
|
48
|
-
*args : `~typing.Any`
|
|
49
|
-
Parameters forwarded to base class constructor.
|
|
50
|
-
**kwargs : `~typing.Any`
|
|
51
|
-
Keyword arguments passed to base class constructor, augmented by
|
|
52
|
-
the ``resource_list`` argument.
|
|
53
|
-
|
|
54
|
-
Notes
|
|
55
|
-
-----
|
|
56
46
|
The number of cores to use is specified in the site configuration, under
|
|
57
47
|
``site.<computeSite>.cores`` (`int`).
|
|
58
48
|
"""
|
|
@@ -75,7 +65,7 @@ class Local(SiteConfig):
|
|
|
75
65
|
|
|
76
66
|
Parameters
|
|
77
67
|
----------
|
|
78
|
-
job : `
|
|
68
|
+
job : `ParslJob`
|
|
79
69
|
Job to be executed.
|
|
80
70
|
|
|
81
71
|
Returns
|
|
@@ -46,16 +46,6 @@ Kwargs = dict[str, Any]
|
|
|
46
46
|
class Slurm(SiteConfig):
|
|
47
47
|
"""Configuration for generic Slurm cluster.
|
|
48
48
|
|
|
49
|
-
Parameters
|
|
50
|
-
----------
|
|
51
|
-
*args : `~typing.Any`
|
|
52
|
-
Parameters forwarded to base class constructor.
|
|
53
|
-
**kwargs : `~typing.Any`
|
|
54
|
-
Keyword arguments passed to base class constructor, augmented by
|
|
55
|
-
the ``resource_list`` argument.
|
|
56
|
-
|
|
57
|
-
Notes
|
|
58
|
-
-----
|
|
59
49
|
This can be used directly as the site configuration for a Slurm cluster by
|
|
60
50
|
setting the BPS config, e.g.:
|
|
61
51
|
|
|
@@ -136,15 +126,13 @@ class Slurm(SiteConfig):
|
|
|
136
126
|
scheduler_options : `str`, optional
|
|
137
127
|
``#SBATCH`` directives to prepend to the Slurm submission script.
|
|
138
128
|
provider_options : `dict`, optional
|
|
139
|
-
Additional arguments for `
|
|
140
|
-
constructor.
|
|
129
|
+
Additional arguments for `SlurmProvider` constructor.
|
|
141
130
|
executor_options : `dict`, optional
|
|
142
|
-
Additional arguments for `
|
|
143
|
-
constructor.
|
|
131
|
+
Additional arguments for `HighThroughputExecutor` constructor.
|
|
144
132
|
|
|
145
133
|
Returns
|
|
146
134
|
-------
|
|
147
|
-
executor : `
|
|
135
|
+
executor : `HighThroughputExecutor`
|
|
148
136
|
Executor for Slurm jobs.
|
|
149
137
|
"""
|
|
150
138
|
nodes = get_bps_config_value(self.site, "nodes", int, nodes, required=True)
|
|
@@ -203,7 +191,7 @@ class Slurm(SiteConfig):
|
|
|
203
191
|
|
|
204
192
|
Parameters
|
|
205
193
|
----------
|
|
206
|
-
job : `
|
|
194
|
+
job : `ParslJob`
|
|
207
195
|
Job to be executed.
|
|
208
196
|
|
|
209
197
|
Returns
|
|
@@ -277,11 +265,11 @@ class TripleSlurm(Slurm):
|
|
|
277
265
|
|
|
278
266
|
Parameters
|
|
279
267
|
----------
|
|
280
|
-
small_options :
|
|
268
|
+
small_options : kwargs
|
|
281
269
|
Options for ``make_executor`` for small executor.
|
|
282
|
-
medium_options :
|
|
270
|
+
medium_options : kwargs
|
|
283
271
|
Options for ``make_executor`` for medium executor.
|
|
284
|
-
large_options :
|
|
272
|
+
large_options : kwargs
|
|
285
273
|
Options for ``make_executor`` for large executor.
|
|
286
274
|
**common_options
|
|
287
275
|
Common options for ``make_executor`` for each of the executors.
|
|
@@ -314,7 +302,7 @@ class TripleSlurm(Slurm):
|
|
|
314
302
|
|
|
315
303
|
Parameters
|
|
316
304
|
----------
|
|
317
|
-
job : `
|
|
305
|
+
job : `ParslJob`
|
|
318
306
|
Job to be executed.
|
|
319
307
|
|
|
320
308
|
Returns
|
|
@@ -47,16 +47,6 @@ Kwargs = dict[str, Any]
|
|
|
47
47
|
class Torque(SiteConfig):
|
|
48
48
|
"""Configuration for generic Torque cluster.
|
|
49
49
|
|
|
50
|
-
Parameters
|
|
51
|
-
----------
|
|
52
|
-
*args : `~typing.Any`
|
|
53
|
-
Parameters forwarded to base class constructor.
|
|
54
|
-
**kwargs : `~typing.Any`
|
|
55
|
-
Keyword arguments passed to base class constructor, augmented by
|
|
56
|
-
the ``resource_list`` argument.
|
|
57
|
-
|
|
58
|
-
Notes
|
|
59
|
-
-----
|
|
60
50
|
This can be used directly as the site configuration for a Torque cluster by
|
|
61
51
|
setting the BPS config, e.g.:
|
|
62
52
|
|
|
@@ -120,20 +110,18 @@ class Torque(SiteConfig):
|
|
|
120
110
|
Default time limit for each Torque job.
|
|
121
111
|
mem_per_worker : `float`, optional
|
|
122
112
|
Minimum memory per worker (GB), limited by the executor.
|
|
113
|
+
worker_init : `str`, optional
|
|
114
|
+
Environment initiation command
|
|
123
115
|
scheduler_options : `str`, optional
|
|
124
116
|
``#SBATCH`` directives to prepend to the Torque submission script.
|
|
125
|
-
worker_init : `str`, optional
|
|
126
|
-
Environment initiation command.
|
|
127
117
|
provider_options : `dict`, optional
|
|
128
|
-
Additional arguments for `
|
|
129
|
-
constructor.
|
|
118
|
+
Additional arguments for `TorqueProvider` constructor.
|
|
130
119
|
executor_options : `dict`, optional
|
|
131
|
-
Additional arguments for `
|
|
132
|
-
constructor.
|
|
120
|
+
Additional arguments for `HighThroughputExecutor` constructor.
|
|
133
121
|
|
|
134
122
|
Returns
|
|
135
123
|
-------
|
|
136
|
-
executor : `
|
|
124
|
+
executor : `HighThroughputExecutor`
|
|
137
125
|
Executor for Torque jobs.
|
|
138
126
|
"""
|
|
139
127
|
nodes = get_bps_config_value(self.site, "nodes", int, nodes, required=True)
|
|
@@ -191,7 +179,7 @@ class Torque(SiteConfig):
|
|
|
191
179
|
|
|
192
180
|
Parameters
|
|
193
181
|
----------
|
|
194
|
-
job : `
|
|
182
|
+
job : `ParslJob`
|
|
195
183
|
Job to be executed.
|
|
196
184
|
|
|
197
185
|
Returns
|
|
@@ -203,20 +191,8 @@ class Torque(SiteConfig):
|
|
|
203
191
|
|
|
204
192
|
|
|
205
193
|
class PbsTorqueProvider(TorqueProvider):
|
|
206
|
-
"""Torque Execution Provider
|
|
207
|
-
|
|
208
|
-
Parameters
|
|
209
|
-
----------
|
|
210
|
-
*args : `~typing.Any`
|
|
211
|
-
Parameters forwarded to base class constructor.
|
|
212
|
-
tasks_per_node : `int`, optional
|
|
213
|
-
Number of tasks per node.
|
|
214
|
-
**kwargs : `~typing.Any`
|
|
215
|
-
Keyword arguments passed to base class constructor, augmented by
|
|
216
|
-
the ``resource_list`` argument.
|
|
217
|
-
|
|
218
|
-
Notes
|
|
219
|
-
-----
|
|
194
|
+
"""Torque Execution Provider
|
|
195
|
+
|
|
220
196
|
This provider uses qsub to submit, qstat for status, and qdel to cancel
|
|
221
197
|
jobs. The qsub script to be used is created from a template file in this
|
|
222
198
|
same module.
|
|
@@ -232,6 +208,12 @@ class PbsTorqueProvider(TorqueProvider):
|
|
|
232
208
|
def submit(self, command, tasks_per_node, job_name="parsl.torque"):
|
|
233
209
|
"""Submit the command onto an Local Resource Manager job.
|
|
234
210
|
|
|
211
|
+
This function returns an ID that corresponds to the task that was just
|
|
212
|
+
submitted.
|
|
213
|
+
|
|
214
|
+
The ``tasks_per_node`` parameter is ignored in this provider, as it is
|
|
215
|
+
set at construction time.
|
|
216
|
+
|
|
235
217
|
Parameters
|
|
236
218
|
----------
|
|
237
219
|
command : `str`
|
|
@@ -244,17 +226,9 @@ class PbsTorqueProvider(TorqueProvider):
|
|
|
244
226
|
|
|
245
227
|
Returns
|
|
246
228
|
-------
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
If job_id (`str`): Identifier for the job.
|
|
250
|
-
|
|
251
|
-
Notes
|
|
252
|
-
-----
|
|
253
|
-
This function returns an ID that corresponds to the task that was just
|
|
254
|
-
submitted.
|
|
229
|
+
None: At capacity, cannot provision more
|
|
230
|
+
job_id (string): Identifier for the job
|
|
255
231
|
|
|
256
|
-
The ``tasks_per_node`` parameter is ignored in this provider, as it is
|
|
257
|
-
set at construction time.
|
|
258
232
|
"""
|
|
259
233
|
return super().submit(
|
|
260
234
|
command=command,
|
|
@@ -267,22 +241,10 @@ class PbsMpiRunLauncher(MpiRunLauncher):
|
|
|
267
241
|
"""Worker launcher that wraps the user's command with the framework to
|
|
268
242
|
launch multiple command invocations via ``mpirun``.
|
|
269
243
|
|
|
270
|
-
Parameters
|
|
271
|
-
----------
|
|
272
|
-
debug : `bool`, optional
|
|
273
|
-
Enable or disable debug logging.
|
|
274
|
-
bash_location : `str`, optional
|
|
275
|
-
Path to the ``bash`` shell binary.
|
|
276
|
-
overrides : `str`, optional
|
|
277
|
-
Any override options.
|
|
278
|
-
|
|
279
|
-
Notes
|
|
280
|
-
-----
|
|
281
244
|
This wrapper sets the bash env variable ``CORES`` to the number of cores on
|
|
282
245
|
the machine.
|
|
283
246
|
|
|
284
247
|
This launcher makes the following assumptions:
|
|
285
|
-
|
|
286
248
|
- mpirun is installed and can be located in ``$PATH``
|
|
287
249
|
- The provider makes available the ``$PBS_NODEFILE`` environment variable
|
|
288
250
|
"""
|
|
@@ -47,11 +47,11 @@ __all__ = ("LocalSrunWorkQueue", "WorkQueue")
|
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
class WorkQueue(SiteConfig):
|
|
50
|
-
"""Base class configuraton for `
|
|
50
|
+
"""Base class configuraton for `WorkQueueExecutor`.
|
|
51
51
|
|
|
52
52
|
Subclasses must provide implementations for ``.get_executors``
|
|
53
53
|
and ``.select_executor``. In ``.get_executors``, the site-specific
|
|
54
|
-
|
|
54
|
+
`ExecutionProvider` must be defined.
|
|
55
55
|
|
|
56
56
|
Parameters
|
|
57
57
|
----------
|
|
@@ -91,20 +91,20 @@ class WorkQueue(SiteConfig):
|
|
|
91
91
|
worker_options: str = "",
|
|
92
92
|
wq_max_retries: int = 1,
|
|
93
93
|
) -> ParslExecutor:
|
|
94
|
-
"""Return a `
|
|
95
|
-
|
|
94
|
+
"""Return a `WorkQueueExecutor`. The ``provider`` contains the
|
|
95
|
+
site-specific configuration.
|
|
96
96
|
|
|
97
97
|
Parameters
|
|
98
98
|
----------
|
|
99
99
|
label : `str`
|
|
100
100
|
Label for executor.
|
|
101
|
-
provider : `
|
|
102
|
-
Parsl execution provider, e.g., `
|
|
101
|
+
provider : `ExecutionProvider`
|
|
102
|
+
Parsl execution provider, e.g., `SlurmProvider`.
|
|
103
103
|
port : `int`, optional
|
|
104
104
|
Port used by work_queue. Default: ``9000``.
|
|
105
105
|
worker_options : `str`, optional
|
|
106
106
|
Extra options to pass to work_queue workers, e.g.,
|
|
107
|
-
``"--memory=90000"``. Default:
|
|
107
|
+
``"--memory=90000"``. Default: `""`.
|
|
108
108
|
wq_max_retries : `int`, optional
|
|
109
109
|
Number of retries for work_queue to attempt per job. Set to
|
|
110
110
|
``None`` to have it try indefinitely; set to ``1`` to have Parsl
|
|
@@ -125,13 +125,13 @@ class WorkQueue(SiteConfig):
|
|
|
125
125
|
|
|
126
126
|
|
|
127
127
|
class LocalSrunWorkQueue(WorkQueue):
|
|
128
|
-
"""Configuration for a `
|
|
129
|
-
|
|
128
|
+
"""Configuration for a `WorkQueueExecutor` that uses a `LocalProvider`
|
|
129
|
+
to manage resources.
|
|
130
130
|
|
|
131
131
|
This can be used directly as the site configuration within a
|
|
132
132
|
multi-node allocation when Slurm is available. For running on a
|
|
133
|
-
single node, e.g., a laptop, a `
|
|
134
|
-
|
|
133
|
+
single node, e.g., a laptop, a `SingleNodeLauncher` is used, and
|
|
134
|
+
Slurm need not be available.
|
|
135
135
|
|
|
136
136
|
The following BPS configuration parameters are recognized, overriding the
|
|
137
137
|
defaults:
|
|
@@ -170,7 +170,7 @@ class LocalSrunWorkQueue(WorkQueue):
|
|
|
170
170
|
|
|
171
171
|
Parameters
|
|
172
172
|
----------
|
|
173
|
-
job : `
|
|
173
|
+
job : `ParslJob`
|
|
174
174
|
Job to be executed.
|
|
175
175
|
|
|
176
176
|
Returns
|
|
@@ -60,7 +60,7 @@ def get_parsl_config(config: BpsConfig) -> parsl.config.Config:
|
|
|
60
60
|
|
|
61
61
|
Parameters
|
|
62
62
|
----------
|
|
63
|
-
config : `
|
|
63
|
+
config : `BpsConfig`
|
|
64
64
|
BPS configuration.
|
|
65
65
|
|
|
66
66
|
Returns
|
|
@@ -83,7 +83,7 @@ class ParslWorkflow(BaseWmsWorkflow):
|
|
|
83
83
|
Generic workflow config.
|
|
84
84
|
path : `str`
|
|
85
85
|
Path prefix for workflow output files.
|
|
86
|
-
jobs : `dict` mapping `str` to `
|
|
86
|
+
jobs : `dict` mapping `str` to `ParslJob`
|
|
87
87
|
Jobs to be executed.
|
|
88
88
|
parents : `dict` mapping `str` to iterable of `str`
|
|
89
89
|
Dependency tree. Keywords are job names, and values are a list of job
|
|
@@ -92,7 +92,7 @@ class ParslWorkflow(BaseWmsWorkflow):
|
|
|
92
92
|
endpoints : iterable of `str`
|
|
93
93
|
Endpoints of the dependency tree. These jobs (specified by name) have
|
|
94
94
|
no children.
|
|
95
|
-
final : `
|
|
95
|
+
final : `ParslJob`, optional
|
|
96
96
|
Final job to be done, e.g., to merge the execution butler. This is done
|
|
97
97
|
locally.
|
|
98
98
|
"""
|
|
@@ -149,7 +149,7 @@ class ParslWorkflow(BaseWmsWorkflow):
|
|
|
149
149
|
|
|
150
150
|
Parameters
|
|
151
151
|
----------
|
|
152
|
-
config : `
|
|
152
|
+
config : `BpsConfig`
|
|
153
153
|
Configuration of the workflow.
|
|
154
154
|
generic_workflow : `lsst.ctrl.bps.generic_workflow.GenericWorkflow`
|
|
155
155
|
Generic representation of a single workflow.
|
|
@@ -233,9 +233,8 @@ class ParslWorkflow(BaseWmsWorkflow):
|
|
|
233
233
|
|
|
234
234
|
Returns
|
|
235
235
|
-------
|
|
236
|
-
futures : `list` of `
|
|
237
|
-
`
|
|
238
|
-
of the endpoint jobs.
|
|
236
|
+
futures : `list` of `Future`
|
|
237
|
+
`Future` objects linked to the execution of the endpoint jobs.
|
|
239
238
|
"""
|
|
240
239
|
futures = [self.execute(name) for name in self.endpoints]
|
|
241
240
|
if block:
|
|
@@ -261,9 +260,9 @@ class ParslWorkflow(BaseWmsWorkflow):
|
|
|
261
260
|
|
|
262
261
|
Returns
|
|
263
262
|
-------
|
|
264
|
-
future : `
|
|
265
|
-
A `
|
|
266
|
-
|
|
263
|
+
future : `Future` or `None`
|
|
264
|
+
A `Future` object linked to the execution of the job, or `None` if
|
|
265
|
+
the job is being reserved to run locally.
|
|
267
266
|
"""
|
|
268
267
|
if name in ("pipetaskInit", "mergeExecutionButler"):
|
|
269
268
|
# These get done outside of parsl
|
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-ctrl-bps-parsl
|
|
3
|
-
Version: 30.
|
|
3
|
+
Version: 30.2025.5000
|
|
4
4
|
Summary: Parsl-based plugin for lsst-ctrl-bps.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License-Expression: BSD-3-Clause OR GPL-3.0-or-later
|
|
7
7
|
Project-URL: Homepage, https://github.com/lsst/ctrl_bps_parsl
|
|
8
|
-
Project-URL: Source, https://github.com/lsst/ctrl_bps_parsl
|
|
9
8
|
Keywords: lsst
|
|
10
9
|
Classifier: Intended Audience :: Science/Research
|
|
11
10
|
Classifier: Operating System :: OS Independent
|
|
12
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
16
16
|
Classifier: Topic :: Scientific/Engineering :: Astronomy
|
|
17
|
-
Requires-Python: >=3.
|
|
17
|
+
Requires-Python: >=3.11.0
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: COPYRIGHT
|
|
20
20
|
License-File: LICENSE
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|