lsst-ctrl-bps-panda 28.2025.600__tar.gz → 29.2025.4700__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lsst_ctrl_bps_panda-28.2025.600/python/lsst_ctrl_bps_panda.egg-info → lsst_ctrl_bps_panda-29.2025.4700}/PKG-INFO +5 -4
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/pyproject.toml +14 -3
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cli/cmd/__init__.py +2 -2
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cli/cmd/panda_auth_commands.py +20 -3
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cli/panda_auth.py +1 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/constants.py +2 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/edgenode/build_cmd_line_decoder.py +25 -2
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/edgenode/cmd_line_decoder.py +111 -42
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/panda_auth_drivers.py +28 -2
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/panda_auth_utils.py +97 -1
- lsst_ctrl_bps_panda-29.2025.4700/python/lsst/ctrl/bps/panda/panda_exceptions.py +34 -0
- lsst_ctrl_bps_panda-29.2025.4700/python/lsst/ctrl/bps/panda/panda_service.py +513 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/utils.py +283 -47
- lsst_ctrl_bps_panda-29.2025.4700/python/lsst/ctrl/bps/panda/version.py +2 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700/python/lsst_ctrl_bps_panda.egg-info}/PKG-INFO +5 -4
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst_ctrl_bps_panda.egg-info/SOURCES.txt +3 -0
- lsst_ctrl_bps_panda-29.2025.4700/tests/test_cmd_line_decoder.py +52 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/tests/test_cmd_line_embedder.py +1 -0
- lsst_ctrl_bps_panda-29.2025.4700/tests/test_panda_auth_utils.py +151 -0
- lsst_ctrl_bps_panda-29.2025.4700/tests/test_panda_service.py +194 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/tests/test_utils.py +2 -1
- lsst_ctrl_bps_panda-28.2025.600/python/lsst/ctrl/bps/panda/panda_service.py +0 -400
- lsst_ctrl_bps_panda-28.2025.600/python/lsst/ctrl/bps/panda/version.py +0 -2
- lsst_ctrl_bps_panda-28.2025.600/tests/test_panda_auth_utils.py +0 -63
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/COPYRIGHT +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/LICENSE +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/README.rst +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/bsd_license.txt +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/gpl-v3.0.txt +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/__init__.py +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cli/__init__.py +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cmd_line_embedder.py +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/conf_example/example_panda_SLAC.yaml +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/conf_example/pipelines_check_idf.yaml +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/conf_example/test_idf.yaml +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/conf_example/test_sdf.yaml +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/conf_example/test_usdf.yaml +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/edgenode/__init__.py +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst_ctrl_bps_panda.egg-info/dependency_links.txt +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst_ctrl_bps_panda.egg-info/requires.txt +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst_ctrl_bps_panda.egg-info/top_level.txt +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst_ctrl_bps_panda.egg-info/zip-safe +0 -0
- {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/setup.cfg +0 -0
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-ctrl-bps-panda
|
|
3
|
-
Version:
|
|
3
|
+
Version: 29.2025.4700
|
|
4
4
|
Summary: PanDA plugin for lsst-ctrl-bps.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
|
-
License: BSD
|
|
6
|
+
License-Expression: BSD-3-Clause OR GPL-3.0-or-later
|
|
7
7
|
Project-URL: Homepage, https://github.com/lsst/ctrl_bps_panda
|
|
8
8
|
Keywords: lsst
|
|
9
9
|
Classifier: Intended Audience :: Science/Research
|
|
10
|
-
Classifier: License :: OSI Approved :: BSD License
|
|
11
10
|
Classifier: Operating System :: OS Independent
|
|
12
11
|
Classifier: Programming Language :: Python :: 3
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
16
|
Classifier: Topic :: Scientific/Engineering :: Astronomy
|
|
17
17
|
Requires-Python: >=3.11.0
|
|
18
18
|
Description-Content-Type: text/x-rst
|
|
@@ -34,6 +34,7 @@ Requires-Dist: lsst-utils
|
|
|
34
34
|
Provides-Extra: test
|
|
35
35
|
Requires-Dist: pytest>=3.2; extra == "test"
|
|
36
36
|
Requires-Dist: pytest-openfiles>=0.5.0; extra == "test"
|
|
37
|
+
Dynamic: license-file
|
|
37
38
|
|
|
38
39
|
##############
|
|
39
40
|
ctrl_bps_panda
|
|
@@ -6,19 +6,20 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
name = "lsst-ctrl-bps-panda"
|
|
7
7
|
requires-python = ">=3.11.0"
|
|
8
8
|
description = "PanDA plugin for lsst-ctrl-bps."
|
|
9
|
-
license =
|
|
9
|
+
license = "BSD-3-Clause OR GPL-3.0-or-later"
|
|
10
|
+
license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
|
|
10
11
|
readme = "README.rst"
|
|
11
12
|
authors = [
|
|
12
13
|
{name="Rubin Observatory Data Management", email="dm-admin@lists.lsst.org"},
|
|
13
14
|
]
|
|
14
15
|
classifiers = [
|
|
15
16
|
"Intended Audience :: Science/Research",
|
|
16
|
-
"License :: OSI Approved :: BSD License",
|
|
17
17
|
"Operating System :: OS Independent",
|
|
18
18
|
"Programming Language :: Python :: 3",
|
|
19
19
|
"Programming Language :: Python :: 3.11",
|
|
20
20
|
"Programming Language :: Python :: 3.12",
|
|
21
21
|
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Programming Language :: Python :: 3.14",
|
|
22
23
|
"Topic :: Scientific/Engineering :: Astronomy",
|
|
23
24
|
]
|
|
24
25
|
keywords = ["lsst"]
|
|
@@ -51,7 +52,6 @@ where = ["python"]
|
|
|
51
52
|
|
|
52
53
|
[tool.setuptools]
|
|
53
54
|
zip-safe = true
|
|
54
|
-
license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
|
|
55
55
|
|
|
56
56
|
[tool.setuptools.package-data]
|
|
57
57
|
"lsst.ctrl.bps.panda" = ["conf_example/*.yaml"]
|
|
@@ -104,6 +104,7 @@ target-version = ["py311"]
|
|
|
104
104
|
[tool.isort]
|
|
105
105
|
profile = "black"
|
|
106
106
|
line_length = 110
|
|
107
|
+
known_first_party = ["lsst"]
|
|
107
108
|
|
|
108
109
|
[tool.lsst_versions]
|
|
109
110
|
write_to = "python/lsst/ctrl/bps/panda/version.py"
|
|
@@ -154,17 +155,27 @@ select = [
|
|
|
154
155
|
"D", # pydocstyle
|
|
155
156
|
"UP", # pyupgrade
|
|
156
157
|
"C4",
|
|
158
|
+
"I", # isort
|
|
159
|
+
"RUF022", # sort __all__
|
|
160
|
+
"B", # bugbear
|
|
157
161
|
]
|
|
158
162
|
extend-select = [
|
|
159
163
|
"RUF100", # Warn about unused noqa
|
|
160
164
|
]
|
|
161
165
|
|
|
166
|
+
[tool.ruff.lint.isort]
|
|
167
|
+
known-first-party = ["lsst"]
|
|
168
|
+
|
|
162
169
|
[tool.ruff.lint.pycodestyle]
|
|
163
170
|
max-doc-length = 79
|
|
164
171
|
|
|
165
172
|
[tool.ruff.lint.pydocstyle]
|
|
166
173
|
convention = "numpy"
|
|
167
174
|
|
|
175
|
+
[tool.ruff.format]
|
|
176
|
+
docstring-code-format = true
|
|
177
|
+
docstring-code-line-length = 79
|
|
178
|
+
|
|
168
179
|
[tool.numpydoc_validation]
|
|
169
180
|
checks = [
|
|
170
181
|
"all", # All except the rules listed below.
|
|
@@ -25,6 +25,6 @@
|
|
|
25
25
|
# You should have received a copy of the GNU General Public License
|
|
26
26
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
27
27
|
|
|
28
|
-
__all__ = ["clean", "reset", "status"]
|
|
28
|
+
__all__ = ["clean", "reset", "refresh", "status"]
|
|
29
29
|
|
|
30
|
-
from .panda_auth_commands import clean, reset, status
|
|
30
|
+
from .panda_auth_commands import clean, reset, refresh, status
|
|
@@ -27,16 +27,23 @@
|
|
|
27
27
|
"""Subcommand definitions for the PanDA auth commands."""
|
|
28
28
|
|
|
29
29
|
__all__ = [
|
|
30
|
-
"status",
|
|
31
|
-
"reset",
|
|
32
30
|
"clean",
|
|
31
|
+
"refresh",
|
|
32
|
+
"reset",
|
|
33
|
+
"status",
|
|
33
34
|
]
|
|
34
35
|
|
|
35
36
|
|
|
36
37
|
import click
|
|
38
|
+
|
|
37
39
|
from lsst.daf.butler.cli.utils import MWCommand
|
|
38
40
|
|
|
39
|
-
from ...panda_auth_drivers import
|
|
41
|
+
from ...panda_auth_drivers import (
|
|
42
|
+
panda_auth_clean_driver,
|
|
43
|
+
panda_auth_refresh_driver,
|
|
44
|
+
panda_auth_reset_driver,
|
|
45
|
+
panda_auth_status_driver,
|
|
46
|
+
)
|
|
40
47
|
|
|
41
48
|
|
|
42
49
|
class PandaAuthCommand(MWCommand):
|
|
@@ -61,3 +68,13 @@ def reset(*args, **kwargs):
|
|
|
61
68
|
def clean(*args, **kwargs):
|
|
62
69
|
"""Clean up token and token cache files."""
|
|
63
70
|
panda_auth_clean_driver(*args, **kwargs)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@click.command(cls=PandaAuthCommand)
|
|
74
|
+
@click.option("--days", default=4, help="The earlist remaining days to refresh the token.")
|
|
75
|
+
@click.option("--verbose", is_flag=True, help="Enable verbose output")
|
|
76
|
+
def refresh(*args, **kwargs):
|
|
77
|
+
"""Refresh auth tocken."""
|
|
78
|
+
days = kwargs.get("days", 4)
|
|
79
|
+
verbose = kwargs.get("verbose", False)
|
|
80
|
+
panda_auth_refresh_driver(days, verbose)
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
# You should have received a copy of the GNU General Public License
|
|
26
26
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
27
27
|
import click
|
|
28
|
+
|
|
28
29
|
from lsst.daf.butler.cli.butler import LoaderCLI
|
|
29
30
|
from lsst.daf.butler.cli.opt import (
|
|
30
31
|
log_file_option,
|
|
@@ -6,6 +6,7 @@ plugin -> PanDA -> Edge node cluster management
|
|
|
6
6
|
of the BPS but a part of the payload wrapper.
|
|
7
7
|
It decodes the hexified command line.
|
|
8
8
|
"""
|
|
9
|
+
|
|
9
10
|
# import base64
|
|
10
11
|
import datetime
|
|
11
12
|
import logging
|
|
@@ -14,8 +15,13 @@ import sys
|
|
|
14
15
|
|
|
15
16
|
from lsst.ctrl.bps.constants import DEFAULT_MEM_FMT, DEFAULT_MEM_UNIT
|
|
16
17
|
from lsst.ctrl.bps.drivers import prepare_driver
|
|
17
|
-
from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS
|
|
18
|
-
from lsst.ctrl.bps.panda.utils import
|
|
18
|
+
from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS, PANDA_DEFAULT_MAX_REQUEST_LENGTH
|
|
19
|
+
from lsst.ctrl.bps.panda.utils import (
|
|
20
|
+
copy_files_for_distribution,
|
|
21
|
+
download_extract_archive,
|
|
22
|
+
get_idds_client,
|
|
23
|
+
get_idds_result,
|
|
24
|
+
)
|
|
19
25
|
from lsst.resources import ResourcePath
|
|
20
26
|
from lsst.utils.timer import time_this
|
|
21
27
|
|
|
@@ -89,6 +95,23 @@ file_distribution_uri = ResourcePath(config["fileDistributionEndPoint"], forceDi
|
|
|
89
95
|
copy_files_for_distribution(bps_workflow.files_to_pre_stage, file_distribution_uri, max_copy_workers)
|
|
90
96
|
|
|
91
97
|
idds_client = get_idds_client(config)
|
|
98
|
+
|
|
99
|
+
# split workflow into steps if the workflow is hughe
|
|
100
|
+
_, max_request_length = config.search("maxRequestLength", opt={"default": PANDA_DEFAULT_MAX_REQUEST_LENGTH})
|
|
101
|
+
workflow_steps = idds_workflow.split_workflow_to_steps(
|
|
102
|
+
request_cache=config["submitPath"], max_request_length=max_request_length
|
|
103
|
+
)
|
|
104
|
+
print(f"workflow_steps: {workflow_steps}")
|
|
105
|
+
for wf_step in workflow_steps:
|
|
106
|
+
ret_step = idds_client.submit(wf_step, username=None, use_dataset_name=False)
|
|
107
|
+
status, result_step, error = get_idds_result(ret_step)
|
|
108
|
+
if status and result_step == 0:
|
|
109
|
+
msg = f"iDDS client manager successfully uploaded workflow step: {wf_step.step_name}"
|
|
110
|
+
print(msg)
|
|
111
|
+
else:
|
|
112
|
+
msg = f"iDDS client manager failed to submit workflow step {wf_step.step_name}: {ret_step}"
|
|
113
|
+
raise RuntimeError(msg)
|
|
114
|
+
|
|
92
115
|
ret = idds_client.update_build_request(request_id, signature, idds_workflow)
|
|
93
116
|
print(f"update_build_request returns: {ret}")
|
|
94
117
|
sys.exit(ret[0])
|
|
@@ -34,6 +34,7 @@ plugin -> PanDA -> Edge node cluster management
|
|
|
34
34
|
of the BPS but a part of the payload wrapper.
|
|
35
35
|
It decodes the hexified command line.
|
|
36
36
|
"""
|
|
37
|
+
|
|
37
38
|
import binascii
|
|
38
39
|
import json
|
|
39
40
|
import os
|
|
@@ -183,6 +184,8 @@ def replace_event_file(params, files):
|
|
|
183
184
|
Example params:
|
|
184
185
|
isr:eventservice_90^10+somethingelse. This part
|
|
185
186
|
'isr:eventservice_90^10' is the EventService parameter.
|
|
187
|
+
isr:orderIdMap_10. This part is using order_id map file. But it
|
|
188
|
+
is not EventService.
|
|
186
189
|
The format for the EventService parameter for LSST is
|
|
187
190
|
'label:eventservice_<baseid>^<localid>'. The '<localid>' should
|
|
188
191
|
start from 1, which means the first event of the file
|
|
@@ -216,6 +219,7 @@ def replace_event_file(params, files):
|
|
|
216
219
|
"""
|
|
217
220
|
ret_status = True
|
|
218
221
|
with_events = False
|
|
222
|
+
with_order_id_map = False
|
|
219
223
|
files = files.split("+")
|
|
220
224
|
file_map = {}
|
|
221
225
|
for file in files:
|
|
@@ -271,51 +275,116 @@ def replace_event_file(params, files):
|
|
|
271
275
|
ret_status = False
|
|
272
276
|
break
|
|
273
277
|
|
|
274
|
-
params_map[param] = {"
|
|
275
|
-
|
|
278
|
+
params_map[param] = {"order_id": event_index, "order_id_map": order_id_map[label]}
|
|
279
|
+
elif "orderIdMap_" in param:
|
|
280
|
+
with_order_id_map = True
|
|
281
|
+
label, event = param.split(":")
|
|
282
|
+
order_id = event.split("_")[1]
|
|
283
|
+
if not order_id_map:
|
|
284
|
+
print("orderIdMap is enabled but order_id_map file doesn't exist.")
|
|
285
|
+
ret_status = False
|
|
286
|
+
break
|
|
276
287
|
|
|
288
|
+
if label not in order_id_map:
|
|
289
|
+
print(
|
|
290
|
+
f"orderIdMap is enabled but label {label} doesn't in the keys"
|
|
291
|
+
f" of order_id_map {order_id_map.keys()}"
|
|
292
|
+
)
|
|
293
|
+
ret_status = False
|
|
294
|
+
break
|
|
295
|
+
if order_id not in order_id_map[label]:
|
|
296
|
+
print(
|
|
297
|
+
f"orderIdMap is enabled but order_id {order_id} is not"
|
|
298
|
+
f" in order_id_map[{label}] {order_id_map[label].keys()}"
|
|
299
|
+
)
|
|
300
|
+
ret_status = False
|
|
301
|
+
break
|
|
277
302
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
data_params = sys.argv[2]
|
|
281
|
-
cmd_line = replace_environment_vars(cmd_line)
|
|
303
|
+
params_map[param] = {"order_id": order_id, "order_id_map": order_id_map[label]}
|
|
304
|
+
return ret_status, with_events, with_order_id_map, params_map
|
|
282
305
|
|
|
283
|
-
# If EventService is enabled, data_params will only contain event information.
|
|
284
|
-
# So we need to convert the event information to LSST pseudo file names.
|
|
285
|
-
# If EventService is not enabled, this part will not change data_params.
|
|
286
|
-
ret_event_status, with_events, event_params_map = replace_event_file(data_params, sys.argv[4])
|
|
287
|
-
print(f"ret_event_status: {ret_event_status}, with_events: {with_events}")
|
|
288
|
-
if not ret_event_status:
|
|
289
|
-
print("failed to map EventService parameters to original LSST pseudo file names")
|
|
290
|
-
exit_code = 1
|
|
291
|
-
sys.exit(exit_code)
|
|
292
306
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
print(f"replacing event {event_param} with event_index {event_index} to: {pseudo_file_name}")
|
|
297
|
-
cmd_line = cmd_line.replace(event_param, pseudo_file_name)
|
|
298
|
-
data_params = data_params.replace(event_param, pseudo_file_name)
|
|
307
|
+
def use_map_file(input_file):
|
|
308
|
+
"""Check whether the input file needs to be replaced
|
|
309
|
+
because enableQnodeMap is enabled.
|
|
299
310
|
|
|
300
|
-
|
|
311
|
+
Parameters
|
|
312
|
+
----------
|
|
313
|
+
input_file : `str`
|
|
314
|
+
Input file either a pseudo file or job name.
|
|
301
315
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
(
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
316
|
+
Returns
|
|
317
|
+
-------
|
|
318
|
+
use_qnode_map: `bool`
|
|
319
|
+
Whether qnode_map is used. There is a placeholder 'PH'
|
|
320
|
+
when enableQnodeMap is true.
|
|
321
|
+
"""
|
|
322
|
+
parts = input_file.split(":")
|
|
323
|
+
use_qnode_map = len(parts) == 2 and parts[0] == "PH"
|
|
324
|
+
return use_qnode_map
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
if __name__ == "__main__":
|
|
328
|
+
deliver_input_files(sys.argv[3], sys.argv[4], sys.argv[5])
|
|
329
|
+
cmd_line = str(binascii.unhexlify(sys.argv[1]).decode())
|
|
330
|
+
data_params = sys.argv[2]
|
|
331
|
+
cmd_line = replace_environment_vars(cmd_line)
|
|
332
|
+
|
|
333
|
+
print(f"cmd_line: {cmd_line}")
|
|
334
|
+
print(f"data_params: {data_params}")
|
|
335
|
+
|
|
336
|
+
# If EventService is enabled, data_params will only contain
|
|
337
|
+
# event information. So we need to convert the event information
|
|
338
|
+
# to LSST pseudo file names. If EventService is not enabled,
|
|
339
|
+
# this part will not change data_params.
|
|
340
|
+
ret_rep = replace_event_file(data_params, sys.argv[4])
|
|
341
|
+
ret_event_status, with_events, with_order_id_map, event_params_map = ret_rep
|
|
342
|
+
print(
|
|
343
|
+
f"ret_event_status: {ret_event_status}, "
|
|
344
|
+
f"with_events: {with_events} "
|
|
345
|
+
f"with_order_id_map: {with_order_id_map}"
|
|
346
|
+
)
|
|
347
|
+
if not ret_event_status:
|
|
348
|
+
print("failed to map EventService/orderIdMap parameters to original LSST pseudo file names")
|
|
349
|
+
exit_code = 1
|
|
350
|
+
sys.exit(exit_code)
|
|
351
|
+
|
|
352
|
+
for event_param in event_params_map:
|
|
353
|
+
order_id = event_params_map[event_param]["order_id"]
|
|
354
|
+
pseudo_file_name = event_params_map[event_param]["order_id_map"][order_id]
|
|
355
|
+
print(f"replacing event {event_param} with order_id {order_id} to: {pseudo_file_name}")
|
|
356
|
+
cmd_line = cmd_line.replace(event_param, pseudo_file_name)
|
|
357
|
+
data_params = data_params.replace(event_param, pseudo_file_name)
|
|
358
|
+
|
|
359
|
+
# If job name map is enabled, data_params will only contain order_id
|
|
360
|
+
# information. Here we will convert order_id information to LSST pseudo
|
|
361
|
+
# file names.
|
|
362
|
+
|
|
363
|
+
data_params = data_params.split("+")
|
|
364
|
+
|
|
365
|
+
"""Replace the pipetask command line placeholders
|
|
366
|
+
with actual data provided in the script call
|
|
367
|
+
in form placeholder1:file1+placeholder2:file2:...
|
|
368
|
+
"""
|
|
369
|
+
cmd_line = replace_files_placeholders(cmd_line, sys.argv[4])
|
|
370
|
+
|
|
371
|
+
jobname = data_params[0]
|
|
372
|
+
if use_map_file(jobname):
|
|
373
|
+
with open("qnode_map.json", encoding="utf-8") as f:
|
|
374
|
+
qnode_map = json.load(f)
|
|
375
|
+
data_params = qnode_map[jobname].split("+")
|
|
376
|
+
|
|
377
|
+
for key_value_pair in data_params[1:]:
|
|
378
|
+
(key, value) = key_value_pair.split(":")
|
|
379
|
+
cmd_line = cmd_line.replace("{" + key + "}", value)
|
|
380
|
+
|
|
381
|
+
print("executable command line:")
|
|
382
|
+
print(cmd_line)
|
|
383
|
+
|
|
384
|
+
exit_status = os.system(cmd_line)
|
|
385
|
+
exit_code = 1
|
|
386
|
+
if os.WIFSIGNALED(exit_status):
|
|
387
|
+
exit_code = os.WTERMSIG(exit_status) + 128
|
|
388
|
+
elif os.WIFEXITED(exit_status):
|
|
389
|
+
exit_code = os.WEXITSTATUS(exit_status)
|
|
390
|
+
sys.exit(exit_code)
|
|
@@ -31,9 +31,9 @@ Driver functions ensure that ensure all setup work is done before running
|
|
|
31
31
|
the subcommand method.
|
|
32
32
|
"""
|
|
33
33
|
|
|
34
|
-
|
|
35
34
|
__all__ = [
|
|
36
35
|
"panda_auth_clean_driver",
|
|
36
|
+
"panda_auth_refresh_driver",
|
|
37
37
|
"panda_auth_reset_driver",
|
|
38
38
|
"panda_auth_status_driver",
|
|
39
39
|
]
|
|
@@ -42,7 +42,19 @@ __all__ = [
|
|
|
42
42
|
import logging
|
|
43
43
|
from datetime import datetime
|
|
44
44
|
|
|
45
|
-
from .
|
|
45
|
+
from lsst.ctrl.bps.panda.panda_exceptions import (
|
|
46
|
+
PandaAuthError,
|
|
47
|
+
TokenExpiredError,
|
|
48
|
+
TokenNotFoundError,
|
|
49
|
+
TokenTooEarlyError,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
from .panda_auth_utils import (
|
|
53
|
+
panda_auth_clean,
|
|
54
|
+
panda_auth_refresh,
|
|
55
|
+
panda_auth_status,
|
|
56
|
+
panda_auth_update,
|
|
57
|
+
)
|
|
46
58
|
|
|
47
59
|
_LOG = logging.getLogger(__name__)
|
|
48
60
|
|
|
@@ -57,6 +69,20 @@ def panda_auth_reset_driver():
|
|
|
57
69
|
panda_auth_update(None, True)
|
|
58
70
|
|
|
59
71
|
|
|
72
|
+
def panda_auth_refresh_driver(days, verbose):
|
|
73
|
+
"""Refresh auth token."""
|
|
74
|
+
try:
|
|
75
|
+
panda_auth_refresh(days, verbose)
|
|
76
|
+
except TokenNotFoundError as e:
|
|
77
|
+
print(f"[ERROR] {e}")
|
|
78
|
+
except TokenExpiredError as e:
|
|
79
|
+
print(f"[ERROR] {e}")
|
|
80
|
+
except TokenTooEarlyError as e:
|
|
81
|
+
print(f"[INFO] {e}")
|
|
82
|
+
except PandaAuthError as e:
|
|
83
|
+
print(f"[FAIL] {e}")
|
|
84
|
+
|
|
85
|
+
|
|
60
86
|
def panda_auth_status_driver():
|
|
61
87
|
"""Gather information about a token if it exists."""
|
|
62
88
|
status = panda_auth_status()
|
|
@@ -27,23 +27,35 @@
|
|
|
27
27
|
|
|
28
28
|
"""Functions for each panda_auth subcommand."""
|
|
29
29
|
|
|
30
|
-
|
|
31
30
|
__all__ = [
|
|
32
31
|
"panda_auth_clean",
|
|
33
32
|
"panda_auth_expiration",
|
|
33
|
+
"panda_auth_refresh",
|
|
34
34
|
"panda_auth_setup",
|
|
35
35
|
"panda_auth_status",
|
|
36
36
|
"panda_auth_update",
|
|
37
37
|
]
|
|
38
38
|
|
|
39
39
|
|
|
40
|
+
import base64
|
|
41
|
+
import json
|
|
40
42
|
import logging
|
|
41
43
|
import os
|
|
44
|
+
from datetime import UTC, datetime, timedelta
|
|
42
45
|
|
|
43
46
|
import idds.common.utils as idds_utils
|
|
44
47
|
import pandaclient.idds_api
|
|
45
48
|
from pandaclient.openidc_utils import OpenIdConnect_Utils
|
|
46
49
|
|
|
50
|
+
from lsst.ctrl.bps.panda.panda_exceptions import (
|
|
51
|
+
AuthConfigError,
|
|
52
|
+
PandaAuthError,
|
|
53
|
+
TokenExpiredError,
|
|
54
|
+
TokenNotFoundError,
|
|
55
|
+
TokenRefreshError,
|
|
56
|
+
TokenTooEarlyError,
|
|
57
|
+
)
|
|
58
|
+
|
|
47
59
|
_LOG = logging.getLogger(__name__)
|
|
48
60
|
|
|
49
61
|
|
|
@@ -152,3 +164,87 @@ def panda_auth_update(idds_server=None, reset=False):
|
|
|
152
164
|
# idds server given. So for now, check result string for keywords.
|
|
153
165
|
if "request_id" not in ret[1][-1] or "status" not in ret[1][-1]:
|
|
154
166
|
raise RuntimeError(f"Error contacting PanDA service: {ret}")
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def panda_auth_refresh(days=4, verbose=False):
|
|
170
|
+
"""
|
|
171
|
+
Refresh the current valid IAM OpenID authentication token.
|
|
172
|
+
|
|
173
|
+
This function checks the expiration time of the existing token stored
|
|
174
|
+
in the local token file and attempts to refresh it if it is close to
|
|
175
|
+
expiring (within a specified number of days).
|
|
176
|
+
|
|
177
|
+
Parameters
|
|
178
|
+
----------
|
|
179
|
+
days : `int`, optional
|
|
180
|
+
The minimum number of days before token expiration to trigger a
|
|
181
|
+
refresh. If the token expires in more than this number of days,
|
|
182
|
+
the refresh is skipped. Default is 4.
|
|
183
|
+
verbose : `bool`, optional
|
|
184
|
+
If True, enables verbose output for debugging or logging.
|
|
185
|
+
Default is False.
|
|
186
|
+
|
|
187
|
+
Returns
|
|
188
|
+
-------
|
|
189
|
+
status: `dict`
|
|
190
|
+
A dictionary containing the refreshed token status
|
|
191
|
+
"""
|
|
192
|
+
panda_url = os.environ.get("PANDA_URL")
|
|
193
|
+
panda_auth_vo = os.environ.get("PANDA_AUTH_VO")
|
|
194
|
+
|
|
195
|
+
if not panda_url or not panda_auth_vo:
|
|
196
|
+
raise PandaAuthError("Missing required environment variables: PANDA_URL or PANDA_AUTH_VO")
|
|
197
|
+
|
|
198
|
+
url_prefix = panda_url.split("/server", 1)[0]
|
|
199
|
+
auth_url = f"{url_prefix}/auth/{panda_auth_vo}_auth_config.json"
|
|
200
|
+
open_id = OpenIdConnect_Utils(auth_url, log_stream=_LOG, verbose=verbose)
|
|
201
|
+
|
|
202
|
+
token_file = open_id.get_token_path()
|
|
203
|
+
if not os.path.exists(token_file):
|
|
204
|
+
raise TokenNotFoundError("Cannot find token file. Use 'panda_auth reset' to obtain a new token.")
|
|
205
|
+
|
|
206
|
+
with open(token_file) as f:
|
|
207
|
+
data = json.load(f)
|
|
208
|
+
enc = data["id_token"].split(".")[1]
|
|
209
|
+
enc += "=" * (-len(enc) % 4)
|
|
210
|
+
dec = json.loads(base64.urlsafe_b64decode(enc.encode()))
|
|
211
|
+
exp_time = datetime.fromtimestamp(dec["exp"], tz=UTC)
|
|
212
|
+
delta = exp_time - datetime.now(UTC)
|
|
213
|
+
minutes = delta.total_seconds() / 60
|
|
214
|
+
print(f"Token will expire in {minutes} minutes.")
|
|
215
|
+
print(f"Token expiration time : {exp_time.strftime('%Y-%m-%d %H:%M:%S')} UTC")
|
|
216
|
+
if delta < timedelta(minutes=0):
|
|
217
|
+
raise TokenExpiredError("Token already expired. Cannot refresh.")
|
|
218
|
+
elif delta > timedelta(days=days):
|
|
219
|
+
raise TokenTooEarlyError(
|
|
220
|
+
f"Too early to refresh. More than {days} day(s) until expiration.\n"
|
|
221
|
+
f"Use '--days' option to adjust threshold, e.g.:\n"
|
|
222
|
+
f" panda_auth refresh --days 10"
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
refresh_token_string = data["refresh_token"]
|
|
226
|
+
|
|
227
|
+
s, auth_config = open_id.fetch_page(open_id.auth_config_url)
|
|
228
|
+
if not s:
|
|
229
|
+
raise AuthConfigError("Failed to get Auth configuration.")
|
|
230
|
+
|
|
231
|
+
s, endpoint_config = open_id.fetch_page(auth_config["oidc_config_url"])
|
|
232
|
+
if not s:
|
|
233
|
+
raise AuthConfigError("Failed to get endpoint configuration.")
|
|
234
|
+
|
|
235
|
+
s, o = open_id.refresh_token(
|
|
236
|
+
endpoint_config["token_endpoint"],
|
|
237
|
+
auth_config["client_id"],
|
|
238
|
+
auth_config["client_secret"],
|
|
239
|
+
refresh_token_string,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
if not s:
|
|
243
|
+
raise TokenRefreshError("Failed to refresh token.")
|
|
244
|
+
|
|
245
|
+
status = panda_auth_status()
|
|
246
|
+
if status:
|
|
247
|
+
exp_time = datetime.fromtimestamp(status["exp"], tz=UTC)
|
|
248
|
+
print(f"{'New expiration time:':23} {exp_time.strftime('%Y-%m-%d %H:%M:%S')} UTC")
|
|
249
|
+
print("Success to refresh token")
|
|
250
|
+
return status
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
class PandaAuthError(Exception):
|
|
2
|
+
"""Base class for authentication errors."""
|
|
3
|
+
|
|
4
|
+
pass
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TokenNotFoundError(PandaAuthError):
|
|
8
|
+
"""Raised when the token file is missing."""
|
|
9
|
+
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TokenExpiredError(PandaAuthError):
|
|
14
|
+
"""Raised when the token has already expired."""
|
|
15
|
+
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class TokenTooEarlyError(PandaAuthError):
|
|
20
|
+
"""Raised when attempting to refresh too early."""
|
|
21
|
+
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class AuthConfigError(PandaAuthError):
|
|
26
|
+
"""Raised when fetching the auth or endpoint configuration fails."""
|
|
27
|
+
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class TokenRefreshError(PandaAuthError):
|
|
32
|
+
"""Raised when token refresh fails."""
|
|
33
|
+
|
|
34
|
+
pass
|