lsst-ctrl-bps-panda 28.2025.600__tar.gz → 29.2025.4700__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {lsst_ctrl_bps_panda-28.2025.600/python/lsst_ctrl_bps_panda.egg-info → lsst_ctrl_bps_panda-29.2025.4700}/PKG-INFO +5 -4
  2. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/pyproject.toml +14 -3
  3. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cli/cmd/__init__.py +2 -2
  4. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cli/cmd/panda_auth_commands.py +20 -3
  5. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cli/panda_auth.py +1 -0
  6. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/constants.py +2 -0
  7. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/edgenode/build_cmd_line_decoder.py +25 -2
  8. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/edgenode/cmd_line_decoder.py +111 -42
  9. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/panda_auth_drivers.py +28 -2
  10. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/panda_auth_utils.py +97 -1
  11. lsst_ctrl_bps_panda-29.2025.4700/python/lsst/ctrl/bps/panda/panda_exceptions.py +34 -0
  12. lsst_ctrl_bps_panda-29.2025.4700/python/lsst/ctrl/bps/panda/panda_service.py +513 -0
  13. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/utils.py +283 -47
  14. lsst_ctrl_bps_panda-29.2025.4700/python/lsst/ctrl/bps/panda/version.py +2 -0
  15. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700/python/lsst_ctrl_bps_panda.egg-info}/PKG-INFO +5 -4
  16. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst_ctrl_bps_panda.egg-info/SOURCES.txt +3 -0
  17. lsst_ctrl_bps_panda-29.2025.4700/tests/test_cmd_line_decoder.py +52 -0
  18. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/tests/test_cmd_line_embedder.py +1 -0
  19. lsst_ctrl_bps_panda-29.2025.4700/tests/test_panda_auth_utils.py +151 -0
  20. lsst_ctrl_bps_panda-29.2025.4700/tests/test_panda_service.py +194 -0
  21. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/tests/test_utils.py +2 -1
  22. lsst_ctrl_bps_panda-28.2025.600/python/lsst/ctrl/bps/panda/panda_service.py +0 -400
  23. lsst_ctrl_bps_panda-28.2025.600/python/lsst/ctrl/bps/panda/version.py +0 -2
  24. lsst_ctrl_bps_panda-28.2025.600/tests/test_panda_auth_utils.py +0 -63
  25. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/COPYRIGHT +0 -0
  26. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/LICENSE +0 -0
  27. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/README.rst +0 -0
  28. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/bsd_license.txt +0 -0
  29. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/gpl-v3.0.txt +0 -0
  30. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/__init__.py +0 -0
  31. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cli/__init__.py +0 -0
  32. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cmd_line_embedder.py +0 -0
  33. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/conf_example/example_panda_SLAC.yaml +0 -0
  34. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/conf_example/pipelines_check_idf.yaml +0 -0
  35. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/conf_example/test_idf.yaml +0 -0
  36. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/conf_example/test_sdf.yaml +0 -0
  37. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/conf_example/test_usdf.yaml +0 -0
  38. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/edgenode/__init__.py +0 -0
  39. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst_ctrl_bps_panda.egg-info/dependency_links.txt +0 -0
  40. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst_ctrl_bps_panda.egg-info/requires.txt +0 -0
  41. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst_ctrl_bps_panda.egg-info/top_level.txt +0 -0
  42. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst_ctrl_bps_panda.egg-info/zip-safe +0 -0
  43. {lsst_ctrl_bps_panda-28.2025.600 → lsst_ctrl_bps_panda-29.2025.4700}/setup.cfg +0 -0
@@ -1,18 +1,18 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: lsst-ctrl-bps-panda
3
- Version: 28.2025.600
3
+ Version: 29.2025.4700
4
4
  Summary: PanDA plugin for lsst-ctrl-bps.
5
5
  Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
6
- License: BSD 3-Clause License
6
+ License-Expression: BSD-3-Clause OR GPL-3.0-or-later
7
7
  Project-URL: Homepage, https://github.com/lsst/ctrl_bps_panda
8
8
  Keywords: lsst
9
9
  Classifier: Intended Audience :: Science/Research
10
- Classifier: License :: OSI Approved :: BSD License
11
10
  Classifier: Operating System :: OS Independent
12
11
  Classifier: Programming Language :: Python :: 3
13
12
  Classifier: Programming Language :: Python :: 3.11
14
13
  Classifier: Programming Language :: Python :: 3.12
15
14
  Classifier: Programming Language :: Python :: 3.13
15
+ Classifier: Programming Language :: Python :: 3.14
16
16
  Classifier: Topic :: Scientific/Engineering :: Astronomy
17
17
  Requires-Python: >=3.11.0
18
18
  Description-Content-Type: text/x-rst
@@ -34,6 +34,7 @@ Requires-Dist: lsst-utils
34
34
  Provides-Extra: test
35
35
  Requires-Dist: pytest>=3.2; extra == "test"
36
36
  Requires-Dist: pytest-openfiles>=0.5.0; extra == "test"
37
+ Dynamic: license-file
37
38
 
38
39
  ##############
39
40
  ctrl_bps_panda
@@ -6,19 +6,20 @@ build-backend = "setuptools.build_meta"
6
6
  name = "lsst-ctrl-bps-panda"
7
7
  requires-python = ">=3.11.0"
8
8
  description = "PanDA plugin for lsst-ctrl-bps."
9
- license = {text = "BSD 3-Clause License"}
9
+ license = "BSD-3-Clause OR GPL-3.0-or-later"
10
+ license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
10
11
  readme = "README.rst"
11
12
  authors = [
12
13
  {name="Rubin Observatory Data Management", email="dm-admin@lists.lsst.org"},
13
14
  ]
14
15
  classifiers = [
15
16
  "Intended Audience :: Science/Research",
16
- "License :: OSI Approved :: BSD License",
17
17
  "Operating System :: OS Independent",
18
18
  "Programming Language :: Python :: 3",
19
19
  "Programming Language :: Python :: 3.11",
20
20
  "Programming Language :: Python :: 3.12",
21
21
  "Programming Language :: Python :: 3.13",
22
+ "Programming Language :: Python :: 3.14",
22
23
  "Topic :: Scientific/Engineering :: Astronomy",
23
24
  ]
24
25
  keywords = ["lsst"]
@@ -51,7 +52,6 @@ where = ["python"]
51
52
 
52
53
  [tool.setuptools]
53
54
  zip-safe = true
54
- license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
55
55
 
56
56
  [tool.setuptools.package-data]
57
57
  "lsst.ctrl.bps.panda" = ["conf_example/*.yaml"]
@@ -104,6 +104,7 @@ target-version = ["py311"]
104
104
  [tool.isort]
105
105
  profile = "black"
106
106
  line_length = 110
107
+ known_first_party = ["lsst"]
107
108
 
108
109
  [tool.lsst_versions]
109
110
  write_to = "python/lsst/ctrl/bps/panda/version.py"
@@ -154,17 +155,27 @@ select = [
154
155
  "D", # pydocstyle
155
156
  "UP", # pyupgrade
156
157
  "C4",
158
+ "I", # isort
159
+ "RUF022", # sort __all__
160
+ "B", # bugbear
157
161
  ]
158
162
  extend-select = [
159
163
  "RUF100", # Warn about unused noqa
160
164
  ]
161
165
 
166
+ [tool.ruff.lint.isort]
167
+ known-first-party = ["lsst"]
168
+
162
169
  [tool.ruff.lint.pycodestyle]
163
170
  max-doc-length = 79
164
171
 
165
172
  [tool.ruff.lint.pydocstyle]
166
173
  convention = "numpy"
167
174
 
175
+ [tool.ruff.format]
176
+ docstring-code-format = true
177
+ docstring-code-line-length = 79
178
+
168
179
  [tool.numpydoc_validation]
169
180
  checks = [
170
181
  "all", # All except the rules listed below.
@@ -25,6 +25,6 @@
25
25
  # You should have received a copy of the GNU General Public License
26
26
  # along with this program. If not, see <https://www.gnu.org/licenses/>.
27
27
 
28
- __all__ = ["clean", "reset", "status"]
28
+ __all__ = ["clean", "reset", "refresh", "status"]
29
29
 
30
- from .panda_auth_commands import clean, reset, status
30
+ from .panda_auth_commands import clean, reset, refresh, status
@@ -27,16 +27,23 @@
27
27
  """Subcommand definitions for the PanDA auth commands."""
28
28
 
29
29
  __all__ = [
30
- "status",
31
- "reset",
32
30
  "clean",
31
+ "refresh",
32
+ "reset",
33
+ "status",
33
34
  ]
34
35
 
35
36
 
36
37
  import click
38
+
37
39
  from lsst.daf.butler.cli.utils import MWCommand
38
40
 
39
- from ...panda_auth_drivers import panda_auth_clean_driver, panda_auth_reset_driver, panda_auth_status_driver
41
+ from ...panda_auth_drivers import (
42
+ panda_auth_clean_driver,
43
+ panda_auth_refresh_driver,
44
+ panda_auth_reset_driver,
45
+ panda_auth_status_driver,
46
+ )
40
47
 
41
48
 
42
49
  class PandaAuthCommand(MWCommand):
@@ -61,3 +68,13 @@ def reset(*args, **kwargs):
61
68
  def clean(*args, **kwargs):
62
69
  """Clean up token and token cache files."""
63
70
  panda_auth_clean_driver(*args, **kwargs)
71
+
72
+
73
+ @click.command(cls=PandaAuthCommand)
74
+ @click.option("--days", default=4, help="The earlist remaining days to refresh the token.")
75
+ @click.option("--verbose", is_flag=True, help="Enable verbose output")
76
+ def refresh(*args, **kwargs):
77
+ """Refresh auth tocken."""
78
+ days = kwargs.get("days", 4)
79
+ verbose = kwargs.get("verbose", False)
80
+ panda_auth_refresh_driver(days, verbose)
@@ -25,6 +25,7 @@
25
25
  # You should have received a copy of the GNU General Public License
26
26
  # along with this program. If not, see <https://www.gnu.org/licenses/>.
27
27
  import click
28
+
28
29
  from lsst.daf.butler.cli.butler import LoaderCLI
29
30
  from lsst.daf.butler.cli.opt import (
30
31
  log_file_option,
@@ -41,3 +41,5 @@ PANDA_DEFAULT_CLOUD = "US"
41
41
  PANDA_DEFAULT_MAX_COPY_WORKERS = 10
42
42
  PANDA_DEFAULT_MAX_PAYLOADS_PER_PANDA_JOB = 10
43
43
  PANDA_DEFAULT_ORDER_ID_MAP_FILE = "panda_order_id_map.json"
44
+ PANDA_DEFAULT_NAME_LENGTH = 4000
45
+ PANDA_DEFAULT_MAX_REQUEST_LENGTH = 400000000 # 400M
@@ -6,6 +6,7 @@ plugin -> PanDA -> Edge node cluster management
6
6
  of the BPS but a part of the payload wrapper.
7
7
  It decodes the hexified command line.
8
8
  """
9
+
9
10
  # import base64
10
11
  import datetime
11
12
  import logging
@@ -14,8 +15,13 @@ import sys
14
15
 
15
16
  from lsst.ctrl.bps.constants import DEFAULT_MEM_FMT, DEFAULT_MEM_UNIT
16
17
  from lsst.ctrl.bps.drivers import prepare_driver
17
- from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS
18
- from lsst.ctrl.bps.panda.utils import copy_files_for_distribution, download_extract_archive, get_idds_client
18
+ from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS, PANDA_DEFAULT_MAX_REQUEST_LENGTH
19
+ from lsst.ctrl.bps.panda.utils import (
20
+ copy_files_for_distribution,
21
+ download_extract_archive,
22
+ get_idds_client,
23
+ get_idds_result,
24
+ )
19
25
  from lsst.resources import ResourcePath
20
26
  from lsst.utils.timer import time_this
21
27
 
@@ -89,6 +95,23 @@ file_distribution_uri = ResourcePath(config["fileDistributionEndPoint"], forceDi
89
95
  copy_files_for_distribution(bps_workflow.files_to_pre_stage, file_distribution_uri, max_copy_workers)
90
96
 
91
97
  idds_client = get_idds_client(config)
98
+
99
+ # split workflow into steps if the workflow is hughe
100
+ _, max_request_length = config.search("maxRequestLength", opt={"default": PANDA_DEFAULT_MAX_REQUEST_LENGTH})
101
+ workflow_steps = idds_workflow.split_workflow_to_steps(
102
+ request_cache=config["submitPath"], max_request_length=max_request_length
103
+ )
104
+ print(f"workflow_steps: {workflow_steps}")
105
+ for wf_step in workflow_steps:
106
+ ret_step = idds_client.submit(wf_step, username=None, use_dataset_name=False)
107
+ status, result_step, error = get_idds_result(ret_step)
108
+ if status and result_step == 0:
109
+ msg = f"iDDS client manager successfully uploaded workflow step: {wf_step.step_name}"
110
+ print(msg)
111
+ else:
112
+ msg = f"iDDS client manager failed to submit workflow step {wf_step.step_name}: {ret_step}"
113
+ raise RuntimeError(msg)
114
+
92
115
  ret = idds_client.update_build_request(request_id, signature, idds_workflow)
93
116
  print(f"update_build_request returns: {ret}")
94
117
  sys.exit(ret[0])
@@ -34,6 +34,7 @@ plugin -> PanDA -> Edge node cluster management
34
34
  of the BPS but a part of the payload wrapper.
35
35
  It decodes the hexified command line.
36
36
  """
37
+
37
38
  import binascii
38
39
  import json
39
40
  import os
@@ -183,6 +184,8 @@ def replace_event_file(params, files):
183
184
  Example params:
184
185
  isr:eventservice_90^10+somethingelse. This part
185
186
  'isr:eventservice_90^10' is the EventService parameter.
187
+ isr:orderIdMap_10. This part is using order_id map file. But it
188
+ is not EventService.
186
189
  The format for the EventService parameter for LSST is
187
190
  'label:eventservice_<baseid>^<localid>'. The '<localid>' should
188
191
  start from 1, which means the first event of the file
@@ -216,6 +219,7 @@ def replace_event_file(params, files):
216
219
  """
217
220
  ret_status = True
218
221
  with_events = False
222
+ with_order_id_map = False
219
223
  files = files.split("+")
220
224
  file_map = {}
221
225
  for file in files:
@@ -271,51 +275,116 @@ def replace_event_file(params, files):
271
275
  ret_status = False
272
276
  break
273
277
 
274
- params_map[param] = {"event_index": event_index, "order_id_map": order_id_map[label]}
275
- return ret_status, with_events, params_map
278
+ params_map[param] = {"order_id": event_index, "order_id_map": order_id_map[label]}
279
+ elif "orderIdMap_" in param:
280
+ with_order_id_map = True
281
+ label, event = param.split(":")
282
+ order_id = event.split("_")[1]
283
+ if not order_id_map:
284
+ print("orderIdMap is enabled but order_id_map file doesn't exist.")
285
+ ret_status = False
286
+ break
276
287
 
288
+ if label not in order_id_map:
289
+ print(
290
+ f"orderIdMap is enabled but label {label} doesn't in the keys"
291
+ f" of order_id_map {order_id_map.keys()}"
292
+ )
293
+ ret_status = False
294
+ break
295
+ if order_id not in order_id_map[label]:
296
+ print(
297
+ f"orderIdMap is enabled but order_id {order_id} is not"
298
+ f" in order_id_map[{label}] {order_id_map[label].keys()}"
299
+ )
300
+ ret_status = False
301
+ break
277
302
 
278
- deliver_input_files(sys.argv[3], sys.argv[4], sys.argv[5])
279
- cmd_line = str(binascii.unhexlify(sys.argv[1]).decode())
280
- data_params = sys.argv[2]
281
- cmd_line = replace_environment_vars(cmd_line)
303
+ params_map[param] = {"order_id": order_id, "order_id_map": order_id_map[label]}
304
+ return ret_status, with_events, with_order_id_map, params_map
282
305
 
283
- # If EventService is enabled, data_params will only contain event information.
284
- # So we need to convert the event information to LSST pseudo file names.
285
- # If EventService is not enabled, this part will not change data_params.
286
- ret_event_status, with_events, event_params_map = replace_event_file(data_params, sys.argv[4])
287
- print(f"ret_event_status: {ret_event_status}, with_events: {with_events}")
288
- if not ret_event_status:
289
- print("failed to map EventService parameters to original LSST pseudo file names")
290
- exit_code = 1
291
- sys.exit(exit_code)
292
306
 
293
- for event_param in event_params_map:
294
- event_index = event_params_map[event_param]["event_index"]
295
- pseudo_file_name = event_params_map[event_param]["order_id_map"][event_index]
296
- print(f"replacing event {event_param} with event_index {event_index} to: {pseudo_file_name}")
297
- cmd_line = cmd_line.replace(event_param, pseudo_file_name)
298
- data_params = data_params.replace(event_param, pseudo_file_name)
307
+ def use_map_file(input_file):
308
+ """Check whether the input file needs to be replaced
309
+ because enableQnodeMap is enabled.
299
310
 
300
- data_params = data_params.split("+")
311
+ Parameters
312
+ ----------
313
+ input_file : `str`
314
+ Input file either a pseudo file or job name.
301
315
 
302
- """Replace the pipetask command line placeholders
303
- with actual data provided in the script call
304
- in form placeholder1:file1+placeholder2:file2:...
305
- """
306
- cmd_line = replace_files_placeholders(cmd_line, sys.argv[4])
307
-
308
- for key_value_pair in data_params[1:]:
309
- (key, value) = key_value_pair.split(":")
310
- cmd_line = cmd_line.replace("{" + key + "}", value)
311
-
312
- print("executable command line:")
313
- print(cmd_line)
314
-
315
- exit_status = os.system(cmd_line)
316
- exit_code = 1
317
- if os.WIFSIGNALED(exit_status):
318
- exit_code = os.WTERMSIG(exit_status) + 128
319
- elif os.WIFEXITED(exit_status):
320
- exit_code = os.WEXITSTATUS(exit_status)
321
- sys.exit(exit_code)
316
+ Returns
317
+ -------
318
+ use_qnode_map: `bool`
319
+ Whether qnode_map is used. There is a placeholder 'PH'
320
+ when enableQnodeMap is true.
321
+ """
322
+ parts = input_file.split(":")
323
+ use_qnode_map = len(parts) == 2 and parts[0] == "PH"
324
+ return use_qnode_map
325
+
326
+
327
+ if __name__ == "__main__":
328
+ deliver_input_files(sys.argv[3], sys.argv[4], sys.argv[5])
329
+ cmd_line = str(binascii.unhexlify(sys.argv[1]).decode())
330
+ data_params = sys.argv[2]
331
+ cmd_line = replace_environment_vars(cmd_line)
332
+
333
+ print(f"cmd_line: {cmd_line}")
334
+ print(f"data_params: {data_params}")
335
+
336
+ # If EventService is enabled, data_params will only contain
337
+ # event information. So we need to convert the event information
338
+ # to LSST pseudo file names. If EventService is not enabled,
339
+ # this part will not change data_params.
340
+ ret_rep = replace_event_file(data_params, sys.argv[4])
341
+ ret_event_status, with_events, with_order_id_map, event_params_map = ret_rep
342
+ print(
343
+ f"ret_event_status: {ret_event_status}, "
344
+ f"with_events: {with_events} "
345
+ f"with_order_id_map: {with_order_id_map}"
346
+ )
347
+ if not ret_event_status:
348
+ print("failed to map EventService/orderIdMap parameters to original LSST pseudo file names")
349
+ exit_code = 1
350
+ sys.exit(exit_code)
351
+
352
+ for event_param in event_params_map:
353
+ order_id = event_params_map[event_param]["order_id"]
354
+ pseudo_file_name = event_params_map[event_param]["order_id_map"][order_id]
355
+ print(f"replacing event {event_param} with order_id {order_id} to: {pseudo_file_name}")
356
+ cmd_line = cmd_line.replace(event_param, pseudo_file_name)
357
+ data_params = data_params.replace(event_param, pseudo_file_name)
358
+
359
+ # If job name map is enabled, data_params will only contain order_id
360
+ # information. Here we will convert order_id information to LSST pseudo
361
+ # file names.
362
+
363
+ data_params = data_params.split("+")
364
+
365
+ """Replace the pipetask command line placeholders
366
+ with actual data provided in the script call
367
+ in form placeholder1:file1+placeholder2:file2:...
368
+ """
369
+ cmd_line = replace_files_placeholders(cmd_line, sys.argv[4])
370
+
371
+ jobname = data_params[0]
372
+ if use_map_file(jobname):
373
+ with open("qnode_map.json", encoding="utf-8") as f:
374
+ qnode_map = json.load(f)
375
+ data_params = qnode_map[jobname].split("+")
376
+
377
+ for key_value_pair in data_params[1:]:
378
+ (key, value) = key_value_pair.split(":")
379
+ cmd_line = cmd_line.replace("{" + key + "}", value)
380
+
381
+ print("executable command line:")
382
+ print(cmd_line)
383
+
384
+ exit_status = os.system(cmd_line)
385
+ exit_code = 1
386
+ if os.WIFSIGNALED(exit_status):
387
+ exit_code = os.WTERMSIG(exit_status) + 128
388
+ elif os.WIFEXITED(exit_status):
389
+ exit_code = os.WEXITSTATUS(exit_status)
390
+ sys.exit(exit_code)
@@ -31,9 +31,9 @@ Driver functions ensure that ensure all setup work is done before running
31
31
  the subcommand method.
32
32
  """
33
33
 
34
-
35
34
  __all__ = [
36
35
  "panda_auth_clean_driver",
36
+ "panda_auth_refresh_driver",
37
37
  "panda_auth_reset_driver",
38
38
  "panda_auth_status_driver",
39
39
  ]
@@ -42,7 +42,19 @@ __all__ = [
42
42
  import logging
43
43
  from datetime import datetime
44
44
 
45
- from .panda_auth_utils import panda_auth_clean, panda_auth_status, panda_auth_update
45
+ from lsst.ctrl.bps.panda.panda_exceptions import (
46
+ PandaAuthError,
47
+ TokenExpiredError,
48
+ TokenNotFoundError,
49
+ TokenTooEarlyError,
50
+ )
51
+
52
+ from .panda_auth_utils import (
53
+ panda_auth_clean,
54
+ panda_auth_refresh,
55
+ panda_auth_status,
56
+ panda_auth_update,
57
+ )
46
58
 
47
59
  _LOG = logging.getLogger(__name__)
48
60
 
@@ -57,6 +69,20 @@ def panda_auth_reset_driver():
57
69
  panda_auth_update(None, True)
58
70
 
59
71
 
72
+ def panda_auth_refresh_driver(days, verbose):
73
+ """Refresh auth token."""
74
+ try:
75
+ panda_auth_refresh(days, verbose)
76
+ except TokenNotFoundError as e:
77
+ print(f"[ERROR] {e}")
78
+ except TokenExpiredError as e:
79
+ print(f"[ERROR] {e}")
80
+ except TokenTooEarlyError as e:
81
+ print(f"[INFO] {e}")
82
+ except PandaAuthError as e:
83
+ print(f"[FAIL] {e}")
84
+
85
+
60
86
  def panda_auth_status_driver():
61
87
  """Gather information about a token if it exists."""
62
88
  status = panda_auth_status()
@@ -27,23 +27,35 @@
27
27
 
28
28
  """Functions for each panda_auth subcommand."""
29
29
 
30
-
31
30
  __all__ = [
32
31
  "panda_auth_clean",
33
32
  "panda_auth_expiration",
33
+ "panda_auth_refresh",
34
34
  "panda_auth_setup",
35
35
  "panda_auth_status",
36
36
  "panda_auth_update",
37
37
  ]
38
38
 
39
39
 
40
+ import base64
41
+ import json
40
42
  import logging
41
43
  import os
44
+ from datetime import UTC, datetime, timedelta
42
45
 
43
46
  import idds.common.utils as idds_utils
44
47
  import pandaclient.idds_api
45
48
  from pandaclient.openidc_utils import OpenIdConnect_Utils
46
49
 
50
+ from lsst.ctrl.bps.panda.panda_exceptions import (
51
+ AuthConfigError,
52
+ PandaAuthError,
53
+ TokenExpiredError,
54
+ TokenNotFoundError,
55
+ TokenRefreshError,
56
+ TokenTooEarlyError,
57
+ )
58
+
47
59
  _LOG = logging.getLogger(__name__)
48
60
 
49
61
 
@@ -152,3 +164,87 @@ def panda_auth_update(idds_server=None, reset=False):
152
164
  # idds server given. So for now, check result string for keywords.
153
165
  if "request_id" not in ret[1][-1] or "status" not in ret[1][-1]:
154
166
  raise RuntimeError(f"Error contacting PanDA service: {ret}")
167
+
168
+
169
+ def panda_auth_refresh(days=4, verbose=False):
170
+ """
171
+ Refresh the current valid IAM OpenID authentication token.
172
+
173
+ This function checks the expiration time of the existing token stored
174
+ in the local token file and attempts to refresh it if it is close to
175
+ expiring (within a specified number of days).
176
+
177
+ Parameters
178
+ ----------
179
+ days : `int`, optional
180
+ The minimum number of days before token expiration to trigger a
181
+ refresh. If the token expires in more than this number of days,
182
+ the refresh is skipped. Default is 4.
183
+ verbose : `bool`, optional
184
+ If True, enables verbose output for debugging or logging.
185
+ Default is False.
186
+
187
+ Returns
188
+ -------
189
+ status: `dict`
190
+ A dictionary containing the refreshed token status
191
+ """
192
+ panda_url = os.environ.get("PANDA_URL")
193
+ panda_auth_vo = os.environ.get("PANDA_AUTH_VO")
194
+
195
+ if not panda_url or not panda_auth_vo:
196
+ raise PandaAuthError("Missing required environment variables: PANDA_URL or PANDA_AUTH_VO")
197
+
198
+ url_prefix = panda_url.split("/server", 1)[0]
199
+ auth_url = f"{url_prefix}/auth/{panda_auth_vo}_auth_config.json"
200
+ open_id = OpenIdConnect_Utils(auth_url, log_stream=_LOG, verbose=verbose)
201
+
202
+ token_file = open_id.get_token_path()
203
+ if not os.path.exists(token_file):
204
+ raise TokenNotFoundError("Cannot find token file. Use 'panda_auth reset' to obtain a new token.")
205
+
206
+ with open(token_file) as f:
207
+ data = json.load(f)
208
+ enc = data["id_token"].split(".")[1]
209
+ enc += "=" * (-len(enc) % 4)
210
+ dec = json.loads(base64.urlsafe_b64decode(enc.encode()))
211
+ exp_time = datetime.fromtimestamp(dec["exp"], tz=UTC)
212
+ delta = exp_time - datetime.now(UTC)
213
+ minutes = delta.total_seconds() / 60
214
+ print(f"Token will expire in {minutes} minutes.")
215
+ print(f"Token expiration time : {exp_time.strftime('%Y-%m-%d %H:%M:%S')} UTC")
216
+ if delta < timedelta(minutes=0):
217
+ raise TokenExpiredError("Token already expired. Cannot refresh.")
218
+ elif delta > timedelta(days=days):
219
+ raise TokenTooEarlyError(
220
+ f"Too early to refresh. More than {days} day(s) until expiration.\n"
221
+ f"Use '--days' option to adjust threshold, e.g.:\n"
222
+ f" panda_auth refresh --days 10"
223
+ )
224
+
225
+ refresh_token_string = data["refresh_token"]
226
+
227
+ s, auth_config = open_id.fetch_page(open_id.auth_config_url)
228
+ if not s:
229
+ raise AuthConfigError("Failed to get Auth configuration.")
230
+
231
+ s, endpoint_config = open_id.fetch_page(auth_config["oidc_config_url"])
232
+ if not s:
233
+ raise AuthConfigError("Failed to get endpoint configuration.")
234
+
235
+ s, o = open_id.refresh_token(
236
+ endpoint_config["token_endpoint"],
237
+ auth_config["client_id"],
238
+ auth_config["client_secret"],
239
+ refresh_token_string,
240
+ )
241
+
242
+ if not s:
243
+ raise TokenRefreshError("Failed to refresh token.")
244
+
245
+ status = panda_auth_status()
246
+ if status:
247
+ exp_time = datetime.fromtimestamp(status["exp"], tz=UTC)
248
+ print(f"{'New expiration time:':23} {exp_time.strftime('%Y-%m-%d %H:%M:%S')} UTC")
249
+ print("Success to refresh token")
250
+ return status
@@ -0,0 +1,34 @@
1
+ class PandaAuthError(Exception):
2
+ """Base class for authentication errors."""
3
+
4
+ pass
5
+
6
+
7
+ class TokenNotFoundError(PandaAuthError):
8
+ """Raised when the token file is missing."""
9
+
10
+ pass
11
+
12
+
13
+ class TokenExpiredError(PandaAuthError):
14
+ """Raised when the token has already expired."""
15
+
16
+ pass
17
+
18
+
19
+ class TokenTooEarlyError(PandaAuthError):
20
+ """Raised when attempting to refresh too early."""
21
+
22
+ pass
23
+
24
+
25
+ class AuthConfigError(PandaAuthError):
26
+ """Raised when fetching the auth or endpoint configuration fails."""
27
+
28
+ pass
29
+
30
+
31
+ class TokenRefreshError(PandaAuthError):
32
+ """Raised when token refresh fails."""
33
+
34
+ pass