lsst-ctrl-bps-panda 29.2025.1600__tar.gz → 29.2025.4700__tar.gz

{lsst_ctrl_bps_panda-29.2025.1600/python/lsst_ctrl_bps_panda.egg-info → lsst_ctrl_bps_panda-29.2025.4700}/PKG-INFO

@@ -1,18 +1,18 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-panda
-Version: 29.2025.1600
+Version: 29.2025.4700
 Summary: PanDA plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
-License: BSD 3-Clause License
+License-Expression: BSD-3-Clause OR GPL-3.0-or-later
 Project-URL: Homepage, https://github.com/lsst/ctrl_bps_panda
 Keywords: lsst
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Astronomy
 Requires-Python: >=3.11.0
 Description-Content-Type: text/x-rst

{lsst_ctrl_bps_panda-29.2025.1600 → lsst_ctrl_bps_panda-29.2025.4700}/pyproject.toml

@@ -6,19 +6,20 @@ build-backend = "setuptools.build_meta"
 name = "lsst-ctrl-bps-panda"
 requires-python = ">=3.11.0"
 description = "PanDA plugin for lsst-ctrl-bps."
-license = {text = "BSD 3-Clause License"}
+license = "BSD-3-Clause OR GPL-3.0-or-later"
+license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
 readme = "README.rst"
 authors = [
     {name="Rubin Observatory Data Management", email="dm-admin@lists.lsst.org"},
 ]
 classifiers = [
     "Intended Audience :: Science/Research",
-    "License :: OSI Approved :: BSD License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Scientific/Engineering :: Astronomy",
 ]
 keywords = ["lsst"]
@@ -51,7 +52,6 @@ where = ["python"]
 
 [tool.setuptools]
 zip-safe = true
-license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
 
 [tool.setuptools.package-data]
 "lsst.ctrl.bps.panda" = ["conf_example/*.yaml"]

{lsst_ctrl_bps_panda-29.2025.1600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cli/cmd/__init__.py

@@ -25,6 +25,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-__all__ = ["clean", "reset", "status"]
+__all__ = ["clean", "reset", "refresh", "status"]
 
-from .panda_auth_commands import clean, reset, status
+from .panda_auth_commands import clean, reset, refresh, status

{lsst_ctrl_bps_panda-29.2025.1600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/cli/cmd/panda_auth_commands.py

@@ -28,6 +28,7 @@
 
 __all__ = [
     "clean",
+    "refresh",
     "reset",
     "status",
 ]
@@ -37,7 +38,12 @@ import click
 
 from lsst.daf.butler.cli.utils import MWCommand
 
-from ...panda_auth_drivers import panda_auth_clean_driver, panda_auth_reset_driver, panda_auth_status_driver
+from ...panda_auth_drivers import (
+    panda_auth_clean_driver,
+    panda_auth_refresh_driver,
+    panda_auth_reset_driver,
+    panda_auth_status_driver,
+)
 
 
 class PandaAuthCommand(MWCommand):
@@ -62,3 +68,13 @@ def reset(*args, **kwargs):
 def clean(*args, **kwargs):
     """Clean up token and token cache files."""
     panda_auth_clean_driver(*args, **kwargs)
+
+
+@click.command(cls=PandaAuthCommand)
+@click.option("--days", default=4, help="The earlist remaining days to refresh the token.")
+@click.option("--verbose", is_flag=True, help="Enable verbose output")
+def refresh(*args, **kwargs):
+    """Refresh auth tocken."""
+    days = kwargs.get("days", 4)
+    verbose = kwargs.get("verbose", False)
+    panda_auth_refresh_driver(days, verbose)

{lsst_ctrl_bps_panda-29.2025.1600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/constants.py

@@ -41,3 +41,5 @@ PANDA_DEFAULT_CLOUD = "US"
 PANDA_DEFAULT_MAX_COPY_WORKERS = 10
 PANDA_DEFAULT_MAX_PAYLOADS_PER_PANDA_JOB = 10
 PANDA_DEFAULT_ORDER_ID_MAP_FILE = "panda_order_id_map.json"
+PANDA_DEFAULT_NAME_LENGTH = 4000
+PANDA_DEFAULT_MAX_REQUEST_LENGTH = 400000000  # 400M

{lsst_ctrl_bps_panda-29.2025.1600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/edgenode/build_cmd_line_decoder.py

@@ -15,8 +15,13 @@ import sys
 
 from lsst.ctrl.bps.constants import DEFAULT_MEM_FMT, DEFAULT_MEM_UNIT
 from lsst.ctrl.bps.drivers import prepare_driver
-from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS
-from lsst.ctrl.bps.panda.utils import copy_files_for_distribution, download_extract_archive, get_idds_client
+from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS, PANDA_DEFAULT_MAX_REQUEST_LENGTH
+from lsst.ctrl.bps.panda.utils import (
+    copy_files_for_distribution,
+    download_extract_archive,
+    get_idds_client,
+    get_idds_result,
+)
 from lsst.resources import ResourcePath
 from lsst.utils.timer import time_this
 
@@ -90,6 +95,23 @@ file_distribution_uri = ResourcePath(config["fileDistributionEndPoint"], forceDi
 copy_files_for_distribution(bps_workflow.files_to_pre_stage, file_distribution_uri, max_copy_workers)
 
 idds_client = get_idds_client(config)
+
+# split workflow into steps if the workflow is hughe
+_, max_request_length = config.search("maxRequestLength", opt={"default": PANDA_DEFAULT_MAX_REQUEST_LENGTH})
+workflow_steps = idds_workflow.split_workflow_to_steps(
+    request_cache=config["submitPath"], max_request_length=max_request_length
+)
+print(f"workflow_steps: {workflow_steps}")
+for wf_step in workflow_steps:
+    ret_step = idds_client.submit(wf_step, username=None, use_dataset_name=False)
+    status, result_step, error = get_idds_result(ret_step)
+    if status and result_step == 0:
+        msg = f"iDDS client manager successfully uploaded workflow step: {wf_step.step_name}"
+        print(msg)
+    else:
+        msg = f"iDDS client manager failed to submit workflow step {wf_step.step_name}: {ret_step}"
+        raise RuntimeError(msg)
+
 ret = idds_client.update_build_request(request_id, signature, idds_workflow)
 print(f"update_build_request returns: {ret}")
 sys.exit(ret[0])

{lsst_ctrl_bps_panda-29.2025.1600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/edgenode/cmd_line_decoder.py

@@ -184,6 +184,8 @@ def replace_event_file(params, files):
         Example params:
             isr:eventservice_90^10+somethingelse. This part
             'isr:eventservice_90^10' is the EventService parameter.
+            isr:orderIdMap_10. This part is using order_id map file. But it
+            is not EventService.
         The format for the EventService parameter for LSST is
         'label:eventservice_<baseid>^<localid>'. The '<localid>' should
         start from 1, which means the first event of the file
@@ -217,6 +219,7 @@ def replace_event_file(params, files):
     """
     ret_status = True
     with_events = False
+    with_order_id_map = False
     files = files.split("+")
     file_map = {}
     for file in files:
@@ -272,51 +275,116 @@ def replace_event_file(params, files):
                 ret_status = False
                 break
 
-            params_map[param] = {"event_index": event_index, "order_id_map": order_id_map[label]}
-    return ret_status, with_events, params_map
+            params_map[param] = {"order_id": event_index, "order_id_map": order_id_map[label]}
+        elif "orderIdMap_" in param:
+            with_order_id_map = True
+            label, event = param.split(":")
+            order_id = event.split("_")[1]
+            if not order_id_map:
+                print("orderIdMap is enabled but order_id_map file doesn't exist.")
+                ret_status = False
+                break
+
+            if label not in order_id_map:
+                print(
+                    f"orderIdMap is enabled but label {label} doesn't in the keys"
+                    f" of order_id_map {order_id_map.keys()}"
+                )
+                ret_status = False
+                break
+            if order_id not in order_id_map[label]:
+                print(
+                    f"orderIdMap is enabled but order_id {order_id} is not"
+                    f" in order_id_map[{label}] {order_id_map[label].keys()}"
+                )
+                ret_status = False
+                break
 
+            params_map[param] = {"order_id": order_id, "order_id_map": order_id_map[label]}
+    return ret_status, with_events, with_order_id_map, params_map
 
-deliver_input_files(sys.argv[3], sys.argv[4], sys.argv[5])
-cmd_line = str(binascii.unhexlify(sys.argv[1]).decode())
-data_params = sys.argv[2]
-cmd_line = replace_environment_vars(cmd_line)
 
-# If EventService is enabled, data_params will only contain event information.
-# So we need to convert the event information to LSST pseudo file names.
-# If EventService is not enabled, this part will not change data_params.
-ret_event_status, with_events, event_params_map = replace_event_file(data_params, sys.argv[4])
-print(f"ret_event_status: {ret_event_status}, with_events: {with_events}")
-if not ret_event_status:
-    print("failed to map EventService parameters to original LSST pseudo file names")
-    exit_code = 1
-    sys.exit(exit_code)
+def use_map_file(input_file):
+    """Check whether the input file needs to be replaced
+    because enableQnodeMap is enabled.
 
-for event_param in event_params_map:
-    event_index = event_params_map[event_param]["event_index"]
-    pseudo_file_name = event_params_map[event_param]["order_id_map"][event_index]
-    print(f"replacing event {event_param} with event_index {event_index} to: {pseudo_file_name}")
-    cmd_line = cmd_line.replace(event_param, pseudo_file_name)
-    data_params = data_params.replace(event_param, pseudo_file_name)
+    Parameters
+    ----------
+    input_file : `str`
+        Input file either a pseudo file or job name.
 
-data_params = data_params.split("+")
+    Returns
+    -------
+    use_qnode_map: `bool`
+        Whether qnode_map is used. There is a placeholder 'PH'
+    when enableQnodeMap is true.
+    """
+    parts = input_file.split(":")
+    use_qnode_map = len(parts) == 2 and parts[0] == "PH"
+    return use_qnode_map
+
+
+if __name__ == "__main__":
+    deliver_input_files(sys.argv[3], sys.argv[4], sys.argv[5])
+    cmd_line = str(binascii.unhexlify(sys.argv[1]).decode())
+    data_params = sys.argv[2]
+    cmd_line = replace_environment_vars(cmd_line)
+
+    print(f"cmd_line: {cmd_line}")
+    print(f"data_params: {data_params}")
+
+    # If EventService is enabled, data_params will only contain
+    # event information. So we need to convert the event information
+    # to LSST pseudo file names. If EventService is not enabled,
+    # this part will not change data_params.
+    ret_rep = replace_event_file(data_params, sys.argv[4])
+    ret_event_status, with_events, with_order_id_map, event_params_map = ret_rep
+    print(
+        f"ret_event_status: {ret_event_status}, "
+        f"with_events: {with_events} "
+        f"with_order_id_map: {with_order_id_map}"
+    )
+    if not ret_event_status:
+        print("failed to map EventService/orderIdMap parameters to original LSST pseudo file names")
+        exit_code = 1
+        sys.exit(exit_code)
+
+    for event_param in event_params_map:
+        order_id = event_params_map[event_param]["order_id"]
+        pseudo_file_name = event_params_map[event_param]["order_id_map"][order_id]
+        print(f"replacing event {event_param} with order_id {order_id} to: {pseudo_file_name}")
+        cmd_line = cmd_line.replace(event_param, pseudo_file_name)
+        data_params = data_params.replace(event_param, pseudo_file_name)
+
+    # If job name map is enabled, data_params will only contain order_id
+    # information. Here we will convert order_id information to LSST pseudo
+    # file names.
+
+    data_params = data_params.split("+")
+
+    """Replace the pipetask command line placeholders
+     with actual data provided in the script call
+     in form placeholder1:file1+placeholder2:file2:...
+    """
+    cmd_line = replace_files_placeholders(cmd_line, sys.argv[4])
 
-"""Replace the pipetask command line placeholders
- with actual data provided in the script call
- in form placeholder1:file1+placeholder2:file2:...
-"""
-cmd_line = replace_files_placeholders(cmd_line, sys.argv[4])
-
-for key_value_pair in data_params[1:]:
-    (key, value) = key_value_pair.split(":")
-    cmd_line = cmd_line.replace("{" + key + "}", value)
-
-print("executable command line:")
-print(cmd_line)
-
-exit_status = os.system(cmd_line)
-exit_code = 1
-if os.WIFSIGNALED(exit_status):
-    exit_code = os.WTERMSIG(exit_status) + 128
-elif os.WIFEXITED(exit_status):
-    exit_code = os.WEXITSTATUS(exit_status)
-sys.exit(exit_code)
+    jobname = data_params[0]
+    if use_map_file(jobname):
+        with open("qnode_map.json", encoding="utf-8") as f:
+            qnode_map = json.load(f)
+            data_params = qnode_map[jobname].split("+")
+
+    for key_value_pair in data_params[1:]:
+        (key, value) = key_value_pair.split(":")
+        cmd_line = cmd_line.replace("{" + key + "}", value)
+
+    print("executable command line:")
+    print(cmd_line)
+
+    exit_status = os.system(cmd_line)
+    exit_code = 1
+    if os.WIFSIGNALED(exit_status):
+        exit_code = os.WTERMSIG(exit_status) + 128
+    elif os.WIFEXITED(exit_status):
+        exit_code = os.WEXITSTATUS(exit_status)
+    sys.exit(exit_code)

{lsst_ctrl_bps_panda-29.2025.1600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/panda_auth_drivers.py

@@ -33,6 +33,7 @@ the subcommand method.
 
 __all__ = [
     "panda_auth_clean_driver",
+    "panda_auth_refresh_driver",
     "panda_auth_reset_driver",
     "panda_auth_status_driver",
 ]
@@ -41,7 +42,19 @@ __all__ = [
 import logging
 from datetime import datetime
 
-from .panda_auth_utils import panda_auth_clean, panda_auth_status, panda_auth_update
+from lsst.ctrl.bps.panda.panda_exceptions import (
+    PandaAuthError,
+    TokenExpiredError,
+    TokenNotFoundError,
+    TokenTooEarlyError,
+)
+
+from .panda_auth_utils import (
+    panda_auth_clean,
+    panda_auth_refresh,
+    panda_auth_status,
+    panda_auth_update,
+)
 
 _LOG = logging.getLogger(__name__)
 
@@ -56,6 +69,20 @@ def panda_auth_reset_driver():
     panda_auth_update(None, True)
 
 
+def panda_auth_refresh_driver(days, verbose):
+    """Refresh auth token."""
+    try:
+        panda_auth_refresh(days, verbose)
+    except TokenNotFoundError as e:
+        print(f"[ERROR] {e}")
+    except TokenExpiredError as e:
+        print(f"[ERROR] {e}")
+    except TokenTooEarlyError as e:
+        print(f"[INFO] {e}")
+    except PandaAuthError as e:
+        print(f"[FAIL] {e}")
+
+
 def panda_auth_status_driver():
     """Gather information about a token if it exists."""
     status = panda_auth_status()

{lsst_ctrl_bps_panda-29.2025.1600 → lsst_ctrl_bps_panda-29.2025.4700}/python/lsst/ctrl/bps/panda/panda_auth_utils.py

@@ -30,19 +30,32 @@
 __all__ = [
     "panda_auth_clean",
     "panda_auth_expiration",
+    "panda_auth_refresh",
     "panda_auth_setup",
     "panda_auth_status",
     "panda_auth_update",
 ]
 
 
+import base64
+import json
 import logging
 import os
+from datetime import UTC, datetime, timedelta
 
 import idds.common.utils as idds_utils
 import pandaclient.idds_api
 from pandaclient.openidc_utils import OpenIdConnect_Utils
 
+from lsst.ctrl.bps.panda.panda_exceptions import (
+    AuthConfigError,
+    PandaAuthError,
+    TokenExpiredError,
+    TokenNotFoundError,
+    TokenRefreshError,
+    TokenTooEarlyError,
+)
+
 _LOG = logging.getLogger(__name__)
 
 
@@ -151,3 +164,87 @@ def panda_auth_update(idds_server=None, reset=False):
         # idds server given.  So for now, check result string for keywords.
         if "request_id" not in ret[1][-1] or "status" not in ret[1][-1]:
             raise RuntimeError(f"Error contacting PanDA service: {ret}")
+
+
+def panda_auth_refresh(days=4, verbose=False):
+    """
+    Refresh the current valid IAM OpenID authentication token.
+
+    This function checks the expiration time of the existing token stored
+    in the local token file and attempts to refresh it if it is close to
+    expiring (within a specified number of days).
+
+    Parameters
+    ----------
+    days : `int`, optional
+        The minimum number of days before token expiration to trigger a
+        refresh. If the token expires in more than this number of days,
+        the refresh is skipped. Default is 4.
+    verbose : `bool`, optional
+        If True, enables verbose output for debugging or logging.
+        Default is False.
+
+    Returns
+    -------
+    status: `dict`
+        A dictionary containing the refreshed token status
+    """
+    panda_url = os.environ.get("PANDA_URL")
+    panda_auth_vo = os.environ.get("PANDA_AUTH_VO")
+
+    if not panda_url or not panda_auth_vo:
+        raise PandaAuthError("Missing required environment variables: PANDA_URL or PANDA_AUTH_VO")
+
+    url_prefix = panda_url.split("/server", 1)[0]
+    auth_url = f"{url_prefix}/auth/{panda_auth_vo}_auth_config.json"
+    open_id = OpenIdConnect_Utils(auth_url, log_stream=_LOG, verbose=verbose)
+
+    token_file = open_id.get_token_path()
+    if not os.path.exists(token_file):
+        raise TokenNotFoundError("Cannot find token file. Use 'panda_auth reset' to obtain a new token.")
+
+    with open(token_file) as f:
+        data = json.load(f)
+    enc = data["id_token"].split(".")[1]
+    enc += "=" * (-len(enc) % 4)
+    dec = json.loads(base64.urlsafe_b64decode(enc.encode()))
+    exp_time = datetime.fromtimestamp(dec["exp"], tz=UTC)
+    delta = exp_time - datetime.now(UTC)
+    minutes = delta.total_seconds() / 60
+    print(f"Token will expire in {minutes} minutes.")
+    print(f"Token expiration time : {exp_time.strftime('%Y-%m-%d %H:%M:%S')} UTC")
+    if delta < timedelta(minutes=0):
+        raise TokenExpiredError("Token already expired. Cannot refresh.")
+    elif delta > timedelta(days=days):
+        raise TokenTooEarlyError(
+            f"Too early to refresh. More than {days} day(s) until expiration.\n"
+            f"Use '--days' option to adjust threshold, e.g.:\n"
+            f"  panda_auth refresh --days 10"
+        )
+
+    refresh_token_string = data["refresh_token"]
+
+    s, auth_config = open_id.fetch_page(open_id.auth_config_url)
+    if not s:
+        raise AuthConfigError("Failed to get Auth configuration.")
+
+    s, endpoint_config = open_id.fetch_page(auth_config["oidc_config_url"])
+    if not s:
+        raise AuthConfigError("Failed to get endpoint configuration.")
+
+    s, o = open_id.refresh_token(
+        endpoint_config["token_endpoint"],
+        auth_config["client_id"],
+        auth_config["client_secret"],
+        refresh_token_string,
+    )
+
+    if not s:
+        raise TokenRefreshError("Failed to refresh token.")
+
+    status = panda_auth_status()
+    if status:
+        exp_time = datetime.fromtimestamp(status["exp"], tz=UTC)
+        print(f"{'New expiration time:':23} {exp_time.strftime('%Y-%m-%d %H:%M:%S')} UTC")
+    print("Success to refresh token")
+    return status

lsst_ctrl_bps_panda-29.2025.4700/python/lsst/ctrl/bps/panda/panda_exceptions.py

@@ -0,0 +1,34 @@
+class PandaAuthError(Exception):
+    """Base class for authentication errors."""
+
+    pass
+
+
+class TokenNotFoundError(PandaAuthError):
+    """Raised when the token file is missing."""
+
+    pass
+
+
+class TokenExpiredError(PandaAuthError):
+    """Raised when the token has already expired."""
+
+    pass
+
+
+class TokenTooEarlyError(PandaAuthError):
+    """Raised when attempting to refresh too early."""
+
+    pass
+
+
+class AuthConfigError(PandaAuthError):
+    """Raised when fetching the auth or endpoint configuration fails."""
+
+    pass
+
+
+class TokenRefreshError(PandaAuthError):
+    """Raised when token refresh fails."""
+
+    pass