lsst-ctrl-bps-panda 29.2025.4400__tar.gz → 29.2025.4500__tar.gz

{lsst_ctrl_bps_panda-29.2025.4400/python/lsst_ctrl_bps_panda.egg-info → lsst_ctrl_bps_panda-29.2025.4500}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-panda
-Version: 29.2025.4400
+Version: 29.2025.4500
 Summary: PanDA plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_ctrl_bps_panda-29.2025.4400 → lsst_ctrl_bps_panda-29.2025.4500}/python/lsst/ctrl/bps/panda/constants.py

@@ -42,3 +42,4 @@ PANDA_DEFAULT_MAX_COPY_WORKERS = 10
 PANDA_DEFAULT_MAX_PAYLOADS_PER_PANDA_JOB = 10
 PANDA_DEFAULT_ORDER_ID_MAP_FILE = "panda_order_id_map.json"
 PANDA_DEFAULT_NAME_LENGTH = 4000
+PANDA_DEFAULT_MAX_REQUEST_LENGTH = 400000000  # 400M

{lsst_ctrl_bps_panda-29.2025.4400 → lsst_ctrl_bps_panda-29.2025.4500}/python/lsst/ctrl/bps/panda/edgenode/build_cmd_line_decoder.py

@@ -15,8 +15,13 @@ import sys
 
 from lsst.ctrl.bps.constants import DEFAULT_MEM_FMT, DEFAULT_MEM_UNIT
 from lsst.ctrl.bps.drivers import prepare_driver
-from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS
-from lsst.ctrl.bps.panda.utils import copy_files_for_distribution, download_extract_archive, get_idds_client
+from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS, PANDA_DEFAULT_MAX_REQUEST_LENGTH
+from lsst.ctrl.bps.panda.utils import (
+    copy_files_for_distribution,
+    download_extract_archive,
+    get_idds_client,
+    get_idds_result,
+)
 from lsst.resources import ResourcePath
 from lsst.utils.timer import time_this
 
@@ -90,6 +95,23 @@ file_distribution_uri = ResourcePath(config["fileDistributionEndPoint"], forceDi
 copy_files_for_distribution(bps_workflow.files_to_pre_stage, file_distribution_uri, max_copy_workers)
 
 idds_client = get_idds_client(config)
+
+# split workflow into steps if the workflow is hughe
+_, max_request_length = config.search("maxRequestLength", opt={"default": PANDA_DEFAULT_MAX_REQUEST_LENGTH})
+workflow_steps = idds_workflow.split_workflow_to_steps(
+    request_cache=config["submitPath"], max_request_length=max_request_length
+)
+print(f"workflow_steps: {workflow_steps}")
+for wf_step in workflow_steps:
+    ret_step = idds_client.submit(wf_step, username=None, use_dataset_name=False)
+    status, result_step, error = get_idds_result(ret_step)
+    if status and result_step == 0:
+        msg = f"iDDS client manager successfully uploaded workflow step: {wf_step.step_name}"
+        print(msg)
+    else:
+        msg = f"iDDS client manager failed to submit workflow step {wf_step.step_name}: {ret_step}"
+        raise RuntimeError(msg)
+
 ret = idds_client.update_build_request(request_id, signature, idds_workflow)
 print(f"update_build_request returns: {ret}")
 sys.exit(ret[0])

{lsst_ctrl_bps_panda-29.2025.4400 → lsst_ctrl_bps_panda-29.2025.4500}/python/lsst/ctrl/bps/panda/panda_service.py

@@ -45,7 +45,10 @@ from lsst.ctrl.bps import (
     WmsRunReport,
     WmsStates,
 )
-from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS
+from lsst.ctrl.bps.panda.constants import (
+    PANDA_DEFAULT_MAX_COPY_WORKERS,
+    PANDA_DEFAULT_MAX_REQUEST_LENGTH,
+)
 from lsst.ctrl.bps.panda.utils import (
     add_final_idds_work,
     add_idds_work,
@@ -111,6 +114,9 @@ class PanDAService(BaseWmsService):
             return idds_build_workflow
 
         else:
+            _, max_request_length = self.config.search(
+                "maxRequestLength", opt={"default": PANDA_DEFAULT_MAX_REQUEST_LENGTH}
+            )
             _, max_copy_workers = self.config.search(
                 "maxCopyWorkers", opt={"default": PANDA_DEFAULT_MAX_COPY_WORKERS}
             )
@@ -122,6 +128,7 @@ class PanDAService(BaseWmsService):
             if not protocol_pattern.match(file_distribution_uri):
                 file_distribution_uri = "file://" + file_distribution_uri
 
+            idds_client = get_idds_client(self.config)
             submit_cmd = workflow.run_attrs.get("bps_iscustom", False)
             if not submit_cmd:
                 copy_files_for_distribution(
@@ -130,7 +137,23 @@ class PanDAService(BaseWmsService):
                     max_copy_workers,
                 )
 
-            idds_client = get_idds_client(self.config)
+                idds_wf = workflow.idds_client_workflow
+                workflow_steps = idds_wf.split_workflow_to_steps(
+                    request_cache=self.config["submitPath"], max_request_length=max_request_length
+                )
+                for wf_step in workflow_steps:
+                    ret_step = idds_client.submit(wf_step, username=None, use_dataset_name=False)
+                    status, result_step, error = get_idds_result(ret_step)
+                    if status and result_step == 0:
+                        msg = f"iDDS client manager successfully uploaded workflow step: {wf_step.step_name}"
+                        _LOG.info(msg)
+                    else:
+                        msg = (
+                            f"iDDS client manager failed to submit workflow step {wf_step.step_name}: "
+                            f"{ret_step}"
+                        )
+                        raise RuntimeError(msg)
+
             ret = idds_client.submit(workflow.idds_client_workflow, username=None, use_dataset_name=False)
             _LOG.debug("iDDS client manager submit returned = %s", ret)
 

lsst_ctrl_bps_panda-29.2025.4500/python/lsst/ctrl/bps/panda/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.4500"

{lsst_ctrl_bps_panda-29.2025.4400 → lsst_ctrl_bps_panda-29.2025.4500/python/lsst_ctrl_bps_panda.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-panda
-Version: 29.2025.4400
+Version: 29.2025.4500
 Summary: PanDA plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

lsst_ctrl_bps_panda-29.2025.4400/python/lsst/ctrl/bps/panda/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.4400"