lsst-ctrl-bps-panda 29.2025.4400__py3-none-any.whl → 29.2025.4500__py3-none-any.whl

lsst/ctrl/bps/panda/constants.py

@@ -42,3 +42,4 @@ PANDA_DEFAULT_MAX_COPY_WORKERS = 10
 PANDA_DEFAULT_MAX_PAYLOADS_PER_PANDA_JOB = 10
 PANDA_DEFAULT_ORDER_ID_MAP_FILE = "panda_order_id_map.json"
 PANDA_DEFAULT_NAME_LENGTH = 4000
+PANDA_DEFAULT_MAX_REQUEST_LENGTH = 400000000  # 400M

lsst/ctrl/bps/panda/edgenode/build_cmd_line_decoder.py

@@ -15,8 +15,13 @@ import sys
 
 from lsst.ctrl.bps.constants import DEFAULT_MEM_FMT, DEFAULT_MEM_UNIT
 from lsst.ctrl.bps.drivers import prepare_driver
-from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS
-from lsst.ctrl.bps.panda.utils import copy_files_for_distribution, download_extract_archive, get_idds_client
+from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS, PANDA_DEFAULT_MAX_REQUEST_LENGTH
+from lsst.ctrl.bps.panda.utils import (
+    copy_files_for_distribution,
+    download_extract_archive,
+    get_idds_client,
+    get_idds_result,
+)
 from lsst.resources import ResourcePath
 from lsst.utils.timer import time_this
 
@@ -90,6 +95,23 @@ file_distribution_uri = ResourcePath(config["fileDistributionEndPoint"], forceDi
 copy_files_for_distribution(bps_workflow.files_to_pre_stage, file_distribution_uri, max_copy_workers)
 
 idds_client = get_idds_client(config)
+
+# split workflow into steps if the workflow is hughe
+_, max_request_length = config.search("maxRequestLength", opt={"default": PANDA_DEFAULT_MAX_REQUEST_LENGTH})
+workflow_steps = idds_workflow.split_workflow_to_steps(
+    request_cache=config["submitPath"], max_request_length=max_request_length
+)
+print(f"workflow_steps: {workflow_steps}")
+for wf_step in workflow_steps:
+    ret_step = idds_client.submit(wf_step, username=None, use_dataset_name=False)
+    status, result_step, error = get_idds_result(ret_step)
+    if status and result_step == 0:
+        msg = f"iDDS client manager successfully uploaded workflow step: {wf_step.step_name}"
+        print(msg)
+    else:
+        msg = f"iDDS client manager failed to submit workflow step {wf_step.step_name}: {ret_step}"
+        raise RuntimeError(msg)
+
 ret = idds_client.update_build_request(request_id, signature, idds_workflow)
 print(f"update_build_request returns: {ret}")
 sys.exit(ret[0])

lsst/ctrl/bps/panda/panda_service.py

@@ -45,7 +45,10 @@ from lsst.ctrl.bps import (
     WmsRunReport,
     WmsStates,
 )
-from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS
+from lsst.ctrl.bps.panda.constants import (
+    PANDA_DEFAULT_MAX_COPY_WORKERS,
+    PANDA_DEFAULT_MAX_REQUEST_LENGTH,
+)
 from lsst.ctrl.bps.panda.utils import (
     add_final_idds_work,
     add_idds_work,
@@ -111,6 +114,9 @@ class PanDAService(BaseWmsService):
             return idds_build_workflow
 
         else:
+            _, max_request_length = self.config.search(
+                "maxRequestLength", opt={"default": PANDA_DEFAULT_MAX_REQUEST_LENGTH}
+            )
             _, max_copy_workers = self.config.search(
                 "maxCopyWorkers", opt={"default": PANDA_DEFAULT_MAX_COPY_WORKERS}
             )
@@ -122,6 +128,7 @@ class PanDAService(BaseWmsService):
             if not protocol_pattern.match(file_distribution_uri):
                 file_distribution_uri = "file://" + file_distribution_uri
 
+            idds_client = get_idds_client(self.config)
             submit_cmd = workflow.run_attrs.get("bps_iscustom", False)
             if not submit_cmd:
                 copy_files_for_distribution(
@@ -130,7 +137,23 @@ class PanDAService(BaseWmsService):
                     max_copy_workers,
                 )
 
-            idds_client = get_idds_client(self.config)
+                idds_wf = workflow.idds_client_workflow
+                workflow_steps = idds_wf.split_workflow_to_steps(
+                    request_cache=self.config["submitPath"], max_request_length=max_request_length
+                )
+                for wf_step in workflow_steps:
+                    ret_step = idds_client.submit(wf_step, username=None, use_dataset_name=False)
+                    status, result_step, error = get_idds_result(ret_step)
+                    if status and result_step == 0:
+                        msg = f"iDDS client manager successfully uploaded workflow step: {wf_step.step_name}"
+                        _LOG.info(msg)
+                    else:
+                        msg = (
+                            f"iDDS client manager failed to submit workflow step {wf_step.step_name}: "
+                            f"{ret_step}"
+                        )
+                        raise RuntimeError(msg)
+
             ret = idds_client.submit(workflow.idds_client_workflow, username=None, use_dataset_name=False)
             _LOG.debug("iDDS client manager submit returned = %s", ret)
 

lsst/ctrl/bps/panda/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.4400"
+__version__ = "29.2025.4500"

{lsst_ctrl_bps_panda-29.2025.4400.dist-info → lsst_ctrl_bps_panda-29.2025.4500.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-panda
-Version: 29.2025.4400
+Version: 29.2025.4500
 Summary: PanDA plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_ctrl_bps_panda-29.2025.4400.dist-info → lsst_ctrl_bps_panda-29.2025.4500.dist-info}/RECORD

@@ -1,12 +1,12 @@
 lsst/ctrl/bps/panda/__init__.py,sha256=oApEOBGMuIv98uQvcDf6DLVvrbD6_8p9o0m2YPTenPY,1351
 lsst/ctrl/bps/panda/cmd_line_embedder.py,sha256=_gyqWQQxM8ZtZzQbWlOdgjfgrotrwV3sFvwWXxe-jfQ,6778
-lsst/ctrl/bps/panda/constants.py,sha256=hhV1CDHW9G-Z6z2wGaAc41EMlJ-yn2NN3A8psDyjTkw,1907
+lsst/ctrl/bps/panda/constants.py,sha256=C8DBJT-jIDF-byBSH5rcefLFkqDNC2DRxMhxmSk-GyI,1960
 lsst/ctrl/bps/panda/panda_auth_drivers.py,sha256=LoD-tP990ELmVks3Vxv76jm4a8j3h3cNRTNX2XtFGHk,3163
 lsst/ctrl/bps/panda/panda_auth_utils.py,sha256=kJnevhvjvUegbXfAyiVcoxinONsc_TJqfK4neTmcN5k,8544
 lsst/ctrl/bps/panda/panda_exceptions.py,sha256=HcOKWMuG79c16Y9j7IJbp990k4DBQ54e7haY1Fsl6XQ,629
-lsst/ctrl/bps/panda/panda_service.py,sha256=tdqAs2ny1FvgULD876heR3rhaIHKCW707aFGlyaJYf8,19308
+lsst/ctrl/bps/panda/panda_service.py,sha256=wQySP0wLppjgvVmgqUk0FHXLIt4hBaCZR0lnuBl6jgc,20450
 lsst/ctrl/bps/panda/utils.py,sha256=fwTMtFHldV8aN8LLqobf0ekkqEfCpUc_7xmKxe7YFaI,44999
-lsst/ctrl/bps/panda/version.py,sha256=c_EQrzHIElqjF0MSUCohtezaryfBlqz_McPyAghglVk,55
+lsst/ctrl/bps/panda/version.py,sha256=ec8d6sw9aRuINU5Qs4LjOV9LhqSISytoOfEoRYO2XzU,55
 lsst/ctrl/bps/panda/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/ctrl/bps/panda/cli/panda_auth.py,sha256=i54ati_HoKSlyslQRBl7QpX1w5z8MjSfHHMpT43ZeXQ,2055
 lsst/ctrl/bps/panda/cli/cmd/__init__.py,sha256=QNhn-2QmXHKVUgwmtuX7wnDv3qOUuU_30juuM_2AGaE,1413
@@ -17,14 +17,14 @@ lsst/ctrl/bps/panda/conf_example/test_idf.yaml,sha256=P-FLBEmKZ2o0QiR6w8GZ9AAAOP
 lsst/ctrl/bps/panda/conf_example/test_sdf.yaml,sha256=e7C8df1h59KA3vBCm_qD1Xmcm20fSFPw307bXDhFB44,487
 lsst/ctrl/bps/panda/conf_example/test_usdf.yaml,sha256=WIbXCJZDaG7zYUHt7U96MUjUsPRdHlk8C0Rg00nsPjs,390
 lsst/ctrl/bps/panda/edgenode/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lsst/ctrl/bps/panda/edgenode/build_cmd_line_decoder.py,sha256=CjB_ESDKLK67QPlcZHWoJzfaqgC733ih_iIQrwYkiUo,3067
+lsst/ctrl/bps/panda/edgenode/build_cmd_line_decoder.py,sha256=Sx7S-jnnyl4IBXWk19B1kcx6cdQGz4RwJDgqQvTwot8,3945
 lsst/ctrl/bps/panda/edgenode/cmd_line_decoder.py,sha256=cqPeJLA7KfB7KnPTR-ykyEoHQ-_YE17h8_EfnqWA5eA,14616
-lsst_ctrl_bps_panda-29.2025.4400.dist-info/licenses/COPYRIGHT,sha256=5ATATZSyXxMNKoJuCJdATg4YNm56ubTwU_hDbShxIWw,116
-lsst_ctrl_bps_panda-29.2025.4400.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps_panda-29.2025.4400.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps_panda-29.2025.4400.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps_panda-29.2025.4400.dist-info/METADATA,sha256=Du1WrzxsCN-B-8oDQN89Bcq7AEsIswImWfY4LXPZb8U,2398
-lsst_ctrl_bps_panda-29.2025.4400.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_ctrl_bps_panda-29.2025.4400.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps_panda-29.2025.4400.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps_panda-29.2025.4400.dist-info/RECORD,,
+lsst_ctrl_bps_panda-29.2025.4500.dist-info/licenses/COPYRIGHT,sha256=5ATATZSyXxMNKoJuCJdATg4YNm56ubTwU_hDbShxIWw,116
+lsst_ctrl_bps_panda-29.2025.4500.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps_panda-29.2025.4500.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps_panda-29.2025.4500.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps_panda-29.2025.4500.dist-info/METADATA,sha256=6feTjMpArcmDON4z5zsTc68-G9dvRjxu7-o3ikRo6Ew,2398
+lsst_ctrl_bps_panda-29.2025.4500.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_ctrl_bps_panda-29.2025.4500.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps_panda-29.2025.4500.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps_panda-29.2025.4500.dist-info/RECORD,,