lsst-ctrl-bps-panda 29.2025.1900__py3-none-any.whl → 29.2025.2100__py3-none-any.whl

lsst/ctrl/bps/panda/edgenode/cmd_line_decoder.py

@@ -184,6 +184,8 @@ def replace_event_file(params, files):
         Example params:
             isr:eventservice_90^10+somethingelse. This part
             'isr:eventservice_90^10' is the EventService parameter.
+            isr:orderIdMap_10. This part is using order_id map file. But it
+            is not EventService.
         The format for the EventService parameter for LSST is
         'label:eventservice_<baseid>^<localid>'. The '<localid>' should
         start from 1, which means the first event of the file
@@ -217,6 +219,7 @@ def replace_event_file(params, files):
     """
     ret_status = True
     with_events = False
+    with_order_id_map = False
     files = files.split("+")
     file_map = {}
     for file in files:
@@ -272,8 +275,33 @@ def replace_event_file(params, files):
                 ret_status = False
                 break
 
-            params_map[param] = {"event_index": event_index, "order_id_map": order_id_map[label]}
-    return ret_status, with_events, params_map
+            params_map[param] = {"order_id": event_index, "order_id_map": order_id_map[label]}
+        elif "orderIdMap_" in param:
+            with_order_id_map = True
+            label, event = param.split(":")
+            order_id = event.split("_")[1]
+            if not order_id_map:
+                print("orderIdMap is enabled but order_id_map file doesn't exist.")
+                ret_status = False
+                break
+
+            if label not in order_id_map:
+                print(
+                    f"orderIdMap is enabled but label {label} doesn't in the keys"
+                    f" of order_id_map {order_id_map.keys()}"
+                )
+                ret_status = False
+                break
+            if order_id not in order_id_map[label]:
+                print(
+                    f"orderIdMap is enabled but order_id {order_id} is not"
+                    f" in order_id_map[{label}] {order_id_map[label].keys()}"
+                )
+                ret_status = False
+                break
+
+            params_map[param] = {"order_id": order_id, "order_id_map": order_id_map[label]}
+    return ret_status, with_events, with_order_id_map, params_map
 
 
 deliver_input_files(sys.argv[3], sys.argv[4], sys.argv[5])
@@ -281,23 +309,33 @@ cmd_line = str(binascii.unhexlify(sys.argv[1]).decode())
 data_params = sys.argv[2]
 cmd_line = replace_environment_vars(cmd_line)
 
+print(f"cmd_line: {cmd_line}")
+print(f"data_params: {data_params}")
+
 # If EventService is enabled, data_params will only contain event information.
 # So we need to convert the event information to LSST pseudo file names.
 # If EventService is not enabled, this part will not change data_params.
-ret_event_status, with_events, event_params_map = replace_event_file(data_params, sys.argv[4])
-print(f"ret_event_status: {ret_event_status}, with_events: {with_events}")
+ret_rep = replace_event_file(data_params, sys.argv[4])
+ret_event_status, with_events, with_order_id_map, event_params_map = ret_rep
+print(
+    f"ret_event_status: {ret_event_status}, with_events: {with_events} with_order_id_map: {with_order_id_map}"
+)
 if not ret_event_status:
-    print("failed to map EventService parameters to original LSST pseudo file names")
+    print("failed to map EventService/orderIdMap parameters to original LSST pseudo file names")
     exit_code = 1
     sys.exit(exit_code)
 
 for event_param in event_params_map:
-    event_index = event_params_map[event_param]["event_index"]
-    pseudo_file_name = event_params_map[event_param]["order_id_map"][event_index]
-    print(f"replacing event {event_param} with event_index {event_index} to: {pseudo_file_name}")
+    order_id = event_params_map[event_param]["order_id"]
+    pseudo_file_name = event_params_map[event_param]["order_id_map"][order_id]
+    print(f"replacing event {event_param} with order_id {order_id} to: {pseudo_file_name}")
     cmd_line = cmd_line.replace(event_param, pseudo_file_name)
     data_params = data_params.replace(event_param, pseudo_file_name)
 
+# If job name map is enabled, data_params will only contain order_id
+# information. Here we will convert order_id information to LSST pseudo
+# file names.
+
 data_params = data_params.split("+")
 
 """Replace the pipetask command line placeholders

lsst/ctrl/bps/panda/panda_service.py

@@ -37,7 +37,14 @@ import re
 
 from idds.workflowv2.workflow import Workflow as IDDS_client_workflow
 
-from lsst.ctrl.bps import BaseWmsService, BaseWmsWorkflow, WmsRunReport, WmsStates
+from lsst.ctrl.bps import (
+    DEFAULT_MEM_FMT,
+    DEFAULT_MEM_UNIT,
+    BaseWmsService,
+    BaseWmsWorkflow,
+    WmsRunReport,
+    WmsStates,
+)
 from lsst.ctrl.bps.panda.constants import PANDA_DEFAULT_MAX_COPY_WORKERS
 from lsst.ctrl.bps.panda.utils import (
     add_final_idds_work,
@@ -48,6 +55,7 @@ from lsst.ctrl.bps.panda.utils import (
     get_idds_result,
 )
 from lsst.resources import ResourcePath
+from lsst.utils.timer import time_this
 
 _LOG = logging.getLogger(__name__)
 
@@ -58,10 +66,22 @@ class PanDAService(BaseWmsService):
     def prepare(self, config, generic_workflow, out_prefix=None):
         # Docstring inherited from BaseWmsService.prepare.
         _LOG.debug("out_prefix = '%s'", out_prefix)
-        workflow = PandaBpsWmsWorkflow.from_generic_workflow(
-            config, generic_workflow, out_prefix, f"{self.__class__.__module__}.{self.__class__.__name__}"
-        )
-        workflow.write(out_prefix)
+
+        _LOG.info("Starting PanDA prepare stage (creating specific implementation of workflow)")
+
+        with time_this(
+            log=_LOG,
+            level=logging.INFO,
+            prefix=None,
+            msg="PanDA prepare stage completed",
+            mem_usage=True,
+            mem_unit=DEFAULT_MEM_UNIT,
+            mem_fmt=DEFAULT_MEM_FMT,
+        ):
+            workflow = PandaBpsWmsWorkflow.from_generic_workflow(
+                config, generic_workflow, out_prefix, f"{self.__class__.__module__}.{self.__class__.__name__}"
+            )
+            workflow.write(out_prefix)
         return workflow
 
     def submit(self, workflow, **kwargs):

lsst/ctrl/bps/panda/utils.py

@@ -222,7 +222,8 @@ def _make_doma_work(
     task_count,
     task_chunk,
     enable_event_service=False,
-    es_files=None,
+    enable_job_name_map=False,
+    order_id_map_files=None,
     es_label=None,
     max_payloads_per_panda_job=PANDA_DEFAULT_MAX_PAYLOADS_PER_PANDA_JOB,
     max_wms_job_wall_time=None,
@@ -248,8 +249,8 @@ def _make_doma_work(
     local_pfns : `dict` [`str`, `str`]
         Files which need to be copied to a workflow staging area.
     """
-    if es_files is None:
-        es_files = {}
+    if order_id_map_files is None:
+        order_id_map_files = {}
     _LOG.debug("Using gwjob %s to create new PanDA task (gwjob=%s)", gwjob.name, gwjob)
     cvals = {"curr_cluster": gwjob.label}
     _, site = config.search("computeSite", opt={"curvals": cvals, "required": True})
@@ -338,13 +339,15 @@ def _make_doma_work(
 
     maxwalltime = gwjob.request_walltime if gwjob.request_walltime else PANDA_DEFAULT_MAX_WALLTIME
     if enable_event_service:
-        for es_name in es_files:
-            local_pfns[es_name] = es_files[es_name]
         if gwjob.request_walltime and max_payloads_per_panda_job:
             maxwalltime = gwjob.request_walltime * max_payloads_per_panda_job
         elif max_wms_job_wall_time:
             maxwalltime = max_wms_job_wall_time
 
+    if enable_event_service or enable_job_name_map:
+        for es_name in order_id_map_files:
+            local_pfns[es_name] = order_id_map_files[es_name]
+
     for gwfile in generic_workflow.get_job_inputs(gwjob.name, transfer_only=True):
         local_pfns[gwfile.name] = gwfile.src_uri
         if os.path.isdir(gwfile.src_uri):
@@ -596,6 +599,14 @@ def add_idds_work(config, generic_workflow, idds_workflow):
     )
     _LOG.info(my_log)
 
+    # job name map: Use a short job name to map the long job name
+    _, enable_job_name_map = config.search("enableJobNameMap", opt={"default": None})
+    _LOG.info(f"enable_job_name_map: {enable_job_name_map}, {type(enable_job_name_map)}")
+    if enable_event_service and not enable_job_name_map:
+        enable_job_name_map = True
+        my_log = "enable_event_service is set, set enable_job_name_map True."
+        _LOG.info(my_log)
+
     # Limit number of jobs in single PanDA task
     _, max_jobs_per_task = config.search("maxJobsPerTask", opt={"default": PANDA_DEFAULT_MAX_JOBS_PER_TASK})
 
@@ -614,14 +625,16 @@ def add_idds_work(config, generic_workflow, idds_workflow):
         archive_filename = create_archive_file(submit_path, archive_filename, files)
         remote_archive_filename = copy_files_to_pandacache(archive_filename)
 
-    es_files = {}
+    order_id_map_files = {}
     name_works = {}
     order_id_map = {}
+    job_name_to_order_id_map = {}
     doma_tree = None
     order_id_map_file = None
     if enable_event_service:
         enable_event_service = enable_event_service.split(",")
         enable_event_service = [i.strip() for i in enable_event_service]
+    if enable_job_name_map:
         doma_tree = DomaTree(name=generic_workflow.name)
         submit_path = config[".bps_defined.submitPath"]
         _, order_id_map_filename = config.search(
@@ -629,8 +642,14 @@ def add_idds_work(config, generic_workflow, idds_workflow):
         )
         order_id_map_file = os.path.join(submit_path, order_id_map_filename)
         order_id_map = doma_tree.order_jobs_from_generic_workflow(generic_workflow, order_id_map_file)
-        es_files = {"orderIdMapFilename": order_id_map_file}
-        files_to_pre_stage.update(es_files)
+        order_id_map_files = {"orderIdMapFilename": order_id_map_file}
+        files_to_pre_stage.update(order_id_map_files)
+
+        # job name to order id map
+        job_name_to_order_id_map = {
+            label: {job_name: order_id for order_id, job_name in orders.items()}
+            for label, orders in order_id_map.items()
+        }
 
     # To avoid dying due to optimizing number of times through workflow,
     # catch dependency issues to loop through again later.
@@ -684,7 +703,8 @@ def add_idds_work(config, generic_workflow, idds_workflow):
                     task_count,
                     task_chunk,
                     enable_event_service=work_enable_event_service,
-                    es_files=es_files,
+                    enable_job_name_map=enable_job_name_map,
+                    order_id_map_files=order_id_map_files,
                     es_label=job_label,
                     max_payloads_per_panda_job=max_payloads_per_panda_job,
                     max_wms_job_wall_time=max_wms_job_wall_time,
@@ -707,19 +727,35 @@ def add_idds_work(config, generic_workflow, idds_workflow):
                     missing_deps = True
                     break
                 else:
+                    if enable_job_name_map:
+                        parent_job = generic_workflow.get_job(parent_job_name)
+                        parent_job_label = parent_job.label
+                        parent_order_id = job_name_to_order_id_map[parent_job_label][parent_job_name]
+                        inputname = f"{parent_job_label}:orderIdMap_{parent_order_id}"
+                    else:
+                        inputname = job_to_pseudo_filename[parent_job_name]
+
                     deps.append(
                         {
                             "task": job_to_task[parent_job_name],
-                            "inputname": job_to_pseudo_filename[parent_job_name],
-                            "available": False,
+                            "inputname": inputname,
                         }
                     )
             if not missing_deps:
+                f_name = f"{job_label}:orderIdMap_{order_id}" if enable_job_name_map else pseudo_filename
                 work.dependency_map.append(
-                    {"name": pseudo_filename, "order_id": order_id, "dependencies": deps}
+                    {
+                        "name": f_name,
+                        "order_id": order_id,
+                        "dependencies": deps,
+                    }
                 )
             else:
-                jobs_with_dependency_issues[gwjob.name] = {"work": work, "order_id": order_id}
+                jobs_with_dependency_issues[gwjob.name] = {
+                    "work": work,
+                    "order_id": order_id,
+                    "label": job_label,
+                }
 
     # If there were any issues figuring out dependencies through earlier loop
     if jobs_with_dependency_issues:
@@ -729,6 +765,7 @@ def add_idds_work(config, generic_workflow, idds_workflow):
             deps = []
             work = work_item["work"]
             order_id = work_item["order_id"]
+            job_label = work_item["label"]
 
             for parent_job_name in generic_workflow.predecessors(job_name):
                 if parent_job_name not in job_to_task:
@@ -736,15 +773,29 @@ def add_idds_work(config, generic_workflow, idds_workflow):
                     raise RuntimeError(
                         "Could not recover from dependency issues ({job_name} missing {parent_job_name})."
                     )
+                if enable_job_name_map:
+                    parent_job = generic_workflow.get_job(parent_job_name)
+                    parent_job_label = parent_job.label
+                    parent_order_id = job_name_to_order_id_map[parent_job_label][parent_job_name]
+                    inputname = f"{parent_job_label}:orderIdMap_{parent_order_id}"
+                else:
+                    inputname = job_to_pseudo_filename[parent_job_name]
+
                 deps.append(
                     {
                         "task": job_to_task[parent_job_name],
-                        "inputname": job_to_pseudo_filename[parent_job_name],
-                        "available": False,
+                        "inputname": inputname,
                     }
                 )
-            pseudo_filename = job_to_pseudo_filename[job_name]
-            work.dependency_map.append({"name": pseudo_filename, "order_id": order_id, "dependencies": deps})
+
+            work.dependency_map.append(
+                {
+                    "name": f"{job_label}:orderIdMap_{order_id}" if enable_job_name_map else pseudo_filename,
+                    "order_id": order_id,
+                    "dependencies": deps,
+                }
+            )
+
         _LOG.info("Successfully recovered.")
 
     for task_name in name_works:
@@ -754,7 +805,7 @@ def add_idds_work(config, generic_workflow, idds_workflow):
         # 2) check to avoid duplicated items.
         work.dependency_map = work.dependency_map
 
-    if enable_event_service:
+    if enable_job_name_map:
         for label_name in order_id_map:
             for order_id in order_id_map[label_name]:
                 job_name = order_id_map[label_name][order_id]

lsst/ctrl/bps/panda/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.1900"
+__version__ = "29.2025.2100"

{lsst_ctrl_bps_panda-29.2025.1900.dist-info → lsst_ctrl_bps_panda-29.2025.2100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-panda
-Version: 29.2025.1900
+Version: 29.2025.2100
 Summary: PanDA plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_panda-29.2025.1900.dist-info → lsst_ctrl_bps_panda-29.2025.2100.dist-info}/RECORD

@@ -3,9 +3,9 @@ lsst/ctrl/bps/panda/cmd_line_embedder.py,sha256=_gyqWQQxM8ZtZzQbWlOdgjfgrotrwV3s
 lsst/ctrl/bps/panda/constants.py,sha256=hhV1CDHW9G-Z6z2wGaAc41EMlJ-yn2NN3A8psDyjTkw,1907
 lsst/ctrl/bps/panda/panda_auth_drivers.py,sha256=Ff0QsrTgQYbHDCK89_Gayu_2ZC1i3RRt-Dnnx10b8G4,2558
 lsst/ctrl/bps/panda/panda_auth_utils.py,sha256=wb-vlB9jvabVIHKlqukE1vILO_0Q9iixE3xXyROeN2s,5093
-lsst/ctrl/bps/panda/panda_service.py,sha256=60rcp18zaVVNwtw55gri7jshDIre6btBlsMnMqp9vsU,16793
-lsst/ctrl/bps/panda/utils.py,sha256=NdPHRNECTJeWBamcWpzhZttbtBo5QRXtdVRcesMLbJA,37414
-lsst/ctrl/bps/panda/version.py,sha256=duakYcQBT5t4KBaOX14WCtq_sSvpoXvNcxi3sUb8GvI,55
+lsst/ctrl/bps/panda/panda_service.py,sha256=iil1bvfmBNhZ3ZQ04UwlP3vshxNyJk5mZhMAhIaObpI,17278
+lsst/ctrl/bps/panda/utils.py,sha256=FmecYPsL3JbUwXQCgx5rpgDNG9yxlbPD0sJON-kbin4,39594
+lsst/ctrl/bps/panda/version.py,sha256=czd5myijXlfzN1l1OFP7x6hn5ASHoE02wXhI32IqLec,55
 lsst/ctrl/bps/panda/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/ctrl/bps/panda/cli/panda_auth.py,sha256=i54ati_HoKSlyslQRBl7QpX1w5z8MjSfHHMpT43ZeXQ,2055
 lsst/ctrl/bps/panda/cli/cmd/__init__.py,sha256=WVcBiZ3z9rnG4FOsYMgp1QYGwlkM9n_edpMbGDBvCrs,1393
@@ -17,13 +17,13 @@ lsst/ctrl/bps/panda/conf_example/test_sdf.yaml,sha256=e7C8df1h59KA3vBCm_qD1Xmcm2
 lsst/ctrl/bps/panda/conf_example/test_usdf.yaml,sha256=WIbXCJZDaG7zYUHt7U96MUjUsPRdHlk8C0Rg00nsPjs,390
 lsst/ctrl/bps/panda/edgenode/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/ctrl/bps/panda/edgenode/build_cmd_line_decoder.py,sha256=CjB_ESDKLK67QPlcZHWoJzfaqgC733ih_iIQrwYkiUo,3067
-lsst/ctrl/bps/panda/edgenode/cmd_line_decoder.py,sha256=QOtG3VP6-iw6Q8n1ThzawJ9NKJ5h1hzzGPihQr8aDfk,12184
-lsst_ctrl_bps_panda-29.2025.1900.dist-info/licenses/COPYRIGHT,sha256=5ATATZSyXxMNKoJuCJdATg4YNm56ubTwU_hDbShxIWw,116
-lsst_ctrl_bps_panda-29.2025.1900.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps_panda-29.2025.1900.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps_panda-29.2025.1900.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps_panda-29.2025.1900.dist-info/METADATA,sha256=ZXcmInJMgn6OzJ7pP76YJ0BLZynsR0jFs2Mm5i0wiqQ,2375
-lsst_ctrl_bps_panda-29.2025.1900.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-lsst_ctrl_bps_panda-29.2025.1900.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps_panda-29.2025.1900.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps_panda-29.2025.1900.dist-info/RECORD,,
+lsst/ctrl/bps/panda/edgenode/cmd_line_decoder.py,sha256=3pxSE672LPF5J51Vc0h4dyyRi0Is8vSmXfNT3NYpcRg,13660
+lsst_ctrl_bps_panda-29.2025.2100.dist-info/licenses/COPYRIGHT,sha256=5ATATZSyXxMNKoJuCJdATg4YNm56ubTwU_hDbShxIWw,116
+lsst_ctrl_bps_panda-29.2025.2100.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps_panda-29.2025.2100.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps_panda-29.2025.2100.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps_panda-29.2025.2100.dist-info/METADATA,sha256=zNwRCa62S-BOkRBNqaVS5KLNeFfIasaDARYkLhya3F0,2375
+lsst_ctrl_bps_panda-29.2025.2100.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+lsst_ctrl_bps_panda-29.2025.2100.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps_panda-29.2025.2100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps_panda-29.2025.2100.dist-info/RECORD,,

{lsst_ctrl_bps_panda-29.2025.1900.dist-info → lsst_ctrl_bps_panda-29.2025.2100.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any