lsst-ctrl-bps-htcondor 29.2025.2300__py3-none-any.whl → 29.2025.3000__py3-none-any.whl

lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml

@@ -38,3 +38,8 @@ provisioning:
 
 # By default, disable automatic provisioning of resources.
 provisionResources: false
+
+# Whether automatic job retries overwrite stdout/stderr of previous attempt.
+overwriteJobFiles: true
+finalJob:
+  overwriteJobFiles: false

lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -706,14 +706,25 @@ def _create_job(subdir_template, cached_values, generic_workflow, gwjob, out_pre
     htc_job_cmds.update(_translate_job_cmds(cached_values, generic_workflow, gwjob))
 
     # job stdout, stderr, htcondor user log.
-    for key in ("output", "error", "log"):
-        htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).{key[:3]}"
+    for key in ("output", "error"):
+        if cached_values["overwriteJobFiles"]:
+            htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).{key[:3]}"
+        else:
+            htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).$$([NumJobStarts ?: 0]).{key[:3]}"
         _LOG.debug("HTCondor %s = %s", key, htc_job_cmds[key])
 
+    key = "log"
+    htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).{key[:3]}"
+    _LOG.debug("HTCondor %s = %s", key, htc_job_cmds[key])
+
     htc_job_cmds.update(
         _handle_job_inputs(generic_workflow, gwjob.name, cached_values["bpsUseShared"], out_prefix)
     )
 
+    htc_job_cmds.update(
+        _handle_job_outputs(generic_workflow, gwjob.name, cached_values["bpsUseShared"], out_prefix)
+    )
+
     # Add the job cmds dict to the job object.
     htc_job.add_job_cmds(htc_job_cmds)
 
@@ -946,13 +957,7 @@ def _replace_file_vars(use_shared, arguments, workflow, gwjob):
                 # Have shared filesystems and jobs can share file.
                 uri = gwfile.src_uri
             else:
-                # Taking advantage of inside knowledge.  Not future-proof.
-                # Temporary fix until have job wrapper that pulls files
-                # within job.
-                if gwfile.name == "butlerConfig" and Path(gwfile.src_uri).suffix != ".yaml":
-                    uri = "butler.yaml"
-                else:
-                    uri = os.path.basename(gwfile.src_uri)
+                uri = os.path.basename(gwfile.src_uri)
         else:  # Using push transfer
             uri = os.path.basename(gwfile.src_uri)
         arguments = arguments.replace(f"<FILE:{gwfile.name}>", uri)
@@ -1001,7 +1006,9 @@ def _replace_cmd_vars(arguments, gwjob):
     return arguments
 
 
-def _handle_job_inputs(generic_workflow: GenericWorkflow, job_name: str, use_shared: bool, out_prefix: str):
+def _handle_job_inputs(
+    generic_workflow: GenericWorkflow, job_name: str, use_shared: bool, out_prefix: str
+) -> dict[str, str]:
     """Add job input files from generic workflow to job.
 
     Parameters
@@ -1020,7 +1027,6 @@ def _handle_job_inputs(generic_workflow: GenericWorkflow, job_name: str, use_sha
     htc_commands : `dict` [`str`, `str`]
         HTCondor commands for the job submission script.
     """
-    htc_commands = {}
     inputs = []
     for gwf_file in generic_workflow.get_job_inputs(job_name, data=True, transfer_only=True):
         _LOG.debug("src_uri=%s", gwf_file.src_uri)
@@ -1030,38 +1036,68 @@ def _handle_job_inputs(generic_workflow: GenericWorkflow, job_name: str, use_sha
         # Note if use_shared and job_shared, don't need to transfer file.
 
         if not use_shared:  # Copy file using push to job
-            inputs.append(str(uri.relative_to(out_prefix)))
+            inputs.append(str(uri))
         elif not gwf_file.job_shared:  # Jobs require own copy
             # if using shared filesystem, but still need copy in job. Use
             # HTCondor's curl plugin for a local copy.
-
-            # Execution butler is represented as a directory which the
-            # curl plugin does not handle. Taking advantage of inside
-            # knowledge for temporary fix until have job wrapper that pulls
-            # files within job.
-            if gwf_file.name == "butlerConfig":
-                # The execution butler directory doesn't normally exist until
-                # the submit phase so checking for suffix instead of using
-                # is_dir().  If other non-yaml file exists they would have a
-                # different gwf_file.name.
-                if uri.suffix == ".yaml":  # Single file, so just copy.
-                    inputs.append(f"file://{uri}")
-                else:
-                    inputs.append(f"file://{uri / 'butler.yaml'}")
-                    inputs.append(f"file://{uri / 'gen3.sqlite3'}")
-            elif uri.is_dir():
+            if uri.is_dir():
                 raise RuntimeError(
                     f"HTCondor plugin cannot transfer directories locally within job {gwf_file.src_uri}"
                 )
-            else:
-                inputs.append(f"file://{uri}")
+            inputs.append(f"file://{uri}")
 
+    htc_commands = {}
     if inputs:
         htc_commands["transfer_input_files"] = ",".join(inputs)
         _LOG.debug("transfer_input_files=%s", htc_commands["transfer_input_files"])
     return htc_commands
 
 
+def _handle_job_outputs(
+    generic_workflow: GenericWorkflow, job_name: str, use_shared: bool, out_prefix: str
+) -> dict[str, str]:
+    """Add job output files from generic workflow to the job if any.
+
+    Parameters
+    ----------
+    generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
+        The generic workflow (e.g., has executable name and arguments).
+    job_name : `str`
+        Unique name for the job.
+    use_shared : `bool`
+        Whether job has access to files via shared filesystem.
+    out_prefix : `str`
+        The root directory into which all WMS-specific files are written.
+
+    Returns
+    -------
+    htc_commands : `dict` [`str`, `str`]
+        HTCondor commands for the job submission script.
+    """
+    outputs = []
+    output_remaps = []
+    for gwf_file in generic_workflow.get_job_outputs(job_name, data=True, transfer_only=True):
+        _LOG.debug("src_uri=%s", gwf_file.src_uri)
+
+        uri = Path(gwf_file.src_uri)
+        if not use_shared:
+            outputs.append(uri.name)
+            output_remaps.append(f"{uri.name}={str(uri)}")
+
+    # Set to an empty string to disable and only update if there are output
+    # files to transfer. Otherwise, HTCondor will transfer back all files in
+    # the job’s temporary working directory that have been modified or created
+    # by the job.
+    htc_commands = {"transfer_output_files": '""'}
+    if outputs:
+        htc_commands["transfer_output_files"] = ",".join(outputs)
+        _LOG.debug("transfer_output_files=%s", htc_commands["transfer_output_files"])
+
+        htc_commands["transfer_output_remaps"] = f'"{";".join(output_remaps)}"'
+        _LOG.debug("transfer_output_remaps=%s", htc_commands["transfer_output_remaps"])
+    return htc_commands
+
+
 def _get_status_from_id(
     wms_workflow_id: str, hist: float, schedds: dict[str, htcondor.Schedd]
 ) -> tuple[WmsStates, str]:
@@ -2342,6 +2378,12 @@ def _gather_label_values(config: BpsConfig, label: str) -> dict[str, Any]:
     if found:
         values["releaseExpr"] = value
 
+    found, value = config.search("overwriteJobFiles", opt=search_opts)
+    if found:
+        values["overwriteJobFiles"] = value
+    else:
+        values["overwriteJobFiles"] = True
+
     if profile_key and profile_key in config:
         for subkey, val in config[profile_key].items():
             if subkey.startswith("+"):

lsst/ctrl/bps/htcondor/lssthtc.py

@@ -205,6 +205,7 @@ HTC_VALID_JOB_KEYS = {
     "transfer_executable",
     "transfer_input_files",
     "transfer_output_files",
+    "transfer_output_remaps",
     "request_cpus",
     "request_memory",
     "request_disk",

lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.2300"
+__version__ = "29.2025.3000"

{lsst_ctrl_bps_htcondor-29.2025.2300.dist-info → lsst_ctrl_bps_htcondor-29.2025.3000.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.2300
+Version: 29.2025.3000
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

lsst_ctrl_bps_htcondor-29.2025.3000.dist-info/RECORD

@@ -0,0 +1,19 @@
+lsst/ctrl/bps/htcondor/__init__.py,sha256=1gTmOVLJILvBqgqHVECo8uqoX8e4fiTeH_dHBUXgDvY,1417
+lsst/ctrl/bps/htcondor/final_post.sh,sha256=chfaQV6Q7rGsK-8Hx58ch52m-PofvBanrl7VwCssHec,248
+lsst/ctrl/bps/htcondor/handlers.py,sha256=2gM3Ac00in4ob9ckcP331W1LSEjs9UDKIqt4MULA4bg,11196
+lsst/ctrl/bps/htcondor/htcondor_config.py,sha256=c4lCiYEwEXFdxgbMfEkbDm4LrvkRMF31SqLtQqzqIV4,1523
+lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=WKN6bC5cJu8X9UpN5DNMDxb8V0gBOWCthvFyEWUkU7s,96467
+lsst/ctrl/bps/htcondor/lssthtc.py,sha256=MOxY30jCwbW1efgVcWtN4x13OO8monW95t6IidDf86Y,80515
+lsst/ctrl/bps/htcondor/provisioner.py,sha256=hPN8YJUtwNHQylw68kfskF1S2vCeQvztF8W0d_QKqqM,7851
+lsst/ctrl/bps/htcondor/version.py,sha256=GKnpddvvmGJ8yPsnT--mhT32jeA31BaGZsJcw6cTK7k,55
+lsst/ctrl/bps/htcondor/etc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml,sha256=C6DKJKmKFKczukpXVXev9u1-vmv2IcgcdtjTtgJWDQM,1561
+lsst_ctrl_bps_htcondor-29.2025.3000.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
+lsst_ctrl_bps_htcondor-29.2025.3000.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps_htcondor-29.2025.3000.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps_htcondor-29.2025.3000.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps_htcondor-29.2025.3000.dist-info/METADATA,sha256=V75dKW9ZV7iHw7h-DMe6x1gvAbZ_X7b31jZ2V7tU6Fs,2139
+lsst_ctrl_bps_htcondor-29.2025.3000.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_ctrl_bps_htcondor-29.2025.3000.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps_htcondor-29.2025.3000.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps_htcondor-29.2025.3000.dist-info/RECORD,,

lsst_ctrl_bps_htcondor-29.2025.2300.dist-info/RECORD

@@ -1,19 +0,0 @@
-lsst/ctrl/bps/htcondor/__init__.py,sha256=1gTmOVLJILvBqgqHVECo8uqoX8e4fiTeH_dHBUXgDvY,1417
-lsst/ctrl/bps/htcondor/final_post.sh,sha256=chfaQV6Q7rGsK-8Hx58ch52m-PofvBanrl7VwCssHec,248
-lsst/ctrl/bps/htcondor/handlers.py,sha256=2gM3Ac00in4ob9ckcP331W1LSEjs9UDKIqt4MULA4bg,11196
-lsst/ctrl/bps/htcondor/htcondor_config.py,sha256=c4lCiYEwEXFdxgbMfEkbDm4LrvkRMF31SqLtQqzqIV4,1523
-lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=4_jm0lIZw3mYXyays1IWyg3pFwpODR_-g6CLIepXu7w,95330
-lsst/ctrl/bps/htcondor/lssthtc.py,sha256=pYxcA5jicuJs1RnhusSuMFrOU92Xy1fb-tleZ9m784Y,80485
-lsst/ctrl/bps/htcondor/provisioner.py,sha256=hPN8YJUtwNHQylw68kfskF1S2vCeQvztF8W0d_QKqqM,7851
-lsst/ctrl/bps/htcondor/version.py,sha256=GnyaHNlpLZJmdCekPuL3VEJCw758_rgDNpEpGPT5mRw,55
-lsst/ctrl/bps/htcondor/etc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml,sha256=xDRts4vHKov2PE_JRh-0nF3jfuNJXtKBXZqveASp_iA,1422
-lsst_ctrl_bps_htcondor-29.2025.2300.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
-lsst_ctrl_bps_htcondor-29.2025.2300.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps_htcondor-29.2025.2300.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps_htcondor-29.2025.2300.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps_htcondor-29.2025.2300.dist-info/METADATA,sha256=x97wexe47Mag59sqQ7et2qQ-y12yBlO2pyGjzNAXBME,2139
-lsst_ctrl_bps_htcondor-29.2025.2300.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_ctrl_bps_htcondor-29.2025.2300.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps_htcondor-29.2025.2300.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps_htcondor-29.2025.2300.dist-info/RECORD,,