lsst-ctrl-bps-htcondor 29.2025.2100__py3-none-any.whl → 29.2025.3500__py3-none-any.whl

lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml

@@ -38,3 +38,8 @@ provisioning:
 
 # By default, disable automatic provisioning of resources.
 provisionResources: false
+
+# Whether automatic job retries overwrite stdout/stderr of previous attempt.
+overwriteJobFiles: true
+finalJob:
+  overwriteJobFiles: false

lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -401,6 +401,54 @@ class HTCondorService(BaseWmsService):
         _LOG.debug("job_ids = %s", job_ids)
         return job_ids
 
+    def get_status(
+        self,
+        wms_workflow_id: str,
+        hist: float = 1,
+        is_global: bool = False,
+    ) -> tuple[WmsStates, str]:
+        """Return status of run based upon given constraints.
+
+        Parameters
+        ----------
+        wms_workflow_id : `str`
+            Limit to specific run based on id (queue id or path).
+        hist : `float`, optional
+            Limit history search to this many days. Defaults to 1.
+        is_global : `bool`, optional
+            If set, all job queues (and their histories) will be queried for
+            job information. Defaults to False which means that only the local
+            job queue will be queried.
+
+        Returns
+        -------
+        state : `lsst.ctrl.bps.WmsStates`
+            Status of single run from given information.
+        message : `str`
+            Extra message for status command to print.  This could be pointers
+            to documentation or to WMS specific commands.
+        """
+        _LOG.debug("get_status: id=%s, hist=%s, is_global=%s", wms_workflow_id, hist, is_global)
+
+        id_type = _wms_id_type(wms_workflow_id)
+        _LOG.debug("id_type = %s", id_type.name)
+
+        if id_type == WmsIdType.LOCAL:
+            schedulers = _locate_schedds(locate_all=is_global)
+            _LOG.debug("schedulers = %s", schedulers)
+            state, message = _get_status_from_id(wms_workflow_id, hist, schedds=schedulers)
+        elif id_type == WmsIdType.GLOBAL:
+            schedulers = _locate_schedds(locate_all=True)
+            _LOG.debug("schedulers = %s", schedulers)
+            state, message = _get_status_from_id(wms_workflow_id, hist, schedds=schedulers)
+        elif id_type == WmsIdType.PATH:
+            state, message = _get_status_from_path(wms_workflow_id)
+        else:
+            state, message = WmsStates.UNKNOWN, "Invalid job id"
+        _LOG.debug("state: %s, %s", state, message)
+
+        return state, message
+
     def report(
         self,
         wms_workflow_id=None,
@@ -658,14 +706,25 @@ def _create_job(subdir_template, cached_values, generic_workflow, gwjob, out_pre
     htc_job_cmds.update(_translate_job_cmds(cached_values, generic_workflow, gwjob))
 
     # job stdout, stderr, htcondor user log.
-    for key in ("output", "error", "log"):
-        htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).{key[:3]}"
+    for key in ("output", "error"):
+        if cached_values["overwriteJobFiles"]:
+            htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).{key[:3]}"
+        else:
+            htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).$$([NumJobStarts ?: 0]).{key[:3]}"
         _LOG.debug("HTCondor %s = %s", key, htc_job_cmds[key])
 
+    key = "log"
+    htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).{key[:3]}"
+    _LOG.debug("HTCondor %s = %s", key, htc_job_cmds[key])
+
     htc_job_cmds.update(
         _handle_job_inputs(generic_workflow, gwjob.name, cached_values["bpsUseShared"], out_prefix)
     )
 
+    htc_job_cmds.update(
+        _handle_job_outputs(generic_workflow, gwjob.name, cached_values["bpsUseShared"], out_prefix)
+    )
+
     # Add the job cmds dict to the job object.
     htc_job.add_job_cmds(htc_job_cmds)
 
@@ -898,13 +957,7 @@ def _replace_file_vars(use_shared, arguments, workflow, gwjob):
                 # Have shared filesystems and jobs can share file.
                 uri = gwfile.src_uri
             else:
-                # Taking advantage of inside knowledge.  Not future-proof.
-                # Temporary fix until have job wrapper that pulls files
-                # within job.
-                if gwfile.name == "butlerConfig" and Path(gwfile.src_uri).suffix != ".yaml":
-                    uri = "butler.yaml"
-                else:
-                    uri = os.path.basename(gwfile.src_uri)
+                uri = os.path.basename(gwfile.src_uri)
         else:  # Using push transfer
             uri = os.path.basename(gwfile.src_uri)
         arguments = arguments.replace(f"<FILE:{gwfile.name}>", uri)
@@ -953,7 +1006,9 @@ def _replace_cmd_vars(arguments, gwjob):
     return arguments
 
 
-def _handle_job_inputs(generic_workflow: GenericWorkflow, job_name: str, use_shared: bool, out_prefix: str):
+def _handle_job_inputs(
+    generic_workflow: GenericWorkflow, job_name: str, use_shared: bool, out_prefix: str
+) -> dict[str, str]:
     """Add job input files from generic workflow to job.
 
     Parameters
@@ -972,7 +1027,6 @@ def _handle_job_inputs(generic_workflow: GenericWorkflow, job_name: str, use_sha
     htc_commands : `dict` [`str`, `str`]
         HTCondor commands for the job submission script.
     """
-    htc_commands = {}
     inputs = []
     for gwf_file in generic_workflow.get_job_inputs(job_name, data=True, transfer_only=True):
         _LOG.debug("src_uri=%s", gwf_file.src_uri)
@@ -982,38 +1036,139 @@ def _handle_job_inputs(generic_workflow: GenericWorkflow, job_name: str, use_sha
         # Note if use_shared and job_shared, don't need to transfer file.
 
         if not use_shared:  # Copy file using push to job
-            inputs.append(str(uri.relative_to(out_prefix)))
+            inputs.append(str(uri))
         elif not gwf_file.job_shared:  # Jobs require own copy
             # if using shared filesystem, but still need copy in job. Use
             # HTCondor's curl plugin for a local copy.
-
-            # Execution butler is represented as a directory which the
-            # curl plugin does not handle. Taking advantage of inside
-            # knowledge for temporary fix until have job wrapper that pulls
-            # files within job.
-            if gwf_file.name == "butlerConfig":
-                # The execution butler directory doesn't normally exist until
-                # the submit phase so checking for suffix instead of using
-                # is_dir().  If other non-yaml file exists they would have a
-                # different gwf_file.name.
-                if uri.suffix == ".yaml":  # Single file, so just copy.
-                    inputs.append(f"file://{uri}")
-                else:
-                    inputs.append(f"file://{uri / 'butler.yaml'}")
-                    inputs.append(f"file://{uri / 'gen3.sqlite3'}")
-            elif uri.is_dir():
+            if uri.is_dir():
                 raise RuntimeError(
                     f"HTCondor plugin cannot transfer directories locally within job {gwf_file.src_uri}"
                 )
-            else:
-                inputs.append(f"file://{uri}")
+            inputs.append(f"file://{uri}")
 
+    htc_commands = {}
     if inputs:
         htc_commands["transfer_input_files"] = ",".join(inputs)
         _LOG.debug("transfer_input_files=%s", htc_commands["transfer_input_files"])
     return htc_commands
 
 
+def _handle_job_outputs(
+    generic_workflow: GenericWorkflow, job_name: str, use_shared: bool, out_prefix: str
+) -> dict[str, str]:
+    """Add job output files from generic workflow to the job if any.
+
+    Parameters
+    ----------
+    generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
+        The generic workflow (e.g., has executable name and arguments).
+    job_name : `str`
+        Unique name for the job.
+    use_shared : `bool`
+        Whether job has access to files via shared filesystem.
+    out_prefix : `str`
+        The root directory into which all WMS-specific files are written.
+
+    Returns
+    -------
+    htc_commands : `dict` [`str`, `str`]
+        HTCondor commands for the job submission script.
+    """
+    outputs = []
+    output_remaps = []
+    for gwf_file in generic_workflow.get_job_outputs(job_name, data=True, transfer_only=True):
+        _LOG.debug("src_uri=%s", gwf_file.src_uri)
+
+        uri = Path(gwf_file.src_uri)
+        if not use_shared:
+            outputs.append(uri.name)
+            output_remaps.append(f"{uri.name}={str(uri)}")
+
+    # Set to an empty string to disable and only update if there are output
+    # files to transfer. Otherwise, HTCondor will transfer back all files in
+    # the job’s temporary working directory that have been modified or created
+    # by the job.
+    htc_commands = {"transfer_output_files": '""'}
+    if outputs:
+        htc_commands["transfer_output_files"] = ",".join(outputs)
+        _LOG.debug("transfer_output_files=%s", htc_commands["transfer_output_files"])
+
+        htc_commands["transfer_output_remaps"] = f'"{";".join(output_remaps)}"'
+        _LOG.debug("transfer_output_remaps=%s", htc_commands["transfer_output_remaps"])
+    return htc_commands
+
+
+def _get_status_from_id(
+    wms_workflow_id: str, hist: float, schedds: dict[str, htcondor.Schedd]
+) -> tuple[WmsStates, str]:
+    """Gather run information using workflow id.
+
+    Parameters
+    ----------
+    wms_workflow_id : `str`
+        Limit to specific run based on id.
+    hist : `float`
+        Limit history search to this many days.
+    schedds : `dict` [ `str`, `htcondor.Schedd` ]
+        HTCondor schedulers which to query for job information. If empty
+        dictionary, all queries will be run against the local scheduler only.
+
+    Returns
+    -------
+    state : `lsst.ctrl.bps.WmsStates`
+        Status for the corresponding run.
+    message : `str`
+        Message with extra error information.
+    """
+    _LOG.debug("_get_status_from_id: id=%s, hist=%s, schedds=%s", wms_workflow_id, hist, schedds)
+
+    message = ""
+
+    # Collect information about the job by querying HTCondor schedd and
+    # HTCondor history.
+    schedd_dag_info = _get_info_from_schedd(wms_workflow_id, hist, schedds)
+    if len(schedd_dag_info) == 1:
+        schedd_name = next(iter(schedd_dag_info))
+        dag_id = next(iter(schedd_dag_info[schedd_name]))
+        dag_ad = schedd_dag_info[schedd_name][dag_id]
+        state = _htc_status_to_wms_state(dag_ad)
+    else:
+        state = WmsStates.UNKNOWN
+        message = f"DAGMan job {wms_workflow_id} not found in queue or history.  Check id or try path."
+    return state, message
+
+
+def _get_status_from_path(wms_path: str | os.PathLike) -> tuple[WmsStates, str]:
+    """Gather run status from a given run directory.
+
+    Parameters
+    ----------
+    wms_path : `str` | `os.PathLike`
+        The directory containing the submit side files (e.g., HTCondor files).
+
+    Returns
+    -------
+    state : `lsst.ctrl.bps.WmsStates`
+        Status for the run.
+    message : `str`
+        Message to be printed.
+    """
+    wms_path = Path(wms_path).resolve()
+    message = ""
+    try:
+        wms_workflow_id, dag_ad = read_dag_log(wms_path)
+    except FileNotFoundError:
+        wms_workflow_id = MISSING_ID
+        message = f"DAGMan log not found in {wms_path}.  Check path."
+
+    if wms_workflow_id == MISSING_ID:
+        state = WmsStates.UNKNOWN
+    else:
+        state = _htc_status_to_wms_state(dag_ad[wms_workflow_id])
+
+    return state, message
+
+
 def _report_from_path(wms_path):
     """Gather run information from a given run directory.
 
@@ -1139,11 +1294,11 @@ def _get_info_from_schedd(
     ----------
     wms_workflow_id : `str`
         Limit to specific run based on id.
-    hist : `int`
+    hist : `float`
         Limit history search to this many days.
-    schedds : `dict` [ `str`, `htcondor.Schedd` ], optional
-        HTCondor schedulers which to query for job information. If None
-        (default), all queries will be run against the local scheduler only.
+    schedds : `dict` [ `str`, `htcondor.Schedd` ]
+        HTCondor schedulers which to query for job information. If empty
+        dictionary, all queries will be run against the local scheduler only.
 
     Returns
     -------
@@ -1152,6 +1307,8 @@ def _get_info_from_schedd(
         Scheduler, local HTCondor job ids are mapped to their respective
         classads.
     """
+    _LOG.debug("_get_info_from_schedd: id=%s, hist=%s, schedds=%s", wms_workflow_id, hist, schedds)
+
     dag_constraint = 'regexp("dagman$", Cmd)'
     try:
         cluster_id = int(float(wms_workflow_id))
@@ -2221,6 +2378,12 @@ def _gather_label_values(config: BpsConfig, label: str) -> dict[str, Any]:
     if found:
         values["releaseExpr"] = value
 
+    found, value = config.search("overwriteJobFiles", opt=search_opts)
+    if found:
+        values["overwriteJobFiles"] = value
+    else:
+        values["overwriteJobFiles"] = True
+
     if profile_key and profile_key in config:
         for subkey, val in config[profile_key].items():
             if subkey.startswith("+"):

lsst/ctrl/bps/htcondor/lssthtc.py

@@ -205,6 +205,7 @@ HTC_VALID_JOB_KEYS = {
     "transfer_executable",
     "transfer_input_files",
     "transfer_output_files",
+    "transfer_output_remaps",
     "request_cpus",
     "request_memory",
     "request_disk",
@@ -1376,6 +1377,7 @@ def condor_search(constraint=None, hist=None, schedds=None):
 
     job_info = condor_q(constraint=constraint, schedds=schedds)
     if hist is not None:
+        _LOG.debug("Searching history going back %s days", hist)
         epoch = (datetime.now() - timedelta(days=hist)).timestamp()
         constraint += f" && (CompletionDate >= {epoch} || JobFinishedHookDone >= {epoch})"
         hist_info = condor_history(constraint, schedds=schedds)

lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.2100"
+__version__ = "29.2025.3500"

{lsst_ctrl_bps_htcondor-29.2025.2100.dist-info → lsst_ctrl_bps_htcondor-29.2025.3500.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.2100
+Version: 29.2025.3500
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

lsst_ctrl_bps_htcondor-29.2025.3500.dist-info/RECORD

@@ -0,0 +1,19 @@
+lsst/ctrl/bps/htcondor/__init__.py,sha256=1gTmOVLJILvBqgqHVECo8uqoX8e4fiTeH_dHBUXgDvY,1417
+lsst/ctrl/bps/htcondor/final_post.sh,sha256=chfaQV6Q7rGsK-8Hx58ch52m-PofvBanrl7VwCssHec,248
+lsst/ctrl/bps/htcondor/handlers.py,sha256=2gM3Ac00in4ob9ckcP331W1LSEjs9UDKIqt4MULA4bg,11196
+lsst/ctrl/bps/htcondor/htcondor_config.py,sha256=c4lCiYEwEXFdxgbMfEkbDm4LrvkRMF31SqLtQqzqIV4,1523
+lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=WKN6bC5cJu8X9UpN5DNMDxb8V0gBOWCthvFyEWUkU7s,96467
+lsst/ctrl/bps/htcondor/lssthtc.py,sha256=MOxY30jCwbW1efgVcWtN4x13OO8monW95t6IidDf86Y,80515
+lsst/ctrl/bps/htcondor/provisioner.py,sha256=hPN8YJUtwNHQylw68kfskF1S2vCeQvztF8W0d_QKqqM,7851
+lsst/ctrl/bps/htcondor/version.py,sha256=UwJkYVJhfyZPaD0yyjLkfsS3KQL2uaxZwFIeRntHr30,55
+lsst/ctrl/bps/htcondor/etc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml,sha256=C6DKJKmKFKczukpXVXev9u1-vmv2IcgcdtjTtgJWDQM,1561
+lsst_ctrl_bps_htcondor-29.2025.3500.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
+lsst_ctrl_bps_htcondor-29.2025.3500.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps_htcondor-29.2025.3500.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps_htcondor-29.2025.3500.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps_htcondor-29.2025.3500.dist-info/METADATA,sha256=QoiKn5Fu2uQs7vEdQy8tYFqnVTW_bg8twcXFzGT8lkI,2139
+lsst_ctrl_bps_htcondor-29.2025.3500.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_ctrl_bps_htcondor-29.2025.3500.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps_htcondor-29.2025.3500.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps_htcondor-29.2025.3500.dist-info/RECORD,,

{lsst_ctrl_bps_htcondor-29.2025.2100.dist-info → lsst_ctrl_bps_htcondor-29.2025.3500.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

lsst_ctrl_bps_htcondor-29.2025.2100.dist-info/RECORD

@@ -1,19 +0,0 @@
-lsst/ctrl/bps/htcondor/__init__.py,sha256=1gTmOVLJILvBqgqHVECo8uqoX8e4fiTeH_dHBUXgDvY,1417
-lsst/ctrl/bps/htcondor/final_post.sh,sha256=chfaQV6Q7rGsK-8Hx58ch52m-PofvBanrl7VwCssHec,248
-lsst/ctrl/bps/htcondor/handlers.py,sha256=2gM3Ac00in4ob9ckcP331W1LSEjs9UDKIqt4MULA4bg,11196
-lsst/ctrl/bps/htcondor/htcondor_config.py,sha256=c4lCiYEwEXFdxgbMfEkbDm4LrvkRMF31SqLtQqzqIV4,1523
-lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=Cn4_muurzMzOSJeP3GZWpi8o_DskNL9Wgz2ZEPrNhng,90979
-lsst/ctrl/bps/htcondor/lssthtc.py,sha256=Rsfr7ZehZHiRWmF-8FMDReZQrGMtKii7CO2O8Vu9hYg,80420
-lsst/ctrl/bps/htcondor/provisioner.py,sha256=hPN8YJUtwNHQylw68kfskF1S2vCeQvztF8W0d_QKqqM,7851
-lsst/ctrl/bps/htcondor/version.py,sha256=czd5myijXlfzN1l1OFP7x6hn5ASHoE02wXhI32IqLec,55
-lsst/ctrl/bps/htcondor/etc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml,sha256=xDRts4vHKov2PE_JRh-0nF3jfuNJXtKBXZqveASp_iA,1422
-lsst_ctrl_bps_htcondor-29.2025.2100.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
-lsst_ctrl_bps_htcondor-29.2025.2100.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps_htcondor-29.2025.2100.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps_htcondor-29.2025.2100.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps_htcondor-29.2025.2100.dist-info/METADATA,sha256=zJ6h3GvvoztRNyTngXqRLYeLjJUL9wSvkZg5GrHZPVA,2139
-lsst_ctrl_bps_htcondor-29.2025.2100.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-lsst_ctrl_bps_htcondor-29.2025.2100.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps_htcondor-29.2025.2100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps_htcondor-29.2025.2100.dist-info/RECORD,,