lsst-ctrl-bps-htcondor 29.2025.2400__tar.gz → 29.2025.3000__tar.gz

{lsst_ctrl_bps_htcondor-29.2025.2400/python/lsst_ctrl_bps_htcondor.egg-info → lsst_ctrl_bps_htcondor-29.2025.3000}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.2400
+Version: 29.2025.3000
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-29.2025.2400 → lsst_ctrl_bps_htcondor-29.2025.3000}/doc/lsst.ctrl.bps.htcondor/CHANGES.rst

@@ -1,3 +1,20 @@
+lsst-ctrl-bps-htcondor v29.1.0 (2025-06-13)
+===========================================
+
+New Features
+------------
+
+- Added capability for ``NOOP`` and ``EXTERNAL SUBDAG`` DAG nodes. (`DM-46294 <https://rubinobs.atlassian.net/browse/DM-46294>`_)
+- Added ability to add job release expression. (`DM-50614 <https://rubinobs.atlassian.net/browse/DM-50614>`_)
+- Added get_status method to ``HTCondorService`` class for quick checking of run status. (`DM-50619 <https://rubinobs.atlassian.net/browse/DM-50619>`_)
+
+
+Other Changes and Additions
+---------------------------
+
+- Explicitly define ``MaxIdle`` to workaround bug where HTCondor overrides config and environment variables when it is responsible for making DAGMan submit file (affects at least certain 24.0.x versions). (`DM-50212 <https://rubinobs.atlassian.net/browse/DM-50212>`_)
+
+
 lsst-ctrl-bps-htcondor v29.0.0 (2025-03-25)
 ===========================================
 

{lsst_ctrl_bps_htcondor-29.2025.2400 → lsst_ctrl_bps_htcondor-29.2025.3000}/doc/lsst.ctrl.bps.htcondor/userguide.rst

@@ -109,6 +109,24 @@ environment variables can be used.  Some examples:
    per cluster, per pipeline task)
 
 
+Overwriting Job Output/Error Files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When jobs are automatically retried, HTCondor keeps the same job id.
+Any existing job's output and error files are overwritten with the new
+ones.  This is not always ideal, for example, when successful parts of
+the failed job is skipped in the retry.  The ``overwriteJobFiles`` value
+(True or False) in the submit yaml controls whether to overwrite job files
+on retry.  When not overwriting job files an extra counter corresponding
+to the retry number appears in the output and error filenames even for
+successful jobs (e.g., cluster1_96908.163.0.err, cluster1_96908.163.1.err,
+cluster1_96908.163.2.err).  ``overwriteJobFiles`` defaults to True for
+payload jobs, but defaults to False for ``finalJob`` because the retries
+for it are always partial.  ``overwriteJobFiles`` can be specified in
+``pipetask`` and ``cluster`` sections as well as the ``finalJob`` section
+or yaml root.
+
+
 Glideins
 ^^^^^^^^
 

{lsst_ctrl_bps_htcondor-29.2025.2400 → lsst_ctrl_bps_htcondor-29.2025.3000}/python/lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml

@@ -38,3 +38,8 @@ provisioning:
 
 # By default, disable automatic provisioning of resources.
 provisionResources: false
+
+# Whether automatic job retries overwrite stdout/stderr of previous attempt.
+overwriteJobFiles: true
+finalJob:
+  overwriteJobFiles: false

{lsst_ctrl_bps_htcondor-29.2025.2400 → lsst_ctrl_bps_htcondor-29.2025.3000}/python/lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -706,14 +706,25 @@ def _create_job(subdir_template, cached_values, generic_workflow, gwjob, out_pre
     htc_job_cmds.update(_translate_job_cmds(cached_values, generic_workflow, gwjob))
 
     # job stdout, stderr, htcondor user log.
-    for key in ("output", "error", "log"):
-        htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).{key[:3]}"
+    for key in ("output", "error"):
+        if cached_values["overwriteJobFiles"]:
+            htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).{key[:3]}"
+        else:
+            htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).$$([NumJobStarts ?: 0]).{key[:3]}"
         _LOG.debug("HTCondor %s = %s", key, htc_job_cmds[key])
 
+    key = "log"
+    htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).{key[:3]}"
+    _LOG.debug("HTCondor %s = %s", key, htc_job_cmds[key])
+
     htc_job_cmds.update(
         _handle_job_inputs(generic_workflow, gwjob.name, cached_values["bpsUseShared"], out_prefix)
     )
 
+    htc_job_cmds.update(
+        _handle_job_outputs(generic_workflow, gwjob.name, cached_values["bpsUseShared"], out_prefix)
+    )
+
     # Add the job cmds dict to the job object.
     htc_job.add_job_cmds(htc_job_cmds)
 
@@ -946,13 +957,7 @@ def _replace_file_vars(use_shared, arguments, workflow, gwjob):
                 # Have shared filesystems and jobs can share file.
                 uri = gwfile.src_uri
             else:
-                # Taking advantage of inside knowledge.  Not future-proof.
-                # Temporary fix until have job wrapper that pulls files
-                # within job.
-                if gwfile.name == "butlerConfig" and Path(gwfile.src_uri).suffix != ".yaml":
-                    uri = "butler.yaml"
-                else:
-                    uri = os.path.basename(gwfile.src_uri)
+                uri = os.path.basename(gwfile.src_uri)
         else:  # Using push transfer
             uri = os.path.basename(gwfile.src_uri)
         arguments = arguments.replace(f"<FILE:{gwfile.name}>", uri)
@@ -1001,7 +1006,9 @@ def _replace_cmd_vars(arguments, gwjob):
     return arguments
 
 
-def _handle_job_inputs(generic_workflow: GenericWorkflow, job_name: str, use_shared: bool, out_prefix: str):
+def _handle_job_inputs(
+    generic_workflow: GenericWorkflow, job_name: str, use_shared: bool, out_prefix: str
+) -> dict[str, str]:
     """Add job input files from generic workflow to job.
 
     Parameters
@@ -1020,7 +1027,6 @@ def _handle_job_inputs(generic_workflow: GenericWorkflow, job_name: str, use_sha
     htc_commands : `dict` [`str`, `str`]
         HTCondor commands for the job submission script.
     """
-    htc_commands = {}
     inputs = []
     for gwf_file in generic_workflow.get_job_inputs(job_name, data=True, transfer_only=True):
         _LOG.debug("src_uri=%s", gwf_file.src_uri)
@@ -1030,38 +1036,68 @@ def _handle_job_inputs(generic_workflow: GenericWorkflow, job_name: str, use_sha
         # Note if use_shared and job_shared, don't need to transfer file.
 
         if not use_shared:  # Copy file using push to job
-            inputs.append(str(uri.relative_to(out_prefix)))
+            inputs.append(str(uri))
         elif not gwf_file.job_shared:  # Jobs require own copy
             # if using shared filesystem, but still need copy in job. Use
             # HTCondor's curl plugin for a local copy.
-
-            # Execution butler is represented as a directory which the
-            # curl plugin does not handle. Taking advantage of inside
-            # knowledge for temporary fix until have job wrapper that pulls
-            # files within job.
-            if gwf_file.name == "butlerConfig":
-                # The execution butler directory doesn't normally exist until
-                # the submit phase so checking for suffix instead of using
-                # is_dir().  If other non-yaml file exists they would have a
-                # different gwf_file.name.
-                if uri.suffix == ".yaml":  # Single file, so just copy.
-                    inputs.append(f"file://{uri}")
-                else:
-                    inputs.append(f"file://{uri / 'butler.yaml'}")
-                    inputs.append(f"file://{uri / 'gen3.sqlite3'}")
-            elif uri.is_dir():
+            if uri.is_dir():
                 raise RuntimeError(
                     f"HTCondor plugin cannot transfer directories locally within job {gwf_file.src_uri}"
                 )
-            else:
-                inputs.append(f"file://{uri}")
+            inputs.append(f"file://{uri}")
 
+    htc_commands = {}
     if inputs:
         htc_commands["transfer_input_files"] = ",".join(inputs)
         _LOG.debug("transfer_input_files=%s", htc_commands["transfer_input_files"])
     return htc_commands
 
 
+def _handle_job_outputs(
+    generic_workflow: GenericWorkflow, job_name: str, use_shared: bool, out_prefix: str
+) -> dict[str, str]:
+    """Add job output files from generic workflow to the job if any.
+
+    Parameters
+    ----------
+    generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
+        The generic workflow (e.g., has executable name and arguments).
+    job_name : `str`
+        Unique name for the job.
+    use_shared : `bool`
+        Whether job has access to files via shared filesystem.
+    out_prefix : `str`
+        The root directory into which all WMS-specific files are written.
+
+    Returns
+    -------
+    htc_commands : `dict` [`str`, `str`]
+        HTCondor commands for the job submission script.
+    """
+    outputs = []
+    output_remaps = []
+    for gwf_file in generic_workflow.get_job_outputs(job_name, data=True, transfer_only=True):
+        _LOG.debug("src_uri=%s", gwf_file.src_uri)
+
+        uri = Path(gwf_file.src_uri)
+        if not use_shared:
+            outputs.append(uri.name)
+            output_remaps.append(f"{uri.name}={str(uri)}")
+
+    # Set to an empty string to disable and only update if there are output
+    # files to transfer. Otherwise, HTCondor will transfer back all files in
+    # the job’s temporary working directory that have been modified or created
+    # by the job.
+    htc_commands = {"transfer_output_files": '""'}
+    if outputs:
+        htc_commands["transfer_output_files"] = ",".join(outputs)
+        _LOG.debug("transfer_output_files=%s", htc_commands["transfer_output_files"])
+
+        htc_commands["transfer_output_remaps"] = f'"{";".join(output_remaps)}"'
+        _LOG.debug("transfer_output_remaps=%s", htc_commands["transfer_output_remaps"])
+    return htc_commands
+
+
 def _get_status_from_id(
     wms_workflow_id: str, hist: float, schedds: dict[str, htcondor.Schedd]
 ) -> tuple[WmsStates, str]:
@@ -2342,6 +2378,12 @@ def _gather_label_values(config: BpsConfig, label: str) -> dict[str, Any]:
     if found:
         values["releaseExpr"] = value
 
+    found, value = config.search("overwriteJobFiles", opt=search_opts)
+    if found:
+        values["overwriteJobFiles"] = value
+    else:
+        values["overwriteJobFiles"] = True
+
     if profile_key and profile_key in config:
         for subkey, val in config[profile_key].items():
             if subkey.startswith("+"):

{lsst_ctrl_bps_htcondor-29.2025.2400 → lsst_ctrl_bps_htcondor-29.2025.3000}/python/lsst/ctrl/bps/htcondor/lssthtc.py

@@ -205,6 +205,7 @@ HTC_VALID_JOB_KEYS = {
     "transfer_executable",
     "transfer_input_files",
     "transfer_output_files",
+    "transfer_output_remaps",
     "request_cpus",
     "request_memory",
     "request_disk",

lsst_ctrl_bps_htcondor-29.2025.3000/python/lsst/ctrl/bps/htcondor/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.3000"

{lsst_ctrl_bps_htcondor-29.2025.2400 → lsst_ctrl_bps_htcondor-29.2025.3000/python/lsst_ctrl_bps_htcondor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.2400
+Version: 29.2025.3000
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-29.2025.2400 → lsst_ctrl_bps_htcondor-29.2025.3000}/tests/test_htcondor_service.py

@@ -40,13 +40,14 @@ from lsst.ctrl.bps import (
     BPS_SEARCH_ORDER,
     BpsConfig,
     GenericWorkflowExec,
+    GenericWorkflowFile,
     GenericWorkflowJob,
     WmsSpecificInfo,
     WmsStates,
 )
 from lsst.ctrl.bps.htcondor import htcondor_service, lssthtc
 from lsst.ctrl.bps.htcondor.htcondor_config import HTC_DEFAULTS_URI
-from lsst.ctrl.bps.tests.gw_test_utils import make_3_label_workflow_groups_sort
+from lsst.ctrl.bps.tests.gw_test_utils import make_3_label_workflow, make_3_label_workflow_groups_sort
 from lsst.utils.tests import temporaryDirectory
 
 logger = logging.getLogger("lsst.ctrl.bps.htcondor")
@@ -1233,7 +1234,11 @@ class GatherLabelValuesTestCase(unittest.TestCase):
         config = BpsConfig(
             {
                 "cluster": {
-                    "label1": {"releaseExpr": "cluster_val", "profile": {"condor": {"prof_val1": 3}}}
+                    "label1": {
+                        "releaseExpr": "cluster_val",
+                        "overwriteJobFiles": False,
+                        "profile": {"condor": {"prof_val1": 3}},
+                    }
                 },
                 "pipetask": {"label1": {"releaseExpr": "pipetask_val"}},
             },
@@ -1242,14 +1247,26 @@ class GatherLabelValuesTestCase(unittest.TestCase):
             wms_service_class_fqn="lsst.ctrl.bps.htcondor.HTCondorService",
         )
         results = htcondor_service._gather_label_values(config, label)
-        self.assertEqual(results, {"attrs": {}, "profile": {"prof_val1": 3}, "releaseExpr": "cluster_val"})
+        self.assertEqual(
+            results,
+            {
+                "attrs": {},
+                "profile": {"prof_val1": 3},
+                "releaseExpr": "cluster_val",
+                "overwriteJobFiles": False,
+            },
+        )
 
     def testPipetaskLabel(self):
         label = "label1"
         config = BpsConfig(
             {
                 "pipetask": {
-                    "label1": {"releaseExpr": "pipetask_val", "profile": {"condor": {"prof_val1": 3}}}
+                    "label1": {
+                        "releaseExpr": "pipetask_val",
+                        "overwriteJobFiles": False,
+                        "profile": {"condor": {"prof_val1": 3}},
+                    }
                 }
             },
             search_order=BPS_SEARCH_ORDER,
@@ -1257,7 +1274,15 @@ class GatherLabelValuesTestCase(unittest.TestCase):
             wms_service_class_fqn="lsst.ctrl.bps.htcondor.HTCondorService",
         )
         results = htcondor_service._gather_label_values(config, label)
-        self.assertEqual(results, {"attrs": {}, "profile": {"prof_val1": 3}, "releaseExpr": "pipetask_val"})
+        self.assertEqual(
+            results,
+            {
+                "attrs": {},
+                "profile": {"prof_val1": 3},
+                "releaseExpr": "pipetask_val",
+                "overwriteJobFiles": False,
+            },
+        )
 
     def testNoSection(self):
         label = "notThere"
@@ -1268,7 +1293,31 @@ class GatherLabelValuesTestCase(unittest.TestCase):
             wms_service_class_fqn="lsst.ctrl.bps.htcondor.HTCondorService",
         )
         results = htcondor_service._gather_label_values(config, label)
-        self.assertEqual(results, {"attrs": {}, "profile": {}})
+        self.assertEqual(results, {"attrs": {}, "profile": {}, "overwriteJobFiles": True})
+
+    def testNoOverwriteSpecified(self):
+        label = "notthere"
+        config = BpsConfig(
+            {},
+            search_order=BPS_SEARCH_ORDER,
+            defaults={},
+            wms_service_class_fqn="lsst.ctrl.bps.htcondor.HTCondorService",
+        )
+        results = htcondor_service._gather_label_values(config, label)
+        self.assertEqual(results, {"attrs": {}, "profile": {}, "overwriteJobFiles": True})
+
+    def testFinalJob(self):
+        label = "finalJob"
+        config = BpsConfig(
+            {"finalJob": {"profile": {"condor": {"prof_val2": 6, "+attr_val1": 5}}}},
+            search_order=BPS_SEARCH_ORDER,
+            defaults=BPS_DEFAULTS,
+            wms_service_class_fqn="lsst.ctrl.bps.htcondor.HTCondorService",
+        )
+        results = htcondor_service._gather_label_values(config, label)
+        self.assertEqual(
+            results, {"attrs": {"attr_val1": 5}, "profile": {"prof_val2": 6}, "overwriteJobFiles": False}
+        )
 
 
 class CreateCheckJobTestCase(unittest.TestCase):
@@ -1418,5 +1467,117 @@ class GetStatusFromPathTestCase(unittest.TestCase):
         self.assertEqual(message, "")
 
 
+class HandleJobOutputsTestCase(unittest.TestCase):
+    """Test _handle_job_outputs function."""
+
+    def setUp(self):
+        self.job_name = "test_job"
+        self.out_prefix = "/test/prefix"
+
+    def tearDown(self):
+        pass
+
+    def testNoOutputsSharedFilesystem(self):
+        """Test with shared filesystem and no outputs."""
+        mock_workflow = unittest.mock.Mock()
+        mock_workflow.get_job_outputs.return_value = []
+
+        result = htcondor_service._handle_job_outputs(mock_workflow, self.job_name, True, self.out_prefix)
+
+        self.assertEqual(result, {"transfer_output_files": '""'})
+
+    def testWithOutputsSharedFilesystem(self):
+        """Test with shared filesystem and outputs present (still empty)."""
+        mock_workflow = unittest.mock.Mock()
+        mock_workflow.get_job_outputs.return_value = [
+            GenericWorkflowFile(name="output.txt", src_uri="/path/to/output.txt")
+        ]
+
+        result = htcondor_service._handle_job_outputs(mock_workflow, self.job_name, True, self.out_prefix)
+
+        self.assertEqual(result, {"transfer_output_files": '""'})
+
+    def testNoOutputsNoSharedFilesystem(self):
+        """Test without shared filesystem and no outputs."""
+        mock_workflow = unittest.mock.Mock()
+        mock_workflow.get_job_outputs.return_value = []
+
+        result = htcondor_service._handle_job_outputs(mock_workflow, self.job_name, False, self.out_prefix)
+
+        self.assertEqual(result, {"transfer_output_files": '""'})
+
+    def testWithAnOutputNoSharedFilesystem(self):
+        """Test without shared filesystem and single output file."""
+        mock_workflow = unittest.mock.Mock()
+        mock_workflow.get_job_outputs.return_value = [
+            GenericWorkflowFile(name="output.txt", src_uri="/path/to/output.txt")
+        ]
+
+        result = htcondor_service._handle_job_outputs(mock_workflow, self.job_name, False, self.out_prefix)
+
+        expected = {
+            "transfer_output_files": "output.txt",
+            "transfer_output_remaps": '"output.txt=/path/to/output.txt"',
+        }
+        self.assertEqual(result, expected)
+
+    def testWithOutputsNoSharedFilesystem(self):
+        """Test without shared filesystem and multiple output files."""
+        mock_workflow = unittest.mock.Mock()
+        mock_workflow.get_job_outputs.return_value = [
+            GenericWorkflowFile(name="output1.txt", src_uri="/path/output1.txt"),
+            GenericWorkflowFile(name="output2.txt", src_uri="/another/path/output2.txt"),
+        ]
+
+        result = htcondor_service._handle_job_outputs(mock_workflow, self.job_name, False, self.out_prefix)
+
+        expected = {
+            "transfer_output_files": "output1.txt,output2.txt",
+            "transfer_output_remaps": '"output1.txt=/path/output1.txt;output2.txt=/another/path/output2.txt"',
+        }
+        self.assertEqual(result, expected)
+
+    @unittest.mock.patch("lsst.ctrl.bps.htcondor.htcondor_service._LOG")
+    def testLogging(self, mock_log):
+        mock_workflow = unittest.mock.Mock()
+        mock_workflow.get_job_outputs.return_value = [
+            GenericWorkflowFile(name="output.txt", src_uri="/path/to/output.txt")
+        ]
+
+        htcondor_service._handle_job_outputs(mock_workflow, self.job_name, False, self.out_prefix)
+
+        self.assertTrue(mock_log.debug.called)
+        debug_calls = mock_log.debug.call_args_list
+        self.assertTrue(any("src_uri=" in str(call) for call in debug_calls))
+        self.assertTrue(any("transfer_output_files=" in str(call) for call in debug_calls))
+        self.assertTrue(any("transfer_output_remaps=" in str(call) for call in debug_calls))
+
+
+class CreateJobTestCase(unittest.TestCase):
+    """Test _create_job function."""
+
+    def setUp(self):
+        self.generic_workflow = make_3_label_workflow("test1", True)
+
+    def testNoOverwrite(self):
+        template = "{label}/{tract}/{patch}/{band}/{subfilter}/{physical_filter}/{visit}/{exposure}"
+        cached_values = {
+            "bpsUseShared": True,
+            "overwriteJobFiles": False,
+            "memoryLimit": 491520,
+            "profile": {},
+            "attrs": {},
+        }
+        gwjob = self.generic_workflow.get_final()
+        out_prefix = "submit"
+        htc_job = htcondor_service._create_job(
+            template, cached_values, self.generic_workflow, gwjob, out_prefix
+        )
+        self.assertEqual(htc_job.name, gwjob.name)
+        self.assertEqual(htc_job.label, gwjob.label)
+        self.assertIn("NumJobStarts", htc_job.cmds["output"])
+        self.assertIn("NumJobStarts", htc_job.cmds["error"])
+
+
 if __name__ == "__main__":
     unittest.main()

lsst_ctrl_bps_htcondor-29.2025.2400/python/lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2400"