lsst-ctrl-bps 29.2025.2800__py3-none-any.whl → 29.2025.3000__py3-none-any.whl

lsst/ctrl/bps/construct.py

@@ -31,9 +31,16 @@ __all__ = ["construct"]
 
 import logging
 import shutil
+from collections.abc import Callable
 from pathlib import Path
 
-from lsst.ctrl.bps import BpsConfig, GenericWorkflow, GenericWorkflowExec, GenericWorkflowJob
+from lsst.ctrl.bps import (
+    BpsConfig,
+    GenericWorkflow,
+    GenericWorkflowExec,
+    GenericWorkflowFile,
+    GenericWorkflowJob,
+)
 from lsst.ctrl.bps.transform import _get_job_values
 
 _LOG = logging.getLogger(__name__)
@@ -73,7 +80,7 @@ def create_custom_workflow(config: BpsConfig) -> tuple[GenericWorkflow, BpsConfi
     generic_workflow_config : `lsst.ctrl.BpsConfig`
         Configuration to accompany created generic workflow.
     """
-    gwjob = create_custom_job(config)
+    gwjob, inputs, outputs = create_custom_job(config)
 
     _, name = config.search("uniqProcName", opt={"required": True})
     generic_workflow = GenericWorkflow(name)
@@ -90,6 +97,10 @@ def create_custom_workflow(config: BpsConfig) -> tuple[GenericWorkflow, BpsConfi
             "bps_runsite": config["computeSite"],
         }
     )
+    if inputs:
+        generic_workflow.add_job_inputs(gwjob.name, inputs)
+    if outputs:
+        generic_workflow.add_job_outputs(gwjob.name, outputs)
 
     generic_workflow_config = BpsConfig(config)
     generic_workflow_config["workflowName"] = config["uniqProcName"]
@@ -98,7 +109,9 @@ def create_custom_workflow(config: BpsConfig) -> tuple[GenericWorkflow, BpsConfi
     return generic_workflow, generic_workflow_config
 
 
-def create_custom_job(config: BpsConfig) -> GenericWorkflowJob:
+def create_custom_job(
+    config: BpsConfig,
+) -> tuple[GenericWorkflowJob, list[GenericWorkflowFile], list[GenericWorkflowFile]]:
     """Create a job that will run a custom command or script.
 
     Parameters
@@ -110,6 +123,10 @@ def create_custom_job(config: BpsConfig) -> GenericWorkflowJob:
     -------
     job : `lsst.ctrl.bps.GenericWorkflowJob`
         A custom job responsible for running the command.
+    inputs : `list` [`lsst.ctrl.bps.GenericWorkflowFile`]
+        List of job's input files, empty if the job has no input files.
+    outputs : `list` [`lsst.ctrl.bps.GenericWorkflowFile`]
+        List of job's output files, empty if the job has no output files.
     """
     prefix = Path(config["submitPath"])
     job_label = "customJob"
@@ -135,6 +152,92 @@ def create_custom_job(config: BpsConfig) -> GenericWorkflowJob:
     job.executable = GenericWorkflowExec(
         name=script_name, src_uri=str(prefix / script_name), transfer_executable=True
     )
-    job.arguments = config[f".{job_label}.arguments"]
+    _, job.arguments = config.search("arguments", opt=search_opts | {"replaceVars": False})
 
-    return job
+    inputs = []
+    found, mapping = config.search("inputs", opt=search_opts)
+    if found:
+        inputs = create_job_files(mapping, prefix, path_creator=create_input_path)
+
+    outputs = []
+    found, mapping = config.search("outputs", opt=search_opts)
+    if found:
+        outputs = create_job_files(mapping, prefix, path_creator=create_output_path)
+
+    for gwfile in inputs + outputs:
+        job.arguments = job.arguments.replace(f"{{{gwfile.name}}}", f"<FILE:{gwfile.name}>")
+
+    return job, inputs, outputs
+
+
+def create_job_files(
+    file_specs: BpsConfig, prefix: str | Path, path_creator: Callable[[Path, Path], Path]
+) -> list[GenericWorkflowFile]:
+    """Create files for a job.
+
+    Parameters
+    ----------
+    file_specs : `lsst.ctrl.bps.BpsConfig`
+        The mapping between file keys and file paths.
+    prefix : `str` | `pathlib.Path`
+        The root directory to which the files will be written.
+    path_creator : `Callable` [[`Path`, `Path`], `Path`]
+        File category that determines actions that need to be taken during
+        file creation.
+
+    Returns
+    -------
+    gwfiles : `list` [`lsst.ctrl.bps.GenericWorkflowFile`]
+        List of files created for the job.
+    """
+    prefix = Path(prefix)
+
+    gwfiles = []
+    for key, path in file_specs.items():
+        src = Path(path)
+        dest = path_creator(src, prefix)
+        gwfiles.append(GenericWorkflowFile(name=key, src_uri=str(dest), wms_transfer=True))
+    return gwfiles
+
+
+def create_input_path(path: Path, prefix: Path) -> Path:
+    """Process an input path.
+
+    Parameters
+    ----------
+    path : `pathlib.Path`
+        The input path.
+    prefix : `pathlib.Path`
+        The root directory to which the file will be written.
+
+    Raises
+    ------
+    ValueError
+        Raised if the input path does not exist or is a directory.
+    """
+    if path.exists():
+        if path.is_dir():
+            raise ValueError(f"input path '{path} is a directory, must be file")
+    else:
+        raise ValueError(f"input path '{path}' does not exist")
+    dest = prefix / path.name
+    shutil.copy2(path, dest)
+    return dest
+
+
+def create_output_path(path: Path, prefix: Path) -> Path:
+    """Process an output path.
+
+    Parameters
+    ----------
+    path : `pathlib.Path`
+        The output path.
+    prefix : `pathlib.Path`
+        The root directory to which the file will be written.
+    """
+    if path.is_absolute():
+        dest = path
+    else:
+        dest = prefix / path
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    return dest

lsst/ctrl/bps/transform.py

@@ -316,13 +316,7 @@ def _fill_arguments(use_shared, generic_workflow, arguments, cmdvals):
                 # Have shared filesystems and jobs can share file.
                 uri = gwfile.src_uri
             else:
-                # Taking advantage of inside knowledge.  Not future-proof.
-                # Temporary fix until have job wrapper that pulls files
-                # within job.
-                if gwfile.name == "butlerConfig" and os.path.splitext(gwfile.src_uri)[1] != ".yaml":
-                    uri = "butler.yaml"
-                else:
-                    uri = os.path.basename(gwfile.src_uri)
+                uri = os.path.basename(gwfile.src_uri)
         else:  # Using push transfer
             uri = os.path.basename(gwfile.src_uri)
 

lsst/ctrl/bps/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.2800"
+__version__ = "29.2025.3000"

{lsst_ctrl_bps-29.2025.2800.dist-info → lsst_ctrl_bps-29.2025.3000.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps
-Version: 29.2025.2800
+Version: 29.2025.3000
 Summary: Pluggable execution of workflow graphs from Rubin pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps-29.2025.2800.dist-info → lsst_ctrl_bps-29.2025.3000.dist-info}/RECORD

@@ -9,7 +9,7 @@ lsst/ctrl/bps/bps_utils.py,sha256=-w0HSUXKw8jgJFl28yzwJuuqLqT_UOw12_XQBELUWrE,12
 lsst/ctrl/bps/cancel.py,sha256=mAdBi-oUpepyo-1MCqx_I34dbm6cqT0VJu3d2-y9T2Y,3317
 lsst/ctrl/bps/clustered_quantum_graph.py,sha256=mBf8s_DlTzGCFq7aAKmD1cAXq6Cqr8QxjwpgXPOQcRc,18765
 lsst/ctrl/bps/constants.py,sha256=dsnsNMqwU6Xl_ln6nQ0PmxsghlbBY9mLJWf2rlzYayQ,1733
-lsst/ctrl/bps/construct.py,sha256=a5Gb_oinowb7m9rLsk0R_hz8GFu9WGCzDd_QWbsK0Mk,4944
+lsst/ctrl/bps/construct.py,sha256=o-JhTF_eehRiIxy3xe2UgBISyKNBg-kduthGiqfa50g,8100
 lsst/ctrl/bps/drivers.py,sha256=M_zWXvzK23MBBNvqdbJCnA_9DUCoFznCjWfwQw9BO6Q,22512
 lsst/ctrl/bps/generic_workflow.py,sha256=BwEFBudncziHTZaTTknOQroY_a-33mGf-mq9PzxH8G0,53720
 lsst/ctrl/bps/initialize.py,sha256=DCQwB9ZSo_36Ncs8jMJwxKZcYCu46jjww6rzyiuMFKY,7554
@@ -21,8 +21,8 @@ lsst/ctrl/bps/report.py,sha256=6qCt6-0apuVZrevtQzJLRmQc_ve_uPyNH3HkgC5KSjc,7898
 lsst/ctrl/bps/restart.py,sha256=yVwxeviLiehyIfPmwU-H3tJ9ou7OWZZcrNf8PMxjr8o,2298
 lsst/ctrl/bps/status.py,sha256=Lrt0cAqROv77B8UvYXGimCa4cDHBD1N0K2Xx7oS6fXk,3362
 lsst/ctrl/bps/submit.py,sha256=Ev-yhcoZwtBPIo5bRt_4XFJRtgQBd8JHUurEfn01HpU,2880
-lsst/ctrl/bps/transform.py,sha256=jGEA3YGIQIJiLv-4Qm00WPNW0EEKughX7lcoJRZhsKU,35600
-lsst/ctrl/bps/version.py,sha256=zkbtcIsz6tbu7rmamcut51DoG62vyYwam95urGw5Lis,55
+lsst/ctrl/bps/transform.py,sha256=P3dGwKStoCLOn7F2Oez6B__f7bGV5jlUF_Vj9lJQcU0,35256
+lsst/ctrl/bps/version.py,sha256=GKnpddvvmGJ8yPsnT--mhT32jeA31BaGZsJcw6cTK7k,55
 lsst/ctrl/bps/wms_service.py,sha256=l3T6i1MG72dhHY0KXMUlBjWUpCLOfaySs7o2W8oCwhs,18891
 lsst/ctrl/bps/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/ctrl/bps/cli/bps.py,sha256=CHfAL-U4mSi-FqeGKtmkX5nI5H9gkRre4XEWNVdeMRk,2559
@@ -35,13 +35,13 @@ lsst/ctrl/bps/cli/opt/options.py,sha256=pZQpjJ2Vrx6kYJGs5vVERBMFstPocwBXHffxkWkm
 lsst/ctrl/bps/etc/bps_defaults.yaml,sha256=UwKAKWl09-Xjtl6kiz-AiXA9SDczeCvPGRXOPG3fzb8,4828
 lsst/ctrl/bps/tests/config_test_utils.py,sha256=WM8Vrigk4OO0TBoL1A73a6hLhf2a6-ACD20fROJ0U7A,3537
 lsst/ctrl/bps/tests/gw_test_utils.py,sha256=zVVQqzwSiQgPgk9TnqDzgR7uDnaTMeuBLYKA8vOp5RI,22452
-lsst_ctrl_bps-29.2025.2800.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
-lsst_ctrl_bps-29.2025.2800.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps-29.2025.2800.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps-29.2025.2800.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps-29.2025.2800.dist-info/METADATA,sha256=z5rzEOPfSHOHJYHZVk8g8Fs_094g69mpw0ZqiaLaeJY,2190
-lsst_ctrl_bps-29.2025.2800.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_ctrl_bps-29.2025.2800.dist-info/entry_points.txt,sha256=d6FhN79h7s9frdBI7YkScsGEInwpGGub49pAjXWbIbI,51
-lsst_ctrl_bps-29.2025.2800.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps-29.2025.2800.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps-29.2025.2800.dist-info/RECORD,,
+lsst_ctrl_bps-29.2025.3000.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
+lsst_ctrl_bps-29.2025.3000.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps-29.2025.3000.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps-29.2025.3000.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps-29.2025.3000.dist-info/METADATA,sha256=zu22RHR5V7RT4op1ilcxRCPPM89Bh347DIiKurkIBXs,2190
+lsst_ctrl_bps-29.2025.3000.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_ctrl_bps-29.2025.3000.dist-info/entry_points.txt,sha256=d6FhN79h7s9frdBI7YkScsGEInwpGGub49pAjXWbIbI,51
+lsst_ctrl_bps-29.2025.3000.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps-29.2025.3000.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps-29.2025.3000.dist-info/RECORD,,