lsst-ctrl-mpexec 29.2025.2100__tar.gz → 29.2025.3100__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. {lsst_ctrl_mpexec-29.2025.2100/python/lsst_ctrl_mpexec.egg-info → lsst_ctrl_mpexec-29.2025.3100}/PKG-INFO +1 -1
  2. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/doc/lsst.ctrl.mpexec/CHANGES.rst +40 -0
  3. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/__init__.py +1 -2
  4. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/cmd/commands.py +1 -1
  5. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py +3 -1
  6. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/run.py +2 -1
  7. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/run_qbb.py +2 -1
  8. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cmdLineFwk.py +23 -23
  9. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/execFixupDataId.py +40 -0
  10. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/executionGraphFixup.py +44 -0
  11. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/log_capture.py +40 -0
  12. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/mpGraphExecutor.py +137 -0
  13. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/quantumGraphExecutor.py +54 -0
  14. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/reports.py +67 -0
  15. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/separablePipelineExecutor.py +44 -0
  16. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/simple_pipeline_executor.py +44 -0
  17. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/singleQuantumExecutor.py +207 -0
  18. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/taskFactory.py +44 -0
  19. lsst_ctrl_mpexec-29.2025.3100/python/lsst/ctrl/mpexec/version.py +2 -0
  20. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100/python/lsst_ctrl_mpexec.egg-info}/PKG-INFO +1 -1
  21. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst_ctrl_mpexec.egg-info/SOURCES.txt +1 -8
  22. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/tests/test_cliCmdQgraph.py +1 -1
  23. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/tests/test_cmdLineFwk.py +3 -1
  24. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/dotTools.py +0 -100
  25. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/execFixupDataId.py +0 -132
  26. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/executionGraphFixup.py +0 -69
  27. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/log_capture.py +0 -226
  28. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/mpGraphExecutor.py +0 -773
  29. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/quantumGraphExecutor.py +0 -124
  30. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/reports.py +0 -243
  31. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/separablePipelineExecutor.py +0 -295
  32. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/simple_pipeline_executor.py +0 -486
  33. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/singleQuantumExecutor.py +0 -664
  34. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/taskFactory.py +0 -70
  35. lsst_ctrl_mpexec-29.2025.2100/python/lsst/ctrl/mpexec/version.py +0 -2
  36. lsst_ctrl_mpexec-29.2025.2100/tests/test_execution_storage_class_conversion.py +0 -317
  37. lsst_ctrl_mpexec-29.2025.2100/tests/test_executors.py +0 -810
  38. lsst_ctrl_mpexec-29.2025.2100/tests/test_reports.py +0 -133
  39. lsst_ctrl_mpexec-29.2025.2100/tests/test_separablePipelineExecutor.py +0 -630
  40. lsst_ctrl_mpexec-29.2025.2100/tests/test_simple_pipeline_executor.py +0 -571
  41. lsst_ctrl_mpexec-29.2025.2100/tests/test_taskFactory.py +0 -134
  42. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/COPYRIGHT +0 -0
  43. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/LICENSE +0 -0
  44. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/MANIFEST.in +0 -0
  45. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/README.rst +0 -0
  46. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/bsd_license.txt +0 -0
  47. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/doc/lsst.ctrl.mpexec/configuring-pipetask-tasks.rst +0 -0
  48. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/doc/lsst.ctrl.mpexec/index.rst +0 -0
  49. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/doc/lsst.ctrl.mpexec/pipetask.rst +0 -0
  50. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/gpl-v3.0.txt +0 -0
  51. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/pyproject.toml +0 -0
  52. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/__init__.py +0 -0
  53. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/__init__.py +0 -0
  54. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/_pipeline_graph_factory.py +0 -0
  55. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/__init__.py +0 -0
  56. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/cmd/__init__.py +0 -0
  57. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/opt/__init__.py +0 -0
  58. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/opt/arguments.py +0 -0
  59. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py +0 -0
  60. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/opt/options.py +0 -0
  61. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/pipetask.py +0 -0
  62. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/__init__.py +0 -0
  63. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/build.py +0 -0
  64. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/cleanup.py +0 -0
  65. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/confirmable.py +0 -0
  66. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/purge.py +0 -0
  67. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/qgraph.py +0 -0
  68. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/report.py +0 -0
  69. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/script/update_graph_run.py +0 -0
  70. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/cli/utils.py +0 -0
  71. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/preExecInit.py +0 -0
  72. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/py.typed +0 -0
  73. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/showInfo.py +0 -0
  74. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst/ctrl/mpexec/util.py +0 -0
  75. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst_ctrl_mpexec.egg-info/dependency_links.txt +0 -0
  76. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst_ctrl_mpexec.egg-info/entry_points.txt +0 -0
  77. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst_ctrl_mpexec.egg-info/requires.txt +0 -0
  78. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst_ctrl_mpexec.egg-info/top_level.txt +0 -0
  79. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/python/lsst_ctrl_mpexec.egg-info/zip-safe +0 -0
  80. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/setup.cfg +0 -0
  81. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/tests/test_cliCmdCleanup.py +0 -0
  82. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/tests/test_cliCmdPurge.py +0 -0
  83. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/tests/test_cliCmdReport.py +0 -0
  84. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/tests/test_cliCmdUpdateGraphRun.py +0 -0
  85. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/tests/test_cliScript.py +0 -0
  86. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/tests/test_cliUtils.py +0 -0
  87. {lsst_ctrl_mpexec-29.2025.2100 → lsst_ctrl_mpexec-29.2025.3100}/tests/test_preExecInit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lsst-ctrl-mpexec
3
- Version: 29.2025.2100
3
+ Version: 29.2025.3100
4
4
  Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
5
5
  Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
6
6
  License: BSD 3-Clause License
@@ -1,3 +1,43 @@
1
+ lsst-ctrl-mpexec v29.1.0 (2025-06-13)
2
+ =====================================
3
+
4
+ New Features
5
+ ------------
6
+
7
+ - Added the ``--view-graph`` option to ``pipetask report`` to show pipeline processing status using a graph-style visualization. (`DM-48206 <https://rubinobs.atlassian.net/browse/DM-48206>`_)
8
+ - Added support for automatically determining the output format of Mermaid visualizations based on file extension, generating either definition files or images. (`DM-48482 <https://rubinobs.atlassian.net/browse/DM-48482>`_)
9
+ - ``pipetask`` now has a ``--select-tasks`` option for subsetting the pipeline according to a pipeline graph expression.
10
+
11
+ ``pipetask`` now has a ``--data-id-table`` option for using tables of data IDs as constraints on quantum graph generation. (`DM-49949 <https://rubinobs.atlassian.net/browse/DM-49949>`_)
12
+ - Added ``use_local_butler`` method to ``SimplePipelineExecutor``.
13
+
14
+ This method allows a quantum graph to be built against a read-only butler and then executed against a local personal data repository by transferring the inputs in advance. (`SP-2103 <https://rubinobs.atlassian.net/browse/SP-2103>`_)
15
+
16
+
17
+ Bug Fixes
18
+ ---------
19
+
20
+ - Fixed failure to fail: not enough arguments for format string. (`DM-49095 <https://rubinobs.atlassian.net/browse/DM-49095>`_)
21
+ - Enabled access to the existing Mermaid visualization of the quantum graphs, allowing users to generate and log the visualization using the ``--qgraph-mermaid`` argument. (`DM-49311 <https://rubinobs.atlassian.net/browse/DM-49311>`_)
22
+ - Re-enabled skipping of already-executed quanta in automatic retries with ``run-qbb``. (`DM-50476 <https://rubinobs.atlassian.net/browse/DM-50476>`_)
23
+ - Fixed handling of storage class conversions, absent output chains, and output dimension records in ``SimplePipelineExecutor.use_local_butler``. (`DM-51359 <https://rubinobs.atlassian.net/browse/DM-51359>`_)
24
+
25
+
26
+ Performance Enhancement
27
+ -----------------------
28
+
29
+ - Added options to speed up ``pipetask report``, with some enabled by default.
30
+
31
+ By default, ``pipetask report --force-v2`` now uses quantum-backed butler and only reads metadata for quanta for which at least one predicted output was not produced (``--read-caveats=lazy``).
32
+ It can also parallelize metadata reads via the ``-j`` option. (`DM-49440 <https://rubinobs.atlassian.net/browse/DM-49440>`_)
33
+
34
+
35
+ Other Changes and Additions
36
+ ---------------------------
37
+
38
+ - Quantum metadata files now include butler metrics reporting on the time spent in butler during execution. (`DM-50490 <https://rubinobs.atlassian.net/browse/DM-50490>`_)
39
+
40
+
1
41
  lsst-ctrl-mpexec v29.0.0 (2025-03-25)
2
42
  =====================================
3
43
 
@@ -27,14 +27,13 @@
27
27
 
28
28
  from ._pipeline_graph_factory import PipelineGraphFactory
29
29
  from .cmdLineFwk import *
30
- from .dotTools import *
31
30
  from .executionGraphFixup import *
32
31
  from .mpGraphExecutor import *
33
- from .preExecInit import *
34
32
  from .quantumGraphExecutor import *
35
33
  from .reports import *
36
34
  from .separablePipelineExecutor import *
37
35
  from .simple_pipeline_executor import *
38
36
  from .singleQuantumExecutor import *
37
+ from .preExecInit import *
39
38
  from .taskFactory import *
40
39
  from .version import * # Generated by sconsUtils
@@ -36,7 +36,6 @@ from typing import Any
36
36
  import click
37
37
 
38
38
  import lsst.pipe.base.cli.opt as pipeBaseOpts
39
- from lsst.ctrl.mpexec import Report
40
39
  from lsst.ctrl.mpexec.showInfo import ShowInfo
41
40
  from lsst.daf.butler.cli.opt import (
42
41
  collections_option,
@@ -49,6 +48,7 @@ from lsst.daf.butler.cli.opt import (
49
48
  where_option,
50
49
  )
51
50
  from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap
51
+ from lsst.pipe.base.quantum_reports import Report
52
52
 
53
53
  from .. import opt as ctrlMpExecOpts
54
54
  from .. import script
@@ -27,7 +27,9 @@
27
27
 
28
28
  from types import SimpleNamespace
29
29
 
30
- from ... import CmdLineFwk, TaskFactory
30
+ from lsst.pipe.base import TaskFactory
31
+
32
+ from ... import CmdLineFwk
31
33
 
32
34
 
33
35
  def pre_exec_init_qbb(
@@ -28,9 +28,10 @@
28
28
  import logging
29
29
  from types import SimpleNamespace
30
30
 
31
+ from lsst.pipe.base import TaskFactory
31
32
  from lsst.utils.threads import disable_implicit_threading
32
33
 
33
- from ... import CmdLineFwk, TaskFactory
34
+ from ... import CmdLineFwk
34
35
 
35
36
  _log = logging.getLogger(__name__)
36
37
 
@@ -28,9 +28,10 @@
28
28
  import logging
29
29
  from types import SimpleNamespace
30
30
 
31
+ from lsst.pipe.base import TaskFactory
31
32
  from lsst.utils.threads import disable_implicit_threading
32
33
 
33
- from ... import CmdLineFwk, TaskFactory
34
+ from ... import CmdLineFwk
34
35
 
35
36
  _log = logging.getLogger(__name__)
36
37
 
@@ -71,19 +71,19 @@ from lsst.pipe.base import (
71
71
  )
72
72
  from lsst.pipe.base.all_dimensions_quantum_graph_builder import AllDimensionsQuantumGraphBuilder
73
73
  from lsst.pipe.base.dot_tools import graph2dot
74
+ from lsst.pipe.base.execution_graph_fixup import ExecutionGraphFixup
74
75
  from lsst.pipe.base.mermaid_tools import graph2mermaid
76
+ from lsst.pipe.base.mp_graph_executor import MPGraphExecutor
75
77
  from lsst.pipe.base.pipeline_graph import NodeType
78
+ from lsst.pipe.base.quantum_reports import Report
79
+ from lsst.pipe.base.single_quantum_executor import SingleQuantumExecutor
76
80
  from lsst.resources import ResourcePath
77
81
  from lsst.utils import doImportType
78
82
  from lsst.utils.logging import VERBOSE, getLogger
79
83
  from lsst.utils.threads import disable_implicit_threading
80
84
 
81
85
  from ._pipeline_graph_factory import PipelineGraphFactory
82
- from .executionGraphFixup import ExecutionGraphFixup
83
- from .mpGraphExecutor import MPGraphExecutor
84
86
  from .preExecInit import PreExecInit, PreExecInitLimited
85
- from .reports import Report
86
- from .singleQuantumExecutor import SingleQuantumExecutor
87
87
 
88
88
  # ----------------------------------
89
89
  # Local non-exported definitions --
@@ -859,24 +859,24 @@ class CmdLineFwk:
859
859
  graphFixup = self._importGraphFixup(args)
860
860
  resources = self._make_execution_resources(args)
861
861
  quantumExecutor = SingleQuantumExecutor(
862
- butler,
863
- taskFactory,
864
- skipExistingIn=args.skip_existing_in,
865
- clobberOutputs=args.clobber_outputs,
866
- enableLsstDebug=args.enableLsstDebug,
862
+ butler=butler,
863
+ task_factory=taskFactory,
864
+ skip_existing_in=args.skip_existing_in,
865
+ clobber_outputs=args.clobber_outputs,
866
+ enable_lsst_debug=args.enableLsstDebug,
867
867
  resources=resources,
868
868
  raise_on_partial_outputs=args.raise_on_partial_outputs,
869
869
  )
870
870
 
871
871
  timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
872
872
  executor = MPGraphExecutor(
873
- numProc=args.processes,
873
+ num_proc=args.processes,
874
874
  timeout=timeout,
875
- startMethod=args.start_method,
876
- quantumExecutor=quantumExecutor,
877
- failFast=args.fail_fast,
875
+ start_method=args.start_method,
876
+ quantum_executor=quantumExecutor,
877
+ fail_fast=args.fail_fast,
878
878
  pdb=args.pdb,
879
- executionGraphFixup=graphFixup,
879
+ execution_graph_fixup=graphFixup,
880
880
  )
881
881
  # Have to reset connection pool to avoid sharing connections with
882
882
  # forked processes.
@@ -1028,24 +1028,24 @@ class CmdLineFwk:
1028
1028
  resources = self._make_execution_resources(args)
1029
1029
  quantumExecutor = SingleQuantumExecutor(
1030
1030
  butler=None,
1031
- taskFactory=task_factory,
1032
- enableLsstDebug=args.enableLsstDebug,
1031
+ task_factory=task_factory,
1032
+ enable_lsst_debug=args.enableLsstDebug,
1033
1033
  limited_butler_factory=_butler_factory,
1034
1034
  resources=resources,
1035
- assumeNoExistingOutputs=args.no_existing_outputs,
1036
- skipExisting=True,
1037
- clobberOutputs=True,
1035
+ assume_no_existing_outputs=args.no_existing_outputs,
1036
+ skip_existing=True,
1037
+ clobber_outputs=True,
1038
1038
  raise_on_partial_outputs=args.raise_on_partial_outputs,
1039
1039
  job_metadata=job_metadata,
1040
1040
  )
1041
1041
 
1042
1042
  timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
1043
1043
  executor = MPGraphExecutor(
1044
- numProc=args.processes,
1044
+ num_proc=args.processes,
1045
1045
  timeout=timeout,
1046
- startMethod=args.start_method,
1047
- quantumExecutor=quantumExecutor,
1048
- failFast=args.fail_fast,
1046
+ start_method=args.start_method,
1047
+ quantum_executor=quantumExecutor,
1048
+ fail_fast=args.fail_fast,
1049
1049
  pdb=args.pdb,
1050
1050
  )
1051
1051
  try:
@@ -0,0 +1,40 @@
1
+ # This file is part of ctrl_mpexec.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (http://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
+
28
+ __all__ = ("ExecFixupDataId",)
29
+
30
+ import warnings
31
+
32
+ from lsst.pipe.base.exec_fixup_data_id import ExecFixupDataId
33
+
34
+ # TODO[DM-51962]: Remove this module.
35
+
36
+ warnings.warn(
37
+ "The execFixupDataId module has moved to lsst.pipe.base.exec_fixup_data_id. "
38
+ "This forwarding shim will be removed after v30.",
39
+ category=FutureWarning,
40
+ )
@@ -0,0 +1,44 @@
1
+ # This file is part of ctrl_mpexec.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (http://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
+
28
+ __all__ = ("ExecutionGraphFixup",)
29
+
30
+ from deprecated.sphinx import deprecated
31
+
32
+ import lsst.pipe.base.execution_graph_fixup
33
+
34
+ # TODO[DM-51962]: Remove this module.
35
+
36
+
37
+ @deprecated(
38
+ "The ExecutionGraphFixup class has moved to lsst.pipe.base.execution_graph_fixup. "
39
+ "This forwarding shim will be removed after v30.",
40
+ version="v30",
41
+ category=FutureWarning,
42
+ )
43
+ class ExecutionGraphFixup(lsst.pipe.base.execution_graph_fixup.ExecutionGraphFixup): # noqa: D101
44
+ pass
@@ -0,0 +1,40 @@
1
+ # This file is part of ctrl_mpexec.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (http://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
+
28
+ __all__ = ("LogCapture",)
29
+
30
+ import warnings
31
+
32
+ from lsst.pipe.base.log_capture import LogCapture
33
+
34
+ # TODO[DM-51962]: Remove this module.
35
+
36
+ warnings.warn(
37
+ "The log_capture module has moved to lsst.pipe.base.log_capture. "
38
+ "This forwarding shim will be removed after v30.",
39
+ category=FutureWarning,
40
+ )
@@ -0,0 +1,137 @@
1
+ # This file is part of ctrl_mpexec.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (http://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
+
28
+ __all__ = ("MPGraphExecutor", "MPGraphExecutorError", "MPTimeoutError")
29
+
30
+ from typing import Literal
31
+
32
+ from deprecated.sphinx import deprecated
33
+
34
+ import lsst.pipe.base.mp_graph_executor
35
+ from lsst.pipe.base.execution_graph_fixup import ExecutionGraphFixup
36
+ from lsst.pipe.base.quantum_graph_executor import QuantumExecutor
37
+ from lsst.pipe.base.quantum_reports import Report
38
+
39
+ # TODO[DM-51962]: Remove this module.
40
+
41
+
42
+ @deprecated(
43
+ "The MPGraphExecutor class has moved to lsst.pipe.base.mp_graph_executor. "
44
+ "This forwarding shim will be removed after v30.",
45
+ version="v30",
46
+ category=FutureWarning,
47
+ )
48
+ class MPGraphExecutor(lsst.pipe.base.mp_graph_executor.MPGraphExecutor):
49
+ """Implementation of QuantumGraphExecutor using same-host multiprocess
50
+ execution of Quanta.
51
+
52
+ This is a deprecated backwards-compatibility shim for
53
+ `lsst.pipe.base.mp_graph_executor.MPGraphExecutor`, which has
54
+ the same functionality with very minor interface changes.
55
+
56
+ Parameters
57
+ ----------
58
+ numProc : `int`
59
+ Number of processes to use for executing tasks.
60
+ timeout : `float`
61
+ Time in seconds to wait for tasks to finish.
62
+ quantumExecutor : `lsst.pipe.base.quantum_graph_executor.QuantumExecutor`
63
+ Executor for single quantum. For multiprocess-style execution when
64
+ ``num_proc`` is greater than one this instance must support pickle.
65
+ startMethod : `str`, optional
66
+ Start method from `multiprocessing` module, `None` selects the best
67
+ one for current platform.
68
+ failFast : `bool`, optional
69
+ If set to ``True`` then stop processing on first error from any task.
70
+ pdb : `str`, optional
71
+ Debugger to import and use (via the ``post_mortem`` function) in the
72
+ event of an exception.
73
+ executionGraphFixup : \
74
+ `lsst.pipe.base.execution_graph_fixup.ExecutionGraphFixup`, \
75
+ optional
76
+ Instance used for modification of execution graph.
77
+ """
78
+
79
+ def __init__(
80
+ self,
81
+ numProc: int,
82
+ timeout: float,
83
+ quantumExecutor: QuantumExecutor,
84
+ *,
85
+ startMethod: Literal["spawn"] | Literal["forkserver"] | None = None,
86
+ failFast: bool = False,
87
+ pdb: str | None = None,
88
+ executionGraphFixup: ExecutionGraphFixup | None = None,
89
+ ):
90
+ super().__init__(
91
+ num_proc=numProc,
92
+ timeout=timeout,
93
+ quantum_executor=quantumExecutor,
94
+ start_method=startMethod,
95
+ fail_fast=failFast,
96
+ pdb=pdb,
97
+ execution_graph_fixup=executionGraphFixup,
98
+ )
99
+
100
+ @property
101
+ def numProc(self) -> int:
102
+ return self._num_proc
103
+
104
+ @property
105
+ def timeout(self) -> float:
106
+ return self._timeout
107
+
108
+ @property
109
+ def quantumExecutor(self) -> QuantumExecutor:
110
+ return self._quantum_executor
111
+
112
+ @property
113
+ def failFast(self) -> bool:
114
+ return self._fail_fast
115
+
116
+ @property
117
+ def pdb(self) -> str | None:
118
+ return self._pdb
119
+
120
+ @property
121
+ def executionGraphFixup(self) -> ExecutionGraphFixup | None:
122
+ return self._execution_graph_fixup
123
+
124
+ @property
125
+ def report(self) -> Report | None:
126
+ return self._report
127
+
128
+ @property
129
+ def startMethod(self) -> str:
130
+ return self._start_method
131
+
132
+
133
+ # We can't make these forwarders warn by subclassing, because an 'except'
134
+ # statement on a derived class won't catch a base class instance.
135
+
136
+ MPGraphExecutorError = lsst.pipe.base.mp_graph_executor.MPGraphExecutorError
137
+ MPTimeoutError = lsst.pipe.base.mp_graph_executor.MPTimeoutError
@@ -0,0 +1,54 @@
1
+ # This file is part of ctrl_mpexec.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (http://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
+
28
+ __all__ = ("QuantumExecutor", "QuantumGraphExecutor")
29
+
30
+ from deprecated.sphinx import deprecated
31
+
32
+ import lsst.pipe.base.quantum_graph_executor
33
+
34
+ # TODO[DM-51962]: Remove this module.
35
+
36
+
37
+ @deprecated(
38
+ "The QuantumExecutor class has moved to lsst.pipe.base.quantum_graph_executor. "
39
+ "This forwarding shim will be removed after v30.",
40
+ version="v30",
41
+ category=FutureWarning,
42
+ )
43
+ class QuantumExecutor(lsst.pipe.base.quantum_graph_executor.QuantumExecutor): # noqa: D101
44
+ pass
45
+
46
+
47
+ @deprecated(
48
+ "The QuantumGraphExecutor class has moved to lsst.pipe.base.quantum_graph_executor. "
49
+ "This forwarding shim will be removed after v30.",
50
+ version="v30",
51
+ category=FutureWarning,
52
+ )
53
+ class QuantumGraphExecutor(lsst.pipe.base.quantum_graph_executor.QuantumGraphExecutor): # noqa: D101
54
+ pass
@@ -0,0 +1,67 @@
1
+ # This file is part of ctrl_mpexec.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (http://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
+
28
+ __all__ = ("ExceptionInfo", "ExecutionStatus", "QuantumReport", "Report")
29
+
30
+ from deprecated.sphinx import deprecated
31
+
32
+ import lsst.pipe.base.quantum_reports
33
+
34
+ # TODO[DM-51962]: Remove this module.
35
+
36
+ # We can't make this shim warn because enums can't be subclassed.
37
+ ExecutionStatus = lsst.pipe.base.quantum_reports.ExecutionStatus
38
+
39
+
40
+ @deprecated(
41
+ "The ExceptionInfo class has moved to lsst.pipe.base.quantum_reports. "
42
+ "This forwarding shim will be removed after v30.",
43
+ version="v30",
44
+ category=FutureWarning,
45
+ )
46
+ class ExceptionInfo(lsst.pipe.base.quantum_reports.ExceptionInfo): # noqa: D101
47
+ pass
48
+
49
+
50
+ @deprecated(
51
+ "The QuantumReport class has moved to lsst.pipe.base.quantum_reports. "
52
+ "This forwarding shim will be removed after v30.",
53
+ version="v30",
54
+ category=FutureWarning,
55
+ )
56
+ class QuantumReport(lsst.pipe.base.quantum_reports.QuantumReport): # noqa: D101
57
+ pass
58
+
59
+
60
+ @deprecated(
61
+ "The Report class has moved to lsst.pipe.base.quantum_reports. "
62
+ "This forwarding shim will be removed after v30.",
63
+ version="v30",
64
+ category=FutureWarning,
65
+ )
66
+ class Report(lsst.pipe.base.quantum_reports.Report): # noqa: D101
67
+ pass
@@ -0,0 +1,44 @@
1
+ # This file is part of ctrl_mpexec.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (http://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
+
28
+ __all__ = ("SeparablePipelineExecutor",)
29
+
30
+ from deprecated.sphinx import deprecated
31
+
32
+ import lsst.pipe.base.separable_pipeline_executor
33
+
34
+ # TODO[DM-51962]: Remove this module.
35
+
36
+
37
+ @deprecated(
38
+ "The SeparablePipelineExecutor class has moved to lsst.pipe.base.separable_pipeline_executor. "
39
+ "This forwarding shim will be removed after v30.",
40
+ version="v30",
41
+ category=FutureWarning,
42
+ )
43
+ class SeparablePipelineExecutor(lsst.pipe.base.separable_pipeline_executor.SeparablePipelineExecutor): # noqa: D101
44
+ pass