lsst-pipe-base 29.2025.4700__tar.gz → 29.2025.4900__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lsst_pipe_base-29.2025.4700/python/lsst_pipe_base.egg-info → lsst_pipe_base-29.2025.4900}/PKG-INFO +1 -1
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/caching_limited_butler.py +3 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/connections.py +11 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/aggregator/_ingester.py +14 -3
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/aggregator/_scanner.py +14 -3
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/script/register_instrument.py +4 -4
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/script/retrieve_artifacts_for_quanta.py +5 -6
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/script/transfer_from_graph.py +42 -42
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/script/zip_from_graph.py +7 -8
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/simple_pipeline_executor.py +4 -3
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/single_quantum_executor.py +20 -2
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/mocks/_repo.py +44 -16
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/simpleQGraph.py +43 -35
- lsst_pipe_base-29.2025.4900/python/lsst/pipe/base/version.py +2 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900/python/lsst_pipe_base.egg-info}/PKG-INFO +1 -1
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst_pipe_base.egg-info/SOURCES.txt +1 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_adjust_all_quanta.py +1 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_aggregator.py +3 -1
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_caching_limited_butler.py +1 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_config_formatter.py +2 -1
- lsst_pipe_base-29.2025.4900/tests/test_deferredDatasetRef.py +69 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_execution_reports.py +5 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_execution_storage_class_conversion.py +3 -1
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_graphBuilder.py +13 -3
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_graph_walker.py +1 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_init_output_run.py +7 -3
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_instrument.py +9 -3
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_mp_graph_executor.py +13 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_predicted_qg.py +1 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_qg_builder_dimensions.py +1 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_separable_pipeline_executor.py +10 -9
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_simple_pipeline_executor.py +11 -8
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_single_quantum_executor.py +3 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_task_factory.py +1 -0
- lsst_pipe_base-29.2025.4700/python/lsst/pipe/base/version.py +0 -2
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/COPYRIGHT +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/LICENSE +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/MANIFEST.in +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/README.md +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/bsd_license.txt +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/doc/lsst.pipe.base/CHANGES.rst +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/doc/lsst.pipe.base/creating-a-pipeline.rst +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/doc/lsst.pipe.base/creating-a-pipelinetask.rst +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/doc/lsst.pipe.base/creating-a-task.rst +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/doc/lsst.pipe.base/index.rst +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/doc/lsst.pipe.base/task-framework-overview.rst +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/doc/lsst.pipe.base/task-retargeting-howto.rst +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/doc/lsst.pipe.base/testing-a-pipeline-task.rst +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/doc/lsst.pipe.base/testing-pipelines-with-mocks.rst +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/doc/lsst.pipe.base/working-with-pipeline-graphs.rst +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/gpl-v3.0.txt +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/pyproject.toml +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/_datasetQueryConstraints.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/_dataset_handle.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/_instrument.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/_observation_dimension_packer.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/_quantumContext.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/_status.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/_task_metadata.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/all_dimensions_quantum_graph_builder.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/automatic_connection_constants.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/cli/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/cli/_get_cli_subcommands.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/cli/cmd/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/cli/cmd/commands.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/cli/opt/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/cli/opt/arguments.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/cli/opt/options.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/config.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/configOverrides.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/connectionTypes.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/dot_tools.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/exec_fixup_data_id.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/execution_graph_fixup.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/execution_reports.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/formatters/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/formatters/pexConfig.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/graph/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/graph/_implDetails.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/graph/_loadHelpers.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/graph/_versionDeserializers.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/graph/graph.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/graph/graphSummary.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/graph/quantumNode.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/graph_walker.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/log_capture.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/mermaid_tools.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/mp_graph_executor.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipelineIR.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipelineTask.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/__main__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/_dataset_types.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/_edges.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/_exceptions.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/_mapping_views.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/_nodes.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/_pipeline_graph.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/_task_subsets.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/_tasks.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/expressions.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/io.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/visualization/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/visualization/_dot.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/visualization/_formatting.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/visualization/_layout.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/visualization/_merge.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/visualization/_mermaid.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/visualization/_options.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/visualization/_printer.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/visualization/_show.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/prerequisite_helpers.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/py.typed +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/_common.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/_multiblock.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/_predicted.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/_provenance.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/aggregator/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/aggregator/_communicators.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/aggregator/_config.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/aggregator/_progress.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/aggregator/_structs.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/aggregator/_writer.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph/visualization.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph_builder.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph_executor.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_graph_skeleton.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_provenance_graph.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/quantum_reports.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/resource_usage.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/script/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/script/utils.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/separable_pipeline_executor.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/struct.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/task.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/taskFactory.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/testUtils.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/in_memory_limited_butler.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/mocks/__init__.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/mocks/_data_id_match.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/mocks/_pipeline_task.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/mocks/_storage_class.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/no_dimensions.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/pipelineStepTester.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/tests/util.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/utils.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst_pipe_base.egg-info/dependency_links.txt +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst_pipe_base.egg-info/entry_points.txt +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst_pipe_base.egg-info/requires.txt +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst_pipe_base.egg-info/top_level.txt +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst_pipe_base.egg-info/zip-safe +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/setup.cfg +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_cliCmdRegisterInstrument.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_configOverrides.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_connections.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_dataid_match.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_dataset_handle.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_dot_tools.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_dynamic_connections.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_mermaid.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_pipeline.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_pipelineIR.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_pipelineLoadSubset.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_pipelineTask.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_pipeline_graph.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_pipeline_graph_expressions.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_quantumGraph.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_quantum_provenance_graph.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_quantum_reports.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_quantum_success_caveats.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_script_utils.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_struct.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_task.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_taskmetadata.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_testUtils.py +0 -0
- {lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_utils.py +0 -0
{lsst_pipe_base-29.2025.4700/python/lsst_pipe_base.egg-info → lsst_pipe_base-29.2025.4900}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-pipe-base
|
|
3
|
-
Version: 29.2025.
|
|
3
|
+
Version: 29.2025.4900
|
|
4
4
|
Summary: Pipeline infrastructure for the Rubin Science Pipelines.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License-Expression: BSD-3-Clause OR GPL-3.0-or-later
|
|
@@ -90,6 +90,9 @@ class CachingLimitedButler(LimitedButler):
|
|
|
90
90
|
self._cache: dict[str, tuple[DatasetId, InMemoryDatasetHandle]] = {}
|
|
91
91
|
self._no_copy_on_cache = no_copy_on_cache
|
|
92
92
|
|
|
93
|
+
def close(self) -> None:
|
|
94
|
+
self._wrapped.close()
|
|
95
|
+
|
|
93
96
|
@property
|
|
94
97
|
def _metrics(self) -> ButlerMetrics:
|
|
95
98
|
# Need to always forward from the wrapped metrics object.
|
{lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/python/lsst/pipe/base/connections.py
RENAMED
|
@@ -506,8 +506,19 @@ class DeferredDatasetRef:
|
|
|
506
506
|
datasetRef: DatasetRef
|
|
507
507
|
|
|
508
508
|
def __getattr__(self, name: str) -> Any:
|
|
509
|
+
# make sure reduce is called on DeferredDatasetRef and not on
|
|
510
|
+
# the DatasetRef
|
|
511
|
+
if name in ("__reduce__", "datasetRef", "__deepcopy__"):
|
|
512
|
+
object.__getattribute__(self, name)
|
|
509
513
|
return getattr(self.datasetRef, name)
|
|
510
514
|
|
|
515
|
+
def __deepcopy__(self, memo: dict) -> DeferredDatasetRef:
|
|
516
|
+
# dataset refs should be immutable deferred version should be too
|
|
517
|
+
return self
|
|
518
|
+
|
|
519
|
+
def __reduce__(self) -> tuple:
|
|
520
|
+
return (self.__class__, (self.datasetRef,))
|
|
521
|
+
|
|
511
522
|
|
|
512
523
|
class PipelineTaskConnections(metaclass=PipelineTaskConnectionsMetaclass):
|
|
513
524
|
"""PipelineTaskConnections is a class used to declare desired IO when a
|
|
@@ -34,6 +34,8 @@ import logging
|
|
|
34
34
|
import time
|
|
35
35
|
import uuid
|
|
36
36
|
from collections import defaultdict
|
|
37
|
+
from contextlib import AbstractContextManager
|
|
38
|
+
from typing import Any, Literal, Self
|
|
37
39
|
|
|
38
40
|
from lsst.daf.butler import Butler, CollectionType, DatasetRef, DimensionGroup
|
|
39
41
|
from lsst.daf.butler.datastore.record_data import DatastoreRecordData
|
|
@@ -46,7 +48,7 @@ from ._communicators import IngesterCommunicator
|
|
|
46
48
|
|
|
47
49
|
|
|
48
50
|
@dataclasses.dataclass
|
|
49
|
-
class Ingester:
|
|
51
|
+
class Ingester(AbstractContextManager):
|
|
50
52
|
"""A helper class for the provenance aggregator that handles ingestion into
|
|
51
53
|
the central butler repository.
|
|
52
54
|
"""
|
|
@@ -107,6 +109,16 @@ class Ingester:
|
|
|
107
109
|
self.comms.log.verbose("Initializing butler.")
|
|
108
110
|
self.butler = Butler.from_config(self.butler_path, writeable=not self.comms.config.dry_run)
|
|
109
111
|
|
|
112
|
+
def __enter__(self) -> Self:
|
|
113
|
+
return self
|
|
114
|
+
|
|
115
|
+
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Literal[False]:
|
|
116
|
+
try:
|
|
117
|
+
self.butler.close()
|
|
118
|
+
except Exception:
|
|
119
|
+
self.comms.log.exception("An exception occurred during Ingester exit")
|
|
120
|
+
return False
|
|
121
|
+
|
|
110
122
|
@property
|
|
111
123
|
def n_datasets_pending(self) -> int:
|
|
112
124
|
"""The number of butler datasets currently pending."""
|
|
@@ -130,8 +142,7 @@ class Ingester:
|
|
|
130
142
|
This method is designed to run as the ``target`` in
|
|
131
143
|
`WorkerContext.make_worker`.
|
|
132
144
|
"""
|
|
133
|
-
with comms:
|
|
134
|
-
ingester = Ingester(predicted_path, butler_path, comms)
|
|
145
|
+
with comms, Ingester(predicted_path, butler_path, comms) as ingester:
|
|
135
146
|
ingester.loop()
|
|
136
147
|
|
|
137
148
|
def loop(self) -> None:
|
|
@@ -32,6 +32,8 @@ __all__ = ("Scanner",)
|
|
|
32
32
|
import dataclasses
|
|
33
33
|
import itertools
|
|
34
34
|
import uuid
|
|
35
|
+
from contextlib import AbstractContextManager
|
|
36
|
+
from typing import Any, Literal, Self
|
|
35
37
|
|
|
36
38
|
import zstandard
|
|
37
39
|
|
|
@@ -56,7 +58,7 @@ from ._structs import IngestRequest, InProgressScan, ScanReport, ScanStatus, Wri
|
|
|
56
58
|
|
|
57
59
|
|
|
58
60
|
@dataclasses.dataclass
|
|
59
|
-
class Scanner:
|
|
61
|
+
class Scanner(AbstractContextManager):
|
|
60
62
|
"""A helper class for the provenance aggregator that reads metadata and log
|
|
61
63
|
files and scans for which outputs exist.
|
|
62
64
|
"""
|
|
@@ -101,6 +103,16 @@ class Scanner:
|
|
|
101
103
|
self.qbb = self.make_qbb(self.butler_path, self.reader.pipeline_graph)
|
|
102
104
|
self.init_quanta = {q.quantum_id: q for q in self.reader.components.init_quanta.root}
|
|
103
105
|
|
|
106
|
+
def __enter__(self) -> Self:
|
|
107
|
+
return self
|
|
108
|
+
|
|
109
|
+
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Literal[False]:
|
|
110
|
+
try:
|
|
111
|
+
self.qbb.close()
|
|
112
|
+
except Exception:
|
|
113
|
+
self.comms.log.exception("An exception occurred during Ingester exit")
|
|
114
|
+
return False
|
|
115
|
+
|
|
104
116
|
@staticmethod
|
|
105
117
|
def make_qbb(butler_config: str, pipeline_graph: PipelineGraph) -> QuantumBackedButler:
|
|
106
118
|
"""Make quantum-backed butler that can operate on the outputs of the
|
|
@@ -155,8 +167,7 @@ class Scanner:
|
|
|
155
167
|
This method is designed to run as the ``target`` in
|
|
156
168
|
`WorkerContext.make_worker`.
|
|
157
169
|
"""
|
|
158
|
-
with comms:
|
|
159
|
-
scanner = Scanner(predicted_path, butler_path, comms)
|
|
170
|
+
with comms, Scanner(predicted_path, butler_path, comms) as scanner:
|
|
160
171
|
scanner.loop()
|
|
161
172
|
|
|
162
173
|
def loop(self) -> None:
|
|
@@ -53,7 +53,7 @@ def register_instrument(repo: str, instrument: list[str], update: bool = False)
|
|
|
53
53
|
Raised iff the instrument is not a subclass of
|
|
54
54
|
`lsst.pipe.base.Instrument`.
|
|
55
55
|
"""
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
56
|
+
with Butler.from_config(repo, writeable=True) as butler:
|
|
57
|
+
for string in instrument:
|
|
58
|
+
instrument_instance = Instrument.from_string(string, butler.registry)
|
|
59
|
+
instrument_instance.register(butler.registry, update=update)
|
|
@@ -93,16 +93,15 @@ def retrieve_artifacts_for_quanta(
|
|
|
93
93
|
dataset_types = {dstype.name: dstype for dstype in qgraph.registryDatasetTypes()}
|
|
94
94
|
|
|
95
95
|
# Make QBB, its config is the same as output Butler.
|
|
96
|
-
|
|
96
|
+
with QuantumBackedButler.from_predicted(
|
|
97
97
|
config=repo,
|
|
98
98
|
predicted_inputs=[ref.id for ref in refs],
|
|
99
99
|
predicted_outputs=[],
|
|
100
100
|
dimensions=qgraph.universe,
|
|
101
101
|
datastore_records=datastore_records,
|
|
102
102
|
dataset_types=dataset_types,
|
|
103
|
-
)
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
)
|
|
103
|
+
) as qbb:
|
|
104
|
+
paths = qbb.retrieve_artifacts(
|
|
105
|
+
refs, dest, transfer=transfer, overwrite=clobber, preserve_path=preserve_path
|
|
106
|
+
)
|
|
108
107
|
return paths
|
|
@@ -85,52 +85,52 @@ def transfer_from_graph(
|
|
|
85
85
|
# Get data repository dataset type definitions from the QuantumGraph.
|
|
86
86
|
dataset_types = {dstype.name: dstype for dstype in qgraph.registryDatasetTypes()}
|
|
87
87
|
|
|
88
|
-
# Make QBB, its config is the same as output Butler.
|
|
89
|
-
qbb = QuantumBackedButler.from_predicted(
|
|
90
|
-
config=dest,
|
|
91
|
-
predicted_inputs=[ref.id for ref in output_refs],
|
|
92
|
-
predicted_outputs=[],
|
|
93
|
-
dimensions=qgraph.universe,
|
|
94
|
-
datastore_records={},
|
|
95
|
-
dataset_types=dataset_types,
|
|
96
|
-
)
|
|
97
|
-
|
|
98
88
|
# Filter the refs based on requested dataset types.
|
|
99
89
|
filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
|
|
100
90
|
_LOG.verbose("After filtering by dataset_type, number of datasets to transfer: %d", len(filtered_refs))
|
|
101
91
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
92
|
+
# Make QBB, its config is the same as output Butler.
|
|
93
|
+
with (
|
|
94
|
+
QuantumBackedButler.from_predicted(
|
|
95
|
+
config=dest,
|
|
96
|
+
predicted_inputs=[ref.id for ref in output_refs],
|
|
97
|
+
predicted_outputs=[],
|
|
98
|
+
dimensions=qgraph.universe,
|
|
99
|
+
datastore_records={},
|
|
100
|
+
dataset_types=dataset_types,
|
|
101
|
+
) as qbb,
|
|
102
|
+
Butler.from_config(dest, writeable=True) as dest_butler,
|
|
103
|
+
):
|
|
104
|
+
# For faster restarts, filter out those the destination already knows.
|
|
105
|
+
filtered_refs = filter_by_existence(dest_butler, filtered_refs)
|
|
106
|
+
|
|
107
|
+
# Transfer in chunks
|
|
108
|
+
chunk_size = 50_000
|
|
109
|
+
n_chunks = math.ceil(len(filtered_refs) / chunk_size)
|
|
110
|
+
chunk_num = 0
|
|
111
|
+
count = 0
|
|
112
|
+
for chunk in chunk_iterable(filtered_refs, chunk_size=chunk_size):
|
|
113
|
+
chunk_num += 1
|
|
114
|
+
if n_chunks > 1:
|
|
115
|
+
_LOG.verbose("Transferring %d datasets in chunk %d/%d", len(chunk), chunk_num, n_chunks)
|
|
116
|
+
transferred = dest_butler.transfer_from(
|
|
117
|
+
qbb,
|
|
118
|
+
chunk,
|
|
119
|
+
transfer="auto",
|
|
120
|
+
register_dataset_types=register_dataset_types,
|
|
121
|
+
transfer_dimensions=transfer_dimensions,
|
|
122
|
+
dry_run=dry_run,
|
|
123
|
+
)
|
|
124
|
+
count += len(transferred)
|
|
125
|
+
|
|
126
|
+
# If asked to do so, update output chain definition.
|
|
127
|
+
if update_output_chain and (metadata := qgraph.metadata) is not None:
|
|
128
|
+
# These are defined in CmdLineFwk.
|
|
129
|
+
output_run = metadata.get("output_run")
|
|
130
|
+
output = metadata.get("output")
|
|
131
|
+
input = metadata.get("input")
|
|
132
|
+
if output_run is not None and output is not None:
|
|
133
|
+
_update_chain(dest_butler, output, output_run, input)
|
|
134
134
|
|
|
135
135
|
return count
|
|
136
136
|
|
|
@@ -72,19 +72,18 @@ def zip_from_graph(
|
|
|
72
72
|
# Get data repository dataset type definitions from the QuantumGraph.
|
|
73
73
|
dataset_types = {dstype.name: dstype for dstype in qgraph.registryDatasetTypes()}
|
|
74
74
|
|
|
75
|
+
# Filter the refs based on requested dataset types.
|
|
76
|
+
filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
|
|
77
|
+
|
|
75
78
|
# Make QBB, its config is the same as output Butler.
|
|
76
|
-
|
|
79
|
+
with QuantumBackedButler.from_predicted(
|
|
77
80
|
config=repo,
|
|
78
81
|
predicted_inputs=[ref.id for ref in output_refs],
|
|
79
82
|
predicted_outputs=[],
|
|
80
83
|
dimensions=qgraph.universe,
|
|
81
84
|
datastore_records={},
|
|
82
85
|
dataset_types=dataset_types,
|
|
83
|
-
)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
|
|
87
|
-
|
|
88
|
-
_LOG.info("Retrieving artifacts for %d datasets and storing in Zip file.", len(filtered_refs))
|
|
89
|
-
zip = qbb.retrieve_artifacts_zip(filtered_refs, dest)
|
|
86
|
+
) as qbb:
|
|
87
|
+
_LOG.info("Retrieving artifacts for %d datasets and storing in Zip file.", len(filtered_refs))
|
|
88
|
+
zip = qbb.retrieve_artifacts_zip(filtered_refs, dest)
|
|
90
89
|
return zip
|
|
@@ -40,6 +40,7 @@ from lsst.daf.butler import (
|
|
|
40
40
|
DatasetRef,
|
|
41
41
|
Quantum,
|
|
42
42
|
)
|
|
43
|
+
from lsst.daf.butler.registry import RegistryDefaults
|
|
43
44
|
from lsst.pex.config import Config
|
|
44
45
|
|
|
45
46
|
from ._instrument import Instrument
|
|
@@ -152,9 +153,9 @@ class SimplePipelineExecutor:
|
|
|
152
153
|
collections = [output_run]
|
|
153
154
|
collections.extend(inputs)
|
|
154
155
|
butler.registry.setCollectionChain(output, collections)
|
|
155
|
-
#
|
|
156
|
-
|
|
157
|
-
return
|
|
156
|
+
# Override the registry defaults. No need to clone.
|
|
157
|
+
butler.registry.defaults = RegistryDefaults(collections=[output], run=output_run)
|
|
158
|
+
return butler
|
|
158
159
|
|
|
159
160
|
@classmethod
|
|
160
161
|
def from_pipeline_filename(
|
|
@@ -176,16 +176,34 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
176
176
|
|
|
177
177
|
Internal implementation of `execute()`.
|
|
178
178
|
"""
|
|
179
|
-
startTime = time.time()
|
|
180
|
-
|
|
181
179
|
# Make a limited butler instance if needed.
|
|
182
180
|
limited_butler: LimitedButler
|
|
181
|
+
used_butler_factory = False
|
|
183
182
|
if self._butler is not None:
|
|
184
183
|
limited_butler = self._butler
|
|
185
184
|
else:
|
|
186
185
|
# We check this in constructor, but mypy needs this check here.
|
|
187
186
|
assert self._limited_butler_factory is not None
|
|
188
187
|
limited_butler = self._limited_butler_factory(quantum)
|
|
188
|
+
used_butler_factory = True
|
|
189
|
+
|
|
190
|
+
try:
|
|
191
|
+
return self._execute_with_limited_butler(
|
|
192
|
+
task_node, limited_butler, quantum=quantum, quantum_id=quantum_id
|
|
193
|
+
)
|
|
194
|
+
finally:
|
|
195
|
+
if used_butler_factory:
|
|
196
|
+
limited_butler.close()
|
|
197
|
+
|
|
198
|
+
def _execute_with_limited_butler(
|
|
199
|
+
self,
|
|
200
|
+
task_node: TaskNode,
|
|
201
|
+
limited_butler: LimitedButler,
|
|
202
|
+
/,
|
|
203
|
+
quantum: Quantum,
|
|
204
|
+
quantum_id: uuid.UUID | None = None,
|
|
205
|
+
) -> Quantum:
|
|
206
|
+
startTime = time.time()
|
|
189
207
|
|
|
190
208
|
if self._butler is not None:
|
|
191
209
|
log_capture = LogCapture.from_full(self._butler)
|
|
@@ -29,11 +29,12 @@ from __future__ import annotations
|
|
|
29
29
|
|
|
30
30
|
__all__ = ("DirectButlerRepo", "InMemoryRepo", "MockRepo")
|
|
31
31
|
|
|
32
|
+
import logging
|
|
32
33
|
import tempfile
|
|
33
|
-
from abc import
|
|
34
|
+
from abc import abstractmethod
|
|
34
35
|
from collections.abc import Iterable, Iterator, Mapping
|
|
35
|
-
from contextlib import contextmanager
|
|
36
|
-
from typing import Any
|
|
36
|
+
from contextlib import AbstractContextManager, contextmanager
|
|
37
|
+
from typing import Any, Literal, Self
|
|
37
38
|
|
|
38
39
|
from lsst.daf.butler import (
|
|
39
40
|
Butler,
|
|
@@ -61,8 +62,10 @@ from ._pipeline_task import (
|
|
|
61
62
|
)
|
|
62
63
|
from ._storage_class import MockDataset, is_mock_name
|
|
63
64
|
|
|
65
|
+
_LOG = logging.getLogger(__name__)
|
|
64
66
|
|
|
65
|
-
|
|
67
|
+
|
|
68
|
+
class MockRepo(AbstractContextManager):
|
|
66
69
|
"""A test helper that populates a butler repository for task execution.
|
|
67
70
|
|
|
68
71
|
Parameters
|
|
@@ -100,6 +103,31 @@ class MockRepo(ABC):
|
|
|
100
103
|
self.last_auto_dataset_type_index = 0
|
|
101
104
|
self.last_auto_task_index = 0
|
|
102
105
|
|
|
106
|
+
def __enter__(self) -> Self:
|
|
107
|
+
return self
|
|
108
|
+
|
|
109
|
+
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Literal[False]:
|
|
110
|
+
try:
|
|
111
|
+
self.close()
|
|
112
|
+
except Exception:
|
|
113
|
+
_LOG.exception("An exception occurred during MockRepo.close()")
|
|
114
|
+
return False
|
|
115
|
+
|
|
116
|
+
def close(self) -> None:
|
|
117
|
+
"""Release all resources associated with this mock instance. The
|
|
118
|
+
instance may no longer be used after this is called.
|
|
119
|
+
|
|
120
|
+
Notes
|
|
121
|
+
-----
|
|
122
|
+
Instead of calling ``close()`` directly, you can use the mock object
|
|
123
|
+
as a context manager. For example::
|
|
124
|
+
|
|
125
|
+
with MockRepo(...) as butler:
|
|
126
|
+
butler.get(...)
|
|
127
|
+
# butler is closed after exiting the block.
|
|
128
|
+
"""
|
|
129
|
+
self.butler.close()
|
|
130
|
+
|
|
103
131
|
def add_task(
|
|
104
132
|
self,
|
|
105
133
|
label: str | None = None,
|
|
@@ -581,18 +609,18 @@ class DirectButlerRepo(MockRepo):
|
|
|
581
609
|
) -> Iterator[tuple[DirectButlerRepo, str]]:
|
|
582
610
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as root:
|
|
583
611
|
config = Butler.makeRepo(root, **kwargs)
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
612
|
+
with Butler.from_config(config, writeable=True) as butler:
|
|
613
|
+
yield (
|
|
614
|
+
cls(
|
|
615
|
+
butler,
|
|
616
|
+
*args,
|
|
617
|
+
input_run=input_run,
|
|
618
|
+
input_chain=input_chain,
|
|
619
|
+
use_import_collections_as_input=use_import_collections_as_input,
|
|
620
|
+
data_root=data_root,
|
|
621
|
+
),
|
|
622
|
+
root,
|
|
623
|
+
)
|
|
596
624
|
|
|
597
625
|
def _insert_datasets_impl(
|
|
598
626
|
self, dataset_type: DatasetType, data_ids: list[DataCoordinate]
|
|
@@ -41,6 +41,7 @@ import lsst.daf.butler.tests as butlerTests
|
|
|
41
41
|
import lsst.pex.config as pexConfig
|
|
42
42
|
from lsst.daf.butler import Butler, Config, DataId, DatasetRef, DatasetType, Formatter, LimitedButler
|
|
43
43
|
from lsst.daf.butler.logging import ButlerLogRecords
|
|
44
|
+
from lsst.daf.butler.registry import RegistryDefaults
|
|
44
45
|
from lsst.resources import ResourcePath
|
|
45
46
|
from lsst.utils import doImportType
|
|
46
47
|
from lsst.utils.introspection import get_full_type_name
|
|
@@ -295,8 +296,8 @@ def makeSimpleButler(
|
|
|
295
296
|
if not inMemory:
|
|
296
297
|
butler_config["registry", "db"] = f"sqlite:///{root_path.ospath}/gen3.sqlite"
|
|
297
298
|
butler_config["datastore", "cls"] = "lsst.daf.butler.datastores.fileDatastore.FileDatastore"
|
|
298
|
-
|
|
299
|
-
butler =
|
|
299
|
+
butler = butlerTests.makeTestRepo(str(root_path), {}, config=butler_config)
|
|
300
|
+
butler.registry.defaults = RegistryDefaults(run=run)
|
|
300
301
|
return butler
|
|
301
302
|
|
|
302
303
|
|
|
@@ -480,44 +481,51 @@ def makeSimpleQGraph(
|
|
|
480
481
|
case _:
|
|
481
482
|
raise TypeError(f"Unexpected pipeline object: {pipeline!r}.")
|
|
482
483
|
|
|
484
|
+
butler_created = False
|
|
483
485
|
if butler is None:
|
|
484
486
|
if root is None:
|
|
485
487
|
raise ValueError("Must provide `root` when `butler` is None")
|
|
486
488
|
if callPopulateButler is False:
|
|
487
489
|
raise ValueError("populateButler can only be False when butler is supplied as an argument")
|
|
488
490
|
butler = makeSimpleButler(root, run=run, inMemory=inMemory)
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
491
|
+
butler_created = True
|
|
492
|
+
|
|
493
|
+
try:
|
|
494
|
+
if callPopulateButler:
|
|
495
|
+
populateButler(pipeline_graph, butler, datasetTypes=datasetTypes, instrument=instrument)
|
|
496
|
+
|
|
497
|
+
# Make the graph
|
|
498
|
+
_LOG.debug(
|
|
499
|
+
"Instantiating QuantumGraphBuilder, "
|
|
500
|
+
"skip_existing_in=%s, input_collections=%r, output_run=%r, where=%r, bind=%s.",
|
|
501
|
+
skipExistingIn,
|
|
502
|
+
butler.collections.defaults,
|
|
503
|
+
run,
|
|
504
|
+
userQuery,
|
|
505
|
+
bind,
|
|
506
|
+
)
|
|
507
|
+
if not run:
|
|
508
|
+
assert butler.run is not None, "Butler must have run defined"
|
|
509
|
+
run = butler.run
|
|
510
|
+
builder = AllDimensionsQuantumGraphBuilder(
|
|
511
|
+
pipeline_graph,
|
|
512
|
+
butler,
|
|
513
|
+
skip_existing_in=skipExistingIn if skipExistingIn is not None else [],
|
|
514
|
+
input_collections=butler.collections.defaults,
|
|
515
|
+
output_run=run,
|
|
516
|
+
where=userQuery,
|
|
517
|
+
bind=bind,
|
|
518
|
+
dataset_query_constraint=datasetQueryConstraint,
|
|
519
|
+
)
|
|
520
|
+
_LOG.debug("Calling QuantumGraphBuilder.build.")
|
|
521
|
+
if not metadata:
|
|
522
|
+
metadata = {}
|
|
523
|
+
metadata["output_run"] = run
|
|
524
|
+
|
|
525
|
+
qgraph = builder.build(metadata=metadata, attach_datastore_records=makeDatastoreRecords)
|
|
526
|
+
except Exception:
|
|
527
|
+
if butler_created:
|
|
528
|
+
butler.close()
|
|
529
|
+
raise
|
|
522
530
|
|
|
523
531
|
return butler, qgraph
|
{lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900/python/lsst_pipe_base.egg-info}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-pipe-base
|
|
3
|
-
Version: 29.2025.
|
|
3
|
+
Version: 29.2025.4900
|
|
4
4
|
Summary: Pipeline infrastructure for the Rubin Science Pipelines.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License-Expression: BSD-3-Clause OR GPL-3.0-or-later
|
|
@@ -147,6 +147,7 @@ tests/test_config_formatter.py
|
|
|
147
147
|
tests/test_connections.py
|
|
148
148
|
tests/test_dataid_match.py
|
|
149
149
|
tests/test_dataset_handle.py
|
|
150
|
+
tests/test_deferredDatasetRef.py
|
|
150
151
|
tests/test_dot_tools.py
|
|
151
152
|
tests/test_dynamic_connections.py
|
|
152
153
|
tests/test_execution_reports.py
|
|
@@ -129,6 +129,7 @@ class AdjustAllQuantaTestCase(unittest.TestCase):
|
|
|
129
129
|
adjust_all_quanta hook, and check that it works as expected.
|
|
130
130
|
"""
|
|
131
131
|
butler = self.make_butler()
|
|
132
|
+
self.enterContext(butler)
|
|
132
133
|
pipeline_graph = PipelineGraph(universe=butler.dimensions)
|
|
133
134
|
pipeline_graph.add_task("grouper", GroupTestTask)
|
|
134
135
|
collections = ["imported_g", "imported_r"]
|
|
@@ -263,7 +263,9 @@ class AggregatorTestCase(unittest.TestCase):
|
|
|
263
263
|
quanta : `~collections.abc.Iterator` [`uuid.UUID`]
|
|
264
264
|
An iterator over all executed quantum IDs (not blocked ones).
|
|
265
265
|
"""
|
|
266
|
-
qg.
|
|
266
|
+
qbb = qg.make_init_qbb(repo)
|
|
267
|
+
self.enterContext(qbb)
|
|
268
|
+
qg.init_output_run(qbb)
|
|
267
269
|
sqe = SingleQuantumExecutor(
|
|
268
270
|
limited_butler_factory=lambda quantum: QuantumBackedButler.initialize(
|
|
269
271
|
repo,
|
{lsst_pipe_base-29.2025.4700 → lsst_pipe_base-29.2025.4900}/tests/test_caching_limited_butler.py
RENAMED
|
@@ -44,6 +44,7 @@ class CachingLimitedButlerTestCase(unittest.TestCase):
|
|
|
44
44
|
|
|
45
45
|
def test_init(self):
|
|
46
46
|
helper = InMemoryRepo("base.yaml")
|
|
47
|
+
self.enterContext(helper)
|
|
47
48
|
helper.add_task()
|
|
48
49
|
refs = helper.insert_datasets("dataset_auto0")
|
|
49
50
|
in_memory_butler = helper.make_limited_butler()
|
|
@@ -51,7 +51,8 @@ class PexConfigFormatterTestCase(unittest.TestCase):
|
|
|
51
51
|
"""Create a new butler root for each test."""
|
|
52
52
|
self.root = makeTestTempDir(TESTDIR)
|
|
53
53
|
Butler.makeRepo(self.root)
|
|
54
|
-
self.butler = Butler(self.root, run="test_run")
|
|
54
|
+
self.butler = Butler.from_config(self.root, run="test_run")
|
|
55
|
+
self.enterContext(self.butler)
|
|
55
56
|
# No dimensions in dataset type so we don't have to worry about
|
|
56
57
|
# inserting dimension data or defining data IDs.
|
|
57
58
|
self.datasetType = DatasetType(
|