lsst-pipe-base 29.2025.3900__py3-none-any.whl → 29.2025.4000__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. lsst/pipe/base/dot_tools.py +14 -152
  2. lsst/pipe/base/exec_fixup_data_id.py +17 -44
  3. lsst/pipe/base/execution_graph_fixup.py +49 -18
  4. lsst/pipe/base/graph/graph.py +28 -9
  5. lsst/pipe/base/graph_walker.py +119 -0
  6. lsst/pipe/base/log_capture.py +5 -2
  7. lsst/pipe/base/mermaid_tools.py +11 -64
  8. lsst/pipe/base/mp_graph_executor.py +298 -236
  9. lsst/pipe/base/quantum_graph/__init__.py +32 -0
  10. lsst/pipe/base/quantum_graph/_common.py +610 -0
  11. lsst/pipe/base/quantum_graph/_multiblock.py +737 -0
  12. lsst/pipe/base/quantum_graph/_predicted.py +1874 -0
  13. lsst/pipe/base/quantum_graph/visualization.py +302 -0
  14. lsst/pipe/base/quantum_graph_builder.py +292 -34
  15. lsst/pipe/base/quantum_graph_executor.py +2 -1
  16. lsst/pipe/base/quantum_provenance_graph.py +16 -7
  17. lsst/pipe/base/separable_pipeline_executor.py +126 -15
  18. lsst/pipe/base/simple_pipeline_executor.py +44 -43
  19. lsst/pipe/base/single_quantum_executor.py +1 -40
  20. lsst/pipe/base/tests/mocks/__init__.py +1 -1
  21. lsst/pipe/base/tests/mocks/_pipeline_task.py +16 -1
  22. lsst/pipe/base/tests/mocks/{_in_memory_repo.py → _repo.py} +324 -45
  23. lsst/pipe/base/tests/mocks/_storage_class.py +6 -0
  24. lsst/pipe/base/tests/simpleQGraph.py +11 -5
  25. lsst/pipe/base/version.py +1 -1
  26. {lsst_pipe_base-29.2025.3900.dist-info → lsst_pipe_base-29.2025.4000.dist-info}/METADATA +2 -1
  27. {lsst_pipe_base-29.2025.3900.dist-info → lsst_pipe_base-29.2025.4000.dist-info}/RECORD +35 -29
  28. {lsst_pipe_base-29.2025.3900.dist-info → lsst_pipe_base-29.2025.4000.dist-info}/WHEEL +0 -0
  29. {lsst_pipe_base-29.2025.3900.dist-info → lsst_pipe_base-29.2025.4000.dist-info}/entry_points.txt +0 -0
  30. {lsst_pipe_base-29.2025.3900.dist-info → lsst_pipe_base-29.2025.4000.dist-info}/licenses/COPYRIGHT +0 -0
  31. {lsst_pipe_base-29.2025.3900.dist-info → lsst_pipe_base-29.2025.4000.dist-info}/licenses/LICENSE +0 -0
  32. {lsst_pipe_base-29.2025.3900.dist-info → lsst_pipe_base-29.2025.4000.dist-info}/licenses/bsd_license.txt +0 -0
  33. {lsst_pipe_base-29.2025.3900.dist-info → lsst_pipe_base-29.2025.4000.dist-info}/licenses/gpl-v3.0.txt +0 -0
  34. {lsst_pipe_base-29.2025.3900.dist-info → lsst_pipe_base-29.2025.4000.dist-info}/top_level.txt +0 -0
  35. {lsst_pipe_base-29.2025.3900.dist-info → lsst_pipe_base-29.2025.4000.dist-info}/zip-safe +0 -0
@@ -27,18 +27,31 @@
27
27
 
28
28
  from __future__ import annotations
29
29
 
30
- __all__ = ("InMemoryRepo",)
30
+ __all__ = ("DirectButlerRepo", "InMemoryRepo", "MockRepo")
31
31
 
32
- from collections.abc import Iterable, Mapping
32
+ import tempfile
33
+ from abc import ABC, abstractmethod
34
+ from collections.abc import Iterable, Iterator, Mapping
35
+ from contextlib import contextmanager
33
36
  from typing import Any
34
37
 
35
- from lsst.daf.butler import CollectionType, DataCoordinate, DatasetRef, DatasetType, RegistryConfig
38
+ from lsst.daf.butler import (
39
+ Butler,
40
+ CollectionType,
41
+ DataCoordinate,
42
+ DatasetRef,
43
+ DatasetType,
44
+ DimensionConfig,
45
+ LimitedButler,
46
+ RegistryConfig,
47
+ )
36
48
  from lsst.daf.butler.tests.utils import create_populated_sqlite_registry
37
49
  from lsst.resources import ResourcePath, ResourcePathExpression
38
50
  from lsst.sphgeom import RangeSet
39
51
 
40
52
  from ...all_dimensions_quantum_graph_builder import AllDimensionsQuantumGraphBuilder
41
53
  from ...pipeline_graph import PipelineGraph
54
+ from ...quantum_graph import PredictedQuantumGraph
42
55
  from ...single_quantum_executor import SingleQuantumExecutor
43
56
  from ..in_memory_limited_butler import InMemoryLimitedButler
44
57
  from ._pipeline_task import (
@@ -49,16 +62,13 @@ from ._pipeline_task import (
49
62
  from ._storage_class import MockDataset, is_mock_name
50
63
 
51
64
 
52
- class InMemoryRepo:
53
- """A test helper that simulates a butler repository for task execution
54
- without any disk I/O.
65
+ class MockRepo(ABC):
66
+ """A test helper that populates a butler repository for task execution.
55
67
 
56
68
  Parameters
57
69
  ----------
58
- *args : `str` or `lsst.resources.ResourcePath`
59
- Butler YAML import files to load into the test repository.
60
- registry_config : `lsst.daf.butler.RegistryConfig`, optional
61
- Registry configuration for the repository.
70
+ butler : `lsst.daf.butler.Butler`
71
+ Butler to use for at least quantum graph building. Must be writeable.
62
72
  input_run : `str`, optional
63
73
  Name of a `~lsst.daf.butler.CollectionType.RUN` collection that will be
64
74
  used as an input to quantum graph generation. Input datasets created
@@ -67,49 +77,22 @@ class InMemoryRepo:
67
77
  Name of a `~lsst.daf.butler.CollectionType.CHAINED` collection that
68
78
  will be the direct input to quantum graph generation. This always
69
79
  includes ``input_run``.
70
- output_run : `str`, optional
71
- Name of a `~lsst.daf.butler.CollectionType.RUN` collection for
72
- execution outputs.
73
- use_import_collections_as_input : `bool` `str`, or \
74
- `~collections.abc.Iterable` [ `str`], optional
75
- Additional collections from YAML import files to include in
76
- ``input_chain``, or `True` to include all such collections (in
77
- chain-flattened lexicographical order).
78
- data_root : convertible to `lsst.resources.ResourcePath`, optional
79
- Root directory to join to each element in ``*args``. Defaults to
80
- the `lsst.daf.butler.tests.registry_data` package.
81
-
82
- Notes
83
- -----
84
- This helper maintains an `..pipeline_graph.PipelineGraph` and a
85
- no-datastore butler backed by an in-memory SQLite database for use in
86
- quantum graph generation.
80
+ input_children : `str` or `~collections.abc.Iterable` [ `str`], optional
81
+ Additional collections to include in ``input_chain``.
87
82
  """
88
83
 
89
84
  def __init__(
90
85
  self,
91
- *args: str | ResourcePath,
92
- registry_config: RegistryConfig | None = None,
86
+ butler: Butler,
93
87
  input_run: str = "input_run",
94
88
  input_chain: str = "input_chain",
95
- output_run: str = "output_run",
96
- use_import_collections_as_input: bool | str | Iterable[str] = True,
97
- data_root: ResourcePathExpression | None = "resource://lsst.daf.butler/tests/registry_data",
89
+ input_children: Iterable[str] = (),
98
90
  ):
99
- if data_root is not None:
100
- data_root = ResourcePath(data_root, forceDirectory=True)
101
- args = tuple(data_root.join(arg) for arg in args)
102
- self.butler = create_populated_sqlite_registry(*args, registry_config=registry_config)
91
+ self.butler = butler
103
92
  input_chain_definition = [input_run]
104
- if use_import_collections_as_input:
105
- if use_import_collections_as_input is True:
106
- use_import_collections_as_input = sorted(
107
- self.butler.collections.query("*", flatten_chains=True)
108
- )
109
- input_chain_definition += list(use_import_collections_as_input)
93
+ input_chain_definition.extend(input_children)
110
94
  self.input_run = input_run
111
95
  self.input_chain = input_chain
112
- self.output_run = output_run
113
96
  self.butler.collections.register(self.input_run)
114
97
  self.butler.collections.register(self.input_chain, CollectionType.CHAINED)
115
98
  self.butler.collections.redefine_chain(self.input_chain, input_chain_definition)
@@ -206,9 +189,55 @@ class InMemoryRepo:
206
189
  label = f"task_auto{self.last_auto_task_index}"
207
190
  self.pipeline_graph.add_task(label, task_class=task_class, config=config)
208
191
 
192
+ def make_quantum_graph(
193
+ self,
194
+ *,
195
+ output: str | None = None,
196
+ output_run: str = "output_run",
197
+ insert_mocked_inputs: bool = True,
198
+ register_output_dataset_types: bool = True,
199
+ ) -> PredictedQuantumGraph:
200
+ """Make a quantum graph from the pipeline task and internal data
201
+ repository.
202
+
203
+ Parameters
204
+ ----------
205
+ output : `str` or `None`, optional
206
+ Name of the output chained collection to embed within the quantum
207
+ graph. Note that this does not actually create this collection.
208
+ output_run : `str`, optional
209
+ Name of the `~lsst.daf.butler.CollectionType.RUN` collection for
210
+ execution outputs. Note that this does not actually create this
211
+ collection.
212
+ insert_mocked_inputs : `bool`, optional
213
+ Whether to automatically insert datasets for all overall inputs to
214
+ the pipeline graph whose dataset types have not already been
215
+ registered. If set to `False`, inputs must be provided by imported
216
+ YAML files or explicit calls to `insert_datasets`, which provides
217
+ more fine-grained control over the data IDs of the datasets.
218
+ register_output_dataset_types : `bool`, optional
219
+ If `True`, register all output dataset types.
220
+
221
+ Returns
222
+ -------
223
+ qg : `..quantum_graph.PredictedQuantumGraph`
224
+ Quantum graph. Datastore records will not be attached, since the
225
+ test helper does not actually have a datastore.
226
+ """
227
+ return (
228
+ self.make_quantum_graph_builder(
229
+ insert_mocked_inputs=insert_mocked_inputs,
230
+ register_output_dataset_types=register_output_dataset_types,
231
+ output_run=output_run,
232
+ )
233
+ .finish(output=output, attach_datastore_records=False)
234
+ .assemble()
235
+ )
236
+
209
237
  def make_quantum_graph_builder(
210
238
  self,
211
239
  *,
240
+ output_run: str = "output_run",
212
241
  insert_mocked_inputs: bool = True,
213
242
  register_output_dataset_types: bool = True,
214
243
  ) -> AllDimensionsQuantumGraphBuilder:
@@ -217,6 +246,10 @@ class InMemoryRepo:
217
246
 
218
247
  Parameters
219
248
  ----------
249
+ output_run : `str`, optional
250
+ Name of the `~lsst.daf.butler.CollectionType.RUN` collection for
251
+ execution outputs. Note that this does not actually create this
252
+ collection.
220
253
  insert_mocked_inputs : `bool`, optional
221
254
  Whether to automatically insert datasets for all overall inputs to
222
255
  the pipeline graph whose dataset types have not already been
@@ -244,7 +277,7 @@ class InMemoryRepo:
244
277
  self.pipeline_graph,
245
278
  self.butler,
246
279
  input_collections=[self.input_chain],
247
- output_run=self.output_run,
280
+ output_run=output_run,
248
281
  )
249
282
  if register_output_dataset_types:
250
283
  self.pipeline_graph.register_dataset_types(self.butler)
@@ -311,6 +344,117 @@ class InMemoryRepo:
311
344
  )
312
345
  else:
313
346
  data_ids = self.butler.query_data_ids(dimensions, *args, **kwargs, explain=False)
347
+ return self._insert_datasets_impl(dataset_type, data_ids)
348
+
349
+ @abstractmethod
350
+ def _insert_datasets_impl(
351
+ self, dataset_type: DatasetType, data_ids: list[DataCoordinate]
352
+ ) -> list[DatasetRef]:
353
+ """Insert datasets after their data IDs have been generated.
354
+
355
+ Parameters
356
+ ----------
357
+ dataset_type : `lsst.daf.butler.DatasetType`
358
+ Type of the datasets.
359
+ data_ids : `list` [ `lsst.daf.butler.DataCoordinate` ]
360
+ Data IDs of all datasets.
361
+
362
+ Returns
363
+ -------
364
+ refs : `list` [ `lsst.daf.butler.DatasetRef` ]
365
+ References to the new datasets.
366
+ """
367
+ raise NotImplementedError()
368
+
369
+ @abstractmethod
370
+ def make_single_quantum_executor(
371
+ self, qg: PredictedQuantumGraph
372
+ ) -> tuple[SingleQuantumExecutor, LimitedButler]:
373
+ """Make a single-quantum executor.
374
+
375
+ Parameters
376
+ ----------
377
+ qg : `..quantum_graph.PredictedQuantumGraph`
378
+ Graph whose quanta the executor must be capable of executing.
379
+
380
+ Returns
381
+ -------
382
+ executor : `..single_quantum_executor.SingleQuantumExecutor`
383
+ An executor for a single quantum.
384
+ butler : `lsst.daf.butler.LimitedButler`
385
+ The butler that the executor will write to.
386
+ """
387
+ raise NotImplementedError()
388
+
389
+
390
+ class InMemoryRepo(MockRepo):
391
+ """A test helper that simulates a butler repository for task execution
392
+ without any disk I/O.
393
+
394
+ Parameters
395
+ ----------
396
+ *args : `str` or `lsst.resources.ResourcePath`
397
+ Butler YAML import files to load into the test repository.
398
+ registry_config : `lsst.daf.butler.RegistryConfig`, optional
399
+ Registry configuration for the repository.
400
+ dimension_config : `lsst.daf.butler.DimensionConfig`, optional
401
+ Dimension universe configuration for the repository.
402
+ input_run : `str`, optional
403
+ Name of a `~lsst.daf.butler.CollectionType.RUN` collection that will be
404
+ used as an input to quantum graph generation. Input datasets created
405
+ by the helper are added to this collection.
406
+ input_chain : `str`, optional
407
+ Name of a `~lsst.daf.butler.CollectionType.CHAINED` collection that
408
+ will be the direct input to quantum graph generation. This always
409
+ includes ``input_run``.
410
+ use_import_collections_as_input : `bool`, `str`, or \
411
+ `~collections.abc.Iterable` [ `str`], optional
412
+ Additional collections from YAML import files to include in
413
+ ``input_chain``, or `True` to include all such collections (in
414
+ chain-flattened lexicographical order).
415
+ data_root : convertible to `lsst.resources.ResourcePath`, optional
416
+ Root directory to join to each element in ``*args``. Defaults to
417
+ the `lsst.daf.butler.tests.registry_data` package.
418
+
419
+ Notes
420
+ -----
421
+ This helper maintains an `..pipeline_graph.PipelineGraph` and a
422
+ no-datastore butler backed by an in-memory SQLite database for use in
423
+ quantum graph generation. It creates a separate in-memory limited butler
424
+ for execution as needed.
425
+ """
426
+
427
+ def __init__(
428
+ self,
429
+ *args: str | ResourcePath,
430
+ registry_config: RegistryConfig | None = None,
431
+ dimension_config: DimensionConfig | None = None,
432
+ input_run: str = "input_run",
433
+ input_chain: str = "input_chain",
434
+ use_import_collections_as_input: bool | str | Iterable[str] = True,
435
+ data_root: ResourcePathExpression | None = "resource://lsst.daf.butler/tests/registry_data",
436
+ ):
437
+ if data_root is not None:
438
+ data_root = ResourcePath(data_root, forceDirectory=True)
439
+ args = tuple(data_root.join(arg) for arg in args)
440
+ butler = create_populated_sqlite_registry(
441
+ *args, registry_config=registry_config, dimension_config=dimension_config
442
+ )
443
+ if use_import_collections_as_input:
444
+ if use_import_collections_as_input is True:
445
+ use_import_collections_as_input = sorted(butler.collections.query("*", flatten_chains=True))
446
+ else:
447
+ use_import_collections_as_input = ()
448
+ super().__init__(
449
+ butler,
450
+ input_run=input_run,
451
+ input_chain=input_chain,
452
+ input_children=list(use_import_collections_as_input),
453
+ )
454
+
455
+ def _insert_datasets_impl(
456
+ self, dataset_type: DatasetType, data_ids: list[DataCoordinate]
457
+ ) -> list[DatasetRef]:
314
458
  return self.butler.registry.insertDatasets(dataset_type, data_ids, run=self.input_run)
315
459
 
316
460
  def make_limited_butler(self) -> InMemoryLimitedButler:
@@ -343,9 +487,16 @@ class InMemoryRepo:
343
487
  )
344
488
  return butler
345
489
 
346
- def make_single_quantum_executor(self) -> tuple[SingleQuantumExecutor, InMemoryLimitedButler]:
490
+ def make_single_quantum_executor(
491
+ self, qg: PredictedQuantumGraph | None = None
492
+ ) -> tuple[SingleQuantumExecutor, InMemoryLimitedButler]:
347
493
  """Make a single-quantum executor backed by a new limited butler.
348
494
 
495
+ Parameters
496
+ ----------
497
+ qg : `..quantum_graph.PredictedQuantumGraph`
498
+ Ignored by this implementation.
499
+
349
500
  Returns
350
501
  -------
351
502
  executor : `..single_quantum_executor.SingleQuantumExecutor`
@@ -355,3 +506,131 @@ class InMemoryRepo:
355
506
  """
356
507
  butler = self.make_limited_butler()
357
508
  return SingleQuantumExecutor(limited_butler_factory=butler.factory), butler
509
+
510
+
511
+ class DirectButlerRepo(MockRepo):
512
+ """A test helper for task execution backed by a local direct butler.
513
+
514
+ Parameters
515
+ ----------
516
+ butler : `lsst.daf.butler.direct_butler.DirectButler`
517
+ Butler to write to.
518
+ *args : `str` or `lsst.resources.ResourcePath`
519
+ Butler YAML import files to load into the test repository.
520
+ input_run : `str`, optional
521
+ Name of a `~lsst.daf.butler.CollectionType.RUN` collection that will be
522
+ used as an input to quantum graph generation. Input datasets created
523
+ by the helper are added to this collection.
524
+ input_chain : `str`, optional
525
+ Name of a `~lsst.daf.butler.CollectionType.CHAINED` collection that
526
+ will be the direct input to quantum graph generation. This always
527
+ includes ``input_run``.
528
+ use_import_collections_as_input : `bool`, `str`, or \
529
+ `~collections.abc.Iterable` [ `str`], optional
530
+ Additional collections from YAML import files to include in
531
+ ``input_chain``, or `True` to include all such collections (in
532
+ chain-flattened lexicographical order).
533
+ data_root : convertible to `lsst.resources.ResourcePath`, optional
534
+ Root directory to join to each element in ``*args``. Defaults to
535
+ the `lsst.daf.butler.tests.registry_data` package.
536
+
537
+ Notes
538
+ -----
539
+ This helper maintains an `..pipeline_graph.PipelineGraph` and a
540
+ no-datastore butler backed by an in-memory SQLite database for use in
541
+ quantum graph generation. It creates a separate in-memory limited butler
542
+ for execution as needed.
543
+ """
544
+
545
+ def __init__(
546
+ self,
547
+ butler: Butler,
548
+ *args: str | ResourcePath,
549
+ input_run: str = "input_run",
550
+ input_chain: str = "input_chain",
551
+ use_import_collections_as_input: bool | str | Iterable[str] = True,
552
+ data_root: ResourcePathExpression | None = "resource://lsst.daf.butler/tests/registry_data",
553
+ ):
554
+ if data_root is not None:
555
+ data_root = ResourcePath(data_root, forceDirectory=True)
556
+ args = tuple(data_root.join(arg) for arg in args)
557
+ for arg in args:
558
+ butler.import_(filename=arg)
559
+ if use_import_collections_as_input:
560
+ if use_import_collections_as_input is True:
561
+ use_import_collections_as_input = sorted(butler.collections.query("*", flatten_chains=True))
562
+ else:
563
+ use_import_collections_as_input = ()
564
+ super().__init__(
565
+ butler,
566
+ input_run=input_run,
567
+ input_chain=input_chain,
568
+ input_children=list(use_import_collections_as_input),
569
+ )
570
+
571
+ @classmethod
572
+ @contextmanager
573
+ def make_temporary(
574
+ cls,
575
+ *args: str | ResourcePath,
576
+ input_run: str = "input_run",
577
+ input_chain: str = "input_chain",
578
+ use_import_collections_as_input: bool | str | Iterable[str] = True,
579
+ data_root: ResourcePathExpression | None = "resource://lsst.daf.butler/tests/registry_data",
580
+ **kwargs: Any,
581
+ ) -> Iterator[tuple[DirectButlerRepo, str]]:
582
+ with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as root:
583
+ config = Butler.makeRepo(root, **kwargs)
584
+ butler = Butler.from_config(config, writeable=True)
585
+ yield (
586
+ cls(
587
+ butler,
588
+ *args,
589
+ input_run=input_run,
590
+ input_chain=input_chain,
591
+ use_import_collections_as_input=use_import_collections_as_input,
592
+ data_root=data_root,
593
+ ),
594
+ root,
595
+ )
596
+
597
+ def _insert_datasets_impl(
598
+ self, dataset_type: DatasetType, data_ids: list[DataCoordinate]
599
+ ) -> list[DatasetRef]:
600
+ if is_mock_name(dataset_type.storageClass_name):
601
+ refs: list[DatasetRef] = []
602
+ for data_id in data_ids:
603
+ data_id = self.butler.registry.expandDataId(data_id)
604
+ ref = DatasetRef(dataset_type, data_id, run=self.input_run)
605
+ self.butler.put(
606
+ MockDataset(
607
+ dataset_id=ref.id,
608
+ dataset_type=ref.datasetType.to_simple(),
609
+ data_id=dict(ref.dataId.mapping),
610
+ run=ref.run,
611
+ ),
612
+ ref,
613
+ )
614
+ refs.append(ref)
615
+ return refs
616
+ else:
617
+ return self.butler.registry.insertDatasets(dataset_type, data_ids, run=self.input_run)
618
+
619
+ def make_single_quantum_executor(
620
+ self, qg: PredictedQuantumGraph | None = None
621
+ ) -> tuple[SingleQuantumExecutor, Butler]:
622
+ """Make a single-quantum executor backed by a new limited butler.
623
+
624
+ Parameters
625
+ ----------
626
+ qg : `..quantum_graph.PredictedQuantumGraph`
627
+ Ignored by this implementation.
628
+
629
+ Returns
630
+ -------
631
+ executor : `..single_quantum_executor.SingleQuantumExecutor`
632
+ An executor for a single quantum.
633
+ butler : `lsst.daf.butler.Butler`
634
+ The butler that the executor will write to.
635
+ """
636
+ return SingleQuantumExecutor(limited_butler_factory=None, butler=self.butler), self.butler
@@ -173,6 +173,12 @@ class MockDataset(pydantic.BaseModel):
173
173
  parameters: dict[str, str] | None = None
174
174
  """`repr` of all parameters applied when reading this dataset."""
175
175
 
176
+ int_value: int | None = None
177
+ """An arbitrary integer value stored in the mock dataset."""
178
+
179
+ str_value: int | None = None
180
+ """An arbitrary string value stored in the mock dataset."""
181
+
176
182
  @property
177
183
  def storage_class(self) -> str:
178
184
  return cast(str, self.dataset_type.storageClass)
@@ -49,7 +49,13 @@ from .. import connectionTypes as cT
49
49
  from .._instrument import Instrument
50
50
  from ..all_dimensions_quantum_graph_builder import AllDimensionsQuantumGraphBuilder
51
51
  from ..all_dimensions_quantum_graph_builder import DatasetQueryConstraintVariant as DSQVariant
52
- from ..automatic_connection_constants import PACKAGES_INIT_OUTPUT_NAME, PACKAGES_INIT_OUTPUT_STORAGE_CLASS
52
+ from ..automatic_connection_constants import (
53
+ CONFIG_INIT_OUTPUT_CONNECTION_NAME,
54
+ LOG_OUTPUT_CONNECTION_NAME,
55
+ METADATA_OUTPUT_CONNECTION_NAME,
56
+ PACKAGES_INIT_OUTPUT_NAME,
57
+ PACKAGES_INIT_OUTPUT_STORAGE_CLASS,
58
+ )
53
59
  from ..config import PipelineTaskConfig
54
60
  from ..connections import PipelineTaskConnections
55
61
  from ..graph import QuantumGraph
@@ -361,13 +367,13 @@ def populateButler(
361
367
  if run is not None:
362
368
  butler.registry.registerRun(run)
363
369
  for dsType in dsTypes:
364
- if dsType == "packages":
370
+ if dsType == PACKAGES_INIT_OUTPUT_NAME:
365
371
  # Version is intentionally inconsistent.
366
372
  # Dict is convertible to Packages if Packages is installed.
367
373
  data: Any = {"python": "9.9.99"}
368
374
  butler.put(data, dsType, run=run)
369
375
  else:
370
- if dsType.endswith("_config"):
376
+ if dsType.endswith(CONFIG_INIT_OUTPUT_CONNECTION_NAME):
371
377
  # find a config from matching task name or make a new one
372
378
  taskLabel, _, _ = dsType.rpartition("_")
373
379
  task_node = pipeline_graph.tasks.get(taskLabel)
@@ -375,9 +381,9 @@ def populateButler(
375
381
  data = task_node.config
376
382
  else:
377
383
  data = AddTaskConfig()
378
- elif dsType.endswith("_metadata"):
384
+ elif dsType.endswith(METADATA_OUTPUT_CONNECTION_NAME):
379
385
  data = _TASK_FULL_METADATA_TYPE()
380
- elif dsType.endswith("_log"):
386
+ elif dsType.endswith(LOG_OUTPUT_CONNECTION_NAME):
381
387
  data = ButlerLogRecords.from_records([])
382
388
  else:
383
389
  data = numpy.array([0.0, 1.0, 2.0, 5.0])
lsst/pipe/base/version.py CHANGED
@@ -1,2 +1,2 @@
1
1
  __all__ = ["__version__"]
2
- __version__ = "29.2025.3900"
2
+ __version__ = "29.2025.4000"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lsst-pipe-base
3
- Version: 29.2025.3900
3
+ Version: 29.2025.4000
4
4
  Summary: Pipeline infrastructure for the Rubin Science Pipelines.
5
5
  Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
6
6
  License: BSD 3-Clause License
@@ -31,6 +31,7 @@ Requires-Dist: wcwidth
31
31
  Requires-Dist: pyyaml>=5.1
32
32
  Requires-Dist: numpy>=1.17
33
33
  Requires-Dist: frozendict
34
+ Requires-Dist: zstandard<0.24,>=0.23.0
34
35
  Provides-Extra: test
35
36
  Requires-Dist: pytest>=3.2; extra == "test"
36
37
  Provides-Extra: mermaid
@@ -15,32 +15,33 @@ lsst/pipe/base/config.py,sha256=yNipVEc6awwhU_O9I01g20OnvQrs28dAwkXuI1hrlYE,1198
15
15
  lsst/pipe/base/configOverrides.py,sha256=B0An8EaX76VzWnC5dJxvyZ2AhVzawMtq7qlE9ma5lkc,14661
16
16
  lsst/pipe/base/connectionTypes.py,sha256=inUDyzbM1sKMCtHaRkhx3dWSPHPBIDVMHOPhzB13Kdw,16720
17
17
  lsst/pipe/base/connections.py,sha256=S_PgywIYoPlaCtGtDtD6S24yewVaPfdS_QgrhUAty7g,66725
18
- lsst/pipe/base/dot_tools.py,sha256=bxwJTIfG58LAxPvBzORgp99kYKJUQFK4mNVjEuj-Pgc,9862
19
- lsst/pipe/base/exec_fixup_data_id.py,sha256=UG-yZboZijOjrPh0bKnAjEYJMpRqGAIgNZxIDYVa0l0,5048
20
- lsst/pipe/base/execution_graph_fixup.py,sha256=_orQ_GT5f-VyRarcpaPD_cNEfo9AIWgum9HkMkcvNG8,2811
18
+ lsst/pipe/base/dot_tools.py,sha256=vriWMaB8YTEKKvhJE5KYdVGE4gB5XmiYfD2f18Fue-c,4285
19
+ lsst/pipe/base/exec_fixup_data_id.py,sha256=9OjOcH-6AHZ1JnD_CemieI0wWX90J_VdaY9v1oXwMdQ,4187
20
+ lsst/pipe/base/execution_graph_fixup.py,sha256=ND0x4hlpeEW-gudo-i2K7HT7MoM5sp_mcoqRMCopSqQ,3815
21
21
  lsst/pipe/base/execution_reports.py,sha256=jYtWCD4PkEAeVUpKIxuiJJVgsCm7qiwCorWVgNHkVgU,17270
22
- lsst/pipe/base/log_capture.py,sha256=tgJcq_eOIwywktagYXL0sCnafqNR0CJ7rfW09iXQ63k,9390
23
- lsst/pipe/base/mermaid_tools.py,sha256=s6TQCBjvrkw2t9yxjm3YEYsBKnUW2yHmt6SawQJ9Bto,7494
24
- lsst/pipe/base/mp_graph_executor.py,sha256=45v915qU1dYQsvtAXtfms6kF3k49bptDe9IZNRpzfOQ,32130
22
+ lsst/pipe/base/graph_walker.py,sha256=7dOx3V7thYr6eN29GRbkudV9Y8249ojDQUlnW2JHtg8,4486
23
+ lsst/pipe/base/log_capture.py,sha256=LylLrxPJyMxU90uSQ15PqFhObDSLWQWGKX6NuXOuwXA,9504
24
+ lsst/pipe/base/mermaid_tools.py,sha256=cdlDJQ1x8k7-VvCLEUqvSC3GR1zCsB-aUTxOjYejNWc,5216
25
+ lsst/pipe/base/mp_graph_executor.py,sha256=FKlFxjtU2-6SFzX_wsdtMMAj5P09ZE8V65-Sg585g2Y,35501
25
26
  lsst/pipe/base/pipeline.py,sha256=FVaiLhgw9Pzo-nzXKS0dLNafegP0AMZKLtPlSvOSkRU,37563
26
27
  lsst/pipe/base/pipelineIR.py,sha256=DDOAYHnMP-iw021RDMYsZnvb21tWumLjYqO5d38q_Zk,44300
27
28
  lsst/pipe/base/pipelineTask.py,sha256=K3GdjJLvy8A7I-jzQiERQZaYF7mC1LM3iB5TmUtbOCI,8394
28
29
  lsst/pipe/base/prerequisite_helpers.py,sha256=bmiebQ4veSrypZgAXjmCBFfj8fUtPW9eRQaVShhxdBQ,28446
29
30
  lsst/pipe/base/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
30
- lsst/pipe/base/quantum_graph_builder.py,sha256=Na8uLsOzPqvQqi9pI9bgHjWREhvWPWIA3_MvLLKl-R0,56621
31
- lsst/pipe/base/quantum_graph_executor.py,sha256=KOOjko2weju02oXnvWTvwXDxbPJp3U1F0KTgDSOJgKg,4342
31
+ lsst/pipe/base/quantum_graph_builder.py,sha256=n5AqYvHVIG0M5RLoh2sERz7oBlM5IOtwCBDLyyMrKtQ,67927
32
+ lsst/pipe/base/quantum_graph_executor.py,sha256=WP41iQmihy1jfgaHV6eu2aSrqQx_Fydq3mbEF6CLQ-s,4419
32
33
  lsst/pipe/base/quantum_graph_skeleton.py,sha256=GhSQjRHaErneGY4A4E0tERqg9QPEeYrlpmdLzqFXy6E,28586
33
- lsst/pipe/base/quantum_provenance_graph.py,sha256=llXcqu-50dtjkt_sVqAhBU10htfkxMiiArNN0_GqL1g,93034
34
+ lsst/pipe/base/quantum_provenance_graph.py,sha256=SChke6lcSzQQQMbVN_mCZ-RxDj7chehjJrSMvXRuhAI,93372
34
35
  lsst/pipe/base/quantum_reports.py,sha256=GRAjhUa7L595AjBywJz95XT7K4OUFa-rudWQtQTrUlU,12230
35
- lsst/pipe/base/separable_pipeline_executor.py,sha256=w7-cZ9koJNDhEVu2lMXqjNMhZvFVn5QbJfsqvxHkQFE,11928
36
- lsst/pipe/base/simple_pipeline_executor.py,sha256=IYda1g258FWWESN5nQv6xwZJL5AZXhnDGC5iN_IohUg,29597
37
- lsst/pipe/base/single_quantum_executor.py,sha256=4Uhg83vZU9P74K6wER4qisDB-dZukdrkJ7hxNjEF-rQ,28277
36
+ lsst/pipe/base/separable_pipeline_executor.py,sha256=vXqJrRI5GNezzGV9QsiaRHEhioDF2Y_W7JQYQCzHR7A,16720
37
+ lsst/pipe/base/simple_pipeline_executor.py,sha256=xYxaCJOorzy5RFXprzOZX8HtSGbd6Ir_nhqLnHr9FBM,29409
38
+ lsst/pipe/base/single_quantum_executor.py,sha256=YWzLJDTRlvMwmtjsIW7XXZEUlLlv7ize-Oxb_0eKTrI,26667
38
39
  lsst/pipe/base/struct.py,sha256=Fa-UkpuXOxdzKWbHrMUkJYOszZuBXCm2NesXNR0IOPQ,5048
39
40
  lsst/pipe/base/task.py,sha256=XHBd-7m1a4-6LgobBYA1DgY4H7EV-_RWKfxbhZbMmD4,15145
40
41
  lsst/pipe/base/taskFactory.py,sha256=MsDGECJqZLSZk8SGhpuVhNaP32UWuNvxZiDcZExPFG8,3412
41
42
  lsst/pipe/base/testUtils.py,sha256=lSBKMhoKflbi8JkMNYfEqqHNl-rtFI8UYT3QneDYpLo,18477
42
43
  lsst/pipe/base/utils.py,sha256=JmEt3l0xrh9uayKrSXuQEq12aXOhDr2YXmbYduaxCko,1940
43
- lsst/pipe/base/version.py,sha256=qaaoasFkyU8Kx3tKT5jPh3H-bKD5Y8pAmsL3Scaq2UU,55
44
+ lsst/pipe/base/version.py,sha256=OZgl6BR-Rxy62NJlHO2yg_Rq4EhfmRSKDkEEnAyc-3Q,55
44
45
  lsst/pipe/base/cli/__init__.py,sha256=861tXIAW7SqtqNUYkjbeEdfg8lDswXsjJQca0gVCFz4,54
45
46
  lsst/pipe/base/cli/_get_cli_subcommands.py,sha256=g_af64klRybBGKAg7fmBSZBdw2LYBAsFON_yQIMZON0,1289
46
47
  lsst/pipe/base/cli/cmd/__init__.py,sha256=BGicstnryQ48rYcNRh4fa6Vy63ZIlZ_pPAEa17jhkwY,1519
@@ -54,7 +55,7 @@ lsst/pipe/base/graph/__init__.py,sha256=Zs2vwSFNiu1bYDsgrWQZ0qegG5F6PIjiQ5ZGT3Eq
54
55
  lsst/pipe/base/graph/_implDetails.py,sha256=QQHVnCW78UnIbALXX_v7EW7g6MTUTuuR1Q_Ss_squUw,6784
55
56
  lsst/pipe/base/graph/_loadHelpers.py,sha256=qUfjIgFezaXZRCFV7PFzmz1SSKFjRWOMWJePuyKiD24,12064
56
57
  lsst/pipe/base/graph/_versionDeserializers.py,sha256=pXk63v6jkQSghSOoU1hpPkxVa82WVGitm2jrop85SeM,27992
57
- lsst/pipe/base/graph/graph.py,sha256=GEauk42yFXVkFfHxo0EKofQmAKLhfnfe0UzlwuvUsVQ,74240
58
+ lsst/pipe/base/graph/graph.py,sha256=d6GVAdpktboDzhunwymyyymbAvkTrqTwGQDb1CrCJeQ,75092
58
59
  lsst/pipe/base/graph/graphSummary.py,sha256=S3O84MddiXrmOD7U9Ek7rR22kdveGLhsor69FRvDjh4,5032
59
60
  lsst/pipe/base/graph/quantumNode.py,sha256=l4mslxBgyUzBAqwjpx6XRP-UPxe-oRMxHJWt-_y3Dm0,7196
60
61
  lsst/pipe/base/pipeline_graph/__init__.py,sha256=yTEuvlzbeKIHIm7GeRmGSsma1wpZFNv8j12WfSH-deY,1516
@@ -79,6 +80,11 @@ lsst/pipe/base/pipeline_graph/visualization/_options.py,sha256=vOIp2T7DLA48lTm5m
79
80
  lsst/pipe/base/pipeline_graph/visualization/_printer.py,sha256=yJMRJ-aXd3nYDgs1FqS2l_hzNbQ50HUVm55VVaNi71s,16537
80
81
  lsst/pipe/base/pipeline_graph/visualization/_show.py,sha256=lPRjO1To2n5r3f_Wgcwy-7TmyJ7UszGGFXAlOtN1wDs,10510
81
82
  lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py,sha256=dp7PXl9Cu7GfWjBi5g8KjXZgnF1KGg_idKKxtICL53Q,8679
83
+ lsst/pipe/base/quantum_graph/__init__.py,sha256=rPvhjKcnPisHsJ1-PbyASSmyaxzw55nGYi4ZKI5LO84,1410
84
+ lsst/pipe/base/quantum_graph/_common.py,sha256=rpfsmf56QM4ivMU9VIQSGcaxRM32usLCaox1JVbQzYM,21247
85
+ lsst/pipe/base/quantum_graph/_multiblock.py,sha256=81L-dqoPqCVkSEuFAELcENY0_hSKChTx0EF0hemwHFk,24648
86
+ lsst/pipe/base/quantum_graph/_predicted.py,sha256=Sa8z_rkWt_Ggcfm9_6rrPl88kSuHuchlupD1-8V7o24,82680
87
+ lsst/pipe/base/quantum_graph/visualization.py,sha256=EbTWhk9aPq7sX6bcHmnEIsr2xuuR6d1SxspQbRe8D0Q,12235
82
88
  lsst/pipe/base/script/__init__.py,sha256=cLEXE7aq5UZ0juL_ScmRw0weFgp4tDgwEX_ts-NEYic,1522
83
89
  lsst/pipe/base/script/register_instrument.py,sha256=TRC2r2tSoYBNWNVQya01ELxAtGH8WVk9Ya-uNgCIL5U,2426
84
90
  lsst/pipe/base/script/retrieve_artifacts_for_quanta.py,sha256=pYI0wNl5PU8ImgzWfGEDrRz3PSKSg2szWLEIVKdm7Og,3939
@@ -89,20 +95,20 @@ lsst/pipe/base/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
89
95
  lsst/pipe/base/tests/in_memory_limited_butler.py,sha256=UzLh416H67nCUhD9y3cniAAjY7VojvhOLjF3gHHgjA4,8679
90
96
  lsst/pipe/base/tests/no_dimensions.py,sha256=58UpyRN8cLAMZtkOmjTm3dJZyRFRekotQ-7-OgEfiAI,4710
91
97
  lsst/pipe/base/tests/pipelineStepTester.py,sha256=KGxdB8gdVpSey2RUGURDIzIfPL-4qvQCsBpMrhG4Z2M,7208
92
- lsst/pipe/base/tests/simpleQGraph.py,sha256=G9C69caX8479JR9h48ERhOFvLTPJCoj5gKf_eRoaALQ,19660
98
+ lsst/pipe/base/tests/simpleQGraph.py,sha256=WHphpVvOA9eSbVtUp884wjk4g6ppFLM7gnddDYSwnTc,19862
93
99
  lsst/pipe/base/tests/util.py,sha256=eWuIRz55HYgNmMkexinN9HjUFmPC3uapO8jMjcQY-ao,4010
94
- lsst/pipe/base/tests/mocks/__init__.py,sha256=YheEqtwDsjkqLNLCYbDrbZmLj9y942fOWC_xKF3xmCk,1582
100
+ lsst/pipe/base/tests/mocks/__init__.py,sha256=fDy9H9vRAIBpKDJEXNZuDWJMzWZfpcBT4TmyOw4o-RY,1572
95
101
  lsst/pipe/base/tests/mocks/_data_id_match.py,sha256=v33QZhZm-srXZAXZ8NbNKGN-_ql4AzaArBUk1lxhyss,7474
96
- lsst/pipe/base/tests/mocks/_in_memory_repo.py,sha256=l5j5b4ff_ATRTskIlzhlHr-w_GiIJ5-rfUd37vT2esA,16664
97
- lsst/pipe/base/tests/mocks/_pipeline_task.py,sha256=xa2vy3HuMQifV0KR5sKfKRySqxSFhy-f1cP4bJ9EXZg,30010
98
- lsst/pipe/base/tests/mocks/_storage_class.py,sha256=gC0czHURMk7PWj8N6dLxnY5V4HWX5i8ukb5SZbgWKy8,25257
99
- lsst_pipe_base-29.2025.3900.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
100
- lsst_pipe_base-29.2025.3900.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
101
- lsst_pipe_base-29.2025.3900.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
102
- lsst_pipe_base-29.2025.3900.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
103
- lsst_pipe_base-29.2025.3900.dist-info/METADATA,sha256=bF-13cEZKbZWDYfVn5FmgfyEykJIQFzbFmibD8Xh6KM,2195
104
- lsst_pipe_base-29.2025.3900.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
105
- lsst_pipe_base-29.2025.3900.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
106
- lsst_pipe_base-29.2025.3900.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
107
- lsst_pipe_base-29.2025.3900.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
108
- lsst_pipe_base-29.2025.3900.dist-info/RECORD,,
102
+ lsst/pipe/base/tests/mocks/_pipeline_task.py,sha256=_n16lDsk3ItWi2J28Qheuqphr4aaCK6CN9acmJ1hAqI,30692
103
+ lsst/pipe/base/tests/mocks/_repo.py,sha256=OTJw_fi37w7bkZbbLa7z51W-45zxySAnLbV7Qv_aSB4,27423
104
+ lsst/pipe/base/tests/mocks/_storage_class.py,sha256=zLZHehfWfzr09suJcPJ55TsTkTIujRqywqXSuYvBDSI,25454
105
+ lsst_pipe_base-29.2025.4000.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
106
+ lsst_pipe_base-29.2025.4000.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
107
+ lsst_pipe_base-29.2025.4000.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
108
+ lsst_pipe_base-29.2025.4000.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
109
+ lsst_pipe_base-29.2025.4000.dist-info/METADATA,sha256=paUAsngc3fOdfUSEsM9R6PpBHT4DX3Ugz-QCeBWJ2wE,2234
110
+ lsst_pipe_base-29.2025.4000.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
111
+ lsst_pipe_base-29.2025.4000.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
112
+ lsst_pipe_base-29.2025.4000.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
113
+ lsst_pipe_base-29.2025.4000.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
114
+ lsst_pipe_base-29.2025.4000.dist-info/RECORD,,