lsst-daf-butler 29.2025.1500__tar.gz → 29.2025.1600__tar.gz

{lsst_daf_butler-29.2025.1500/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.1600}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.1500
+Version: 29.2025.1600
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.1500 → lsst_daf_butler-29.2025.1600}/python/lsst/daf/butler/_config.py

@@ -1254,7 +1254,7 @@ class ConfigSubset(Config):
 
         Global defaults, at lowest priority, are found in the ``config``
         directory of the butler source tree. Additional defaults can be
-        defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS``
+        defined using the environment variable ``$DAF_BUTLER_CONFIG_PATH``
         which is a PATH-like variable where paths at the front of the list
         have priority over those later.
 

{lsst_daf_butler-29.2025.1500 → lsst_daf_butler-29.2025.1600}/python/lsst/daf/butler/configs/datastores/formatters.yaml

@@ -86,7 +86,7 @@ TaskMetadata: lsst.daf.butler.formatters.json.JsonFormatter
 SpectractorSpectrum: lsst.atmospec.formatters.SpectractorSpectrumFormatter
 SpectractorImage: lsst.atmospec.formatters.SpectractorImageFormatter
 SpectractorFitParameters: lsst.atmospec.formatters.SpectractorFitParametersFormatter
-ScarletModelData: lsst.daf.butler.formatters.json.JsonFormatter
+ScarletModelData: lsst.meas.extensions.scarlet.io.ScarletModelFormatter
 MetricMeasurementBundle: lsst.daf.butler.formatters.json.JsonFormatter
 MultipleCellCoadd: lsst.cell_coadds.CellCoaddFitsFormatter
 NNModelPackagePayload: lsst.meas.transiNet.modelPackages.NNModelPackageFormatter

{lsst_daf_butler-29.2025.1500 → lsst_daf_butler-29.2025.1600}/python/lsst/daf/butler/configs/storageClasses.yaml

@@ -409,6 +409,9 @@ storageClasses:
     pytype: spectractor.fit.fitter.FitParameters
   ScarletModelData:
     pytype: lsst.scarlet.lite.io.ScarletModelData
+    parameters:
+      - blend_id
+    delegate: lsst.meas.extensions.scarlet.io.ScarletModelDelegate
   MetricMeasurementBundle:
     pytype: lsst.analysis.tools.interfaces.MetricMeasurementBundle
   MultipleCellCoadd:

{lsst_daf_butler-29.2025.1500 → lsst_daf_butler-29.2025.1600}/python/lsst/daf/butler/direct_query_driver/_driver.py

@@ -56,6 +56,7 @@ from ..queries.driver import (
     QueryDriver,
     ResultPage,
 )
+from ..queries.predicate_constraints_summary import PredicateConstraintsSummary
 from ..queries.result_specs import (
     DataCoordinateResultSpec,
     DatasetRefResultSpec,
@@ -68,7 +69,6 @@ from ..registry.interfaces import ChainedCollectionRecord, CollectionRecord
 from ..registry.managers import RegistryManagerInstances
 from ..registry.wildcards import CollectionWildcard
 from ._postprocessing import Postprocessing
-from ._predicate_constraints_summary import PredicateConstraintsSummary
 from ._query_analysis import (
     QueryCollectionAnalysis,
     QueryFindFirstAnalysis,

{lsst_daf_butler-29.2025.1500 → lsst_daf_butler-29.2025.1600}/python/lsst/daf/butler/queries/_query.py

@@ -33,6 +33,8 @@ from collections.abc import Iterable, Mapping, Set
 from types import EllipsisType
 from typing import Any, final
 
+import astropy.table
+
 from lsst.utils.iteration import ensure_iterable
 
 from .._dataset_type import DatasetType
@@ -49,6 +51,7 @@ from ._identifiers import IdentifierContext, interpret_identifier
 from .convert_args import convert_where_args
 from .driver import QueryDriver
 from .expression_factory import ExpressionFactory
+from .predicate_constraints_summary import PredicateConstraintsSummary
 from .result_specs import (
     DataCoordinateResultSpec,
     DatasetRefResultSpec,
@@ -586,7 +589,7 @@ class Query(QueryBase):
         return query
 
     def join_data_coordinates(self, iterable: Iterable[DataCoordinate]) -> Query:
-        """Return a new query that joins in an explicit table of data IDs.
+        """Return a new query that joins in an explicit iterable of data IDs.
 
         Parameters
         ----------
@@ -618,6 +621,50 @@ class Query(QueryBase):
             driver=self._driver,
         )
 
+    def join_data_coordinate_table(self, table: astropy.table.Table) -> Query:
+        """Return a new query that joins in an explicit table of data IDs.
+
+        Parameters
+        ----------
+        table : `astropy.table.Table`
+            A table of data IDs to join.  Columns must be dimension names, and
+            columns for dimensions whose values that are implied by others are
+            ignored.  If there is no column for a required dimension is missing
+            but is fully constrained to a literal by a previous `where` call, a
+            constant-valued column will be added.
+
+        Returns
+        -------
+        query : `Query`
+            A new query object with the data IDs joined in.
+        """
+        if not len(table):
+            raise InvalidQueryError("Cannot upload an empty data coordinate set.")
+        column_names = set(table.colnames)
+        dimensions = self._driver.universe.conform(column_names)
+        # To avoid numpy scalar types that will upset SQLAlchemy, we turn the
+        # columns we care about into lists of regular Python scalars.  We do
+        # this in dimensions.required order so we can zip the values of this
+        # dict later to make data ID 'required_values' tuples.
+        column_lists = {d: table[d].data.tolist() if d in column_names else None for d in dimensions.required}
+        if not column_names.issuperset(dimensions.required):
+            # If columns are missing, see if they're fixed by a previous
+            # `where` call or equivalent.
+            predicate_summary = PredicateConstraintsSummary(self._tree.predicate)
+            missing = dimensions.required - column_names
+            provided_by_predicate = predicate_summary.constraint_data_id.keys() & missing
+            missing -= provided_by_predicate
+            if missing:
+                raise InvalidQueryError(f"Data coordinate table is missing required dimension(s) {missing}.")
+            if provided_by_predicate:
+                for k in provided_by_predicate:
+                    column_lists[k] = [predicate_summary.constraint_data_id[k]] * len(table)
+        key = self._driver.upload_data_coordinates(dimensions, zip(*column_lists.values(), strict=True))
+        return Query(
+            tree=self._tree.join_data_coordinate_upload(dimensions=dimensions, key=key),
+            driver=self._driver,
+        )
+
     def join_dimensions(self, dimensions: Iterable[str] | DimensionGroup) -> Query:
         """Return a new query that joins the logical tables for additional
         dimensions.

lsst_daf_butler-29.2025.1500/python/lsst/daf/butler/direct_query_driver/_predicate_constraints_summary.py → lsst_daf_butler-29.2025.1600/python/lsst/daf/butler/queries/predicate_constraints_summary.py

@@ -31,8 +31,8 @@ from typing import Any
 
 from .._exceptions import InvalidQueryError
 from ..dimensions import DataCoordinate, DataIdValue, DimensionGroup, DimensionUniverse
-from ..queries import tree as qt
-from ..queries.visitors import ColumnExpressionVisitor, PredicateVisitFlags, SimplePredicateVisitor
+from . import tree as qt
+from .visitors import ColumnExpressionVisitor, PredicateVisitFlags, SimplePredicateVisitor
 
 
 class PredicateConstraintsSummary:

lsst_daf_butler-29.2025.1600/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.1600"

{lsst_daf_butler-29.2025.1500 → lsst_daf_butler-29.2025.1600/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.1500
+Version: 29.2025.1600
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.1500 → lsst_daf_butler-29.2025.1600}/python/lsst_daf_butler.egg-info/SOURCES.txt

@@ -147,7 +147,6 @@ python/lsst/daf/butler/direct_butler/_direct_butler_collections.py
 python/lsst/daf/butler/direct_query_driver/__init__.py
 python/lsst/daf/butler/direct_query_driver/_driver.py
 python/lsst/daf/butler/direct_query_driver/_postprocessing.py
-python/lsst/daf/butler/direct_query_driver/_predicate_constraints_summary.py
 python/lsst/daf/butler/direct_query_driver/_query_analysis.py
 python/lsst/daf/butler/direct_query_driver/_query_builder.py
 python/lsst/daf/butler/direct_query_driver/_result_page_converter.py
@@ -177,6 +176,7 @@ python/lsst/daf/butler/queries/convert_args.py
 python/lsst/daf/butler/queries/driver.py
 python/lsst/daf/butler/queries/expression_factory.py
 python/lsst/daf/butler/queries/overlaps.py
+python/lsst/daf/butler/queries/predicate_constraints_summary.py
 python/lsst/daf/butler/queries/result_specs.py
 python/lsst/daf/butler/queries/visitors.py
 python/lsst/daf/butler/queries/tree/__init__.py

{lsst_daf_butler-29.2025.1500 → lsst_daf_butler-29.2025.1600}/tests/test_config.py

@@ -42,11 +42,11 @@ TESTDIR = os.path.abspath(os.path.dirname(__file__))
 def modified_environment(**environ):
     """Temporarily set environment variables.
 
-    >>> with modified_environment(DAF_BUTLER_CONFIG_PATHS="/somewhere"):
-    ...     os.environ["DAF_BUTLER_CONFIG_PATHS"] == "/somewhere"
+    >>> with modified_environment(DAF_BUTLER_CONFIG_PATH="/somewhere"):
+    ...     os.environ["DAF_BUTLER_CONFIG_PATH"] == "/somewhere"
     True
 
-    >>> "DAF_BUTLER_CONFIG_PATHS" != "/somewhere"
+    >>> "DAF_BUTLER_CONFIG_PATH" != "/somewhere"
     True
 
     Parameters

{lsst_daf_butler-29.2025.1500 → lsst_daf_butler-29.2025.1600}/tests/test_query_interface.py

@@ -25,7 +25,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-"""Tests for the public Butler._query interface and the Pydantic models that
+"""Tests for the public Butler.query interface and the Pydantic models that
 back it using a mock column-expression visitor and a mock QueryDriver
 implementation.
 
@@ -46,7 +46,9 @@ import uuid
 from collections.abc import Iterable, Iterator, Mapping, Set
 from typing import Any
 
+import astropy.table
 import astropy.time
+import numpy as np
 
 from lsst.daf.butler import (
     CollectionType,
@@ -1475,6 +1477,42 @@ class QueryTestCase(unittest.TestCase):
                 ]
             )
 
+    def test_join_data_coordinate_table(self) -> None:
+        """Test joining a data coordinates via a table."""
+        query = self.query()
+        x = query.expression_factory
+        self.assertFalse(query.constraint_dimensions)
+        query = query.where(x.skymap == "m")
+        self.assertEqual(query.constraint_dimensions, self.universe.conform(["skymap"]))
+        with self.assertRaises(InvalidQueryError):
+            # Cannot join a table with no rows.
+            query.join_data_coordinate_table(
+                astropy.table.Table({"instrument": np.array([]), "visit": np.array([])})
+            )
+        with self.assertRaises(InvalidQueryError):
+            # Cannot join a table with missing required dimensions.
+            query.join_data_coordinate_table(astropy.table.Table({"visit": np.array([1, 3, 5])}))
+        upload_table = astropy.table.Table({"tract": np.array([2, 5, 7], dtype=np.int64)})
+        raw_rows = frozenset([("m", 2), ("m", 5), ("m", 7)])
+        query = query.join_data_coordinate_table(upload_table)
+        self.assertEqual(query.constraint_dimensions, self.universe.conform(["skymap", "tract"]))
+        results = query.dimension_records("patch")
+        with self.assertRaises(_TestQueryExecution) as cm:
+            list(results)
+        tree = cm.exception.tree
+        driver = cm.exception.driver
+        self.assertEqual(str(tree.predicate), "skymap == 'm'")
+        self.assertEqual(tree.dimensions, self.universe.conform(["patch"]))
+        self.assertFalse(tree.materializations)
+        self.assertFalse(tree.datasets)
+        ((key, upload_dimensions),) = tree.data_coordinate_uploads.items()
+        self.assertEqual(upload_dimensions, self.universe.conform(["tract"]))
+        self.assertEqual(driver.data_coordinate_uploads[key], (upload_dimensions, raw_rows))
+        for row in driver.data_coordinate_uploads[key][1]:
+            for value in row:
+                # Make sure numpy scalar types have been cast to builtins.
+                self.assertIsInstance(value, (int, str))
+
     def test_dimension_record_iteration(self) -> None:
         """Tests for DimensionRecordQueryResult iteration."""
 

lsst_daf_butler-29.2025.1500/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.1500"