lsst-daf-butler 29.2025.1000__tar.gz → 29.2025.1100__tar.gz

{lsst_daf_butler-29.2025.1000/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.1100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lsst-daf-butler
-Version: 29.2025.1000
+Version: 29.2025.1100
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/doc/lsst.daf.butler/queries.rst

@@ -167,6 +167,8 @@ filter query results based on that property of datasets.
 
 Registry methods accepting user expressions also accept a ``bind`` parameter, which is a mapping from identifier name to its corresponding value.
 Identifiers appearing in user expressions will be replaced with the corresponding value from this mapping.
+Bind identifiers in expressions are specified with a preceding colon (e.g. ``:id``).
+The legacy format where bind identifiers can be specified without a colon is still supported, but discouraged, and will be deprecated and removed in the future.
 Using the ``bind`` parameter is encouraged when possible to simplify rendering of the query strings.
 A partial example of comparing two approaches, without and with ``bind``:
 
@@ -186,7 +188,7 @@ A partial example of comparing two approaches, without and with ``bind``:
     # Same functionality using bind parameter
     result = butler.query_datasets(
         dataset_type,
-        where="instrument = instrument_name AND visit = visit_id",
+        where="instrument = :instrument_name AND visit = :visit_id",
         bind={"instrument_name": instrument_name, "visit_id": visit_id},
     )
 
@@ -201,7 +203,7 @@ An example of this feature:
     visit_ids = (12345, 12346, 12350)
     result = butler.query_datasets(
         dataset_type,
-        where="instrument = instrument_name AND visit IN (visit_ids)",
+        where="instrument = :instrument_name AND visit IN (:visit_ids)",
         bind={"instrument_name": instrument_name, "visit_ids": visit_ids},
     )
 

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/direct_query_driver/_driver.py

@@ -156,7 +156,7 @@ class DirectQueryDriver(QueryDriver):
         self._exit_stack: ExitStack | None = None
         self._raw_page_size = raw_page_size
         self._postprocessing_filter_factor = postprocessing_filter_factor
-        self._constant_rows_limit = constant_rows_limit
+        self._constant_rows_limit = min(constant_rows_limit, db.get_constant_rows_max())
         self._cursors: set[_Cursor] = set()
 
     def __enter__(self) -> None:

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/queries/_expression_strings.py

@@ -230,6 +230,13 @@ class _ConversionVisitor(TreeVisitor[_VisitorResult]):
         else:
             return _ColExpr(column_expression)
 
+    def visitBind(self, name: str, node: Node) -> _VisitorResult:
+        name = name.lower()
+        if name not in self.context.bind:
+            raise InvalidQueryError("Name {name!r} is not in the bind map.")
+        # Logic in visitIdentifier handles binds.
+        return self.visitIdentifier(name, node)
+
     def visitNumericLiteral(self, value: str, node: Node) -> _VisitorResult:
         numeric: int | float
         try:

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/registry/databases/sqlite.py

@@ -398,6 +398,12 @@ class SqliteDatabase(Database):
         ]
         return sqlalchemy.sql.union_all(*selects).alias(name)
 
+    def get_constant_rows_max(self) -> int:
+        # Docstring inherited.
+        # This is the default SQLITE_MAX_COMPOUND_SELECT (see
+        # https://www.sqlite.org/limits.html):
+        return 500
+
     @property
     def has_distinct_on(self) -> bool:
         # Docstring inherited.

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/registry/interfaces/_database.py

@@ -1918,7 +1918,7 @@ class Database(ABC):
         This should reflect typical performance profiles (or a guess at these),
         not just hard database engine limits.
         """
-        return 100
+        return 1000
 
     @property
     @abstractmethod

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/registry/queries/expressions/_predicate.py

@@ -353,25 +353,7 @@ class PredicateConversionVisitor(TreeVisitor[VisitorResult]):
     def visitIdentifier(self, name: str, node: Node) -> VisitorResult:
         # Docstring inherited.
         if name in self.bind:
-            value = self.bind[name]
-            if isinstance(value, list | tuple | Set):
-                elements = []
-                all_dtypes = set()
-                for item in value:
-                    dtype = type(item)
-                    all_dtypes.add(dtype)
-                    elements.append(ColumnExpression.literal(item, dtype=dtype))
-                if len(all_dtypes) > 1:
-                    raise ExpressionTypeError(
-                        f"Mismatched types in bind iterable: {value} has a mix of {all_dtypes}."
-                    )
-                elif not elements:
-                    # Empty container
-                    return ColumnContainer.sequence([])
-                else:
-                    (dtype,) = all_dtypes
-                    return ColumnContainer.sequence(elements, dtype=dtype)
-            return ColumnExpression.literal(value, dtype=type(value))
+            return self.visitBind(name, node)
         tag: ColumnTag
         match categorizeConstant(name):
             case ExpressionConstant.INGEST_DATE:
@@ -403,6 +385,31 @@ class PredicateConversionVisitor(TreeVisitor[VisitorResult]):
             assert isinstance(element, Dimension)
             return ColumnExpression.reference(tag, element.primaryKey.getPythonType())
 
+    def visitBind(self, name: str, node: Node) -> VisitorResult:
+        # Docstring inherited.
+        if name not in self.bind:
+            raise UserExpressionError(f"Name {name!r} is not in the bind map.")
+
+        value = self.bind[name]
+        if isinstance(value, list | tuple | Set):
+            elements = []
+            all_dtypes = set()
+            for item in value:
+                dtype = type(item)
+                all_dtypes.add(dtype)
+                elements.append(ColumnExpression.literal(item, dtype=dtype))
+            if len(all_dtypes) > 1:
+                raise ExpressionTypeError(
+                    f"Mismatched types in bind iterable: {value} has a mix of {all_dtypes}."
+                )
+            elif not elements:
+                # Empty container
+                return ColumnContainer.sequence([])
+            else:
+                (dtype,) = all_dtypes
+                return ColumnContainer.sequence(elements, dtype=dtype)
+        return ColumnExpression.literal(value, dtype=type(value))
+
     def visitIsIn(
         self, lhs: VisitorResult, values: list[VisitorResult], not_in: bool, node: Node
     ) -> VisitorResult:

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/registry/queries/expressions/check.py

@@ -212,16 +212,8 @@ class InspectionVisitor(TreeVisitor[TreeSummary]):
     def visitIdentifier(self, name: str, node: Node) -> TreeSummary:
         # Docstring inherited from TreeVisitor.visitIdentifier
         if name in self.bind:
-            value = self.bind[name]
-            if isinstance(value, list | tuple | Set):
-                # This can happen on rhs of IN operator, if there is only one
-                # element in the list then take it.
-                if len(value) == 1:
-                    return TreeSummary(dataIdValue=next(iter(value)))
-                else:
-                    return TreeSummary()
-            else:
-                return TreeSummary(dataIdValue=value)
+            return self.visitBind(name, node)
+
         constant = categorizeConstant(name)
         if constant is ExpressionConstant.INGEST_DATE:
             return TreeSummary(hasIngestDate=True)
@@ -238,6 +230,23 @@ class InspectionVisitor(TreeVisitor[TreeSummary]):
         else:
             return TreeSummary(dimensions=set(element.minimal_group.names), columns={element.name: {column}})
 
+    def visitBind(self, name: str, node: Node) -> TreeSummary:
+        if name not in self.bind:
+            raise UserExpressionError(f"Name {name!r} is not in the bind map.")
+        value = self.bind[name]
+        match value:
+            case list() | tuple() | Set():
+                # This can happen on rhs of IN operator, if there is only one
+                # element in the list then take it.
+                match len(value):
+                    case 1:
+                        (value,) = value
+                        return TreeSummary(dataIdValue=value)
+                    case _:
+                        return TreeSummary()
+            case _:
+                return TreeSummary(dataIdValue=value)
+
     def visitUnaryOp(self, operator: str, operand: TreeSummary, node: Node) -> TreeSummary:
         # Docstring inherited from TreeVisitor.visitUnaryOp
         return operand

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/registry/queries/expressions/normalForm.py

@@ -1040,6 +1040,10 @@ class TransformationVisitor(TreeVisitor[TransformationWrapper]):
         # Docstring inherited from TreeVisitor.visitIdentifier
         return Opaque(node, PrecedenceTier.TOKEN)
 
+    def visitBind(self, name: str, node: Node) -> TransformationWrapper:
+        # Docstring inherited from TreeVisitor.visitBind
+        return Opaque(node, PrecedenceTier.TOKEN)
+
     def visitUnaryOp(
         self,
         operator: str,

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/registry/queries/expressions/parser/exprTree.py

@@ -256,6 +256,29 @@ class Identifier(Node):
         return "{name}".format(**vars(self))
 
 
+class BindName(Node):
+    """Node representing a bind name.
+
+    Value of the bind is its name, which is a simple identifier.
+
+    Parameters
+    ----------
+    name : str
+        Bind name.
+    """
+
+    def __init__(self, name: str):
+        Node.__init__(self)
+        self.name = name
+
+    def visit(self, visitor: TreeVisitor) -> Any:
+        # Docstring inherited from Node.visit
+        return visitor.visitBind(self.name, self)
+
+    def __str__(self) -> str:
+        return "{name}".format(**vars(self))
+
+
 class RangeLiteral(Node):
     """Node representing range literal appearing in `IN` list.
 

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/registry/queries/expressions/parser/parserLex.py

@@ -128,6 +128,7 @@ class ParserLex:
         # 'DURATION_LITERAL',
         "QUALIFIED_IDENTIFIER",
         "SIMPLE_IDENTIFIER",
+        "BIND_NAME",
         "LPAREN",
         "RPAREN",
         "EQ",
@@ -221,6 +222,14 @@ class ParserLex:
             t.type = "SIMPLE_IDENTIFIER"
         return t
 
+    # we only support ASCII in identifier names
+    def t_BIND_NAME(self, t: LexToken) -> LexToken:
+        """[:][a-zA-Z_][a-zA-Z0-9_]*"""
+        # Drop colon to get the name.
+        t.value = t.value[1:]
+        t.type = "BIND_NAME"
+        return t
+
     def t_error(self, t: LexToken) -> None:
         """Error handling rule"""
         lexer = t.lexer

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/registry/queries/expressions/parser/parserYacc.py

@@ -47,6 +47,7 @@ except ImportError:
 
 from .exprTree import (
     BinaryOp,
+    BindName,
     Identifier,
     IsIn,
     Node,
@@ -59,7 +60,7 @@ from .exprTree import (
     UnaryOp,
     function_call,
 )
-from .parserLex import LexToken, ParserLex
+from .parserLex import LexToken, ParserLex, ParserLexError
 from .ply import yacc
 
 
@@ -223,8 +224,7 @@ class ParseError(ParserYaccError):
         self.pos = pos
         self.lineno = lineno
         self.posInLine = self._posInLine()
-        msg = "Syntax error at or near '{0}' (line: {1}, pos: {2})"
-        msg = msg.format(token, lineno, self.posInLine + 1)
+        msg = f"Syntax error at or near '{token}' (line: {lineno}, pos: {self.posInLine + 1})"
         ParserYaccError.__init__(self, msg)
 
     def _posInLine(self) -> int:
@@ -288,7 +288,11 @@ class ParserYacc:
         # make lexer
         if lexer is None:
             lexer = ParserLex.make_lexer()
-        tree = self.parser.parse(input=input, lexer=lexer, debug=debug, tracking=tracking)
+        try:
+            tree = self.parser.parse(input=input, lexer=lexer, debug=debug, tracking=tracking)
+        except ParserLexError as exc:
+            # Convert it into ParserYaccError
+            raise ParserYaccError(str(exc)) from exc
         return tree
 
     tokens = ParserLex.tokens[:]
@@ -371,14 +375,21 @@ class ParserYacc:
     def p_literal_or_id_list(cls, p: YaccProduction) -> None:
         """literal_or_id_list : literal_or_id_list COMMA literal
         | literal_or_id_list COMMA identifier
+        | literal_or_id_list COMMA bind_name
         | literal
         | identifier
+        | bind_name
         """
         if len(p) == 2:
             p[0] = [p[1]]
         else:
             p[0] = p[1] + [p[3]]
 
+    @classmethod
+    def p_bind_name(cls, p: YaccProduction) -> None:
+        """bind_name : BIND_NAME"""
+        p[0] = BindName(p[1])
+
     @classmethod
     def p_bit_expr(cls, p: YaccProduction) -> None:
         """bit_expr : bit_expr ADD bit_expr
@@ -395,17 +406,11 @@ class ParserYacc:
 
     @classmethod
     def p_simple_expr_lit(cls, p: YaccProduction) -> None:
-        """simple_expr : literal"""
-        p[0] = p[1]
-
-    @classmethod
-    def p_simple_expr_id(cls, p: YaccProduction) -> None:
-        """simple_expr : identifier"""
-        p[0] = p[1]
-
-    @classmethod
-    def p_simple_expr_function_call(cls, p: YaccProduction) -> None:
-        """simple_expr : function_call"""
+        """simple_expr : literal
+        | identifier
+        | bind_name
+        | function_call
+        """
         p[0] = p[1]
 
     @classmethod

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/registry/queries/expressions/parser/treeVisitor.py

@@ -29,7 +29,7 @@ from __future__ import annotations
 
 __all__ = ["TreeVisitor"]
 
-from abc import abstractmethod
+from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Generic, TypeVar
 
 if TYPE_CHECKING:
@@ -41,7 +41,7 @@ if TYPE_CHECKING:
 T = TypeVar("T")
 
 
-class TreeVisitor(Generic[T]):
+class TreeVisitor(Generic[T], ABC):
     """Definition of interface for visitor classes.
 
     Visitors and tree node classes implement Visitor pattern for tree
@@ -121,6 +121,18 @@ class TreeVisitor(Generic[T]):
             Corresponding tree node, mostly useful for diagnostics.
         """
 
+    @abstractmethod
+    def visitBind(self, name: str, node: Node) -> T:
+        """Visit BindName node.
+
+        Parameters
+        ----------
+        name : `str`
+            Bind name.
+        node : `Node`
+            Corresponding tree node, mostly useful for diagnostics.
+        """
+
     @abstractmethod
     def visitUnaryOp(self, operator: str, operand: T, node: Node) -> T:
         """Visit UnaryOp node.

{lsst_daf_butler-29.2025.1000 → lsst_daf_butler-29.2025.1100}/python/lsst/daf/butler/registry/tests/_registry.py

@@ -2812,7 +2812,7 @@ class RegistryTests(ABC):
             self.assertEqual(len(datasets), expect_len)
 
             # same with bind using datetime or astropy Time
-            where = f"ingest_date {op} ingest_time"
+            where = f"ingest_date {op} :ingest_time"
             datasets = list(
                 registry.queryDatasets(..., collections=..., where=where, bind={"ingest_time": dt})
             )
@@ -2881,33 +2881,33 @@ class RegistryTests(ABC):
         # the expression.
 
         # t1 is before the start of i1, so this should not include i1.
-        self.assertEqual(ids[:i1], query("visit.timespan OVERLAPS (null, t1)"))
+        self.assertEqual(ids[:i1], query("visit.timespan OVERLAPS (null, :t1)"))
         # t2 is exactly at the start of i2, but ends are exclusive, so these
         # should not include i2.
-        self.assertEqual(ids[i1:i2], query("(t1, t2) OVERLAPS visit.timespan"))
+        self.assertEqual(ids[i1:i2], query("(:t1, :t2) OVERLAPS visit.timespan"))
         if self.supportsExtendedTimeQueryOperators:
-            self.assertEqual(ids[:i2], query("visit.timespan < (t2, t4)"))
+            self.assertEqual(ids[:i2], query("visit.timespan < (:t2, :t4)"))
         # t3 is in the middle of i3, so this should include i3.
-        self.assertEqual(ids[i2 : i3 + 1], query("visit.timespan OVERLAPS ts23"))
+        self.assertEqual(ids[i2 : i3 + 1], query("visit.timespan OVERLAPS :ts23"))
         # This one should not include t3 by the same reasoning.
         if self.supportsExtendedTimeQueryOperators:
-            self.assertEqual(ids[i3 + 1 :], query("visit.timespan > (t1, t3)"))
+            self.assertEqual(ids[i3 + 1 :], query("visit.timespan > (:t1, :t3)"))
         # t4 is exactly at the end of i4, so this should include i4.
-        self.assertEqual(ids[i3 : i4 + 1], query(f"visit.timespan OVERLAPS (T'{t3.tai.isot}', t4)"))
+        self.assertEqual(ids[i3 : i4 + 1], query(f"visit.timespan OVERLAPS (T'{t3.tai.isot}', :t4)"))
         # i4's upper bound of t4 is exclusive so this should not include t4.
-        self.assertEqual(ids[i4 + 1 :], query("visit.timespan OVERLAPS (t4, NULL)"))
+        self.assertEqual(ids[i4 + 1 :], query("visit.timespan OVERLAPS (:t4, NULL)"))
 
         # Now some timespan vs. time scalar queries.
-        self.assertEqual(ids[i3 : i3 + 1], query("visit.timespan OVERLAPS t3"))
+        self.assertEqual(ids[i3 : i3 + 1], query("visit.timespan OVERLAPS :t3"))
         self.assertEqual(ids[i3 : i3 + 1], query(f"T'{t3.tai.isot}' OVERLAPS visit.timespan"))
         if self.supportsExtendedTimeQueryOperators:
-            self.assertEqual(ids[:i2], query("visit.timespan < t2"))
-            self.assertEqual(ids[:i2], query("t2 > visit.timespan"))
-            self.assertEqual(ids[i3 + 1 :], query("visit.timespan > t3"))
-            self.assertEqual(ids[i3 + 1 :], query("t3 < visit.timespan"))
+            self.assertEqual(ids[:i2], query("visit.timespan < :t2"))
+            self.assertEqual(ids[:i2], query(":t2 > visit.timespan"))
+            self.assertEqual(ids[i3 + 1 :], query("visit.timespan > :t3"))
+            self.assertEqual(ids[i3 + 1 :], query(":t3 < visit.timespan"))
 
         # Empty timespans should not overlap anything.
-        self.assertEqual([], query("visit.timespan OVERLAPS (t3, t2)"))
+        self.assertEqual([], query("visit.timespan OVERLAPS (:t3, :t2)"))
 
         # Make sure that expanded data IDs include the timespans.
         results = list(
@@ -2984,7 +2984,9 @@ class RegistryTests(ABC):
         self.load_data(butler, "base.yaml", "datasets.yaml")
         self.assertEqual(
             set(registry.queryDatasets("flat", band="r", collections=...)),
-            set(registry.queryDatasets("flat", where="band=my_band", bind={"my_band": "r"}, collections=...)),
+            set(
+                registry.queryDatasets("flat", where="band=:my_band", bind={"my_band": "r"}, collections=...)
+            ),
         )
 
     def testQueryIntRangeExpressions(self):
@@ -3223,7 +3225,7 @@ class RegistryTests(ABC):
             "(Dimension element name cannot be inferred in this context.)"
             "|(Unrecognized identifier 'timespan')",
         ):
-            list(registry.queryDataIds(["detector"], where="timespan.end < time", bind=bind))
+            list(registry.queryDataIds(["detector"], where="timespan.end < :time", bind=bind))
 
     def testQueryDataIdsOrderBy(self):
         """Test order_by and limit on result returned by queryDataIds()."""
@@ -3401,13 +3403,13 @@ class RegistryTests(ABC):
             Test("tract,visit", where="instrument='Cam1' AND skymap='SkyMap5'", exception=DataIdValueError),
             Test(
                 "tract,visit",
-                where="instrument=cam AND skymap=map",
+                where="instrument=:cam AND skymap=:map",
                 bind={"cam": "Cam1", "map": "SkyMap1"},
                 count=6,
             ),
             Test(
                 "tract,visit",
-                where="instrument=cam AND skymap=map",
+                where="instrument=:cam AND skymap=:map",
                 bind={"cam": "Cam", "map": "SkyMap"},
                 exception=DataIdValueError,
             ),
@@ -3541,7 +3543,7 @@ class RegistryTests(ABC):
 
         result = registry.queryDimensionRecords("detector", where="instrument='Cam1'")
         self.assertEqual(result.count(), 4)
-        result = registry.queryDimensionRecords("detector", where="instrument=instr", bind={"instr": "Cam1"})
+        result = registry.queryDimensionRecords("detector", where="instrument=:instr", bind={"instr": "Cam1"})
         self.assertTrue(result.any())
         self.assertEqual(result.count(), 4)
 
@@ -3568,7 +3570,7 @@ class RegistryTests(ABC):
                     DataIdValueError, "Unknown values specified for governor dimension"
                 ):
                     result = registry.queryDimensionRecords(
-                        "detector", where="instrument=instr", bind={"instr": "NotCam1"}
+                        "detector", where="instrument=:instr", bind={"instr": "NotCam1"}
                     )
                     result.count()