lsst-daf-butler 29.2025.3700__tar.gz → 29.2025.3800__tar.gz

{lsst_daf_butler-29.2025.3700/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.3800}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.3700
+Version: 29.2025.3800
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License
@@ -63,7 +63,8 @@ Please cite [the SPIE paper](https://doi.org/10.1117/12.2629569) when using this
 
 This is a **Python 3 only** package (we assume Python 3.11 or higher).
 
-* SPIE Paper from 2022: [The Vera C. Rubin Observatory Data Butler and Pipeline Execution System](https://doi.org/10.1117/12.2629569) ([arXiv](https://arxiv.org/abs/2206.14941))
+* SPIE paper from 2024: [Converting Rubin Observatory's data butler to a client/server architecture](https://doi.org/10.1117/12.3019130) ([Tech note](https://dmtn-288.lsst.io))
+* SPIE Paper from 2022: [The Vera C. Rubin Observatory Data Butler and Pipeline Execution System](https://doi.org/10.1117/12.2629569) ([arXiv](https://arxiv.org/abs/2206.14941)) \[Primary reference]
 * ADASS paper from 2019: [Abstracting the Storage and Retrieval of Image Data at the LSST](https://ui.adsabs.harvard.edu/abs/2019ASPC..523..653J/abstract).
 * Early design note: [DMTN-056](https://dmtn-056.lsst.io)
 
@@ -71,3 +72,9 @@ PyPI: [lsst-daf-butler](https://pypi.org/project/lsst-daf-butler/)
 
 This software is dual licensed under the GNU General Public License (version 3 of the License, or (at your option) any later version, and also under a 3-clause BSD license).
 Recipients may choose which of these licenses to use; please see the files gpl-3.0.txt and/or bsd_license.txt, respectively.
+
+## Arrow Memory Leaks
+
+From version 18 of arrow we have seen significant memory leaks when accessing parquet files using the default memory allocator.
+If you see such leaks the workaround is to set the `ARROW_DEFAULT_MEMORY_POOL` environment variable to `jemalloc` following the advice from [apache/arrow#45882](https://github.com/apache/arrow/issues/45882).
+For EUPS users this variable is automatically set.

{lsst_daf_butler-29.2025.3700 → lsst_daf_butler-29.2025.3800}/README.md

@@ -8,7 +8,8 @@ Please cite [the SPIE paper](https://doi.org/10.1117/12.2629569) when using this
 
 This is a **Python 3 only** package (we assume Python 3.11 or higher).
 
-* SPIE Paper from 2022: [The Vera C. Rubin Observatory Data Butler and Pipeline Execution System](https://doi.org/10.1117/12.2629569) ([arXiv](https://arxiv.org/abs/2206.14941))
+* SPIE paper from 2024: [Converting Rubin Observatory's data butler to a client/server architecture](https://doi.org/10.1117/12.3019130) ([Tech note](https://dmtn-288.lsst.io))
+* SPIE Paper from 2022: [The Vera C. Rubin Observatory Data Butler and Pipeline Execution System](https://doi.org/10.1117/12.2629569) ([arXiv](https://arxiv.org/abs/2206.14941)) \[Primary reference]
 * ADASS paper from 2019: [Abstracting the Storage and Retrieval of Image Data at the LSST](https://ui.adsabs.harvard.edu/abs/2019ASPC..523..653J/abstract).
 * Early design note: [DMTN-056](https://dmtn-056.lsst.io)
 
@@ -16,3 +17,9 @@ PyPI: [lsst-daf-butler](https://pypi.org/project/lsst-daf-butler/)
 
 This software is dual licensed under the GNU General Public License (version 3 of the License, or (at your option) any later version, and also under a 3-clause BSD license).
 Recipients may choose which of these licenses to use; please see the files gpl-3.0.txt and/or bsd_license.txt, respectively.
+
+## Arrow Memory Leaks
+
+From version 18 of arrow we have seen significant memory leaks when accessing parquet files using the default memory allocator.
+If you see such leaks the workaround is to set the `ARROW_DEFAULT_MEMORY_POOL` environment variable to `jemalloc` following the advice from [apache/arrow#45882](https://github.com/apache/arrow/issues/45882).
+For EUPS users this variable is automatically set.

{lsst_daf_butler-29.2025.3700 → lsst_daf_butler-29.2025.3800}/python/lsst/daf/butler/column_spec.py

@@ -52,7 +52,6 @@ from typing import (
     Any,
     ClassVar,
     Literal,
-    Optional,
     TypeAlias,
     Union,
     final,
@@ -254,7 +253,7 @@ class _BaseColumnSpec(pydantic.BaseModel, ABC):
         """
         base = self._get_base_annotated_type()
         if self.nullable:
-            return Optional[base]
+            return base | None
         return base
 
     @abstractmethod

{lsst_daf_butler-29.2025.3700 → lsst_daf_butler-29.2025.3800}/python/lsst/daf/butler/delegates/arrowtable.py

@@ -32,6 +32,7 @@ from __future__ import annotations
 __all__ = ["ArrowTableDelegate"]
 
 from collections.abc import Mapping
+from fnmatch import fnmatchcase
 from typing import TYPE_CHECKING, Any
 
 import pyarrow as pa
@@ -162,28 +163,33 @@ class ArrowTableDelegate(StorageClassDelegate):
                     parameters["columns"],
                     stringify=False,
                 )
+                # Ensure uniqueness, keeping order.
+                readColumns = list(dict.fromkeys(readColumns))
             else:
-                readColumns = list(ensure_iterable(parameters["columns"]))
+                readColumnsIn = list(dict.fromkeys(ensure_iterable(parameters["columns"])))
 
-                for column in readColumns:
+                if typeString == "pandas":
+                    # Exclude index columns from the subset.
+                    readColumnsIn = [
+                        name for name in readColumnsIn if name not in inMemoryDataset.index.names
+                    ]
+
+                readColumnsDict = {}
+                for column in readColumnsIn:
                     if not isinstance(column, str):
                         raise NotImplementedError(
                             f"InMemoryDataset of a {get_full_type_name(inMemoryDataset)} only "
                             "supports string column names."
                         )
-                    if column not in allColumns:
+                    found = False
+                    for allColumn in allColumns:
+                        if fnmatchcase(allColumn, column):
+                            found = True
+                            readColumnsDict[allColumn] = True
+                    if not found:
                         raise ValueError(f"Unrecognized column name {column!r}.")
 
-                if typeString == "pandas":
-                    # Exclude index columns from the subset.
-                    readColumns = [
-                        name
-                        for name in ensure_iterable(parameters["columns"])
-                        if name not in inMemoryDataset.index.names
-                    ]
-
-            # Ensure uniqueness, keeping order.
-            readColumns = list(dict.fromkeys(readColumns))
+                readColumns = list(readColumnsDict.keys())
 
             if typeString == "arrow":
                 return inMemoryDataset.select(readColumns)

{lsst_daf_butler-29.2025.3700 → lsst_daf_butler-29.2025.3800}/python/lsst/daf/butler/formatters/parquet.py

@@ -54,6 +54,7 @@ import json
 import logging
 import re
 from collections.abc import Iterable, Sequence
+from fnmatch import fnmatchcase
 from typing import TYPE_CHECKING, Any, cast
 
 import pyarrow as pa
@@ -154,14 +155,23 @@ class ParquetFormatter(FormatterV2):
 
                 if not has_pandas_multi_index:
                     # Ensure uniqueness, keeping order.
-                    par_columns = list(dict.fromkeys(ensure_iterable(par_columns)))
+                    par_columns_in = list(dict.fromkeys(ensure_iterable(par_columns)))
                     file_columns = [name for name in schema.names if not name.startswith("__")]
 
-                    for par_column in par_columns:
-                        if par_column not in file_columns:
+                    # Do case-sensitive glob-style matching, again ensuring
+                    # uniqueness and ordering.
+                    par_columns = {}
+                    for par_column in par_columns_in:
+                        found = False
+                        for file_column in file_columns:
+                            if fnmatchcase(file_column, par_column):
+                                found = True
+                                par_columns[file_column] = True
+                        if not found:
                             raise ValueError(
                                 f"Column {par_column} specified in parameters not available in parquet file."
                             )
+                    par_columns = list(par_columns.keys())
                 else:
                     par_columns = _standardize_multi_index_columns(
                         arrow_schema_to_pandas_index(schema),

{lsst_daf_butler-29.2025.3700 → lsst_daf_butler-29.2025.3800}/python/lsst/daf/butler/logging.py

@@ -36,7 +36,7 @@ from collections.abc import Callable, Generator, Iterable, Iterator
 from contextlib import contextmanager
 from logging import Formatter, LogRecord, StreamHandler
 from types import TracebackType
-from typing import IO, Any, ClassVar, overload
+from typing import IO, Any, ClassVar, Literal, overload
 
 from pydantic import BaseModel, ConfigDict, PrivateAttr, RootModel
 
@@ -174,7 +174,7 @@ class ButlerMDC:
             lno: int,
             msg: str,
             args: tuple,
-            exc_info: tuple | None,
+            exc_info: tuple | None | Literal[False],
             func: str | None = None,
             sinfo: TracebackType | None = None,
             **kwargs: Any,
@@ -182,7 +182,7 @@ class ButlerMDC:
             record = old_factory(name, level, fn, lno, msg, args, exc_info, func, sinfo, **kwargs)
             # Make sure we send a copy of the global dict in the record.
             mdc = MDCDict(cls._MDC)
-            if exc_info is not None:
+            if exc_info is not None and exc_info is not False:
                 _, ex, _ = exc_info
                 # TODO: this doesn't handle chained exceptions, fix on DM-47546
                 if hasattr(ex, "mdc"):

{lsst_daf_butler-29.2025.3700 → lsst_daf_butler-29.2025.3800}/python/lsst/daf/butler/tests/registry_data/spatial.py

@@ -55,7 +55,7 @@ from astropy.time import Time
 from astropy.wcs import WCS
 from matplotlib import pyplot
 
-import lsst.daf.butler  # noqa:F401; register Time/YAML conversions.
+import lsst.daf.butler  # register Time/YAML conversions. # noqa: F401
 from lsst.sphgeom import (
     ConvexPolygon,
     HealpixPixelization,

lsst_daf_butler-29.2025.3800/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.3800"

{lsst_daf_butler-29.2025.3700 → lsst_daf_butler-29.2025.3800/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.3700
+Version: 29.2025.3800
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License
@@ -63,7 +63,8 @@ Please cite [the SPIE paper](https://doi.org/10.1117/12.2629569) when using this
 
 This is a **Python 3 only** package (we assume Python 3.11 or higher).
 
-* SPIE Paper from 2022: [The Vera C. Rubin Observatory Data Butler and Pipeline Execution System](https://doi.org/10.1117/12.2629569) ([arXiv](https://arxiv.org/abs/2206.14941))
+* SPIE paper from 2024: [Converting Rubin Observatory's data butler to a client/server architecture](https://doi.org/10.1117/12.3019130) ([Tech note](https://dmtn-288.lsst.io))
+* SPIE Paper from 2022: [The Vera C. Rubin Observatory Data Butler and Pipeline Execution System](https://doi.org/10.1117/12.2629569) ([arXiv](https://arxiv.org/abs/2206.14941)) \[Primary reference]
 * ADASS paper from 2019: [Abstracting the Storage and Retrieval of Image Data at the LSST](https://ui.adsabs.harvard.edu/abs/2019ASPC..523..653J/abstract).
 * Early design note: [DMTN-056](https://dmtn-056.lsst.io)
 
@@ -71,3 +72,9 @@ PyPI: [lsst-daf-butler](https://pypi.org/project/lsst-daf-butler/)
 
 This software is dual licensed under the GNU General Public License (version 3 of the License, or (at your option) any later version, and also under a 3-clause BSD license).
 Recipients may choose which of these licenses to use; please see the files gpl-3.0.txt and/or bsd_license.txt, respectively.
+
+## Arrow Memory Leaks
+
+From version 18 of arrow we have seen significant memory leaks when accessing parquet files using the default memory allocator.
+If you see such leaks the workaround is to set the `ARROW_DEFAULT_MEMORY_POOL` environment variable to `jemalloc` following the advice from [apache/arrow#45882](https://github.com/apache/arrow/issues/45882).
+For EUPS users this variable is automatically set.

{lsst_daf_butler-29.2025.3700 → lsst_daf_butler-29.2025.3800}/tests/test_logging.py

@@ -118,8 +118,13 @@ class LoggingTestCase(unittest.TestCase):
         self.log.warning("warning message")
         self.log.critical("critical message")
         self.log.verbose("verbose message")
+        self.log.error("error message")
+        try:
+            raise RuntimeError("An error has occurred")
+        except RuntimeError:
+            self.log.exception("exception message")
 
-        self.assertEqual(len(self.handler.records), 4)
+        self.assertEqual(len(self.handler.records), 6)
 
         format_default = str(self.handler.records)
 
@@ -128,7 +133,7 @@ class LoggingTestCase(unittest.TestCase):
         format_override = str(self.handler.records)
 
         self.assertNotEqual(format_default, format_override)
-        self.assertEqual(format_override, "DEBUG\nWARNING\nCRITICAL\nVERBOSE")
+        self.assertEqual(format_override, "DEBUG\nWARNING\nCRITICAL\nVERBOSE\nERROR\nERROR")
 
         # Reset the log format and it should match the original text.
         self.handler.records.set_log_format(None)
@@ -284,11 +289,14 @@ class LoggingTestCase(unittest.TestCase):
             self.log.exception("Exception raised:")
             self.log.warning("Implied exception", exc_info=True)
             self.log.critical("Explicit exception", exc_info=e)
+            self.log.error("Explicit exception info", exc_info=(Exception, e, e.__traceback__))
             # Is original context used *only* when logging the exception?
             self.log.error("Something went wrong")
-        for i in [-4, -3, -2]:
+            self.log.error("Boolean disabled exc info", exc_info=False)
+        for i in [-6, -5, -4, -3]:
             self.assertEqual(self.handler.records[i].MDC, {"foo": "fum", "answer": "42"})
-        self.assertEqual(self.handler.records[-1].MDC, {"foo": "bar"})
+        for i in [-2, -1]:
+            self.assertEqual(self.handler.records[i].MDC, {"foo": "bar"})
 
         self.log.setLevel(logging.INFO)
         self.log.info("Normal log")

{lsst_daf_butler-29.2025.3700 → lsst_daf_butler-29.2025.3800}/tests/test_parquet.py

@@ -400,6 +400,10 @@ class ParquetFormatterDataFrameTestCase(unittest.TestCase):
         self.assertTrue(df1.loc[:, ["ddd"]].equals(df6))
         df7 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["a", "a"]})
         self.assertTrue(df1.loc[:, ["a"]].equals(df7))
+        df8 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["d*"]})
+        self.assertTrue(df1.loc[:, ["ddd", "dtn", "dtu"]].equals(df8))
+        df9 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["d*", "d*"]})
+        self.assertTrue(df1.loc[:, ["ddd", "dtn", "dtu"]].equals(df9))
         # Passing an unrecognized column should be a ValueError.
         with self.assertRaises(ValueError):
             self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["e"]})
@@ -989,6 +993,10 @@ class ParquetFormatterArrowAstropyTestCase(unittest.TestCase):
         _checkAstropyTableEquality(tab1[("ddd",)], tab6)
         tab7 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["a", "a"]})
         _checkAstropyTableEquality(tab1[("a",)], tab7)
+        tab8 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["d??"]})
+        _checkAstropyTableEquality(tab1[("ddd", "dtn", "dtu")], tab8)
+        tab9 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["d??", "a*"]})
+        _checkAstropyTableEquality(tab1[("ddd", "dtn", "dtu", "a")], tab9)
         # Passing an unrecognized column should be a ValueError.
         with self.assertRaises(ValueError):
             self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["e"]})
@@ -1401,6 +1409,18 @@ class ParquetFormatterArrowNumpyTestCase(unittest.TestCase):
             ],
             tab7,
         )
+        tab8 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["d??", "a*"]})
+        _checkNumpyTableEquality(
+            tab1[
+                [
+                    "ddd",
+                    "dtn",
+                    "dtu",
+                    "a",
+                ]
+            ],
+            tab8,
+        )
         # Passing an unrecognized column should be a ValueError.
         with self.assertRaises(ValueError):
             self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["e"]})
@@ -1673,6 +1693,8 @@ class ParquetFormatterArrowTableTestCase(unittest.TestCase):
         self.assertEqual(tab6, tab1.select(("ddd",)))
         tab7 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["a", "a"]})
         self.assertEqual(tab7, tab1.select(("a",)))
+        tab8 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["a*", "d??"]})
+        self.assertEqual(tab8, tab1.select(("a", "ddd", "dtn", "dtu")))
         # Passing an unrecognized column should be a ValueError.
         with self.assertRaises(ValueError):
             self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["e"]})
@@ -1984,6 +2006,9 @@ class ParquetFormatterArrowNumpyDictTestCase(unittest.TestCase):
         tab7 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["a", "a"]})
         subdict = {key: dict1[key] for key in ["a"]}
         _checkNumpyDictEquality(subdict, tab7)
+        tab8 = self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["d??", "a*"]})
+        subdict = {key: dict1[key] for key in ["ddd", "dtn", "dtu", "a"]}
+        _checkNumpyDictEquality(subdict, tab8)
         # Passing an unrecognized column should be a ValueError.
         with self.assertRaises(ValueError):
             self.butler.get(self.datasetType, dataId={}, parameters={"columns": ["e"]})

lsst_daf_butler-29.2025.3700/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.3700"