lsst-daf-butler 29.2025.4800__py3-none-any.whl → 30.0.0rc1__py3-none-any.whl

lsst/daf/butler/_butler.py

@@ -36,7 +36,7 @@ from abc import abstractmethod
 from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
 from contextlib import AbstractContextManager
 from types import EllipsisType
-from typing import TYPE_CHECKING, Any, Literal, Self, TextIO
+from typing import TYPE_CHECKING, Any, TextIO
 
 from lsst.resources import ResourcePath, ResourcePathExpression
 from lsst.utils import doImportType
@@ -94,7 +94,7 @@ class _DeprecatedDefault:
     """Default value for a deprecated parameter."""
 
 
-class Butler(LimitedButler, AbstractContextManager):  # numpydoc ignore=PR02
+class Butler(LimitedButler):  # numpydoc ignore=PR02
     """Interface for data butler and factory for Butler instances.
 
     Parameters
@@ -358,16 +358,6 @@ class Butler(LimitedButler, AbstractContextManager):  # numpydoc ignore=PR02
             case _:
                 raise TypeError(f"Unknown Butler type '{butler_type}'")
 
-    def __enter__(self) -> Self:
-        return self
-
-    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Literal[False]:
-        try:
-            self.close()
-        except Exception:
-            _LOG.exception("An exception occured during Butler.close()")
-        return False
-
     @staticmethod
     def makeRepo(
         root: ResourcePathExpression,
@@ -2236,15 +2226,4 @@ class Butler(LimitedButler, AbstractContextManager):  # numpydoc ignore=PR02
 
     @abstractmethod
     def close(self) -> None:
-        """Release all resources associated with this Butler instance.  The
-        instance may no longer be used after this is called.
-
-        Notes
-        -----
-        Instead of calling ``close()``directly, you can use the Butler object
-        as a context manager.  For example::
-          with Butler(...) as butler:
-              butler.get(...)
-          # butler is closed after exiting the block.
-        """
         raise NotImplementedError()

lsst/daf/butler/_limited_butler.py

@@ -30,10 +30,10 @@ from __future__ import annotations
 __all__ = ("LimitedButler",)
 
 import logging
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from collections.abc import Iterable, Iterator
-from contextlib import contextmanager
-from typing import Any, ClassVar
+from contextlib import AbstractContextManager, contextmanager
+from typing import Any, ClassVar, Literal, Self
 
 from lsst.resources import ResourcePath
 
@@ -48,7 +48,7 @@ from .dimensions import DimensionUniverse
 log = logging.getLogger(__name__)
 
 
-class LimitedButler(ABC):
+class LimitedButler(AbstractContextManager):
     """A minimal butler interface that is sufficient to back
     `~lsst.pipe.base.PipelineTask` execution.
     """
@@ -100,6 +100,31 @@ class LimitedButler(ABC):
         """
         raise NotImplementedError()
 
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Literal[False]:
+        try:
+            self.close()
+        except Exception:
+            log.exception("An exception occurred during Butler.close()")
+        return False
+
+    def close(self) -> None:
+        """Release all resources associated with this Butler instance.  The
+        instance may no longer be used after this is called.
+
+        Notes
+        -----
+        Instead of calling ``close()`` directly, you can use the Butler object
+        as a context manager.  For example::
+
+          with Butler(...) as butler:
+              butler.get(...)
+          # butler is closed after exiting the block.
+        """
+        pass
+
     def get(
         self,
         ref: DatasetRef,

lsst/daf/butler/formatters/parquet.py

@@ -49,13 +49,14 @@ __all__ = (
 )
 
 import collections.abc
+import contextlib
 import itertools
 import json
 import logging
 import re
-from collections.abc import Iterable, Sequence
+from collections.abc import Generator, Iterable, Sequence
 from fnmatch import fnmatchcase
-from typing import TYPE_CHECKING, Any, cast
+from typing import IO, TYPE_CHECKING, Any, cast
 
 import pyarrow as pa
 import pyarrow.parquet as pq
@@ -84,6 +85,16 @@ TARGET_ROW_GROUP_BYTES = 1_000_000_000
 ASTROPY_PANDAS_INDEX_KEY = "lsst::arrow::astropy_pandas_index"
 
 
+@contextlib.contextmanager
+def generic_open(path: str, fs: AbstractFileSystem | None) -> Generator[IO]:
+    if fs is None:
+        with open(path, "rb") as fh:
+            yield fh
+    else:
+        with fs.open(path) as fh:
+            yield fh
+
+
 class ParquetFormatter(FormatterV2):
     """Interface for reading and writing Arrow Table objects to and from
     Parquet files.
@@ -120,7 +131,8 @@ class ParquetFormatter(FormatterV2):
         component: str | None = None,
         expected_size: int = -1,
     ) -> Any:
-        schema = pq.read_schema(path, filesystem=fs)
+        with generic_open(path, fs) as handle:
+            schema = pq.read_schema(handle)
 
         schema_names = ["ArrowSchema", "DataFrameSchema", "ArrowAstropySchema", "ArrowNumpySchema"]
 
@@ -133,13 +145,13 @@ class ParquetFormatter(FormatterV2):
             if b"lsst::arrow::rowcount" in schema.metadata:
                 return int(schema.metadata[b"lsst::arrow::rowcount"])
 
-            temp_table = pq.read_table(
-                path,
-                filesystem=fs,
-                columns=[schema.names[0]],
-                use_threads=False,
-                use_pandas_metadata=False,
-            )
+            with generic_open(path, fs) as handle:
+                temp_table = pq.read_table(
+                    handle,
+                    columns=[schema.names[0]],
+                    use_threads=False,
+                    use_pandas_metadata=False,
+                )
 
             return len(temp_table[schema.names[0]])
 
@@ -148,7 +160,7 @@ class ParquetFormatter(FormatterV2):
             par_columns = self.file_descriptor.parameters.pop("columns", None)
             if par_columns:
                 has_pandas_multi_index = False
-                if b"pandas" in schema.metadata:
+                if schema.metadata and b"pandas" in schema.metadata:
                     md = json.loads(schema.metadata[b"pandas"])
                     if len(md["column_indexes"]) > 1:
                         has_pandas_multi_index = True
@@ -184,13 +196,13 @@ class ParquetFormatter(FormatterV2):
                 )
 
         metadata = schema.metadata if schema.metadata is not None else {}
-        arrow_table = pq.read_table(
-            path,
-            filesystem=fs,
-            columns=par_columns,
-            use_threads=False,
-            use_pandas_metadata=(b"pandas" in metadata),
-        )
+        with generic_open(path, fs) as handle:
+            arrow_table = pq.read_table(
+                handle,
+                columns=par_columns,
+                use_threads=False,
+                use_pandas_metadata=(b"pandas" in metadata),
+            )
 
         return arrow_table
 

lsst/daf/butler/logging.py

@@ -537,6 +537,9 @@ class ButlerLogRecords(MutableSequence[ButlerLogRecord]):
         if cls._generic_startswith(startdata, "["):
             # This is a JSON array of records.
             return _ButlerLogRecordsModelV1
+        elif cls._generic_startswith(startdata, cls.STREAMING_EXTRA_DELIMITER):
+            # This is an empty log file with a log record per line format.
+            return None
         elif not cls._generic_startswith(startdata, "{"):
             # Limit the length of string reported in error message in case
             # this is an enormous file.
@@ -570,9 +573,12 @@ class ButlerLogRecords(MutableSequence[ButlerLogRecord]):
         """
         first_line = stream.readline()
 
+        empty_stream = False
         if not first_line:
             # Empty file, return zero records.
             return cls.from_records([])
+        elif cls._generic_startswith(first_line, cls.STREAMING_EXTRA_DELIMITER):
+            empty_stream = True
 
         model_type = cls._detect_model(first_line)
 
@@ -583,13 +589,17 @@ class ButlerLogRecords(MutableSequence[ButlerLogRecord]):
             return model_type.model_validate_json(all).wrap(cls)
 
         # A stream of records with one record per line.
-        records = [ButlerLogRecord.model_validate_json(first_line)]
-        for line in stream:
-            line = line.rstrip()
-            if cls._generic_startswith(line, "###EXTRA###"):
-                break
-            elif line:  # skip blank lines
-                records.append(ButlerLogRecord.model_validate_json(line))
+        if not empty_stream:
+            records = [ButlerLogRecord.model_validate_json(first_line)]
+            for line in stream:
+                line = line.rstrip()
+                if cls._generic_startswith(line, cls.STREAMING_EXTRA_DELIMITER):
+                    break
+                elif line:  # skip blank lines
+                    records.append(ButlerLogRecord.model_validate_json(line))
+        else:
+            # No records but might have extra metadata.
+            records = []
         extra_data = stream.read()
         if extra_data:
             extra = pydantic_core.from_json(extra_data)

lsst/daf/butler/script/_pruneDatasets.py

@@ -243,6 +243,7 @@ def pruneDatasets(
             find_first=not find_all,
             show_uri=False,
         )
+        dataset_refs = list(itertools.chain.from_iterable(datasets_found.getDatasets()))
 
         result = PruneDatasetsResult(list(datasets_found.getTables()))
 
@@ -258,7 +259,7 @@ def pruneDatasets(
     def doPruneDatasets() -> PruneDatasetsResult:
         with Butler.from_config(repo, writeable=True) as butler:
             butler.pruneDatasets(
-                refs=list(itertools.chain(*datasets_found.getDatasets())),
+                refs=dataset_refs,
                 disassociate=disassociate,
                 tags=disassociate_tags or (),
                 purge=purge,

lsst/daf/butler/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.4800"
+__version__ = "30.0.0rc1"

{lsst_daf_butler-29.2025.4800.dist-info → lsst_daf_butler-30.0.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.4800
+Version: 30.0.0rc1
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_daf_butler-29.2025.4800.dist-info → lsst_daf_butler-30.0.0rc1.dist-info}/RECORD

@@ -1,7 +1,7 @@
 lsst/__init__.py,sha256=_2bZAHuDVAx7MM7KA7pt3DYp641NY4RzSoRAwesWKfU,67
 lsst/daf/__init__.py,sha256=_2bZAHuDVAx7MM7KA7pt3DYp641NY4RzSoRAwesWKfU,67
 lsst/daf/butler/__init__.py,sha256=Ku-Gzm95RVSJtlvA3Ptr050S8JFM_QFZVwnxpkC136o,3519
-lsst/daf/butler/_butler.py,sha256=0Tr9E28uV43M5aGhlkxpwOd_QLyTJgfNRXNtdvuUMNE,96780
+lsst/daf/butler/_butler.py,sha256=Unvzmm2BzJy1J50F9G5M63ZJ-MJIqtIVNp_TGczXx3g,96021
 lsst/daf/butler/_butler_collections.py,sha256=tRZaFjPyQzKz--9v7HNmJNjDwB4hi9LJUIyvx1adVFc,19384
 lsst/daf/butler/_butler_config.py,sha256=q_iJf4rk-L5tMdvIZwEqOvQrt5VEMvDsVzoWmvoxx2A,9619
 lsst/daf/butler/_butler_instance_options.py,sha256=6l4hF7nfxb77Rf0r2PAmiaSid7kM1z6kQPW4VzRUjYs,2046
@@ -22,7 +22,7 @@ lsst/daf/butler/_file_dataset.py,sha256=KF_o5zi01L0gK4t_Ncb3BSbCTiyf9zmwwTpWoNga
 lsst/daf/butler/_file_descriptor.py,sha256=PIYT9O2NRbNndtI2rMXfx7tP6nUcTnd9fxFVsq-Hu1s,3841
 lsst/daf/butler/_formatter.py,sha256=BZTpbAwATykj6Omz4pl5f5SdI_Qa0rWxhSUk1G4_UIc,83156
 lsst/daf/butler/_labeled_butler_factory.py,sha256=Ly1oaSBDTNDaDtEJFEEys4KRc359Gcr9KLLwi-ggZHE,9052
-lsst/daf/butler/_limited_butler.py,sha256=85na5IWR3nRRWEgMcuy-lY46FPrxBRqsvgEMEnZoOkQ,16696
+lsst/daf/butler/_limited_butler.py,sha256=GTsAzymQf11daCtDhI4WBqee8zyBzKN9CD85gOeG-WU,17514
 lsst/daf/butler/_location.py,sha256=oX-AKZeSdToeKBxvB6y-pxLMiFwF-3nz3YleH5SiA3A,10451
 lsst/daf/butler/_named.py,sha256=-AvkTP5JIsCwj5tAmZO8epoRoF1_dGuz2aC0ItLg7-M,19541
 lsst/daf/butler/_quantum.py,sha256=xqWk2mHLydf4VWpRTCR74eBjApESuaTb46_iOdLKFDo,25575
@@ -39,7 +39,7 @@ lsst/daf/butler/arrow_utils.py,sha256=_vADFsvZZD-vFRYOZ_cZ9FkUkbKnJxD4KaiUmrQCGD
 lsst/daf/butler/column_spec.py,sha256=XFijHMSv2Gwa2olqDjCEF5mELr7IhGJK6npNwZ5iMRE,14639
 lsst/daf/butler/ddl.py,sha256=Yi8_8IgSh7omR1oW1M2kOTw83zyIqC0E041FShSYFBw,23376
 lsst/daf/butler/json.py,sha256=hOCbNsVBXc9q9ZksKP_-5Zh5-lSGXZqjBFaCGbobBcI,5374
-lsst/daf/butler/logging.py,sha256=Bfcmw5lk0UInO_TyQez250O4YkAHAq3vCjYuxaTDAvg,25942
+lsst/daf/butler/logging.py,sha256=SO6aq8vgnEYuMqNI-TOBf370x9Eaa9dZp_FkXBR9iGo,26431
 lsst/daf/butler/mapping_factory.py,sha256=tLMTD03Rgz3xUwGs2vLysyahpEgt3Ob7F5zrXfJHNv4,11504
 lsst/daf/butler/name_shrinker.py,sha256=Pc5S8cbauT4KJ7ALKurEO-kaVR6Ih5AC5lF3NOZqMRg,4269
 lsst/daf/butler/nonempty_mapping.py,sha256=YnIowyh4v-bBf5Q032egN7DUOgv1Ip0uS6C80GIEJTU,4356
@@ -51,7 +51,7 @@ lsst/daf/butler/repo_relocation.py,sha256=Ivhx2xU4slc53Z6RExhNnquMr2Hx-S8h62emml
 lsst/daf/butler/time_utils.py,sha256=MVTfOFI2xt3IeA46pa-fWY2kJRwSzaQyq1uzeUABcTM,11805
 lsst/daf/butler/timespan_database_representation.py,sha256=rYeQ_vp6gneRjboqV-gvNW0DWhm1QJM-KnVzFTDVZ0I,24550
 lsst/daf/butler/utils.py,sha256=5u50COK5z4u31grOhmQF7mFz55biNLOvSMRdQjEdsjo,5140
-lsst/daf/butler/version.py,sha256=MHZJr_Yoevn1SeqerO6dmfTP3xtJLGxPmEi-9Jpl380,55
+lsst/daf/butler/version.py,sha256=vSgpHsF0PG5ox9cJDNTmDJ8a4r5c8EKOc2gxM3iszgs,52
 lsst/daf/butler/_rubin/__init__.py,sha256=9z5kmc6LJ3C_iPFV46cvdlQ2qOGJbZh-2Ft5Z-rbE28,1569
 lsst/daf/butler/_rubin/file_datasets.py,sha256=P5_BIhxpVj9qfLuLiI2_dClMHsjO5Qm5oDXVr3WntNU,3607
 lsst/daf/butler/_utilities/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
@@ -143,7 +143,7 @@ lsst/daf/butler/formatters/json.py,sha256=n8IDbIH4yAsbppXVOtMar3WvMaZm4chfXQmm9D
 lsst/daf/butler/formatters/logs.py,sha256=p_J2v9VFpUIRzCqtttP_QdgiX0FepTG4C4f7EQCLJoE,3050
 lsst/daf/butler/formatters/matplotlib.py,sha256=nEqFJOab970b5EeEjSm-T1fzM9M7cHKGmTubjr4bgNA,2159
 lsst/daf/butler/formatters/packages.py,sha256=XzOUptSjXX-NJ2CUfYkTdI09M1m6hj-mdy4SZMCyey8,3409
-lsst/daf/butler/formatters/parquet.py,sha256=fGNoBMoixA3UxHo3f5rseCIgeUi4Uozo524Bwm2356I,48618
+lsst/daf/butler/formatters/parquet.py,sha256=sPZG1sid0qpWquC9d5CL9hIimozXUL1IkiO_WWAoVw8,49049
 lsst/daf/butler/formatters/pickle.py,sha256=gxZ9VumvJm1EGQiPtwC8D3x5CPjihfUzE4GDVrv7YRQ,2218
 lsst/daf/butler/formatters/typeless.py,sha256=Q_Nh5bt0zrCExA19XA_VdAVvVUPlGwCCQ463RWGArkM,9158
 lsst/daf/butler/formatters/yaml.py,sha256=-zqKTioad3wPNc1Xp6GYgx_TRhQNaxydGQ4VOCzfEZU,4764
@@ -285,7 +285,7 @@ lsst/daf/butler/remote_butler/server/handlers/_query_streaming.py,sha256=uSWaJ8A
 lsst/daf/butler/remote_butler/server/handlers/_utils.py,sha256=ALCX9Hi59izfEGifHBs9lP7Udjptv0lXpf4mGpvorCA,2332
 lsst/daf/butler/script/__init__.py,sha256=mbYs1becM_mW1ejNDDePqWyt47yBPARL2pO9Z7i0DC0,2326
 lsst/daf/butler/script/_associate.py,sha256=Uxarvv5cHQmDRIpu4CC4EY2JN3suXOvga8-AktWXw60,2930
-lsst/daf/butler/script/_pruneDatasets.py,sha256=0uKuDUxSNBIsr82ixasG9IvUyUl6WPfbJeWKAZVK810,10461
+lsst/daf/butler/script/_pruneDatasets.py,sha256=VtpKCjC7-qaXwVdDwpfZgwNhTFy9UW-4zjncwWg4OrA,10510
 lsst/daf/butler/script/butlerImport.py,sha256=j4dfnYSJ6KlRKLT6V2ZjugamJq8DhpI0PjEnSLBBOrk,3132
 lsst/daf/butler/script/certifyCalibrations.py,sha256=kDtdeO9HKTFzd68eWGn_meNQnqnrA2JG7f9du_xrYxg,4026
 lsst/daf/butler/script/collectionChain.py,sha256=4SeGm1CyWk6_qAvTua9yWyQSY9NlTCMQUuX3xPCu1HQ,6164
@@ -337,13 +337,13 @@ lsst/daf/butler/transfers/__init__.py,sha256=M1YcFszSkNB5hB2pZwwGXqbJE2dKt4YXDin
 lsst/daf/butler/transfers/_context.py,sha256=Ro_nf9NDw9IAr-Pw_NtcdotQKx34RbBbNubt20zwRXU,16449
 lsst/daf/butler/transfers/_interfaces.py,sha256=Ia1NqcFR5E-Ik4zsXEe2fuMtNCJj5Yfe_gVHLTBtJDw,7490
 lsst/daf/butler/transfers/_yaml.py,sha256=w_0GmrueuHVLfOfAXGHFBbWAl18tX6eSElbTC-2jRoc,32632
-lsst_daf_butler-29.2025.4800.dist-info/licenses/COPYRIGHT,sha256=k1Vq0-Be_K-puaeW4UZnckPjksEL-MJh4XKiWcjMxJE,312
-lsst_daf_butler-29.2025.4800.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_daf_butler-29.2025.4800.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_daf_butler-29.2025.4800.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_daf_butler-29.2025.4800.dist-info/METADATA,sha256=lHXvwM7V9rhuuLWAqceEi7LucKY8jZ3runxAfycxoUw,3813
-lsst_daf_butler-29.2025.4800.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_daf_butler-29.2025.4800.dist-info/entry_points.txt,sha256=XsRxyTK3c-jGlKVuVnbpch3gtaO0lAA_fS3i2NGS5rw,59
-lsst_daf_butler-29.2025.4800.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_daf_butler-29.2025.4800.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_daf_butler-29.2025.4800.dist-info/RECORD,,
+lsst_daf_butler-30.0.0rc1.dist-info/licenses/COPYRIGHT,sha256=k1Vq0-Be_K-puaeW4UZnckPjksEL-MJh4XKiWcjMxJE,312
+lsst_daf_butler-30.0.0rc1.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_daf_butler-30.0.0rc1.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_daf_butler-30.0.0rc1.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_daf_butler-30.0.0rc1.dist-info/METADATA,sha256=jMWZliOVZ-7FuZBEA5n0FNiRsm531aSDDPQdkSAQ87k,3810
+lsst_daf_butler-30.0.0rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_daf_butler-30.0.0rc1.dist-info/entry_points.txt,sha256=XsRxyTK3c-jGlKVuVnbpch3gtaO0lAA_fS3i2NGS5rw,59
+lsst_daf_butler-30.0.0rc1.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_daf_butler-30.0.0rc1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_daf_butler-30.0.0rc1.dist-info/RECORD,,