lsst-daf-butler 29.2025.4800__tar.gz → 30.0.0rc1__tar.gz

{lsst_daf_butler-29.2025.4800/python/lsst_daf_butler.egg-info → lsst_daf_butler-30.0.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.4800
+Version: 30.0.0rc1
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_daf_butler-29.2025.4800 → lsst_daf_butler-30.0.0rc1}/python/lsst/daf/butler/_butler.py

@@ -36,7 +36,7 @@ from abc import abstractmethod
 from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
 from contextlib import AbstractContextManager
 from types import EllipsisType
-from typing import TYPE_CHECKING, Any, Literal, Self, TextIO
+from typing import TYPE_CHECKING, Any, TextIO
 
 from lsst.resources import ResourcePath, ResourcePathExpression
 from lsst.utils import doImportType
@@ -94,7 +94,7 @@ class _DeprecatedDefault:
     """Default value for a deprecated parameter."""
 
 
-class Butler(LimitedButler, AbstractContextManager):  # numpydoc ignore=PR02
+class Butler(LimitedButler):  # numpydoc ignore=PR02
     """Interface for data butler and factory for Butler instances.
 
     Parameters
@@ -358,16 +358,6 @@ class Butler(LimitedButler, AbstractContextManager):  # numpydoc ignore=PR02
             case _:
                 raise TypeError(f"Unknown Butler type '{butler_type}'")
 
-    def __enter__(self) -> Self:
-        return self
-
-    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Literal[False]:
-        try:
-            self.close()
-        except Exception:
-            _LOG.exception("An exception occured during Butler.close()")
-        return False
-
     @staticmethod
     def makeRepo(
         root: ResourcePathExpression,
@@ -2236,15 +2226,4 @@ class Butler(LimitedButler, AbstractContextManager):  # numpydoc ignore=PR02
 
     @abstractmethod
     def close(self) -> None:
-        """Release all resources associated with this Butler instance.  The
-        instance may no longer be used after this is called.
-
-        Notes
-        -----
-        Instead of calling ``close()``directly, you can use the Butler object
-        as a context manager.  For example::
-          with Butler(...) as butler:
-              butler.get(...)
-          # butler is closed after exiting the block.
-        """
         raise NotImplementedError()

{lsst_daf_butler-29.2025.4800 → lsst_daf_butler-30.0.0rc1}/python/lsst/daf/butler/_limited_butler.py

@@ -30,10 +30,10 @@ from __future__ import annotations
 __all__ = ("LimitedButler",)
 
 import logging
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from collections.abc import Iterable, Iterator
-from contextlib import contextmanager
-from typing import Any, ClassVar
+from contextlib import AbstractContextManager, contextmanager
+from typing import Any, ClassVar, Literal, Self
 
 from lsst.resources import ResourcePath
 
@@ -48,7 +48,7 @@ from .dimensions import DimensionUniverse
 log = logging.getLogger(__name__)
 
 
-class LimitedButler(ABC):
+class LimitedButler(AbstractContextManager):
     """A minimal butler interface that is sufficient to back
     `~lsst.pipe.base.PipelineTask` execution.
     """
@@ -100,6 +100,31 @@ class LimitedButler(ABC):
         """
         raise NotImplementedError()
 
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Literal[False]:
+        try:
+            self.close()
+        except Exception:
+            log.exception("An exception occurred during Butler.close()")
+        return False
+
+    def close(self) -> None:
+        """Release all resources associated with this Butler instance.  The
+        instance may no longer be used after this is called.
+
+        Notes
+        -----
+        Instead of calling ``close()`` directly, you can use the Butler object
+        as a context manager.  For example::
+
+          with Butler(...) as butler:
+              butler.get(...)
+          # butler is closed after exiting the block.
+        """
+        pass
+
     def get(
         self,
         ref: DatasetRef,

{lsst_daf_butler-29.2025.4800 → lsst_daf_butler-30.0.0rc1}/python/lsst/daf/butler/formatters/parquet.py

@@ -49,13 +49,14 @@ __all__ = (
 )
 
 import collections.abc
+import contextlib
 import itertools
 import json
 import logging
 import re
-from collections.abc import Iterable, Sequence
+from collections.abc import Generator, Iterable, Sequence
 from fnmatch import fnmatchcase
-from typing import TYPE_CHECKING, Any, cast
+from typing import IO, TYPE_CHECKING, Any, cast
 
 import pyarrow as pa
 import pyarrow.parquet as pq
@@ -84,6 +85,16 @@ TARGET_ROW_GROUP_BYTES = 1_000_000_000
 ASTROPY_PANDAS_INDEX_KEY = "lsst::arrow::astropy_pandas_index"
 
 
+@contextlib.contextmanager
+def generic_open(path: str, fs: AbstractFileSystem | None) -> Generator[IO]:
+    if fs is None:
+        with open(path, "rb") as fh:
+            yield fh
+    else:
+        with fs.open(path) as fh:
+            yield fh
+
+
 class ParquetFormatter(FormatterV2):
     """Interface for reading and writing Arrow Table objects to and from
     Parquet files.
@@ -120,7 +131,8 @@ class ParquetFormatter(FormatterV2):
         component: str | None = None,
         expected_size: int = -1,
     ) -> Any:
-        schema = pq.read_schema(path, filesystem=fs)
+        with generic_open(path, fs) as handle:
+            schema = pq.read_schema(handle)
 
         schema_names = ["ArrowSchema", "DataFrameSchema", "ArrowAstropySchema", "ArrowNumpySchema"]
 
@@ -133,13 +145,13 @@ class ParquetFormatter(FormatterV2):
             if b"lsst::arrow::rowcount" in schema.metadata:
                 return int(schema.metadata[b"lsst::arrow::rowcount"])
 
-            temp_table = pq.read_table(
-                path,
-                filesystem=fs,
-                columns=[schema.names[0]],
-                use_threads=False,
-                use_pandas_metadata=False,
-            )
+            with generic_open(path, fs) as handle:
+                temp_table = pq.read_table(
+                    handle,
+                    columns=[schema.names[0]],
+                    use_threads=False,
+                    use_pandas_metadata=False,
+                )
 
             return len(temp_table[schema.names[0]])
 
@@ -148,7 +160,7 @@ class ParquetFormatter(FormatterV2):
             par_columns = self.file_descriptor.parameters.pop("columns", None)
             if par_columns:
                 has_pandas_multi_index = False
-                if b"pandas" in schema.metadata:
+                if schema.metadata and b"pandas" in schema.metadata:
                     md = json.loads(schema.metadata[b"pandas"])
                     if len(md["column_indexes"]) > 1:
                         has_pandas_multi_index = True
@@ -184,13 +196,13 @@ class ParquetFormatter(FormatterV2):
                 )
 
         metadata = schema.metadata if schema.metadata is not None else {}
-        arrow_table = pq.read_table(
-            path,
-            filesystem=fs,
-            columns=par_columns,
-            use_threads=False,
-            use_pandas_metadata=(b"pandas" in metadata),
-        )
+        with generic_open(path, fs) as handle:
+            arrow_table = pq.read_table(
+                handle,
+                columns=par_columns,
+                use_threads=False,
+                use_pandas_metadata=(b"pandas" in metadata),
+            )
 
         return arrow_table
 

{lsst_daf_butler-29.2025.4800 → lsst_daf_butler-30.0.0rc1}/python/lsst/daf/butler/logging.py

@@ -537,6 +537,9 @@ class ButlerLogRecords(MutableSequence[ButlerLogRecord]):
         if cls._generic_startswith(startdata, "["):
             # This is a JSON array of records.
             return _ButlerLogRecordsModelV1
+        elif cls._generic_startswith(startdata, cls.STREAMING_EXTRA_DELIMITER):
+            # This is an empty log file with a log record per line format.
+            return None
         elif not cls._generic_startswith(startdata, "{"):
             # Limit the length of string reported in error message in case
             # this is an enormous file.
@@ -570,9 +573,12 @@ class ButlerLogRecords(MutableSequence[ButlerLogRecord]):
         """
         first_line = stream.readline()
 
+        empty_stream = False
         if not first_line:
             # Empty file, return zero records.
             return cls.from_records([])
+        elif cls._generic_startswith(first_line, cls.STREAMING_EXTRA_DELIMITER):
+            empty_stream = True
 
         model_type = cls._detect_model(first_line)
 
@@ -583,13 +589,17 @@ class ButlerLogRecords(MutableSequence[ButlerLogRecord]):
             return model_type.model_validate_json(all).wrap(cls)
 
         # A stream of records with one record per line.
-        records = [ButlerLogRecord.model_validate_json(first_line)]
-        for line in stream:
-            line = line.rstrip()
-            if cls._generic_startswith(line, "###EXTRA###"):
-                break
-            elif line:  # skip blank lines
-                records.append(ButlerLogRecord.model_validate_json(line))
+        if not empty_stream:
+            records = [ButlerLogRecord.model_validate_json(first_line)]
+            for line in stream:
+                line = line.rstrip()
+                if cls._generic_startswith(line, cls.STREAMING_EXTRA_DELIMITER):
+                    break
+                elif line:  # skip blank lines
+                    records.append(ButlerLogRecord.model_validate_json(line))
+        else:
+            # No records but might have extra metadata.
+            records = []
         extra_data = stream.read()
         if extra_data:
             extra = pydantic_core.from_json(extra_data)

{lsst_daf_butler-29.2025.4800 → lsst_daf_butler-30.0.0rc1}/python/lsst/daf/butler/script/_pruneDatasets.py

@@ -243,6 +243,7 @@ def pruneDatasets(
             find_first=not find_all,
             show_uri=False,
         )
+        dataset_refs = list(itertools.chain.from_iterable(datasets_found.getDatasets()))
 
         result = PruneDatasetsResult(list(datasets_found.getTables()))
 
@@ -258,7 +259,7 @@ def pruneDatasets(
     def doPruneDatasets() -> PruneDatasetsResult:
         with Butler.from_config(repo, writeable=True) as butler:
             butler.pruneDatasets(
-                refs=list(itertools.chain(*datasets_found.getDatasets())),
+                refs=dataset_refs,
                 disassociate=disassociate,
                 tags=disassociate_tags or (),
                 purge=purge,

lsst_daf_butler-30.0.0rc1/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "30.0.0rc1"

{lsst_daf_butler-29.2025.4800 → lsst_daf_butler-30.0.0rc1/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.4800
+Version: 30.0.0rc1
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_daf_butler-29.2025.4800 → lsst_daf_butler-30.0.0rc1}/tests/test_cliCmdPruneDatasets.py

@@ -27,6 +27,7 @@
 
 """Unit tests for daf_butler CLI prune-datasets subcommand."""
 
+import os
 import unittest
 from itertools import chain
 from unittest.mock import ANY, patch
@@ -36,7 +37,7 @@ from astropy.table import Table
 # Tests require the SqlRegistry
 import lsst.daf.butler.registry.sql_registry
 import lsst.daf.butler.script
-from lsst.daf.butler import CollectionInfo, CollectionType
+from lsst.daf.butler import Butler, CollectionInfo, CollectionType
 from lsst.daf.butler.cli.butler import cli as butlerCli
 from lsst.daf.butler.cli.cmd.commands import (
     pruneDatasets_askContinueMsg,
@@ -57,7 +58,14 @@ from lsst.daf.butler.cli.cmd.commands import (
 from lsst.daf.butler.cli.utils import LogCliRunner, astropyTablesToStr, clickResultMsg
 from lsst.daf.butler.direct_butler import DirectButler
 from lsst.daf.butler.script import QueryDatasets
+from lsst.daf.butler.tests.utils import (
+    ButlerTestHelper,
+    MetricTestRepo,
+    makeTestTempDir,
+    removeTestTempDir,
+)
 
+TESTDIR = os.path.abspath(os.path.dirname(__file__))
 doFindTables = True
 
 
@@ -211,8 +219,6 @@ class PruneDatasetsTestCase(unittest.TestCase):
             # QueryDatasets.getDatasets also does not get called.
             if exPruneDatasetsCallArgs:
                 mockQueryDatasets_getDatasets.assert_called_once()
-            else:
-                mockQueryDatasets_getDatasets.assert_not_called()
             if exGetTablesCalled:
                 mockQueryDatasets_getTables.assert_called_once()
             else:
@@ -520,5 +526,58 @@ class PruneDatasetsTestCase(unittest.TestCase):
         )
 
 
+class QueryDatasetsCLITest(unittest.TestCase, ButlerTestHelper):
+    """Test that the command works when run as a script."""
+
+    def setUp(self):
+        self.root = makeTestTempDir(TESTDIR)
+        self.testRepo = MetricTestRepo(
+            self.root, configFile=os.path.join(TESTDIR, "config/basic/butler.yaml")
+        )
+        self.enterContext(self.testRepo.butler)
+        self.runner = LogCliRunner()
+
+    def tearDown(self):
+        removeTestTempDir(self.root)
+
+    def test_no_confirm(self):
+        """Test that prune-datasets can execute with --no-confirm."""
+        result = self.runner.invoke(
+            butlerCli,
+            [
+                "prune-datasets",
+                "--no-confirm",
+                self.root,
+                "--purge",
+                "ingest/run",
+                "--where",
+                "instrument='DummyCamComp' and visit=423",
+            ],
+        )
+        self.assertEqual(result.exit_code, 0, clickResultMsg(result))
+        with Butler(self.root) as butler:
+            self.assertIsNone(butler.get_dataset(self.testRepo.ref1.id))
+            self.assertIsNotNone(butler.get_dataset(self.testRepo.ref2.id))
+
+    def test_confirm(self):
+        """Test that prune-datasets can execute with the default --confirm."""
+        result = self.runner.invoke(
+            butlerCli,
+            [
+                "prune-datasets",
+                self.root,
+                "--purge",
+                "ingest/run",
+                "--where",
+                "instrument='DummyCamComp' and visit=423",
+            ],
+            input="y\n",
+        )
+        self.assertEqual(result.exit_code, 0, clickResultMsg(result))
+        with Butler(self.root) as butler:
+            self.assertIsNone(butler.get_dataset(self.testRepo.ref1.id))
+            self.assertIsNotNone(butler.get_dataset(self.testRepo.ref2.id))
+
+
 if __name__ == "__main__":
     unittest.main()

{lsst_daf_butler-29.2025.4800 → lsst_daf_butler-30.0.0rc1}/tests/test_cliUtilSplitKv.py

@@ -176,10 +176,12 @@ class SplitKvCmdTestCase(unittest.TestCase):
         # double separator "==" should fail:
         result = self.runner.invoke(cli, ["--value", "first==1"])
         self.assertEqual(result.exit_code, 1)
+        # Check first 137 characters because python 3.14 adds more information
+        # to the error message.
         self.assertEqual(
-            result.output,
+            result.output[:137],
             "Error: Could not parse key-value pair 'first==1' using separator '=', with "
-            "multiple values allowed: too many values to unpack (expected 2)\n",
+            "multiple values allowed: too many values to unpack (expected 2",
         )
 
     def test_choice(self):

{lsst_daf_butler-29.2025.4800 → lsst_daf_butler-30.0.0rc1}/tests/test_logFormatter.py

@@ -106,6 +106,24 @@ class ButlerLogRecordsFormatterTestCase(unittest.TestCase):
         self.assertEqual(records, handler.records)
         self.assertEqual(len(records), 2)
 
+    def testButlerLogRecordsFormatterEmptyExtra(self):
+        """Test that we can store extra information even with no log
+        records written.
+        """
+        handler = ButlerLogRecordHandler()
+
+        log = logging.getLogger(self.id())
+        log.setLevel(logging.INFO)
+        log.addHandler(handler)
+
+        extra_data = {"extra1": 1, "extra2": 2}
+        handler.records.extra = extra_data
+        ref = self.butler.put(handler.records, self.datasetType)
+        records = self.butler.get(ref)
+
+        self.assertEqual(records, handler.records)
+        self.assertEqual(len(records), 0)
+
     @unittest.skipIf(
         sys.version_info < (3, 12, 0),
         "This test requires NamedTemporaryFile behavior not available in older Python versions.",
@@ -205,6 +223,36 @@ class ButlerLogRecordsFormatterTestCase(unittest.TestCase):
             self.assertEqual(len(records), 2)
             self.assertEqual(records.extra, extra_data)
 
+    @unittest.skipIf(
+        sys.version_info < (3, 12, 0),
+        "This test requires NamedTemporaryFile behavior not available in older Python versions.",
+    )
+    def testJsonLogRecordsFormatterExtraEmpty(self):
+        """Test that externally created JSON format stream files work with
+        extra JSON data appended even if no log messages written.
+        """
+        log = logging.getLogger(self.id())
+        log.setLevel(logging.INFO)
+
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="butler-log-") as tmp:
+            handler = FileHandler(tmp.name)
+            handler.setFormatter(JsonLogFormatter())
+            log.addHandler(handler)
+
+            handler.close()
+            extra_data = {"extra1": 1, "extra2": 2}
+            with open(tmp.name, "a") as stream:
+                ButlerLogRecords.write_streaming_extra(stream, json.dumps(extra_data))
+
+            # Now ingest the file.
+            ref = DatasetRef(self.datasetType, dataId={}, run=self.run)
+            dataset = FileDataset(path=tmp.name, refs=ref)
+            self.butler.ingest(dataset, transfer="move")
+
+            records: ButlerLogRecords = self.butler.get(ref)
+            self.assertEqual(len(records), 0)
+            self.assertEqual(records.extra, extra_data)
+
 
 if __name__ == "__main__":
     unittest.main()

lsst_daf_butler-29.2025.4800/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.4800"