astro-metadata-translator 29.2025.4300__tar.gz → 29.2025.4800__tar.gz

{astro_metadata_translator-29.2025.4300/python/astro_metadata_translator.egg-info → astro_metadata_translator-29.2025.4800}/PKG-INFO

@@ -1,25 +1,26 @@
 Metadata-Version: 2.4
 Name: astro-metadata-translator
-Version: 29.2025.4300
+Version: 29.2025.4800
 Summary: A translator for astronomical metadata.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
-License: BSD 3-Clause License
+License-Expression: BSD-3-Clause
 Project-URL: Homepage, https://github.com/lsst/astro_metadata_translator
 Keywords: lsst
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering :: Astronomy
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.10
-Requires-Python: >=3.10.0
+Requires-Python: >=3.11.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy>=3.0.5
 Requires-Dist: pyyaml>=3.13
+Requires-Dist: lsst-resources
+Requires-Dist: fsspec
 Requires-Dist: click>=8
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"

{astro_metadata_translator-29.2025.4300 → astro_metadata_translator-29.2025.4800}/pyproject.toml

@@ -4,9 +4,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "astro-metadata-translator"
-requires-python = ">=3.10.0"
+requires-python = ">=3.11.0"
 description = "A translator for astronomical metadata."
-license = {text = "BSD 3-Clause License"}
+license = "BSD-3-Clause"
+license-files = ["LICENSE"]
 authors = [
     {name="Rubin Observatory Data Management", email="dm-admin@lists.lsst.org"},
 ]
@@ -14,19 +15,20 @@ readme = "README.md"
 classifiers = [
     "Intended Audience :: Science/Research",
     "Topic :: Scientific/Engineering :: Astronomy",
-    "License :: OSI Approved :: BSD License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.14",
     "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.10",
 ]
 keywords = ["lsst"]
 dynamic = ["version"]
 dependencies = [
     "astropy >=3.0.5",
     "pyyaml >=3.13",
+    "lsst-resources",
+    "fsspec",  # Do not include S3/http variants by default.
     "click >= 8"
 ]
 
@@ -43,7 +45,6 @@ astrometadata = "astro_metadata_translator.cli.astrometadata:main"
 
 [tool.setuptools]
 zip-safe = true
-license-files = ["LICENSE"]
 
 [tool.setuptools.packages.find]
 where = ["python"]

{astro_metadata_translator-29.2025.4300 → astro_metadata_translator-29.2025.4800}/python/astro_metadata_translator/bin/translate.py

@@ -23,18 +23,22 @@ import math
 import traceback
 from collections import defaultdict
 from collections.abc import Sequence
-from typing import IO, Any
+from typing import IO, TYPE_CHECKING, Any
 
 import astropy.time
 import astropy.units as u
 import yaml
 from astropy.table import Column, MaskedColumn, QTable
+from lsst.resources import ResourcePath
 
 from astro_metadata_translator import MetadataTranslator, ObservationInfo, fix_header
 
 from ..file_helpers import find_files, read_basic_metadata_from_file
 from ..properties import PROPERTIES
 
+if TYPE_CHECKING:
+    from lsst.resources import ResourcePathExpression
+
 log = logging.getLogger(__name__)
 
 # Output mode choices
@@ -74,7 +78,7 @@ TABLE_COLUMNS = (
 
 
 def read_file(
-    file: str,
+    file: ResourcePathExpression,
     hdrnum: int,
     print_trace: bool,
     output_columns: defaultdict[str, list],
@@ -86,7 +90,7 @@ def read_file(
 
     Parameters
     ----------
-    file : `str`
+    file : `str` or `lsst.resources.ResourcePathExpression`
         The file from which the header is to be read.
     hdrnum : `int`
         The HDU number to read. The primary header is always read and
@@ -127,10 +131,11 @@ def read_file(
     if output_mode != "table":
         log.info("Analyzing %s...", file)
 
+    uri = ResourcePath(file, forceDirectory=False)
     try:
-        md = read_basic_metadata_from_file(file, hdrnum, can_raise=True)
+        md = read_basic_metadata_from_file(uri, hdrnum, can_raise=True)
         if md is None:
-            raise RuntimeError(f"Failed to read file {file} HDU={hdrnum}")
+            raise RuntimeError(f"Failed to read file {uri} HDU={hdrnum}")
 
         if output_mode.endswith("native"):
             # Strip native and don't change type of md
@@ -141,21 +146,21 @@ def read_file(
 
         if output_mode in ("yaml", "fixed"):
             if output_mode == "fixed":
-                fix_header(md, filename=file)
+                fix_header(md, filename=str(uri))
 
             # The header should be written out in the insertion order
             print(yaml.dump(md, sort_keys=False), file=outstream)
             return True
 
         # Try to work out a translator class.
-        translator_class = MetadataTranslator.determine_translator(md, filename=file)
+        translator_class = MetadataTranslator.determine_translator(md, filename=str(uri))
 
         # Work out which headers to translate, assuming the default if
         # we have a YAML test file.
-        if file.endswith(".yaml"):
+        if uri.getExtension() == ".yaml":
             headers = [md]
         else:
-            headers = list(translator_class.determine_translatable_headers(file, md))
+            headers = list(translator_class.determine_translatable_headers(uri, md))
         if output_mode == "auto":
             output_mode = "table" if len(headers) > 1 else "verbose"
 
@@ -163,7 +168,7 @@ def read_file(
             raise ValueError("Table mode requires output columns")
 
         for md in headers:
-            obs_info = ObservationInfo(md, pedantic=False, filename=file)
+            obs_info = ObservationInfo(md, pedantic=False, filename=str(uri))
             if output_mode == "table":
                 for c in TABLE_COLUMNS:
                     output_columns[c["label"]].append(getattr(obs_info, c["attr"]))
@@ -178,7 +183,7 @@ def read_file(
         if print_trace:
             traceback.print_exc(file=outstream)
         else:
-            print(f"Failure processing {file}: {e}", file=outstream)
+            print(f"Failure processing {uri}: {e}", file=outstream)
         return False
     return True
 
@@ -308,9 +313,9 @@ def translate_or_dump_headers(
         isok = read_file(path, hdrnum, print_trace, output_columns, outstream, output_mode, heading)
         heading = False
         if isok:
-            okay.append(path)
+            okay.append(str(path))
         else:
-            failed.append(path)
+            failed.append(str(path))
 
         # Check if we have exceeded the page size and should dump the table.
         if output_mode == "table" and len(output_columns[TABLE_COLUMNS[0]["label"]]) >= MAX_TABLE_PAGE_SIZE:

{astro_metadata_translator-29.2025.4300 → astro_metadata_translator-29.2025.4800}/python/astro_metadata_translator/bin/writeindex.py

@@ -19,6 +19,8 @@ import os
 from collections.abc import MutableMapping, Sequence
 from typing import IO
 
+from lsst.resources import ResourcePath
+
 from ..file_helpers import find_files
 from ..indexing import index_files
 
@@ -87,15 +89,19 @@ def write_index_files(
 
     failed = []
     okay = []
-    files_per_directory: MutableMapping[str, list[str]] = {}
+    files_per_directory: MutableMapping[ResourcePath, list[ResourcePath]] = {}
 
     # Group each file by directory if no explicit output path
     if outpath is None:
         for path in found_files:
-            head, tail = os.path.split(path)
-            files_per_directory.setdefault(head, []).append(tail)
+            head, tail = path.split()
+            files_per_directory.setdefault(head, []).append(ResourcePath(tail, forceAbsolute=False))
     else:
-        files_per_directory["."] = list(found_files)
+        # We want the requested files to be paths relative to the current
+        # directory. For now this assumes that all the input files are
+        # local -- we are not trying to discover a shared root directory.
+        cwd = ResourcePath(".", forceAbsolute=True, forceDirectory=True)
+        files_per_directory[cwd] = list(found_files)
 
     # Extract translated metadata for each file in each directory
     for directory, files_in_dir in files_per_directory.items():
@@ -113,11 +119,10 @@ def write_index_files(
 
         # Write the index file
         if outpath is None:
-            index_file = os.path.join(directory, "_index.json")
+            index_file = directory.join("_index.json")
         else:
-            index_file = outpath
-        with open(index_file, "w") as fd:
-            print(json.dumps(output), file=fd)
-            log.info("Wrote index file to %s", index_file)
+            index_file = ResourcePath(outpath, forceAbsolute=False)
+        index_file.write(json.dumps(output).encode())
+        log.info("Wrote index file to %s", index_file)
 
     return okay, failed

{astro_metadata_translator-29.2025.4300 → astro_metadata_translator-29.2025.4800}/python/astro_metadata_translator/bin/writesidecar.py

@@ -14,45 +14,22 @@ from __future__ import annotations
 __all__ = ("write_sidecar_file", "write_sidecar_files")
 
 import logging
-import os
 import traceback
 from collections.abc import Sequence
-from typing import IO
+from typing import IO, TYPE_CHECKING
 
-from ..file_helpers import find_files, read_file_info
-
-log = logging.getLogger(__name__)
-
-
-def _split_ext(file: str) -> tuple[str, str]:
-    """Split the extension from the file name and return it and the root.
-
-    Parameters
-    ----------
-    file : `str`
-        The file name to examine.
+from lsst.resources import ResourcePath
 
-    Returns
-    -------
-    root : `str`
-        The root of the file name.
-    ext : `str`
-        The file extension. There is special case handling of .gz and other
-        compression extensions.
-    """
-    special = {".gz", ".bz2", ".xz", ".fz"}
-
-    root, ext = os.path.splitext(file)
+from ..file_helpers import find_files, read_file_info
 
-    if ext in special:
-        root, second_ext = os.path.splitext(root)
-        ext = second_ext + ext
+if TYPE_CHECKING:
+    from lsst.resources import ResourcePathExpression
 
-    return root, ext
+log = logging.getLogger(__name__)
 
 
 def write_sidecar_file(
-    file: str,
+    file: ResourcePathExpression,
     hdrnum: int,
     content_mode: str,
     print_trace: bool,
@@ -62,7 +39,7 @@ def write_sidecar_file(
 
     Parameters
     ----------
-    file : `str`
+    file : `str` or `lsst.resources.ResourcePathExpression`
         The file from which the header is to be read.
     hdrnum : `int`
         The HDU number to read. The primary header is always read and
@@ -103,12 +80,11 @@ def write_sidecar_file(
             return False
 
         # Calculate sidecar file name derived from this file.
-        # Match the ButlerURI behavior in that .fits.gz should be replaced
+        # .fits.gz should be replaced
         # with .json, and not resulting in .fits.json.
-        root, ext = _split_ext(file)
-        newfile = root + ".json"
-        with open(newfile, "w") as fd:
-            print(json_str, file=fd)
+        uri = ResourcePath(file)
+        newfile = uri.updatedExtension(".json")
+        newfile.write(str(json_str).encode())
         log.debug("Writing sidecar file %s", newfile)
 
     except Exception as e:
@@ -169,8 +145,8 @@ def write_sidecar_files(
     for path in sorted(found_files):
         isok = write_sidecar_file(path, hdrnum, content_mode, print_trace, outstream)
         if isok:
-            okay.append(path)
+            okay.append(str(path))
         else:
-            failed.append(path)
+            failed.append(str(path))
 
     return okay, failed

{astro_metadata_translator-29.2025.4300 → astro_metadata_translator-29.2025.4800}/python/astro_metadata_translator/file_helpers.py

@@ -20,16 +20,21 @@ __all__ = ("find_files", "read_basic_metadata_from_file", "read_file_info")
 
 import json
 import logging
-import os
 import re
 import traceback
 from collections.abc import Iterable, MutableMapping
-from typing import IO, Any
+from typing import IO, TYPE_CHECKING, Any
+
+from astropy.io import fits
+from lsst.resources import ResourcePath
 
 from .headers import merge_headers
 from .observationInfo import ObservationInfo
 from .tests import read_test_file
 
+if TYPE_CHECKING:
+    from lsst.resources import ResourcePathExpression
+
 log = logging.getLogger(__name__)
 
 # Prefer afw over Astropy
@@ -37,30 +42,15 @@ try:
     import lsst.daf.base  # noqa: F401 need PropertyBase for readMetadata
     from lsst.afw.fits import FitsError, readMetadata
 
-    def _read_fits_metadata(file: str, hdu: int, can_raise: bool = False) -> MutableMapping[str, Any] | None:
-        """Read a FITS header using afw.
-
-        Parameters
-        ----------
-        file : `str`
-            The file to read.
-        hdu : `int`
-            The header number to read.
-        can_raise : `bool`, optional
-            Indicate whether the function can raise and exception (default)
-            or should return `None` on error. Can still raise if an unexpected
-            error is encountered.
-
-        Returns
-        -------
-        md : `dict`
-            The requested header. `None` if it could not be read and
-            ``can_raise`` is `False`.
-
-        Notes
-        -----
-        Tries to catch a FitsError 104 and convert to `FileNotFoundError`.
-        """
+    have_afw = True
+
+    def _read_fits_metadata_afw(
+        file: str, hdu: int, can_raise: bool = False
+    ) -> MutableMapping[str, Any] | None:
+        # Only works with local files.
+        # Tries to catch a FitsError 104 and convert to `FileNotFoundError`.
+        # For detailed docstrings see the _read_fits_metadata implementation
+        # below.
         try:
             return readMetadata(file, hdu=hdu)
         except FitsError as e:
@@ -72,31 +62,84 @@ try:
             return None
 
 except ImportError:
-    from astropy.io import fits
+    have_afw = False
+
+
+def _read_fits_metadata_astropy(
+    file: ResourcePathExpression, hdu: int, can_raise: bool = False
+) -> MutableMapping[str, Any] | None:
+    """Read a FITS header using astropy.
+
+    Parameters
+    ----------
+    file : `str` or `lsst.resources.ResourcePathExpression`
+        The file to read.
+    hdu : `int`
+        The header number to read.
+    can_raise : `bool`, optional
+        Indicate whether the function can raise and exception (default)
+        or should return `None` on error. Can still raise if an unexpected
+        error is encountered.
+
+    Returns
+    -------
+    md : `dict`
+        The requested header. `None` if it could not be read and
+        ``can_raise`` is `False`.
+    """
+    header = None
+    uri = ResourcePath(file, forceDirectory=False)
+    try:
+        fs, fspath = uri.to_fsspec()
+        with fs.open(fspath) as f, fits.open(f) as fits_file:
+            try:
+                # Copy forces a download of the remote resource.
+                header = fits_file[hdu].header.copy()
+            except IndexError as e:
+                if can_raise:
+                    raise e
+    except Exception as e:
+        if can_raise:
+            raise e
+    return header
 
-    def _read_fits_metadata(file: str, hdu: int, can_raise: bool = False) -> MutableMapping[str, Any] | None:
-        """Read a FITS header using astropy."""
-        # For detailed docstrings see the afw implementation above
-        header = None
-        try:
-            with fits.open(file) as fits_file:
-                try:
-                    header = fits_file[hdu].header
-                except IndexError as e:
-                    if can_raise:
-                        raise e
-        except Exception as e:
-            if can_raise:
-                raise e
-        return header
 
+def _read_fits_metadata(
+    file: ResourcePathExpression, hdu: int, can_raise: bool = False
+) -> MutableMapping[str, Any] | None:
+    """Read a FITS header using afw or astropy.
+
+    Prefer afw for local reads if available.
 
-def find_files(files: Iterable[str], regex: str) -> list[str]:
+    Parameters
+    ----------
+    file : `str` or `lsst.resources.ResourcePathExpression`
+        The file to read.
+    hdu : `int`
+        The header number to read.
+    can_raise : `bool`, optional
+        Indicate whether the function can raise and exception (default)
+        or should return `None` on error. Can still raise if an unexpected
+        error is encountered.
+
+    Returns
+    -------
+    md : `dict`
+        The requested header. `None` if it could not be read and
+        ``can_raise`` is `False`.
+    """
+    uri = ResourcePath(file, forceAbsolute=False)
+    if have_afw and uri.isLocal:
+        return _read_fits_metadata_afw(uri.ospath, hdu, can_raise=can_raise)
+    return _read_fits_metadata_astropy(uri, hdu, can_raise=can_raise)
+
+
+def find_files(files: Iterable[ResourcePathExpression], regex: str) -> list[ResourcePath]:
     """Find files for processing.
 
     Parameters
     ----------
-    files : iterable of `str`
+    files : iterable of `lsst.resources.ResourcePathExpression`
         The files or directories from which the headers are to be read.
     regex : `str`
         Regular expression string used to filter files when a directory is
@@ -104,34 +147,31 @@ def find_files(files: Iterable[str], regex: str) -> list[str]:
 
     Returns
     -------
-    found_files : `list` of `str`
+    found_files : `list` of `lsst.resources.ResourcePath`
         The files that were found.
     """
     file_regex = re.compile(regex)
-    found_files = []
+    found_files: list[ResourcePath] = []
 
     # Find all the files of interest
-    for file in files:
-        if os.path.isdir(file):
-            for root, _, files in os.walk(file):
-                for name in files:
-                    path = os.path.join(root, name)
-                    if os.path.isfile(path) and file_regex.search(name):
-                        found_files.append(path)
+    for candidate in files:
+        uri = ResourcePath(candidate, forceAbsolute=False)
+        if uri.isdir():
+            found_files.extend(ResourcePath.findFileResources([uri], file_filter=file_regex, grouped=False))
         else:
-            found_files.append(file)
+            found_files.append(uri)
 
     return found_files
 
 
 def read_basic_metadata_from_file(
-    file: str, hdrnum: int, can_raise: bool = True
+    file: ResourcePathExpression, hdrnum: int, can_raise: bool = True
 ) -> MutableMapping[str, Any] | None:
     """Read a raw header from a file, merging if necessary.
 
     Parameters
     ----------
-    file : `str`
+    file : `str` or `lsst.resources.ResourcePathExpression`
         Name of file to read. Can be FITS, YAML or JSON. YAML or JSON must be
         a simple top-level dict.
     hdrnum : `int`
@@ -151,10 +191,11 @@ def read_basic_metadata_from_file(
         The header as a dict. Can be `None` if there was a problem reading
         the file.
     """
-    if file.endswith(".yaml") or file.endswith(".json"):
+    uri = ResourcePath(file, forceAbsolute=False)
+    if uri.getExtension() in (".yaml", ".json"):
         try:
             md = read_test_file(
-                file,
+                uri,
             )
         except Exception as e:
             if not can_raise:
@@ -165,7 +206,7 @@ def read_basic_metadata_from_file(
             # YAML can't have HDUs so skip merging below
             hdrnum = 0
     else:
-        md = _read_fits_metadata(file, 0, can_raise=can_raise)
+        md = _read_fits_metadata(uri, 0, can_raise=can_raise)
     if md is None:
         log.warning("Unable to open file %s", file)
         return None
@@ -174,19 +215,19 @@ def read_basic_metadata_from_file(
             hdrnum = 1
     if hdrnum > 0:
         # Allow this to fail
-        mdn = _read_fits_metadata(file, int(hdrnum), can_raise=False)
+        mdn = _read_fits_metadata(uri, int(hdrnum), can_raise=False)
         # Astropy does not allow append mode since it does not
         # convert lists to multiple cards. Overwrite for now
         if mdn is not None:
             md = merge_headers([md, mdn], mode="overwrite")
         else:
-            log.warning("HDU %d was not found in file %s. Ignoring request.", hdrnum, file)
+            log.warning("HDU %d was not found in file %s. Ignoring request.", hdrnum, uri)
 
     return md
 
 
 def read_file_info(
-    file: str,
+    file: ResourcePathExpression,
     hdrnum: int,
     print_trace: bool | None = None,
     content_mode: str = "translated",
@@ -232,9 +273,10 @@ def read_file_info(
     if content_type not in ("native", "simple", "json"):
         raise ValueError(f"Unrecognized content type request {content_type}")
 
+    uri = ResourcePath(file, forceAbsolute=False)
     try:
         # Calculate the JSON from the file
-        md = read_basic_metadata_from_file(file, hdrnum, can_raise=True if print_trace is None else False)
+        md = read_basic_metadata_from_file(uri, hdrnum, can_raise=True if print_trace is None else False)
         if md is None:
             return None
         if content_mode == "metadata":
@@ -249,7 +291,7 @@ def read_file_info(
                     json_str = json.dumps(dict(md))
                 return json_str
             return md
-        obs_info = ObservationInfo(md, pedantic=True, filename=file)
+        obs_info = ObservationInfo(md, pedantic=True, filename=str(uri))
         if content_type == "native":
             return obs_info
         simple = obs_info.to_simple()