lsst-resources 29.2025.4800__py3-none-any.whl → 30.0.0rc3__py3-none-any.whl

lsst/resources/_resourcePath.py

@@ -386,6 +386,11 @@ class ResourcePath:  # numpydoc ignore=PR02
                 from .mem import InMemoryResourcePath
 
                 subclass = InMemoryResourcePath
+            elif parsed.scheme == "eups":
+                # EUPS package root.
+                from .eups import EupsResourcePath
+
+                subclass = EupsResourcePath
             else:
                 raise NotImplementedError(
                     f"No URI support for scheme: '{parsed.scheme}' in {parsed.geturl()}"
@@ -396,11 +401,15 @@ class ResourcePath:  # numpydoc ignore=PR02
             )
 
             # It is possible for the class to change from schemeless
-            # to file so handle that
+            # to file or eups so handle that
             if parsed.scheme == "file":
                 from .file import FileResourcePath
 
                 subclass = FileResourcePath
+            elif parsed.scheme == "eups":
+                from .eups import EupsResourcePath
+
+                subclass = EupsResourcePath
 
         # Now create an instance of the correct subclass and set the
         # attributes directly
@@ -410,8 +419,13 @@ class ResourcePath:  # numpydoc ignore=PR02
         if isTemporary is None:
             isTemporary = False
         self.isTemporary = isTemporary
+        self._set_proxy()
         return self
 
+    def _set_proxy(self) -> None:
+        """Calculate internal proxy for externally visible resource path."""
+        pass
+
     @property
     def scheme(self) -> str:
         """Return the URI scheme.

lsst/resources/eups.py

@@ -0,0 +1,101 @@
+# This file is part of lsst-resources.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# Use of this source code is governed by a 3-clause BSD-style
+# license that can be found in the LICENSE file.
+
+from __future__ import annotations
+
+__all__ = ("EupsResourcePath",)
+
+import logging
+import posixpath
+import urllib.parse
+
+from lsst.utils import getPackageDir
+
+from ._resourcePath import ResourcePath
+from .proxied import ProxiedResourcePath
+from .utils import os2posix
+
+log = logging.getLogger(__name__)
+
+
+class EupsResourcePath(ProxiedResourcePath):
+    """URI referring to an EUPS package.
+
+    These URIs look like: ``eups://daf_butler/configs/file.yaml``
+    where the network location is the EUPS package name.
+
+    Internally they are proxied by either ``file`` URIs or ``resource`` URIs.
+    If an ``{product}_DIR`` environment variable is found it will be used
+    and internally a ``file`` URI will be created. If no environment variable
+    is found an attempt will be made to convert the EUPS product name to
+    a python package and a ``resource`` URI will be returned with
+    ``/resources`` appended. The convention is that any package supporting
+    an EUPS URI outside an EUPS environment will also have made available
+    the support files as package resources.
+
+    If it is known that the package supports package resources it is always
+    better to use that URI form explicitly since it is more robust since
+    not all EUPS packages can reliably be converted to python packages
+    without EUPS.
+    """
+
+    quotePaths = False
+    _proxy: ResourcePath | None = None
+    _default_namespace: str = "lsst"
+
+    def _set_proxy(self) -> None:
+        """Calculate the internal `ResourcePath` corresponding to the public
+        version.
+        """
+        # getPackageDir returns an absolute path.
+        try:
+            eups_path = getPackageDir(self.netloc)
+            log.debug("Found EUPS package %s via env var", self.netloc)
+        except LookupError:
+            eups_path = ""
+        if eups_path:
+            # Must convert this path into a file URI.
+            new_path = posixpath.join(os2posix(eups_path), os2posix(self.path.lstrip("/")))
+            parsed = self._uri._replace(path=urllib.parse.quote(new_path), scheme="file", netloc="")
+            self._proxy = ResourcePath(parsed)
+            return
+
+        # If there is no _DIR env var we need to look for python package
+        # resource. There is no guaranteed way to generated a python package
+        # from an EUPS product name.
+        # daf_butler -> lsst.daf.butler
+        # image_cutout_backend -> lsst.image_cutout_backend
+        # astro_metadata_translator -> astro_metadata_translator
+        product = self.netloc
+        variants = (
+            product,
+            self._default_namespace + "." + product.replace("_", "."),
+            self._default_namespace + "." + product,
+            product.replace("_", "."),
+        )
+        for variant in variants:
+            proxy = ResourcePath(f"resource://{variant}/resources", forceDirectory=True)
+            # This can be slow because package imports happen but there is
+            # no other way to check that we have the correct variant.
+            log.debug("Trying variant %s", proxy)
+            if proxy.exists():
+                self._proxy = proxy
+                if self.path:
+                    self._proxy = self._proxy.join(self.path.lstrip("/"))
+                log.debug(f"Found variant {variant}")
+                return
+
+        # Can not find an actively set up package or resources for this EUPS
+        # product. Return without setting a proxy to allow standard URI
+        # path manipulations to happen but defer failure until someone tries
+        # to use the proxy for read.
+        log.debug("Could not find any files corresponding to %s", self)
+        return

lsst/resources/packageresource.py

@@ -42,6 +42,8 @@ class PackageResourcePath(ResourcePath):
     resource name.
     """
 
+    quotePaths = False
+
     def _get_ref(self) -> resources.abc.Traversable | None:
         """Obtain the object representing the resource.
 

lsst/resources/proxied.py

@@ -0,0 +1,148 @@
+# This file is part of lsst-resources.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# Use of this source code is governed by a 3-clause BSD-style
+# license that can be found in the LICENSE file.
+
+from __future__ import annotations
+
+__all__ = ("ProxiedResourcePath",)
+
+import contextlib
+import logging
+import re
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+
+from ._resourcePath import ResourceHandleProtocol, ResourcePath, ResourcePathExpression
+from .utils import TransactionProtocol
+
+try:
+    import fsspec
+    from fsspec.spec import AbstractFileSystem
+except ImportError:
+    fsspec = None
+    AbstractFileSystem = type
+
+
+log = logging.getLogger(__name__)
+
+
+class ProxiedResourcePath(ABC, ResourcePath):
+    """URI that is represented internally by another type of URI for file I/O.
+
+    For example ``abc://xyz/file.txt`` could be the public URI form but
+    internally all file access is forwarded to a ``file`` URI.
+    """
+
+    _proxy: ResourcePath | None = None
+
+    @abstractmethod
+    def _set_proxy(self) -> None:
+        """Calculate the internal `ResourcePath` corresponding to the public
+        version.
+        """
+        raise NotImplementedError("Proxy must be configured")
+
+    def _get_proxy(self) -> ResourcePath:
+        """Retrieve the proxied ResourcePath."""
+        proxy = self._proxy
+        if proxy is None:
+            raise FileNotFoundError(f"Internal error: No proxy ResourcePath available for {self}")
+        return proxy
+
+    def to_fsspec(self) -> tuple[AbstractFileSystem, str]:
+        try:
+            proxy = self._get_proxy()
+        except FileNotFoundError:
+            raise NotImplementedError(f"No proxy registered for {self}. Resource does not exist.") from None
+        return proxy.to_fsspec()
+
+    def isdir(self) -> bool:
+        if self.dirLike is None:
+            try:
+                proxy = self._get_proxy()
+            except FileNotFoundError:
+                return False
+            self.dirLike = proxy.isdir()
+        return self.dirLike
+
+    def exists(self) -> bool:
+        try:
+            proxy = self._get_proxy()
+        except FileNotFoundError:
+            # If there is no proxy registered then the resource can not exist.
+            return False
+        return proxy.exists()
+
+    def remove(self) -> None:
+        proxy = self._get_proxy()
+        proxy.remove()
+
+    def read(self, size: int = -1) -> bytes:
+        proxy = self._get_proxy()
+        return proxy.read(size=size)
+
+    @contextlib.contextmanager
+    def as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePathExpression | None = None
+    ) -> Iterator[ResourcePath]:
+        proxy = self._get_proxy()
+        with proxy.as_local(multithreaded=multithreaded, tmpdir=tmpdir) as loc:
+            yield loc
+
+    @contextlib.contextmanager
+    def open(
+        self,
+        mode: str = "r",
+        *,
+        encoding: str | None = None,
+        prefer_file_temporary: bool = False,
+    ) -> Iterator[ResourceHandleProtocol]:
+        proxy = self._get_proxy()
+        with proxy.open(mode, encoding=encoding, prefer_file_temporary=prefer_file_temporary) as fh:
+            yield fh
+
+    def walk(
+        self, file_filter: str | re.Pattern | None = None
+    ) -> Iterator[list | tuple[ResourcePath, list[str], list[str]]]:
+        try:
+            proxy = self._get_proxy()
+        except FileNotFoundError as e:
+            raise ValueError(str(e)) from None
+        for proxied_root, dirs, files in proxy.walk(file_filter=file_filter):
+            # Need to return the directory in the original form and not the
+            # proxy form.
+            relative_to_self = proxied_root.path.removeprefix(proxy.path)
+            root = self.replace(path=self._pathModule.join(self.path, relative_to_self))
+            yield root, dirs, files
+
+    def size(self) -> int:
+        proxy = self._get_proxy()
+        return proxy.size()
+
+    def write(self, data: bytes, overwrite: bool = True) -> None:
+        proxy = self._get_proxy()
+        proxy.write(data, overwrite=overwrite)
+
+    def mkdir(self) -> None:
+        proxy = self._get_proxy()
+        proxy.mkdir()
+
+    def transfer_from(
+        self,
+        src: ResourcePath,
+        transfer: str = "copy",
+        overwrite: bool = False,
+        transaction: TransactionProtocol | None = None,
+        multithreaded: bool = True,
+    ) -> None:
+        proxy = self._get_proxy()
+        proxy.transfer_from(
+            src, transfer=transfer, overwrite=overwrite, transaction=transaction, multithreaded=multithreaded
+        )

lsst/resources/schemeless.py

@@ -16,6 +16,7 @@ __all__ = ("SchemelessResourcePath",)
 import logging
 import os
 import os.path
+import re
 import stat
 import urllib.parse
 from pathlib import PurePath
@@ -211,12 +212,23 @@ class SchemelessResourcePath(FileResourcePath):
         expandedPath = os.path.expanduser(urllib.parse.unquote(parsed.path))
 
         # We might also be receiving a path containing environment variables
-        # so expand those here
+        # so expand those here, although we treat $X_DIR at the start of the
+        # path as a special EUPS URI. This allows us to handle EUPS-style
+        # env var specifications even if EUPS has not set them.
+        # Support $X_DIR and ${X_DIR} variants at the start of the path.
+        if eups := re.match(r"(\$\{?([A-Z_]+)_DIR\}?)/", expandedPath):
+            replacements["scheme"] = "eups"
+            # Two matching groups: the entire env var, and the EUPS product.
+            replacements["netloc"] = eups.group(2).lower()
+            expandedPath = expandedPath.removeprefix(eups.group(1))
+
         expandedPath = os.path.expandvars(expandedPath)
 
-        # Ensure that this becomes a file URI if it is already absolute
+        # Ensure that this becomes a file URI if it is already absolute, unless
+        # we already overrode it above.
         if os.path.isabs(expandedPath):
-            replacements["scheme"] = "file"
+            if "scheme" not in replacements:
+                replacements["scheme"] = "file"
             # Keep in OS form for now to simplify later logic
             replacements["path"] = os.path.normpath(expandedPath)
         elif forceAbsolute:
@@ -265,9 +277,9 @@ class SchemelessResourcePath(FileResourcePath):
             if not replacements["path"].endswith(os.sep):
                 replacements["path"] += os.sep
 
-        if "scheme" in replacements:
+        if "scheme" in replacements and replacements["scheme"] == "file":
             # This is now meant to be a URI path so force to posix
-            # and quote
+            # and quote. EUPS URIs are not quoted.
             replacements["path"] = urllib.parse.quote(os2posix(replacements["path"]))
 
         # ParseResult is a NamedTuple so _replace is standard API

lsst/resources/tests.py

@@ -417,11 +417,19 @@ class GenericTestCase(_GenericTestCase):
     def test_escapes(self) -> None:
         """Special characters in file paths."""
         src = self.root_uri.join("bbb/???/test.txt")
-        self.assertNotIn("???", src.path)
+        quotes = src.quotePaths
+
+        if quotes:
+            self.assertNotIn("???", src.path)
+        else:
+            self.assertIn("???", src.path)
         self.assertIn("???", src.unquoted_path)
 
         file = src.updatedFile("tests??.txt")
-        self.assertNotIn("??.txt", file.path)
+        if quotes:
+            self.assertNotIn("??.txt", file.path)
+        else:
+            self.assertIn("??.txt", file.path)
 
         src = src.updatedFile("tests??.txt")
         self.assertIn("??.txt", src.unquoted_path)
@@ -456,7 +464,10 @@ class GenericTestCase(_GenericTestCase):
 
         fnew2 = fdir.join(new2name)
         self.assertTrue(fnew2.unquoted_path.endswith(new2name))
-        self.assertNotIn("###", fnew2.path)
+        if quotes:
+            self.assertNotIn("###", fnew2.path)
+        else:
+            self.assertIn("###", fnew2.path)
 
         # Test that children relative to schemeless and file schemes
         # still return the same unquoted name

lsst/resources/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.4800"
+__version__ = "30.0.0rc3"

{lsst_resources-29.2025.4800.dist-info → lsst_resources-30.0.0rc3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.4800
+Version: 30.0.0rc3
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause

{lsst_resources-29.2025.4800.dist-info → lsst_resources-30.0.0rc3.dist-info}/RECORD

@@ -1,31 +1,33 @@
 lsst/__init__.py,sha256=9I6UQ9gj-ZcPlvsa0OPBo76UujxXVehVzw9yMAOQvyM,466
 lsst/resources/__init__.py,sha256=BDj6uokvd0ZQNGl-Xgz5gZd83Z0L2gFqGSk0KJpylP8,778
-lsst/resources/_resourcePath.py,sha256=fnB8XNUfk25lV378U0kg1O6c1QnSJL7F9iqaEt08IOA,74508
+lsst/resources/_resourcePath.py,sha256=VLW9YDADwnvK2WOypmWPCstd6iTnvDns1Jtf6UAR03E,74980
 lsst/resources/dav.py,sha256=ZYP7PnQzS7epm5woxnn1_t1XhsPQZm6_q1kv8baUfn4,32100
 lsst/resources/davutils.py,sha256=5geEl_44lrWX-Si1VDfYJ6WP1rg22PBqlyD_v1HE4yI,100300
+lsst/resources/eups.py,sha256=imN_qrmtHVQ0g-rCrGNYrlabWBJONRCT02hZkN7ikvc,4056
 lsst/resources/file.py,sha256=v2XLOzflfhI6kjUGB1mE8p-1e1B2eE58PW-qsQSCqdA,24360
 lsst/resources/gs.py,sha256=3qMEqO1wIK05BJmuUHtsEunuYWgR4-eB5Z3ffxEtb0o,12827
 lsst/resources/http.py,sha256=WSx2VXKFd6486TytV2NMfdgLntioL6FvliZWpn9LtDE,92426
 lsst/resources/location.py,sha256=x3Tq0x5o1OXYmZDxYBenUG1N71wtDhnjVAr3s2ZEiu8,7937
 lsst/resources/mem.py,sha256=xCpGgvxF2gmO5gLkOikKvIet2RPvaPCiARenR9pUWCk,1115
-lsst/resources/packageresource.py,sha256=vnfeRlpVwpC5cDQZE6Lnh8EH6oZy1sH2vLz9ONYjJ4k,6817
+lsst/resources/packageresource.py,sha256=FJ2t7HSnAgtfLUEvg5TTEYrWEXdqVagTmWGfP0hCfC4,6841
+lsst/resources/proxied.py,sha256=9hpfVBbzkC66QNHSKgd4lXBv3XTMzqPwXalkLZv7A34,4775
 lsst/resources/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/resources/s3.py,sha256=NGJPM4BjtqFIPvg9vbp_blrIRt009NbOm06cr65Wqmw,29662
 lsst/resources/s3utils.py,sha256=ojWf9BPrK9mhGQ8jvs4_8Nsqf9360e79U5FnPTxe24A,14576
-lsst/resources/schemeless.py,sha256=9tgqf0eQI3ErGpGSscTRFk_8amF6GwpykPBaTa-KqLI,10909
-lsst/resources/tests.py,sha256=UD2Pql8olpW9oCDlsA_jtl23SZtknp7ReuJHLcMPSa0,46237
+lsst/resources/schemeless.py,sha256=Bhcjy-M3k7D-Kmf1ShXYaPqceGb0bOns4Rw9ecXOJjA,11658
+lsst/resources/tests.py,sha256=0tDhVtF1euMH0Y7QR-RbQyo83GPe9baIPI6pM9VyZ-Y,46516
 lsst/resources/utils.py,sha256=6O3Mq7JbPEtqyD2lM77pRpwcPMfV5SxiNMknw-F2vNs,8097
-lsst/resources/version.py,sha256=MHZJr_Yoevn1SeqerO6dmfTP3xtJLGxPmEi-9Jpl380,55
+lsst/resources/version.py,sha256=Tt7y1Knxuicvk026_H8m0GXmxEFM8XEWyfbDoHcAxbs,52
 lsst/resources/_resourceHandles/__init__.py,sha256=zOcZ8gVEBdAWcHJaZabA8Vdq-wAVcxjbmA_1b1IWM6M,76
 lsst/resources/_resourceHandles/_baseResourceHandle.py,sha256=lQwxDOmFUNJndTxsjpz-HxrQBL0L-z4aXQocHdOEI7c,4676
 lsst/resources/_resourceHandles/_davResourceHandle.py,sha256=xcJNFUj8VzlPOKlHdXXoFFyiLNiSFiT-RFNqJRzKniQ,6799
 lsst/resources/_resourceHandles/_fileResourceHandle.py,sha256=2nC8tfP_ynAfjpzrtkw_1ahx1CuMEFpZ5mLmofSShUk,3676
 lsst/resources/_resourceHandles/_httpResourceHandle.py,sha256=Yami8IVGeru4bLQCag-OvGG0ltz1qyEg57FY4IEB87Y,10995
 lsst/resources/_resourceHandles/_s3ResourceHandle.py,sha256=Cp-eBtptskbmthy3DwLKPpYPLvU_lrqtK10X37inHt0,12406
-lsst_resources-29.2025.4800.dist-info/licenses/COPYRIGHT,sha256=yazVsoMmFwhiw5itGrdT4YPmXbpsQyUFjlpOyZIa77M,148
-lsst_resources-29.2025.4800.dist-info/licenses/LICENSE,sha256=7wrtgl8meQ0_RIuv2TjIKpAnNrl-ODH-QLwyHe9citI,1516
-lsst_resources-29.2025.4800.dist-info/METADATA,sha256=c5UbrCKnTjqtgCZiLq5INC7gvEVVL3rDlzT5Ev1I6PU,2240
-lsst_resources-29.2025.4800.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_resources-29.2025.4800.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_resources-29.2025.4800.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_resources-29.2025.4800.dist-info/RECORD,,
+lsst_resources-30.0.0rc3.dist-info/licenses/COPYRIGHT,sha256=yazVsoMmFwhiw5itGrdT4YPmXbpsQyUFjlpOyZIa77M,148
+lsst_resources-30.0.0rc3.dist-info/licenses/LICENSE,sha256=7wrtgl8meQ0_RIuv2TjIKpAnNrl-ODH-QLwyHe9citI,1516
+lsst_resources-30.0.0rc3.dist-info/METADATA,sha256=iTiiS1jfRbFhYIVbouFR6TCSnoHjqKYeI64RJElTofY,2237
+lsst_resources-30.0.0rc3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_resources-30.0.0rc3.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_resources-30.0.0rc3.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_resources-30.0.0rc3.dist-info/RECORD,,