lsst-resources 29.2025.1900__tar.gz → 29.2025.2100__tar.gz

{lsst_resources-29.2025.1900/python/lsst_resources.egg-info → lsst_resources-29.2025.2100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.1900
+Version: 29.2025.2100
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

lsst_resources-29.2025.2100/doc/lsst.resources/dav.rst

@@ -0,0 +1,251 @@
+webDAV ResourcePath
+===================
+
+The basic syntax for using a webDAV `~lsst.resources.ResourcePath` is:
+
+.. code-block::
+
+    ResourcePath("davs://host.example.org:1234/path/to/file")
+
+Configuration
+-------------
+Interacting with WebDAV servers may require specific client configuration.
+That is for instance the case when the server exposes a host certificate
+which is not issued by a certificate authority trusted by the system where
+the client runs. Another case is when the server requires the client
+presents its identity for authentication purposes.
+
+In order to configure `~lsst.resources.ResourcePath` to interact with one or
+more WebDAV servers which require specific configuration, you must initialize
+the environment variable ``LSST_RESOURCES_WEBDAV_CONFIG`` to point to a
+configuration YAML file. This configuration file must contain the settings for
+each of the webDAV servers which host any of the
+`~lsst.resources.ResourcePath` objects with ``davs://`` or ``dav://`` scheme
+you need to instantiate.
+
+The general structure of the configuration file is as follows:
+
+.. code-block:: yaml
+
+    # These settings apply to all ResourcePath objects hosted by
+    # server 'host.example.org:1234'.
+    - base_url: "davs://host.example.org:1234/"
+      user_cert: "${X509_USER_PROXY}"
+      trusted_authorities: "/etc/grid-security/certificates"
+
+    # These settings apply to all ResourcePath objects hosted by
+    # server 'host.domain.org:5432'.
+    - base_url: "davs://host.domain.org:5432/"
+      token: "${HOME}/.token"
+
+    # These settings apply to all ResourcePath objects hosted by
+    # server 'webdav.example.org:9876'.
+    - base_url: "davs://webdav.example.org:9876/"
+      timeout_connect: 30.0
+      timeout_read: 120.0
+      retries: 5
+
+For each WebDAV endpoint you can specify several settings that all instances
+of `~lsst.resources.ResourcePath` hosted at that specific endpoint will use.
+If no settings are found in the configuration for a given endpoint, sensible
+defaults values are used (see below).
+
+Please note that if your WebDAV endpoint requires client authentication you
+must have an entry for that endpoint in your configuration file. Currently
+two client authentication mechanisms are supported: either a client
+X.509 certificate (and associated private key) or a token. If both are
+specified, token authentication is preferred.
+
+Similarly, if the WebDAV server exposes a host certificate that is not
+issued by a certification authority trusted by the operating system where
+your client runs, you must add an entry for that endpoint to the configuration
+file which includes the local path to the file or directory where the trusted
+authorities can be found.
+
+Configuration settings
+^^^^^^^^^^^^^^^^^^^^^^
+These are the settings you can specify in the configuration file for each
+WebDAV endpoint:
+
+``user_cert``
+    Path to the client certificate the WebDAV client must present to the
+    server for authentication purposes.
+
+    If not specified, no client certificate is presented to the server.
+
+    This setting is ignored if ``token`` is specified (see below).
+
+``user_key``
+    Path to the private key associated to the client certificate the WebDAV
+    client must present to the server for authentication purposes.
+
+    If not specified but ``user_cert`` is specified, it is assumed that
+    the private key and the user certificate are both in the same file
+    specified in the value for ``user_cert``.
+
+``trusted_authorities``
+    Path to a local directory or certificate bundle file where the
+    certificates of the trusted certificate authorities can be found.
+    Those certificates will be used by the client of the WebDAV endpoint
+    to verify the server's host certificate.
+
+    If not specified, the certificates trusted by the system are used. The
+    server's host certificate is always verified.
+
+``token``
+    Value of the token the WebDAV client must sent to the server for
+    authentication purposes. The value of this entry may be the token itself
+    or the path to a local file where the value of the token can be found. It
+    is however discouraged to specify the token value directly in the
+    configuration file. Instead store the value of the token in a protected
+    file and specify the path to that file as the value of this entry.
+
+    If ``token`` is specified that mechanism is preferred for authentication
+    of client requests, even if ``user_cert`` is also specified.
+
+    If you specify a value for ``token`` which is the path to a local file
+    which contains the actual token, that file must be protected for
+    reading and writing only by its owner. The contents of that file is
+    automatically reloaded when every time it is modified.
+
+``timeout_connect``
+    Timeout in seconds to establish a network connection with the remote
+    server.
+
+    Default: 10 seconds (``float``).
+
+``timeout_read``
+    Timeout in seconds to read the response to a request sent to the server.
+    This is total time for reading both the headers and the response body.
+    It must be large enough to allow for upload and download of files
+    of typical size.
+
+    Default: 300 seconds (``float``).
+
+``retries``
+    Number of times to retry requests before failing. Retry happens only
+    under certain conditions.
+
+    Default: 3 (``int``).
+
+``retry_backoff_min``
+    Minimal retry backoff (in seconds) for the client to compute
+    the wait time before retrying a request.
+    A value in the interval [``retry_backoff_min``, ``retry_backoff_max``]
+    is randomly selected as the backoff factor every time a request is
+    retried.
+
+    Default: 1.0 seconds (``float``).
+
+``retry_backoff_max``
+    Maximum retry backoff (in seconds) for the client to compute
+    the wait time before retrying a request.
+    A value in the interval [``retry_backoff_min``, ``retry_backoff_max``]
+    is randomly selected as the backoff factor every time a request is
+    retried.
+
+    Default: 3.0 seconds (``float``).
+
+``buffer_size``
+    Size of the buffer (in mebibytes, i.e. 1024*1024 bytes) the WebDAV
+    client of this endpoint will use when sending requests and receiving
+    responses.
+
+    Default: 5 mebibytes (``int``).
+
+``persistent_connections_frontend``
+    Maximum number of network connections to persist against each one of
+    the hosts in the server frontend.
+
+    Default: 50 (``int``).
+
+``persistent_connections_backend``
+    Maximum number of network connections to persist against each one of
+    the hosts in the server backend.
+
+    Default: 100 (``int``).
+
+``enable_fsspec``
+    If specified, expose a `fsspec <https://filesystem-spec.read>`_-compatible,
+    read-only file system for accessing a  `~lsst.resources.ResourcePath`
+    object.
+
+    Default: ``true`` (``boolean``).
+
+``request_checksum``
+    If specified, the WebDAV client will request the server to compute
+    a checksum of the file contents every time a file is uploaded.
+
+    Note that it is the server decision to compute the checksum. Some
+    servers simply ignore that request.
+
+    Accepted values: ``adler32``, ``md5``,  ``sha-256``, ``sha-512``.
+
+``collect_memory_usage``
+    If specified, memory usage data is collected when running in debug mode.
+    Collecting memory usage data is expensive, so this setting should not
+    be used in production.
+
+    Accepted values: ``true``,  ``false``.
+
+    Default: ``false`` (``boolean``).
+
+
+Configuration Examples
+----------------------
+
+These are examples of configuration files for
+`dCache <https://www.dcache.org>`_ and `XRootD <https://xrootd.org>`_ WebDAV
+servers.
+
+Set the environment variable ``LSST_RESOURCES_WEBDAV_CONFIG`` to point to
+your configuration file:
+
+.. code-block:: bash
+
+    export LSST_RESOURCES_WEBDAV_CONFIG="${HOME}/.lsst/dav_conf.yaml"
+
+
+dCache
+^^^^^^
+
+Typical configuration settings for a **dCache** endpoint are:
+
+.. code-block:: yaml
+
+    - base_url: "davs://dcache.example.org:2880/"
+      trusted_authorities: "/etc/grid-security/certificates"
+      user_cert: "${X509_USER_PROXY}"
+      request_checksum: "md5"
+
+The example above uses X.509 grid proxy client authentication. If you prefer
+to use a token use instead:
+
+.. code-block:: yaml
+
+    - base_url: "davs://dcache.example.org:2880/"
+      trusted_authorities: "/etc/grid-security/certificates"
+      request_checksum: "md5"
+      token: "${HOME}/.lsst/dcache_token"
+
+and ensure the file at path ``${HOME}/.lsst/dcache_token`` contains the
+client authentication token and is only readable and writable by you.
+
+In this example we configure `~lsst.resources.ResourcePath` to request the
+dCache server to compute and record the MD5 checksum when a file is uploaded,
+in addition to the ADLER32 checksum dCache always computes and records for
+each file.
+
+XRootD
+^^^^^^
+
+Typical configuration settings for a **XRootD** endpoint are:
+
+.. code-block:: yaml
+
+    - base_url: "davs://xrootd.example.org:1094/"
+      trusted_authorities: "/etc/grid-security/certificates"
+      user_cert: "${X509_USER_PROXY}"
+
+If you prefer to use a token for client authentication use the ``token``
+setting instead (see dCache example above).

{lsst_resources-29.2025.1900 → lsst_resources-29.2025.2100}/doc/lsst.resources/index.rst

@@ -29,6 +29,7 @@ Configuration
 
 .. toctree::
    s3.rst
+   dav.rst
 
 .. _lsst.resources-pyapi:
 

lsst_resources-29.2025.2100/python/lsst/resources/_resourceHandles/_davResourceHandle.py

@@ -0,0 +1,190 @@
+# This file is part of lsst-resources.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# Use of this source code is governed by a 3-clause BSD-style
+# license that can be found in the LICENSE file.
+
+from __future__ import annotations
+
+__all__ = ("DavReadResourceHandle",)
+
+import io
+import logging
+from collections.abc import Callable, Iterable
+from typing import TYPE_CHECKING, AnyStr
+
+from ..davutils import DavFileMetadata
+from ._baseResourceHandle import BaseResourceHandle, CloseStatus
+
+if TYPE_CHECKING:
+    from ..dav import DavResourcePath
+
+
+class DavReadResourceHandle(BaseResourceHandle[bytes]):
+    """WebDAV-based specialization of `.BaseResourceHandle`.
+
+    Parameters
+    ----------
+    mode : `str`
+        Handle modes as described in the python `io` module.
+    log : `~logging.Logger`
+        Logger to used when writing messages.
+    uri : `lsst.resources.dav.DavResourcePath`
+        URI of remote resource.
+    newline : `str` or `None`, optional
+        When doing multiline operations, break the stream on given character.
+        Defaults to newline. If a file is opened in binary mode, this argument
+        is not used, as binary files will only split lines on the binary
+        newline representation.
+    """
+
+    def __init__(
+        self,
+        mode: str,
+        log: logging.Logger,
+        uri: DavResourcePath,
+        stat: DavFileMetadata,
+        *,
+        newline: AnyStr | None = None,
+    ) -> None:
+        super().__init__(mode, log, uri, newline=newline)
+        self._uri: DavResourcePath = uri
+        self._stat: DavFileMetadata = stat
+        self._current_position = 0
+        self._cache: io.BytesIO | None = None
+        self._buffer: io.BytesIO | None = None
+        self._closed = CloseStatus.OPEN
+
+    def close(self) -> None:
+        self._closed = CloseStatus.CLOSED
+        self._cache = None
+
+    @property
+    def closed(self) -> bool:
+        return self._closed == CloseStatus.CLOSED
+
+    def fileno(self) -> int:
+        raise io.UnsupportedOperation("DavReadResourceHandle does not have a file number")
+
+    def flush(self) -> None:
+        modes = set(self._mode)
+        if {"w", "x", "a", "+"} & modes:
+            raise io.UnsupportedOperation("DavReadResourceHandles are read only")
+
+    @property
+    def isatty(self) -> bool | Callable[[], bool]:
+        return False
+
+    def readable(self) -> bool:
+        return True
+
+    def readline(self, size: int = -1) -> bytes:
+        raise io.UnsupportedOperation("DavReadResourceHandles Do not support line by line reading")
+
+    def readlines(self, size: int = -1) -> Iterable[bytes]:
+        raise io.UnsupportedOperation("DavReadResourceHandles Do not support line by line reading")
+
+    def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
+        match whence:
+            case io.SEEK_SET:
+                if offset < 0:
+                    raise ValueError(f"negative seek value {offset}")
+                self._current_position = offset
+            case io.SEEK_CUR:
+                self._current_position += offset
+            case io.SEEK_END:
+                self._current_position = self._stat.size + offset
+            case _:
+                raise ValueError(f"unexpected value {whence} for whence in seek()")
+
+        if self._current_position < 0:
+            self._current_position = 0
+
+        return self._current_position
+
+    def seekable(self) -> bool:
+        return True
+
+    def tell(self) -> int:
+        return self._current_position
+
+    def truncate(self, size: int | None = None) -> int:
+        raise io.UnsupportedOperation("DavReadResourceHandles Do not support truncation")
+
+    def writable(self) -> bool:
+        return False
+
+    def write(self, b: bytes, /) -> int:
+        raise io.UnsupportedOperation("DavReadResourceHandles are read only")
+
+    def writelines(self, b: Iterable[bytes], /) -> None:
+        raise io.UnsupportedOperation("DavReadResourceHandles are read only")
+
+    @property
+    def _eof(self) -> bool:
+        return self._current_position >= self._stat.size
+
+    def _download_to_cache(self) -> io.BytesIO:
+        """Download the entire content of the remote resource to an internal
+        memory buffer.
+        """
+        if self._cache is None:
+            self._cache = io.BytesIO()
+            self._cache.write(self._uri.read())
+
+        return self._cache
+
+    def read(self, size: int = -1) -> bytes:
+        if self._eof or size == 0:
+            return b""
+
+        # If this file's size is small than the buffer size configured for
+        # this URI's client, download the entire file in one request and cache
+        # its content. This avoids multiple roundtrips to the server
+        # for retrieving small chunks.
+        if self._stat.size <= self._uri._client._config.buffer_size:
+            self._download_to_cache()
+
+        # If we are asked to read the whole file content, cache the entire
+        # file content and return a copy-on-write memory view of our internal
+        # cache.
+        if self._current_position == 0 and size == -1:
+            cache = self._download_to_cache()
+            self._current_position = self._stat.size
+            return cache.getvalue()
+
+        # This is a partial read. If we have already cached the whole file
+        # content use the cache to build the return value.
+        if self._cache is not None:
+            start = self._current_position
+            end = self._current_position = self._stat.size if size < 0 else start + size
+            return self._cache.getvalue()[start:end]
+
+        # We need to make a partial read from the server. Reuse our internal
+        # I/O buffer to reduce memory allocations.
+        if self._buffer is None:
+            self._buffer = io.BytesIO()
+
+        start = self._current_position
+        end = self._stat.size if size < 0 else min(start + size, self._stat.size)
+        self._buffer.seek(0)
+        self._buffer.write(self._uri.read_range(start=start, end=end - 1))
+        count = self._buffer.tell()
+        self._current_position += count
+        return self._buffer.getvalue()[0:count]
+
+    def readinto(self, output: bytearray) -> int:
+        """Read up to `len(output)` bytes into `output` and return the number
+        of bytes read.
+        """
+        if self._eof or len(output) == 0:
+            return 0
+
+        data = self.read(len(output))
+        output[:] = data
+        return len(data)

{lsst_resources-29.2025.1900 → lsst_resources-29.2025.2100}/python/lsst/resources/_resourcePath.py

@@ -368,9 +368,9 @@ class ResourcePath:  # numpydoc ignore=PR02
 
                 subclass = HttpResourcePath
             elif parsed.scheme in {"dav", "davs"}:
-                from .http import HttpResourcePath
+                from .dav import DavResourcePath
 
-                subclass = HttpResourcePath
+                subclass = DavResourcePath
             elif parsed.scheme == "gs":
                 from .gs import GSResourcePath