pathapi-uri 0.1.0__py3-none-any.whl

uripath/schemes/_http/_path.py

@@ -0,0 +1,371 @@
+from __future__ import annotations
+
+import errno as _errno
+import os as _os
+import re as _re
+import typing as _ty
+import urllib.parse as _urlparse
+
+import pathapi.http as _http
+from pathapi import FileStat
+from pathapi import providers as _providers
+from pathapi.http import (
+    HTTPResponse,
+    drop_body,
+    is_redirect,
+    os_error,
+    raise_for_status,
+    server_size,
+    translate_errors,
+)
+from .. import _httpdate
+from .._path import URIPath
+from . import _settings
+from ._backend import HTTPBackend, _DerivedHTTPBackend
+from ._headers import _IDENTITY_ENCODING
+from ._listing import _DirectoryListingParser, _FileEntry
+from ._streams import (
+    HTTPAppendStream,
+    HTTPWriteStream,
+    _ResponseReader,
+    _accept,
+    _charset,
+    _response_reader,
+)
+
+_CHUNK = 64 * 1024
+
+
+# A listing is parsed only from an HTML reply (or one with no Content-Type).
+_LISTING_TYPES = ("text/html", "application/xhtml+xml")
+
+
+_RE_URL_SCHEME = _re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://")
+
+
+class HTTPURIPath(URIPath):
+    """`http`/`https` scheme: read/write access over HTTP (`PUT`/`DELETE`
+    for writes/deletes, configurable via `with_transport()`), listing
+    directories by scraping an Apache/nginx-style HTML index with a
+    zero-dependency in-house parser (`_DirectoryListingParser`). Needs no
+    third-party library: requests go through `HTTPTransport` unless a
+    transport is given."""
+
+    __SCHEMES = ("http", "https")
+    __slots__ = ()
+
+    if _ty.TYPE_CHECKING:
+        backend: HTTPBackend
+
+    def _initbackend(self):
+        return _DerivedHTTPBackend(timeout=_settings.DEFAULT_TIMEOUT)
+
+    def _make_child_relpath(self, name, stat_hint=None, **kwargs):
+        """The child `name`. A name given by the caller keeps the query and
+        fragment like every join. An entry of a listing (it comes with its
+        stat) is addressed by its href resolved against the directory
+        (RFC 3986 5.2), so the query that shapes the index (`?sort=name`) and
+        the fragment belong to the request for the listing, not to the entry."""
+        if stat_hint is None:
+            return super()._make_child_relpath(name, **kwargs)
+        directory = self.path if self.path.endswith("/") else self.path + "/"
+        child = self._from_parsed_parts(self.source, directory + name, "", "", **kwargs)
+        child._stat_hint = stat_hint
+        return child
+
+    def _listdir(self) -> list[_FileEntry]:
+        # The slash form first: a server redirects a directory's slash-less URL to it, at a
+        # round trip per listing. One that answers 404 for it (a file, or a server that
+        # serves the bare path only) is asked again for the path as given.
+        slashed = self if self.path.endswith("/") else self.with_path(self.path + "/")
+        try:
+            req = self._get(slashed)
+        except FileNotFoundError:
+            if slashed is self:
+                raise
+            req = self._get(self)
+        try:
+            content_type = req.headers.get("Content-Type") or ""
+            mime = content_type.split(";", 1)[0].strip().lower()
+            if mime and mime not in _LISTING_TYPES:
+                # A file: pathlib's iterdir() raises, and its body -- maybe
+                # gigabytes -- is never downloaded to look for links.
+                raise os_error(NotADirectoryError, self)
+            text = self._read_listing(req)
+        finally:
+            req.close()
+        # Scoped by the URL actually answered (after redirects), not by the
+        # page's own <title>.
+        parser = _DirectoryListingParser(base_url=req.url)
+        parser.feed(text)
+        parser.close()
+        return parser.listing
+
+    def _read_listing(self, req: HTTPResponse) -> str:
+        """The body of the directory index `req` as text, read the way
+        `open()` reads a file: a cut-short, stalled or undecodable body
+        raises `TimeoutError`/`OSError` naming the path, and one longer than
+        `pathapi.http.MAX_REPLY_BYTES` raises `OSError(EFBIG)`."""
+        limit = _http.MAX_REPLY_BYTES
+        too_large = OSError(
+            _errno.EFBIG,
+            f"Directory index of {self} is larger than {limit} bytes "
+            "(MAX_REPLY_BYTES)",
+        )
+        stated = server_size(req.headers.get("Content-Length"))
+        if stated is not None and stated > limit:
+            raise too_large
+        reader = _ResponseReader(self, req, _CHUNK)
+        chunks = []
+        size = 0
+        while chunk := reader.read(_CHUNK):
+            size += len(chunk)
+            if size > limit:
+                raise too_large
+            chunks.append(chunk)
+        body = b"".join(chunks)
+        try:
+            return body.decode(_charset(req.headers), "replace")
+        except LookupError:
+            return body.decode("utf-8", "replace")
+
+    def _get(self, uri, **kwargs) -> HTTPResponse:
+        """A streamed `GET` of `uri` that succeeded, its body unread. A
+        failure is translated to the pathlib exception for this path, and the
+        connection is released first: `stream=True` leaves it held."""
+        with translate_errors(self):
+            req = self.backend.request("GET", uri, stream=True, **kwargs)
+            try:
+                raise_for_status(req)
+            except BaseException:
+                drop_body(req)
+                raise
+        return req
+
+    def _scandir(self):
+        # `_listdir()`'s single GET already carries type/size/mtime for
+        # every child -- reuse it instead of `iterdir()` + a HEAD per child.
+        for entry in self._listdir():
+            # A subdirectory entry carries a trailing "/" (htmllistparse's name is "sub/"):
+            # strip it, or the child's path ends in "/" and `Pathname.name` (the last segment)
+            # is "".
+            is_dir = entry.name.endswith("/")
+            name = entry.name.removesuffix("/")
+            if not _providers.is_safe_child_name(name):
+                # A listing is untrusted: wsgidav renders its parent row as `<a href="..">` and a
+                # `<pre>` index can carry `./`. "." or ".." as a child would make `unlink()` DELETE
+                # the parent collection and `walk()` loop forever. Filtered here for every parser.
+                continue
+            if is_dir:
+                stat = FileStat(
+                    st_size=0, st_mtime=_httpdate.parsedate(entry.modified), is_dir=True
+                )
+            elif entry.size is not None and entry.size_exact and entry.modified:
+                stat = FileStat(
+                    st_size=entry.size,
+                    st_mtime=_httpdate.parsedate(entry.modified),
+                    is_dir=False,
+                )
+            else:
+                # No exact byte count or no date in the listing (the stdlib `<ul>` index, a "1.2K"
+                # column): the entry is a file, but a size or time would be made up, so the hint is
+                # type-only and `stat()` asks the server.
+                stat = FileStat(is_dir=False, size_known=False)
+            yield name, stat
+
+    def _is_dir(self, resp: HTTPResponse):
+        # Judged on the FINAL response, after redirects (http->https, CDN and "latest"
+        # download links redirect to files), by the path of that URL: a query or fragment
+        # does not make a directory a file.
+        return _urlparse.urlsplit(resp.url).path.endswith(("/", "/..", "/."))
+
+    def stat(self, *, follow_symlinks=True, walk_up_last_modified=False):
+        hint = self._pop_stat_hint()
+        if hint is not None:
+            return hint
+        return self._head_stat(walk_up_last_modified)[0]
+
+    def _head_stat(self, walk_up_last_modified=False) -> "tuple[FileStat, bool]":
+        """`stat()` without the listing hint, and whether the server stated
+        the size: a file whose HEAD reply has no `Content-Length` reports
+        `st_size` 0, which is not its length."""
+        with translate_errors(self):
+            # The caller's own spelling first: a trailing-slash path the
+            # server answers is a directory, even when the slash-less URL
+            # answers too (wsgidav serves both `/d` and `/d/` with 200).
+            check = (
+                [self, self.with_path(self.path.removesuffix("/"))]
+                if self.path.endswith("/")
+                else [self]
+            )
+            for uri in check:
+                resp = self.backend.request(
+                    "HEAD", uri, allow_redirects=False, headers=_IDENTITY_ENCODING
+                )
+                resp.close()
+                if resp.status == 405:
+                    # Some servers reject HEAD outright; fall back to GET.
+                    resp = self.backend.request(
+                        "GET",
+                        uri,
+                        allow_redirects=False,
+                        stream=True,
+                        headers=_IDENTITY_ENCODING,
+                    )
+                    resp.close()
+                if resp.status < 400:
+                    break
+
+            if is_redirect(resp):
+                # Where the first answer pointed, not the same request again;
+                # `redirect()` knows the origin that `Authorization` is for.
+                moved = resp
+                resp = self.backend.redirect(
+                    "HEAD", uri, moved, headers=_IDENTITY_ENCODING
+                )
+                resp.close()
+                if resp.status == 405:
+                    # The HEAD-405 fallback again after a redirect: a server or proxy that rejects HEAD
+                    # everywhere would otherwise raise PermissionError for an existing path.
+                    resp = self.backend.redirect(
+                        "GET", uri, moved, stream=True, headers=_IDENTITY_ENCODING
+                    )
+                    resp.close()
+            raise_for_status(resp)
+            # From the final URL, once any redirect has been followed.
+            is_dir = self._is_dir(resp)
+
+        # A missing, non-numeric or negative Content-Length is no size.
+        stated = None if is_dir else server_size(resp.headers.get("Content-Length"))
+        st_size = stated or 0
+        lm = resp.headers.get("Last-Modified")
+        if lm is None and walk_up_last_modified:
+            parent = self.parent
+            if self != parent:
+                try:
+                    entry = next(
+                        filter(
+                            lambda p: p.name.removesuffix("/") == self.name,
+                            parent._listdir(),
+                        )
+                    )
+                    if entry and entry.modified:
+                        lm = entry.modified
+                except (StopIteration, OSError):
+                    pass
+
+        size_stated = is_dir or stated is not None
+        stat = FileStat(
+            st_size=st_size, st_mtime=_httpdate.parsedate(lm), is_dir=is_dir
+        )
+        return stat, size_stated
+
+    def _ensure_missing(self) -> None:
+        """Raise `FileExistsError` unless `stat()` says the path is not
+        found. Any other failure of the probe (a 5xx, a 403, a timeout)
+        propagates: it must not read as permission to create."""
+        try:
+            self.stat()
+        except FileNotFoundError:
+            return
+        raise os_error(FileExistsError, self)
+
+    def _open(
+        self,
+        mode="r",
+        buffering=-1,
+    ):
+        if mode == "r":
+            req = self._get(self.as_uri(), headers=_IDENTITY_ENCODING)
+            return _response_reader(self, req, buffering)
+        if mode == "a":
+            return HTTPAppendStream(self)
+        if mode not in ("w", "x"):
+            raise NotImplementedError(f"open(mode={mode!r})")
+        if mode == "x":
+            self._ensure_missing()
+        return HTTPWriteStream(self, exclusive=mode == "x")
+
+    def unlink(self, missing_ok=False):
+        # A server that honours DELETE on a collection (WebDAV, RFC 4918) removes the whole
+        # tree, and unlink() never removes a directory: refuse one. Over http: this uses the
+        # HEAD `stat()` heuristic, which misses some collections (see `dav:` for reliable ones).
+        if self.is_dir():
+            raise IsADirectoryError(
+                _errno.EISDIR, _os.strerror(_errno.EISDIR), str(self)
+            )
+        self._delete(missing_ok=missing_ok)
+
+    def _delete(self, missing_ok=False):
+        # The raw DELETE, with no collection guard: `rmdir()` has already
+        # verified an empty directory before calling it.
+        with translate_errors(self):
+            resp = self.backend.request("DELETE", self, stream=True)
+            if resp.status == 404:
+                drop_body(resp)
+                if missing_ok:
+                    return
+                raise FileNotFoundError(self)
+            _accept(resp)
+
+    def rmdir(self):
+        # An empty directory's listing and a file whose body parses to zero entries look
+        # alike from `_listdir()`: without this check rmdir() of a file would DELETE it
+        # instead of raising NotADirectoryError.
+        if not self.is_dir():
+            raise NotADirectoryError(self)
+        # `_scandir()`, not the raw `_listdir()`: a listing's own "."/".."
+        # rows are not children and must not make an empty directory
+        # look non-empty.
+        for _ in self._scandir():
+            raise OSError(_errno.ENOTEMPTY, "Directory not empty", str(self))
+        self._delete()
+
+    def with_transport(
+        self,
+        transport,
+        *,
+        headers=None,
+        timeout=_settings.DEFAULT_TIMEOUT,
+        write_method: str = "PUT",
+        append_mode: str = "rewrite",
+    ):
+        """This path with a backend that sends through `transport` (any object
+        with a `send()` method as `pathapi.http.HTTPTransport.send()`;
+        the caller closes
+        it). `headers` go with every request; `write_method` (`"PUT"`, or
+        e.g. `"POST"`) uploads on close, and `append_mode` is `"rewrite"` or
+        `"patch"`."""
+        backend = HTTPBackend(
+            transport,
+            headers=headers,
+            timeout=timeout,
+            write_method=write_method,
+            append_mode=append_mode,
+        )
+        return type(self)(self, backend=backend)
+
+    def with_session(
+        self,
+        session,
+        write_method: str = "PUT",
+        append_mode: str = "rewrite",
+        **requests_args,
+    ):
+        """This path with a backend that sends through a `requests.Session`
+        (`session=None` makes a new one). `headers` and `timeout` in
+        `requests_args` are the backend's; the others (`auth`, `verify`,
+        `proxies`, `cert`, ...) go to the `requests` transport of
+        `pathapi-requests`, which must be installed: without it this raises
+        `ImportError` naming it. The caller closes the transport."""
+        from pathapi.http import get_transport
+
+        headers = requests_args.pop("headers", None)
+        timeout = requests_args.pop("timeout", _settings.DEFAULT_TIMEOUT)
+        return self.with_transport(
+            get_transport("requests")(session, **requests_args),
+            headers=headers,
+            timeout=timeout,
+            write_method=write_method,
+            append_mode=append_mode,
+        )

uripath/schemes/_http/_settings.py

@@ -0,0 +1,6 @@
+"""The module-level setting of the HTTP schemes. Code reads it as an attribute
+of this module, so assigning it applies at once."""
+
+from __future__ import annotations
+
+from pathapi.http import DEFAULT_TIMEOUT as DEFAULT_TIMEOUT

uripath/schemes/_http/_streams.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import errno as _errno
+import io as _io
+import typing as _ty
+
+from pathapi import providers as _providers
+from pathapi.http import (
+    HTTPResponse,
+    drop_body,
+    os_error,
+    raise_for_status,
+    translate_errors,
+)
+
+if _ty.TYPE_CHECKING:
+    from ._path import HTTPURIPath
+
+
+def _charset(headers) -> str:
+    """The charset a `Content-Type` names, `utf-8` when it names none."""
+    for part in (headers.get("Content-Type") or "").split(";")[1:]:
+        name, _, value = part.partition("=")
+        if name.strip().lower() == "charset":
+            return value.strip().strip("\"'") or "utf-8"
+    return "utf-8"
+
+
+class _ResponseReader(_io.RawIOBase):
+    """Raw read stream over a streamed response body.
+
+    The transport has decoded a body the server encoded anyway (despite
+    `Accept-Encoding: identity`), and a failure in the middle of the body
+    surfaces as `TimeoutError`/`ConnectionError`/`OSError` naming the path.
+    A decoded chunk larger than the caller's buffer is kept for the next
+    `readinto()`."""
+
+    def __init__(self, path, response: HTTPResponse, chunk_size: int):
+        super().__init__()
+        self._path = path
+        self._response = response
+        self._chunks = response.iter_bytes(chunk_size)
+        self._pending = b""
+        self._offset = 0
+
+    def readable(self):
+        return True
+
+    def _next_chunk(self) -> bytes:
+        path = self._path
+        with translate_errors(path):
+            try:
+                return next(self._chunks, b"")
+            except OSError as error:
+                if type(error) is OSError and error.errno == _errno.EIO:
+                    raise OSError(_errno.EIO, f"{error.strerror} for {path}") from None
+                raise
+
+    def readinto(self, buffer):
+        view = memoryview(buffer).cast("B")
+        if not view:
+            return 0
+        while self._offset >= len(self._pending):
+            chunk = self._next_chunk()
+            if not chunk:
+                return 0
+            self._pending, self._offset = chunk, 0
+        count = min(len(view), len(self._pending) - self._offset)
+        view[:count] = self._pending[self._offset : self._offset + count]
+        self._offset += count
+        return count
+
+    def close(self):
+        if not self.closed:
+            try:
+                self._response.close()
+            finally:
+                super().close()
+
+
+def _response_reader(path, response: HTTPResponse, buffering: int):
+    buffer_size = _io.DEFAULT_BUFFER_SIZE if buffering < 0 else buffering
+    raw = _ResponseReader(path, response, buffer_size or _io.DEFAULT_BUFFER_SIZE)
+    return raw if buffer_size == 0 else _io.BufferedReader(raw, buffer_size)
+
+
+_EXCLUSIVE_CREATE = {"headers": {"If-None-Match": "*"}}
+"""Request keywords that make a `PUT` create the file only if it is absent
+(RFC 9110 13.1.2)."""
+
+
+class _UploadStream(_providers.DiscardOnError, _io.BytesIO):
+    """A write buffer whose `close()` uploads its content through `_upload()`.
+
+    `_ready` is set by a subclass as the last step of its constructor. A
+    stream whose constructor raised stays inert: `IOBase.__del__` still calls
+    `close()` on it, and that must neither upload a half-built buffer nor
+    fail on state the constructor never reached. A `with` block that raised
+    through the stream, or collecting it unclosed, discards it
+    (`DiscardOnError`): nothing is sent."""
+
+    _ready = False
+
+    def __init__(self, path: "HTTPURIPath"):
+        super().__init__()
+        self._path = path
+
+    def _upload(self, data: bytes) -> None:
+        raise NotImplementedError
+
+    def _holds_uncommitted(self):
+        return self._ready
+
+    def discard(self):
+        """Drop the buffer: `close()` sends nothing."""
+        self._ready = False
+        if not self.closed:
+            self.truncate(0)
+
+    def close(self):
+        if self.closed:
+            return
+        try:
+            if self._ready:
+                self._upload(self.getvalue())
+        finally:
+            # Closed even when the upload fails: a later close() (context
+            # manager cleanup, or garbage collection) must not send it again.
+            super().close()
+
+
+class HTTPWriteStream(_UploadStream):
+    def __init__(self, path: "HTTPURIPath", exclusive: bool = False):
+        super().__init__(path)
+        # `exclusive` (mode "x"): the upload carries `If-None-Match: *`, and a
+        # 412 reply means the file appeared since the caller looked.
+        self._exclusive = exclusive
+        self._ready = True
+
+    def _upload(self, data):
+        backend = self._path.backend
+        extra = _EXCLUSIVE_CREATE if self._exclusive else {}
+        with translate_errors(self._path, conflict=FileNotFoundError):
+            resp = backend.request(
+                backend.write_method,
+                self._path.as_uri(),
+                data=data,
+                stream=True,
+                **extra,
+            )
+            try:
+                if self._exclusive and resp.status == 412:
+                    raise os_error(FileExistsError, self._path)
+                raise_for_status(resp)
+            finally:
+                drop_body(resp)
+
+
+class HTTPAppendStream(_UploadStream):
+    def __init__(self, path: "HTTPURIPath"):
+        super().__init__(path)
+        if path.backend.append_mode == "rewrite":
+            try:
+                existing = path.read_bytes()
+            except FileNotFoundError:
+                existing = b""
+            self.write(existing)
+        else:
+            # "patch" mode: stat the resource to get its size as start offset.
+            # Drop a listing-derived hint first: an offset taken from a hint
+            # would overwrite the file wherever the listing was wrong.
+            path._pop_stat_hint()
+            try:
+                stat, size_stated = path._head_stat()
+            except FileNotFoundError:
+                self._start_offset = 0
+                self._existed = False
+            else:
+                if not size_stated:
+                    raise OSError(
+                        _errno.EIO,
+                        f"Cannot append to {path} in patch mode: the HEAD reply "
+                        "has no Content-Length, so the end of the file is unknown",
+                    )
+                self._start_offset = stat.st_size
+                self._existed = True
+        self._ready = True
+
+    def _upload(self, data):
+        backend = self._path.backend
+        with translate_errors(self._path, conflict=FileNotFoundError):
+            if backend.append_mode == "rewrite":
+                resp = backend.request(
+                    backend.write_method, self._path.as_uri(), data=data, stream=True
+                )
+                _accept(resp)
+            else:
+                # "patch" mode: send only new content via Content-Range
+                if not data:
+                    # `bytes N-(N-1)/*` is not a valid range: there is
+                    # nothing to append, only a missing file to create.
+                    if not self._existed:
+                        resp = backend.request(
+                            backend.write_method,
+                            self._path.as_uri(),
+                            data=b"",
+                            stream=True,
+                        )
+                        _accept(resp)
+                    return
+                start = self._start_offset
+                end = start + len(data) - 1
+                headers = {"Content-Range": f"bytes {start}-{end}/*"}
+                resp = backend.request(
+                    "PATCH",
+                    self._path.as_uri(),
+                    data=data,
+                    headers=headers,
+                    stream=True,
+                )
+                _accept(resp)
+
+
+def _accept(resp: HTTPResponse) -> None:
+    """Raise for an error status of `resp`, a reply whose body is not used:
+    the body is dropped (see `drop_body`) whichever way this goes."""
+    try:
+        raise_for_status(resp)
+    finally:
+        drop_body(resp)

uripath/schemes/_httpdate.py

@@ -0,0 +1,48 @@
+"""HTTP and WebDAV date headers as epoch seconds."""
+
+from __future__ import annotations
+
+import calendar as _calendar
+import time as _time
+import typing as _ty
+from email.utils import parsedate_tz as _parsedate_tz
+
+
+def parsedate(date: _ty.Union[str, _time.struct_time, tuple, int, float, None]):
+    """Convert a date to UTC epoch seconds, independent of the host timezone.
+
+    - `str` (RFC 1123, RFC 850 or asctime, as in `Last-Modified`): parsed
+      with `email.utils.parsedate_tz`, and its zone offset applied. A string
+      with no zone is read as UTC -- HTTP dates are GMT by definition.
+    - `int`/`float`: already epoch seconds, returned unchanged.
+    - `struct_time`/tuple: read as UTC (`calendar.timegm`), minus its
+      `tm_gmtoff` (or a `parsedate_tz`-style 10th element) when one is set.
+    - `None` or an unparseable string: `0`. A string naming a date that does
+      not exist ("31 Feb") is not refused: it rolls over to the next month.
+
+    Nothing else is accepted: `bytes`, a `datetime` and a tuple of fewer than
+    six items raise `ValueError` or `TypeError` instead of returning `0`.
+
+    Local time (`time.mktime`) is never involved: it shifted every HTTP/DAV
+    `st_mtime` by the host's UTC offset and overflowed on Windows for dates
+    near the epoch.
+    """
+    # Missing/unparseable dates yield epoch 0, not "now" -- a caller with no
+    # Last-Modified header shouldn't have that read as "just modified" and
+    # poison checksum/sync freshness comparisons.
+    if date is None:
+        return 0
+    if isinstance(date, (int, float)):
+        return date
+    if isinstance(date, str):
+        try:
+            parsed = _parsedate_tz(date)
+            if parsed is None:
+                return 0
+            return _calendar.timegm(parsed[:6]) - (parsed[9] or 0)
+        except (TypeError, ValueError, IndexError, OverflowError):
+            return 0
+    offset = getattr(date, "tm_gmtoff", None)
+    if offset is None and len(date) > 9:
+        offset = date[9]
+    return _calendar.timegm(tuple(date)[:6]) - (offset or 0)