repopath 0.1.0__py3-none-any.whl

repopath/_backend.py

@@ -0,0 +1,219 @@
+"""The backends of a repository path: the contract, and the one that sends the
+requests through `pathapi.http`."""
+
+from __future__ import annotations
+
+import contextlib as _contextlib
+import errno as _errno
+import ipaddress as _ipaddress
+import urllib.parse as _urlparse
+import warnings as _warnings
+
+from pathapi import providers as _providers
+from pathapi.http import (
+    DEFAULT_TIMEOUT,
+    HTTPClient,
+    HTTPResponse,
+    HTTPStatusError,
+    get_transport,
+    peer_text,
+)
+
+from ._version import __version__
+from .exceptions import InsecureTransportWarning, RateLimitError
+
+_UNSET = object()
+"""The `timeout` of a call that did not give one: the backend's own."""
+
+
+def _is_loopback(host: "str | None") -> bool:
+    if not host:
+        return False
+    host = host.lower()
+    if host == "localhost" or host.endswith(".localhost"):
+        return True
+    try:
+        return _ipaddress.ip_address(host).is_loopback
+    except ValueError:
+        return False
+
+
+def _warn_if_plaintext(url: str, authorized: bool) -> None:
+    if not url.lower().startswith("http://"):
+        return
+    if not authorized:
+        return
+    host = _urlparse.urlsplit(url).hostname
+    if _is_loopback(host):
+        return
+    _warnings.warn(
+        f"Sending an Authorization header over plain http to {host}; "
+        "use an https API root",
+        InsecureTransportWarning,
+        stacklevel=3,
+    )
+
+
+def _rate_limit_error(response, path) -> "RateLimitError | None":
+    """The `RateLimitError` for a rate-limited reply, else None. Both hosts
+    signal a primary limit with 403 or 429 and a `(X-)RateLimit-Remaining: 0`
+    header, and a secondary limit with 403 or 429 and a `Retry-After` header;
+    a bare 429 is a limit too. Checked before the generic mapping, which would
+    read a secondary-limit 403 as PermissionError and a 429 as EIO."""
+    if response is None or response.status not in (403, 429):
+        return None
+    headers = response.headers
+    remaining = headers.get("X-RateLimit-Remaining", headers.get("RateLimit-Remaining"))
+    retry_after = headers.get("Retry-After")
+    if response.status == 403 and remaining != "0" and retry_after is None:
+        return None
+    if retry_after is not None:
+        when = f"retry after {peer_text(retry_after, 40)}s"
+    else:
+        reset = headers.get("X-RateLimit-Reset", headers.get("RateLimit-Reset", "?"))
+        when = f"resets at {peer_text(reset, 40)}"
+    return RateLimitError(
+        _errno.EAGAIN,
+        f"API rate limit exceeded (HTTP {response.status}) ({when})",
+        str(path),
+    )
+
+
+@_contextlib.contextmanager
+def translate_repo_errors(path):
+    """Maps what a request raised to what `pathlib` raises for `path`: a
+    rate-limited reply is `RateLimitError`, 404 `FileNotFoundError`, 401 and
+    403 `PermissionError`, any other status `OSError(EIO)`; a timeout or a
+    refused connection keeps its type and names `path`."""
+    # `from None`: a chained cause is printed by every traceback. The status
+    # code or exception type stays in the message.
+    try:
+        yield
+    except HTTPStatusError as e:
+        response = e.response
+        status = response.status
+        reason = peer_text(response.reason or "")
+        limited = _rate_limit_error(response, path)
+        if limited is not None:
+            raise limited from None
+        if status == 404:
+            raise FileNotFoundError(_errno.ENOENT, "No such file", str(path)) from None
+        elif status in (401, 403):
+            raise PermissionError(
+                _errno.EACCES, "Permission denied", str(path)
+            ) from None
+        else:
+            message = f"HTTP Error {status} {reason}".rstrip()
+            raise OSError(_errno.EIO, message, str(path)) from None
+    except TimeoutError as e:
+        raise TimeoutError(f"Timeout for {path} ({type(e).__name__})") from None
+    except ConnectionError as e:
+        raise ConnectionError(
+            f"Connection error for {path} ({type(e).__name__})"
+        ) from None
+
+
+class BaseRepoBackend(object):
+    """Protocol for issuing an authenticated HTTP request against the REST API
+    of a hosting service. Subclass this to plug in custom auth or session
+    handling; `RepoBackend` is the one that sends through `pathapi.http`.
+
+    A backend may carry an `api_base` (the API root, replacing the one derived
+    from the instance of a path) and a `cache` (a dict, to memoize the default
+    branch of a repository); a path uses either only when the backend has
+    one."""
+
+    # Weakly referenceable so a path can record which backends it derived.
+    __slots__ = ("__weakref__",)
+
+    @_providers.notimplemented
+    def request(self, method, url, headers=None, params=None, **kwargs):
+        """Send one request to the API and return its `HTTPResponse`."""
+
+
+class RepoBackend(HTTPClient, BaseRepoBackend):
+    """The transport and the optional bearer `token` sent on every request.
+    Unauthenticated works fine for public repositories, subject to the host's
+    anonymous rate limit (60 requests an hour for github.com). A context
+    manager: leaving the block closes a transport the backend built.
+
+    `transport` is any object with a `send()` method as
+    `pathapi.http.HTTPTransport.send()`; without one the built-in
+    `HTTPTransport` is built on first use. `session=` (a `requests.Session`)
+    and `requests_args` (`verify=`, `proxies=`, ...) select the `requests`
+    transport of `pathapi-requests`; without it they raise `ImportError`.
+    `headers` go with every request, under the request's own. `api_base`, when
+    set, overrides the API root a path derives from its instance entirely
+    (e.g. `https://api.github.com` or `https://{host}/api/v3`) -- the seam a
+    test fake, or a reverse-proxied self-hosted setup, plugs into, since the
+    derived root is always `https`; a request that carries an `Authorization`
+    header to a plain `http://` URL on a host other than loopback warns
+    (`InsecureTransportWarning`). Every request gets
+    `timeout=DEFAULT_TIMEOUT` (`(10, 60)` seconds) unless `timeout` here or in
+    the call says otherwise; `None` waits without limit. A `GET` follows
+    redirects, dropping the token when the host changes."""
+
+    __slots__ = ("token", "api_base", "cache")
+
+    def __init__(
+        self,
+        token: str = None,
+        session=None,
+        api_base: str = None,
+        *,
+        transport=None,
+        headers=None,
+        timeout=DEFAULT_TIMEOUT,
+        **requests_args,
+    ):
+        if session is not None or requests_args:
+            if transport is not None:
+                raise TypeError("give a transport, or a session and its arguments")
+            transport = get_transport("requests")(session, **requests_args)
+        super().__init__(
+            transport,
+            headers=headers,
+            timeout=timeout,
+            user_agent=f"repopath/{__version__}",
+        )
+        self.token = token or None
+        self.api_base = api_base.rstrip("/") if api_base else None
+        # Memoization shared by every path that shares this backend (the
+        # resolved default branch of a repository, see `_hosted.py`).
+        self.cache = {}
+
+    def __repr__(self) -> str:
+        # The token and the headers are left out: they are credentials.
+        return f"{type(self).__name__}(api_base={self.api_base!r})"
+
+    def request(
+        self,
+        method,
+        url,
+        headers=None,
+        params=None,
+        *,
+        data=None,
+        timeout=_UNSET,
+        stream=False,
+    ) -> HTTPResponse:
+        sent = dict(headers or {})
+        authorized = bool(self.token) or any(
+            name.lower() == "authorization" for name in (*self.headers, *sent)
+        )
+        if params:
+            query = _urlparse.urlencode(
+                [(k, v) for k, v in dict(params).items() if v is not None], doseq=True
+            )
+            if query:
+                url = url + ("&" if "?" in url else "?") + query
+        _warn_if_plaintext(url, authorized)
+        return super().request(
+            method,
+            url,
+            data=data,
+            headers=sent,
+            stream=stream,
+            timeout=self.timeout if timeout is _UNSET else timeout,
+            auth=f"Bearer {self.token}" if self.token else None,
+        )

repopath/_github.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+
+import base64 as _base64
+import errno as _errno
+
+from pathapi import FileStat
+from pathapi import providers as _providers
+from pathapi.http import drop_body, raise_for_status, server_size
+
+from ._backend import translate_repo_errors
+from ._hosted import HostedRepoPath
+
+
+class GitHubPath(HostedRepoPath):
+    """A path in a GitHub repository, read-only, through the REST contents API
+    (`GET /repos/{owner}/{repo}/contents/{path}`).
+
+    `GitHubPath("src/mod.py", owner="acme", repo="widgets", ref="v1.2")`; see
+    `HostedRepoPath` for the keywords. The default instance is `github.com`
+    (API at `api.github.com`); any other is GitHub Enterprise (API at
+    `https://{instance}/api/v3`). File bodies are fetched with the `raw` media
+    type (skips base64 and its ~1MB inline-content cap) instead of the default
+    JSON+base64 envelope. A `symlink` or `submodule` tree entry is treated as a
+    plain file. No mtime (would need a separate commits-history call per
+    path). Needs no third-party library."""
+
+    TYPE = "github"
+    INSTANCES = ("github.com", "www.github.com")
+    DEFAULT_INSTANCE = "github.com"
+    __slots__ = ()
+
+    _RAW_ACCEPT = "application/vnd.github.raw+json"
+    # The contents API returns at most this many entries per directory, with
+    # no pagination and no truncation flag.
+    _CONTENTS_LIMIT = 1000
+
+    def _default_api_base(self) -> str:
+        if self.instance.lower() in ("github.com", "www.github.com"):
+            return "https://api.github.com"
+        return f"https://{self._api_authority()}/api/v3"
+
+    @property
+    def _repo_url(self) -> str:
+        owner, repo = self._api_quote(self.repo_owner), self._api_quote(self.repo)
+        return f"{self._api_base}/repos/{owner}/{repo}"
+
+    def _contents_url(self, path: "str | None" = None) -> str:
+        url = f"{self._repo_url}/contents"
+        path = self.repo_path if path is None else path
+        if path:
+            url += f"/{self._api_quote(path, '/')}"
+        return url
+
+    def _params(self) -> dict:
+        ref = self.ref
+        return {"ref": ref} if ref else {}
+
+    def _request(self, headers=None, url=None, params=None):
+        """A streamed `GET` that succeeded: the caller decodes or reads the
+        body, which closes the response."""
+        with translate_repo_errors(self):
+            resp = self._backend.request(
+                "GET",
+                url or self._contents_url(),
+                params=self._params() if params is None else params,
+                headers=headers,
+                stream=True,
+            )
+            # A rate-limited reply (403/429) becomes `RateLimitError` in
+            # `translate_repo_errors`.
+            try:
+                raise_for_status(resp)
+            except BaseException:
+                drop_body(resp)
+                raise
+        return resp
+
+    def _object(self, data: dict) -> dict:
+        """`data` if it describes one entry of a repository (it has a
+        `type`), else `BadReplyError`."""
+        if not isinstance(data.get("type"), str):
+            raise self._bad_reply("an object without a type")
+        return data
+
+    def _entry(self, item) -> dict:
+        if not (
+            isinstance(item, dict)
+            and isinstance(item.get("name"), str)
+            and isinstance(item.get("type"), str)
+        ):
+            raise self._bad_reply("a directory entry without a name and a type")
+        self._size_of(item)
+        return item
+
+    def _head_size(self) -> "int | None":
+        """The size of the file here from a request without a body, or `None`
+        when the reply does not settle it. Asked for the raw media type, the
+        API answers a file with that type and the file's length and a
+        directory with a JSON type; a reply without `X-GitHub-Media-Type` is
+        not the API's own (a proxy, a server without `HEAD`)."""
+        with translate_repo_errors(self):
+            resp = self._backend.request(
+                "HEAD",
+                self._contents_url(),
+                params=self._params(),
+                # A compressed reply would state the compressed length.
+                headers={"Accept": self._RAW_ACCEPT, "Accept-Encoding": "identity"},
+            )
+            if "X-GitHub-Media-Type" not in resp.headers:
+                return None
+            raise_for_status(resp)
+        headers = resp.headers
+        media_type = headers.get("Content-Type", "").partition(";")[0].strip().lower()
+        if (
+            resp.status != 200
+            or not media_type.startswith("application/vnd.github")
+            or "raw" not in media_type
+            or headers.get("Content-Encoding", "identity").strip().lower() != "identity"
+        ):
+            return None
+        return server_size(headers.get("Content-Length"))
+
+    def stat(self, *, follow_symlinks=True):
+        hint = self._pop_stat_hint()
+        if hint is not None:
+            return hint
+        # The root of a repository is never a file.
+        size = self._head_size() if self.repo_path else None
+        if size is not None:
+            return FileStat(st_size=size, is_dir=False)
+        data = self._decode(self._request(), list, dict)
+        if isinstance(data, list):
+            return FileStat(is_dir=True)
+        return FileStat(st_size=self._size_of(self._object(data)), is_dir=False)
+
+    def _entries(self, path: str) -> "list[dict]":
+        """Contents-API-shaped entries of the directory `path`. A listing
+        that hits the contents API's 1,000-entry cap is re-read through the
+        Git Trees API, which has no such cap."""
+        data = self._decode(self._request(url=self._contents_url(path)), list, dict)
+        if isinstance(data, dict):
+            self._object(data)
+            raise NotADirectoryError(_errno.ENOTDIR, "Not a directory", str(self))
+        entries = [self._entry(item) for item in data]
+        if len(entries) >= self._CONTENTS_LIMIT:
+            entries = self._tree_entries(self._tree_sha(path))
+        return entries
+
+    def _tree_sha(self, path: str) -> str:
+        if not path:
+            return self.ref or self._default_branch()
+        parent, _, name = path.rpartition("/")
+        for entry in self._entries(parent):
+            if entry["name"] == name and entry["type"] == "dir":
+                sha = entry.get("sha")
+                if not isinstance(sha, str):
+                    raise self._bad_reply("a directory entry without a sha")
+                return sha
+        raise FileNotFoundError(_errno.ENOENT, "No such file", str(self))
+
+    def _default_branch(self) -> str:
+        cache = self._backend_cache()
+        key = ("github_default_branch", self._repo_url)
+        if cache is not None and key in cache:
+            return cache[key]
+        data = self._decode(self._request(url=self._repo_url, params={}), dict)
+        branch = data.get("default_branch")
+        if not (isinstance(branch, str) and branch):
+            raise self._bad_reply("a repository without a default_branch")
+        if cache is not None:
+            cache[key] = branch
+        return branch
+
+    def _tree_entries(self, sha: str) -> "list[dict]":
+        url = f"{self._repo_url}/git/trees/{self._api_quote(sha, '/')}"
+        data = self._decode(self._request(url=url, params={}), dict)
+        if data.get("truncated"):
+            # Only for trees far past any directory listing (100,000
+            # entries); never return a partial listing as a complete one.
+            raise OSError(_errno.EIO, "Git tree listing truncated", str(self))
+        tree = data.get("tree")
+        if not isinstance(tree, list):
+            raise self._bad_reply("a tree without a tree list")
+        kinds = {"tree": "dir", "blob": "file", "commit": "submodule"}
+        entries = []
+        for item in tree:
+            if not (
+                isinstance(item, dict)
+                and isinstance(item.get("path"), str)
+                and isinstance(item.get("type"), str)
+            ):
+                raise self._bad_reply("a tree entry without a path and a type")
+            entries.append(
+                self._entry(
+                    {
+                        "name": item["path"],
+                        "type": kinds.get(item["type"], item["type"]),
+                        "size": item.get("size"),
+                        "sha": item.get("sha"),
+                    }
+                )
+            )
+        return entries
+
+    def _scandir(self):
+        for entry in self._entries(self.repo_path):
+            # The API's names are not trusted to be one path component.
+            if not _providers.is_safe_child_name(entry["name"]):
+                continue
+            is_dir = entry["type"] == "dir"
+            yield entry["name"], FileStat(
+                st_size=0 if is_dir else self._size_of(entry), is_dir=is_dir
+            )
+
+    def _open(self, mode="r", buffering=-1):
+        self._check_read_mode(mode)
+        resp = self._request(headers={"Accept": self._RAW_ACCEPT})
+        try:
+            return self._read_contents(resp)
+        finally:
+            resp.close()
+
+    def _read_contents(self, resp):
+        content_type = resp.headers.get("Content-Type", "")
+        if content_type.startswith("application/json"):
+            # "raw" is ignored by the API for a directory listing (and for
+            # a symlink/submodule entry, which carries its own JSON shape).
+            data = self._decode(resp, list, dict)
+            if isinstance(data, list):
+                raise IsADirectoryError(_errno.EISDIR, "Is a directory", str(self))
+            content = data.get("content")
+            if data.get("encoding") == "base64" and isinstance(content, str):
+                try:
+                    return self._reader(_base64.b64decode(content))
+                except ValueError:
+                    raise self._bad_reply("content that is not base64") from None
+            raise self._bad_reply("an unsupported content response")
+        media_type = content_type.partition(";")[0].strip().lower()
+        if media_type == "text/html" and "X-GitHub-Media-Type" not in resp.headers:
+            # Every reply of the API names its media type in that header, so
+            # an HTML page without it is a proxy's or a portal's, not a file.
+            raise self._bad_reply("an HTML page that is not the API's answer")
+        return self._reader(resp.read())