repopath 0.1.0__py3-none-any.whl

repopath/_gitlab.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+
+import errno as _errno
+import itertools as _itertools
+import urllib.parse as _urlparse
+
+from pathapi import FileStat
+from pathapi import providers as _providers
+from pathapi.http import drop_body, raise_for_status, server_size
+
+from ._backend import translate_repo_errors
+from ._hosted import HostedRepoPath
+
+
+class GitLabPath(HostedRepoPath):
+    """A path in a GitLab project's repository, read-only, through the REST
+    API v4 (the project is the URL-encoded `owner/repo`).
+
+    `GitLabPath("src/mod.py", owner="acme", repo="widgets", ref="v1.2")`; see
+    `HostedRepoPath` for the keywords. A project in a subgroup has the group
+    path as its owner: `owner="acme/team", repo="widgets"`. The default
+    instance is `gitlab.com`; a self-hosted one is just another instance (and
+    port), always at `https://{instance}/api/v4`. The tree-listing endpoint
+    doesn't carry file size, so `_scandir()` only pre-seeds a stat hint for
+    `tree` (directory) entries (`size=0` is truthful for a directory, not a
+    placeholder) -- `blob` (file) entries get a real `stat()` lazily instead of
+    guessing a size that could poison a caller trusting the hint. No mtime.
+    Needs no third-party library."""
+
+    TYPE = "gitlab"
+    INSTANCES = ("gitlab.com", "www.gitlab.com")
+    DEFAULT_INSTANCE = "gitlab.com"
+    OWNER_NESTS = True
+    __slots__ = ()
+
+    def _default_api_base(self) -> str:
+        return f"https://{self._api_authority()}/api/v4"
+
+    def _uri_segments(self, names):
+        # GitLab's own `/-/` ends the project path in its web URLs. It is
+        # needed when the project is in a group, and when a name is `-`, which
+        # would otherwise read as the separator.
+        owner = self.repo_owner.split("/")
+        if len(owner) > 1 or "-" in names:
+            return [*owner, self.repo, "-", *names]
+        return [*owner, self.repo, *names]
+
+    @classmethod
+    def _from_uri_segments(cls, parts):
+        # The first `-` after an owner and a repository ends the project path;
+        # without one the first two segments are the owner and the repository.
+        try:
+            index = list(parts).index("-", 2)
+        except ValueError:
+            return super()._from_uri_segments(parts)
+        return "/".join(parts[: index - 1]), parts[index - 1], list(parts[index + 1 :])
+
+    @property
+    def _project_id(self) -> str:
+        return _urlparse.quote(f"{self.repo_owner}/{self.repo}", safe="")
+
+    def _params(self, **extra) -> dict:
+        ref = self.ref
+        if ref:
+            extra["ref"] = ref
+        return extra
+
+    def _file_url(self, path: str, suffix: str = "") -> str:
+        encoded = self._api_quote(path)
+        return f"{self._api_base}/projects/{self._project_id}/repository/files/{encoded}{suffix}"
+
+    def _tree_url(self) -> str:
+        return f"{self._api_base}/projects/{self._project_id}/repository/tree"
+
+    def _request(self, method, url, **kwargs):
+        """A streamed request that succeeded: the caller decodes or reads the
+        body, which closes the response."""
+        with translate_repo_errors(self):
+            resp = self._backend.request(method, url, stream=True, **kwargs)
+            try:
+                raise_for_status(resp)
+            except BaseException:
+                drop_body(resp)
+                raise
+        return resp
+
+    def _resolved_ref(self) -> str:
+        # Unlike the tree endpoint (ref optional), GitLab's repository/files endpoints
+        # (metadata AND raw) answer 400 "ref is missing" without a `ref`: resolve and
+        # cache the project's default branch once per backend.
+        ref = self.ref
+        if ref:
+            return ref
+        # A custom `BaseRepoBackend` may have no cache: no memoization then.
+        cache = self._backend_cache()
+        key = ("gitlab_default_branch", self._api_base, self._project_id)
+        if cache is not None and key in cache:
+            return cache[key]
+        resp = self._request("GET", f"{self._api_base}/projects/{self._project_id}")
+        branch = self._decode(resp, dict).get("default_branch")
+        if not (isinstance(branch, str) and branch):
+            raise self._bad_reply("a project without a default_branch")
+        if cache is not None:
+            cache[key] = branch
+        return branch
+
+    def _file_size(self, path: str) -> "int | None":
+        """The size of the file at `path`, or `None` when there is no such
+        file. `HEAD` on the files endpoint answers with the size in
+        `X-Gitlab-Size` and no body; a reply without it (or a server that
+        refuses `HEAD`) is followed by a `GET` of the file's metadata."""
+        url = self._file_url(path)
+        params = {"ref": self._resolved_ref()}
+        try:
+            with translate_repo_errors(self):
+                resp = self._backend.request("HEAD", url, params=params)
+                if resp.status not in (405, 501):
+                    raise_for_status(resp)
+            size = server_size(resp.headers.get("X-Gitlab-Size"))
+            if size is not None and resp.status < 300:
+                return size
+            meta = self._decode(self._request("GET", url, params=params), dict)
+        except FileNotFoundError:
+            return None
+        if "size" not in meta:
+            raise self._bad_reply("file metadata without a size")
+        return self._size_of(meta)
+
+    def _tree_entries(self, path: str):
+        # The tree endpoint is paginated (at most 100 per page): follow
+        # `X-Next-Page` until the last page.
+        page = 1
+        while True:
+            resp = self._request(
+                "GET",
+                self._tree_url(),
+                params=self._params(path=path, per_page=100, page=page),
+            )
+            for item in self._decode(resp, list):
+                if not (
+                    isinstance(item, dict)
+                    and isinstance(item.get("name"), str)
+                    and isinstance(item.get("type"), str)
+                ):
+                    raise self._bad_reply("a tree entry without a name and a type")
+                yield item
+            next_page = resp.headers.get("X-Next-Page", "")
+            if not next_page.isdigit() or int(next_page) <= page:
+                return
+            page = int(next_page)
+
+    def stat(self, *, follow_symlinks=True):
+        hint = self._pop_stat_hint()
+        if hint is not None:
+            return hint
+        path = self.repo_path
+        if not path:
+            # Ask the server: a missing project, owner-only URI or unknown
+            # ref must not read as an existing directory.
+            self._decode(
+                self._request("GET", self._tree_url(), params=self._params(per_page=1)),
+                list,
+            )
+            return FileStat(is_dir=True)
+        size = self._file_size(path)
+        if size is not None:
+            return FileStat(st_size=size, is_dir=False)
+        # Not a file at this exact path. Git has no empty directories, so a
+        # path whose tree listing has any entry is a directory; an empty
+        # listing (or a 404) means it does not exist.
+        try:
+            resp = self._request(
+                "GET", self._tree_url(), params=self._params(path=path, per_page=1)
+            )
+        except FileNotFoundError:
+            raise FileNotFoundError(_errno.ENOENT, "No such file", str(self)) from None
+        if self._decode(resp, list):
+            return FileStat(is_dir=True)
+        raise FileNotFoundError(_errno.ENOENT, "No such file", str(self))
+
+    def _scandir(self):
+        entries = self._tree_entries(self.repo_path)
+        try:
+            first = next(entries, None)
+        except FileNotFoundError:
+            # The tree endpoint 404s for a blob path as for a missing one.
+            if self.repo_path and self._file_size(self.repo_path) is not None:
+                raise NotADirectoryError(
+                    _errno.ENOTDIR, "Not a directory", str(self)
+                ) from None
+            raise
+        if first is None:
+            # Git has no empty directories, so an empty listing of a
+            # non-root path is a file or a missing path (a server may answer
+            # either with an empty 200): pathlib's iterdir() raises for both.
+            if self.repo_path:
+                self._hint = None
+                if not self.stat().is_dir():
+                    raise NotADirectoryError(
+                        _errno.ENOTDIR, "Not a directory", str(self)
+                    )
+            return
+        for entry in _itertools.chain((first,), entries):
+            # The API's names are not trusted to be one path component.
+            if not _providers.is_safe_child_name(entry["name"]):
+                continue
+            # The tree listing carries no size: a blob is known to be a file
+            # (and a tree a directory), which is all `walk()` and `glob()` ask.
+            # Any other type (a submodule) is left to a `stat()`.
+            if entry["type"] == "tree":
+                hint = FileStat(is_dir=True)
+            elif entry["type"] == "blob":
+                hint = FileStat(is_dir=False, size_known=False)
+            else:
+                hint = None
+            yield entry["name"], hint
+
+    def _open(self, mode="r", buffering=-1):
+        self._check_read_mode(mode)
+        path = self.repo_path
+        if not path:
+            raise IsADirectoryError(_errno.EISDIR, "Is a directory", str(self))
+        try:
+            resp = self._request(
+                "GET",
+                self._file_url(path, "/raw"),
+                params={"ref": self._resolved_ref()},
+            )
+        except FileNotFoundError:
+            # The files endpoint 404s for a tree path as for a missing one.
+            try:
+                is_dir = self.stat().is_dir()
+            except FileNotFoundError:
+                is_dir = False
+            if is_dir:
+                raise IsADirectoryError(
+                    _errno.EISDIR, "Is a directory", str(self)
+                ) from None
+            raise
+        try:
+            return self._reader(resp.read())
+        finally:
+            resp.close()

repopath/_hosted.py

@@ -0,0 +1,284 @@
+"""`HostedRepoPath`, what `GitHubPath` and `GitLabPath` share: a repository
+named by an owner and a name on an instance, at a ref, read through the REST
+API of the service."""
+
+from __future__ import annotations
+
+import errno as _errno
+import typing as _ty
+import urllib.parse as _urlparse
+
+from pathapi import providers as _providers
+from pathapi.http import json_body, peer_text
+
+from ._backend import BaseRepoBackend, RepoBackend
+from ._kinds import RepoPath
+from .exceptions import BadReplyError
+
+if _ty.TYPE_CHECKING:
+    from pathapi import FileStat
+
+#: What a path segment keeps unescaped when the path is spelled as a URI.
+_SAFE_SEGMENT = "!$&'()*+,;=:@"
+
+_FORBIDDEN_IN_INSTANCE = set("/\\?#@ \t\r\n") | {chr(i) for i in range(32)}
+
+
+def checked_instance(text: object) -> str:
+    """`text` if it is a host name, an IP address (an IPv6 one in brackets) or
+    either followed by `:port`; else `TypeError` for a non-string and
+    `ValueError`. Nothing in it can reach another part of a URL."""
+    if not isinstance(text, str):
+        raise TypeError(f"instance= must be a str, not {type(text).__name__!r}")
+    if not text or _FORBIDDEN_IN_INSTANCE & set(text):
+        raise ValueError(f"instance= must be a host and an optional port, not {text!r}")
+    try:
+        parts = _urlparse.urlsplit("//" + text)
+        host, _port = parts.hostname, parts.port
+    except ValueError:
+        host = None
+    if not host:
+        raise ValueError(f"instance= must be a host and an optional port, not {text!r}")
+    return text
+
+
+def _checked_name(path, what: str, value: object, inherited, nested: bool) -> str:
+    if value is None:
+        if inherited is None:
+            raise TypeError(
+                f"{type(path).__name__} needs {what}= (or a segment that is a "
+                f"{type(path).__name__})"
+            )
+        return inherited
+    if not isinstance(value, str):
+        raise TypeError(f"{what}= must be a str, not {type(value).__name__!r}")
+    parts = value.split("/") if nested else [value]
+    if any(part in ("", ".", "..") for part in parts) or (not nested and "/" in value):
+        raise ValueError(
+            f"{what}= must be a name"
+            + (" or a '/' separated group path" if nested else " with no '/'")
+            + f", with no empty or dot segment, not {value!r}"
+        )
+    return value
+
+
+class HostedRepoPath(RepoPath):
+    """A repository on a hosting service that has an owner (a user, an
+    organization or, on GitLab, a group path) and a name, at a ref.
+
+    `Kind(*segments, owner=, repo=, ref=None, instance=None, token=None,
+    backend=None)`: the segments are the path inside the repository.
+    `instance` is the host of the service, with a port if it is not the
+    default one (`github.com`, `ghe.example:8443`, `[fd00::5]`); a service
+    other than the public one is GitHub Enterprise or a self-hosted GitLab,
+    and its API root is derived from the instance. `ref` is a branch, a tag
+    or a commit; without one the service's default branch is read.
+    `token` builds the path's own `RepoBackend` that sends it as a bearer
+    token; `backend=` shares one backend between paths built separately, or
+    plugs in a `BaseRepoBackend` of your own, and giving it with `token=` is a
+    `TypeError`. A path joined from another keeps all of these, and a keyword
+    moves it (`ref=""` back to the default branch). Without a backend or a
+    token, the path builds an anonymous `RepoBackend`.
+
+    The tree is read-only: `open()` in a writing mode is `NotImplementedError`.
+    A token is shown by no `str()`, `repr()` or error, and is not pickled."""
+
+    __slots__ = ()
+
+    #: The instance a path is on when none is given.
+    DEFAULT_INSTANCE: str
+    #: Whether the owner may be a `/` separated group path.
+    OWNER_NESTS = False
+
+    if _ty.TYPE_CHECKING:
+        _backend: BaseRepoBackend
+        _place: "tuple[str, str, str, str | None]"
+
+    def __init__(
+        self,
+        *segments: str | RepoPath,
+        owner: str | None = None,
+        repo: str | None = None,
+        ref: str | None = None,
+        instance: str | None = None,
+        token: str | None = None,
+        backend: BaseRepoBackend | None = None,
+    ):
+        text, donor = self._join(segments)
+        place = self._place_of(owner, repo, ref, instance, donor)
+        if backend is not None and token is not None:
+            raise TypeError("pass either backend= or token=, not both")
+        if backend is None:
+            if token is not None:
+                backend = RepoBackend(token=token)
+            elif donor is not None:
+                backend = donor._backend
+            else:
+                backend = RepoBackend()
+        elif not callable(getattr(backend, "request", None)):
+            raise TypeError("a backend is an object with a request() method")
+        self._initialize(text, place, backend)
+
+    def _place_of(self, owner, repo, ref, instance, donor) -> tuple:
+        """`(instance, owner, repo, ref)` from the keywords, else from the
+        path this one is joined from, else the defaults."""
+        inherited = donor._place if donor is not None else (None, None, None, None)
+        if instance is None:
+            instance = inherited[0] or self.DEFAULT_INSTANCE
+        else:
+            checked_instance(instance)
+        owner = _checked_name(self, "owner", owner, inherited[1], self.OWNER_NESTS)
+        repo = _checked_name(self, "repo", repo, inherited[2], False)
+        if ref is None:
+            ref = inherited[3]
+        elif isinstance(ref, str):
+            ref = ref or None
+        else:
+            raise TypeError(f"ref= must be a str, not {type(ref).__name__!r}")
+        return instance, owner, repo, ref
+
+    def _keywords(self) -> dict:
+        instance, owner, repo, ref = self._place
+        return {"owner": owner, "repo": repo, "ref": ref, "instance": instance}
+
+    def _place_key(self) -> tuple:
+        # A host name is case-insensitive (RFC 3986 3.2.2); the other parts are not.
+        instance, owner, repo, ref = self._place
+        return instance.lower(), owner, repo, ref or ""
+
+    @property
+    def instance(self) -> str:
+        """The host (and port) of the service, as given."""
+        return self._place[0]
+
+    @property
+    def repo_owner(self) -> str:
+        """The user, organization or group path that owns the repository."""
+        return self._place[1]
+
+    @property
+    def repo(self) -> str:
+        """The name of the repository."""
+        return self._place[2]
+
+    @property
+    def ref(self) -> "str | None":
+        """The branch, tag or commit this path reads, or None for the default
+        branch of the repository."""
+        return self._place[3]
+
+    @property
+    def repo_path(self) -> str:
+        """The path inside the repository, with no leading or trailing `/`."""
+        return "/".join(name for name in self._segments if name)
+
+    def __repr__(self) -> str:
+        shown = [
+            repr(self.as_posix()),
+            f"owner={self.repo_owner!r}",
+            f"repo={self.repo!r}",
+        ]
+        if self.ref:
+            shown.append(f"ref={self.ref!r}")
+        if self.instance.lower() != self.DEFAULT_INSTANCE:
+            shown.append(f"instance={self.instance!r}")
+        backend = self._backend
+        if not (isinstance(backend, RepoBackend) and backend.api_base is None):
+            # A backend shows its API root and never a credential.
+            shown.append(f"backend={backend!r}")
+        return f"{type(self).__name__}({', '.join(shown)})"
+
+    def __str__(self) -> str:
+        return self.as_uri()
+
+    def _uri_segments(self, names: "_ty.Sequence[str]") -> "list[str]":
+        """The path segments of `as_uri()`: the owner, the repository and the
+        names, with whatever separator the service's URLs use."""
+        return [self.repo_owner, self.repo, *names]
+
+    @classmethod
+    def _from_uri_segments(
+        cls, parts: "_ty.Sequence[str]"
+    ) -> "tuple[str, str, list[str]] | None":
+        """`(owner, repo, names)` from the non-empty segments of the path of a
+        URI of this kind, or None when they do not hold an owner and a
+        repository. The inverse of `_uri_segments()`."""
+        if len(parts) < 2:
+            return None
+        return parts[0], parts[1], list(parts[2:])
+
+    def as_uri(self) -> str:
+        """The `<kind>://instance/owner/repo/path` URI of this path, with
+        `?ref=` when it has a ref. It names no token."""
+        names = [name for name in self._segments if name]
+        path = "/".join(
+            _urlparse.quote(segment, safe=_SAFE_SEGMENT, errors="surrogateescape")
+            for segment in self._uri_segments(names)
+        )
+        text = f"{self._family().TYPE}://{self.instance}/{path}"
+        if self.ref:
+            text += "?ref=" + _urlparse.quote(
+                self.ref, safe="", errors="surrogateescape"
+            )
+        return text
+
+    @property
+    def _api_base(self) -> str:
+        return getattr(self._backend, "api_base", None) or self._default_api_base()
+
+    @_providers.notimplemented
+    def _default_api_base(self) -> str:
+        """The API root of this instance when the backend names none."""
+
+    def _api_authority(self) -> str:
+        """`host[:port]` of the instance in the case a URL spells it."""
+        return self.instance.lower()
+
+    def _same_filesystem(self, other: "HostedRepoPath") -> bool:
+        # The tree is what the API root, the repository and the ref name; a
+        # token only decides what may be read of it.
+        return (self._api_base, *self._place[1:]) == (
+            other._api_base,
+            *other._place[1:],
+        )
+
+    def _bad_reply(self, what: str) -> BadReplyError:
+        return BadReplyError(_errno.EIO, f"Unexpected API reply: {what}", str(self))
+
+    def _decode(self, resp, *expected: type):
+        """The JSON body of `resp`, which must be one of `expected` (`dict`,
+        `list`): anything else -- a captive portal's page, a gateway's JSON,
+        `null`, a truncated body -- is `BadReplyError` naming this path, so
+        `exists()` answers `False` and `stat()` raises."""
+        try:
+            data = json_body(resp, self)
+        except ValueError:
+            data = None
+        if not isinstance(data, expected):
+            kinds = " or ".join({dict: "object", list: "array"}[k] for k in expected)
+            raise self._bad_reply(f"not a JSON {kinds}")
+        return data
+
+    def _size_of(self, entry: dict) -> int:
+        """The `size` of an API object: a non-negative integer, `0` when it
+        has none."""
+        size = entry.get("size")
+        if size is None:
+            return 0
+        if isinstance(size, int) and not isinstance(size, bool) and size >= 0:
+            return size
+        raise self._bad_reply(f"size {peer_text(repr(size), 60)}")
+
+    def _api_quote(self, text: str, safe: str = "") -> str:
+        """`text` as part of an API URL path: percent-encoded (`safe` stays
+        bare), and never a `.` or `..` segment, which the client would
+        resolve out of the repository's API root."""
+        if any(part in (".", "..") for part in text.split("/")):
+            raise ValueError(f"A dot segment cannot be part of an API URL: {self}")
+        return _urlparse.quote(text, safe=safe, errors="surrogateescape")
+
+    def _backend_cache(self) -> "dict | None":
+        """The backend's memoization dict, or None for a custom
+        `BaseRepoBackend` that has none (memoization is then skipped)."""
+        cache = getattr(self._backend, "cache", None)
+        return cache if isinstance(cache, dict) else None