monorepoly 0.2.0__py3-none-any.whl

monorepoly/git.py

@@ -0,0 +1,394 @@
+"""Bare clones and history reads through the git CLI."""
+
+import base64
+import dataclasses
+import datetime
+import hashlib
+import io
+import logging
+import os
+import re
+import subprocess
+import tempfile
+import urllib.parse
+from collections.abc import Iterable, Iterator, Sequence
+from pathlib import Path
+from types import TracebackType
+
+from monorepoly.exceptions import (
+    MonorepolyGitError,
+    MonorepolyGitTransientError,
+    MonorepolyValueError,
+)
+
+_LOGGER = logging.getLogger(__name__)
+
+_FIELD = '\x1f'
+_RECORD = '\x1e'
+_LOG_FORMAT = f'{_RECORD}%H{_FIELD}%P{_FIELD}%an{_FIELD}%ae{_FIELD}%aI{_FIELD}%cI{_FIELD}%B{_FIELD}'
+_MERGE_REQUEST_REFS = 'refs/mr'
+# Message lines can start with any byte; only the marker plus a full commit id starts a record.
+_RECORD_START = re.compile(f'{_RECORD}[0-9a-f]{{40,64}}{_FIELD}')
+# Read limits per line and kept text per commit, so one huge or minified file cannot exhaust
+# memory; both are far above what the indexer stores of a diff.
+_MAX_LINE_CHARS = 64 * 1024
+_MAX_PATCH_CHARS = 1_000_000
+# Patch lines `_parse_patch` reads; kept past `_MAX_PATCH_CHARS` so line ranges stay complete.
+_PATCH_STRUCTURE = ('diff --git ', '+++ ', '@@ ')
+_HUNK = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@')
+_TIMEOUT_SECONDS = 600
+# Every run clones into an empty container; the big repositories take 3-5 minutes, sometimes 10+.
+_FETCH_TIMEOUT_SECONDS = 1_200
+# Server errors and connection failures as git and curl word them; 4xx (auth, missing) are not.
+_TRANSIENT = re.compile(
+    r'(?:HTTP|error:) 5\d\d\b|Could not resolve host|Failed to connect|Connection (?:refused|reset'
+    r'|timed out)|early EOF|transfer closed|unexpected disconnect',
+    re.IGNORECASE,
+)
+# These can print the Authorization header into stderr, which ends up in error messages.
+_LEAKY_TRACE_VARIABLES = ('GIT_CURL_VERBOSE', 'GIT_TRACE', 'GIT_TRACE_CURL')
+
+
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class RawCommit:
+    author_email: str
+    author_name: str
+    authored_at: datetime.datetime
+    committed_at: datetime.datetime
+    diff: str
+    line_ranges: dict[str, tuple[tuple[int, int], ...]]
+    message: str
+    parents: tuple[str, ...]
+    sha: str
+
+
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class LogEntry:
+    author_email: str
+    author_name: str
+    authored_at: datetime.datetime
+    subject: str
+
+
+def fetch(*, cache_dir: Path, token: str, url: str) -> Path:
+    """Returns the path of an up-to-date bare clone of `url`."""
+    # Fail closed: credentials in the url would be echoed in git errors and stored in the clone.
+    if '@' in urllib.parse.urlsplit(url).netloc:
+        raise MonorepolyValueError('Git url must not embed credentials!')
+    repo = cache_dir / f'{hashlib.sha256(url.encode()).hexdigest()[:16]}.git'
+    if not repo.exists():
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        # The files backend cannot store refs that differ only in case on case-insensitive
+        # filesystems (Windows, macOS), which fails the fetch of such a repository.
+        _run('init', '--quiet', '--bare', '--ref-format=reftable', str(repo), repo=None)
+    elif _run('rev-parse', '--show-ref-format', repo=repo).strip() == 'files':
+        _LOGGER.info("Migrating '%s' to reftable refs.", repo)
+        _run('refs', 'migrate', '--ref-format=reftable', repo=repo)
+    _run(
+        'fetch',
+        '--quiet',
+        '--prune',
+        '--force',
+        url,
+        '+refs/heads/*:refs/heads/*',
+        # GitLab keeps every merge request's head here, even after a squash merge or once the
+        # source branch is deleted. Matching nothing is fine.
+        f'+refs/merge-requests/*/head:{_MERGE_REQUEST_REFS}/*/head',
+        env=_auth_env(token),
+        repo=repo,
+        timeout=_FETCH_TIMEOUT_SECONDS,
+    )
+    return repo
+
+
+def branches(repo: Path) -> dict[str, str]:
+    """Returns branch name to head sha."""
+    output = _run(
+        'for-each-ref', '--format=%(refname:lstrip=2)%00%(objectname)', 'refs/heads', repo=repo
+    )
+    return dict(line.split('\0') for line in output.splitlines())
+
+
+def ancestors(*, head: str, repo: Path) -> set[str]:
+    """Returns the shas of every commit reachable from `head`, including it."""
+    return set(_run('rev-list', head, repo=repo).split())
+
+
+def merged_branches(*, head: str, repo: Path) -> set[str]:
+    """Returns the branches whose head is reachable from `head`."""
+    output = _run(
+        'for-each-ref', '--format=%(refname:lstrip=2)', f'--merged={head}', 'refs/heads', repo=repo
+    )
+    return set(output.splitlines())
+
+
+def merge_request_heads(repo: Path) -> dict[str, str]:
+    """Returns merge request head ref to sha."""
+    output = _run(
+        'for-each-ref', '--format=%(refname)%00%(objectname)', _MERGE_REQUEST_REFS, repo=repo
+    )
+    return dict(line.split('\0') for line in output.splitlines())
+
+
+def has_commit(*, repo: Path, sha: str) -> bool:
+    try:
+        _run('cat-file', '-e', f'{sha}^{{commit}}', repo=repo)
+    except MonorepolyGitError:
+        return False
+    return True
+
+
+def list_files(*, ref: str, repo: Path) -> list[str]:
+    return _run('ls-tree', '-r', '-z', '--name-only', ref, repo=repo).split('\0')[:-1]
+
+
+def changed_paths(*, base: str, head: str, repo: Path) -> list[str]:
+    output = _run('diff', '--name-only', '-z', '--no-renames', base, head, repo=repo)
+    return output.split('\0')[:-1]
+
+
+def merge_base(*, a: str, b: str, repo: Path) -> str | None:
+    """Returns the best common ancestor, or None for unrelated histories."""
+    try:
+        return _run('merge-base', a, b, repo=repo).strip()
+    except MonorepolyGitError:
+        return None
+
+
+def log_entries(*, limit: int, repo: Path, revs: Sequence[str]) -> list[LogEntry]:
+    """Returns up to `limit` commits reachable from `revs`, newest first."""
+    output = _run(
+        'log',
+        f'--max-count={limit}',
+        f'--format=%an{_FIELD}%ae{_FIELD}%aI{_FIELD}%s',
+        *revs,
+        '--',
+        repo=repo,
+    )
+    entries = []
+    for line in output.splitlines():
+        name, email, at, subject = line.split(_FIELD, 3)
+        entries.append(
+            LogEntry(
+                author_email=email,
+                author_name=name,
+                authored_at=datetime.datetime.fromisoformat(at),
+                subject=subject,
+            )
+        )
+    return entries
+
+
+def iter_commits(
+    *, exclude: Iterable[str], include: Iterable[str], repo: Path
+) -> Iterator[RawCommit]:
+    """Yields commits reachable from `include` but not from `exclude`, with their patches."""
+    revs = [*include, *(f'^{sha}' for sha in exclude)]
+    if not revs:
+        return
+    command = [
+        'git',
+        '-c',
+        'core.quotePath=false',
+        'log',
+        '--patch',
+        '--unified=0',
+        '--no-color',
+        '--no-renames',
+        '--no-ext-diff',
+        # Git for Windows' system attributes textconv *.pdf/*.rtf via temp files named after the
+        # file, which fails on names invalid on Windows ("unable to create temp-file").
+        '--no-textconv',
+        f'--format={_LOG_FORMAT}',
+        # Revisions go through stdin: thousands of merge request heads overflow Windows'
+        # 32767-character command line (WinError 206).
+        '--stdin',
+    ]
+    with tempfile.TemporaryFile() as stderr:
+        process = subprocess.Popen(
+            command, cwd=repo, stderr=stderr, stdin=subprocess.PIPE, stdout=subprocess.PIPE
+        )
+        try:
+            # git reads all of stdin before it writes anything, so this cannot deadlock.
+            process.stdin.write(''.join(f'{rev}\n' for rev in revs).encode())
+            process.stdin.close()
+            # newline='\n' keeps CRLF intact and, unlike newline='', does not split on a lone
+            # '\r' inside file content.
+            lines = io.TextIOWrapper(
+                process.stdout, encoding='utf-8', errors='replace', newline='\n'
+            )
+            record: _Record | None = None
+            line_start = True
+            # Pieces of at most `_MAX_LINE_CHARS`: a minified file is one enormous line.
+            while piece := lines.readline(_MAX_LINE_CHARS):
+                if line_start and _RECORD_START.match(piece):
+                    if record:
+                        yield record.parse()
+                    record = _Record(piece[1:])
+                elif record:
+                    record.add(line_start=line_start, piece=piece)
+                line_start = piece.endswith('\n')
+            if record:
+                yield record.parse()
+        except BaseException:
+            # The consumer abandoned the generator (GeneratorExit) or parsing raised.
+            process.kill()
+            raise
+        finally:
+            process.stdout.close()
+            returncode = process.wait(timeout=_TIMEOUT_SECONDS)
+        if returncode:
+            stderr.seek(0)
+            message = stderr.read().decode(errors='replace').strip()
+            raise MonorepolyGitError(f"`git log` failed in '{repo}': {message}")
+
+
+class BlobReader:
+    """Reads files at revisions through one long-running `git cat-file --batch`."""
+
+    def __init__(self, repo: Path) -> None:
+        self._process = subprocess.Popen(
+            ['git', 'cat-file', '--batch'],
+            cwd=repo,
+            stderr=subprocess.DEVNULL,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+
+    def __enter__(self) -> BlobReader:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self._process.stdin.close()
+        self._process.stdout.close()
+        self._process.wait(timeout=_TIMEOUT_SECONDS)
+
+    def read(self, *, path: str, rev: str) -> str | None:
+        """Returns `rev:path` decoded as UTF-8, or None if it is not a file there."""
+        self._process.stdin.write(f'{rev}:{path}\n'.encode())
+        self._process.stdin.flush()
+        header = self._process.stdout.readline().decode().split()
+        if header[-1:] == ['missing']:
+            return None
+        if len(header) != 3:
+            raise MonorepolyValueError(f"Unexpected cat-file header {header!r} for '{rev}:{path}'!")
+        _, kind, size = header
+        data = self._process.stdout.read(int(size) + 1)[:-1]
+        return data.decode('utf-8', errors='replace') if kind == 'blob' else None
+
+
+class _Record:
+    """One `git log` record: its text up to `_MAX_PATCH_CHARS`, then only patch structure."""
+
+    def __init__(self, first: str) -> None:
+        self._cut = False
+        self._kept = [first]
+        self._size = len(first)
+        self._structure: list[str] = []
+
+    def add(self, *, line_start: bool, piece: str) -> None:
+        if not self._cut and self._size + len(piece) <= _MAX_PATCH_CHARS:
+            self._kept.append(piece)
+            self._size += len(piece)
+            return
+        # Once cut, nothing more is kept, even a later piece that would still fit.
+        self._cut = True
+        if line_start and piece.startswith(_PATCH_STRUCTURE):
+            self._structure.append(piece.removesuffix('\n'))
+
+    def parse(self) -> RawCommit:
+        return _parse_record(structure=self._structure, text=''.join(self._kept))
+
+
+def _parse_record(*, structure: Sequence[str] = (), text: str) -> RawCommit:
+    sha, parents, name, email, authored_at, committed_at, body, patch = text.split(_FIELD, 7)
+    return RawCommit(
+        author_email=email,
+        author_name=name,
+        authored_at=datetime.datetime.fromisoformat(authored_at),
+        committed_at=datetime.datetime.fromisoformat(committed_at),
+        diff=patch.lstrip('\n'),
+        # The kept text may end mid-line, so the structure starts on a line of its own.
+        line_ranges=_parse_patch('\n'.join([patch, *structure])),
+        message=body.rstrip('\n'),
+        parents=tuple(parents.split()),
+        sha=sha,
+    )
+
+
+def _parse_patch(patch: str) -> dict[str, tuple[tuple[int, int], ...]]:
+    """Returns new-side changed line ranges per path; deleted and hunkless files get none."""
+    ranges: dict[str, list[tuple[int, int]]] = {}
+    in_header = False
+    path = None
+    # Not splitlines(): it would also split on form feeds and the like inside content.
+    for line in patch.split('\n'):
+        if line.startswith('diff --git '):
+            # With --no-renames both paths are equal, and unlike `---`/`+++` this line has no
+            # trailing TAB after paths containing spaces.
+            rest = line.removeprefix('diff --git ')
+            path = rest[2 : 2 + (len(rest) - 5) // 2]
+            ranges[path] = []
+            in_header = True
+        elif in_header and line.startswith('+++ /dev/null'):
+            path = None
+        elif match := _HUNK.match(line):
+            in_header = False
+            if path is None:
+                continue
+            start = int(match[1])
+            count = 1 if match[2] is None else int(match[2])
+            # A pure deletion has count 0 and points at the line before it.
+            ranges[path].append((start, start + count - 1) if count else (max(start, 1),) * 2)
+    return {name: tuple(spans) for name, spans in ranges.items()}
+
+
+def _auth_env(token: str) -> dict[str, str]:
+    # The header travels in the environment, never in argv, so errors cannot leak it.
+    credentials = base64.b64encode(f'oauth2:{token}'.encode()).decode()
+    return {
+        'GIT_CONFIG_COUNT': '2',
+        'GIT_CONFIG_KEY_0': 'http.extraHeader',
+        'GIT_CONFIG_VALUE_0': f'Authorization: Basic {credentials}',
+        # An empty value clears the helper list: on a 401, Git Credential Manager would
+        # otherwise open a login window.
+        'GIT_CONFIG_KEY_1': 'credential.helper',
+        'GIT_CONFIG_VALUE_1': '',
+        'GCM_INTERACTIVE': 'never',
+        'GIT_TERMINAL_PROMPT': '0',
+    }
+
+
+def _run(
+    *args: str,
+    env: dict[str, str] | None = None,
+    repo: Path | None,
+    timeout: float = _TIMEOUT_SECONDS,
+) -> str:
+    environment = os.environ | (env or {})
+    if env:
+        for name in _LEAKY_TRACE_VARIABLES:
+            environment.pop(name, None)
+    try:
+        result = subprocess.run(
+            ['git', *args],
+            capture_output=True,
+            check=True,
+            cwd=repo,
+            encoding='utf-8',
+            env=environment,
+            errors='replace',
+            timeout=timeout,
+        )
+    except subprocess.CalledProcessError as err:
+        stderr = err.stderr.strip()
+        error = MonorepolyGitTransientError if _TRANSIENT.search(stderr) else MonorepolyGitError
+        raise error(f"`git {args[0]}` failed in '{repo}': {stderr}") from err
+    except subprocess.TimeoutExpired as err:
+        raise MonorepolyGitError(f"`git {args[0]}` timed out in '{repo}'!") from err
+    return result.stdout

monorepoly/gitlab.py

@@ -0,0 +1,234 @@
+"""Read-only GitLab API access, translated into models."""
+
+import dataclasses
+import datetime
+import logging
+from collections.abc import Iterator, Mapping
+
+import gitlab
+from gitlab.exceptions import GitlabAuthenticationError, GitlabGetError
+
+from monorepoly.exceptions import MonorepolyAccessError, MonorepolyValueError
+from monorepoly.models import ActivityKind, Discussion, Issue, MergeRequest, Project, State
+
+_LOGGER = logging.getLogger(__name__)
+
+_PER_PAGE = 100
+_TIMEOUT_SECONDS = 30
+
+
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class User:
+    emails: frozenset[str]
+    id: int
+
+
+class GitLab:
+    def __init__(self, *, client: gitlab.Gitlab) -> None:
+        self._client = client
+
+    def current_user(self) -> User:
+        """Returns the token's user, raising if GitLab rejects the token."""
+        try:
+            self._client.auth()
+        except GitlabAuthenticationError as err:
+            raise MonorepolyAccessError(
+                f"GitLab at '{self._client.url}' rejected the token!"
+            ) from err
+        attributes = self._client.user.attributes
+        emails = {attributes.get(name) for name in ('email', 'commit_email', 'public_email')}
+        return User(emails=frozenset(email for email in emails if email), id=attributes['id'])
+
+    def group_path(self, group_id: int | str) -> str:
+        """Returns the group's full path, raising if the token cannot see the group."""
+        try:
+            group = self._client.groups.get(group_id)
+        except GitlabGetError as err:
+            raise MonorepolyAccessError(
+                f"GitLab group '{group_id}' is not visible to the token!"
+            ) from err
+        return group.attributes['full_path']
+
+    def group_projects(
+        self,
+        *,
+        group_id: int | str,
+        include_archived: bool = True,
+        min_access_level: int | None = None,
+    ) -> list[Project]:
+        """Returns the group's visible projects, archived too unless `include_archived` is False."""
+        group = self._client.groups.get(group_id, lazy=True)
+        filters: dict[str, object] = {}
+        if not include_archived:
+            filters['archived'] = False
+        if min_access_level is not None:
+            filters['min_access_level'] = min_access_level
+        # Projects shared into the group from elsewhere are not part of this group's index.
+        listing = group.projects.list(
+            include_subgroups=True,
+            iterator=True,
+            per_page=_PER_PAGE,
+            with_shared=False,
+            **filters,
+        )
+        return [_project(item.attributes) for item in listing]
+
+    def project(self, path: str) -> Project:
+        return _project(self._client.projects.get(path).attributes)
+
+    def iter_issues(
+        self, *, project_id: int, updated_after: datetime.datetime | None
+    ) -> Iterator[Issue]:
+        manager = self._client.projects.get(project_id, lazy=True).issues
+        for item in manager.list(**_list_filters(updated_after)):
+            yield _issue(attributes=item.attributes, project_id=project_id)
+
+    def iter_merge_requests(
+        self,
+        *,
+        project_id: int,
+        updated_after: datetime.datetime | None,
+    ) -> Iterator[MergeRequest]:
+        manager = self._client.projects.get(project_id, lazy=True).mergerequests
+        for item in manager.list(**_list_filters(updated_after)):
+            yield _merge_request(attributes=item.attributes, project_id=project_id)
+
+    def iter_discussions(self, parent: Issue | MergeRequest) -> Iterator[Discussion]:
+        """Yields the parent's threads that contain human notes."""
+        project = self._client.projects.get(parent.project_id, lazy=True)
+        match parent:
+            case Issue():
+                kind, manager = ActivityKind.ISSUE, project.issues
+            case MergeRequest():
+                kind, manager = ActivityKind.MERGE_REQUEST, project.mergerequests
+            case _:
+                raise MonorepolyValueError(f'Unexpected discussion parent {parent!r}!')
+        listing = manager.get(parent.iid, lazy=True).discussions.list(
+            iterator=True, per_page=_PER_PAGE
+        )
+        for item in listing:
+            notes = [note for note in item.attributes['notes'] if not note.get('system')]
+            if not notes:
+                continue
+            # Internal notes are visible to Reporter and above only, like confidential issues.
+            internal = any(note.get('internal') or note.get('confidential') for note in notes)
+            yield Discussion(
+                authors=tuple(dict.fromkeys(note['author']['username'] for note in notes)),
+                confidential=getattr(parent, 'confidential', False) or internal,
+                discussion_id=item.attributes['id'],
+                notes=tuple(
+                    f'{note["author"]["username"]}: {note.get("body") or ""}' for note in notes
+                ),
+                parent_iid=parent.iid,
+                parent_kind=kind,
+                parent_title=parent.title,
+                project_id=parent.project_id,
+                state=parent.state,
+                updated_at=_datetime(notes[-1].get('updated_at')),
+                web_url=f'{parent.web_url}#note_{notes[0]["id"]}',
+            )
+
+    def merge_request_commit_shas(self, *, iid: int, project_id: int) -> list[str]:
+        merge_request = self._client.projects.get(project_id, lazy=True).mergerequests.get(
+            iid, lazy=True
+        )
+        return [commit.id for commit in merge_request.commits()]
+
+
+def make_gitlab(*, token: str, url: str) -> GitLab:
+    return GitLab(client=_client(private_token=token, url=url))
+
+
+def make_oauth_gitlab(*, token: str, url: str) -> GitLab:
+    """Returns the API for a user's OAuth access token."""
+    return GitLab(client=_client(oauth_token=token, url=url))
+
+
+def _client(*, url: str, **auth: str) -> gitlab.Gitlab:
+    return gitlab.Gitlab(url, retry_transient_errors=True, timeout=_TIMEOUT_SECONDS, **auth)
+
+
+def _list_filters(updated_after: datetime.datetime | None) -> dict[str, object]:
+    filters = {
+        'iterator': True,
+        'order_by': 'updated_at',
+        'per_page': _PER_PAGE,
+        'scope': 'all',
+        'sort': 'asc',
+    }
+    if updated_after:
+        filters['updated_after'] = updated_after.isoformat()
+    return filters
+
+
+def _project(attributes: Mapping) -> Project:
+    # Fail closed: a missing visibility or feature level counts as members-only.
+    return Project(
+        default_branch=attributes.get('default_branch'),
+        empty_repo=bool(attributes.get('empty_repo')),
+        http_url_to_repo=attributes['http_url_to_repo'],
+        id=attributes['id'],
+        issues_access_level=attributes.get('issues_access_level') or 'private',
+        merge_requests_access_level=attributes.get('merge_requests_access_level') or 'private',
+        path_with_namespace=attributes['path_with_namespace'],
+        repository_access_level=attributes.get('repository_access_level') or 'private',
+        visibility=attributes.get('visibility') or 'private',
+        web_url=attributes['web_url'],
+    )
+
+
+def _issue(*, attributes: Mapping, project_id: int) -> Issue:
+    return Issue(
+        assignee_ids=_ids(attributes.get('assignees')),
+        author_id=(attributes.get('author') or {}).get('id'),
+        closed_at=_datetime(attributes.get('closed_at')),
+        confidential=bool(attributes.get('confidential')),
+        created_at=_datetime(attributes.get('created_at')),
+        description=attributes.get('description') or '',
+        iid=attributes['iid'],
+        labels=tuple(attributes.get('labels') or ()),
+        project_id=project_id,
+        state=_state(attributes['state']),
+        title=attributes['title'],
+        updated_at=_datetime(attributes.get('updated_at')),
+        web_url=attributes['web_url'],
+    )
+
+
+def _merge_request(*, attributes: Mapping, project_id: int) -> MergeRequest:
+    return MergeRequest(
+        assignee_ids=_ids(attributes.get('assignees')),
+        author_id=(attributes.get('author') or {}).get('id'),
+        created_at=_datetime(attributes.get('created_at')),
+        description=attributes.get('description') or '',
+        iid=attributes['iid'],
+        merged_at=_datetime(attributes.get('merged_at')),
+        project_id=project_id,
+        reviewer_ids=_ids(attributes.get('reviewers')),
+        source_branch=attributes['source_branch'],
+        state=_state(attributes['state']),
+        symbol_keys=(),
+        target_branch=attributes['target_branch'],
+        title=attributes['title'],
+        updated_at=_datetime(attributes.get('updated_at')),
+        web_url=attributes['web_url'],
+    )
+
+
+def _ids(users: list[Mapping] | None) -> tuple[int, ...]:
+    return tuple(user['id'] for user in users or ())
+
+
+def _state(value: str) -> State:
+    try:
+        return State(value)
+    except ValueError as err:
+        raise MonorepolyValueError(f"Unknown GitLab state '{value}'!") from err
+
+
+def _datetime(value: object) -> datetime.datetime | None:
+    """Returns the parsed timestamp, or None for anything unparsable."""
+    try:
+        return datetime.datetime.fromisoformat(value)
+    except TypeError, ValueError:
+        return None