pr-analyzer-cli 0.1.0__py3-none-any.whl

pr_analyzer/github/ratelimit.py

@@ -0,0 +1,277 @@
+"""Waiting out GitHub's rate limits instead of failing the run.
+
+GitHub refuses a rate-limited request with 403 or 429 and doesn't carry it out, so the request,
+even a POST, can be sent again once the limit allows:
+
+- primary limit (`x-ratelimit-remaining: 0`): wait until `x-ratelimit-reset` (epoch seconds)
+- secondary limit: wait `retry-after` (seconds, or an HTTP date), or a minute when GitHub
+  doesn't say
+
+A wait longer than GITHUB_RATE_LIMIT_MAX_WAIT_SECONDS raises RateLimitExceededException instead,
+which the client reports as RateLimitError, with the time GitHub lets the request through again:
+while the PR is fetched that stops the run (exit code 2), while the review is posted it becomes
+a warning. Waits are logged as warnings, and they can be cancelled: when a parallel fetch fails
+or is interrupted (Ctrl-C), its waits end at once and none of its requests is sent again.
+
+This is a urllib3 Retry of our own rather than PyGithub's GithubRetry because GithubRetry (2.10)
+caps only primary-limit waits: a 403 with Retry-After falls through to urllib3, which sleeps for
+whatever it says (up to six hours, silently). It ignores 429, retries 5xx errors on POST too (a
+502 after GitHub created a review would post the review twice), and logs through the standard
+logging module, which this project doesn't show. The plain urllib3 Retry used before had the
+same silent Retry-After sleep on 429 and 503 responses.
+"""
+from __future__ import annotations
+
+import json
+import math
+import threading
+import time
+from collections.abc import Callable, Mapping
+from datetime import timezone
+from email.utils import parsedate_to_datetime
+from types import TracebackType
+from typing import TYPE_CHECKING, Any, Self
+
+import structlog
+from github import RateLimitExceededException
+from github.Requester import Requester
+from urllib3.exceptions import MaxRetryError
+from urllib3.util.retry import Retry
+
+if TYPE_CHECKING:
+    # annotations only: BaseHTTPResponse is new in urllib3 2, and PyGithub and requests still
+    # allow urllib3 1.26
+    from urllib3.connectionpool import ConnectionPool
+    from urllib3.response import BaseHTTPResponse
+
+log = structlog.get_logger(__name__)
+
+FORBIDDEN = 403
+TOO_MANY_REQUESTS = 429
+RATE_LIMIT_STATUSES = frozenset({FORBIDDEN, TOO_MANY_REQUESTS})
+# GitHub: without a retry-after header, "wait for at least one minute before retrying"
+SECONDARY_LIMIT_WAIT_SECONDS = 60.0
+DEFAULT_MAX_WAIT_SECONDS = 60.0
+# transient server errors, retried on idempotent requests only
+SERVER_ERRORS = (500, 502, 503, 504)
+MAX_RETRIES = 3
+
+
+def lowercase(headers: Mapping[str, str]) -> dict[str, str]:
+    return {key.lower(): value for key, value in headers.items()}
+
+
+def has_limit_headers(headers: Mapping[str, str]) -> bool:
+    return "retry-after" in headers or headers.get("x-ratelimit-remaining") == "0"
+
+
+def number(value: str | None) -> float | None:
+    try:
+        parsed = float(value) if value is not None else None
+    except ValueError:
+        return None
+    return parsed if parsed is not None and math.isfinite(parsed) else None
+
+
+def retry_after(value: str | None, now: float) -> float | None:
+    """A Retry-After header in seconds from `now`: it gives either seconds or an HTTP date."""
+    seconds = number(value)
+    if seconds is None and value:
+        try:
+            date = parsedate_to_datetime(value)
+            if date.tzinfo is None:   # "-0000": UTC, says RFC 5322
+                date = date.replace(tzinfo=timezone.utc)
+            seconds = date.timestamp() - now
+        except (ValueError, OverflowError):
+            return None
+    return max(seconds, 0.0) if seconds is not None else None
+
+
+def rate_limit_wait(headers: Mapping[str, str], now: float) -> float:
+    """Seconds GitHub asks for before a rate-limited request may be sent again."""
+    headers = lowercase(headers)
+    after = retry_after(headers.get("retry-after"), now)
+    if after is not None:
+        return after
+    reset = number(headers.get("x-ratelimit-reset"))
+    if headers.get("x-ratelimit-remaining") == "0" and reset is not None:
+        # +1: the reset time is in whole seconds, and when in that second it comes is unknown
+        return max(reset - now, 0.0) + 1
+    return SECONDARY_LIMIT_WAIT_SECONDS
+
+
+def read_json(response: BaseHTTPResponse) -> Any:
+    # parsed the way PyGithub parses an error body, so the exception raised from it matches
+    text = response.read(decode_content=True).decode("utf-8", errors="replace")
+    if not text:
+        return None
+    try:
+        return json.loads(text)
+    except ValueError:
+        return {"data": text}
+
+
+class RateLimitExceeded(RateLimitExceededException):
+    """A rate limit that isn't waited out, and when GitHub lets the request through again."""
+
+    def __init__(self, status: int, data: Any, headers: dict[str, str], retry_at: float):
+        super().__init__(status, data, headers)
+        self.retry_at = retry_at   # epoch seconds
+
+
+class RequestsCancelled(Exception):
+    """Raised instead of waiting, retrying or sending a request once its fetch is cancelled."""
+
+
+class _Deadline:
+    """When the run's requests may go on, on the monotonic clock: shared by a pause and its
+    scopes."""
+
+    def __init__(self) -> None:
+        self.lock = threading.Lock()
+        self.at = 0.0
+
+
+class RateLimitPause:
+    """The rate-limit waits of one run, shared by all its GitHub clients and threads: while one
+    request waits for a limit to reset, the others hold back instead of adding to it.
+
+    Waits are on an Event rather than time.sleep, so cancel() ends them at once: Ctrl-C reaches
+    only the main thread, and nothing else could stop a worker thread's wait. Deadlines are on
+    the monotonic clock, which a change to the system time doesn't move; `clock` is the wall
+    clock that GitHub's reset times are read against."""
+
+    def __init__(
+        self,
+        clock: Callable[[], float] = time.time,
+        monotonic: Callable[[], float] = time.monotonic,
+        new_event: Callable[[], threading.Event] = threading.Event,
+    ):
+        self.clock = clock
+        self.monotonic = monotonic
+        self.new_event = new_event
+        self.cancelled = new_event()
+        self._resume = _Deadline()
+
+    def scope(self) -> RateLimitPause:
+        """A pause with this one's waits but a cancel() of its own: one per parallel fetch, so
+        that ending the fetch stops its requests, not the client's later ones."""
+        scoped = RateLimitPause(self.clock, self.monotonic, self.new_event)
+        scoped._resume = self._resume
+        return scoped
+
+    def cancel(self) -> None:
+        """End every wait at once; from then on each wait, retry and request raises."""
+        self.cancelled.set()
+
+    def raise_if_cancelled(self) -> None:
+        if self.cancelled.is_set():
+            raise RequestsCancelled("the GitHub requests were cancelled")
+
+    def sleep(self, seconds: float) -> None:
+        """Wait `seconds` without holding back other requests, unless cancel() comes first."""
+        if seconds > 0 and not self.cancelled.is_set():
+            self.cancelled.wait(seconds)
+        self.raise_if_cancelled()
+
+    def wait(self, seconds: float, **context: Any) -> None:
+        with self._resume.lock:
+            now = self.monotonic()
+            self._resume.at = max(self._resume.at, now + seconds)
+            remaining = self._resume.at - now
+        # a warning, so the default log level shows it: the run can stand still for minutes
+        log.warning("github_rate_limit_wait", seconds=round(remaining, 1), **context)
+        self.sleep(remaining)
+
+    def wait_if_paused(self) -> None:
+        with self._resume.lock:
+            remaining = self._resume.at - self.monotonic()
+        self.sleep(remaining)
+
+
+class RateLimitRetry(Retry):
+    """urllib3's retry policy for GitHub requests: transient server errors on idempotent requests,
+    as before, plus rate limits on any request, waited out for up to `max_wait` seconds."""
+
+    def __init__(
+        self,
+        max_wait: float = DEFAULT_MAX_WAIT_SECONDS,
+        pause: RateLimitPause | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.max_wait = max_wait
+        self.pause = pause if pause is not None else RateLimitPause()
+        self.rate_limit_wait: float | None = None   # set on the copy that increment() returns
+
+    def new(self, **kw: Any) -> Self:
+        kw.setdefault("max_wait", self.max_wait)
+        kw.setdefault("pause", self.pause)
+        return super().new(**kw)
+
+    def is_retry(self, method: str, status_code: int, has_retry_after: bool = False) -> bool:
+        if status_code in RATE_LIMIT_STATUSES:
+            return True   # increment() tells a rate limit from a refusal
+        return super().is_retry(method, status_code, has_retry_after)
+
+    def increment(
+        self,
+        method: str | None = None,
+        url: str | None = None,
+        response: BaseHTTPResponse | None = None,
+        error: Exception | None = None,
+        _pool: ConnectionPool | None = None,
+        _stacktrace: TracebackType | None = None,
+    ) -> Self:
+        self.pause.raise_if_cancelled()   # a cancelled fetch sends nothing again, whatever failed
+        if response is None or response.status not in RATE_LIMIT_STATUSES:
+            return super().increment(method, url, response, error, _pool, _stacktrace)
+
+        headers = lowercase(response.headers)
+        data = None
+        if response.status == FORBIDDEN and not has_limit_headers(headers):
+            # without the headers, only the message tells a secondary limit from "forbidden".
+            # Reading it uses up the body, so a refusal is raised here, as PyGithub would raise it
+            data = read_json(response)
+            message = data.get("message") if isinstance(data, dict) else None
+            if not Requester.isRateLimitError(message or ""):
+                raise Requester.createException(response.status, headers, data)
+
+        now = self.pause.clock()
+        wait = rate_limit_wait(headers, now)
+        if wait > self.max_wait:
+            raise RateLimitExceeded(response.status, data, headers, retry_at=now + wait)
+        try:
+            retry = super().increment(method, url, response, error, _pool, _stacktrace)
+        except MaxRetryError:
+            raise RateLimitExceeded(response.status, data, headers, retry_at=now + wait) from None
+        retry.rate_limit_wait = wait
+        return retry
+
+    def sleep(self, response: BaseHTTPResponse | None = None) -> None:
+        if self.rate_limit_wait is None:
+            # the back-off before retrying a server error or a lost connection (github_retry has
+            # urllib3 ignore their Retry-After), which cancel() cuts short too
+            self.pause.sleep(self.get_backoff_time())
+            return
+        last = self.history[-1] if self.history else None
+        self.pause.wait(
+            self.rate_limit_wait,
+            status=last.status if last else None,
+            url=last.url if last else None,
+            attempt=len(self.history),
+        )
+
+
+def github_retry(max_wait: float, pause: RateLimitPause | None = None) -> RateLimitRetry:
+    return RateLimitRetry(
+        max_wait=max_wait,
+        pause=pause,
+        total=MAX_RETRIES,
+        backoff_factor=1,
+        status_forcelist=SERVER_ERRORS,
+        raise_on_status=False,
+        # urllib3 would otherwise sleep out a Retry-After on a 429 or 503 itself: for up to six
+        # hours, without a word, and past GITHUB_RATE_LIMIT_MAX_WAIT_SECONDS
+        respect_retry_after_header=False,
+    )

pr_analyzer/llm/chunking.py

@@ -0,0 +1,149 @@
+"""Splitting a PR's diff into LLM requests of at most MAX_DIFF_LINES diff lines each.
+
+A PR whose diff fits is one request, with exactly the prompt it always had. A bigger one becomes
+several requests, each holding whole files in PR order. A file too big for one request is cut at
+hunk boundaries, and a hunk too big for one request (a large new file is a single hunk) at line
+boundaries, under a hunk header that keeps the line numbers right. A new 5,000-line file with
+MAX_DIFF_LINES=3000 goes out in two parts:
+
+    @@ -0,0 +1,5000 @@        lines 1-2998 (the file's name and this header use up two lines)
+    @@ -0 +2999 @@            lines 2999-5000
+
+Each request's prompt is built by build_prompt from a copy of the PR that holds only that
+request's files, so every request is laid out, and guarded, like a single one.
+"""
+import math
+import re
+from collections import Counter
+from dataclasses import dataclass, replace
+
+from pr_analyzer.llm.prompts import build_prompt
+from pr_analyzer.models import FileChange, PRSummary, Suggestion
+
+# a hunk header with both start lines: @@ -<old>[,<count>] +<new>[,<count>] @@
+HUNK_RANGES = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@")
+# rough size of a prompt in tokens: about four characters each
+CHARS_PER_TOKEN = 4
+
+
+@dataclass
+class ReviewRequest:
+    prompt: str
+    filenames: list[str]   # the files whose diff, or part of it, the prompt holds
+
+
+def estimate_tokens(text: str) -> int:
+    return math.ceil(len(text) / CHARS_PER_TOKEN)
+
+
+def diff_lines(file: FileChange) -> int:
+    # what build_prompt sends for a file: its "--- name ---" line and every line of its patch
+    return 1 + len((file.patch or "").splitlines())
+
+
+def hunks(lines: list[str]) -> list[list[str]]:
+    """The patch's lines, one list per hunk (anything before the first header is a list too)."""
+    groups: list[list[str]] = []
+    for line in lines:
+        if not groups or HUNK_RANGES.match(line):
+            groups.append([line])
+        else:
+            groups[-1].append(line)
+    return groups
+
+
+def advance(old: int, new: int, lines: list[str]) -> tuple[int, int]:
+    """Line numbers in the old and new file after `lines`, counted the way annotate_patch does."""
+    for line in lines:
+        if line.startswith("-"):
+            old += 1
+        elif line.startswith("+"):
+            new += 1
+        elif not line.startswith("\\"):   # "" is no line at all
+            old += 1
+            new += 1
+    return old, new
+
+
+def cut_hunk(hunk: list[str], max_lines: int) -> list[list[str]]:
+    """Cut one hunk into parts of at most `max_lines` lines. Every part after the first starts
+    with a header naming the lines it starts at, so its lines keep their numbers."""
+    ranges = HUNK_RANGES.match(hunk[0])
+    if ranges is None:   # text before the first hunk: it has no line numbers to keep
+        return [hunk[i:i + max_lines] for i in range(0, len(hunk), max(max_lines, 1))]
+
+    room = max(max_lines - 1, 1)   # lines that fit beside a part's header
+    old, new = int(ranges.group(1)), int(ranges.group(2))
+    parts = []
+    for start in range(1, len(hunk), room):
+        body = hunk[start:start + room]
+        header = hunk[0] if start == 1 else f"@@ -{old} +{new} @@"
+        parts.append([header, *body])
+        old, new = advance(old, new, body)
+    return parts or [hunk]
+
+
+def split_patch(patch: str, max_lines: int) -> list[str]:
+    """Cut a patch into pieces of at most `max_lines` lines, at hunk boundaries where it can."""
+    pieces: list[list[str]] = []
+    for hunk in hunks(patch.splitlines()):
+        for part in cut_hunk(hunk, max_lines) if len(hunk) > max_lines else [hunk]:
+            if pieces and len(pieces[-1]) + len(part) <= max_lines:
+                pieces[-1].extend(part)
+            else:
+                pieces.append(list(part))
+    return ["\n".join(piece) for piece in pieces]
+
+
+def plan_chunks(files: list[FileChange], max_lines: int) -> list[list[FileChange]]:
+    """Group the files that have a patch into chunks of at most `max_lines` diff lines, in order.
+    A file too big for one chunk is split into copies that each hold one piece of its patch."""
+    chunks: list[list[FileChange]] = []
+    size = 0
+    for file in files:
+        if not file.patch:
+            continue
+        pieces = [file]
+        if diff_lines(file) > max_lines:
+            pieces = [replace(file, patch=p) for p in split_patch(file.patch, max_lines - 1)]
+        for piece in pieces:
+            lines = diff_lines(piece)
+            if chunks and size + lines <= max_lines:
+                chunks[-1].append(piece)
+                size += lines
+            else:
+                chunks.append([piece])
+                size = lines
+    return chunks
+
+
+def review_requests(
+    pr: PRSummary, existing_findings: list[Suggestion], max_lines: int
+) -> list[ReviewRequest]:
+    chunks = plan_chunks(pr.files, max_lines)
+    if len(chunks) <= 1:
+        # a diff that fits is sent as it always was
+        names = [f.filename for f in pr.files if f.patch]
+        return [ReviewRequest(build_prompt(pr, existing_findings, max_lines), names)]
+
+    split = {f.filename for f in pr.files if f.patch and diff_lines(f) > max_lines}
+    requests = []
+    for number, chunk in enumerate(chunks, start=1):
+        names = list(dict.fromkeys(f.filename for f in chunk))
+        # the findings already reported for this chunk's files, so the model doesn't repeat them
+        findings = [s for s in existing_findings if s.filename in names]
+        prompt = build_prompt(replace(pr, files=chunk), findings, max_lines)
+        partly = " (some of them only in part)" if split.intersection(names) else ""
+        note = (
+            f"This is part {number} of {len(chunks)} of the review of this pull request: the diff "
+            f"above covers {len(names)} of its {len(pr.files)} changed files{partly}. The other "
+            f"parts are reviewed in separate requests."
+        )
+        requests.append(ReviewRequest(f"{prompt}\n\n{note}", names))
+    return requests
+
+
+def split_files(requests: list[ReviewRequest]) -> set[str]:
+    """The files reviewed in parts: those whose diff went out in more than one request."""
+    counts = Counter(name for request in requests for name in request.filenames)
+    return {name for name, count in counts.items() if count > 1}