ghstack 0.9.6__tar.gz → 0.11.0__tar.gz

{ghstack-0.9.6 → ghstack-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ghstack
-Version: 0.9.6
+Version: 0.11.0
 Summary: Stack diff support for GitHub
 Author: Edward Z. Yang
 Author-email: Edward Z. Yang <ezyang@mit.edu>
@@ -23,7 +23,7 @@ Description-Content-Type: text/markdown
 Conveniently submit stacks of diffs to GitHub as separate pull requests.
 
 ```
-pip3 install ghstack
+uv tool install ghstack
 ```
 
 ghstack is tested with 
@@ -69,13 +69,14 @@ resolve conflicts in *each* PR, not just for the entire stack.)
 **How do I start a new feature?**  Just checkout master on a new
 branch, and start working on a fresh branch.
 
-**WARNING.**  You will NOT be able to merge these commits using the
-normal GitHub UI, as their branch bases won't be master.  Use
-`ghstack land $PR_URL` (or alternatively `ghstack land #PR_NUM`) to land
+**How do I merge my changes?**  Warning: You will NOT be able to merge these 
+commits using the normal GitHub UI, as their branch bases won't be master.  
+Use `ghstack land $PR_URL` (or alternatively `ghstack land #PR_NUM`) to land
 a ghstack'ed pull request.
 
 You can also setup a GitHub action to allow a bot land.
-https://github.com/Chillee/ghstack_land_example/blob/main/.github/workflows/issue_comment.yml is an end-to-end example of how to do this.
+[ghstack_land_example](https://github.com/Chillee/ghstack_land_example/blob/main/.github/workflows/ghstack_land.yml)
+is an end-to-end example of how to do this.
 
 ## Structure of submitted pull requests
 

{ghstack-0.9.6 → ghstack-0.11.0}/README.md

@@ -3,7 +3,7 @@
 Conveniently submit stacks of diffs to GitHub as separate pull requests.
 
 ```
-pip3 install ghstack
+uv tool install ghstack
 ```
 
 ghstack is tested with 
@@ -49,13 +49,14 @@ resolve conflicts in *each* PR, not just for the entire stack.)
 **How do I start a new feature?**  Just checkout master on a new
 branch, and start working on a fresh branch.
 
-**WARNING.**  You will NOT be able to merge these commits using the
-normal GitHub UI, as their branch bases won't be master.  Use
-`ghstack land $PR_URL` (or alternatively `ghstack land #PR_NUM`) to land
+**How do I merge my changes?**  Warning: You will NOT be able to merge these 
+commits using the normal GitHub UI, as their branch bases won't be master.  
+Use `ghstack land $PR_URL` (or alternatively `ghstack land #PR_NUM`) to land
 a ghstack'ed pull request.
 
 You can also setup a GitHub action to allow a bot land.
-https://github.com/Chillee/ghstack_land_example/blob/main/.github/workflows/issue_comment.yml is an end-to-end example of how to do this.
+[ghstack_land_example](https://github.com/Chillee/ghstack_land_example/blob/main/.github/workflows/ghstack_land.yml)
+is an end-to-end example of how to do this.
 
 ## Structure of submitted pull requests
 

{ghstack-0.9.6 → ghstack-0.11.0}/pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
     "flake8<8.0.0,>=7.0.0",
 ]
 name = "ghstack"
-version = "0.9.6"
+version = "0.11.0"
 description = "Stack diff support for GitHub"
 readme = "README.md"
 classifiers = [

{ghstack-0.9.6 → ghstack-0.11.0}/src/ghstack/diff.py

@@ -14,7 +14,7 @@ RE_GH_METADATA = re.compile(
 
 
 RAW_PULL_REQUEST_RESOLVED = (
-    r"(Pull Request resolved|Pull-Request-resolved): "
+    r"(Pull Request resolved|Pull-Request-resolved|Pull-Request): "
     r"https://{github_url}/(?P<owner>[^/]+)/(?P<repo>[^/]+)/pull/(?P<number>[0-9]+)"
 )
 
@@ -93,7 +93,7 @@ class Diff:
     # be the same.
     source_id: str
 
-    # The contents of 'Pull-Request-resolved'.  This is None for
+    # The contents of 'Pull-Request'.  This is None for
     # diffs that haven't been submitted by ghstack.  For BC reasons,
     # this also accepts gh-metadata.
     pull_request_resolved: Optional[PullRequestResolved]

{ghstack-0.9.6 → ghstack-0.11.0}/src/ghstack/github_real.py

@@ -3,12 +3,16 @@
 import json
 import logging
 import re
+import time
 from typing import Any, Dict, Optional, Sequence, Tuple, Union
 
 import requests
 
 import ghstack.github
 
+MAX_RETRIES = 5
+INITIAL_BACKOFF_SECONDS = 60
+
 
 class RealGitHubEndpoint(ghstack.github.GitHubEndpoint):
     """
@@ -155,32 +159,63 @@ class RealGitHubEndpoint(ghstack.github.GitHubEndpoint):
         }
 
         url = self.rest_endpoint.format(github_url=self.github_url) + "/" + path
-        logging.debug("# {} {}".format(method, url))
-        logging.debug("Request body:\n{}".format(json.dumps(kwargs, indent=1)))
 
-        resp: requests.Response = getattr(requests, method)(
-            url,
-            json=kwargs,
-            headers=headers,
-            proxies=self._proxies(),
-            verify=self.verify,
-            cert=self.cert,
-        )
+        backoff_seconds = INITIAL_BACKOFF_SECONDS
+        for attempt in range(0, MAX_RETRIES):
+            logging.debug("# {} {}".format(method, url))
+            logging.debug("Request body:\n{}".format(json.dumps(kwargs, indent=1)))
 
-        logging.debug("Response status: {}".format(resp.status_code))
+            resp: requests.Response = getattr(requests, method)(
+                url,
+                json=kwargs,
+                headers=headers,
+                proxies=self._proxies(),
+                verify=self.verify,
+                cert=self.cert,
+            )
 
-        try:
-            r = resp.json()
-        except ValueError:
-            logging.debug("Response body:\n{}".format(r.text))
-            raise
-        else:
-            pretty_json = json.dumps(r, indent=1)
-            logging.debug("Response JSON:\n{}".format(pretty_json))
+            logging.debug("Response status: {}".format(resp.status_code))
 
-        if resp.status_code == 404:
-            raise ghstack.github.NotFoundError(
-                """\
+            try:
+                r = resp.json()
+            except ValueError:
+                logging.debug("Response body:\n{}".format(r.text))
+                raise
+            else:
+                pretty_json = json.dumps(r, indent=1)
+                logging.debug("Response JSON:\n{}".format(pretty_json))
+
+            # Per Github rate limiting: https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#exceeding-the-rate-limit
+            if resp.status_code in (403, 429):
+                remaining_count = resp.headers.get("x-ratelimit-remaining")
+                reset_time = resp.headers.get("x-ratelimit-reset")
+
+                if remaining_count == "0" and reset_time:
+                    sleep_time = int(reset_time) - int(time.time())
+                    logging.warning(
+                        f"Rate limit exceeded. Sleeping until reset in {sleep_time} seconds."
+                    )
+                    time.sleep(sleep_time)
+                    continue
+                else:
+                    retry_after_seconds = resp.headers.get("retry-after")
+                    if retry_after_seconds:
+                        sleep_time = int(retry_after_seconds)
+                        logging.warning(
+                            f"Secondary rate limit hit. Sleeping for {sleep_time} seconds."
+                        )
+                    else:
+                        sleep_time = backoff_seconds
+                        logging.warning(
+                            f"Secondary rate limit hit. Sleeping for {sleep_time} seconds (exponential backoff)."
+                        )
+                        backoff_seconds *= 2
+                    time.sleep(sleep_time)
+                    continue
+
+            if resp.status_code == 404:
+                raise ghstack.github.NotFoundError(
+                    """\
 GitHub raised a 404 error on the request for
 {url}.
 Usually, this doesn't actually mean the page doesn't exist; instead, it
@@ -190,13 +225,15 @@ https://{github_url}/settings/tokens and DOUBLE CHECK that you checked
 "public_repo" for permissions, and update ~/.ghstackrc with your new
 value.
 """.format(
-                    url=url, github_url=self.github_url
+                        url=url, github_url=self.github_url
+                    )
                 )
-            )
 
-        try:
-            resp.raise_for_status()
-        except requests.HTTPError:
-            raise RuntimeError(pretty_json)
+            try:
+                resp.raise_for_status()
+            except requests.HTTPError:
+                raise RuntimeError(pretty_json)
 
-        return r
+            return r
+
+        raise RuntimeError("Exceeded maximum retries due to GitHub rate limiting")

{ghstack-0.9.6 → ghstack-0.11.0}/src/ghstack/land.py

@@ -152,11 +152,18 @@ to complain to the ghstack authors."""
                 sh.git("cherry-pick", "--abort")
                 raise
 
+        # All good! Push!
+        maybe_force_arg = []
+        if needs_force:
+            maybe_force_arg = ["--force-with-lease"]
+        sh.git(
+            "push", *maybe_force_arg, remote_name, f"HEAD:refs/heads/{default_branch}"
+        )
+
         # Advance base to head to "close" the PR for all PRs.
-        # This has to happen before the push because the push
-        # will trigger a closure, but we want a *merge*.  This should
-        # happen after the cherry-pick, because the cherry-picks can
-        # fail
+        # This happens after the cherry-pick and push, because the cherry-picks
+        # can fail (merge conflict) and the push can also fail (race condition)
+
         # TODO: It might be helpful to advance orig to reflect the true
         # state of upstream at the time we are doing the land, and then
         # directly *merge* head into base, so that the PR accurately
@@ -168,24 +175,24 @@ to complain to the ghstack authors."""
             # TODO: regex here so janky
             base_ref = re.sub(r"/orig$", "/base", orig_ref)
             head_ref = re.sub(r"/orig$", "/head", orig_ref)
-            sh.git("push", remote_name, f"{remote_name}/{head_ref}:{base_ref}")
+            sh.git(
+                "push", remote_name, f"{remote_name}/{head_ref}:refs/heads/{base_ref}"
+            )
             github.notify_merged(pr_resolved)
 
-        # All good! Push!
-        maybe_force_arg = []
-        if needs_force:
-            maybe_force_arg = ["--force-with-lease"]
-        sh.git(
-            "push", *maybe_force_arg, remote_name, f"HEAD:refs/heads/{default_branch}"
-        )
-
         # Delete the branches
         for orig_ref, _ in stack_orig_refs:
             # TODO: regex here so janky
             base_ref = re.sub(r"/orig$", "/base", orig_ref)
             head_ref = re.sub(r"/orig$", "/head", orig_ref)
             try:
-                sh.git("push", remote_name, "--delete", orig_ref, base_ref, head_ref)
+                sh.git("push", remote_name, "--delete", orig_ref, base_ref)
+            except RuntimeError:
+                # Whatever, keep going
+                logging.warning("Failed to delete branch, continuing", exc_info=True)
+            # Try deleting head_ref separately since often after it's merged it doesn't exist anymore
+            try:
+                sh.git("push", remote_name, "--delete", head_ref)
             except RuntimeError:
                 # Whatever, keep going
                 logging.warning("Failed to delete branch, continuing", exc_info=True)

{ghstack-0.9.6 → ghstack-0.11.0}/src/ghstack/submit.py

@@ -15,6 +15,7 @@ import ghstack.github_utils
 import ghstack.gpg_sign
 import ghstack.logs
 import ghstack.shell
+import ghstack.trailers
 from ghstack.types import GhNumber, GitCommitHash, GitHubNumber, GitHubRepositoryId
 
 # Either "base", "head" or "orig"; which of the ghstack generated
@@ -467,12 +468,6 @@ class Submitter:
             raise RuntimeError(
                 "There appears to be no commits to process, based on the revs you passed me."
             )
-        elif commit_count > 8 and not self.force:
-            raise RuntimeError(
-                "Cowardly refusing to handle a stack with more than eight PRs.  "
-                "You are likely to get rate limited by GitHub if you try to create or "
-                "manipulate this many PRs.  You can bypass this throttle using --force"
-            )
 
         # This is not really accurate if you're doing a fancy pattern;
         # if this is a problem file us a bug.
@@ -815,7 +810,7 @@ as it supports bidirectional syncing.  However, there is no way to
 convert a pre-existing PR in the old style to the new format which
 supports bidirectional syncing.  If you would like to blow away the old
 PR and start anew, edit the Summary in the Phabricator diff to delete
-the line 'Pull-Request-resolved' and then run ghexport again.
+the line 'Pull-Request' and then run ghexport again.
 """
             )
 
@@ -832,7 +827,7 @@ the Phabricator diff page?  If so, please continue to use that button
 to export your diff.
 
 If you think this is in error, edit the Summary in the Phabricator diff
-to delete the line 'Pull-Request-resolved' and then run ghexport again.
+to delete the line 'Pull-Request' and then run ghexport again.
 """
                 )
             else:
@@ -963,20 +958,16 @@ to disassociate the commit with the pull request, and then try again.
             commit_msg = self._update_source_id(diff.summary, elab_diff)
         else:
             # Need to insert metadata for the first time
-            # TODO: Probably quicker if we reimplement reinterpret-trailers
-            # in Python
-            commit_msg = self.sh.git(
-                "interpret-trailers",
-                "--trailer",
-                f"ghstack-source-id: {diff.source_id}",
-                *(
-                    ["--trailer", f"ghstack-comment-id: {elab_diff.comment_id}"]
-                    if self.direct
-                    else []
-                ),
-                "--trailer",
-                f"Pull-Request-resolved: {pull_request_resolved.url()}",
-                input=strip_mentions(diff.summary.rstrip()),
+            # Using our Python implementation of interpret-trailers
+            trailers_to_add = [f"ghstack-source-id: {diff.source_id}"]
+
+            if self.direct:
+                trailers_to_add.append(f"ghstack-comment-id: {elab_diff.comment_id}")
+
+            trailers_to_add.append(f"Pull-Request: {pull_request_resolved.url()}")
+
+            commit_msg = ghstack.trailers.interpret_trailers(
+                strip_mentions(diff.summary.rstrip()), trailers_to_add
             )
 
         return DiffMeta(
@@ -1780,7 +1771,7 @@ is closed (likely due to being merged).  Please rebase to upstream and try again
         commit_body = RE_GHSTACK_SOURCE_ID.sub("", commit_body)
         # Comment ID is not necessary; source of truth is orig commit
         commit_body = RE_GHSTACK_COMMENT_ID.sub("", commit_body)
-        # Don't store Pull request resolved in the PR body; it's
+        # Don't store Pull request in the PR body; it's
         # unnecessary
         commit_body = ghstack.diff.re_pull_request_resolved_w_sp(self.github_url).sub(
             "", commit_body

ghstack-0.11.0/src/ghstack/trailers.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+
+import re
+from typing import List, Tuple
+
+# Compile regexes once at module level for better performance
+TRAILER_RE = re.compile(r"^([A-Za-z0-9_-]+)(\s*:\s*)(.*)$")
+CONTINUATION_RE = re.compile(r"^\s+\S.*$")
+
+# Git-generated trailer prefixes
+GIT_GENERATED_PREFIXES = ["Signed-off-by: ", "(cherry picked from commit "]
+
+
+def parse_message(message: str) -> Tuple[str, str, str]:
+    """
+    Parse a Git commit message into subject, body, and trailers.
+
+    According to the Git documentation, trailers are:
+    - A group of one or more lines that is all trailers, or contains at least one
+      Git-generated or user-configured trailer and consists of at least 25% trailers.
+    - The group must be preceded by one or more empty (or whitespace-only) lines.
+    - The group must either be at the end of the message or be the last non-whitespace
+      lines before a line that starts with "---" (the "divider").
+
+    Args:
+        message: The commit message to parse.
+
+    Returns:
+        A tuple containing:
+            - subject: The first line of the message
+            - body: The body of the message (may be empty)
+            - trailers: The trailer block as a raw string (may be empty)
+    """
+    if not message:
+        return "", "", ""
+
+    # Split into lines and get the subject (first line)
+    lines = message.splitlines()
+    subject = lines[0] if lines else ""
+
+    if len(lines) <= 1:
+        return subject, "", ""
+
+    # Remove subject
+    message_lines = lines[1:]
+
+    if not message_lines:
+        return subject, "", ""
+
+    # Find where the trailer block starts
+    trailer_start = find_trailer_block_start(message_lines)
+
+    if trailer_start == -1:
+        # No trailer block found, everything after subject is body
+        body = "\n".join(message_lines).strip()
+        return subject, body, ""
+
+    # Body is everything between subject and trailers (with empty lines trimmed)
+    body = "\n".join(message_lines[:trailer_start]).strip()
+
+    # Keep trailers as a raw string
+    trailers = "\n".join(message_lines[trailer_start:]).strip()
+
+    return subject, body, trailers
+
+
+def find_trailer_block_start(lines: List[str]) -> int:
+    """
+    Find the start index of the trailer block in a list of lines.
+
+    Args:
+        lines: List of message lines (without subject and divider).
+
+    Returns:
+        Index of the first line of the trailer block, or -1 if no trailer block is found.
+    """
+    # Remove trailing empty lines
+    trimmed_lines = list(reversed([line for line in reversed(lines) if line.strip()]))
+
+    if not trimmed_lines:
+        return -1
+
+    # Find the last non-empty block
+    block_indices = [-1] + [i for i, line in enumerate(lines) if not line.strip()]
+
+    # Try blocks from last to first
+    for i in range(len(block_indices) - 1, -1, -1):
+        start_idx = block_indices[i] + 1
+        # If we're at the beginning or checking the whole message
+        if i == 0 or start_idx == 0:
+            # Check if the whole remaining content is a trailer block
+            if is_trailer_block(lines[start_idx:]):
+                return start_idx
+            # No more blocks to check
+            return -1
+
+        # Check if the block after this blank line is a trailer block
+        end_idx = block_indices[i + 1] if i + 1 < len(block_indices) else len(lines)
+        if is_trailer_block(lines[start_idx:end_idx]):
+            return start_idx
+
+    return -1
+
+
+def is_trailer_block(lines: List[str]) -> bool:
+    """
+    Determine if the given lines form a trailer block.
+
+    A block is a trailer block if:
+    1. All lines are trailers, or
+    2. At least one Git-generated trailer exists and at least 25% of lines are trailers
+
+    Args:
+        lines: List of lines to check.
+
+    Returns:
+        True if the lines form a trailer block, False otherwise.
+    """
+    # Filter out empty lines
+    content_lines = [line for line in lines if line.strip()]
+
+    if not content_lines:
+        return False
+
+    trailer_lines = 0
+    non_trailer_lines = 0
+    has_git_generated_trailer = False
+
+    i = 0
+    while i < len(content_lines):
+        line = content_lines[i]
+
+        # Skip continuation lines (they belong to the previous trailer)
+        if CONTINUATION_RE.match(line):
+            i += 1
+            continue
+
+        # Check if it's a git-generated trailer
+        if any(line.startswith(prefix) for prefix in GIT_GENERATED_PREFIXES):
+            has_git_generated_trailer = True
+            trailer_lines += 1
+        elif TRAILER_RE.match(line):
+            # Regular trailer
+            trailer_lines += 1
+        else:
+            # Not a trailer line
+            non_trailer_lines += 1
+
+        i += 1
+
+    # A block is a trailer block if all lines are trailers OR
+    # it has at least one git-generated trailer and >= 25% of lines are trailers
+    return (trailer_lines > 0 and non_trailer_lines == 0) or (
+        has_git_generated_trailer and trailer_lines * 3 >= non_trailer_lines
+    )
+
+
+def interpret_trailers(message: str, trailers_to_add: List[str]) -> str:
+    """
+    Add trailers to a commit message, mimicking git interpret-trailers.
+
+    Args:
+        message: The commit message to add trailers to
+        trailers_to_add: List of trailers to add in the format "Key: Value"
+
+    Returns:
+        The commit message with trailers added
+    """
+    subject, body, existing_trailers = parse_message(message)
+
+    # Create a new list with all trailers (existing + new)
+    all_trailers = []
+    if existing_trailers:
+        all_trailers.append(existing_trailers)
+
+    all_trailers.extend(trailers_to_add)
+
+    # Build the new message
+    new_message = subject
+
+    if body:
+        new_message += "\n\n" + body
+
+    if all_trailers:
+        if body or (not body and existing_trailers):
+            new_message += "\n"
+        if not existing_trailers:
+            new_message += "\n"
+        new_message += "\n" + "\n".join(all_trailers)
+
+    return new_message