ghstack 0.16.0__tar.gz → 0.18.0__tar.gz

{ghstack-0.16.0 → ghstack-0.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ghstack
-Version: 0.16.0
+Version: 0.18.0
 Summary: Stack diff support for GitHub
 Author: Edward Z. Yang
 Author-email: Edward Z. Yang <ezyang@mit.edu>

{ghstack-0.16.0 → ghstack-0.18.0}/pyproject.toml

@@ -16,7 +16,7 @@ dependencies = [
     "flake8<8.0.0,>=7.0.0",
 ]
 name = "ghstack"
-version = "0.16.0"
+version = "0.18.0"
 description = "Stack diff support for GitHub"
 readme = "README.md"
 classifiers = [

{ghstack-0.16.0 → ghstack-0.18.0}/src/ghstack/cli.py

@@ -61,6 +61,8 @@ def cli_context(
             oauth_token=config.github_oauth,
             proxy=config.proxy,
             github_url=config.github_url,
+            max_retries=config.max_retries,
+            initial_backoff_seconds=config.initial_backoff_seconds,
         )
         yield shell, config, github
 

{ghstack-0.16.0 → ghstack-0.18.0}/src/ghstack/config.py

@@ -111,6 +111,11 @@ Config = NamedTuple(
         ("reviewer", Optional[str]),
         # Default labels to add to new pull requests (comma-separated labels)
         ("label", Optional[str]),
+        # Retry/backoff tuning
+        ("max_retries", int),
+        # The initial backoff time to use, in seconds.  We will double this
+        # time for each retry.
+        ("initial_backoff_seconds", int),
         # Command to generate a per-PR update description from diff contents
         ("automsg", Optional[str]),
     ],
@@ -334,6 +339,16 @@ def read_config(
     else:
         automsg = None
 
+    if config.has_option("ghstack", "max_retries"):
+        max_retries = config.getint("ghstack", "max_retries")
+    else:
+        max_retries = 5
+
+    if config.has_option("ghstack", "initial_backoff_seconds"):
+        initial_backoff_seconds = config.getint("ghstack", "initial_backoff_seconds")
+    else:
+        initial_backoff_seconds = 60
+
     if write_back:
         with open(config_path, "w") as f:
             config.write(f)
@@ -351,6 +366,8 @@ def read_config(
         remote_name=remote_name,
         reviewer=reviewer,
         label=label,
+        max_retries=max_retries,
+        initial_backoff_seconds=initial_backoff_seconds,
         automsg=automsg,
     )
     logging.debug(f"conf = {conf}")

{ghstack-0.16.0 → ghstack-0.18.0}/src/ghstack/github_fake.py

@@ -125,13 +125,32 @@ class GitHubState:
         return r
 
     def push_hook(self, refs: Sequence[str]) -> None:
-        # updated_refs = set(refs)
-        # for pr in self.pull_requests:
-        #    # TODO: this assumes only origin repository
-        #    # if pr.headRefName in updated_refs:
-        #    #    pr.headRef =
-        #    pass
-        pass
+        self._refs_dirty = True
+
+    async def detect_merged(self) -> None:
+        """
+        GitHub closes a pull request as merged once its head becomes
+        reachable from its base, which in direct mode can happen through an
+        ordinary push to some other pull request's head branch.  Only scan
+        after something moved; a scan is a git call per open pull request.
+        """
+        if not self._refs_dirty or self.upstream_sh is None:
+            self._refs_dirty = False
+            return
+        self._refs_dirty = False
+        for pr in self.pull_requests.values():
+            if pr.closed:
+                continue
+            reachable = await self.upstream_sh.agit(
+                "merge-base",
+                "--is-ancestor",
+                pr.headRefName,
+                pr.baseRefName,
+                exitcode=True,
+            )
+            if reachable:
+                pr.closed = True
+                pr.merged = True
 
     def notify_merged(self, pr_resolved: ghstack.diff.PullRequestResolved) -> None:
         repo = self.repository(pr_resolved.owner, pr_resolved.repo)
@@ -140,6 +159,7 @@ class GitHubState:
         # TODO: model merged too
 
     def __init__(self, upstream_sh: Optional[ghstack.shell.Shell]) -> None:
+        self._refs_dirty = False
         self.repositories = {}
         self.pull_requests = {}
         self.issue_comments = {}
@@ -275,6 +295,7 @@ class PullRequest(Node):
     # state: PullRequestState
     title: str
     url: str
+    merged: bool = False
     reviewers: List[str] = dataclasses.field(default_factory=list)
     labels: List[str] = dataclasses.field(default_factory=list)
 
@@ -342,6 +363,7 @@ class FakeGitHubEndpoint(ghstack.github.GitHubEndpoint):
         self.state = GitHubState(upstream_sh)
 
     async def graphql(self, query: str, **kwargs: Any) -> Any:
+        await self.state.detect_merged()
         r = await graphql.graphql(
             schema=GITHUB_SCHEMA,
             source=query,
@@ -419,6 +441,7 @@ class FakeGitHubEndpoint(ghstack.github.GitHubEndpoint):
         if "base" in input and input["base"] is not None:
             pr.baseRefName = input["base"]
             pr.baseRef = await repo._make_ref_async(state, pr.baseRefName)
+            state._refs_dirty = True
         if "body" in input and input["body"] is not None:
             pr.body = input["body"]
 
@@ -463,6 +486,7 @@ class FakeGitHubEndpoint(ghstack.github.GitHubEndpoint):
         )
 
     async def arest(self, method: str, path: str, **kwargs: Any) -> Any:
+        await self.state.detect_merged()
         return await self._arest_impl(method, path, **kwargs)
 
     async def _arest_impl(self, method: str, path: str, **kwargs: Any) -> Any:

{ghstack-0.16.0 → ghstack-0.18.0}/src/ghstack/github_real.py

@@ -13,9 +13,6 @@ import aiohttp
 
 import ghstack.github
 
-MAX_RETRIES = 5
-INITIAL_BACKOFF_SECONDS = 60
-
 
 class RealGitHubEndpoint(ghstack.github.GitHubEndpoint):
     """
@@ -61,6 +58,9 @@ class RealGitHubEndpoint(ghstack.github.GitHubEndpoint):
     cert: Optional[Union[str, Tuple[str, str]]]
     _session: Optional[aiohttp.ClientSession]
 
+    max_retries: int
+    initial_backoff_seconds: int
+
     def __init__(
         self,
         oauth_token: Optional[str],
@@ -68,12 +68,16 @@ class RealGitHubEndpoint(ghstack.github.GitHubEndpoint):
         proxy: Optional[str] = None,
         verify: Optional[Union[str, bool]] = None,
         cert: Optional[Union[str, Tuple[str, str]]] = None,
+        max_retries: int = 5,
+        initial_backoff_seconds: int = 60,
     ):
         self.oauth_token = oauth_token
         self.proxy = proxy
         self.github_url = github_url
         self.verify = verify
         self.cert = cert
+        self.max_retries = max_retries
+        self.initial_backoff_seconds = initial_backoff_seconds
         self._rest_request_ids = itertools.count(1)
         self._session = None
 
@@ -212,11 +216,11 @@ class RealGitHubEndpoint(ghstack.github.GitHubEndpoint):
         if aiohttp_ssl is not None:
             request_kwargs["ssl"] = aiohttp_ssl
 
-        backoff_seconds = INITIAL_BACKOFF_SECONDS
+        backoff_seconds = self.initial_backoff_seconds
         request_id = next(self._rest_request_ids)
         log_prefix = f"rest[{request_id}]"
         session = self._get_session()
-        for attempt in range(0, MAX_RETRIES):
+        for attempt in range(0, self.max_retries):
             logging.debug("# %s %s %s", log_prefix, method, url)
             logging.debug(
                 "%s request body:\n%s", log_prefix, json.dumps(kwargs, indent=1)
@@ -224,12 +228,11 @@ class RealGitHubEndpoint(ghstack.github.GitHubEndpoint):
 
             async with getattr(session, method)(url, **request_kwargs) as resp:
                 logging.debug("%s response status: %s", log_prefix, resp.status)
+                resp_text = await resp.text()
                 try:
-                    r = await resp.json()
-                except (aiohttp.ContentTypeError, ValueError):
-                    logging.debug(
-                        "%s response body:\n%s", log_prefix, await resp.text()
-                    )
+                    r = json.loads(resp_text)
+                except ValueError:
+                    logging.debug("%s response body:\n%s", log_prefix, resp_text)
                     raise
                 else:
                     pretty_json = json.dumps(r, indent=1)
@@ -240,32 +243,32 @@ class RealGitHubEndpoint(ghstack.github.GitHubEndpoint):
                 if resp.status in (403, 429):
                     remaining_count = resp.headers.get("x-ratelimit-remaining")
                     reset_time = resp.headers.get("x-ratelimit-reset")
+                    more_attempts = attempt < self.max_retries - 1
 
                     if remaining_count == "0" and reset_time:
-                        sleep_time = int(reset_time) - int(time.time())
-                        logging.warning(
-                            f"Rate limit exceeded. Sleeping until reset in {sleep_time} seconds."
-                        )
-                        await asyncio.sleep(sleep_time)
-                        continue
+                        sleep_time = max(0, int(reset_time) - int(time.time()))
+                        if more_attempts:
+                            logging.warning(
+                                f"Rate limit exceeded. Sleeping until reset in {sleep_time} seconds."
+                            )
+                            await asyncio.sleep(sleep_time)
+                            continue
                     # GitHub doesn't document the content of these messages, but this
                     # seems to be an accurate way to find secondary rate limits.  Any
                     # other reason for 403 or 429 will fall through to the error below.
-                    elif b"rate limit" in resp.content.lower():
+                    elif "rate limit" in resp_text.lower():
                         retry_after_seconds = resp.headers.get("retry-after")
                         if retry_after_seconds:
                             sleep_time = int(retry_after_seconds)
-                            logging.warning(
-                                f"Secondary rate limit hit. Sleeping for {sleep_time} seconds."
-                            )
                         else:
                             sleep_time = backoff_seconds
+                            backoff_seconds *= 2
+                        if more_attempts:
                             logging.warning(
-                                f"Secondary rate limit hit. Sleeping for {sleep_time} seconds (exponential backoff)."
+                                f"Secondary rate limit hit. Sleeping for {sleep_time} seconds."
                             )
-                            backoff_seconds *= 2
-                        await asyncio.sleep(sleep_time)
-                        continue
+                            await asyncio.sleep(max(0, sleep_time))
+                            continue
 
                 if resp.status == 404:
                     raise ghstack.github.NotFoundError(

{ghstack-0.16.0 → ghstack-0.18.0}/src/ghstack/github_utils.py

@@ -32,7 +32,10 @@ async def get_github_repo_name_with_owner(
     # where commits will actually be pushed to
     remote_url = await sh.agit("remote", "get-url", "--push", remote_name)
     while True:
-        match = r"^git@{github_url}:/?([^/]+)/(.+?)(?:\.git)?$".format(
+        # The SSH user can be "git" (e.g. git@github.com:owner/repo.git) or an
+        # arbitrary identity such as the org-scoped deploy user GitHub now hands
+        # out (e.g. org-21003710@github.com:owner/repo.git), so match any user.
+        match = r"^[^@]+@{github_url}:/?([^/]+)/(.+?)(?:\.git)?$".format(
             github_url=github_url
         )
         m = re.match(match, remote_url)
@@ -175,7 +178,9 @@ GitHubPullRequestParams = TypedDict(
 def _normalize_remote_url(remote_url: str) -> str:
     """Convert SSH remote URL to HTTPS format, strip .git suffix."""
     # git@github.com:owner/repo.git -> https://github.com/owner/repo
-    m = re.match(r"^git@([^:]+):/?(.+?)(?:\.git)?$", remote_url)
+    # The SSH user may be "git" or an org-scoped identity such as
+    # org-21003710@github.com:owner/repo.git, so match any user.
+    m = re.match(r"^[^@]+@([^:]+):/?(.+?)(?:\.git)?$", remote_url)
     if m:
         return f"https://{m.group(1)}/{m.group(2)}"
     return re.sub(r"\.git$", "", remote_url)

{ghstack-0.16.0 → ghstack-0.18.0}/src/ghstack/submit.py

@@ -516,8 +516,10 @@ class Submitter:
 
         object.__setattr__(self, "base", default_branch)
 
-        # Check if direct should be used, if the user didn't explicitly
-        # specify an option
+    # ~~~~~~~~~~~~~~~~~~~~~~~~
+    # The main algorithm
+
+    async def _initialize_direct(self, pr_info_cache: Dict[GitHubNumber, Any]) -> None:
         direct = self.direct_opt
         if direct is None:
             direct_r = await self.sh.agit(
@@ -525,12 +527,22 @@ class Submitter:
             )
             assert isinstance(direct_r, bool)
             direct = direct_r
-
+        if self.direct_opt is None and not direct:
+            styles = {
+                re.fullmatch(r"gh/[^/]+/[0-9]+/base", base_ref) is None
+                for pr_info in pr_info_cache.values()
+                if (base_ref := self._pr_ref_name(pr_info, "base")) is not None
+            }
+            if len(styles) > 1:
+                raise RuntimeError(
+                    "Cannot infer ghstack submission style: the stack contains "
+                    "both direct and non-direct pull requests. Pass --direct or "
+                    "--no-direct explicitly."
+                )
+            if styles:
+                direct = styles.pop()
         object.__setattr__(self, "direct", direct)
 
-    # ~~~~~~~~~~~~~~~~~~~~~~~~
-    # The main algorithm
-
     async def run(self) -> List[DiffMeta]:
         timer = _Timer() if _TIMING_ENABLED else None
 
@@ -601,6 +613,7 @@ class Submitter:
         )
 
         pr_info_cache = await self._prefetch_pr_info(commits_to_rebase)
+        await self._initialize_direct(pr_info_cache)
         if not self.no_fetch:
             await self._fetch_foreign_pr_refs(pr_info_cache.values())
 
@@ -1992,6 +2005,25 @@ Current PR description:
         # otherwise GitHub can spuriously think that the user pushed a number
         # of patches as part of the PR, when actually they were just from the
         # new upstream branch.
+        # In direct mode a pull request's base is another pull request's head
+        # branch, so a reorder can leave a pull request's head reachable from
+        # the base GitHub still has on file, and GitHub closes any pull request
+        # in that state as merged.  Park the ones whose base is moving on the
+        # default branch, which no head branch is ever reachable from, until
+        # their real base has been pushed.
+        if self.direct:
+            await _gather_ordered(
+                self.github.arest(
+                    "patch",
+                    "repos/{}/{}/pulls/{}".format(
+                        self.repo_owner, self.repo_name, s.number
+                    ),
+                    base=self.base,
+                )
+                for s in diffs_to_submit
+                if not s.closed and s.base != s.elab_diff.base_ref
+            )
+
         all_push_specs: List[str] = []
 
         for s in reversed(diffs_to_submit):

{ghstack-0.16.0 → ghstack-0.18.0}/src/ghstack/test_prelude.py

@@ -122,9 +122,9 @@ class Context:
     github: ghstack.github.GitHubEndpoint
     upstream_sh: ghstack.shell.Shell
     sh: ghstack.shell.Shell
-    direct: bool
+    direct: Optional[bool]
 
-    def __init__(self, direct: bool) -> None:
+    def __init__(self, direct: Optional[bool]) -> None:
         # Set up a "parent" repository with an empty initial commit that we'll operate on
         upstream_dir = tempfile.mkdtemp()
         self.upstream_sh = ghstack.shell.Shell(cwd=upstream_dir, testing=True)
@@ -154,7 +154,7 @@ class Context:
                 onerror=handle_remove_read_only,
             )
 
-    async def check_global_github_invariants(self, direct: bool) -> None:
+    async def check_global_github_invariants(self, direct: Optional[bool]) -> None:
         r = await self.github.graphql(
             """
           query {
@@ -177,7 +177,7 @@ class Context:
                 continue
             # In direct mode, only head refs may not be reused;
             # base refs can be reused in octopus situations
-            if not direct:
+            if direct is False:
                 assert pr["baseRefName"] not in seen_refs
                 seen_refs.add(pr["baseRefName"])
             assert pr["headRefName"] not in seen_refs
@@ -200,7 +200,7 @@ async def init_test() -> Context:
 
 
 @contextlib.asynccontextmanager
-async def scoped_test(direct: bool) -> AsyncIterator[None]:
+async def scoped_test(direct: Optional[bool]) -> AsyncIterator[None]:
     global CTX
     assert CTX is None
     try:
@@ -224,8 +224,10 @@ async def gh_submit(
     reviewer: Optional[str] = None,
     label: Optional[str] = None,
     automsg: Optional[str] = None,
+    **submit_kwargs: Any,
 ) -> List[ghstack.submit.DiffMeta]:
     self = CTX
+    direct_opt = submit_kwargs.pop("direct_opt", self.direct)
     r = await ghstack.submit.main(
         msg=msg,
         username="ezyang",
@@ -236,7 +238,7 @@ async def gh_submit(
         repo_owner_opt="pytorch",
         repo_name_opt="pytorch",
         short=short,
-        direct_opt=self.direct,
+        direct_opt=direct_opt,
         no_skip=no_skip,
         github_url="github.com",
         remote_name="origin",
@@ -247,6 +249,7 @@ async def gh_submit(
         reviewer=reviewer,
         label=label,
         automsg=automsg,
+        **submit_kwargs,
     )
     await self.check_global_github_invariants(self.direct)
     return r
@@ -451,7 +454,7 @@ async def assert_github_state(expect: str, *, skip: int = 0) -> None:
     assert_expected_inline(await dump_github(), expect, skip=skip + 1)
 
 
-def is_direct() -> bool:
+def is_direct() -> Optional[bool]:
     return CTX.direct