imbi-plugin-github 2.43.0__py3-none-any.whl → 2.44.0__py3-none-any.whl

imbi/plugins/github/README.md

@@ -61,9 +61,12 @@ dispatches from webhook deliveries. A `WebhookRule.handler` is
 | `sync_commits`        | `github#sync_commits`        | `commits`               |
 | `sync_tags`           | `github#sync_tags`           | `tags`                  |
 | `sync_pull_requests`  | `github#sync_pull_requests`  | `pull_requests`         |
+| `sync_pull_request_reviews` | `github#sync_pull_request_reviews` | `pull_request_reviews` |
 
 `sync_commits` and `sync_tags` are dispatched from `push` deliveries;
-`sync_pull_requests` from `pull_request` deliveries.
+`sync_pull_requests` from `pull_request` deliveries, and
+`sync_pull_request_reviews` from `pull_request_review` deliveries (the
+webhook or GitHub App must subscribe to "Pull request reviews").
 
 `sync_commits` fetches the full set of commits in a push via the compare
 API (paginated, so it isn't capped by the 20-commit inline payload limit);

imbi/plugins/github/plugin.py

@@ -38,6 +38,7 @@ from imbi.plugins.github.identity import DEFAULT_SCOPES, GitHubIdentity
 from imbi.plugins.github.lifecycle import GitHubLifecycle
 from imbi.plugins.github.pull_requests import (
     GitHubPullRequestSync,
+    sync_pull_request_reviews_descriptor,
     sync_pull_requests_descriptor,
 )
 from imbi.plugins.github.secret_provider import GitHubApp
@@ -58,6 +59,7 @@ class GitHubWebhookActions(WebhookActionsCapability):
             sync_commits_descriptor,
             sync_tags_descriptor,
             sync_pull_requests_descriptor,
+            sync_pull_request_reviews_descriptor,
         ]
 
     @classmethod

imbi/plugins/github/pull_requests.py

@@ -22,6 +22,7 @@ gateway's own event recording behaves.
 
 from __future__ import annotations
 
+import asyncio
 import datetime
 import logging
 import typing
@@ -30,17 +31,27 @@ import httpx
 import jsonpointer
 import pydantic
 
-from imbi.common import iggy
+from imbi.common import clickhouse, iggy
 from imbi.common.json_pointer import JsonPointer
-from imbi.common.models import PullRequestRecord
+from imbi.common.models import (
+    PullRequestRecord,
+    PullRequestReviewRecord,
+    PullRequestReviewRequestsRecord,
+)
 from imbi.common.plugins.base import (
     ActionDescriptor,
     PluginContext,
     PullRequestSyncCapability,
 )
+from imbi.common.plugins.errors import PluginRateLimited
 from imbi.plugins.github._app_auth import AppNotInstalledError
 from imbi.plugins.github._hosts import host_to_api_base
-from imbi.plugins.github._repos import resolve_owner_repo
+from imbi.plugins.github._repos import (
+    RepositoryTarget,
+    parse_owner_repo,
+    resolve_owner_repo,
+    resolve_repository_target,
+)
 from imbi.plugins.github.commits import (
     _BACKFILL_MAX_WAIT_SECONDS,  # pyright: ignore[reportPrivateUsage]
     _client,  # pyright: ignore[reportPrivateUsage]
@@ -64,8 +75,61 @@ _SELF_SLUG = 'github-pr-sync'
 _MAX_HISTORY_PAGES = 100
 # Webhook actions handle these PR lifecycle transitions; synchronize
 # fires when new commits are pushed to the PR branch (title/state may
-# also change).
-_SYNC_ACTIONS = frozenset({'opened', 'closed', 'reopened', 'synchronize'})
+# also change). The review-request actions keep ``requested_reviewers``
+# current.
+_SYNC_ACTIONS = frozenset(
+    {
+        'opened',
+        'closed',
+        'reopened',
+        'synchronize',
+        'review_requested',
+        'review_request_removed',
+    }
+)
+# ``pull_request_review`` actions that change a review's state.
+_REVIEW_ACTIONS = frozenset({'submitted', 'edited', 'dismissed'})
+# GitHub's review states, lower-cased.  ``PENDING`` is an unsubmitted
+# draft review only its author can see, so it is not recorded.
+_REVIEW_STATES = frozenset(
+    {'approved', 'changes_requested', 'commented', 'dismissed'}
+)
+# Cap per-PR review pages during backfill (100 reviews per page).
+_MAX_REVIEW_PAGES = 10
+# Open PRs whose reviews the backfill fetches at once.
+_REVIEW_FETCH_CONCURRENCY = 5
+# Stale rows: the project's ``open`` rows on the synced host that a
+# complete listing didn't return.  Only ``open`` rows, because that is
+# the symptom (a PR stuck open forever); closed and merged history
+# stays even when the listing no longer has it, e.g. after the project
+# moves to a new repository.  The ``url`` prefix limits it to the synced
+# host: a project can bind more than one pr-sync Integration.
+# ``cutoff`` spares rows recorded during (or just before) the sync.
+_SELECT_STALE_SQL = (
+    'SELECT pr_id, url FROM pull_requests FINAL '
+    'WHERE project_id = {project_id:String} '
+    "AND state = 'open' "
+    'AND startsWith(url, {url_prefix:String}) '
+    "AND recorded_at < {cutoff:DateTime64(3, 'UTC')}"
+)
+_DELETE_STALE_SQL = (
+    'DELETE FROM pull_requests '
+    'WHERE project_id = {project_id:String} '
+    'AND startsWith(url, {url_prefix:String}) '
+    "AND recorded_at < {cutoff:DateTime64(3, 'UTC')}"
+)
+# ``cutoff`` is typed ``DateTime64(3, 'UTC')``: clickhouse-connect formats
+# a bound datetime in the zone its placeholder type names, so the text and
+# the type agree on any server.  Without the zone the text is server-local
+# wall time, which is ambiguous for an hour when DST falls back.
+# ``recorded_at`` is stamped by whichever host wrote the row (this
+# worker, or the gateway for webhook rows), so back the cutoff off to
+# absorb clock skew and GitHub's listing lag.  Stale rows are old.
+_PRUNE_MARGIN = datetime.timedelta(minutes=5)
+# Each bound array goes to ClickHouse as one HTTP field, capped at
+# 128 KiB by default.  A full 10k-PR listing in one array overruns it,
+# so the listed IDs are split across arrays of at most this many.
+_PRUNE_CHUNK_SIZE = 1000
 
 
 def _parse_pr_datetime(value: object) -> datetime.datetime | None:
@@ -120,17 +184,211 @@ def _pr_record(
     )
 
 
-async def _fetch_all_prs(
+def _review_requests_record(
+    pr: dict[str, typing.Any],
+    *,
+    project_id: str,
+) -> PullRequestReviewRequestsRecord | None:
+    """The PR's pending review requests, or ``None`` without an id.
+
+    Both ``pull_request`` and ``pull_request_review`` payloads carry
+    the PR's current request lists, so either one can write this row.
+    The row is versioned by the PR's ``updated_at``, not the time Imbi
+    wrote it: a backfill builds rows from a listing fetched at the start
+    of a long walk, and must not replace a newer webhook row.
+    """
+    pr_id = pr.get('id')
+    if not pr_id:
+        return None
+    record = PullRequestReviewRequestsRecord(
+        project_id=project_id,
+        pr_id=str(pr_id),
+        requested_reviewers=_logins(pr.get('requested_reviewers'), 'login'),
+        requested_teams=_logins(pr.get('requested_teams'), 'slug'),
+    )
+    updated_at = _parse_pr_datetime(pr.get('updated_at'))
+    if updated_at is not None:
+        record.recorded_at = updated_at
+    return record
+
+
+def _logins(value: object, key: str) -> list[str]:
+    """Pull ``key`` out of each object in a GitHub user/team list."""
+    if not isinstance(value, list):
+        return []
+    out: list[str] = []
+    for item in typing.cast('list[object]', value):
+        if isinstance(item, dict):
+            name = typing.cast('dict[str, typing.Any]', item).get(key)
+            if isinstance(name, str) and name:
+                out.append(name)
+    return out
+
+
+def _review_record(
+    review: dict[str, typing.Any],
+    *,
+    project_id: str,
+    pr_id: str,
+) -> PullRequestReviewRecord | None:
+    """Map a GitHub review object onto a review record.
+
+    Returns ``None`` for pending (unsubmitted) reviews and for reviews
+    missing an id, reviewer, or submission time.
+    """
+    review_id = review.get('id')
+    state = str(review.get('state') or '').lower()
+    user: dict[str, typing.Any] = review.get('user') or {}
+    login = user.get('login')
+    submitted_at = _parse_pr_datetime(review.get('submitted_at'))
+    if (
+        not review_id
+        or state not in _REVIEW_STATES
+        or not isinstance(login, str)
+        or not login
+        or submitted_at is None
+    ):
+        return None
+    return PullRequestReviewRecord(
+        project_id=project_id,
+        pr_id=pr_id,
+        review_id=str(review_id),
+        reviewer=login,
+        state=state,
+        submitted_at=submitted_at,
+    )
+
+
+async def _fetch_reviews(
     client: httpx.AsyncClient,
+    pr_number: int,
     *,
     max_wait: float,
 ) -> list[dict[str, typing.Any]]:
-    """Walk ``GET /pulls?state=all`` and return every PR object.
+    """Walk ``GET /pulls/{number}/reviews`` for one PR.
+
+    A 404 (PR or repo gone) returns what was collected so far.  GitHub
+    lists reviews oldest first, so hitting the page cap drops the
+    newest; that is logged.
+    """
+    out: list[dict[str, typing.Any]] = []
+    params: dict[str, str] = {'per_page': '100'}
+    for _ in range(_MAX_REVIEW_PAGES):
+        resp = await _request(
+            client,
+            'GET',
+            f'/pulls/{pr_number}/reviews',
+            params=params,
+            max_wait=max_wait,
+        )
+        if resp.status_code == 404:
+            return out
+        resp.raise_for_status()
+        page: object = resp.json()
+        if not isinstance(page, list):
+            LOGGER.warning(
+                'github-pr-sync: reviews for PR #%d were not a list; skipping',
+                pr_number,
+            )
+            break
+        out.extend(
+            typing.cast('dict[str, typing.Any]', item)
+            for item in typing.cast('list[object]', page)
+            if isinstance(item, dict)
+        )
+        next_url = _next_page_url(resp.headers.get('link'))
+        next_page = _query_param(next_url, 'page') if next_url else None
+        if not page or next_page is None:
+            break
+        params['page'] = next_page
+    else:
+        LOGGER.warning(
+            'github-pr-sync: reached %d-page cap fetching reviews for PR '
+            '#%d; its newest reviews are missing',
+            _MAX_REVIEW_PAGES,
+            pr_number,
+        )
+    return out
+
+
+async def _open_pr_reviews(
+    client: httpx.AsyncClient,
+    raw_prs: list[dict[str, typing.Any]],
+    *,
+    project_id: str,
+    max_wait: float,
+) -> list[pydantic.BaseModel]:
+    """Fetch review records for the open PRs in ``raw_prs``.
+
+    Only open PRs: the report that shows reviewers lists open PRs, and
+    walking reviews for a repo's whole history would cost one request
+    per PR ever opened.  Best-effort per PR: a failed fetch is logged
+    and that PR's reviews are skipped (its webhooks fill them in).  A
+    rate limit stops the rest of the fetches but keeps what came back.
+    """
+    semaphore = asyncio.Semaphore(_REVIEW_FETCH_CONCURRENCY)
+    limited = asyncio.Event()
+
+    async def one(pr_id: str, number: int) -> list[pydantic.BaseModel]:
+        async with semaphore:
+            if limited.is_set():
+                return []
+            try:
+                reviews = await _fetch_reviews(
+                    client, number, max_wait=max_wait
+                )
+            except PluginRateLimited:
+                if not limited.is_set():
+                    LOGGER.warning(
+                        'github-pr-sync: rate limited fetching reviews for '
+                        'project %s; skipping the rest',
+                        project_id,
+                    )
+                limited.set()
+                return []
+            except httpx.HTTPError, ValueError:
+                # ValueError covers a body that is not valid JSON.
+                LOGGER.exception(
+                    'github-pr-sync: failed to fetch reviews for PR #%d of '
+                    'project %s',
+                    number,
+                    project_id,
+                )
+                return []
+        out: list[pydantic.BaseModel] = []
+        for review in reviews:
+            record = _review_record(review, project_id=project_id, pr_id=pr_id)
+            if record is not None:
+                out.append(record)
+        return out
+
+    jobs = [
+        one(str(pr['id']), pr['number'])
+        for pr in raw_prs
+        if pr.get('state') == 'open'
+        and pr.get('id')
+        and isinstance(pr.get('number'), int)
+    ]
+    results = await asyncio.gather(*jobs)
+    return [record for batch in results for record in batch]
+
+
+async def _fetch_all_prs(
+    client: httpx.AsyncClient,
+    *,
+    max_wait: float,
+) -> tuple[list[dict[str, typing.Any]], bool]:
+    """Walk ``GET /pulls?state=all`` and return ``(prs, complete)``.
 
     GitHub's list-PRs endpoint omits ``additions``, ``deletions``, and
     ``changed_files``; backfill rows store ``0`` for those fields.
     Caps at ``_MAX_HISTORY_PAGES`` pages (10k PRs) so a repo with a very
     large PR history can't pin the worker indefinitely.
+
+    ``complete`` is ``False`` when any page returns 404, a next-page
+    link can't be followed, or the page cap is reached, so the caller
+    knows the listing may be missing PRs.  Only a page with no next
+    link (or an empty page) ends the listing as complete.
     """
     out: list[dict[str, typing.Any]] = []
     params: dict[str, str] = {'state': 'all', 'per_page': '100'}
@@ -152,7 +410,7 @@ async def _fetch_all_prs(
                     'or removed); nothing to sync',
                     str(client.base_url).rstrip('/'),
                 )
-            return out
+            return out, False
         resp.raise_for_status()
         page: list[dict[str, typing.Any]] = resp.json()
         if not page:
@@ -163,7 +421,13 @@ async def _fetch_all_prs(
             break
         next_page = _query_param(next_url, 'page')
         if next_page is None:
-            break
+            LOGGER.warning(
+                'github-pr-sync: next-page link without a page param '
+                '(%s); stopping after %d PRs',
+                next_url,
+                len(out),
+            )
+            return out, False
         params['page'] = next_page
     else:
         LOGGER.warning(
@@ -171,7 +435,102 @@ async def _fetch_all_prs(
             'history may be truncated',
             _MAX_HISTORY_PAGES,
         )
-    return out
+        return out, False
+    return out, True
+
+
+def _is_repository_link(
+    ctx: PluginContext, host: str, target: RepositoryTarget
+) -> bool:
+    """Whether ``target`` came from the project's repository link.
+
+    ``resolve_repository_target`` also accepts any other same-host
+    link, which may name some other repo.  Only the Integration's own
+    link key or ``github-repository`` says which repo this project is.
+    """
+    for key in (ctx.integration_slug, 'github-repository'):
+        url = ctx.project_links.get(key) if key else None
+        if url is None:
+            continue
+        if parse_owner_repo(url, host.lower()) == (target.owner, target.repo):
+            return True
+    return False
+
+
+def _id_arrays(
+    parameters: dict[str, typing.Any], prefix: str, ids: list[str]
+) -> list[str]:
+    """Bind ``ids`` as arrays of at most ``_PRUNE_CHUNK_SIZE``.
+
+    Returns the placeholder for each array, e.g.
+    ``{prefix_0:Array(String)}``.
+    """
+    placeholders: list[str] = []
+    for offset in range(0, len(ids), _PRUNE_CHUNK_SIZE):
+        name = f'{prefix}_{offset // _PRUNE_CHUNK_SIZE}'
+        parameters[name] = ids[offset : offset + _PRUNE_CHUNK_SIZE]
+        placeholders.append(f'{{{name}:Array(String)}}')
+    return placeholders
+
+
+async def _prune_stale_prs(
+    project_id: str,
+    host: str,
+    started_at: datetime.datetime,
+    pr_ids: list[str],
+) -> int:
+    """Delete ``project_id``'s stale ``open`` rows on ``host``.
+
+    A row is stale when its ``pr_id`` is not in ``pr_ids`` (the complete
+    listing) and it was recorded before ``started_at`` less
+    ``_PRUNE_MARGIN``.  The stale rows are selected and logged first,
+    and only those ``pr_id``s are deleted, so the log says exactly what
+    was removed and a sync with nothing stale issues no mutation.
+
+    Returns the number of PRs deleted.  Best-effort: a failure is logged
+    and swallowed so a ClickHouse problem never fails the sync.
+    """
+    parameters: dict[str, typing.Any] = {
+        'project_id': project_id,
+        'url_prefix': f'https://{host.lower()}/',
+        # clickhouse-connect formats a plain ``datetime`` to whole
+        # seconds; the ``_64`` suffix binds it as ``cutoff`` with its
+        # fractional seconds kept.
+        'cutoff_64': started_at - _PRUNE_MARGIN,
+    }
+    select = _SELECT_STALE_SQL + ''.join(
+        f' AND pr_id NOT IN {placeholder}'
+        for placeholder in _id_arrays(parameters, 'listed', pr_ids)
+    )
+    try:
+        rows = await clickhouse.query(select, parameters)
+        if not rows:
+            return 0
+        stale = sorted(str(row['pr_id']) for row in rows)
+        LOGGER.warning(
+            'github-pr-sync: removing %d stale open PR row(s) for project '
+            '%s that %s no longer lists: %s',
+            len(stale),
+            project_id,
+            host,
+            ', '.join(sorted(str(row['url']) for row in rows)),
+        )
+        delete = _DELETE_STALE_SQL + (
+            ' AND ('
+            + ' OR '.join(
+                f'pr_id IN {placeholder}'
+                for placeholder in _id_arrays(parameters, 'stale', stale)
+            )
+            + ')'
+        )
+        await clickhouse.command(delete, parameters)
+    except Exception:
+        LOGGER.exception(
+            'github-pr-sync: failed to remove stale PR rows for project %s',
+            project_id,
+        )
+        return 0
+    return len(stale)
 
 
 class SyncPRsConfig(pydantic.BaseModel):
@@ -212,10 +571,10 @@ async def sync_pull_requests(
 ) -> None:
     """Sync a pull_request event delivery into the ``pull_requests`` table.
 
-    Handles ``opened``, ``closed``, ``reopened``, and ``synchronize``
-    actions.  Other actions (e.g. ``labeled``, ``review_requested``) are
-    silently ignored -- the rule's CEL filter can pre-screen them, but
-    this action is defensive too.
+    Handles ``opened``, ``closed``, ``reopened``, ``synchronize``,
+    ``review_requested``, and ``review_request_removed`` actions.  Other
+    actions (e.g. ``labeled``) are silently ignored -- the rule's CEL
+    filter can pre-screen them, but this action is defensive too.
 
     The full PR object is in the webhook payload so no extra GitHub API
     call is needed.
@@ -245,6 +604,27 @@ async def sync_pull_requests(
             record.pr_number,
             ctx.project_id,
         )
+    await _publish_review_requests(pr_obj, ctx.project_id)
+
+
+async def _publish_review_requests(
+    pr: dict[str, typing.Any], project_id: str
+) -> None:
+    """Record the PR's pending review requests, best-effort."""
+    requests = _review_requests_record(pr, project_id=project_id)
+    if requests is None:
+        return
+    try:
+        await iggy.publish(
+            'pull_request_review_requests', 'github', [requests]
+        )
+    except Exception:
+        LOGGER.exception(
+            'github-pr-sync: failed to record review requests for PR %s '
+            'of project %s',
+            requests.pr_id,
+            project_id,
+        )
 
 
 sync_pull_requests_descriptor = ActionDescriptor(
@@ -252,7 +632,8 @@ sync_pull_requests_descriptor = ActionDescriptor(
     label='Sync Pull Request History',
     description=(
         'Record pull_request webhook events (opened, closed, reopened, '
-        'synchronize) directly into the ClickHouse pull_requests table.'
+        'synchronize, review_requested, review_request_removed) directly '
+        'into the ClickHouse pull_requests table.'
     ),
     callable=typing.cast(
         'typing.Any',
@@ -265,6 +646,86 @@ sync_pull_requests_descriptor = ActionDescriptor(
 )
 
 
+class SyncPRReviewsConfig(pydantic.BaseModel):
+    """``WebhookRule.handler_config`` for ``sync_pull_request_reviews``."""
+
+    action_selector: JsonPointer = pydantic.Field(
+        default_factory=lambda: jsonpointer.JsonPointer('/payload/action')
+    )
+    review_selector: JsonPointer = pydantic.Field(
+        default_factory=lambda: jsonpointer.JsonPointer('/payload/review')
+    )
+    pr_selector: JsonPointer = pydantic.Field(
+        default_factory=lambda: jsonpointer.JsonPointer(
+            '/payload/pull_request'
+        )
+    )
+
+
+async def sync_pull_request_reviews(
+    *,
+    ctx: PluginContext,
+    credentials: dict[str, str],
+    external_identifier: str,
+    action_config: SyncPRReviewsConfig,
+    event: object,
+) -> None:
+    """Record a ``pull_request_review`` delivery in ``pull_request_reviews``.
+
+    Handles ``submitted``, ``edited``, and ``dismissed``.  A dismissal
+    carries the dismissed review with state ``dismissed``, which
+    replaces that review's row.  GitHub drops a reviewer from the PR's
+    requested reviewers when they submit, so the payload's request
+    lists are recorded too; otherwise the reviewer would still read as
+    requested.  The PR row itself is not touched: the PR object in a
+    review payload has no diff stats, so writing it would zero them.
+    """
+    del credentials, external_identifier
+    action = _resolve(action_config.action_selector, event)
+    if not isinstance(action, str) or action not in _REVIEW_ACTIONS:
+        return
+    review_raw = _resolve(action_config.review_selector, event)
+    pr_raw = _resolve(action_config.pr_selector, event)
+    if not isinstance(review_raw, dict) or not isinstance(pr_raw, dict):
+        return
+    pr_obj = typing.cast('dict[str, typing.Any]', pr_raw)
+    if not pr_obj.get('id'):
+        return
+    record = _review_record(
+        typing.cast('dict[str, typing.Any]', review_raw),
+        project_id=ctx.project_id,
+        pr_id=str(pr_obj['id']),
+    )
+    if record is not None:
+        try:
+            await iggy.publish('pull_request_reviews', 'github', [record])
+        except Exception:
+            LOGGER.exception(
+                'github-pr-sync: failed to record review %s for project %s',
+                record.review_id,
+                ctx.project_id,
+            )
+    await _publish_review_requests(pr_obj, ctx.project_id)
+
+
+sync_pull_request_reviews_descriptor = ActionDescriptor(
+    name='sync_pull_request_reviews',
+    label='Sync Pull Request Reviews',
+    description=(
+        'Record pull_request_review webhook events (submitted, edited, '
+        'dismissed) into the ClickHouse pull_request_reviews table.'
+    ),
+    callable=typing.cast(
+        'typing.Any',
+        'imbi.plugins.github.pull_requests:sync_pull_request_reviews',
+    ),
+    config_model=typing.cast(
+        'typing.Any',
+        'imbi.plugins.github.pull_requests:SyncPRReviewsConfig',
+    ),
+)
+
+
 class GitHubPullRequestSync(PullRequestSyncCapability):
     """Pull-request history sync capability handler.
 
@@ -310,9 +771,25 @@ class GitHubPullRequestSync(PullRequestSyncCapability):
         PAT-or-App resolution the webhook actions use.
 
         Walks every PR in the repo (``state=all``, paginated) and upserts
-        into the ClickHouse ``pull_requests`` table.
-        ``ReplacingMergeTree`` dedupes against rows the webhook already
-        recorded, so re-running is safe.
+        into the ClickHouse ``pull_requests`` table, records the review
+        requests and reviews of every open PR (best-effort, after the PR
+        rows are published), then reconciles: it
+        deletes the project's ``open`` rows on this host whose ``pr_id``
+        GitHub did not return, so a PR filed under the project from
+        another repository doesn't stay open forever.  Closed and merged
+        rows are never deleted, and neither are rows from another host
+        (a second pr-sync Integration) or rows recorded less than
+        ``_PRUNE_MARGIN`` before the sync started.  A PR GitHub returned
+        with an ``id`` keeps its row even when it is too malformed to
+        record.  The rows deleted are logged.
+
+        The reconcile is skipped when the listing may be incomplete (a
+        404 on any page, a next link it can't follow, or the page cap),
+        when the GitHub App is not installed, and when the repo did not
+        come from the project's repository link (the Integration's link
+        key or ``github-repository``).  It runs whether or not the
+        publish succeeded, and a ClickHouse failure is logged and
+        swallowed.
 
         ``additions``, ``deletions``, and ``changed_files`` are ``0`` for
         backfill rows (GitHub's list-PRs API omits them); they are
@@ -338,7 +815,8 @@ class GitHubPullRequestSync(PullRequestSyncCapability):
                 'on-demand sync: set the Integration flavor/host'
             )
         base = host_to_api_base(host)
-        owner, repo = resolve_owner_repo(ctx, host, _SELF_SLUG)
+        target = resolve_repository_target(ctx, host, _SELF_SLUG)
+        owner, repo = target.owner, target.repo
         try:
             token = await _resolve_bearer(credentials, base, owner, repo)
         except AppNotInstalledError as exc:
@@ -348,15 +826,44 @@ class GitHubPullRequestSync(PullRequestSyncCapability):
                 ctx.project_id,
             )
             return 0
+        started_at = datetime.datetime.now(datetime.UTC)
         async with _client(base, owner, repo, token) as client:
-            raw_prs = await _fetch_all_prs(
+            raw_prs, complete = await _fetch_all_prs(
                 client, max_wait=_BACKFILL_MAX_WAIT_SECONDS
             )
-        records: list[pydantic.BaseModel] = []
-        for pr in raw_prs:
-            record = _pr_record(pr, project_id=ctx.project_id)
-            if record is not None:
-                records.append(record)
-        return await _publish_best_effort(
-            'pull_requests', records, ctx.project_id
-        )
+            records: list[pydantic.BaseModel] = []
+            requests: list[pydantic.BaseModel] = []
+            for pr in raw_prs:
+                record = _pr_record(pr, project_id=ctx.project_id)
+                if record is not None:
+                    records.append(record)
+                if pr.get('state') == 'open':
+                    request = _review_requests_record(
+                        pr, project_id=ctx.project_id
+                    )
+                    if request is not None:
+                        requests.append(request)
+            # PRs go out before the review fetches, so a review fetch
+            # can never cost the PR rows.
+            recorded = await _publish_best_effort(
+                'pull_requests', records, ctx.project_id
+            )
+            await _publish_best_effort(
+                'pull_request_review_requests', requests, ctx.project_id
+            )
+            reviews = await _open_pr_reviews(
+                client,
+                raw_prs,
+                project_id=ctx.project_id,
+                max_wait=_BACKFILL_MAX_WAIT_SECONDS,
+            )
+            await _publish_best_effort(
+                'pull_request_reviews', reviews, ctx.project_id
+            )
+        # Only the project's repository link is authoritative enough to
+        # delete against; another same-host link or a project-type/slug
+        # guess may name some other repo entirely.
+        if complete and _is_repository_link(ctx, host, target):
+            pr_ids = [str(pr['id']) for pr in raw_prs if pr.get('id')]
+            await _prune_stale_prs(ctx.project_id, host, started_at, pr_ids)
+        return recorded

{imbi_plugin_github-2.43.0.dist-info → imbi_plugin_github-2.44.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: imbi-plugin-github
-Version: 2.43.0
+Version: 2.44.0
 Summary: GitHub identity plugin for Imbi (github.com / GHEC / GHES)
 Author-email: "Gavin M. Roy" <gavinr@aweber.com>
 License-Expression: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.14
 Requires-Dist: httpx>=0.27
-Requires-Dist: imbi-common[databases]==2.43.0
+Requires-Dist: imbi-common[databases]==2.44.0
 Requires-Dist: pydantic>=2
 Requires-Dist: pyjwt[crypto]>=2.8
 Description-Content-Type: text/markdown
@@ -107,6 +107,35 @@ credential in one of two modes, resolved per call:
   from the pushed repository (`GET /repos/{owner}/{repo}/installation`).
   The App needs **Contents: Read-only**.
 
+### Pull request history sync
+
+`GitHubPullRequestSync.sync_all_history` is the on-demand `pr-sync`
+backfill. It walks `GET /repos/{owner}/{repo}/pulls?state=all` (up to 100
+pages of 100) and publishes a `pull_requests` row for every PR.
+
+After a complete walk it reconciles: it deletes the project's `open` rows
+on the synced host whose `pr_id` GitHub did not list, so a PR filed under
+the project from another repository does not stay `open` forever. It
+selects those rows first, logs their URLs, and deletes exactly those
+`pr_id`s; a sync with nothing stale issues no delete.
+
+It never deletes:
+
+- closed or merged rows, so PR history survives a move to a new
+  repository;
+- rows on another host, from a second `pr-sync` Integration;
+- rows recorded less than five minutes before the sync started, which
+  covers webhook rows written during the sync and clock skew between
+  hosts;
+- the row of a PR GitHub lists but is too malformed to record.
+
+The reconcile is skipped when the listing may be incomplete (a 404 on any
+page, a next link it cannot follow, or the page cap), when the GitHub App
+is not installed for the repository, and when the repository did not come
+from the project's repository link (the Integration's link key or
+`github-repository`). It runs even if the publish failed, and a
+ClickHouse failure is logged and does not fail the sync.
+
 ## Integration options
 
 Asked once per Integration and delivered to every capability on

{imbi_plugin_github-2.43.0.dist-info → imbi_plugin_github-2.44.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-imbi/plugins/github/README.md,sha256=HUD61Mvn9fSykQ4I-oUoZ62wd9tdhJ8jqoaPnpuPasM,7121
+imbi/plugins/github/README.md,sha256=WF_NszzEcKoCMZ6SmB2mgQxkonVwlMbla5Dl0N3B4mU,7355
 imbi/plugins/github/__init__.py,sha256=L_5_kr22VkjJob6SVLVzCkIvnWddxi54GIrM2cNs2rg,441
 imbi/plugins/github/_app_auth.py,sha256=qr3azBExZSI0vDIIKMKV8nSllJNKCJLKBXV86lIp9jE,19015
 imbi/plugins/github/_hosts.py,sha256=sYuE3f7934125VZ6GGTqm8t3sVa3m0DNt3jL9Hb8nM0,4014
@@ -8,11 +8,11 @@ imbi/plugins/github/deployment.py,sha256=bC5qMvbO-N0f5DTNo7ObyZIUtN4cl8uyjHgpp57
 imbi/plugins/github/doctor.py,sha256=sIps9YyDFLva7JHyKJU0IkKSW8tiI-PArC0s1ytxv3w,33177
 imbi/plugins/github/identity.py,sha256=9t_e-_jUbQ1iL0Vb6vaK2d7bT1ki6pw9aWwc4ics6gI,10354
 imbi/plugins/github/lifecycle.py,sha256=KJSzB36zFqetQBY5HNXLsRGRoIwGhrBJ-I1mA8G9H_Q,37711
-imbi/plugins/github/plugin.py,sha256=KmAu1lrGX2tkaNqTd3GV9DwLaPd78ZrODGi6jUt2Z0c,15981
+imbi/plugins/github/plugin.py,sha256=LDFCqj6e2jfF9io3U7cC0eVOUAxw1ENvFS-0AWOMuDc,16073
 imbi/plugins/github/public.py,sha256=GZnTLbj_Gl5sHWuRqFuajnoO2xPudNQumAt4h_uJpYw,3114
-imbi/plugins/github/pull_requests.py,sha256=FiHbjdGqghJ2Q1--d15XfOtpUe_KZE7hYRb8JLQ5U1I,13510
+imbi/plugins/github/pull_requests.py,sha256=DH9CnJoCpY5Ats5sZ_ktLvy54kBwK9g_-CDMlAIOq7Y,32553
 imbi/plugins/github/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 imbi/plugins/github/secret_provider.py,sha256=83BQwbG6AuIlyCWAskkN5hrVbRc9X5UqQJl5qt7Pij0,27317
-imbi_plugin_github-2.43.0.dist-info/METADATA,sha256=IJBROVQR2tagTrCI0PddNXFWxlbGmcPZSkieQLVPQbQ,7618
-imbi_plugin_github-2.43.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-imbi_plugin_github-2.43.0.dist-info/RECORD,,
+imbi_plugin_github-2.44.0.dist-info/METADATA,sha256=o6mVXl-pbwOEL5FSJnvFz8dQo292FnGptle6Rj57fT0,8977
+imbi_plugin_github-2.44.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+imbi_plugin_github-2.44.0.dist-info/RECORD,,