imbi-plugin-github 2.32.3__tar.gz → 2.33.0__tar.gz

{imbi_plugin_github-2.32.3 → imbi_plugin_github-2.33.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: imbi-plugin-github
-Version: 2.32.3
+Version: 2.33.0
 Summary: GitHub identity plugin for Imbi (github.com / GHEC / GHES)
 Author-email: "Gavin M. Roy" <gavinr@aweber.com>
 License-Expression: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.14
 Requires-Dist: httpx>=0.27
-Requires-Dist: imbi-common[databases]==2.32.3
+Requires-Dist: imbi-common[databases]==2.33.0
 Requires-Dist: pydantic>=2
 Requires-Dist: pyjwt[crypto]>=2.8
 Description-Content-Type: text/markdown

{imbi_plugin_github-2.32.3 → imbi_plugin_github-2.33.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "imbi-plugin-github"
-version = "2.32.3"
+version = "2.33.0"
 description = "GitHub identity plugin for Imbi (github.com / GHEC / GHES)"
 readme = "README.md"
 requires-python = ">=3.14"
@@ -18,7 +18,7 @@ classifiers = [
 ]
 dependencies = [
   "httpx>=0.27",
-  "imbi-common[databases]==2.32.3",
+  "imbi-common[databases]==2.33.0",
   "pydantic>=2",
   "pyjwt[crypto]>=2.8",
 ]

{imbi_plugin_github-2.32.3 → imbi_plugin_github-2.33.0}/src/imbi/plugins/github/commits.py

@@ -570,6 +570,50 @@ async def _last_known_sha(project_id: str) -> str | None:
     return None
 
 
+async def _stored_pushed_at(
+    project_id: str, shas: collections.abc.Iterable[str]
+) -> dict[str, datetime.datetime]:
+    """``pushed_at`` already recorded for each of *shas*, keyed by sha.
+
+    A re-sync must carry the stored push time forward rather than stamp
+    the commit with the current clock.  ``pushed_at`` is what orders the
+    recent-commits feed and picks :func:`_last_known_sha`, and the
+    ``commits`` table is a ``ReplacingMergeTree`` versioned on
+    ``recorded_at`` -- so a row re-inserted with a fresh ``pushed_at``
+    (a ``workflow_run`` delivery re-syncing a commit stored days earlier)
+    replaces the original and moves the commit out of push order.
+
+    Reads the *earliest* ``pushed_at`` still on disk for each sha, which
+    is the closest thing to the real push time when an earlier re-sync has
+    already re-stamped the row and the parts have not merged yet.
+    Best-effort: a lookup failure answers ``{}`` and the caller stamps
+    every row with the current time, as before.
+    """
+    wanted = sorted({sha for sha in shas if sha})
+    if not wanted:
+        return {}
+    try:
+        rows = await clickhouse.query(
+            'SELECT sha, min(pushed_at) AS pushed_at FROM commits '
+            'WHERE project_id = {pid:String} '
+            'AND sha IN {shas:Array(String)} GROUP BY sha',
+            {'pid': project_id, 'shas': wanted},
+        )
+    except Exception:  # noqa: BLE001
+        LOGGER.debug(
+            'github-commit-sync stored pushed_at lookup failed for %s',
+            project_id,
+            exc_info=True,
+        )
+        return {}
+    out: dict[str, datetime.datetime] = {}
+    for row in rows:
+        pushed_at = clickhouse.as_utc_or_none(row.get('pushed_at'))
+        if row.get('sha') and pushed_at is not None:
+            out[str(row['sha'])] = pushed_at
+    return out
+
+
 async def _fetch_recent_commits(
     client: httpx.AsyncClient, head: str, limit: int, *, max_wait: float
 ) -> list[dict[str, typing.Any]]:
@@ -1030,12 +1074,17 @@ async def sync_commits(
     user_map = await _resolve_author_users(
         raw, ctx.resolve_user_by_identity, base
     )
+    # A workflow_run re-sync of an already-stored commit must keep the
+    # push time it was first recorded with; see _stored_pushed_at.
+    stored = await _stored_pushed_at(
+        ctx.project_id, (str(i['sha']) for i in raw if i.get('sha'))
+    )
     records: list[pydantic.BaseModel] = [
         _commit_record(
             item,
             project_id=ctx.project_id,
             ref=ref,
-            pushed_at=pushed_at,
+            pushed_at=stored.get(str(item['sha']), pushed_at),
             author_user=_author_user(item, user_map),
             ci_status=ci_by_sha.get(str(item['sha']), 'unknown'),
         )
@@ -1558,12 +1607,18 @@ class GitHubCommitSync(CommitSyncCapability):
         user_map = await _resolve_author_users(
             raw_commits, ctx.resolve_user_by_identity, base
         )
+        # Re-running the backfill must not collapse every stored commit
+        # onto one push time; only commits new to the table get now().
+        stored = await _stored_pushed_at(
+            ctx.project_id,
+            (str(i['sha']) for i in raw_commits if i.get('sha')),
+        )
         commit_records: list[pydantic.BaseModel] = [
             _commit_record(
                 item,
                 project_id=ctx.project_id,
                 ref=branch,
-                pushed_at=pushed_at,
+                pushed_at=stored.get(str(item['sha']), pushed_at),
                 author_user=_author_user(item, user_map),
                 ci_status=ci_by_sha.get(str(item['sha']), 'unknown'),
             )
@@ -1721,12 +1776,15 @@ class GitHubCommitSync(CommitSyncCapability):
         user_map = await _resolve_author_users(
             raw, ctx.resolve_user_by_identity, base
         )
+        stored = await _stored_pushed_at(
+            ctx.project_id, (str(i['sha']) for i in raw if i.get('sha'))
+        )
         records: list[pydantic.BaseModel] = [
             _commit_record(
                 item,
                 project_id=ctx.project_id,
                 ref=branch,
-                pushed_at=pushed_at,
+                pushed_at=stored.get(str(item['sha']), pushed_at),
                 author_user=_author_user(item, user_map),
                 ci_status=ci_by_sha.get(str(item['sha']), 'unknown'),
             )

{imbi_plugin_github-2.32.3 → imbi_plugin_github-2.33.0}/src/imbi/plugins/github/deployment.py

@@ -199,6 +199,10 @@ _ZERO_SHA = '0' * 40
 # GitHub's compare endpoint lists at most 300 changed files and offers
 # no pagination for them -- a list this long may be incomplete.
 _COMPARE_FILES_CAP = 300
+# The compare endpoint pages its commits (250 per page) while ``ahead_by``
+# counts the whole range, so ``compare`` follows the ``Link`` chain -- up
+# to this many pages before it gives up on a pathological range.
+_MAX_COMPARE_PAGES = 20
 # How many note-blob reads run at once when listing a whole notes tree.
 _NOTE_BLOB_CONCURRENCY = 10
 
@@ -1078,12 +1082,38 @@ class GitHubDeployment(DeploymentCapability):
             ctx, credentials, SCOPE_CONTENTS_READ
         ) as client:
             quoted = urllib.parse.quote(f'{base}...{head}', safe='.')
-            resp = await client.get(f'/compare/{quoted}')
-            resp.raise_for_status()
-            payload = typing.cast(dict[str, typing.Any], resp.json())
-            commits_raw: list[dict[str, typing.Any]] = (
-                payload.get('commits') or []
-            )
+            path = f'/compare/{quoted}'
+            params: dict[str, str] = {'per_page': '250'}
+            payload: dict[str, typing.Any] = {}
+            commits_raw: list[dict[str, typing.Any]] = []
+            for page in range(1, _MAX_COMPARE_PAGES + 1):
+                resp = await client.get(path, params=params)
+                resp.raise_for_status()
+                page_payload = typing.cast(dict[str, typing.Any], resp.json())
+                if page == 1:
+                    # Only the first page carries ``files``, the
+                    # ``ahead_by``/``behind_by`` totals and the base.
+                    payload = page_payload
+                commits_raw.extend(page_payload.get('commits') or [])
+                next_url = _next_page_url(resp.headers.get('link'))
+                if next_url is None:
+                    break
+                next_page = _query_param(next_url, 'page')
+                if next_page is None:
+                    break
+                params['page'] = next_page
+            else:
+                # Still more pages after the cap: ``ahead`` keeps the
+                # true count so a consumer can tell the list is short.
+                LOGGER.warning(
+                    'Compare %s...%s truncated at %d pages (%d of %s '
+                    'commits); the commit list is incomplete',
+                    base,
+                    head,
+                    _MAX_COMPARE_PAGES,
+                    len(commits_raw),
+                    payload.get('ahead_by'),
+                )
             commits: list[Commit] = [
                 _commit_from_payload(item) for item in commits_raw
             ]

{imbi_plugin_github-2.32.3 → imbi_plugin_github-2.33.0}/src/imbi/plugins/github/pull_requests.py

@@ -106,7 +106,10 @@ def _pr_record(
         state=str(pr.get('state') or 'open'),
         author=str(user.get('login') or ''),
         draft=bool(pr.get('draft', False)),
-        merged=bool(pr.get('merged', False)),
+        # The list endpoint (``GET /pulls``) omits ``merged``; only the
+        # single-PR endpoint carries it. ``merged_at`` is set on both, so
+        # treat a merge timestamp as merged.
+        merged=bool(pr.get('merged') or pr.get('merged_at')),
         created_at=created_at,
         updated_at=updated_at,
         merged_at=_parse_pr_datetime(pr.get('merged_at')),

{imbi_plugin_github-2.32.3 → imbi_plugin_github-2.33.0}/tests/test_commits.py

@@ -121,6 +121,35 @@ def _event(body: dict[str, typing.Any]) -> dict[str, typing.Any]:
     }
 
 
+def _query_router(
+    *,
+    last_known: str | None = None,
+    stored: dict[str, datetime.datetime] | None = None,
+) -> mock.AsyncMock:
+    """A ``clickhouse.query`` stand-in that answers by statement shape.
+
+    ``stored`` maps sha -> the ``pushed_at`` already on record for it,
+    answering the carry-forward lookup; ``last_known`` answers the
+    ``_last_known_sha`` probe.  Anything else reads as an empty table.
+    """
+
+    async def _answer(
+        sql: str, params: dict[str, typing.Any] | None = None
+    ) -> list[dict[str, typing.Any]]:
+        params = params or {}
+        if 'min(pushed_at)' in sql:
+            return [
+                {'sha': sha, 'pushed_at': pushed_at}
+                for sha, pushed_at in (stored or {}).items()
+                if sha in params.get('shas', ())
+            ]
+        if 'argMax(sha, pushed_at)' in sql:
+            return [{'sha': last_known}] if last_known else []
+        return []
+
+    return mock.AsyncMock(side_effect=_answer)
+
+
 def _commit(
     sha: str, *, login: str = 'octocat', author_id: int = 583231
 ) -> dict[str, object]:
@@ -392,6 +421,96 @@ class SyncCommitsTestCase(unittest.IsolatedAsyncioTestCase):
         self.assertEqual('main', records[0].ref)
         self.assertEqual('pass', records[0].ci_status)
 
+    @respx.mock
+    async def test_resync_keeps_the_stored_push_time(self) -> None:
+        """A re-sync of a stored commit must not re-stamp ``pushed_at``.
+
+        The re-inserted row wins under ``ReplacingMergeTree``, so a fresh
+        ``pushed_at`` lifts a commit stored days earlier above everything
+        pushed since -- and the recent-commits feed, and every release
+        range sliced out of it, is ordered by that column (#308).  The
+        re-sync still refreshes what it is for: the CI status.
+        """
+        head = 'b' * 40
+        first_pushed = datetime.datetime(
+            2026, 9, 9, 15, 8, 54, tzinfo=datetime.UTC
+        )
+        respx.get(
+            f'https://api.github.com/repos/octo/demo/commits/{head}'
+        ).mock(return_value=httpx.Response(200, json=_commit(head)))
+        respx.get(_check_runs_url(head)).mock(
+            return_value=httpx.Response(200, json=_check_runs('success'))
+        )
+        query = _query_router(stored={head: first_pushed})
+        with mock.patch(_QUERY, new=query):
+            with mock.patch(_INSERT, new=mock.AsyncMock()) as insert:
+                await commits.sync_commits(
+                    ctx=_ctx(),
+                    credentials=_CREDS,
+                    external_identifier='',
+                    action_config=_WORKFLOW_RUN_CONFIG,
+                    event=_event(_workflow_run(head=head)),
+                )
+        _, records = _await_args(insert)
+        self.assertEqual(first_pushed, records[0].pushed_at)
+        self.assertEqual('pass', records[0].ci_status)
+        lookup = next(
+            c for c in query.await_args_list if 'min(pushed_at)' in c.args[0]
+        )
+        self.assertEqual([head], lookup.args[1]['shas'])
+
+    @respx.mock
+    async def test_only_new_commits_get_the_current_push_time(self) -> None:
+        base, head = 'a' * 40, 'b' * 40
+        respx.get(
+            f'https://api.github.com/repos/octo/demo/compare/{base}...{head}'
+        ).mock(
+            return_value=httpx.Response(
+                200, json={'commits': [_commit('c' * 40), _commit('d' * 40)]}
+            )
+        )
+        first_pushed = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC)
+        with mock.patch(
+            _QUERY, new=_query_router(stored={'c' * 40: first_pushed})
+        ):
+            with mock.patch(_INSERT, new=mock.AsyncMock()) as insert:
+                await commits.sync_commits(
+                    ctx=_ctx(),
+                    credentials=_CREDS,
+                    external_identifier='',
+                    action_config=commits.SyncCommitsConfig(),
+                    event=_event(_push(before=base, after=head)),
+                )
+        _, records = _await_args(insert)
+        by_sha = {r.sha: r for r in records}
+        self.assertEqual(first_pushed, by_sha['c' * 40].pushed_at)
+        self.assertGreater(by_sha['d' * 40].pushed_at, first_pushed)
+
+    @respx.mock
+    async def test_push_time_lookup_failure_stamps_now(self) -> None:
+        """The carry-forward is best-effort; a lookup error must not
+        cost the sync."""
+        head = 'b' * 40
+        respx.get(
+            f'https://api.github.com/repos/octo/demo/commits/{head}'
+        ).mock(return_value=httpx.Response(200, json=_commit(head)))
+        respx.get(_check_runs_url(head)).mock(
+            return_value=httpx.Response(200, json=_check_runs('success'))
+        )
+        failing = mock.AsyncMock(side_effect=RuntimeError('clickhouse down'))
+        with mock.patch(_QUERY, new=failing):
+            with mock.patch(_INSERT, new=mock.AsyncMock()) as insert:
+                await commits.sync_commits(
+                    ctx=_ctx(),
+                    credentials=_CREDS,
+                    external_identifier='',
+                    action_config=_WORKFLOW_RUN_CONFIG,
+                    event=_event(_workflow_run(head=head)),
+                )
+        insert.assert_awaited_once()
+        _, records = _await_args(insert)
+        self.assertEqual(head, records[0].sha)
+
     @respx.mock
     async def test_absent_before_never_consults_last_known_sha(self) -> None:
         """The single-commit path must not fall through to a compare.
@@ -407,8 +526,11 @@ class SyncCommitsTestCase(unittest.IsolatedAsyncioTestCase):
         respx.get(_check_runs_url(head)).mock(
             return_value=httpx.Response(200, json=_check_runs('success'))
         )
+        compare = respx.get(url__regex=r'.*/compare/.+\.\.\..+').mock(
+            return_value=httpx.Response(200, json={'commits': []})
+        )
         with (
-            mock.patch(_QUERY, new=mock.AsyncMock()) as query,
+            mock.patch(_QUERY, new=_query_router(last_known=head)) as query,
             mock.patch(_INSERT, new=mock.AsyncMock()) as insert,
         ):
             await commits.sync_commits(
@@ -418,7 +540,16 @@ class SyncCommitsTestCase(unittest.IsolatedAsyncioTestCase):
                 action_config=_WORKFLOW_RUN_CONFIG,
                 event=_event(_workflow_run(head=head)),
             )
-        query.assert_not_awaited()
+        # The pushed_at carry-forward reads the table too; what must not
+        # run is the last-known-sha probe (and the compare it would feed).
+        self.assertFalse(
+            [
+                c
+                for c in query.await_args_list
+                if 'argMax(sha, pushed_at)' in c.args[0]
+            ]
+        )
+        self.assertFalse(compare.called)
         insert.assert_awaited_once()
 
     @respx.mock
@@ -1551,6 +1682,37 @@ class SyncAllHistoryTestCase(unittest.IsolatedAsyncioTestCase):
             urls,
         )
 
+    @respx.mock
+    async def test_rerun_keeps_stored_push_times(self) -> None:
+        """A backfill re-run must not collapse the stored history onto one
+        push time; only commits new to the table are stamped now."""
+        self._mock_default_branch()
+        respx.get(f'{self._REPO}/commits').mock(
+            return_value=httpx.Response(
+                200, json=[_commit('c' * 40), _commit('d' * 40)]
+            )
+        )
+        respx.get(f'{self._REPO}/releases').mock(
+            return_value=httpx.Response(200, json=[])
+        )
+        respx.get(f'{self._REPO}/git/matching-refs/tags').mock(
+            return_value=httpx.Response(200, json=[])
+        )
+        first_pushed = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC)
+        with mock.patch(
+            _QUERY, new=_query_router(stored={'d' * 40: first_pushed})
+        ):
+            with mock.patch(_INSERT, new=mock.AsyncMock()) as insert:
+                await commits.GitHubCommitSync().sync_all_history(
+                    ctx=self._ctx(), credentials=_CREDS
+                )
+        commit_call = next(
+            c for c in insert.await_args_list if c.args[0] == 'commits'
+        )
+        by_sha = {r.sha: r for r in commit_call.args[1]}
+        self.assertEqual(first_pushed, by_sha['d' * 40].pushed_at)
+        self.assertGreater(by_sha['c' * 40].pushed_at, first_pushed)
+
     @respx.mock
     async def test_bounds_ci_status_to_recent_commits(self) -> None:
         deployment._CHECKS_DISABLED_TOKENS.clear()
@@ -2125,6 +2287,27 @@ class SyncNewCommitsTestCase(unittest.IsolatedAsyncioTestCase):
         record = insert.await_args_list[0].args[1][0]
         self.assertEqual('main', record.ref)
 
+    @respx.mock
+    async def test_keeps_the_stored_push_time_on_resync(self) -> None:
+        self._mock_default_branch()
+        respx.get(url__regex=r'.*/compare/.+\.\.\..+').mock(
+            return_value=httpx.Response(
+                200, json={'commits': [_commit('c' * 40), _commit('d' * 40)]}
+            )
+        )
+        first_pushed = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC)
+        with mock.patch(
+            _QUERY,
+            new=_query_router(
+                last_known='a' * 40, stored={'c' * 40: first_pushed}
+            ),
+        ):
+            written, insert = await self._sync()
+        self.assertEqual(2, written)
+        by_sha = {r.sha: r for r in insert.await_args_list[0].args[1]}
+        self.assertEqual(first_pushed, by_sha['c' * 40].pushed_at)
+        self.assertGreater(by_sha['d' * 40].pushed_at, first_pushed)
+
     @respx.mock
     async def test_since_window_when_nothing_stored(self) -> None:
         self._mock_default_branch()

{imbi_plugin_github-2.32.3 → imbi_plugin_github-2.33.0}/tests/test_deployment.py

@@ -6,6 +6,7 @@ import datetime
 import json
 import time
 import unittest
+import unittest.mock
 
 import httpx
 import respx
@@ -717,6 +718,98 @@ class CompareTestCase(unittest.IsolatedAsyncioTestCase):
         self.assertEqual(result.commits[1].message, 'Subject only')
         self.assertIsNone(result.commits[1].body)
 
+    @respx.mock
+    async def test_compare_follows_next_link(self) -> None:
+        url = 'https://api.github.com/repos/octo/demo/compare/base...head'
+        page2_link = f'{url}?per_page=250&page=2'
+        # More-specific (page=2) matcher first; see ListRefsPagination.
+        respx.get(url, params={'per_page': '250', 'page': '2'}).mock(
+            return_value=httpx.Response(
+                200,
+                json={
+                    'ahead_by': 3,
+                    'commits': [
+                        {
+                            'sha': 'c3',
+                            'commit': {
+                                'message': 'three',
+                                'author': {'name': 'A', 'date': None},
+                            },
+                        },
+                    ],
+                },
+            )
+        )
+        respx.get(url, params={'per_page': '250'}).mock(
+            return_value=httpx.Response(
+                200,
+                json={
+                    'ahead_by': 3,
+                    'behind_by': 0,
+                    'base_commit': {'sha': 'base-sha'},
+                    'commits': [
+                        {
+                            'sha': 'c1',
+                            'commit': {
+                                'message': 'one',
+                                'author': {'name': 'A', 'date': None},
+                            },
+                        },
+                        {
+                            'sha': 'c2',
+                            'commit': {
+                                'message': 'two',
+                                'author': {'name': 'A', 'date': None},
+                            },
+                        },
+                    ],
+                    'files': [{'additions': 1, 'deletions': 0}],
+                },
+                headers={'Link': f'<{page2_link}>; rel="next"'},
+            )
+        )
+        plugin = GitHubDeployment()
+        result = await plugin.compare(_ctx(), _CREDS, 'base', 'head')
+        self.assertEqual([c.sha for c in result.commits], ['c1', 'c2', 'c3'])
+        self.assertEqual(result.head_sha, 'c3')
+        self.assertEqual(result.base_sha, 'base-sha')
+        self.assertEqual(result.ahead, 3)
+        self.assertEqual(result.files_changed, 1)
+        self.assertEqual(result.additions, 1)
+
+    @respx.mock
+    async def test_compare_warns_when_page_walk_truncates(self) -> None:
+        url = 'https://api.github.com/repos/octo/demo/compare/base...head'
+        respx.get(url).mock(
+            return_value=httpx.Response(
+                200,
+                json={
+                    'ahead_by': 500,
+                    'commits': [
+                        {
+                            'sha': 'c1',
+                            'commit': {
+                                'message': 'one',
+                                'author': {'name': 'A', 'date': None},
+                            },
+                        },
+                    ],
+                },
+                headers={'Link': f'<{url}?per_page=250&page=2>; rel="next"'},
+            )
+        )
+        plugin = GitHubDeployment()
+        with unittest.mock.patch(
+            'imbi.plugins.github.deployment._MAX_COMPARE_PAGES', 1
+        ):
+            with self.assertLogs(
+                'imbi.plugins.github', level='WARNING'
+            ) as logs:
+                result = await plugin.compare(_ctx(), _CREDS, 'base', 'head')
+        self.assertEqual(len(result.commits), 1)
+        self.assertEqual(result.ahead, 500)
+        self.assertIn('truncated', logs.output[0])
+
 
 class TriggerDeploymentTestCase(unittest.IsolatedAsyncioTestCase):
     @respx.mock

{imbi_plugin_github-2.32.3 → imbi_plugin_github-2.33.0}/tests/test_pull_requests.py

@@ -103,6 +103,31 @@ class ManifestTestCase(unittest.TestCase):
         self.assertIn('sync_pull_requests', names)
 
 
+class PrRecordTestCase(unittest.TestCase):
+    def test_merged_from_flag(self) -> None:
+        record = pull_requests._pr_record(
+            _pr(state='closed', merged=True, merged_at='2026-01-03T00:00:00Z'),
+            project_id='p1',
+        )
+        assert record is not None
+        self.assertTrue(record.merged)
+
+    def test_merged_inferred_from_merged_at_on_list_payload(self) -> None:
+        # ``GET /pulls`` (the backfill source) has no ``merged`` key.
+        pr = _pr(state='closed', merged_at='2026-01-03T00:00:00Z')
+        del pr['merged']
+        record = pull_requests._pr_record(pr, project_id='p1')
+        assert record is not None
+        self.assertTrue(record.merged)
+
+    def test_closed_without_merge_is_not_merged(self) -> None:
+        pr = _pr(state='closed')
+        del pr['merged']
+        record = pull_requests._pr_record(pr, project_id='p1')
+        assert record is not None
+        self.assertFalse(record.merged)
+
+
 class SyncPullRequestsActionTestCase(unittest.IsolatedAsyncioTestCase):
     async def test_opened_records_one_row(self) -> None:
         with mock.patch(_INSERT_WEBHOOK, new=mock.AsyncMock()) as insert: