imbi-plugin-github 2.24.0__tar.gz → 2.26.0__tar.gz

{imbi_plugin_github-2.24.0 → imbi_plugin_github-2.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: imbi-plugin-github
-Version: 2.24.0
+Version: 2.26.0
 Summary: GitHub identity plugin for Imbi (github.com / GHEC / GHES)
 Author-email: "Gavin M. Roy" <gavinr@aweber.com>
 License-Expression: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.14
 Requires-Dist: httpx>=0.27
-Requires-Dist: imbi-common[databases]==2.24.0
+Requires-Dist: imbi-common[databases]==2.26.0
 Requires-Dist: pydantic>=2
 Requires-Dist: pyjwt[crypto]>=2.8
 Description-Content-Type: text/markdown

{imbi_plugin_github-2.24.0 → imbi_plugin_github-2.26.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "imbi-plugin-github"
-version = "2.24.0"
+version = "2.26.0"
 description = "GitHub identity plugin for Imbi (github.com / GHEC / GHES)"
 readme = "README.md"
 requires-python = ">=3.14"
@@ -18,7 +18,7 @@ classifiers = [
 ]
 dependencies = [
   "httpx>=0.27",
-  "imbi-common[databases]==2.24.0",
+  "imbi-common[databases]==2.26.0",
   "pydantic>=2",
   "pyjwt[crypto]>=2.8",
 ]

{imbi_plugin_github-2.24.0 → imbi_plugin_github-2.26.0}/src/imbi/plugins/github/deployment.py

@@ -2,7 +2,10 @@
 
 :class:`GitHubDeployment` resolves its host from the Integration's
 ``flavor`` + ``host`` options on ``ctx.integration_options`` (github.com,
-a ``*.ghe.com`` GHEC tenant, or an operator-managed GHES appliance).
+a ``*.ghe.com`` GHEC tenant, or an operator-managed GHES appliance), and
+reads ``mainline_branches`` from the same map to know which deployment
+refs are branches rather than release tags (see
+:func:`_mainline_branches`).
 
 *Deploying* drives the GitHub Deployments API
 (``POST /repos/{owner}/{repo}/deployments``) rather than
@@ -29,6 +32,7 @@ access token through the Integration credential blob's
 from __future__ import annotations
 
 import asyncio
+import base64
 import collections.abc
 import contextlib
 import datetime
@@ -136,6 +140,31 @@ def _auth_headers(token: str) -> dict[str, str]:
     }
 
 
+#: A push event's ``before`` when the ref did not exist until now.
+_ZERO_SHA = '0' * 40
+# GitHub's compare endpoint lists at most 300 changed files and offers
+# no pagination for them -- a list this long may be incomplete.
+_COMPARE_FILES_CAP = 300
+# How many note-blob reads run at once when listing a whole notes tree.
+_NOTE_BLOB_CONCURRENCY = 10
+
+_FULL_SHA_PATTERN = re.compile(r'^[0-9a-f]{40}$')
+
+
+def _note_sha(path: object) -> str | None:
+    """Annotated commit SHA for a notes-tree path, or ``None``.
+
+    Notes trees key blobs by the annotated commit's full SHA, optionally
+    fanned out into subtrees (``ab/cdef...``); flattening the path
+    recovers the SHA.  Non-note files (``README`` and friends can live
+    on a notes ref) do not flatten to 40 hex chars and are skipped.
+    """
+    if not isinstance(path, str):
+        return None
+    flattened = path.replace('/', '').lower()
+    return flattened if _FULL_SHA_PATTERN.match(flattened) else None
+
+
 def _short_sha(sha: str) -> str:
     return sha[:7]
 
@@ -371,17 +400,52 @@ def _record_releases_forbidden(client: httpx.AsyncClient) -> None:
     _RELEASES_FORBIDDEN_TOKENS[_releases_cache_key(client)] = now
 
 
+# Fallback for the ``mainline_branches`` integration option: deployment
+# refs that are branch names rather than release tags. A repo that deploys
+# off its default branch reports ``ref == 'main'`` on every deployment, so
+# ``GET /releases/tags/main`` is a guaranteed 404 — skip the request
+# outright instead of paying for it once per deployment row.
+_DEFAULT_MAINLINE_BRANCHES = frozenset({'main', 'master'})
+
+
+def _mainline_branches(
+    integration_options: dict[str, typing.Any],
+) -> frozenset[str]:
+    """Resolve the ``mainline_branches`` integration option to a set.
+
+    Declared integration-level (beside ``flavor`` / ``host``) because
+    mainline branch naming is a property of the org's repos rather than of
+    any one capability, so every capability can read the same value.  The
+    manifest's ``default`` is a form pre-fill only -- the host does not
+    substitute it -- so an absent or blank value resolves to
+    :data:`_DEFAULT_MAINLINE_BRANCHES` here, mirroring how
+    ``artifact_version_input`` re-applies its own default.  Consequently
+    the guard can be *retargeted* but not switched off; a repo that cuts
+    releases tagged with a branch name is not a case worth supporting.
+
+    Operator-entered, so tolerate commas as well as the advertised spaces
+    and discard surrounding whitespace -- a stray separator would
+    otherwise register as a branch named ``''``.
+    """
+    raw = integration_options.get('mainline_branches')
+    if not isinstance(raw, str):
+        return _DEFAULT_MAINLINE_BRANCHES
+    configured = frozenset(raw.replace(',', ' ').split())
+    return configured or _DEFAULT_MAINLINE_BRANCHES
+
+
 def _commit_from_payload(payload: dict[str, typing.Any]) -> Commit:
     """Convert a GitHub commit list/object payload into a :class:`Commit`."""
     sha = str(payload.get('sha', ''))
     commit_meta: dict[str, typing.Any] = payload.get('commit') or {}
     author_meta: dict[str, typing.Any] = commit_meta.get('author') or {}
     raw_message = str(commit_meta.get('message') or '')
-    message_lines = raw_message.splitlines()
+    subject, _, body = raw_message.partition('\n')
     return Commit(
         sha=sha,
         short_sha=_short_sha(sha),
-        message=message_lines[0] if message_lines else '',
+        message=subject,
+        body=body.strip() or None,
         author=author_meta.get('name'),
         authored_at=_parse_iso(author_meta.get('date')),
         url=payload.get('html_url'),
@@ -811,6 +875,238 @@ class GitHubDeployment(DeploymentCapability):
                 deletions=deletions,
             )
 
+    # -- Git notes ----------------------------------------------------------
+
+    async def get_commit_note(
+        self,
+        ctx: PluginContext,
+        credentials: dict[str, str],
+        namespace: str,
+        committish: str,
+    ) -> str | None:
+        """Read the note on ``committish`` from ``refs/notes/<namespace>``.
+
+        The notes tree keys blobs by the *full* SHA of the annotated
+        commit (with optional fan-out subtrees like ``ab/cdef...``), so a
+        short committish is resolved through the commits endpoint first.
+        """
+        async with self._client(ctx, credentials) as client:
+            full_sha = committish.lower()
+            if not _FULL_SHA_PATTERN.match(full_sha):
+                commit = await client.get(
+                    f'/commits/{urllib.parse.quote(committish, safe="")}'
+                )
+                commit.raise_for_status()
+                full_sha = str(commit.json()['sha']).lower()
+            notes = await self._notes_tree(client, namespace)
+            if notes is None:
+                return None
+            blob_sha = notes.get(full_sha)
+            if blob_sha is None:
+                return None
+            return await self._blob_text(client, blob_sha)
+
+    async def diff_commit_notes(
+        self,
+        ctx: PluginContext,
+        credentials: dict[str, str],
+        namespace: str,
+        before: str,
+        after: str,
+    ) -> dict[str, str | None]:
+        """Diff ``refs/notes/<namespace>`` between two of its commits.
+
+        ``before``/``after`` come from a push event on the notes ref;
+        the *files* changed between them are the notes of the annotated
+        commits, so the tree diff -- not the push's ``commits`` list --
+        is what names the commits whose notes changed.  An all-zero
+        ``before`` (the ref was just created) returns every note at
+        ``after``.
+
+        Known limitation: GitHub's compare endpoint is three-dot only
+        (``after`` against the merge base), so a *force-pushed* notes
+        ref whose ``before`` is not an ancestor of ``after`` can miss a
+        note that the rewrite flipped or removed.  The 404 fallback
+        below covers a ``before`` GitHub no longer has at all, and the
+        sweep backfill only repairs verdicts that are still ``null`` --
+        a stale non-null verdict from this window persists until the
+        next ordinary push touches the note.
+        """
+        async with self._client(ctx, credentials) as client:
+            if before == _ZERO_SHA:
+                return await self._all_notes(client, after)
+            quoted = urllib.parse.quote(f'{before}...{after}', safe='.')
+            resp = await client.get(f'/compare/{quoted}')
+            if resp.status_code == 404:
+                # ``before`` was garbage-collected or the ref history
+                # was rewritten; fall back to the full tree at ``after``
+                # so the push still lands rather than being dropped.
+                return await self._all_notes(client, after)
+            resp.raise_for_status()
+            payload = typing.cast('dict[str, typing.Any]', resp.json())
+            files: list[dict[str, typing.Any]] = payload.get('files') or []
+            if len(files) >= _COMPARE_FILES_CAP:
+                # The compare endpoint stops listing files at 300 and
+                # offers no pagination for them, so a full list this
+                # long may be missing entries.  Fall back to the whole
+                # tree at ``after`` -- same trade-off as the 404 path:
+                # removals in the window are missed until the sweep or
+                # the next push touches them.
+                LOGGER.warning(
+                    'Notes compare %s...%s hit the %d-file cap; '
+                    'reading the full tree instead',
+                    before,
+                    after,
+                    _COMPARE_FILES_CAP,
+                )
+                return await self._all_notes(client, after)
+            out: dict[str, str | None] = {}
+            for item in files:
+                status = str(item.get('status') or '')
+                previous = _note_sha(item.get('previous_filename'))
+                if status == 'renamed' and previous is not None:
+                    # Fan-out reshuffle: the note moved paths.  The old
+                    # path's flattened SHA only differs from the new one
+                    # if the note now annotates a different commit.
+                    out.setdefault(previous, None)
+                annotated = _note_sha(item.get('filename'))
+                if annotated is None:
+                    continue
+                if status == 'removed':
+                    out[annotated] = None
+                    continue
+                blob_sha = item.get('sha')
+                if not blob_sha:
+                    continue
+                try:
+                    body = await self._blob_text(client, str(blob_sha))
+                except httpx.HTTPError:
+                    LOGGER.warning(
+                        'Could not read note blob %s for %s; skipping',
+                        blob_sha,
+                        annotated,
+                    )
+                    body = None
+                if body is None:
+                    # Skip rather than record ``None``: in the diff a
+                    # ``None`` means "note removed" and would resolve a
+                    # drift blocker over a note we merely could not
+                    # read (``_blob_text`` already logged why).
+                    out.pop(annotated, None)
+                    continue
+                out[annotated] = body
+            return out
+
+    async def _notes_tree(
+        self, client: httpx.AsyncClient, namespace: str
+    ) -> dict[str, str] | None:
+        """Map annotated full SHA -> note blob SHA, or ``None`` sans ref."""
+        ref = await client.get(
+            f'/git/ref/{urllib.parse.quote(f"notes/{namespace}", safe="/")}'
+        )
+        if ref.status_code == 404:
+            return None
+        ref.raise_for_status()
+        tip = str(ref.json()['object']['sha'])
+        return await self._tree_notes(client, tip)
+
+    async def _tree_notes(
+        self, client: httpx.AsyncClient, commit_sha: str
+    ) -> dict[str, str]:
+        """Flatten one notes-ref commit's tree to annotated SHA -> blob."""
+        commit = await client.get(f'/git/commits/{commit_sha}')
+        commit.raise_for_status()
+        tree_sha = str(commit.json()['tree']['sha'])
+        tree = await client.get(f'/git/trees/{tree_sha}?recursive=1')
+        tree.raise_for_status()
+        payload = typing.cast('dict[str, typing.Any]', tree.json())
+        if payload.get('truncated'):
+            LOGGER.warning(
+                'Notes tree %s is truncated; some notes will be missed',
+                tree_sha,
+            )
+        entries: list[dict[str, typing.Any]] = payload.get('tree') or []
+        out: dict[str, str] = {}
+        for entry in entries:
+            if entry.get('type') != 'blob':
+                continue
+            annotated = _note_sha(entry.get('path'))
+            if annotated is not None:
+                out[annotated] = str(entry['sha'])
+        return out
+
+    async def _all_notes(
+        self, client: httpx.AsyncClient, commit_sha: str
+    ) -> dict[str, str | None]:
+        """Every note at one notes-ref commit, bodies included.
+
+        Blob reads run a few at a time (one request per note) and an
+        unreadable note is skipped rather than failing the batch or
+        recording a false "removed".
+        """
+        notes = await self._tree_notes(client, commit_sha)
+        gate = asyncio.Semaphore(_NOTE_BLOB_CONCURRENCY)
+
+        async def _read(blob_sha: str) -> str | BaseException | None:
+            async with gate:
+                try:
+                    return await self._blob_text(client, blob_sha)
+                except httpx.HTTPError as exc:
+                    return exc
+
+        items = list(notes.items())
+        bodies = await asyncio.gather(
+            *(_read(blob_sha) for _, blob_sha in items)
+        )
+        out: dict[str, str | None] = {}
+        for (annotated, blob_sha), body in zip(items, bodies, strict=True):
+            if isinstance(body, BaseException):
+                LOGGER.warning(
+                    'Could not read note blob %s for %s; skipping',
+                    blob_sha,
+                    annotated,
+                )
+                continue
+            if body is None:
+                # ``_blob_text`` could not decode it and logged why;
+                # a ``None`` here would read as "note removed".
+                continue
+            out[annotated] = body
+        return out
+
+    @staticmethod
+    async def _blob_text(
+        client: httpx.AsyncClient, blob_sha: str
+    ) -> str | None:
+        """Fetch and decode one note blob; ``None`` when undecodable."""
+        resp = await client.get(f'/git/blobs/{blob_sha}')
+        resp.raise_for_status()
+        payload = typing.cast('dict[str, typing.Any]', resp.json())
+        content = str(payload.get('content') or '')
+        if payload.get('encoding') != 'base64':
+            if not content:
+                # ``encoding: none`` with an empty body is how GitHub
+                # answers for blobs above the inline size limit --
+                # "cannot read", not "empty note".
+                LOGGER.warning(
+                    'Note blob %s answered encoding %r with no content',
+                    blob_sha,
+                    payload.get('encoding'),
+                )
+                return None
+            return content
+        try:
+            # Strip GitHub's line wrapping, then decode strictly --
+            # the default decoder silently discards invalid characters,
+            # turning garbage like '%%%%' into an empty body instead of
+            # landing on this "cannot read" path.
+            return base64.b64decode(
+                ''.join(content.split()), validate=True
+            ).decode('utf-8')
+        except (ValueError, UnicodeDecodeError):
+            LOGGER.warning('Could not decode note blob %s', blob_sha)
+            return None
+
     # -- Tags / Releases ----------------------------------------------------
 
     async def create_tag(
@@ -1221,11 +1517,25 @@ class GitHubDeployment(DeploymentCapability):
         # the triggering-actor lookup by run id so we resolve each run
         # at most once. Shared across the parallel per-env fan-out.
         run_cache: dict[str, tuple[str, str] | None] = {}
+        # Likewise, deployments in a sweep share a handful of refs; the
+        # lookup for each is memoised here as a task so it happens once
+        # per ref instead of once per deployment. Holding the task (not
+        # its result) also coalesces the per-env fan-out below: envs that
+        # deployed the same ref await one in-flight request rather than
+        # each issuing their own, which is the common shape at the host's
+        # default limit=1 where no single env repeats a ref.
+        release_lookups: dict[str, asyncio.Task[RemoteRelease | None]] = {}
+        mainline = _mainline_branches(ctx.integration_options)
         async with self._client(ctx, credentials) as client:
             per_env = await asyncio.gather(
                 *(
                     self._list_deployments_for_env(
-                        client, env, page_size, run_cache
+                        client,
+                        env,
+                        page_size,
+                        run_cache,
+                        release_lookups,
+                        mainline,
                     )
                     for env in environments
                 )
@@ -1248,8 +1558,9 @@ class GitHubDeployment(DeploymentCapability):
         which 404s to ``None`` for tags without a release and caches 403s
         so a scope-limited token short-circuits.
         """
+        mainline = _mainline_branches(ctx.integration_options)
         async with self._client(ctx, credentials) as client:
-            return await self._release_notes_for_ref(client, tag)
+            return await self._release_notes_for_ref(client, tag, mainline)
 
     async def _list_deployments_for_env(
         self,
@@ -1257,6 +1568,8 @@ class GitHubDeployment(DeploymentCapability):
         environment: str,
         page_size: int,
         run_cache: dict[str, tuple[str, str] | None],
+        release_lookups: dict[str, asyncio.Task[RemoteRelease | None]],
+        mainline: frozenset[str],
     ) -> list[RemoteDeployment]:
         try:
             resp = await client.get(
@@ -1289,7 +1602,12 @@ class GitHubDeployment(DeploymentCapability):
         observed: list[RemoteDeployment] = []
         for deployment in deployments:
             run = await self._observe_deployment(
-                client, environment, deployment, run_cache
+                client,
+                environment,
+                deployment,
+                run_cache,
+                release_lookups,
+                mainline,
             )
             if run is not None:
                 observed.append(run)
@@ -1301,6 +1619,8 @@ class GitHubDeployment(DeploymentCapability):
         environment: str,
         deployment: dict[str, typing.Any],
         run_cache: dict[str, tuple[str, str] | None],
+        release_lookups: dict[str, asyncio.Task[RemoteRelease | None]],
+        mainline: frozenset[str],
     ) -> RemoteDeployment | None:
         deployment_id = deployment.get('id')
         sha = deployment.get('sha')
@@ -1318,7 +1638,9 @@ class GitHubDeployment(DeploymentCapability):
         ref_value = deployment.get('ref')
         description = deployment.get('description')
         release_notes = (
-            await self._release_notes_for_ref(client, str(ref_value))
+            await self._release_notes_for_ref(
+                client, str(ref_value), mainline, release_lookups
+            )
             if ref_value
             else None
         )
@@ -1411,18 +1733,53 @@ class GitHubDeployment(DeploymentCapability):
         return result
 
     async def _release_for_ref(
-        self, client: httpx.AsyncClient, ref: str
+        self,
+        client: httpx.AsyncClient,
+        ref: str,
+        mainline: frozenset[str],
+        release_lookups: dict[str, asyncio.Task[RemoteRelease | None]]
+        | None = None,
     ) -> RemoteRelease | None:
         """Return the GitHub release for ``ref``, metadata included.
 
-        The single release lookup every caller degrades through: a 404 /
-        non-200 / parse failure yields ``None``, and a ``403`` (token lacks
-        scope to read releases) is cached process-wide so a forbidden token
-        short-circuits instead of re-issuing the request for every
+        Refs in ``mainline`` (the resolved ``mainline_branches`` option)
+        never name a release, so they never reach the API.
+
+        ``release_lookups`` memoises one lookup per ref for the duration of
+        a sweep.  It holds the in-flight task rather than its result, so
+        the parallel per-env fan-out in
+        :meth:`list_recent_deployments` coalesces onto a single request
+        when several environments deployed the same ref -- checking a
+        result-only cache would let every env past the check before the
+        first response landed.  Like ``run_cache`` it memoises every
+        outcome, failures included, so one sweep never retries a ref that
+        just errored; it is scoped the same way (one sweep, one repo, one
+        token), and omitting it disables the memo for one-off host-facing
+        lookups.
+        """
+        if ref in mainline:
+            return None
+        if release_lookups is None:
+            return await self._fetch_release(client, ref)
+        task = release_lookups.get(ref)
+        if task is None:
+            task = asyncio.ensure_future(self._fetch_release(client, ref))
+            release_lookups[ref] = task
+        return await task
+
+    async def _fetch_release(
+        self, client: httpx.AsyncClient, ref: str
+    ) -> RemoteRelease | None:
+        """Issue the release lookup for ``ref`` and shape the response.
+
+        The single release request every caller degrades through: a 404 /
+        410 / non-200 / parse failure yields ``None``, and a ``403`` (token
+        lacks scope to read releases) is cached process-wide so a forbidden
+        token short-circuits instead of re-issuing the request for every
         deployment on every resync sweep.  ``author`` is the login GitHub
-        credits with the release and ``author_subject`` its numeric user id,
-        which the host resolves to an Imbi user through the identity plugins
-        on the same service.
+        credits with the release and ``author_subject`` its numeric user
+        id, which the host resolves to an Imbi user through the identity
+        plugins on the same service.
         """
         if _releases_forbidden(client):
             return None
@@ -1464,11 +1821,17 @@ class GitHubDeployment(DeploymentCapability):
         tag: str,
     ) -> RemoteRelease | None:
         """Return the GitHub release for ``tag`` with its author."""
+        mainline = _mainline_branches(ctx.integration_options)
         async with self._client(ctx, credentials) as client:
-            return await self._release_for_ref(client, tag)
+            return await self._release_for_ref(client, tag, mainline)
 
     async def _release_notes_for_ref(
-        self, client: httpx.AsyncClient, ref: str
+        self,
+        client: httpx.AsyncClient,
+        ref: str,
+        mainline: frozenset[str],
+        release_lookups: dict[str, asyncio.Task[RemoteRelease | None]]
+        | None = None,
     ) -> str | None:
         """Return the GitHub release notes body for a deployed ref.
 
@@ -1479,7 +1842,9 @@ class GitHubDeployment(DeploymentCapability):
         ``None`` -- resync is never blocked by a missing or unreadable
         release.
         """
-        release = await self._release_for_ref(client, ref)
+        release = await self._release_for_ref(
+            client, ref, mainline, release_lookups
+        )
         return release.body_markdown if release else None
 
     async def _latest_status(

{imbi_plugin_github-2.24.0 → imbi_plugin_github-2.26.0}/src/imbi/plugins/github/plugin.py

@@ -81,6 +81,22 @@ _OPTIONS: list[PluginOption] = [
         ),
         type='string',
     ),
+    PluginOption(
+        name='mainline_branches',
+        label='Mainline branches (space-separated)',
+        description=(
+            'Branch names your repos deploy from, which therefore never '
+            'name a GitHub Release.  Deployment resync skips the release '
+            'lookup for a deployment whose ref is one of these instead of '
+            'issuing a request that is certain to 404 -- a repo deploying '
+            'off its default branch would otherwise pay for one doomed '
+            'lookup per deployment row.  Commas or spaces both separate; '
+            'leave blank for the ``main master`` default.'
+        ),
+        type='string',
+        required=False,
+        default='main master',
+    ),
 ]
 
 # The single credential store for the Integration. Every capability of