airbyte-source-github 2.1.12__tar.gz → 2.1.13__tar.gz

{airbyte_source_github-2.1.12 → airbyte_source_github-2.1.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 2.1.12
+Version: 2.1.13
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_github-2.1.12 → airbyte_source_github-2.1.13}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.1.12"
+version = "2.1.13"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]

{airbyte_source_github-2.1.12 → airbyte_source_github-2.1.13}/source_github/graphql.py

@@ -165,6 +165,54 @@ def get_query_issue_reactions(owner, name, first, after, number=None):
     return str(op)
 
 
+def get_query_releases(owner, name, first, after):
+    kwargs = {"first": first, "order_by": {"field": "CREATED_AT", "direction": "ASC"}}
+    if after:
+        kwargs["after"] = after
+
+    op = sgqlc.operation.Operation(_schema_root.query_type)
+    repository = op.repository(owner=owner, name=name)
+    repository.name()
+    repository.owner.login()
+    releases = repository.releases(**kwargs)
+    releases.nodes.__fields__(
+        id="node_id",
+        database_id="id",
+        name=True,
+        tag_name="tag_name",
+        created_at="created_at",
+        published_at="published_at",
+        updated_at="updated_at",
+        is_draft="draft",
+        is_prerelease="prerelease",
+        description="body",
+        description_html="body_html",
+        url="html_url",
+    )
+    releases.nodes.tag_commit.oid(__alias__="target_commitish")
+    author = releases.nodes.author(__alias__="author").__as__(_schema_root.User)
+    select_user_fields(author)
+    release_assets = releases.nodes.release_assets(first=100, __alias__="assets")
+    release_assets.nodes.__fields__(
+        id="node_id",
+        name=True,
+        content_type="content_type",
+        size=True,
+        download_count="download_count",
+        created_at="created_at",
+        updated_at="updated_at",
+        download_url="browser_download_url",
+        url=True,
+    )
+    release_assets.nodes.uploaded_by(__alias__="uploader").__as__(_schema_root.User).__fields__(database_id="id")
+    release_assets.page_info.__fields__(has_next_page=True)
+    releases.nodes.reaction_groups(__alias__="reaction_groups").__fields__(content=True)
+    releases.nodes.reaction_groups(__alias__="reaction_groups").reactors.__fields__(total_count=True)
+    releases.nodes.mentions(first=0, __alias__="mentions_connection").total_count()
+    releases.page_info.__fields__(has_next_page=True, end_cursor=True)
+    return str(op)
+
+
 class QueryReactions:
     # AVERAGE_REVIEWS - optimal number of reviews to fetch inside every pull request.
     # If we try to fetch too many (up to 100) we will spend too many scores of query cost.

{airbyte_source_github-2.1.12 → airbyte_source_github-2.1.13}/source_github/schemas/releases.json

@@ -7,7 +7,7 @@
       "type": "string"
     },
     "url": {
-      "description": "The URL for the release.",
+      "description": "The REST API URL for the release (synthesized from GraphQL data).",
       "type": ["null", "string"]
     },
     "html_url": {
@@ -15,19 +15,19 @@
       "type": ["null", "string"]
     },
     "assets_url": {
-      "description": "The URL to fetch information about the assets linked to this release.",
+      "description": "The URL to fetch information about the assets linked to this release (synthesized from GraphQL data).",
       "type": ["null", "string"]
     },
     "upload_url": {
-      "description": "The URL for uploading assets to the release.",
+      "description": "The URL for uploading assets to the release (synthesized from GraphQL data).",
       "type": ["null", "string"]
     },
     "tarball_url": {
-      "description": "The URL for the tarball file of the release.",
+      "description": "The URL for the tarball file of the release (synthesized from GraphQL data).",
       "type": ["null", "string"]
     },
     "zipball_url": {
-      "description": "The URL for the zipball file of the release.",
+      "description": "The URL for the zipball file of the release (synthesized from GraphQL data).",
       "type": ["null", "string"]
     },
     "id": {
@@ -43,7 +43,7 @@
       "type": ["null", "string"]
     },
     "target_commitish": {
-      "description": "The commit SHA or branch name for the release.",
+      "description": "The commit SHA the release is based on. Note: the GraphQL API returns the commit SHA here, not the branch name returned by the REST API.",
       "type": ["null", "string"]
     },
     "name": {
@@ -72,12 +72,17 @@
       "type": ["null", "string"],
       "format": "date-time"
     },
+    "updated_at": {
+      "description": "The timestamp of when the release was last updated.",
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
     "author": {
       "description": "The author of the release.",
       "$ref": "user.json"
     },
     "assets": {
-      "description": "List of assets (e.g., downloadable files) associated with the release",
+      "description": "List of assets (e.g., downloadable files) associated with the release. Capped at 100 per release.",
       "type": ["null", "array"],
       "items": {
         "description": "Details of an individual asset",
@@ -92,7 +97,7 @@
             "type": ["null", "string"]
           },
           "id": {
-            "description": "The unique identifier for the asset.",
+            "description": "The unique numeric identifier of the asset (extracted from the GraphQL Node ID).",
             "type": ["null", "integer"]
           },
           "node_id": {
@@ -103,14 +108,6 @@
             "description": "The name of the asset.",
             "type": ["null", "string"]
           },
-          "label": {
-            "description": "The label assigned to the asset.",
-            "type": ["null", "string"]
-          },
-          "state": {
-            "description": "The state of the asset.",
-            "type": ["null", "string"]
-          },
           "content_type": {
             "description": "The content type of the asset.",
             "type": ["null", "string"]
@@ -144,18 +141,10 @@
       "description": "The HTML body of the release.",
       "type": ["null", "string"]
     },
-    "body_text": {
-      "description": "The text body of the release.",
-      "type": ["null", "string"]
-    },
     "mentions_count": {
       "description": "The count of mentions in the release.",
       "type": ["null", "integer"]
     },
-    "discussion_url": {
-      "description": "The URL for the discussion related to the release.",
-      "type": ["null", "string"]
-    },
     "reactions": {
       "description": "The reactions associated with the release.",
       "$ref": "reactions.json"

{airbyte_source_github-2.1.12 → airbyte_source_github-2.1.13}/source_github/streams.py

@@ -2,7 +2,9 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+import base64
 import re
+import struct
 from abc import ABC, abstractmethod
 from datetime import timedelta, timezone
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
@@ -39,6 +41,7 @@ from .graphql import (
     get_query_issue_reactions,
     get_query_projectsV2,
     get_query_pull_requests,
+    get_query_releases,
     get_query_reviews,
 )
 from .utils import GitHubAPILimitException, getter
@@ -502,24 +505,6 @@ class Users(Organizations):
 # Below are semi incremental streams
 
 
-class Releases(SemiIncrementalMixin, GithubStream):
-    """
-    API docs: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#list-releases
-    """
-
-    cursor_field = "created_at"
-
-    def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
-        record = super().transform(record=record, stream_slice=stream_slice)
-
-        assets = record.get("assets", [])
-        for asset in assets:
-            uploader = asset.pop("uploader", None)
-            asset["uploader_id"] = uploader.get("id") if uploader else None
-
-        return record
-
-
 class Events(SemiIncrementalMixin, GithubStream):
     """
     API docs: https://docs.github.com/en/rest/activity/events?apiVersion=2022-11-28#list-repository-events
@@ -808,6 +793,132 @@ class GitHubGraphQLStream(GithubStream, ABC):
         return {}
 
 
+class Releases(SemiIncrementalMixin, GitHubGraphQLStream):
+    """
+    API docs: https://docs.github.com/en/graphql/reference/objects#release
+    Uses GraphQL API to avoid the REST API's 10,000 result pagination limit.
+    """
+
+    cursor_field = "created_at"
+    is_sorted = "asc"
+
+    GRAPHQL_REACTION_TO_REST = {
+        "THUMBS_UP": "plus_one",
+        "THUMBS_DOWN": "minus_one",
+        "LAUGH": "laugh",
+        "HOORAY": "hooray",
+        "CONFUSED": "confused",
+        "HEART": "heart",
+        "ROCKET": "rocket",
+        "EYES": "eyes",
+    }
+
+    @staticmethod
+    def _extract_database_id_from_node_id(node_id: str) -> Optional[int]:
+        """Extract the numeric database ID from a GitHub GraphQL Node ID.
+
+        GitHub Node IDs with type prefixes (e.g. 'RA_...') are URL-safe base64
+        encodings of a msgpack array: [type_flag, repo_database_id, entity_database_id].
+        The last 4 bytes encode the entity's numeric database ID as a big-endian uint32.
+        """
+        if not node_id or "_" not in node_id:
+            return None
+        try:
+            encoded = node_id.split("_", 1)[1]
+            decoded = base64.urlsafe_b64decode(encoded + "==")
+            if len(decoded) >= 4:
+                return struct.unpack(">I", decoded[-4:])[0]
+        except Exception:
+            return None
+        return None
+
+    def _get_assets_from_release(self, record: Mapping) -> list:
+        assets_data = record.get("assets", {})
+        if assets_data.get("pageInfo", {}).get("hasNextPage"):
+            self.logger.warning(
+                "Release %s in %s has >100 assets; only the first 100 were synced. "
+                "Sub-pagination for release assets is not yet implemented.",
+                record.get("id"),
+                record.get("repository"),
+            )
+        assets = assets_data.get("nodes", [])
+        for asset in assets:
+            uploader = asset.pop("uploader", None)
+            asset["uploader_id"] = uploader.get("id") if uploader else None
+            asset["id"] = self._extract_database_id_from_node_id(asset.get("node_id"))
+        return assets
+
+    def _get_reactions_from_release(self, record: Mapping) -> Optional[Mapping]:
+        reaction_groups = record.pop("reaction_groups", None)
+        if reaction_groups is None:
+            return None
+        reactions = {rest_key: 0 for rest_key in self.GRAPHQL_REACTION_TO_REST.values()}
+        total = 0
+        for group in reaction_groups:
+            content = group.get("content")
+            count = group.get("reactors", {}).get("totalCount", 0)
+            rest_key = self.GRAPHQL_REACTION_TO_REST.get(content)
+            if rest_key:
+                reactions[rest_key] = count
+                total += count
+        reactions["total_count"] = total
+        return reactions
+
+    def _build_rest_urls(self, repository: str, release_id: int, tag_name: str) -> Mapping[str, str]:
+        """Synthesize REST-compatible URL fields from GraphQL data.
+
+        The GraphQL API does not return reference URLs the way the REST API does,
+        but we can construct them from the repository, release ID, and tag name
+        to retain backwards compatibility with the previous REST-based schema.
+        """
+        api_url = self.api_url.rstrip("/")
+        upload_url = api_url.replace("api.github.com", "uploads.github.com")
+        return {
+            "url": f"{api_url}/repos/{repository}/releases/{release_id}",
+            "assets_url": f"{api_url}/repos/{repository}/releases/{release_id}/assets",
+            "upload_url": f"{upload_url}/repos/{repository}/releases/{release_id}/assets{{?name,label}}",
+            "tarball_url": f"{api_url}/repos/{repository}/tarball/{tag_name}" if tag_name else None,
+            "zipball_url": f"{api_url}/repos/{repository}/zipball/{tag_name}" if tag_name else None,
+        }
+
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        repository = response.json().get("data", {}).get("repository")
+        if repository:
+            nodes = repository.get("releases", {}).get("nodes", [])
+            for record in nodes:
+                record["repository"] = self._get_repository_name(repository)
+                if record.get("author"):
+                    record["author"]["type"] = record["author"].pop("__typename", "User")
+                record["assets"] = self._get_assets_from_release(record)
+                record["reactions"] = self._get_reactions_from_release(record)
+                mentions_connection = record.pop("mentions_connection", None)
+                if mentions_connection is not None:
+                    record["mentions_count"] = mentions_connection.get("totalCount", 0)
+                tag_commit = record.pop("tagCommit", None)
+                record["target_commitish"] = tag_commit.get("target_commitish") if tag_commit else None
+                record.update(self._build_rest_urls(record["repository"], record.get("id"), record.get("tag_name")))
+                yield record
+
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        repository = response.json().get("data", {}).get("repository")
+        if repository:
+            page_info = repository.get("releases", {}).get("pageInfo", {})
+            if page_info.get("hasNextPage"):
+                return {"after": page_info["endCursor"]}
+        return None
+
+    def request_body_json(
+        self,
+        stream_state: Mapping[str, Any],
+        stream_slice: Mapping[str, Any] = None,
+        next_page_token: Mapping[str, Any] = None,
+    ) -> Optional[Mapping]:
+        organization, name = stream_slice["repository"].split("/")
+        after = next_page_token["after"] if next_page_token else None
+        query = get_query_releases(owner=organization, name=name, first=self.page_size, after=after)
+        return {"query": query}
+
+
 class PullRequestStats(SemiIncrementalMixin, GitHubGraphQLStream):
     """
     API docs: https://docs.github.com/en/graphql/reference/objects#pullrequest