airbyte-source-github 2.1.10__tar.gz → 2.1.10.dev202602202300__tar.gz

{airbyte_source_github-2.1.10 → airbyte_source_github-2.1.10.dev202602202300}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 2.1.10
+Version: 2.1.10.dev202602202300
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_github-2.1.10 → airbyte_source_github-2.1.10.dev202602202300}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.1.10"
+version = "2.1.10.dev.202602202300"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]

{airbyte_source_github-2.1.10 → airbyte_source_github-2.1.10.dev202602202300}/source_github/graphql.py

@@ -165,6 +165,51 @@ def get_query_issue_reactions(owner, name, first, after, number=None):
     return str(op)
 
 
+def get_query_releases(owner, name, first, after, direction):
+    kwargs = {"first": first, "order_by": {"field": "CREATED_AT", "direction": direction}}
+    if after:
+        kwargs["after"] = after
+
+    op = sgqlc.operation.Operation(_schema_root.query_type)
+    repository = op.repository(owner=owner, name=name)
+    repository.name()
+    repository.owner.login()
+    releases = repository.releases(**kwargs)
+    releases.nodes.__fields__(
+        id="node_id",
+        database_id="id",
+        name=True,
+        tag_name="tag_name",
+        created_at="created_at",
+        published_at="published_at",
+        updated_at="updated_at",
+        is_draft="draft",
+        is_prerelease="prerelease",
+        description="body",
+        description_html="body_html",
+        url="html_url",
+    )
+    releases.nodes.tag_commit.oid(__alias__="target_commitish")
+    author = releases.nodes.author(__alias__="author").__as__(_schema_root.User)
+    select_user_fields(author)
+    release_assets = releases.nodes.release_assets(first=100, __alias__="assets")
+    release_assets.nodes.__fields__(
+        id="node_id",
+        name=True,
+        content_type="content_type",
+        size=True,
+        download_count="download_count",
+        created_at="created_at",
+        updated_at="updated_at",
+        download_url="browser_download_url",
+        url=True,
+    )
+    release_assets.nodes.uploaded_by(__alias__="uploader").__as__(_schema_root.User).__fields__(database_id="id")
+    releases.nodes.mentions(first=0, __alias__="mentions_connection").total_count()
+    releases.page_info.__fields__(has_next_page=True, end_cursor=True)
+    return str(op)
+
+
 class QueryReactions:
     # AVERAGE_REVIEWS - optimal number of reviews to fetch inside every pull request.
     # If we try to fetch too many (up to 100) we will spend too many scores of query cost.

{airbyte_source_github-2.1.10 → airbyte_source_github-2.1.10.dev202602202300}/source_github/streams.py

@@ -39,6 +39,7 @@ from .graphql import (
     get_query_issue_reactions,
     get_query_projectsV2,
     get_query_pull_requests,
+    get_query_releases,
     get_query_reviews,
 )
 from .utils import GitHubAPILimitException, getter
@@ -502,24 +503,6 @@ class Users(Organizations):
 # Below are semi incremental streams
 
 
-class Releases(SemiIncrementalMixin, GithubStream):
-    """
-    API docs: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#list-releases
-    """
-
-    cursor_field = "created_at"
-
-    def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
-        record = super().transform(record=record, stream_slice=stream_slice)
-
-        assets = record.get("assets", [])
-        for asset in assets:
-            uploader = asset.pop("uploader", None)
-            asset["uploader_id"] = uploader.get("id") if uploader else None
-
-        return record
-
-
 class Events(SemiIncrementalMixin, GithubStream):
     """
     API docs: https://docs.github.com/en/rest/activity/events?apiVersion=2022-11-28#list-repository-events
@@ -808,6 +791,55 @@ class GitHubGraphQLStream(GithubStream, ABC):
         return {}
 
 
+class Releases(SemiIncrementalMixin, GitHubGraphQLStream):
+    """
+    API docs: https://docs.github.com/en/graphql/reference/objects#release
+    Uses GraphQL API to avoid the REST API's 10,000 result pagination limit.
+    """
+
+    cursor_field = "created_at"
+    is_sorted = "asc"
+
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        repository = response.json().get("data", {}).get("repository")
+        if repository:
+            nodes = repository.get("releases", {}).get("nodes", [])
+            for record in nodes:
+                record["repository"] = self._get_repository_name(repository)
+                if record.get("author"):
+                    record["author"]["type"] = record["author"].pop("__typename", "User")
+                assets = record.get("assets", {}).get("nodes", [])
+                for asset in assets:
+                    uploader = asset.pop("uploader", None)
+                    asset["uploader_id"] = uploader.get("id") if uploader else None
+                record["assets"] = assets
+                mentions_connection = record.pop("mentions_connection", None)
+                if mentions_connection is not None:
+                    record["mentions_count"] = mentions_connection.get("totalCount", 0)
+                tag_commit = record.pop("tagCommit", record.pop("tag_commit", None))
+                if tag_commit:
+                    record["target_commitish"] = tag_commit.get("target_commitish", tag_commit.get("oid"))
+                yield record
+
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        repository = response.json().get("data", {}).get("repository")
+        if repository:
+            page_info = repository.get("releases", {}).get("pageInfo", {})
+            if page_info.get("hasNextPage"):
+                return {"after": page_info["endCursor"]}
+
+    def request_body_json(
+        self,
+        stream_state: Mapping[str, Any],
+        stream_slice: Mapping[str, Any] = None,
+        next_page_token: Mapping[str, Any] = None,
+    ) -> Optional[Mapping]:
+        organization, name = stream_slice["repository"].split("/")
+        after = next_page_token["after"] if next_page_token else None
+        query = get_query_releases(owner=organization, name=name, first=self.page_size, after=after, direction=self.is_sorted.upper())
+        return {"query": query}
+
+
 class PullRequestStats(SemiIncrementalMixin, GitHubGraphQLStream):
     """
     API docs: https://docs.github.com/en/graphql/reference/objects#pullrequest