airbyte-source-github 2.1.27__tar.gz → 2.1.28__tar.gz

{airbyte_source_github-2.1.27 → airbyte_source_github-2.1.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 2.1.27
+Version: 2.1.28
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_github-2.1.27 → airbyte_source_github-2.1.28}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.1.27"
+version = "2.1.28"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]

{airbyte_source_github-2.1.27 → airbyte_source_github-2.1.28}/source_github/errors_handlers.py

@@ -199,11 +199,25 @@ class GitHubGraphQLErrorHandler(GithubStreamABCErrorHandler):
     def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
         if isinstance(response_or_exception, requests.Response):
             if response_or_exception.status_code in (requests.codes.BAD_GATEWAY, requests.codes.GATEWAY_TIMEOUT):
-                self.stream.page_size = int(self.stream.page_size / 2)
+                # Halve the page size on every 502/504 to reduce GraphQL query cost,
+                # but never let it drop below 1 — a page_size of 0 would request no
+                # records and cause infinite paging.
+                previous_page_size = self.stream.page_size
+                self.stream.page_size = max(1, int(self.stream.page_size / 2))
+                self._logger.info(
+                    "GitHub GraphQL endpoint returned HTTP %s for stream `%s`; reducing GraphQL page_size from %s to %s and retrying.",
+                    response_or_exception.status_code,
+                    self.stream.name,
+                    previous_page_size,
+                    self.stream.page_size,
+                )
                 return ErrorResolution(
                     response_action=ResponseAction.RETRY,
                     failure_type=FailureType.transient_error,
-                    error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
+                    error_message=(
+                        f"GitHub GraphQL endpoint returned HTTP {response_or_exception.status_code} "
+                        f"for stream `{self.stream.name}`. Reducing GraphQL page size and retrying."
+                    ),
                 )
 
             self.stream.page_size = (
@@ -214,7 +228,10 @@ class GitHubGraphQLErrorHandler(GithubStreamABCErrorHandler):
                 return ErrorResolution(
                     response_action=ResponseAction.RETRY,
                     failure_type=FailureType.transient_error,
-                    error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
+                    error_message=(
+                        f"GitHub GraphQL endpoint returned errors in the response body "
+                        f"for stream `{self.stream.name}` (HTTP {response_or_exception.status_code}). Retrying."
+                    ),
                 )
 
         return super().interpret_response(response_or_exception)

{airbyte_source_github-2.1.27 → airbyte_source_github-2.1.28}/source_github/streams.py

@@ -201,6 +201,12 @@ class GithubStreamABC(HttpStream, ABC):
                     f"GitHub returned HTTP 502 Bad Gateway for stream `{self.name}` after exhausting retries. "
                     f"This is usually transient — the next sync attempt should succeed."
                 )
+            elif e._exception.response.status_code == requests.codes.GATEWAY_TIMEOUT:
+                error_msg = (
+                    f"GitHub returned HTTP 504 Gateway Timeout for stream `{self.name}` after exhausting retries "
+                    f"and reducing the GraphQL page size. The next sync attempt should succeed; "
+                    f'if 504s persist, lower "Page size for large streams" in the source configuration.'
+                )
             else:
                 self.logger.error(f"Undefined error while reading records: {e._exception.response.text}")
                 raise e
@@ -232,11 +238,15 @@ class GithubStream(GithubStreamABC):
     def get_error_display_message(self, exception: BaseException) -> Optional[str]:
         if (
             isinstance(exception, DefaultBackoffException)
-            and exception.response.status_code == requests.codes.BAD_GATEWAY
+            and exception.response.status_code in (requests.codes.BAD_GATEWAY, requests.codes.GATEWAY_TIMEOUT)
             and self.large_stream
             and self.page_size > 1
         ):
-            return f'Please try to decrease the "Page size for large streams" below {self.page_size}. The stream "{self.name}" is a large stream, such streams can fail with 502 for high "page_size" values.'
+            return (
+                f'Please try to decrease the "Page size for large streams" below {self.page_size}. '
+                f'The stream "{self.name}" is a large stream, such streams can fail with '
+                f'{exception.response.status_code} for high "page_size" values.'
+            )
         return super().get_error_display_message(exception)
 
     def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
@@ -850,6 +860,16 @@ class Releases(SemiIncrementalMixin, GitHubGraphQLStream):
 
     cursor_field = "created_at"
     is_sorted = "asc"
+    # The Releases GraphQL query is high-cost on the server side: every node
+    # materializes `description` and `descriptionHTML`, which forces GitHub
+    # to render each release body to HTML. On repositories with long release
+    # notes, a page_size of 100 pushes the resolver past its internal 10s
+    # deadline and returns 504 Gateway Timeout (reproduced deterministically
+    # against nodejs/node: first=100 -> 504 in ~11s; first=10 -> 200 in ~3s).
+    # `releaseAssets` / `reactionGroups` / `mentions` are NOT the cost driver
+    # — stripping them does not fix the timeout, only lowering `first` does.
+    # Mark as large_stream so it picks up the smaller default page size.
+    large_stream = True
 
     GRAPHQL_REACTION_TO_REST = {
         "THUMBS_UP": "plus_one",