airbyte-source-github 2.2.0.dev202601122028__tar.gz → 2.2.1__tar.gz

airbyte_source_github-2.2.1/.airbyte-pypi-readme.md

@@ -0,0 +1,7 @@
+# GitHub
+
+This package contains the Airbyte GitHub connector.
+
+Airbyte is open-source data movement for ELT pipelines and AI agents. It moves data from APIs, databases, and files to warehouses, lakes, and AI applications.
+
+Airbyte provides a catalog of 600+ connectors for APIs, databases, data warehouses, data lakes, and AI applications. Use this package with [PyAirbyte](https://docs.airbyte.com/developers/pyairbyte) to run the connector from Python.

airbyte_source_github-2.2.1/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.1
+Name: airbyte-source-github
+Version: 2.2.1
+Summary: Source implementation for GitHub.
+Home-page: https://airbyte.com
+License: ELv2
+Author: Airbyte
+Author-email: contact@airbyte.io
+Requires-Python: >=3.10,<3.14
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: airbyte-cdk (>=7.28.2,<8.0.0)
+Requires-Dist: sgqlc (==16.3)
+Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/github
+Project-URL: Repository, https://github.com/airbytehq/airbyte
+Description-Content-Type: text/markdown
+
+# GitHub
+
+This package contains the Airbyte GitHub connector.
+
+Airbyte is open-source data movement for ELT pipelines and AI agents. It moves data from APIs, databases, and files to warehouses, lakes, and AI applications.
+
+Airbyte provides a catalog of 600+ connectors for APIs, databases, data warehouses, data lakes, and AI applications. Use this package with [PyAirbyte](https://docs.airbyte.com/developers/pyairbyte) to run the connector from Python.
+

{airbyte_source_github-2.2.0.dev202601122028 → airbyte_source_github-2.2.1}/pyproject.toml

@@ -3,12 +3,12 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.2.0.dev.202601122028"
+version = "2.2.1"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]
 license = "ELv2"
-readme = "README.md"
+readme = ".airbyte-pypi-readme.md"
 documentation = "https://docs.airbyte.com/integrations/sources/github"
 homepage = "https://airbyte.com"
 repository = "https://github.com/airbytehq/airbyte"
@@ -17,7 +17,7 @@ include = "source_github"
 
 [tool.poetry.dependencies]
 python = "^3.10,<3.14"
-airbyte-cdk = "^7.4.1"
+airbyte-cdk = "^7.28.2"
 sgqlc = "==16.3"
 
 [tool.poetry.scripts]

airbyte_source_github-2.2.1/source_github/backoff_strategies.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import time
+from typing import Any, Optional, Union
+
+import requests
+
+from airbyte_cdk import BackoffStrategy
+from airbyte_cdk.sources.streams.http import HttpStream
+
+from .errors_handlers import is_rate_limited_response
+
+
+class GithubStreamABCBackoffStrategy(BackoffStrategy):
+    def __init__(self, stream: HttpStream, max_wait_time_seconds: float = 120 * 60, **kwargs):  # type: ignore # noqa
+        self.stream = stream
+        self.max_wait_time_seconds = max_wait_time_seconds
+        super().__init__(**kwargs)
+
+    def backoff_time(
+        self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], **kwargs: Any
+    ) -> Optional[float]:
+        if isinstance(response_or_exception, requests.Response):
+            if not is_rate_limited_response(
+                response_or_exception,
+                self.stream.check_graphql_rate_limited,
+                self.stream.logger,
+            ):
+                return None
+
+            min_backoff_time = 60.0
+            retry_after = response_or_exception.headers.get("Retry-After")
+            if retry_after is not None:
+                wait_time = max(float(retry_after), min_backoff_time)
+            else:
+                reset_time = response_or_exception.headers.get("X-RateLimit-Reset")
+                if not reset_time:
+                    return None
+                wait_time = max(float(reset_time) - time.time(), min_backoff_time)
+
+            if wait_time > self.max_wait_time_seconds:
+                self.stream.logger.warning(
+                    "Rate limit wait for stream `%s` is %.2f seconds, exceeding the maximum of %.2f seconds; using default backoff.",
+                    self.stream.name,
+                    wait_time,
+                    self.max_wait_time_seconds,
+                )
+                return None
+            return wait_time
+        return None
+
+
+class ContributorActivityBackoffStrategy(BackoffStrategy):
+    def backoff_time(
+        self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], **kwargs: Any
+    ) -> Optional[float]:
+        if isinstance(response_or_exception, requests.Response) and response_or_exception.status_code == requests.codes.ACCEPTED:
+            return 90
+        return None

airbyte_source_github-2.2.1/source_github/errors_handlers.py

@@ -0,0 +1,244 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from typing import Callable, Optional, Union
+
+import requests
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.streams.http import HttpStream
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
+
+from . import constants
+
+
+logger = logging.getLogger("airbyte")
+
+
+GITHUB_DEFAULT_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | {
+    401: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.config_error,
+        error_message="Conflict.",
+    ),
+    403: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.config_error,
+        error_message=(
+            "GitHub denied access (HTTP 403). Your token may be missing required scopes "
+            "(this connector typically needs: repo, read:org, read:user, read:project, workflow), "
+            "or this organization may require SAML SSO authorization. "
+            "See https://docs.github.com/en/rest/using-the-rest-api/troubleshooting-the-rest-api"
+        ),
+    ),
+    404: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.config_error,
+        error_message="Conflict.",
+    ),
+    409: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.config_error,
+        error_message="Conflict.",
+    ),
+    410: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.config_error,
+        error_message=(
+            "GitHub returned 410 Gone for an unexpected reason. "
+            "The endpoint or API version may be deprecated. "
+            "Verify the connector version is current and the endpoint is still supported."
+        ),
+    ),
+}
+
+
+def is_conflict_with_empty_repository(response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> bool:
+    if isinstance(response_or_exception, requests.Response) and response_or_exception.status_code == requests.codes.CONFLICT:
+        try:
+            response_data = response_or_exception.json()
+        except ValueError:
+            logger.warning(
+                "is_conflict_with_empty_repository received non-JSON 409 response (first 50 chars: %r).",
+                response_or_exception.text[:50],
+            )
+            return False
+        return response_data.get("message") == "Git Repository is empty."
+    return False
+
+
+def is_gone_with_feature_disabled(response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> bool:
+    if isinstance(response_or_exception, requests.Response) and response_or_exception.status_code == requests.codes.GONE:
+        try:
+            message = (response_or_exception.json().get("message") or "").lower()
+        except ValueError:
+            logger.warning(
+                "is_gone_with_feature_disabled received non-JSON 410 response (first 50 chars: %r).",
+                response_or_exception.text[:50],
+            )
+            return False
+        return "are disabled" in message or "is disabled" in message
+    return False
+
+
+def is_rate_limited_response(
+    response: requests.Response, graphql_rate_limit_checker: Callable[[dict], bool], logger: logging.Logger
+) -> bool:
+    # GitHub GraphQL resource limitations:
+    # https://docs.github.com/en/graphql/overview/resource-limitations
+    if response.headers.get("X-RateLimit-Resource") == "graphql":
+        try:
+            body = response.json()
+        except ValueError:
+            logger.warning(
+                "GraphQL rate-limit check received non-JSON response (HTTP %s, first 50 chars: %r).",
+                response.status_code,
+                response.text[:50],
+            )
+            graphql_rate_limited = False
+        else:
+            graphql_rate_limited = graphql_rate_limit_checker(body or {})
+
+        if graphql_rate_limited:
+            return True
+
+    # GitHub REST rate-limit HTTP headers:
+    # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers
+    # GitHub secondary rate limits:
+    # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits
+    return (response.status_code != 200 and response.headers.get("X-RateLimit-Remaining") == "0") or "Retry-After" in response.headers
+
+
+class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
+    def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
+        self.stream = stream
+        super().__init__(**kwargs)
+
+    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
+        if isinstance(response_or_exception, requests.Response):
+            retry_flag = is_rate_limited_response(
+                response_or_exception,
+                self.stream.check_graphql_rate_limited,
+                self._logger,
+            )
+            if retry_flag:
+                headers = [
+                    "X-RateLimit-Resource",
+                    "X-RateLimit-Remaining",
+                    "X-RateLimit-Reset",
+                    "X-RateLimit-Limit",
+                    "X-RateLimit-Used",
+                    "Retry-After",
+                ]
+                string_headers = ", ".join(
+                    [f"{h}: {response_or_exception.headers[h]}" for h in headers if h in response_or_exception.headers]
+                )
+                if string_headers:
+                    string_headers = f"HTTP headers: {string_headers},"
+
+                self._logger.info(
+                    f"Rate limit handling for stream `{self.stream.name}` for the response with {response_or_exception.status_code} status code, {string_headers} with message: {response_or_exception.text}"
+                )
+                return ErrorResolution(
+                    response_action=ResponseAction.RATE_LIMITED,
+                    failure_type=FailureType.transient_error,
+                    error_message=(
+                        f"GitHub rate limit hit for stream `{self.stream.name}` "
+                        f"(HTTP {response_or_exception.status_code}). "
+                        f"Waiting for the rate limit window to reset before retrying."
+                    ),
+                )
+
+            if is_conflict_with_empty_repository(response_or_exception=response_or_exception):
+                log_message = (
+                    f"Skipping `{self.stream.name}` for this repository: GitHub returned 409 Conflict "
+                    f"with message 'Git Repository is empty.' This means the repository has no commits."
+                )
+                return ErrorResolution(
+                    response_action=ResponseAction.IGNORE,
+                    failure_type=FailureType.config_error,
+                    error_message=log_message,
+                )
+
+            if is_gone_with_feature_disabled(response_or_exception=response_or_exception):
+                log_message = f"Skipping stream slice for '{response_or_exception.url}': {response_or_exception.json().get('message', 'Feature disabled')}."
+                return ErrorResolution(
+                    response_action=ResponseAction.IGNORE,
+                    failure_type=FailureType.config_error,
+                    error_message=log_message,
+                )
+
+        return super().interpret_response(response_or_exception)
+
+
+class ContributorActivityErrorHandler(GithubStreamABCErrorHandler):
+    """
+    This custom error handler is needed for streams based on repository statistics endpoints like ContributorActivity because
+    when requesting data that hasn't been cached yet when the request is made, you'll receive a 202 response. And these requests
+    need to retried to get the actual results.
+
+    See the docs for more info:
+    https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#a-word-about-caching
+    """
+
+    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
+        if isinstance(response_or_exception, requests.Response) and response_or_exception.status_code == requests.codes.ACCEPTED:
+            return ErrorResolution(
+                response_action=ResponseAction.RETRY,
+                failure_type=FailureType.transient_error,
+                error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
+            )
+
+        return super().interpret_response(response_or_exception)
+
+
+class GitHubGraphQLErrorHandler(GithubStreamABCErrorHandler):
+    def _safe_json_get_errors(self, response: requests.Response) -> bool:
+        try:
+            body = response.json()
+        except ValueError:
+            return False
+        return bool((body or {}).get("errors"))
+
+    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
+        if isinstance(response_or_exception, requests.Response):
+            if response_or_exception.status_code in (requests.codes.BAD_GATEWAY, requests.codes.GATEWAY_TIMEOUT):
+                # Halve the page size on every 502/504 to reduce GraphQL query cost,
+                # but never let it drop below 1 — a page_size of 0 would request no
+                # records and cause infinite paging.
+                previous_page_size = self.stream.page_size
+                self.stream.page_size = max(1, int(self.stream.page_size / 2))
+                self._logger.info(
+                    "GitHub GraphQL endpoint returned HTTP %s for stream `%s`; reducing GraphQL page_size from %s to %s and retrying.",
+                    response_or_exception.status_code,
+                    self.stream.name,
+                    previous_page_size,
+                    self.stream.page_size,
+                )
+                return ErrorResolution(
+                    response_action=ResponseAction.RETRY,
+                    failure_type=FailureType.transient_error,
+                    error_message=(
+                        f"GitHub GraphQL endpoint returned HTTP {response_or_exception.status_code} "
+                        f"for stream `{self.stream.name}`. Reducing GraphQL page size and retrying."
+                    ),
+                )
+
+            self.stream.page_size = (
+                constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM if self.stream.large_stream else constants.DEFAULT_PAGE_SIZE
+            )
+
+            if self._safe_json_get_errors(response_or_exception):
+                return ErrorResolution(
+                    response_action=ResponseAction.RETRY,
+                    failure_type=FailureType.transient_error,
+                    error_message=(
+                        f"GitHub GraphQL endpoint returned errors in the response body "
+                        f"for stream `{self.stream.name}` (HTTP {response_or_exception.status_code}). Retrying."
+                    ),
+                )
+
+        return super().interpret_response(response_or_exception)

{airbyte_source_github-2.2.0.dev202601122028 → airbyte_source_github-2.2.1}/source_github/graphql.py

@@ -165,6 +165,54 @@ def get_query_issue_reactions(owner, name, first, after, number=None):
     return str(op)
 
 
+def get_query_releases(owner, name, first, after):
+    kwargs = {"first": first, "order_by": {"field": "CREATED_AT", "direction": "ASC"}}
+    if after:
+        kwargs["after"] = after
+
+    op = sgqlc.operation.Operation(_schema_root.query_type)
+    repository = op.repository(owner=owner, name=name)
+    repository.name()
+    repository.owner.login()
+    releases = repository.releases(**kwargs)
+    releases.nodes.__fields__(
+        id="node_id",
+        database_id="id",
+        name=True,
+        tag_name="tag_name",
+        created_at="created_at",
+        published_at="published_at",
+        updated_at="updated_at",
+        is_draft="draft",
+        is_prerelease="prerelease",
+        description="body",
+        description_html="body_html",
+        url="html_url",
+    )
+    releases.nodes.tag_commit.oid(__alias__="target_commitish")
+    author = releases.nodes.author(__alias__="author").__as__(_schema_root.User)
+    select_user_fields(author)
+    release_assets = releases.nodes.release_assets(first=100, __alias__="assets")
+    release_assets.nodes.__fields__(
+        id="node_id",
+        name=True,
+        content_type="content_type",
+        size=True,
+        download_count="download_count",
+        created_at="created_at",
+        updated_at="updated_at",
+        download_url="browser_download_url",
+        url=True,
+    )
+    release_assets.nodes.uploaded_by(__alias__="uploader").__as__(_schema_root.User).__fields__(database_id="id")
+    release_assets.page_info.__fields__(has_next_page=True)
+    releases.nodes.reaction_groups(__alias__="reaction_groups").__fields__(content=True)
+    releases.nodes.reaction_groups(__alias__="reaction_groups").reactors.__fields__(total_count=True)
+    releases.nodes.mentions(first=0, __alias__="mentions_connection").total_count()
+    releases.page_info.__fields__(has_next_page=True, end_cursor=True)
+    return str(op)
+
+
 class QueryReactions:
     # AVERAGE_REVIEWS - optimal number of reviews to fetch inside every pull request.
     # If we try to fetch too many (up to 100) we will spend too many scores of query cost.