airbyte-source-github 2.2.0.dev202601122124__py3-none-any.whl → 2.2.2__py3-none-any.whl

airbyte_source_github-2.2.2.dist-info/METADATA

@@ -0,0 +1,29 @@
+Metadata-Version: 2.1
+Name: airbyte-source-github
+Version: 2.2.2
+Summary: Source implementation for GitHub.
+Home-page: https://airbyte.com
+License: ELv2
+Author: Airbyte
+Author-email: contact@airbyte.io
+Requires-Python: >=3.10,<3.14
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: airbyte-cdk (>=7.28.2,<8.0.0)
+Requires-Dist: sgqlc (==16.3)
+Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/github
+Project-URL: Repository, https://github.com/airbytehq/airbyte
+Description-Content-Type: text/markdown
+
+# GitHub
+
+This package contains the Airbyte GitHub connector.
+
+Airbyte is open-source data movement for ELT pipelines and AI agents. It moves data from APIs, databases, and files to warehouses, lakes, and AI applications.
+
+Airbyte provides a catalog of 600+ connectors for APIs, databases, data warehouses, data lakes, and AI applications. Use this package with [PyAirbyte](https://docs.airbyte.com/developers/pyairbyte) to run the connector from Python.
+

{airbyte_source_github-2.2.0.dev202601122124.dist-info → airbyte_source_github-2.2.2.dist-info}/RECORD

@@ -1,11 +1,12 @@
 source_github/__init__.py,sha256=8hLLjFC1wI1mMU442Y5qCcND9qIwyhaANQwT-GAIK0s,1135
-source_github/backoff_strategies.py,sha256=ChkWEmqZL7Qrr1zidOfjZWlMWIVRVO2yqZ7QE_HaZP8,2287
+source_github/backoff_strategies.py,sha256=H1n3xNkkrlto7MqJY-Yp0el2G5S4yuJ7z3kg6aimab0,2343
 source_github/config_migrations.py,sha256=guUJAdNP-liUciVaJB4ackEEMCN4jd7bNd85QFmDNlU,3932
 source_github/constants.py,sha256=Hj3Q4y7OoU-Iff4m9gEC2CjwmWJYXhNbHVNjg8EBLmQ,238
-source_github/errors_handlers.py,sha256=nKd3iWnrOjFXqVfqBCWzLLWpVxLpBflv676txxSI6x8,7254
+source_github/errors_handlers.py,sha256=y0SjYuxKo3zBRhk2kZe0NZWsKen4W9x-wD84Q0S3sjU,11211
 source_github/github_schema.py,sha256=2AXmTN_s_VqvZAR1GPqus5HhSXWokS4N7HMQQqefaYw,1600315
-source_github/graphql.py,sha256=edd7EoXwD5rRDHrjm4ZBy5VGkruTScL_Xipy4CBmTFc,11625
-source_github/run.py,sha256=onA-rP2aVhWHvDquKZdR1381CU66rnzqJ7EFMS5dd4Q,407
+source_github/graphql.py,sha256=fGuYzDynYjJOR7u3GX0t6eBJMzdFxKIp4gHP73MCpos,13518
+source_github/manifest.yaml,sha256=JDINDUIMSofsNdDqjHlEES5s9d7k83mVa9mUvwOKcA0,57776
+source_github/run.py,sha256=gfWy8TcMHa0XRwaVUyq4KmslooST38fvfCLuP5D-5Fo,2391
 source_github/schemas/assignees.json,sha256=_WmGIwYmTmEi9WWkvXdFF3CLsMzG2usoIjaqw-tb4NA,2335
 source_github/schemas/branches.json,sha256=3CIkdEi2XVFoVZrpO_U04mMI7-jSb9i-CT3wbIEgPik,2734
 source_github/schemas/collaborators.json,sha256=ZeRsZvj9lUr43hnjX0l84rl2TTeKKbFKk-tmzBYdOmo,3505
@@ -32,8 +33,7 @@ source_github/schemas/pull_request_comment_reactions.json,sha256=shrzajYnEHg_UlU
 source_github/schemas/pull_request_commits.json,sha256=WICcuxUsExJQe1qlXvXVN-VvkqbjWxR_E__WKO-Ta7E,4800
 source_github/schemas/pull_request_stats.json,sha256=harNH5ha6su-2OhxszHOqDKJ34OZzUsL_4nBnaOD16o,3497
 source_github/schemas/pull_requests.json,sha256=Yp9Sq3jgpfBn0YFbcZC_sy06rlJ-AMkCbHNq90KXT_Y,12196
-source_github/schemas/releases.json,sha256=MnoP0oSioKLmXvo6JyuqCoesIOFhtcRli7j6umlkn8k,5140
-source_github/schemas/repositories.json,sha256=WpC5ZjoycnHUP5DNz8ImEXROYAel8DGRkBN10W1mQ2w,13319
+source_github/schemas/releases.json,sha256=3ITMlMFML3ZSfo_EAkVajJdaS_Xxz_8V4LhL9GRwWrY,5121
 source_github/schemas/review_comments.json,sha256=HVH3OE-x_wYVvg0Qxn6rzXOwAGEk0y3De8_VLCgE3mA,4905
 source_github/schemas/reviews.json,sha256=OPqo9Dz6slRGojamq8pBMKKRZgz96kDRkYiUJjJQH_E,2731
 source_github/schemas/shared/events/comment.json,sha256=6dcatpi1gitgcR_JscPllyGGoSb8Pgce5OS9vh1HKP4,7400
@@ -52,14 +52,12 @@ source_github/schemas/team_memberships.json,sha256=ECr0ICzCwcUVp-btBicm0EemVxiOP
 source_github/schemas/teams.json,sha256=_GQOy0IOO9d8BcpnvqIxQLjtPJusfU7fC7IUJmSi6R4,1796
 source_github/schemas/users.json,sha256=by1CEx3MW740S5F54KZfUDc2N1_UZ98Bb3fWlgCtgxc,2530
 source_github/schemas/workflow_jobs.json,sha256=Kk3N3FcCYBqOdR7FYvz2nAAFtLZBuVfR8Lxg-ORD39U,3934
-source_github/schemas/workflow_run_attempts.json,sha256=XDmIsjtzka-ItEonImD3ZATZjxRNkbFo5-MPTtZDARA,19453
 source_github/schemas/workflow_runs.json,sha256=XDmIsjtzka-ItEonImD3ZATZjxRNkbFo5-MPTtZDARA,19453
 source_github/schemas/workflows.json,sha256=gSNw8WZaVKbX4AL97PbjZHzvxcOltXqv9Ao1RNQOFXM,1470
-source_github/source.py,sha256=MGJ5alBYUUsauouTBEv2svoLk8LcI0LmSkqBblAe9fE,14710
-source_github/spec.json,sha256=7LOQm01fP_RvPF-HifhNPJ7i0AxT2LTNPaLAA3uOfNY,7443
-source_github/streams.py,sha256=PyTTBjmB37ClebdIue36zu7qgz1mUO7FZUV53noedRw,82103
-source_github/utils.py,sha256=1mKZzLEaDe0m8BTw8Y-MxCXrJr1bZfaLPeSUbPN3vyQ,7243
-airbyte_source_github-2.2.0.dev202601122124.dist-info/METADATA,sha256=4D0R1s1DRhGA2AvAVDN5o6GU7y-IkemTkHDqx8h_knY,5314
-airbyte_source_github-2.2.0.dev202601122124.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_source_github-2.2.0.dev202601122124.dist-info/entry_points.txt,sha256=gYhqVrTAZvMwuYByg0b_-o115yUFLLcfNxMrLZmiW9k,55
-airbyte_source_github-2.2.0.dev202601122124.dist-info/RECORD,,
+source_github/source.py,sha256=oAwpfXCMwZf8_lbxr49rSm5qzGffhSq0qIbvIFEy2xs,22559
+source_github/streams.py,sha256=MYPvzAEbGk9nyN0mYamFXEAxepbf7O7aN5Hepf-Bj80,88297
+source_github/utils.py,sha256=rkOk8VG6ey1xDl-9t4B99N6HrvTTPCTGhvifrKwYG9Y,706
+airbyte_source_github-2.2.2.dist-info/METADATA,sha256=1YJ8kLVAVSrdY-KZwq86eicGzaX0EwIbKSYIzL0avn4,1248
+airbyte_source_github-2.2.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_source_github-2.2.2.dist-info/entry_points.txt,sha256=gYhqVrTAZvMwuYByg0b_-o115yUFLLcfNxMrLZmiW9k,55
+airbyte_source_github-2.2.2.dist-info/RECORD,,

source_github/backoff_strategies.py

@@ -10,38 +10,47 @@ import requests
 from airbyte_cdk import BackoffStrategy
 from airbyte_cdk.sources.streams.http import HttpStream
 
+from .errors_handlers import is_rate_limited_response
+
 
 class GithubStreamABCBackoffStrategy(BackoffStrategy):
-    def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
+    def __init__(self, stream: HttpStream, max_wait_time_seconds: float = 120 * 60, **kwargs):  # type: ignore # noqa
         self.stream = stream
+        self.max_wait_time_seconds = max_wait_time_seconds
         super().__init__(**kwargs)
 
     def backoff_time(
         self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], **kwargs: Any
     ) -> Optional[float]:
-        # This method is called if we run into the rate limit. GitHub limits requests to 5000 per hour and provides
-        # `X-RateLimit-Reset` header which contains time when this hour will be finished and limits will be reset so
-        # we again could have 5000 per another hour.
         if isinstance(response_or_exception, requests.Response):
+            if not is_rate_limited_response(
+                response_or_exception,
+                self.stream.check_graphql_rate_limited,
+                self.stream.logger,
+            ):
+                return None
+
             min_backoff_time = 60.0
             retry_after = response_or_exception.headers.get("Retry-After")
             if retry_after is not None:
-                backoff_time_in_seconds = max(float(retry_after), min_backoff_time)
-                return self.get_waiting_time(backoff_time_in_seconds)
-
-            reset_time = response_or_exception.headers.get("X-RateLimit-Reset")
-            if reset_time:
-                backoff_time_in_seconds = max(float(reset_time) - time.time(), min_backoff_time)
-                return self.get_waiting_time(backoff_time_in_seconds)
+                wait_time = max(float(retry_after), min_backoff_time)
+            else:
+                reset_time = response_or_exception.headers.get("X-RateLimit-Reset")
+                if not reset_time:
+                    return None
+                wait_time = max(float(reset_time) - time.time(), min_backoff_time)
+
+            if wait_time > self.max_wait_time_seconds:
+                self.stream.logger.warning(
+                    "Rate limit wait for stream `%s` is %.2f seconds, exceeding the maximum of %.2f seconds; using default backoff.",
+                    self.stream.name,
+                    wait_time,
+                    self.max_wait_time_seconds,
+                )
+                return None
+            return wait_time
         return None
 
-    def get_waiting_time(self, backoff_time_in_seconds: Optional[float]) -> Optional[float]:
-        if backoff_time_in_seconds < 60 * 10:  # type: ignore[operator]
-            return backoff_time_in_seconds
-        else:
-            self.stream._http_client._session.auth.update_token()  # New token will be used in next request
-            return 1
-
 
 class ContributorActivityBackoffStrategy(BackoffStrategy):
     def backoff_time(

source_github/errors_handlers.py

@@ -2,7 +2,8 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from typing import Optional, Union
+import logging
+from typing import Callable, Optional, Union
 
 import requests
 
@@ -14,6 +15,9 @@ from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping impor
 from . import constants
 
 
+logger = logging.getLogger("airbyte")
+
+
 GITHUB_DEFAULT_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | {
     401: ErrorResolution(
         response_action=ResponseAction.RETRY,
@@ -21,9 +25,14 @@ GITHUB_DEFAULT_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | {
         error_message="Conflict.",
     ),
     403: ErrorResolution(
-        response_action=ResponseAction.RETRY,
+        response_action=ResponseAction.FAIL,
         failure_type=FailureType.config_error,
-        error_message="Conflict.",
+        error_message=(
+            "GitHub denied access (HTTP 403). Your token may be missing required scopes "
+            "(this connector typically needs: repo, read:org, read:user, read:project, workflow), "
+            "or this organization may require SAML SSO authorization. "
+            "See https://docs.github.com/en/rest/using-the-rest-api/troubleshooting-the-rest-api"
+        ),
     ),
     404: ErrorResolution(
         response_action=ResponseAction.RETRY,
@@ -36,20 +45,73 @@ GITHUB_DEFAULT_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | {
         error_message="Conflict.",
     ),
     410: ErrorResolution(
-        response_action=ResponseAction.RETRY,
+        response_action=ResponseAction.FAIL,
         failure_type=FailureType.config_error,
-        error_message="Gone. Please ensure the url is valid.",
+        error_message=(
+            "GitHub returned 410 Gone for an unexpected reason. "
+            "The endpoint or API version may be deprecated. "
+            "Verify the connector version is current and the endpoint is still supported."
+        ),
     ),
 }
 
 
 def is_conflict_with_empty_repository(response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> bool:
     if isinstance(response_or_exception, requests.Response) and response_or_exception.status_code == requests.codes.CONFLICT:
-        response_data = response_or_exception.json()
+        try:
+            response_data = response_or_exception.json()
+        except ValueError:
+            logger.warning(
+                "is_conflict_with_empty_repository received non-JSON 409 response (first 50 chars: %r).",
+                response_or_exception.text[:50],
+            )
+            return False
         return response_data.get("message") == "Git Repository is empty."
     return False
 
 
+def is_gone_with_feature_disabled(response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> bool:
+    if isinstance(response_or_exception, requests.Response) and response_or_exception.status_code == requests.codes.GONE:
+        try:
+            message = (response_or_exception.json().get("message") or "").lower()
+        except ValueError:
+            logger.warning(
+                "is_gone_with_feature_disabled received non-JSON 410 response (first 50 chars: %r).",
+                response_or_exception.text[:50],
+            )
+            return False
+        return "are disabled" in message or "is disabled" in message
+    return False
+
+
+def is_rate_limited_response(
+    response: requests.Response, graphql_rate_limit_checker: Callable[[dict], bool], logger: logging.Logger
+) -> bool:
+    # GitHub GraphQL resource limitations:
+    # https://docs.github.com/en/graphql/overview/resource-limitations
+    if response.headers.get("X-RateLimit-Resource") == "graphql":
+        try:
+            body = response.json()
+        except ValueError:
+            logger.warning(
+                "GraphQL rate-limit check received non-JSON response (HTTP %s, first 50 chars: %r).",
+                response.status_code,
+                response.text[:50],
+            )
+            graphql_rate_limited = False
+        else:
+            graphql_rate_limited = graphql_rate_limit_checker(body or {})
+
+        if graphql_rate_limited:
+            return True
+
+    # GitHub REST rate-limit HTTP headers:
+    # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers
+    # GitHub secondary rate limits:
+    # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits
+    return (response.status_code != 200 and response.headers.get("X-RateLimit-Remaining") == "0") or "Retry-After" in response.headers
+
+
 class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
     def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
         self.stream = stream
@@ -57,19 +119,10 @@ class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
 
     def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
         if isinstance(response_or_exception, requests.Response):
-            retry_flag = (
-                # The GitHub GraphQL API has limitations
-                # https://docs.github.com/en/graphql/overview/resource-limitations
-                (
-                    response_or_exception.headers.get("X-RateLimit-Resource") == "graphql"
-                    and self.stream.check_graphql_rate_limited(response_or_exception.json())
-                )
-                # Rate limit HTTP headers
-                # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers
-                or (response_or_exception.status_code != 200 and response_or_exception.headers.get("X-RateLimit-Remaining") == "0")
-                # Secondary rate limits
-                # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits
-                or "Retry-After" in response_or_exception.headers
+            retry_flag = is_rate_limited_response(
+                response_or_exception,
+                self.stream.check_graphql_rate_limited,
+                self._logger,
             )
             if retry_flag:
                 headers = [
@@ -92,11 +145,26 @@ class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
                 return ErrorResolution(
                     response_action=ResponseAction.RATE_LIMITED,
                     failure_type=FailureType.transient_error,
-                    error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
+                    error_message=(
+                        f"GitHub rate limit hit for stream `{self.stream.name}` "
+                        f"(HTTP {response_or_exception.status_code}). "
+                        f"Waiting for the rate limit window to reset before retrying."
+                    ),
                 )
 
             if is_conflict_with_empty_repository(response_or_exception=response_or_exception):
-                log_message = f"Ignoring response for '{response_or_exception.request.method}' request to '{response_or_exception.url}' with response code '{response_or_exception.status_code}' as the repository is empty."
+                log_message = (
+                    f"Skipping `{self.stream.name}` for this repository: GitHub returned 409 Conflict "
+                    f"with message 'Git Repository is empty.' This means the repository has no commits."
+                )
+                return ErrorResolution(
+                    response_action=ResponseAction.IGNORE,
+                    failure_type=FailureType.config_error,
+                    error_message=log_message,
+                )
+
+            if is_gone_with_feature_disabled(response_or_exception=response_or_exception):
+                log_message = f"Skipping stream slice for '{response_or_exception.url}': {response_or_exception.json().get('message', 'Feature disabled')}."
                 return ErrorResolution(
                     response_action=ResponseAction.IGNORE,
                     failure_type=FailureType.config_error,
@@ -106,7 +174,7 @@ class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
         return super().interpret_response(response_or_exception)
 
 
-class ContributorActivityErrorHandler(HttpStatusErrorHandler):
+class ContributorActivityErrorHandler(GithubStreamABCErrorHandler):
     """
     This custom error handler is needed for streams based on repository statistics endpoints like ContributorActivity because
     when requesting data that hasn't been cached yet when the request is made, you'll receive a 202 response. And these requests
@@ -128,25 +196,49 @@ class ContributorActivityErrorHandler(HttpStatusErrorHandler):
 
 
 class GitHubGraphQLErrorHandler(GithubStreamABCErrorHandler):
+    def _safe_json_get_errors(self, response: requests.Response) -> bool:
+        try:
+            body = response.json()
+        except ValueError:
+            return False
+        return bool((body or {}).get("errors"))
+
     def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
         if isinstance(response_or_exception, requests.Response):
             if response_or_exception.status_code in (requests.codes.BAD_GATEWAY, requests.codes.GATEWAY_TIMEOUT):
-                self.stream.page_size = int(self.stream.page_size / 2)
+                # Halve the page size on every 502/504 to reduce GraphQL query cost,
+                # but never let it drop below 1 — a page_size of 0 would request no
+                # records and cause infinite paging.
+                previous_page_size = self.stream.page_size
+                self.stream.page_size = max(1, int(self.stream.page_size / 2))
+                self._logger.info(
+                    "GitHub GraphQL endpoint returned HTTP %s for stream `%s`; reducing GraphQL page_size from %s to %s and retrying.",
+                    response_or_exception.status_code,
+                    self.stream.name,
+                    previous_page_size,
+                    self.stream.page_size,
+                )
                 return ErrorResolution(
                     response_action=ResponseAction.RETRY,
                     failure_type=FailureType.transient_error,
-                    error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
+                    error_message=(
+                        f"GitHub GraphQL endpoint returned HTTP {response_or_exception.status_code} "
+                        f"for stream `{self.stream.name}`. Reducing GraphQL page size and retrying."
+                    ),
                 )
 
             self.stream.page_size = (
                 constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM if self.stream.large_stream else constants.DEFAULT_PAGE_SIZE
             )
 
-            if response_or_exception.json().get("errors"):
+            if self._safe_json_get_errors(response_or_exception):
                 return ErrorResolution(
                     response_action=ResponseAction.RETRY,
                     failure_type=FailureType.transient_error,
-                    error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
+                    error_message=(
+                        f"GitHub GraphQL endpoint returned errors in the response body "
+                        f"for stream `{self.stream.name}` (HTTP {response_or_exception.status_code}). Retrying."
+                    ),
                 )
 
         return super().interpret_response(response_or_exception)

source_github/graphql.py

@@ -165,6 +165,54 @@ def get_query_issue_reactions(owner, name, first, after, number=None):
     return str(op)
 
 
+def get_query_releases(owner, name, first, after):
+    kwargs = {"first": first, "order_by": {"field": "CREATED_AT", "direction": "ASC"}}
+    if after:
+        kwargs["after"] = after
+
+    op = sgqlc.operation.Operation(_schema_root.query_type)
+    repository = op.repository(owner=owner, name=name)
+    repository.name()
+    repository.owner.login()
+    releases = repository.releases(**kwargs)
+    releases.nodes.__fields__(
+        id="node_id",
+        database_id="id",
+        name=True,
+        tag_name="tag_name",
+        created_at="created_at",
+        published_at="published_at",
+        updated_at="updated_at",
+        is_draft="draft",
+        is_prerelease="prerelease",
+        description="body",
+        description_html="body_html",
+        url="html_url",
+    )
+    releases.nodes.tag_commit.oid(__alias__="target_commitish")
+    author = releases.nodes.author(__alias__="author").__as__(_schema_root.User)
+    select_user_fields(author)
+    release_assets = releases.nodes.release_assets(first=100, __alias__="assets")
+    release_assets.nodes.__fields__(
+        id="node_id",
+        name=True,
+        content_type="content_type",
+        size=True,
+        download_count="download_count",
+        created_at="created_at",
+        updated_at="updated_at",
+        download_url="browser_download_url",
+        url=True,
+    )
+    release_assets.nodes.uploaded_by(__alias__="uploader").__as__(_schema_root.User).__fields__(database_id="id")
+    release_assets.page_info.__fields__(has_next_page=True)
+    releases.nodes.reaction_groups(__alias__="reaction_groups").__fields__(content=True)
+    releases.nodes.reaction_groups(__alias__="reaction_groups").reactors.__fields__(total_count=True)
+    releases.nodes.mentions(first=0, __alias__="mentions_connection").total_count()
+    releases.page_info.__fields__(has_next_page=True, end_cursor=True)
+    return str(op)
+
+
 class QueryReactions:
     # AVERAGE_REVIEWS - optimal number of reviews to fetch inside every pull request.
     # If we try to fetch too many (up to 100) we will spend too many scores of query cost.