airbyte-source-github 1.7.12.dev202407111311__tar.gz → 1.8.0__tar.gz

{airbyte_source_github-1.7.12.dev202407111311 → airbyte_source_github-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 1.7.12.dev202407111311
+Version: 1.8.0
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: MIT
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (==0.90.0)
+Requires-Dist: airbyte-cdk (>=3,<4)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/github
 Project-URL: Repository, https://github.com/airbytehq/airbyte

{airbyte_source_github-1.7.12.dev202407111311 → airbyte_source_github-1.8.0}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.7.12.dev202407111311"
+version = "1.8.0"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [
@@ -22,7 +22,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.9,<3.12"
-airbyte-cdk = "0.90.0"
+airbyte-cdk = "^3"
 sgqlc = "==16.3"
 
 [tool.poetry.scripts]

airbyte_source_github-1.8.0/source_github/backoff_strategies.py

@@ -0,0 +1,51 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import time
+from typing import Any, Optional, Union
+
+import requests
+from airbyte_cdk import BackoffStrategy
+from airbyte_cdk.sources.streams.http import HttpStream
+
+
+class GithubStreamABCBackoffStrategy(BackoffStrategy):
+    def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
+        self.stream = stream
+        super().__init__(**kwargs)
+
+    def backoff_time(
+        self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], **kwargs: Any
+    ) -> Optional[float]:
+        # This method is called if we run into the rate limit. GitHub limits requests to 5000 per hour and provides
+        # `X-RateLimit-Reset` header which contains time when this hour will be finished and limits will be reset so
+        # we again could have 5000 per another hour.
+        if isinstance(response_or_exception, requests.Response):
+            min_backoff_time = 60.0
+            retry_after = response_or_exception.headers.get("Retry-After")
+            if retry_after is not None:
+                backoff_time_in_seconds = max(float(retry_after), min_backoff_time)
+                return self.get_waiting_time(backoff_time_in_seconds)
+
+            reset_time = response_or_exception.headers.get("X-RateLimit-Reset")
+            if reset_time:
+                backoff_time_in_seconds = max(float(reset_time) - time.time(), min_backoff_time)
+                return self.get_waiting_time(backoff_time_in_seconds)
+        return None
+
+    def get_waiting_time(self, backoff_time_in_seconds: Optional[float]) -> Optional[float]:
+        if backoff_time_in_seconds < 60 * 10:  # type: ignore[operator]
+            return backoff_time_in_seconds
+        else:
+            self.stream._http_client._session.auth.update_token()  # New token will be used in next request
+            return 1
+
+
+class ContributorActivityBackoffStrategy(BackoffStrategy):
+    def backoff_time(
+        self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], **kwargs: Any
+    ) -> Optional[float]:
+        if isinstance(response_or_exception, requests.Response) and response_or_exception.status_code == requests.codes.ACCEPTED:
+            return 90
+        return None

airbyte_source_github-1.8.0/source_github/errors_handlers.py

@@ -0,0 +1,126 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Optional, Union
+
+import requests
+from airbyte_cdk.sources.streams.http import HttpStream
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
+from airbyte_protocol.models import FailureType
+
+from . import constants
+
+GITHUB_DEFAULT_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | {
+    401: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.config_error,
+        error_message="Conflict.",
+    ),
+    403: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.config_error,
+        error_message="Conflict.",
+    ),
+    404: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.config_error,
+        error_message="Conflict.",
+    ),
+    409: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.config_error,
+        error_message="Conflict.",
+    ),
+    410: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.config_error,
+        error_message="Gone. Please ensure the url is valid.",
+    ),
+}
+
+
+class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
+    def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
+        self.stream = stream
+        super().__init__(**kwargs)
+
+    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
+        if isinstance(response_or_exception, requests.Response):
+            retry_flag = (
+                # The GitHub GraphQL API has limitations
+                # https://docs.github.com/en/graphql/overview/resource-limitations
+                (
+                    response_or_exception.headers.get("X-RateLimit-Resource") == "graphql"
+                    and self.stream.check_graphql_rate_limited(response_or_exception.json())
+                )
+                # Rate limit HTTP headers
+                # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers
+                or (response_or_exception.status_code != 200 and response_or_exception.headers.get("X-RateLimit-Remaining") == "0")
+                # Secondary rate limits
+                # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits
+                or "Retry-After" in response_or_exception.headers
+            )
+            if retry_flag:
+                headers = [
+                    "X-RateLimit-Resource",
+                    "X-RateLimit-Remaining",
+                    "X-RateLimit-Reset",
+                    "X-RateLimit-Limit",
+                    "X-RateLimit-Used",
+                    "Retry-After",
+                ]
+                string_headers = ", ".join(
+                    [f"{h}: {response_or_exception.headers[h]}" for h in headers if h in response_or_exception.headers]
+                )
+                if string_headers:
+                    string_headers = f"HTTP headers: {string_headers},"
+
+                self._logger.info(
+                    f"Rate limit handling for stream `{self.stream.name}` for the response with {response_or_exception.status_code} status code, {string_headers} with message: {response_or_exception.text}"
+                )
+                return ErrorResolution(
+                    response_action=ResponseAction.RATE_LIMITED,
+                    failure_type=FailureType.transient_error,
+                    error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
+                )
+
+        return super().interpret_response(response_or_exception)
+
+
+class ContributorActivityErrorHandler(HttpStatusErrorHandler):
+    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
+        if isinstance(response_or_exception, requests.Response) and response_or_exception.status_code == requests.codes.ACCEPTED:
+            return ErrorResolution(
+                response_action=ResponseAction.RETRY,
+                failure_type=FailureType.transient_error,
+                error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
+            )
+
+        return super().interpret_response(response_or_exception)
+
+
+class GitHubGraphQLErrorHandler(GithubStreamABCErrorHandler):
+    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
+        if isinstance(response_or_exception, requests.Response):
+            if response_or_exception.status_code in (requests.codes.BAD_GATEWAY, requests.codes.GATEWAY_TIMEOUT):
+                self.stream.page_size = int(self.stream.page_size / 2)
+                return ErrorResolution(
+                    response_action=ResponseAction.RETRY,
+                    failure_type=FailureType.transient_error,
+                    error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
+                )
+
+            self.stream.page_size = (
+                constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM if self.stream.large_stream else constants.DEFAULT_PAGE_SIZE
+            )
+
+            if response_or_exception.json().get("errors"):
+                return ErrorResolution(
+                    response_action=ResponseAction.RETRY,
+                    failure_type=FailureType.transient_error,
+                    error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
+                )
+
+        return super().interpret_response(response_or_exception)

{airbyte_source_github-1.7.12.dev202407111311 → airbyte_source_github-1.8.0}/source_github/source.py

@@ -65,7 +65,7 @@ class SourceGithub(AbstractSource):
 
     @staticmethod
     def _get_org_repositories(
-        config: Mapping[str, Any], authenticator: MultipleTokenAuthenticator
+        config: Mapping[str, Any], authenticator: MultipleTokenAuthenticator, is_check_connection: bool = False
     ) -> Tuple[List[str], List[str], Optional[str]]:
         """
         Parse config/repositories and produce two lists: organizations, repositories.
@@ -92,6 +92,7 @@ class SourceGithub(AbstractSource):
             org_names = [org.split("/")[0] for org in unchecked_orgs]
             pattern = "|".join([f"({org.replace('*', '.*')})" for org in unchecked_orgs])
             stream = Repositories(authenticator=authenticator, organizations=org_names, api_url=config.get("api_url"), pattern=pattern)
+            stream.exit_on_rate_limit = True if is_check_connection else False
             for record in read_full_refresh(stream):
                 repositories.add(record["full_name"])
                 organizations.add(record["organization"])
@@ -105,6 +106,7 @@ class SourceGithub(AbstractSource):
                 # This parameter is deprecated and in future will be used sane default, page_size: 10
                 page_size_for_large_streams=config.get("page_size_for_large_streams", constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM),
             )
+            stream.exit_on_rate_limit = True if is_check_connection else False
             for record in read_full_refresh(stream):
                 repositories.add(record["full_name"])
                 organization = record.get("organization", {}).get("login")
@@ -194,7 +196,7 @@ class SourceGithub(AbstractSource):
         config = self._validate_and_transform_config(config)
         try:
             authenticator = self._get_authenticator(config)
-            _, repositories, _ = self._get_org_repositories(config=config, authenticator=authenticator)
+            _, repositories, _ = self._get_org_repositories(config=config, authenticator=authenticator, is_check_connection=True)
             if not repositories:
                 return (
                     False,

{airbyte_source_github-1.7.12.dev202407111311 → airbyte_source_github-1.8.0}/source_github/streams.py

@@ -3,24 +3,31 @@
 #
 
 import re
-import time
 from abc import ABC, abstractmethod
-from typing import Any, Iterable, List, Mapping, MutableMapping, Optional
+from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 from urllib import parse
 
 import pendulum
 import requests
+from airbyte_cdk import BackoffStrategy
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
-from airbyte_cdk.sources.streams.core import CheckpointMixin
+from airbyte_cdk.sources.streams.core import CheckpointMixin, Stream
 from airbyte_cdk.sources.streams.http import HttpStream
-from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction
+from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException
 from airbyte_cdk.utils import AirbyteTracedException
 from airbyte_protocol.models import FailureType
-from requests.exceptions import HTTPError
 
 from . import constants
+from .backoff_strategies import ContributorActivityBackoffStrategy, GithubStreamABCBackoffStrategy
+from .errors_handlers import (
+    GITHUB_DEFAULT_ERROR_MAPPING,
+    ContributorActivityErrorHandler,
+    GitHubGraphQLErrorHandler,
+    GithubStreamABCErrorHandler,
+)
 from .graphql import (
     CursorStorage,
     QueryReactions,
@@ -38,7 +45,7 @@ class GithubStreamABC(HttpStream, ABC):
 
     # Detect streams with high API load
     large_stream = False
-
+    max_retries: int = 5
     stream_base_params = {}
 
     def __init__(self, api_url: str = "https://api.github.com", access_token_type: str = "", **kwargs):
@@ -93,62 +100,13 @@ class GithubStreamABC(HttpStream, ABC):
         for record in response.json():  # GitHub puts records in an array.
             yield self.transform(record=record, stream_slice=stream_slice)
 
-    def should_retry(self, response: requests.Response) -> bool:
-        if super().should_retry(response):
-            return True
-
-        retry_flag = (
-            # The GitHub GraphQL API has limitations
-            # https://docs.github.com/en/graphql/overview/resource-limitations
-            (response.headers.get("X-RateLimit-Resource") == "graphql" and self.check_graphql_rate_limited(response.json()))
-            # Rate limit HTTP headers
-            # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers
-            or (response.status_code != 200 and response.headers.get("X-RateLimit-Remaining") == "0")
-            # Secondary rate limits
-            # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits
-            or "Retry-After" in response.headers
+    def get_error_handler(self) -> Optional[ErrorHandler]:
+        return GithubStreamABCErrorHandler(
+            logger=self.logger, max_retries=self.max_retries, error_mapping=GITHUB_DEFAULT_ERROR_MAPPING, stream=self
         )
-        if retry_flag:
-            headers = [
-                "X-RateLimit-Resource",
-                "X-RateLimit-Remaining",
-                "X-RateLimit-Reset",
-                "X-RateLimit-Limit",
-                "X-RateLimit-Used",
-                "Retry-After",
-            ]
-            headers = ", ".join([f"{h}: {response.headers[h]}" for h in headers if h in response.headers])
-            if headers:
-                headers = f"HTTP headers: {headers},"
-
-            self.logger.info(
-                f"Rate limit handling for stream `{self.name}` for the response with {response.status_code} status code, {headers} with message: {response.text}"
-            )
-
-        return retry_flag
-
-    def backoff_time(self, response: requests.Response) -> Optional[float]:
-        # This method is called if we run into the rate limit. GitHub limits requests to 5000 per hour and provides
-        # `X-RateLimit-Reset` header which contains time when this hour will be finished and limits will be reset so
-        # we again could have 5000 per another hour.
-
-        min_backoff_time = 60.0
-        retry_after = response.headers.get("Retry-After")
-        if retry_after is not None:
-            backoff_time_in_seconds = max(float(retry_after), min_backoff_time)
-            return self.get_waiting_time(backoff_time_in_seconds)
 
-        reset_time = response.headers.get("X-RateLimit-Reset")
-        if reset_time:
-            backoff_time_in_seconds = max(float(reset_time) - time.time(), min_backoff_time)
-            return self.get_waiting_time(backoff_time_in_seconds)
-
-    def get_waiting_time(self, backoff_time_in_seconds):
-        if backoff_time_in_seconds < self.max_time:
-            return backoff_time_in_seconds
-        else:
-            self._session.auth.update_token()  # New token will be used in next request
-            return 1
+    def get_backoff_strategy(self) -> Optional[Union[BackoffStrategy, List[BackoffStrategy]]]:
+        return GithubStreamABCBackoffStrategy(stream=self)
 
     @staticmethod
     def check_graphql_rate_limited(response_json: dict) -> bool:
@@ -166,7 +124,7 @@ class GithubStreamABC(HttpStream, ABC):
         # Reading records while handling the errors
         try:
             yield from super().read_records(stream_slice=stream_slice, **kwargs)
-        except HTTPError as e:
+        except DefaultBackoffException as e:
             # This whole try/except situation in `read_records()` isn't good but right now in `self._send_request()`
             # function we have `response.raise_for_status()` so we don't have much choice on how to handle errors.
             # Bocked on https://github.com/airbytehq/airbyte/issues/3514.
@@ -802,12 +760,10 @@ class GitHubGraphQLStream(GithubStream, ABC):
     ) -> str:
         return "graphql"
 
-    def should_retry(self, response: requests.Response) -> bool:
-        if response.status_code in (requests.codes.BAD_GATEWAY, requests.codes.GATEWAY_TIMEOUT):
-            self.page_size = int(self.page_size / 2)
-            return True
-        self.page_size = constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM if self.large_stream else constants.DEFAULT_PAGE_SIZE
-        return super().should_retry(response) or response.json().get("errors")
+    def get_error_handler(self) -> Optional[ErrorHandler]:
+        return GitHubGraphQLErrorHandler(
+            logger=self.logger, max_retries=self.max_retries, error_mapping=GITHUB_DEFAULT_ERROR_MAPPING, stream=self
+        )
 
     def _get_repository_name(self, repository: Mapping[str, Any]) -> str:
         return repository["owner"]["login"] + "/" + repository["name"]
@@ -1656,16 +1612,11 @@ class ContributorActivity(GithubStream):
         record.update(record.pop("author"))
         return record
 
-    def should_retry(self, response: requests.Response) -> bool:
-        """
-        If the data hasn't been cached when you query a repository's statistics, you'll receive a 202 response, need to retry to get results
-        see for more info https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#a-word-about-caching
-        """
-        if super().should_retry(response) or response.status_code == requests.codes.ACCEPTED:
-            return True
+    def get_error_handler(self) -> Optional[ErrorHandler]:
+        return ContributorActivityErrorHandler(logger=self.logger, max_retries=self.max_retries, error_mapping=GITHUB_DEFAULT_ERROR_MAPPING)
 
-    def backoff_time(self, response: requests.Response) -> Optional[float]:
-        return 90 if response.status_code == requests.codes.ACCEPTED else super().backoff_time(response)
+    def get_backoff_strategy(self) -> Optional[Union[BackoffStrategy, List[BackoffStrategy]]]:
+        return ContributorActivityBackoffStrategy()
 
     def parse_response(
         self,
@@ -1685,7 +1636,7 @@ class ContributorActivity(GithubStream):
         repository = stream_slice.get("repository", "")
         try:
             yield from super().read_records(stream_slice=stream_slice, **kwargs)
-        except HTTPError as e:
+        except UserDefinedBackoffException as e:
             if e.response.status_code == requests.codes.ACCEPTED:
                 yield AirbyteMessage(
                     type=MessageType.LOG,