airbyte-source-github 2.1.41__tar.gz → 2.2.0__tar.gz

{airbyte_source_github-2.1.41 → airbyte_source_github-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 2.1.41
+Version: 2.2.0
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: ELv2
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: airbyte-cdk (>=7.12.0,<8.0.0)
+Requires-Dist: airbyte-cdk (>=7.28.2,<8.0.0)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/github
 Project-URL: Repository, https://github.com/airbytehq/airbyte

{airbyte_source_github-2.1.41 → airbyte_source_github-2.2.0}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.1.41"
+version = "2.2.0"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]
@@ -17,7 +17,7 @@ include = "source_github"
 
 [tool.poetry.dependencies]
 python = "^3.10,<3.14"
-airbyte-cdk = "^7.12.0"
+airbyte-cdk = "^7.28.2"
 sgqlc = "==16.3"
 
 [tool.poetry.scripts]

airbyte_source_github-2.2.0/source_github/backoff_strategies.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import time
+from typing import Any, Optional, Union
+
+import requests
+
+from airbyte_cdk import BackoffStrategy
+from airbyte_cdk.sources.streams.http import HttpStream
+
+from .errors_handlers import is_rate_limited_response
+
+
+class GithubStreamABCBackoffStrategy(BackoffStrategy):
+    def __init__(self, stream: HttpStream, max_wait_time_seconds: float = 120 * 60, **kwargs):  # type: ignore # noqa
+        self.stream = stream
+        self.max_wait_time_seconds = max_wait_time_seconds
+        super().__init__(**kwargs)
+
+    def backoff_time(
+        self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], **kwargs: Any
+    ) -> Optional[float]:
+        if isinstance(response_or_exception, requests.Response):
+            if not is_rate_limited_response(
+                response_or_exception,
+                self.stream.check_graphql_rate_limited,
+                self.stream.logger,
+            ):
+                return None
+
+            min_backoff_time = 60.0
+            retry_after = response_or_exception.headers.get("Retry-After")
+            if retry_after is not None:
+                wait_time = max(float(retry_after), min_backoff_time)
+            else:
+                reset_time = response_or_exception.headers.get("X-RateLimit-Reset")
+                if not reset_time:
+                    return None
+                wait_time = max(float(reset_time) - time.time(), min_backoff_time)
+
+            if wait_time > self.max_wait_time_seconds:
+                self.stream.logger.warning(
+                    "Rate limit wait for stream `%s` is %.2f seconds, exceeding the maximum of %.2f seconds; using default backoff.",
+                    self.stream.name,
+                    wait_time,
+                    self.max_wait_time_seconds,
+                )
+                return None
+            return wait_time
+        return None
+
+
+class ContributorActivityBackoffStrategy(BackoffStrategy):
+    def backoff_time(
+        self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], **kwargs: Any
+    ) -> Optional[float]:
+        if isinstance(response_or_exception, requests.Response) and response_or_exception.status_code == requests.codes.ACCEPTED:
+            return 90
+        return None

{airbyte_source_github-2.1.41 → airbyte_source_github-2.2.0}/source_github/errors_handlers.py

@@ -3,7 +3,7 @@
 #
 
 import logging
-from typing import Optional, Union
+from typing import Callable, Optional, Union
 
 import requests
 
@@ -84,38 +84,45 @@ def is_gone_with_feature_disabled(response_or_exception: Optional[Union[requests
     return False
 
 
-class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
-    def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
-        self.stream = stream
-        super().__init__(**kwargs)
-
-    def _safe_json_check_graphql_rate_limited(self, response: requests.Response) -> bool:
+def is_rate_limited_response(
+    response: requests.Response, graphql_rate_limit_checker: Callable[[dict], bool], logger: logging.Logger
+) -> bool:
+    # GitHub GraphQL resource limitations:
+    # https://docs.github.com/en/graphql/overview/resource-limitations
+    if response.headers.get("X-RateLimit-Resource") == "graphql":
         try:
             body = response.json()
         except ValueError:
-            self._logger.warning(
+            logger.warning(
                 "GraphQL rate-limit check received non-JSON response (HTTP %s, first 50 chars: %r).",
                 response.status_code,
                 response.text[:50],
             )
-            return False
-        return self.stream.check_graphql_rate_limited(body or {})
+            graphql_rate_limited = False
+        else:
+            graphql_rate_limited = graphql_rate_limit_checker(body or {})
+
+        if graphql_rate_limited:
+            return True
+
+    # GitHub REST rate-limit HTTP headers:
+    # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers
+    # GitHub secondary rate limits:
+    # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits
+    return (response.status_code != 200 and response.headers.get("X-RateLimit-Remaining") == "0") or "Retry-After" in response.headers
+
+
+class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
+    def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
+        self.stream = stream
+        super().__init__(**kwargs)
 
     def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
         if isinstance(response_or_exception, requests.Response):
-            retry_flag = (
-                # The GitHub GraphQL API has limitations
-                # https://docs.github.com/en/graphql/overview/resource-limitations
-                (
-                    response_or_exception.headers.get("X-RateLimit-Resource") == "graphql"
-                    and self._safe_json_check_graphql_rate_limited(response_or_exception)
-                )
-                # Rate limit HTTP headers
-                # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers
-                or (response_or_exception.status_code != 200 and response_or_exception.headers.get("X-RateLimit-Remaining") == "0")
-                # Secondary rate limits
-                # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits
-                or "Retry-After" in response_or_exception.headers
+            retry_flag = is_rate_limited_response(
+                response_or_exception,
+                self.stream.check_graphql_rate_limited,
+                self._logger,
             )
             if retry_flag:
                 headers = [