airbyte-source-github 2.1.41__py3-none-any.whl → 2.2.0__py3-none-any.whl

{airbyte_source_github-2.1.41.dist-info → airbyte_source_github-2.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 2.1.41
+Version: 2.2.0
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: ELv2
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: airbyte-cdk (>=7.12.0,<8.0.0)
+Requires-Dist: airbyte-cdk (>=7.28.2,<8.0.0)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/github
 Project-URL: Repository, https://github.com/airbytehq/airbyte

{airbyte_source_github-2.1.41.dist-info → airbyte_source_github-2.2.0.dist-info}/RECORD

@@ -1,11 +1,11 @@
 source_github/__init__.py,sha256=8hLLjFC1wI1mMU442Y5qCcND9qIwyhaANQwT-GAIK0s,1135
-source_github/backoff_strategies.py,sha256=ChkWEmqZL7Qrr1zidOfjZWlMWIVRVO2yqZ7QE_HaZP8,2287
+source_github/backoff_strategies.py,sha256=H1n3xNkkrlto7MqJY-Yp0el2G5S4yuJ7z3kg6aimab0,2343
 source_github/config_migrations.py,sha256=guUJAdNP-liUciVaJB4ackEEMCN4jd7bNd85QFmDNlU,3932
 source_github/constants.py,sha256=Hj3Q4y7OoU-Iff4m9gEC2CjwmWJYXhNbHVNjg8EBLmQ,238
-source_github/errors_handlers.py,sha256=t3jD8TwnQlbYXed1l_OJvQ3Z98k1e4XOvKQ5ZqA0WIA,11168
+source_github/errors_handlers.py,sha256=y0SjYuxKo3zBRhk2kZe0NZWsKen4W9x-wD84Q0S3sjU,11211
 source_github/github_schema.py,sha256=2AXmTN_s_VqvZAR1GPqus5HhSXWokS4N7HMQQqefaYw,1600315
 source_github/graphql.py,sha256=fGuYzDynYjJOR7u3GX0t6eBJMzdFxKIp4gHP73MCpos,13518
-source_github/manifest.yaml,sha256=4nIg4j3n6D_jokf_2Crgcy9wezTZ6PKC7VMD16ejiME,100
+source_github/manifest.yaml,sha256=D6z6gNZoy8yd7SZ50AxwhEx3Ky6gmsz16B4CL4ILXxM,56068
 source_github/run.py,sha256=gfWy8TcMHa0XRwaVUyq4KmslooST38fvfCLuP5D-5Fo,2391
 source_github/schemas/assignees.json,sha256=_WmGIwYmTmEi9WWkvXdFF3CLsMzG2usoIjaqw-tb4NA,2335
 source_github/schemas/branches.json,sha256=3CIkdEi2XVFoVZrpO_U04mMI7-jSb9i-CT3wbIEgPik,2734
@@ -34,7 +34,6 @@ source_github/schemas/pull_request_commits.json,sha256=WICcuxUsExJQe1qlXvXVN-Vvk
 source_github/schemas/pull_request_stats.json,sha256=harNH5ha6su-2OhxszHOqDKJ34OZzUsL_4nBnaOD16o,3497
 source_github/schemas/pull_requests.json,sha256=Yp9Sq3jgpfBn0YFbcZC_sy06rlJ-AMkCbHNq90KXT_Y,12196
 source_github/schemas/releases.json,sha256=3ITMlMFML3ZSfo_EAkVajJdaS_Xxz_8V4LhL9GRwWrY,5121
-source_github/schemas/repositories.json,sha256=WpC5ZjoycnHUP5DNz8ImEXROYAel8DGRkBN10W1mQ2w,13319
 source_github/schemas/review_comments.json,sha256=HVH3OE-x_wYVvg0Qxn6rzXOwAGEk0y3De8_VLCgE3mA,4905
 source_github/schemas/reviews.json,sha256=OPqo9Dz6slRGojamq8pBMKKRZgz96kDRkYiUJjJQH_E,2731
 source_github/schemas/shared/events/comment.json,sha256=6dcatpi1gitgcR_JscPllyGGoSb8Pgce5OS9vh1HKP4,7400
@@ -55,11 +54,10 @@ source_github/schemas/users.json,sha256=by1CEx3MW740S5F54KZfUDc2N1_UZ98Bb3fWlgCt
 source_github/schemas/workflow_jobs.json,sha256=Kk3N3FcCYBqOdR7FYvz2nAAFtLZBuVfR8Lxg-ORD39U,3934
 source_github/schemas/workflow_runs.json,sha256=XDmIsjtzka-ItEonImD3ZATZjxRNkbFo5-MPTtZDARA,19453
 source_github/schemas/workflows.json,sha256=gSNw8WZaVKbX4AL97PbjZHzvxcOltXqv9Ao1RNQOFXM,1470
-source_github/source.py,sha256=mYjAHEAXeL8d6Uo0yNgOI5fEqZqDr958l1pSiUG0qIk,18062
-source_github/spec.json,sha256=zzvSRPLbLNTNxGvygTSUb1ZuTIhLcFPGg3vowPCZLSc,8243
-source_github/streams.py,sha256=_RPdavX-AC9MVdzd6_UqXnt7PV3POovJ25DcUuy3w4U,88216
-source_github/utils.py,sha256=aUaJjBoM0xzkpV_Ajbh02p5O-gWQU83MI5_vRIJZL2o,11602
-airbyte_source_github-2.1.41.dist-info/METADATA,sha256=KZTIKYHKeBDT2XC8LG_94Dj0P7ElozoX5HtGNsc9YzQ,1249
-airbyte_source_github-2.1.41.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_source_github-2.1.41.dist-info/entry_points.txt,sha256=gYhqVrTAZvMwuYByg0b_-o115yUFLLcfNxMrLZmiW9k,55
-airbyte_source_github-2.1.41.dist-info/RECORD,,
+source_github/source.py,sha256=eqjqxGcR0EfoSgYb6ZHqKUYvwl609_8hMsRWoFOPD4k,22235
+source_github/streams.py,sha256=icnFwlrjGpIrgY-OlOLQqv5gkKcXC5OKxpxTJYLRMxI,86936
+source_github/utils.py,sha256=rkOk8VG6ey1xDl-9t4B99N6HrvTTPCTGhvifrKwYG9Y,706
+airbyte_source_github-2.2.0.dist-info/METADATA,sha256=bDlXZ8k2l0QqzAgYtDrkon94cpabg8diK-tKr5GnvlM,1248
+airbyte_source_github-2.2.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_source_github-2.2.0.dist-info/entry_points.txt,sha256=gYhqVrTAZvMwuYByg0b_-o115yUFLLcfNxMrLZmiW9k,55
+airbyte_source_github-2.2.0.dist-info/RECORD,,

source_github/backoff_strategies.py

@@ -10,38 +10,47 @@ import requests
 from airbyte_cdk import BackoffStrategy
 from airbyte_cdk.sources.streams.http import HttpStream
 
+from .errors_handlers import is_rate_limited_response
+
 
 class GithubStreamABCBackoffStrategy(BackoffStrategy):
-    def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
+    def __init__(self, stream: HttpStream, max_wait_time_seconds: float = 120 * 60, **kwargs):  # type: ignore # noqa
         self.stream = stream
+        self.max_wait_time_seconds = max_wait_time_seconds
         super().__init__(**kwargs)
 
     def backoff_time(
         self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], **kwargs: Any
     ) -> Optional[float]:
-        # This method is called if we run into the rate limit. GitHub limits requests to 5000 per hour and provides
-        # `X-RateLimit-Reset` header which contains time when this hour will be finished and limits will be reset so
-        # we again could have 5000 per another hour.
         if isinstance(response_or_exception, requests.Response):
+            if not is_rate_limited_response(
+                response_or_exception,
+                self.stream.check_graphql_rate_limited,
+                self.stream.logger,
+            ):
+                return None
+
             min_backoff_time = 60.0
             retry_after = response_or_exception.headers.get("Retry-After")
             if retry_after is not None:
-                backoff_time_in_seconds = max(float(retry_after), min_backoff_time)
-                return self.get_waiting_time(backoff_time_in_seconds)
-
-            reset_time = response_or_exception.headers.get("X-RateLimit-Reset")
-            if reset_time:
-                backoff_time_in_seconds = max(float(reset_time) - time.time(), min_backoff_time)
-                return self.get_waiting_time(backoff_time_in_seconds)
+                wait_time = max(float(retry_after), min_backoff_time)
+            else:
+                reset_time = response_or_exception.headers.get("X-RateLimit-Reset")
+                if not reset_time:
+                    return None
+                wait_time = max(float(reset_time) - time.time(), min_backoff_time)
+
+            if wait_time > self.max_wait_time_seconds:
+                self.stream.logger.warning(
+                    "Rate limit wait for stream `%s` is %.2f seconds, exceeding the maximum of %.2f seconds; using default backoff.",
+                    self.stream.name,
+                    wait_time,
+                    self.max_wait_time_seconds,
+                )
+                return None
+            return wait_time
         return None
 
-    def get_waiting_time(self, backoff_time_in_seconds: Optional[float]) -> Optional[float]:
-        if backoff_time_in_seconds < 60 * 10:  # type: ignore[operator]
-            return backoff_time_in_seconds
-        else:
-            self.stream._http_client._session.auth.update_token()  # New token will be used in next request
-            return 1
-
 
 class ContributorActivityBackoffStrategy(BackoffStrategy):
     def backoff_time(

source_github/errors_handlers.py

@@ -3,7 +3,7 @@
 #
 
 import logging
-from typing import Optional, Union
+from typing import Callable, Optional, Union
 
 import requests
 
@@ -84,38 +84,45 @@ def is_gone_with_feature_disabled(response_or_exception: Optional[Union[requests
     return False
 
 
-class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
-    def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
-        self.stream = stream
-        super().__init__(**kwargs)
-
-    def _safe_json_check_graphql_rate_limited(self, response: requests.Response) -> bool:
+def is_rate_limited_response(
+    response: requests.Response, graphql_rate_limit_checker: Callable[[dict], bool], logger: logging.Logger
+) -> bool:
+    # GitHub GraphQL resource limitations:
+    # https://docs.github.com/en/graphql/overview/resource-limitations
+    if response.headers.get("X-RateLimit-Resource") == "graphql":
         try:
             body = response.json()
         except ValueError:
-            self._logger.warning(
+            logger.warning(
                 "GraphQL rate-limit check received non-JSON response (HTTP %s, first 50 chars: %r).",
                 response.status_code,
                 response.text[:50],
             )
-            return False
-        return self.stream.check_graphql_rate_limited(body or {})
+            graphql_rate_limited = False
+        else:
+            graphql_rate_limited = graphql_rate_limit_checker(body or {})
+
+        if graphql_rate_limited:
+            return True
+
+    # GitHub REST rate-limit HTTP headers:
+    # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers
+    # GitHub secondary rate limits:
+    # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits
+    return (response.status_code != 200 and response.headers.get("X-RateLimit-Remaining") == "0") or "Retry-After" in response.headers
+
+
+class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
+    def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
+        self.stream = stream
+        super().__init__(**kwargs)
 
     def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
         if isinstance(response_or_exception, requests.Response):
-            retry_flag = (
-                # The GitHub GraphQL API has limitations
-                # https://docs.github.com/en/graphql/overview/resource-limitations
-                (
-                    response_or_exception.headers.get("X-RateLimit-Resource") == "graphql"
-                    and self._safe_json_check_graphql_rate_limited(response_or_exception)
-                )
-                # Rate limit HTTP headers
-                # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers
-                or (response_or_exception.status_code != 200 and response_or_exception.headers.get("X-RateLimit-Remaining") == "0")
-                # Secondary rate limits
-                # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits
-                or "Retry-After" in response_or_exception.headers
+            retry_flag = is_rate_limited_response(
+                response_or_exception,
+                self.stream.check_graphql_rate_limited,
+                self._logger,
             )
             if retry_flag:
                 headers = [