airbyte-source-github 1.9.0rc2__py3-none-any.whl → 1.9.1rc1__py3-none-any.whl

{airbyte_source_github-1.9.0rc2.dist-info → airbyte_source_github-1.9.1rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 1.9.0rc2
+Version: 1.9.1rc1
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: ELv2
@@ -11,7 +11,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=7,<8)
+Requires-Dist: airbyte-cdk (>=7.3.4,<8.0.0)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/github
 Project-URL: Repository, https://github.com/airbytehq/airbyte

{airbyte_source_github-1.9.0rc2.dist-info → airbyte_source_github-1.9.1rc1.dist-info}/RECORD

@@ -54,11 +54,11 @@ source_github/schemas/users.json,sha256=by1CEx3MW740S5F54KZfUDc2N1_UZ98Bb3fWlgCt
 source_github/schemas/workflow_jobs.json,sha256=Kk3N3FcCYBqOdR7FYvz2nAAFtLZBuVfR8Lxg-ORD39U,3934
 source_github/schemas/workflow_runs.json,sha256=XDmIsjtzka-ItEonImD3ZATZjxRNkbFo5-MPTtZDARA,19453
 source_github/schemas/workflows.json,sha256=gSNw8WZaVKbX4AL97PbjZHzvxcOltXqv9Ao1RNQOFXM,1470
-source_github/source.py,sha256=1o8eayigi4xSUeNHdCd-mhNswGUq_XQrVk2eihTjm1o,14246
+source_github/source.py,sha256=rY85d7X_StM0_GYIRYwnvoGQcsbNA_ZtdRnNJdh9-lk,14588
 source_github/spec.json,sha256=7LOQm01fP_RvPF-HifhNPJ7i0AxT2LTNPaLAA3uOfNY,7443
 source_github/streams.py,sha256=uvCHATVyObHCz76tuknNqPETuGlyUdXGsa1YZc7B4JI,78285
-source_github/utils.py,sha256=nSARcWkLTsoyh6c7j8AvxGCgK1uUPDwOe970_1Ywj94,5586
-airbyte_source_github-1.9.0rc2.dist-info/METADATA,sha256=IJOa4twAg9c5Yb_mQSEijUwZaobg0hSenP9hG-UG6yE,5191
-airbyte_source_github-1.9.0rc2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_source_github-1.9.0rc2.dist-info/entry_points.txt,sha256=gYhqVrTAZvMwuYByg0b_-o115yUFLLcfNxMrLZmiW9k,55
-airbyte_source_github-1.9.0rc2.dist-info/RECORD,,
+source_github/utils.py,sha256=1mKZzLEaDe0m8BTw8Y-MxCXrJr1bZfaLPeSUbPN3vyQ,7243
+airbyte_source_github-1.9.1rc1.dist-info/METADATA,sha256=AMbb8fqUONHqjtukcD-PcF9ky8ByW4C4IU-zobIUg2E,5199
+airbyte_source_github-1.9.1rc1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_source_github-1.9.1rc1.dist-info/entry_points.txt,sha256=gYhqVrTAZvMwuYByg0b_-o115yUFLLcfNxMrLZmiW9k,55
+airbyte_source_github-1.9.1rc1.dist-info/RECORD,,

source_github/source.py

@@ -9,6 +9,7 @@ from urllib.parse import urlparse
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors
 from airbyte_cdk.sources.streams.http.requests_native_auth import MultipleTokenAuthenticator
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter
@@ -184,7 +185,7 @@ class SourceGithub(AbstractSource):
             # 404 Client Error: Not Found for url: https://api.github.com/orgs/airbytehqBLA/repos?per_page=100
             org_name = message.split("https://api.github.com/orgs/")[1].split("/")[0]
             user_message = f'Organization name: "{org_name}" is unknown, "repository" config option should be updated. Please validate your repository config.'
-        elif "401 Client Error: Unauthorized for url" in message:
+        elif "401 Client Error: Unauthorized for url" in message or ("Error: Unauthorized" in message and "401" in message):
             # 401 Client Error: Unauthorized for url: https://api.github.com/orgs/datarootsio/repos?per_page=100&sort=updated&direction=desc
             user_message = (
                 "Github credentials have expired or changed, please review your credentials and re-authenticate or renew your access token."
@@ -203,6 +204,9 @@ class SourceGithub(AbstractSource):
                 )
             return True, None
 
+        except MessageRepresentationAirbyteTracedErrors as e:
+            user_message = self.user_friendly_error_message(e.message)
+            return False, user_message or e.message
         except Exception as e:
             message = repr(e)
             user_message = self.user_friendly_error_message(message)

source_github/utils.py

@@ -1,7 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+import logging
 import time
 from dataclasses import dataclass
 from datetime import timedelta
@@ -10,10 +10,12 @@ from typing import Any, List, Mapping
 
 import requests
 
-from airbyte_cdk.models import SyncMode
+from airbyte_cdk.models import FailureType, SyncMode
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http import HttpClient
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from airbyte_cdk.utils import AirbyteTracedException
 from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
 
 
@@ -59,14 +61,30 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
     DURATION = timedelta(seconds=3600)  # Duration at which the current rate limit window resets
 
     def __init__(self, tokens: List[str], auth_method: str = "token", auth_header: str = "Authorization"):
+        self._logger = logging.getLogger("airbyte")
         self._auth_method = auth_method
         self._auth_header = auth_header
         self._tokens = {t: Token() for t in tokens}
+        # It would've been nice to instantiate a single client on this authenticator. However, we are checking
+        # the limits of each token which is associated with a TokenAuthenticator. And each HttpClient can only
+        # correspond to one authenticator.
+        self._token_to_http_client: Mapping[str, HttpClient] = self._initialize_http_clients(tokens)
         self.check_all_tokens()
         self._tokens_iter = cycle(self._tokens)
         self._active_token = next(self._tokens_iter)
         self._max_time = 60 * 10  # 10 minutes as default
 
+    def _initialize_http_clients(self, tokens: List[str]) -> Mapping[str, HttpClient]:
+        return {
+            token: HttpClient(
+                name="token_validator",
+                logger=self._logger,
+                authenticator=TokenAuthenticator(token, auth_method=self._auth_method),
+                use_cache=False,  # We don't want to reuse cached valued because rate limit values change frequently
+            )
+            for token in tokens
+        }
+
     @property
     def auth_header(self) -> str:
         return self._auth_header
@@ -114,14 +132,27 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
 
     def _check_token_limits(self, token: str):
         """check that token is not limited"""
-        headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}
-        rate_limit_info = (
-            requests.get(
-                "https://api.github.com/rate_limit", headers=headers, auth=TokenAuthenticator(token, auth_method=self._auth_method)
-            )
-            .json()
-            .get("resources")
+
+        http_client = self._token_to_http_client.get(token)
+        if not http_client:
+            raise ValueError("No HttpClient was initialized for this token. This is unexpected. Please contact Airbyte support.")
+
+        _, response = http_client.send_request(
+            http_method="GET",
+            url="https://api.github.com/rate_limit",
+            headers={"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"},
+            request_kwargs={},
         )
+
+        response_body = response.json()
+        if "resources" not in response_body:
+            raise AirbyteTracedException(
+                failure_type=FailureType.config_error,
+                internal_message=f"Token rate limit info response did not contain expected key: resources",
+                message="Unable to validate token. Please double check that specified authentication tokens are correct",
+            )
+
+        rate_limit_info = response_body.get("resources")
         token_info = self._tokens[token]
         remaining_info_core = rate_limit_info.get("core")
         token_info.count_rest, token_info.reset_at_rest = (
@@ -144,7 +175,7 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
             setattr(current_token, count_attr, getattr(current_token, count_attr) - 1)
             return True
         elif all(getattr(x, count_attr) == 0 for x in self._tokens.values()):
-            min_time_to_wait = min((getattr(x, reset_attr) - ab_datetime_now()).seconds for x in self._tokens.values())
+            min_time_to_wait = min((getattr(x, reset_attr) - ab_datetime_now()).total_seconds() for x in self._tokens.values())
             if min_time_to_wait < self.max_time:
                 time.sleep(min_time_to_wait if min_time_to_wait > 0 else 0)
                 self.check_all_tokens()