airbyte-source-github 1.8.39__tar.gz → 2.1.5__tar.gz

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/PKG-INFO

@@ -1,17 +1,19 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 1.8.39
+Version: 2.1.5
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
-License: MIT
+License: ELv2
 Author: Airbyte
 Author-email: contact@airbyte.io
-Requires-Python: >=3.10,<3.12
-Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.10,<3.14
+Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=4,<5)
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: airbyte-cdk (>=7.4.1,<8.0.0)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/github
 Project-URL: Repository, https://github.com/airbytehq/airbyte

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/pyproject.toml

@@ -3,11 +3,11 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.8.39"
+version = "2.1.5"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]
-license = "MIT"
+license = "ELv2"
 readme = "README.md"
 documentation = "https://docs.airbyte.com/integrations/sources/github"
 homepage = "https://airbyte.com"
@@ -16,8 +16,8 @@ repository = "https://github.com/airbytehq/airbyte"
 include = "source_github"
 
 [tool.poetry.dependencies]
-python = "^3.10,<3.12"
-airbyte-cdk = "^4"
+python = "^3.10,<3.14"
+airbyte-cdk = "^7.4.1"
 sgqlc = "==16.3"
 
 [tool.poetry.scripts]
@@ -45,4 +45,4 @@ include = [
 test-unit-tests.shell = '''
 poetry run pytest --junitxml=build/test-results/pytest-unit-tests-junit.xml --ignore=unit_tests/integration/ unit_tests
 poetry run pytest --junitxml=build/test-results/pytest-unit-integration-tests-junit.xml unit_tests/integration/
-'''
+'''

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/source_github/__init__.py

@@ -24,4 +24,5 @@ SOFTWARE.
 
 from .source import SourceGithub
 
+
 __all__ = ["SourceGithub"]

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/source_github/config_migrations.py

@@ -6,8 +6,11 @@ import logging
 from abc import ABC
 from typing import Any, List, Mapping
 
+import orjson
+
 from airbyte_cdk.config_observation import create_connector_config_control_message
 from airbyte_cdk.entrypoint import AirbyteEntrypoint
+from airbyte_cdk.models import AirbyteMessageSerializer
 from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 
 from .source import SourceGithub
@@ -72,7 +75,7 @@ class MigrateStringToArray(ABC):
         cls.message_repository.emit_message(create_connector_config_control_message(migrated_config))
         # emit the Airbyte Control Message from message queue to stdout
         for message in cls.message_repository._message_queue:
-            print(message.json(exclude_unset=True))
+            print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
 
     @classmethod
     def migrate(cls, args: List[str], source: SourceGithub) -> None:

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/source_github/errors_handlers.py

@@ -6,10 +6,10 @@ from typing import Optional, Union
 
 import requests
 
+from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction
 from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
-from airbyte_protocol.models import FailureType
 
 from . import constants
 

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/source_github/schemas/shared/events/comment.json

@@ -174,8 +174,8 @@
       "properties": {
         "url": { "type": "string" },
         "total_count": { "type": "integer" },
-        "+1": { "type": "integer" },
-        "-1": { "type": "integer" },
+        "plus_one": { "type": "integer" },
+        "minus_one": { "type": "integer" },
         "laugh": { "type": "integer" },
         "confused": { "type": "integer" },
         "heart": { "type": "integer" },

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/source_github/schemas/shared/events/commented.json

@@ -99,8 +99,8 @@
             "properties": {
               "url": { "type": "string" },
               "total_count": { "type": "integer" },
-              "+1": { "type": "integer" },
-              "-1": { "type": "integer" },
+              "plus_one": { "type": "integer" },
+              "minus_one": { "type": "integer" },
               "laugh": { "type": "integer" },
               "confused": { "type": "integer" },
               "heart": { "type": "integer" },

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/source_github/schemas/shared/events/cross_referenced.json

@@ -752,10 +752,10 @@
                   "total_count": {
                     "type": "integer"
                   },
-                  "+1": {
+                  "plus_one": {
                     "type": "integer"
                   },
-                  "-1": {
+                  "minus_one": {
                     "type": "integer"
                   },
                   "laugh": {
@@ -788,10 +788,10 @@
                 "total_count": {
                   "type": "integer"
                 },
-                "+1": {
+                "plus_one": {
                   "type": "integer"
                 },
-                "-1": {
+                "minus_one": {
                   "type": "integer"
                 },
                 "laugh": {

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/source_github/schemas/shared/reactions.json

@@ -7,10 +7,10 @@
     "total_count": {
       "type": ["null", "integer"]
     },
-    "+1": {
+    "plus_one": {
       "type": ["null", "integer"]
     },
-    "-1": {
+    "minus_one": {
       "type": ["null", "integer"]
     },
     "laugh": {

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/source_github/source.py

@@ -9,6 +9,7 @@ from urllib.parse import urlparse
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors
 from airbyte_cdk.sources.streams.http.requests_native_auth import MultipleTokenAuthenticator
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter
@@ -184,7 +185,7 @@ class SourceGithub(AbstractSource):
             # 404 Client Error: Not Found for url: https://api.github.com/orgs/airbytehqBLA/repos?per_page=100
             org_name = message.split("https://api.github.com/orgs/")[1].split("/")[0]
             user_message = f'Organization name: "{org_name}" is unknown, "repository" config option should be updated. Please validate your repository config.'
-        elif "401 Client Error: Unauthorized for url" in message:
+        elif "401 Client Error: Unauthorized for url" in message or ("Error: Unauthorized" in message and "401" in message):
             # 401 Client Error: Unauthorized for url: https://api.github.com/orgs/datarootsio/repos?per_page=100&sort=updated&direction=desc
             user_message = (
                 "Github credentials have expired or changed, please review your credentials and re-authenticate or renew your access token."
@@ -203,6 +204,9 @@ class SourceGithub(AbstractSource):
                 )
             return True, None
 
+        except MessageRepresentationAirbyteTracedErrors as e:
+            user_message = self.user_friendly_error_message(e.message)
+            return False, user_message or e.message
         except Exception as e:
             message = repr(e)
             user_message = self.user_friendly_error_message(message)

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/source_github/streams.py

@@ -4,14 +4,15 @@
 
 import re
 from abc import ABC, abstractmethod
+from datetime import timedelta, timezone
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 from urllib import parse
 
-import pendulum
 import requests
+from dateutil.parser import parse as date_parse
 
 from airbyte_cdk import BackoffStrategy, StreamSlice
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level, SyncMode
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
 from airbyte_cdk.sources.streams.checkpoint.substream_resumable_full_refresh_cursor import SubstreamResumableFullRefreshCursor
@@ -19,8 +20,9 @@ from airbyte_cdk.sources.streams.core import CheckpointMixin, Stream
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction
 from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException
+from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors
 from airbyte_cdk.utils import AirbyteTracedException
-from airbyte_protocol.models import FailureType
+from airbyte_cdk.utils.datetime_helpers import ab_datetime_format, ab_datetime_parse
 
 from . import constants
 from .backoff_strategies import ContributorActivityBackoffStrategy, GithubStreamABCBackoffStrategy
@@ -128,11 +130,14 @@ class GithubStreamABC(HttpStream, ABC):
         # Reading records while handling the errors
         try:
             yield from super().read_records(stream_slice=stream_slice, **kwargs)
-        except DefaultBackoffException as e:
+        # HTTP Client wraps DefaultBackoffException into MessageRepresentationAirbyteTracedErrors
+        except MessageRepresentationAirbyteTracedErrors as e:
             # This whole try/except situation in `read_records()` isn't good but right now in `self._send_request()`
             # function we have `response.raise_for_status()` so we don't have much choice on how to handle errors.
             # Bocked on https://github.com/airbytehq/airbyte/issues/3514.
-            if e.response.status_code == requests.codes.NOT_FOUND:
+            if not hasattr(e, "_exception") and not hasattr(e._exception, "response"):
+                raise e
+            if e._exception.response.status_code == requests.codes.NOT_FOUND:
                 # A lot of streams are not available for repositories owned by a user instead of an organization.
                 if isinstance(self, Organizations):
                     error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for organization `{organisation}`."
@@ -140,8 +145,8 @@ class GithubStreamABC(HttpStream, ABC):
                     error_msg = f"Syncing `{self.__class__.__name__}` stream for organization `{organisation}`, team `{stream_slice.get('team_slug')}` and user `{stream_slice.get('username')}` isn't available: User has no team membership. Skipping..."
                 else:
                     error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for repository `{repository}`."
-            elif e.response.status_code == requests.codes.FORBIDDEN:
-                error_msg = str(e.response.json().get("message"))
+            elif e._exception.response.status_code == requests.codes.FORBIDDEN:
+                error_msg = str(e._exception.response.json().get("message"))
                 # When using the `check_connection` method, we should raise an error if we do not have access to the repository.
                 if isinstance(self, Repositories):
                     raise e
@@ -157,27 +162,27 @@ class GithubStreamABC(HttpStream, ABC):
                     error_msg = (
                         f"Syncing `{self.name}` stream isn't available for repository `{repository}`. Full error message: {error_msg}"
                     )
-            elif e.response.status_code == requests.codes.UNAUTHORIZED:
+            elif e._exception.response.status_code == requests.codes.UNAUTHORIZED:
                 if self.access_token_type == constants.PERSONAL_ACCESS_TOKEN_TITLE:
-                    error_msg = str(e.response.json().get("message"))
+                    error_msg = str(e._exception.response.json().get("message"))
                     self.logger.error(f"{self.access_token_type} renewal is required: {error_msg}")
                 raise e
-            elif e.response.status_code == requests.codes.GONE and isinstance(self, Projects):
+            elif e._exception.response.status_code == requests.codes.GONE and isinstance(self, Projects):
                 # Some repos don't have projects enabled and we we get "410 Client Error: Gone for
                 # url: https://api.github.com/repos/xyz/projects?per_page=100" error.
                 error_msg = f"Syncing `Projects` stream isn't available for repository `{stream_slice['repository']}`."
-            elif e.response.status_code == requests.codes.CONFLICT:
+            elif e._exception.response.status_code == requests.codes.CONFLICT:
                 error_msg = (
                     f"Syncing `{self.name}` stream isn't available for repository "
                     f"`{stream_slice['repository']}`, it seems like this repository is empty."
                 )
-            elif e.response.status_code == requests.codes.SERVER_ERROR and isinstance(self, WorkflowRuns):
+            elif e._exception.response.status_code == requests.codes.SERVER_ERROR and isinstance(self, WorkflowRuns):
                 error_msg = f"Syncing `{self.name}` stream isn't available for repository `{stream_slice['repository']}`."
-            elif e.response.status_code == requests.codes.BAD_GATEWAY:
+            elif e._exception.response.status_code == requests.codes.BAD_GATEWAY:
                 error_msg = f"Stream {self.name} temporary failed. Try to re-run sync later"
             else:
                 # most probably here we're facing a 500 server error and a risk to get a non-json response, so lets output response.text
-                self.logger.error(f"Undefined error while reading records: {e.response.text}")
+                self.logger.error(f"Undefined error while reading records: {e._exception.response.text}")
                 raise e
 
             self.logger.warning(error_msg)
@@ -216,6 +221,14 @@ class GithubStream(GithubStreamABC):
 
     def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
         record["repository"] = stream_slice["repository"]
+
+        if "reactions" in record and record["reactions"]:
+            reactions = record["reactions"]
+            if "+1" in reactions:
+                reactions["plus_one"] = reactions.pop("+1")
+            if "-1" in reactions:
+                reactions["minus_one"] = reactions.pop("-1")
+
         return record
 
     def parse_response(
@@ -1437,7 +1450,8 @@ class Workflows(SemiIncrementalMixin, GithubStream):
             yield self.transform(record=record, stream_slice=stream_slice)
 
     def convert_cursor_value(self, value):
-        return pendulum.parse(value).in_tz(tz="UTC").format("YYYY-MM-DDTHH:mm:ss[Z]")
+        parsed_value = date_parse(value).astimezone(timezone.utc)
+        return ab_datetime_format(parsed_value, "%Y-%m-%dT%H:%M:%SZ")
 
 
 class WorkflowRuns(SemiIncrementalMixin, GithubStream):
@@ -1478,7 +1492,7 @@ class WorkflowRuns(SemiIncrementalMixin, GithubStream):
         # the state is updated only in the end of the sync as records are sorted in reverse order
         new_state = self.state
         if start_point:
-            break_point = (pendulum.parse(start_point) - pendulum.duration(days=self.re_run_period)).to_iso8601_string()
+            break_point = (ab_datetime_parse(start_point) - timedelta(days=self.re_run_period)).isoformat()
         for record in super(SemiIncrementalMixin, self).read_records(
             sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state
         ):
@@ -1663,22 +1677,24 @@ class ContributorActivity(GithubStream):
         repository = stream_slice.get("repository", "")
         try:
             yield from super().read_records(stream_slice=stream_slice, **kwargs)
-        except UserDefinedBackoffException as e:
-            if e.response.status_code == requests.codes.ACCEPTED:
-                yield AirbyteMessage(
-                    type=MessageType.LOG,
-                    log=AirbyteLogMessage(
-                        level=Level.INFO,
-                        message=f"Syncing `{self.__class__.__name__}` " f"stream isn't available for repository `{repository}`.",
-                    ),
-                )
+        # HTTP Client wraps BackoffException into MessageRepresentationAirbyteTracedErrors
+        except MessageRepresentationAirbyteTracedErrors as e:
+            if hasattr(e, "_exception") and hasattr(e._exception, "response"):
+                if e._exception.response.status_code == requests.codes.ACCEPTED:
+                    yield AirbyteMessage(
+                        type=MessageType.LOG,
+                        log=AirbyteLogMessage(
+                            level=Level.INFO,
+                            message=f"Syncing `{self.__class__.__name__}` stream isn't available for repository `{repository}`.",
+                        ),
+                    )
 
-                # In order to retain the existing stream behavior before we added RFR to this stream, we need to close out the
-                # partition after we give up the maximum number of retries on the 202 response. This does lead to the question
-                # of if we should prematurely exit in the first place, but for now we're going to aim for feature parity
-                partition_obj = stream_slice.get("partition")
-                if self.cursor and partition_obj:
-                    self.cursor.close_slice(StreamSlice(cursor_slice={}, partition=partition_obj))
+                    # In order to retain the existing stream behavior before we added RFR to this stream, we need to close out the
+                    # partition after we give up the maximum number of retries on the 202 response. This does lead to the question
+                    # of if we should prematurely exit in the first place, but for now we're going to aim for feature parity
+                    partition_obj = stream_slice.get("partition")
+                    if self.cursor and partition_obj:
+                        self.cursor.close_slice(StreamSlice(cursor_slice={}, partition=partition_obj))
             else:
                 raise e
 

{airbyte_source_github-1.8.39 → airbyte_source_github-2.1.5}/source_github/utils.py

@@ -1,19 +1,22 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+import logging
 import time
 from dataclasses import dataclass
+from datetime import timedelta
 from itertools import cycle
 from typing import Any, List, Mapping
 
-import pendulum
 import requests
 
-from airbyte_cdk.models import SyncMode
+from airbyte_cdk.models import FailureType, SyncMode
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http import HttpClient
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
 
 
 def getter(D: dict, key_or_keys, strict=True):
@@ -43,8 +46,8 @@ class GitHubAPILimitException(Exception):
 class Token:
     count_rest: int = 5000
     count_graphql: int = 5000
-    reset_at_rest: pendulum.DateTime = pendulum.now()
-    reset_at_graphql: pendulum.DateTime = pendulum.now()
+    reset_at_rest: AirbyteDateTime = ab_datetime_now()
+    reset_at_graphql: AirbyteDateTime = ab_datetime_now()
 
 
 class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
@@ -55,17 +58,33 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
     the first token becomes available again.
     """
 
-    DURATION = pendulum.duration(seconds=3600)  # Duration at which the current rate limit window resets
+    DURATION = timedelta(seconds=3600)  # Duration at which the current rate limit window resets
 
     def __init__(self, tokens: List[str], auth_method: str = "token", auth_header: str = "Authorization"):
+        self._logger = logging.getLogger("airbyte")
         self._auth_method = auth_method
         self._auth_header = auth_header
         self._tokens = {t: Token() for t in tokens}
+        # It would've been nice to instantiate a single client on this authenticator. However, we are checking
+        # the limits of each token which is associated with a TokenAuthenticator. And each HttpClient can only
+        # correspond to one authenticator.
+        self._token_to_http_client: Mapping[str, HttpClient] = self._initialize_http_clients(tokens)
         self.check_all_tokens()
         self._tokens_iter = cycle(self._tokens)
         self._active_token = next(self._tokens_iter)
         self._max_time = 60 * 10  # 10 minutes as default
 
+    def _initialize_http_clients(self, tokens: List[str]) -> Mapping[str, HttpClient]:
+        return {
+            token: HttpClient(
+                name="token_validator",
+                logger=self._logger,
+                authenticator=TokenAuthenticator(token, auth_method=self._auth_method),
+                use_cache=False,  # We don't want to reuse cached valued because rate limit values change frequently
+            )
+            for token in tokens
+        }
+
     @property
     def auth_header(self) -> str:
         return self._auth_header
@@ -113,25 +132,38 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
 
     def _check_token_limits(self, token: str):
         """check that token is not limited"""
-        headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}
-        rate_limit_info = (
-            requests.get(
-                "https://api.github.com/rate_limit", headers=headers, auth=TokenAuthenticator(token, auth_method=self._auth_method)
-            )
-            .json()
-            .get("resources")
+
+        http_client = self._token_to_http_client.get(token)
+        if not http_client:
+            raise ValueError("No HttpClient was initialized for this token. This is unexpected. Please contact Airbyte support.")
+
+        _, response = http_client.send_request(
+            http_method="GET",
+            url="https://api.github.com/rate_limit",
+            headers={"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"},
+            request_kwargs={},
         )
+
+        response_body = response.json()
+        if "resources" not in response_body:
+            raise AirbyteTracedException(
+                failure_type=FailureType.config_error,
+                internal_message=f"Token rate limit info response did not contain expected key: resources",
+                message="Unable to validate token. Please double check that specified authentication tokens are correct",
+            )
+
+        rate_limit_info = response_body.get("resources")
         token_info = self._tokens[token]
         remaining_info_core = rate_limit_info.get("core")
         token_info.count_rest, token_info.reset_at_rest = (
             remaining_info_core.get("remaining"),
-            pendulum.from_timestamp(remaining_info_core.get("reset")),
+            ab_datetime_parse(remaining_info_core.get("reset")),
         )
 
         remaining_info_graphql = rate_limit_info.get("graphql")
         token_info.count_graphql, token_info.reset_at_graphql = (
             remaining_info_graphql.get("remaining"),
-            pendulum.from_timestamp(remaining_info_graphql.get("reset")),
+            ab_datetime_parse(remaining_info_graphql.get("reset")),
         )
 
     def check_all_tokens(self):
@@ -143,7 +175,7 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
             setattr(current_token, count_attr, getattr(current_token, count_attr) - 1)
             return True
         elif all(getattr(x, count_attr) == 0 for x in self._tokens.values()):
-            min_time_to_wait = min((getattr(x, reset_attr) - pendulum.now()).in_seconds() for x in self._tokens.values())
+            min_time_to_wait = min((getattr(x, reset_attr) - ab_datetime_now()).total_seconds() for x in self._tokens.values())
             if min_time_to_wait < self.max_time:
                 time.sleep(min_time_to_wait if min_time_to_wait > 0 else 0)
                 self.check_all_tokens()