airbyte-source-github 1.8.13__tar.gz → 2.1.5__tar.gz

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/PKG-INFO

@@ -1,17 +1,19 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 1.8.13
+Version: 2.1.5
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
-License: MIT
+License: ELv2
 Author: Airbyte
 Author-email: contact@airbyte.io
-Requires-Python: >=3.10,<3.12
-Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.10,<3.14
+Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=4,<5)
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: airbyte-cdk (>=7.4.1,<8.0.0)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/github
 Project-URL: Repository, https://github.com/airbytehq/airbyte

airbyte_source_github-2.1.5/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = [ "poetry-core>=1.0.0",]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+version = "2.1.5"
+name = "airbyte-source-github"
+description = "Source implementation for GitHub."
+authors = [ "Airbyte <contact@airbyte.io>",]
+license = "ELv2"
+readme = "README.md"
+documentation = "https://docs.airbyte.com/integrations/sources/github"
+homepage = "https://airbyte.com"
+repository = "https://github.com/airbytehq/airbyte"
+[[tool.poetry.packages]]
+include = "source_github"
+
+[tool.poetry.dependencies]
+python = "^3.10,<3.14"
+airbyte-cdk = "^7.4.1"
+sgqlc = "==16.3"
+
+[tool.poetry.scripts]
+source-github = "source_github.run:run"
+
+[tool.poetry.group.dev.dependencies]
+requests-mock = "^1.9.3"
+freezegun = "^1.2"
+pytest-mock = "^3.6.1"
+pytest = "^8.0.0"
+responses = "^0.23.1"
+
+
+[tool.poe]
+include = [
+    # Shared tasks definition file(s) can be imported here.
+    # Run `poe` or `poe --help` to see the list of available tasks.
+    "${POE_GIT_DIR}/poe-tasks/poetry-connector-tasks.toml",
+]
+
+[tool.poe.tasks]
+# This overrides the default task for running unit tests specifically for the source-github connector.
+# This is a workaround to an issue that arises when running all of the unit tests in one execution of pytest 
+# due to incompatibilities between request mocking libraries requests-mock and responses.
+test-unit-tests.shell = '''
+poetry run pytest --junitxml=build/test-results/pytest-unit-tests-junit.xml --ignore=unit_tests/integration/ unit_tests
+poetry run pytest --junitxml=build/test-results/pytest-unit-integration-tests-junit.xml unit_tests/integration/
+'''

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/__init__.py

@@ -24,4 +24,5 @@ SOFTWARE.
 
 from .source import SourceGithub
 
+
 __all__ = ["SourceGithub"]

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/backoff_strategies.py

@@ -6,6 +6,7 @@ import time
 from typing import Any, Optional, Union
 
 import requests
+
 from airbyte_cdk import BackoffStrategy
 from airbyte_cdk.sources.streams.http import HttpStream
 

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/config_migrations.py

@@ -6,12 +6,16 @@ import logging
 from abc import ABC
 from typing import Any, List, Mapping
 
+import orjson
+
 from airbyte_cdk.config_observation import create_connector_config_control_message
 from airbyte_cdk.entrypoint import AirbyteEntrypoint
+from airbyte_cdk.models import AirbyteMessageSerializer
 from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 
 from .source import SourceGithub
 
+
 logger = logging.getLogger("airbyte_logger")
 
 
@@ -30,13 +34,11 @@ class MigrateStringToArray(ABC):
 
     @property
     @abc.abstractmethod
-    def migrate_from_key(self) -> str:
-        ...
+    def migrate_from_key(self) -> str: ...
 
     @property
     @abc.abstractmethod
-    def migrate_to_key(self) -> str:
-        ...
+    def migrate_to_key(self) -> str: ...
 
     @classmethod
     def _should_migrate(cls, config: Mapping[str, Any]) -> bool:
@@ -73,7 +75,7 @@ class MigrateStringToArray(ABC):
         cls.message_repository.emit_message(create_connector_config_control_message(migrated_config))
         # emit the Airbyte Control Message from message queue to stdout
         for message in cls.message_repository._message_queue:
-            print(message.json(exclude_unset=True))
+            print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
 
     @classmethod
     def migrate(cls, args: List[str], source: SourceGithub) -> None:
@@ -95,12 +97,10 @@ class MigrateStringToArray(ABC):
 
 
 class MigrateRepository(MigrateStringToArray):
-
     migrate_from_key: str = "repository"
     migrate_to_key: str = "repositories"
 
 
 class MigrateBranch(MigrateStringToArray):
-
     migrate_from_key: str = "branch"
     migrate_to_key: str = "branches"

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/errors_handlers.py

@@ -5,13 +5,15 @@
 from typing import Optional, Union
 
 import requests
+
+from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction
 from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
-from airbyte_protocol.models import FailureType
 
 from . import constants
 
+
 GITHUB_DEFAULT_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | {
     401: ErrorResolution(
         response_action=ResponseAction.RETRY,
@@ -41,6 +43,13 @@ GITHUB_DEFAULT_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | {
 }
 
 
+def is_conflict_with_empty_repository(response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> bool:
+    if isinstance(response_or_exception, requests.Response) and response_or_exception.status_code == requests.codes.CONFLICT:
+        response_data = response_or_exception.json()
+        return response_data.get("message") == "Git Repository is empty."
+    return False
+
+
 class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
     def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
         self.stream = stream
@@ -86,6 +95,14 @@ class GithubStreamABCErrorHandler(HttpStatusErrorHandler):
                     error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
                 )
 
+            if is_conflict_with_empty_repository(response_or_exception=response_or_exception):
+                log_message = f"Ignoring response for '{response_or_exception.request.method}' request to '{response_or_exception.url}' with response code '{response_or_exception.status_code}' as the repository is empty."
+                return ErrorResolution(
+                    response_action=ResponseAction.IGNORE,
+                    failure_type=FailureType.config_error,
+                    error_message=log_message,
+                )
+
         return super().interpret_response(response_or_exception)
 
 

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/github_schema.py

@@ -6,6 +6,7 @@ import sgqlc.types
 import sgqlc.types.datetime
 import sgqlc.types.relay
 
+
 github_schema = sgqlc.types.Schema()
 
 

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/schemas/shared/events/comment.json

@@ -174,8 +174,8 @@
       "properties": {
         "url": { "type": "string" },
         "total_count": { "type": "integer" },
-        "+1": { "type": "integer" },
-        "-1": { "type": "integer" },
+        "plus_one": { "type": "integer" },
+        "minus_one": { "type": "integer" },
         "laugh": { "type": "integer" },
         "confused": { "type": "integer" },
         "heart": { "type": "integer" },

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/schemas/shared/events/commented.json

@@ -99,8 +99,8 @@
             "properties": {
               "url": { "type": "string" },
               "total_count": { "type": "integer" },
-              "+1": { "type": "integer" },
-              "-1": { "type": "integer" },
+              "plus_one": { "type": "integer" },
+              "minus_one": { "type": "integer" },
               "laugh": { "type": "integer" },
               "confused": { "type": "integer" },
               "heart": { "type": "integer" },

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/schemas/shared/events/cross_referenced.json

@@ -752,10 +752,10 @@
                   "total_count": {
                     "type": "integer"
                   },
-                  "+1": {
+                  "plus_one": {
                     "type": "integer"
                   },
-                  "-1": {
+                  "minus_one": {
                     "type": "integer"
                   },
                   "laugh": {
@@ -788,10 +788,10 @@
                 "total_count": {
                   "type": "integer"
                 },
-                "+1": {
+                "plus_one": {
                   "type": "integer"
                 },
-                "-1": {
+                "minus_one": {
                   "type": "integer"
                 },
                 "laugh": {

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/schemas/shared/reactions.json

@@ -7,10 +7,10 @@
     "total_count": {
       "type": ["null", "integer"]
     },
-    "+1": {
+    "plus_one": {
       "type": ["null", "integer"]
     },
-    "-1": {
+    "minus_one": {
       "type": ["null", "integer"]
     },
     "laugh": {

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/source.py

@@ -9,6 +9,7 @@ from urllib.parse import urlparse
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors
 from airbyte_cdk.sources.streams.http.requests_native_auth import MultipleTokenAuthenticator
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter
@@ -60,7 +61,6 @@ from .utils import read_full_refresh
 
 
 class SourceGithub(AbstractSource):
-
     continue_sync_on_stream_failure = True
 
     @staticmethod
@@ -185,7 +185,7 @@ class SourceGithub(AbstractSource):
             # 404 Client Error: Not Found for url: https://api.github.com/orgs/airbytehqBLA/repos?per_page=100
             org_name = message.split("https://api.github.com/orgs/")[1].split("/")[0]
             user_message = f'Organization name: "{org_name}" is unknown, "repository" config option should be updated. Please validate your repository config.'
-        elif "401 Client Error: Unauthorized for url" in message:
+        elif "401 Client Error: Unauthorized for url" in message or ("Error: Unauthorized" in message and "401" in message):
             # 401 Client Error: Unauthorized for url: https://api.github.com/orgs/datarootsio/repos?per_page=100&sort=updated&direction=desc
             user_message = (
                 "Github credentials have expired or changed, please review your credentials and re-authenticate or renew your access token."
@@ -204,6 +204,9 @@ class SourceGithub(AbstractSource):
                 )
             return True, None
 
+        except MessageRepresentationAirbyteTracedErrors as e:
+            user_message = self.user_friendly_error_message(e.message)
+            return False, user_message or e.message
         except Exception as e:
             message = repr(e)
             user_message = self.user_friendly_error_message(message)

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/streams.py

@@ -4,13 +4,15 @@
 
 import re
 from abc import ABC, abstractmethod
+from datetime import timedelta, timezone
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 from urllib import parse
 
-import pendulum
 import requests
+from dateutil.parser import parse as date_parse
+
 from airbyte_cdk import BackoffStrategy, StreamSlice
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level, SyncMode
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
 from airbyte_cdk.sources.streams.checkpoint.substream_resumable_full_refresh_cursor import SubstreamResumableFullRefreshCursor
@@ -18,8 +20,9 @@ from airbyte_cdk.sources.streams.core import CheckpointMixin, Stream
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction
 from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException
+from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors
 from airbyte_cdk.utils import AirbyteTracedException
-from airbyte_protocol.models import FailureType
+from airbyte_cdk.utils.datetime_helpers import ab_datetime_format, ab_datetime_parse
 
 from . import constants
 from .backoff_strategies import ContributorActivityBackoffStrategy, GithubStreamABCBackoffStrategy
@@ -28,6 +31,7 @@ from .errors_handlers import (
     ContributorActivityErrorHandler,
     GitHubGraphQLErrorHandler,
     GithubStreamABCErrorHandler,
+    is_conflict_with_empty_repository,
 )
 from .graphql import (
     CursorStorage,
@@ -41,7 +45,6 @@ from .utils import GitHubAPILimitException, getter
 
 
 class GithubStreamABC(HttpStream, ABC):
-
     primary_key = "id"
 
     # Detect streams with high API load
@@ -80,7 +83,6 @@ class GithubStreamABC(HttpStream, ABC):
     def request_params(
         self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
     ) -> MutableMapping[str, Any]:
-
         params = {"per_page": self.page_size}
 
         if next_page_token:
@@ -128,11 +130,14 @@ class GithubStreamABC(HttpStream, ABC):
         # Reading records while handling the errors
         try:
             yield from super().read_records(stream_slice=stream_slice, **kwargs)
-        except DefaultBackoffException as e:
+        # HTTP Client wraps DefaultBackoffException into MessageRepresentationAirbyteTracedErrors
+        except MessageRepresentationAirbyteTracedErrors as e:
             # This whole try/except situation in `read_records()` isn't good but right now in `self._send_request()`
             # function we have `response.raise_for_status()` so we don't have much choice on how to handle errors.
             # Bocked on https://github.com/airbytehq/airbyte/issues/3514.
-            if e.response.status_code == requests.codes.NOT_FOUND:
+            if not hasattr(e, "_exception") and not hasattr(e._exception, "response"):
+                raise e
+            if e._exception.response.status_code == requests.codes.NOT_FOUND:
                 # A lot of streams are not available for repositories owned by a user instead of an organization.
                 if isinstance(self, Organizations):
                     error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for organization `{organisation}`."
@@ -140,8 +145,8 @@ class GithubStreamABC(HttpStream, ABC):
                     error_msg = f"Syncing `{self.__class__.__name__}` stream for organization `{organisation}`, team `{stream_slice.get('team_slug')}` and user `{stream_slice.get('username')}` isn't available: User has no team membership. Skipping..."
                 else:
                     error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for repository `{repository}`."
-            elif e.response.status_code == requests.codes.FORBIDDEN:
-                error_msg = str(e.response.json().get("message"))
+            elif e._exception.response.status_code == requests.codes.FORBIDDEN:
+                error_msg = str(e._exception.response.json().get("message"))
                 # When using the `check_connection` method, we should raise an error if we do not have access to the repository.
                 if isinstance(self, Repositories):
                     raise e
@@ -157,27 +162,27 @@ class GithubStreamABC(HttpStream, ABC):
                     error_msg = (
                         f"Syncing `{self.name}` stream isn't available for repository `{repository}`. Full error message: {error_msg}"
                     )
-            elif e.response.status_code == requests.codes.UNAUTHORIZED:
+            elif e._exception.response.status_code == requests.codes.UNAUTHORIZED:
                 if self.access_token_type == constants.PERSONAL_ACCESS_TOKEN_TITLE:
-                    error_msg = str(e.response.json().get("message"))
+                    error_msg = str(e._exception.response.json().get("message"))
                     self.logger.error(f"{self.access_token_type} renewal is required: {error_msg}")
                 raise e
-            elif e.response.status_code == requests.codes.GONE and isinstance(self, Projects):
+            elif e._exception.response.status_code == requests.codes.GONE and isinstance(self, Projects):
                 # Some repos don't have projects enabled and we we get "410 Client Error: Gone for
                 # url: https://api.github.com/repos/xyz/projects?per_page=100" error.
                 error_msg = f"Syncing `Projects` stream isn't available for repository `{stream_slice['repository']}`."
-            elif e.response.status_code == requests.codes.CONFLICT:
+            elif e._exception.response.status_code == requests.codes.CONFLICT:
                 error_msg = (
                     f"Syncing `{self.name}` stream isn't available for repository "
                     f"`{stream_slice['repository']}`, it seems like this repository is empty."
                 )
-            elif e.response.status_code == requests.codes.SERVER_ERROR and isinstance(self, WorkflowRuns):
+            elif e._exception.response.status_code == requests.codes.SERVER_ERROR and isinstance(self, WorkflowRuns):
                 error_msg = f"Syncing `{self.name}` stream isn't available for repository `{stream_slice['repository']}`."
-            elif e.response.status_code == requests.codes.BAD_GATEWAY:
+            elif e._exception.response.status_code == requests.codes.BAD_GATEWAY:
                 error_msg = f"Stream {self.name} temporary failed. Try to re-run sync later"
             else:
                 # most probably here we're facing a 500 server error and a risk to get a non-json response, so lets output response.text
-                self.logger.error(f"Undefined error while reading records: {e.response.text}")
+                self.logger.error(f"Undefined error while reading records: {e._exception.response.text}")
                 raise e
 
             self.logger.warning(error_msg)
@@ -216,8 +221,34 @@ class GithubStream(GithubStreamABC):
 
     def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
         record["repository"] = stream_slice["repository"]
+
+        if "reactions" in record and record["reactions"]:
+            reactions = record["reactions"]
+            if "+1" in reactions:
+                reactions["plus_one"] = reactions.pop("+1")
+            if "-1" in reactions:
+                reactions["minus_one"] = reactions.pop("-1")
+
         return record
 
+    def parse_response(
+        self,
+        response: requests.Response,
+        stream_state: Mapping[str, Any],
+        stream_slice: Mapping[str, Any] = None,
+        next_page_token: Mapping[str, Any] = None,
+    ) -> Iterable[Mapping]:
+        if is_conflict_with_empty_repository(response):
+            # I would expect that this should be handled (skipped) by the error handler, but it seems like
+            # ignored this error but continue to processing records. This may be fixed in latest CDK versions.
+            return
+        yield from super().parse_response(
+            response=response,
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+            next_page_token=next_page_token,
+        )
+
 
 class SemiIncrementalMixin(CheckpointMixin):
     """
@@ -756,7 +787,6 @@ class ReviewComments(IncrementalMixin, GithubStream):
 
 
 class GitHubGraphQLStream(GithubStream, ABC):
-
     http_method = "POST"
 
     def path(
@@ -976,7 +1006,6 @@ class ProjectsV2(SemiIncrementalMixin, GitHubGraphQLStream):
 
 
 class ReactionStream(GithubStream, CheckpointMixin, ABC):
-
     parent_key = "id"
     copy_parent_key = "comment_id"
     cursor_field = "created_at"
@@ -1394,9 +1423,9 @@ class ProjectCards(GithubStream):
         stream_state_value = current_stream_state.get(repository, {}).get(project_id, {}).get(column_id, {}).get(self.cursor_field)
         if stream_state_value:
             updated_state = max(updated_state, stream_state_value)
-        current_stream_state.setdefault(repository, {}).setdefault(project_id, {}).setdefault(column_id, {})[
-            self.cursor_field
-        ] = updated_state
+        current_stream_state.setdefault(repository, {}).setdefault(project_id, {}).setdefault(column_id, {})[self.cursor_field] = (
+            updated_state
+        )
         return current_stream_state
 
     def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
@@ -1421,7 +1450,8 @@ class Workflows(SemiIncrementalMixin, GithubStream):
             yield self.transform(record=record, stream_slice=stream_slice)
 
     def convert_cursor_value(self, value):
-        return pendulum.parse(value).in_tz(tz="UTC").format("YYYY-MM-DDTHH:mm:ss[Z]")
+        parsed_value = date_parse(value).astimezone(timezone.utc)
+        return ab_datetime_format(parsed_value, "%Y-%m-%dT%H:%M:%SZ")
 
 
 class WorkflowRuns(SemiIncrementalMixin, GithubStream):
@@ -1462,7 +1492,7 @@ class WorkflowRuns(SemiIncrementalMixin, GithubStream):
         # the state is updated only in the end of the sync as records are sorted in reverse order
         new_state = self.state
         if start_point:
-            break_point = (pendulum.parse(start_point) - pendulum.duration(days=self.re_run_period)).to_iso8601_string()
+            break_point = (ab_datetime_parse(start_point) - timedelta(days=self.re_run_period)).isoformat()
         for record in super(SemiIncrementalMixin, self).read_records(
             sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state
         ):
@@ -1617,11 +1647,13 @@ class ContributorActivity(GithubStream):
 
     def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
         record["repository"] = stream_slice["repository"]
-        record.update(record.pop("author"))
+        author = record.pop("author", None)
+        # It's been found that the author field can be None, so we check for it
+        if author:
+            record.update(author)
         return record
 
     def get_error_handler(self) -> Optional[ErrorHandler]:
-
         return ContributorActivityErrorHandler(logger=self.logger, max_retries=5, error_mapping=GITHUB_DEFAULT_ERROR_MAPPING)
 
     def get_backoff_strategy(self) -> Optional[Union[BackoffStrategy, List[BackoffStrategy]]]:
@@ -1645,22 +1677,24 @@ class ContributorActivity(GithubStream):
         repository = stream_slice.get("repository", "")
         try:
             yield from super().read_records(stream_slice=stream_slice, **kwargs)
-        except UserDefinedBackoffException as e:
-            if e.response.status_code == requests.codes.ACCEPTED:
-                yield AirbyteMessage(
-                    type=MessageType.LOG,
-                    log=AirbyteLogMessage(
-                        level=Level.INFO,
-                        message=f"Syncing `{self.__class__.__name__}` " f"stream isn't available for repository `{repository}`.",
-                    ),
-                )
+        # HTTP Client wraps BackoffException into MessageRepresentationAirbyteTracedErrors
+        except MessageRepresentationAirbyteTracedErrors as e:
+            if hasattr(e, "_exception") and hasattr(e._exception, "response"):
+                if e._exception.response.status_code == requests.codes.ACCEPTED:
+                    yield AirbyteMessage(
+                        type=MessageType.LOG,
+                        log=AirbyteLogMessage(
+                            level=Level.INFO,
+                            message=f"Syncing `{self.__class__.__name__}` stream isn't available for repository `{repository}`.",
+                        ),
+                    )
 
-                # In order to retain the existing stream behavior before we added RFR to this stream, we need to close out the
-                # partition after we give up the maximum number of retries on the 202 response. This does lead to the question
-                # of if we should prematurely exit in the first place, but for now we're going to aim for feature parity
-                partition_obj = stream_slice.get("partition")
-                if self.cursor and partition_obj:
-                    self.cursor.close_slice(StreamSlice(cursor_slice={}, partition=partition_obj))
+                    # In order to retain the existing stream behavior before we added RFR to this stream, we need to close out the
+                    # partition after we give up the maximum number of retries on the 202 response. This does lead to the question
+                    # of if we should prematurely exit in the first place, but for now we're going to aim for feature parity
+                    partition_obj = stream_slice.get("partition")
+                    if self.cursor and partition_obj:
+                        self.cursor.close_slice(StreamSlice(cursor_slice={}, partition=partition_obj))
             else:
                 raise e
 

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/utils.py

@@ -1,18 +1,22 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+import logging
 import time
 from dataclasses import dataclass
+from datetime import timedelta
 from itertools import cycle
 from typing import Any, List, Mapping
 
-import pendulum
 import requests
-from airbyte_cdk.models import SyncMode
+
+from airbyte_cdk.models import FailureType, SyncMode
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http import HttpClient
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
 
 
 def getter(D: dict, key_or_keys, strict=True):
@@ -42,8 +46,8 @@ class GitHubAPILimitException(Exception):
 class Token:
     count_rest: int = 5000
     count_graphql: int = 5000
-    reset_at_rest: pendulum.DateTime = pendulum.now()
-    reset_at_graphql: pendulum.DateTime = pendulum.now()
+    reset_at_rest: AirbyteDateTime = ab_datetime_now()
+    reset_at_graphql: AirbyteDateTime = ab_datetime_now()
 
 
 class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
@@ -54,17 +58,33 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
     the first token becomes available again.
     """
 
-    DURATION = pendulum.duration(seconds=3600)  # Duration at which the current rate limit window resets
+    DURATION = timedelta(seconds=3600)  # Duration at which the current rate limit window resets
 
     def __init__(self, tokens: List[str], auth_method: str = "token", auth_header: str = "Authorization"):
+        self._logger = logging.getLogger("airbyte")
         self._auth_method = auth_method
         self._auth_header = auth_header
         self._tokens = {t: Token() for t in tokens}
+        # It would've been nice to instantiate a single client on this authenticator. However, we are checking
+        # the limits of each token which is associated with a TokenAuthenticator. And each HttpClient can only
+        # correspond to one authenticator.
+        self._token_to_http_client: Mapping[str, HttpClient] = self._initialize_http_clients(tokens)
         self.check_all_tokens()
         self._tokens_iter = cycle(self._tokens)
         self._active_token = next(self._tokens_iter)
         self._max_time = 60 * 10  # 10 minutes as default
 
+    def _initialize_http_clients(self, tokens: List[str]) -> Mapping[str, HttpClient]:
+        return {
+            token: HttpClient(
+                name="token_validator",
+                logger=self._logger,
+                authenticator=TokenAuthenticator(token, auth_method=self._auth_method),
+                use_cache=False,  # We don't want to reuse cached valued because rate limit values change frequently
+            )
+            for token in tokens
+        }
+
     @property
     def auth_header(self) -> str:
         return self._auth_header
@@ -99,7 +119,6 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
 
     @property
     def token(self) -> str:
-
         token = self.current_active_token
         return f"{self._auth_method} {token}"
 
@@ -113,23 +132,38 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
 
     def _check_token_limits(self, token: str):
         """check that token is not limited"""
-        headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}
-        rate_limit_info = (
-            requests.get(
-                "https://api.github.com/rate_limit", headers=headers, auth=TokenAuthenticator(token, auth_method=self._auth_method)
-            )
-            .json()
-            .get("resources")
+
+        http_client = self._token_to_http_client.get(token)
+        if not http_client:
+            raise ValueError("No HttpClient was initialized for this token. This is unexpected. Please contact Airbyte support.")
+
+        _, response = http_client.send_request(
+            http_method="GET",
+            url="https://api.github.com/rate_limit",
+            headers={"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"},
+            request_kwargs={},
         )
+
+        response_body = response.json()
+        if "resources" not in response_body:
+            raise AirbyteTracedException(
+                failure_type=FailureType.config_error,
+                internal_message=f"Token rate limit info response did not contain expected key: resources",
+                message="Unable to validate token. Please double check that specified authentication tokens are correct",
+            )
+
+        rate_limit_info = response_body.get("resources")
         token_info = self._tokens[token]
         remaining_info_core = rate_limit_info.get("core")
-        token_info.count_rest, token_info.reset_at_rest = remaining_info_core.get("remaining"), pendulum.from_timestamp(
-            remaining_info_core.get("reset")
+        token_info.count_rest, token_info.reset_at_rest = (
+            remaining_info_core.get("remaining"),
+            ab_datetime_parse(remaining_info_core.get("reset")),
         )
 
         remaining_info_graphql = rate_limit_info.get("graphql")
-        token_info.count_graphql, token_info.reset_at_graphql = remaining_info_graphql.get("remaining"), pendulum.from_timestamp(
-            remaining_info_graphql.get("reset")
+        token_info.count_graphql, token_info.reset_at_graphql = (
+            remaining_info_graphql.get("remaining"),
+            ab_datetime_parse(remaining_info_graphql.get("reset")),
         )
 
     def check_all_tokens(self):
@@ -141,7 +175,7 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
             setattr(current_token, count_attr, getattr(current_token, count_attr) - 1)
             return True
         elif all(getattr(x, count_attr) == 0 for x in self._tokens.values()):
-            min_time_to_wait = min((getattr(x, reset_attr) - pendulum.now()).in_seconds() for x in self._tokens.values())
+            min_time_to_wait = min((getattr(x, reset_attr) - ab_datetime_now()).total_seconds() for x in self._tokens.values())
             if min_time_to_wait < self.max_time:
                 time.sleep(min_time_to_wait if min_time_to_wait > 0 else 0)
                 self.check_all_tokens()

airbyte_source_github-1.8.13/pyproject.toml

@@ -1,36 +0,0 @@
-[build-system]
-requires = [
-    "poetry-core>=1.0.0",
-]
-build-backend = "poetry.core.masonry.api"
-
-[tool.poetry]
-version = "1.8.13"
-name = "airbyte-source-github"
-description = "Source implementation for GitHub."
-authors = [
-    "Airbyte <contact@airbyte.io>",
-]
-license = "MIT"
-readme = "README.md"
-documentation = "https://docs.airbyte.com/integrations/sources/github"
-homepage = "https://airbyte.com"
-repository = "https://github.com/airbytehq/airbyte"
-packages = [
-    { include = "source_github" },
-]
-
-[tool.poetry.dependencies]
-python = "^3.10,<3.12"
-airbyte-cdk = "^4"
-sgqlc = "==16.3"
-
-[tool.poetry.scripts]
-source-github = "source_github.run:run"
-
-[tool.poetry.group.dev.dependencies]
-requests-mock = "^1.9.3"
-freezegun = "^1.2"
-pytest-mock = "^3.6.1"
-pytest = "^6.2"
-responses = "^0.23.1"

{airbyte_source_github-1.8.13 → airbyte_source_github-2.1.5}/source_github/graphql.py

@@ -11,6 +11,7 @@ from sgqlc.operation import Selector
 
 from . import github_schema
 
+
 _schema = github_schema
 _schema_root = _schema.github_schema
 
@@ -165,7 +166,6 @@ def get_query_issue_reactions(owner, name, first, after, number=None):
 
 
 class QueryReactions:
-
     # AVERAGE_REVIEWS - optimal number of reviews to fetch inside every pull request.
     # If we try to fetch too many (up to 100) we will spend too many scores of query cost.
     # https://docs.github.com/en/graphql/overview/resource-limitations#calculating-a-rate-limit-score-before-running-the-call