airbyte-source-github 1.8.41__tar.gz → 1.9.0rc1__tar.gz

{airbyte_source_github-1.8.41 → airbyte_source_github-1.9.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 1.8.41
+Version: 1.9.0rc1
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: MIT
@@ -11,7 +11,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=4,<5)
+Requires-Dist: airbyte-cdk (>=7,<8)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/github
 Project-URL: Repository, https://github.com/airbytehq/airbyte

{airbyte_source_github-1.8.41 → airbyte_source_github-1.9.0rc1}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.8.41"
+version = "1.9.0-rc.1"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]
@@ -17,7 +17,7 @@ include = "source_github"
 
 [tool.poetry.dependencies]
 python = "^3.10,<3.12"
-airbyte-cdk = "^4"
+airbyte-cdk = "^7"
 sgqlc = "==16.3"
 
 [tool.poetry.scripts]

{airbyte_source_github-1.8.41 → airbyte_source_github-1.9.0rc1}/source_github/config_migrations.py

@@ -6,8 +6,11 @@ import logging
 from abc import ABC
 from typing import Any, List, Mapping
 
+import orjson
+
 from airbyte_cdk.config_observation import create_connector_config_control_message
 from airbyte_cdk.entrypoint import AirbyteEntrypoint
+from airbyte_cdk.models import AirbyteMessageSerializer
 from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 
 from .source import SourceGithub
@@ -72,7 +75,7 @@ class MigrateStringToArray(ABC):
         cls.message_repository.emit_message(create_connector_config_control_message(migrated_config))
         # emit the Airbyte Control Message from message queue to stdout
         for message in cls.message_repository._message_queue:
-            print(message.json(exclude_unset=True))
+            print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
 
     @classmethod
     def migrate(cls, args: List[str], source: SourceGithub) -> None:

{airbyte_source_github-1.8.41 → airbyte_source_github-1.9.0rc1}/source_github/errors_handlers.py

@@ -6,10 +6,10 @@ from typing import Optional, Union
 
 import requests
 
+from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction
 from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
-from airbyte_protocol.models import FailureType
 
 from . import constants
 

{airbyte_source_github-1.8.41 → airbyte_source_github-1.9.0rc1}/source_github/streams.py

@@ -4,14 +4,14 @@
 
 import re
 from abc import ABC, abstractmethod
+from datetime import timedelta
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 from urllib import parse
 
-import pendulum
 import requests
 
 from airbyte_cdk import BackoffStrategy, StreamSlice
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level, SyncMode
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
 from airbyte_cdk.sources.streams.checkpoint.substream_resumable_full_refresh_cursor import SubstreamResumableFullRefreshCursor
@@ -19,8 +19,9 @@ from airbyte_cdk.sources.streams.core import CheckpointMixin, Stream
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction
 from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException
+from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors
 from airbyte_cdk.utils import AirbyteTracedException
-from airbyte_protocol.models import FailureType
+from airbyte_cdk.utils.datetime_helpers import ab_datetime_format, ab_datetime_parse
 
 from . import constants
 from .backoff_strategies import ContributorActivityBackoffStrategy, GithubStreamABCBackoffStrategy
@@ -128,11 +129,14 @@ class GithubStreamABC(HttpStream, ABC):
         # Reading records while handling the errors
         try:
             yield from super().read_records(stream_slice=stream_slice, **kwargs)
-        except DefaultBackoffException as e:
+        # HTTP Client wraps DefaultBackoffException into MessageRepresentationAirbyteTracedErrors
+        except MessageRepresentationAirbyteTracedErrors as e:
             # This whole try/except situation in `read_records()` isn't good but right now in `self._send_request()`
             # function we have `response.raise_for_status()` so we don't have much choice on how to handle errors.
             # Bocked on https://github.com/airbytehq/airbyte/issues/3514.
-            if e.response.status_code == requests.codes.NOT_FOUND:
+            if not hasattr(e, "_exception") and not hasattr(e._exception, "response"):
+                raise e
+            if e._exception.response.status_code == requests.codes.NOT_FOUND:
                 # A lot of streams are not available for repositories owned by a user instead of an organization.
                 if isinstance(self, Organizations):
                     error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for organization `{organisation}`."
@@ -140,8 +144,8 @@ class GithubStreamABC(HttpStream, ABC):
                     error_msg = f"Syncing `{self.__class__.__name__}` stream for organization `{organisation}`, team `{stream_slice.get('team_slug')}` and user `{stream_slice.get('username')}` isn't available: User has no team membership. Skipping..."
                 else:
                     error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for repository `{repository}`."
-            elif e.response.status_code == requests.codes.FORBIDDEN:
-                error_msg = str(e.response.json().get("message"))
+            elif e._exception.response.status_code == requests.codes.FORBIDDEN:
+                error_msg = str(e._exception.response.json().get("message"))
                 # When using the `check_connection` method, we should raise an error if we do not have access to the repository.
                 if isinstance(self, Repositories):
                     raise e
@@ -157,27 +161,27 @@ class GithubStreamABC(HttpStream, ABC):
                     error_msg = (
                         f"Syncing `{self.name}` stream isn't available for repository `{repository}`. Full error message: {error_msg}"
                     )
-            elif e.response.status_code == requests.codes.UNAUTHORIZED:
+            elif e._exception.response.status_code == requests.codes.UNAUTHORIZED:
                 if self.access_token_type == constants.PERSONAL_ACCESS_TOKEN_TITLE:
-                    error_msg = str(e.response.json().get("message"))
+                    error_msg = str(e._exception.response.json().get("message"))
                     self.logger.error(f"{self.access_token_type} renewal is required: {error_msg}")
                 raise e
-            elif e.response.status_code == requests.codes.GONE and isinstance(self, Projects):
+            elif e._exception.response.status_code == requests.codes.GONE and isinstance(self, Projects):
                 # Some repos don't have projects enabled and we we get "410 Client Error: Gone for
                 # url: https://api.github.com/repos/xyz/projects?per_page=100" error.
                 error_msg = f"Syncing `Projects` stream isn't available for repository `{stream_slice['repository']}`."
-            elif e.response.status_code == requests.codes.CONFLICT:
+            elif e._exception.response.status_code == requests.codes.CONFLICT:
                 error_msg = (
                     f"Syncing `{self.name}` stream isn't available for repository "
                     f"`{stream_slice['repository']}`, it seems like this repository is empty."
                 )
-            elif e.response.status_code == requests.codes.SERVER_ERROR and isinstance(self, WorkflowRuns):
+            elif e._exception.response.status_code == requests.codes.SERVER_ERROR and isinstance(self, WorkflowRuns):
                 error_msg = f"Syncing `{self.name}` stream isn't available for repository `{stream_slice['repository']}`."
-            elif e.response.status_code == requests.codes.BAD_GATEWAY:
+            elif e._exception.response.status_code == requests.codes.BAD_GATEWAY:
                 error_msg = f"Stream {self.name} temporary failed. Try to re-run sync later"
             else:
                 # most probably here we're facing a 500 server error and a risk to get a non-json response, so lets output response.text
-                self.logger.error(f"Undefined error while reading records: {e.response.text}")
+                self.logger.error(f"Undefined error while reading records: {e._exception.response.text}")
                 raise e
 
             self.logger.warning(error_msg)
@@ -1437,7 +1441,7 @@ class Workflows(SemiIncrementalMixin, GithubStream):
             yield self.transform(record=record, stream_slice=stream_slice)
 
     def convert_cursor_value(self, value):
-        return pendulum.parse(value).in_tz(tz="UTC").format("YYYY-MM-DDTHH:mm:ss[Z]")
+        return ab_datetime_format(value, "YYYY-MM-DDTHH:mm:ss[Z]")
 
 
 class WorkflowRuns(SemiIncrementalMixin, GithubStream):
@@ -1478,7 +1482,7 @@ class WorkflowRuns(SemiIncrementalMixin, GithubStream):
         # the state is updated only in the end of the sync as records are sorted in reverse order
         new_state = self.state
         if start_point:
-            break_point = (pendulum.parse(start_point) - pendulum.duration(days=self.re_run_period)).to_iso8601_string()
+            break_point = (ab_datetime_parse(start_point) - timedelta(days=self.re_run_period)).isoformat()
         for record in super(SemiIncrementalMixin, self).read_records(
             sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state
         ):
@@ -1663,22 +1667,24 @@ class ContributorActivity(GithubStream):
         repository = stream_slice.get("repository", "")
         try:
             yield from super().read_records(stream_slice=stream_slice, **kwargs)
-        except UserDefinedBackoffException as e:
-            if e.response.status_code == requests.codes.ACCEPTED:
-                yield AirbyteMessage(
-                    type=MessageType.LOG,
-                    log=AirbyteLogMessage(
-                        level=Level.INFO,
-                        message=f"Syncing `{self.__class__.__name__}` " f"stream isn't available for repository `{repository}`.",
-                    ),
-                )
+        # HTTP Client wraps BackoffException into MessageRepresentationAirbyteTracedErrors
+        except MessageRepresentationAirbyteTracedErrors as e:
+            if hasattr(e, "_exception") and hasattr(e._exception, "response"):
+                if e._exception.response.status_code == requests.codes.ACCEPTED:
+                    yield AirbyteMessage(
+                        type=MessageType.LOG,
+                        log=AirbyteLogMessage(
+                            level=Level.INFO,
+                            message=f"Syncing `{self.__class__.__name__}` " f"stream isn't available for repository `{repository}`.",
+                        ),
+                    )
 
-                # In order to retain the existing stream behavior before we added RFR to this stream, we need to close out the
-                # partition after we give up the maximum number of retries on the 202 response. This does lead to the question
-                # of if we should prematurely exit in the first place, but for now we're going to aim for feature parity
-                partition_obj = stream_slice.get("partition")
-                if self.cursor and partition_obj:
-                    self.cursor.close_slice(StreamSlice(cursor_slice={}, partition=partition_obj))
+                    # In order to retain the existing stream behavior before we added RFR to this stream, we need to close out the
+                    # partition after we give up the maximum number of retries on the 202 response. This does lead to the question
+                    # of if we should prematurely exit in the first place, but for now we're going to aim for feature parity
+                    partition_obj = stream_slice.get("partition")
+                    if self.cursor and partition_obj:
+                        self.cursor.close_slice(StreamSlice(cursor_slice={}, partition=partition_obj))
             else:
                 raise e
 

{airbyte_source_github-1.8.41 → airbyte_source_github-1.9.0rc1}/source_github/utils.py

@@ -4,16 +4,17 @@
 
 import time
 from dataclasses import dataclass
+from datetime import timedelta
 from itertools import cycle
 from typing import Any, List, Mapping
 
-import pendulum
 import requests
 
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
 
 
 def getter(D: dict, key_or_keys, strict=True):
@@ -43,8 +44,8 @@ class GitHubAPILimitException(Exception):
 class Token:
     count_rest: int = 5000
     count_graphql: int = 5000
-    reset_at_rest: pendulum.DateTime = pendulum.now()
-    reset_at_graphql: pendulum.DateTime = pendulum.now()
+    reset_at_rest: AirbyteDateTime = ab_datetime_now()
+    reset_at_graphql: AirbyteDateTime = ab_datetime_now()
 
 
 class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
@@ -55,7 +56,7 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
     the first token becomes available again.
     """
 
-    DURATION = pendulum.duration(seconds=3600)  # Duration at which the current rate limit window resets
+    DURATION = timedelta(seconds=3600)  # Duration at which the current rate limit window resets
 
     def __init__(self, tokens: List[str], auth_method: str = "token", auth_header: str = "Authorization"):
         self._auth_method = auth_method
@@ -125,13 +126,13 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
         remaining_info_core = rate_limit_info.get("core")
         token_info.count_rest, token_info.reset_at_rest = (
             remaining_info_core.get("remaining"),
-            pendulum.from_timestamp(remaining_info_core.get("reset")),
+            ab_datetime_parse(remaining_info_core.get("reset")),
         )
 
         remaining_info_graphql = rate_limit_info.get("graphql")
         token_info.count_graphql, token_info.reset_at_graphql = (
             remaining_info_graphql.get("remaining"),
-            pendulum.from_timestamp(remaining_info_graphql.get("reset")),
+            ab_datetime_parse(remaining_info_graphql.get("reset")),
         )
 
     def check_all_tokens(self):
@@ -143,7 +144,7 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
             setattr(current_token, count_attr, getattr(current_token, count_attr) - 1)
             return True
         elif all(getattr(x, count_attr) == 0 for x in self._tokens.values()):
-            min_time_to_wait = min((getattr(x, reset_attr) - pendulum.now()).in_seconds() for x in self._tokens.values())
+            min_time_to_wait = min((getattr(x, reset_attr) - ab_datetime_now()).seconds for x in self._tokens.values())
             if min_time_to_wait < self.max_time:
                 time.sleep(min_time_to_wait if min_time_to_wait > 0 else 0)
                 self.check_all_tokens()