airbyte-source-github 2.1.21__tar.gz → 2.1.22__tar.gz

{airbyte_source_github-2.1.21 → airbyte_source_github-2.1.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 2.1.21
+Version: 2.1.22
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_github-2.1.21 → airbyte_source_github-2.1.22}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.1.21"
+version = "2.1.22"
 name = "airbyte-source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]

{airbyte_source_github-2.1.21 → airbyte_source_github-2.1.22}/source_github/source.py

@@ -241,7 +241,7 @@ class SourceGithub(AbstractSource):
         # This parameter is deprecated and in future will be used sane default, page_size: 10
         page_size = config.get("page_size_for_large_streams", constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM)
         access_token_type, _ = self.get_access_token(config)
-        max_waiting_time = config.get("max_waiting_time", 10) * 60
+        max_waiting_time = config.get("max_waiting_time", 120) * 60
         organization_args = {
             "authenticator": authenticator,
             "organizations": organizations,

{airbyte_source_github-2.1.21 → airbyte_source_github-2.1.22}/source_github/spec.json

@@ -135,11 +135,11 @@
       "max_waiting_time": {
         "type": "integer",
         "title": "Max Waiting Time (in minutes)",
-        "examples": [10, 30, 60],
-        "default": 10,
+        "examples": [10, 60, 120],
+        "default": 120,
         "minimum": 1,
-        "maximum": 60,
-        "description": "Max Waiting Time for rate limit. Set higher value to wait till rate limits will be resetted to continue sync",
+        "maximum": 240,
+        "description": "Max time (in minutes) the connector will wait when all API tokens are rate-limited before failing. GitHub rate limits reset every 60 minutes, so values above 60 allow the connector to wait for a full reset cycle.",
         "order": 5
       }
     }

{airbyte_source_github-2.1.21 → airbyte_source_github-2.1.22}/source_github/streams.py

@@ -190,11 +190,11 @@ class GithubStreamABC(HttpStream, ABC):
 
             self.logger.warning(error_msg)
         except GitHubAPILimitException as e:
-            internal_message = (
-                f"Stream: `{self.name}`, slice: `{stream_slice}`. Limits for all provided tokens are reached, please try again later"
-            )
-            message = "Rate Limits for all provided tokens are reached. For more information please refer to documentation: https://docs.airbyte.com/integrations/sources/github#limitations--troubleshooting"
-            raise AirbyteTracedException(internal_message=internal_message, message=message, failure_type=FailureType.config_error) from e
+            internal_message = f"Stream: `{self.name}`, slice: `{stream_slice}`. {e}"
+            message = "Rate limit exceeded for all configured GitHub API tokens."
+            raise AirbyteTracedException(
+                internal_message=internal_message, message=message, failure_type=FailureType.transient_error
+            ) from e
 
 
 class GithubStream(GithubStreamABC):

{airbyte_source_github-2.1.21 → airbyte_source_github-2.1.22}/source_github/utils.py

@@ -56,9 +56,16 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
     If a token exceeds the capacity limit, the system switches to another token.
     If all tokens are exhausted, the system will enter a sleep state until
     the first token becomes available again.
+
+    An API budget mechanism throttles requests proactively: when a token's
+    remaining quota drops below a configurable reserve, a small delay is
+    injected before the request so that the connector never fully exhausts
+    all tokens at once.
     """
 
     DURATION = timedelta(seconds=3600)  # Duration at which the current rate limit window resets
+    BUDGET_RESERVE_FRACTION = 0.1  # Start throttling when only 10% of quota remains
+    BUDGET_MIN_RESERVE = 50  # Always keep at least this many calls in reserve per token
 
     def __init__(self, tokens: List[str], auth_method: str = "token", auth_header: str = "Authorization"):
         self._logger = logging.getLogger("airbyte")
@@ -72,7 +79,8 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
         self.check_all_tokens()
         self._tokens_iter = cycle(self._tokens)
         self._active_token = next(self._tokens_iter)
-        self._max_time = 60 * 10  # 10 minutes as default
+        self._max_time = 60 * 120  # 120 minutes as default (must exceed GitHub's 60-min rate limit window)
+        self._budget_logged = False  # avoid log spam for throttle messages
 
     def _initialize_http_clients(self, tokens: List[str]) -> Mapping[str, HttpClient]:
         return {
@@ -169,15 +177,94 @@ class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
     def check_all_tokens(self):
         for token in self._tokens:
             self._check_token_limits(token)
+        self._budget_logged = False
+
+    def _get_budget_reserve(self, token: Token, count_attr: str) -> int:
+        """Return the minimum number of calls to keep in reserve for a token.
+
+        The reserve is the larger of ``BUDGET_MIN_RESERVE`` and
+        ``BUDGET_RESERVE_FRACTION`` of the token's actual remaining count.
+        We use the current remaining value as an approximation when the
+        original limit is unknown.
+        """
+        remaining = getattr(token, count_attr)
+        # Use 5000 (GitHub default) as the basis, but fall back to the
+        # remaining count if it's higher (e.g. enterprise tokens).
+        limit_estimate = max(5000, remaining)
+        return max(self.BUDGET_MIN_RESERVE, int(limit_estimate * self.BUDGET_RESERVE_FRACTION))
+
+    def _apply_budget_throttle(self, token: Token, count_attr: str, reset_attr: str) -> None:
+        """Optionally sleep a little to spread remaining calls over the reset window.
+
+        When the remaining count for *all* tokens is below the budget
+        reserve, we inject a short delay proportional to how much time
+        remains until the rate-limit window resets.  This avoids hitting
+        the wall and having to do a long blocking sleep.
+        """
+        reserve = self._get_budget_reserve(token, count_attr)
+        remaining = getattr(token, count_attr)
+        if remaining > reserve:
+            return  # plenty of headroom — no throttling needed
+
+        # Only throttle when *every* token is running low so that we don't
+        # slow down needlessly while other tokens still have capacity.
+        if not all(getattr(t, count_attr) <= self._get_budget_reserve(t, count_attr) for t in self._tokens.values()):
+            return
+
+        # Calculate a proportional delay: spread the remaining calls evenly
+        # across the time left until the earliest reset.
+        seconds_to_reset = max((getattr(token, reset_attr) - ab_datetime_now()).total_seconds(), 0)
+        total_remaining = sum(max(getattr(t, count_attr), 0) for t in self._tokens.values())
+        if total_remaining <= 0 or seconds_to_reset <= 0:
+            return
+
+        delay = seconds_to_reset / total_remaining
+        # Cap the delay to avoid extremely long pauses on single requests
+        delay = min(delay, 10.0)
+        if delay >= 0.1:
+            if not self._budget_logged:
+                self._logger.info(
+                    "API budget: throttling requests (%.1fs delay). %d calls remaining across %d token(s), " "%.0fs until reset.",
+                    delay,
+                    total_remaining,
+                    len(self._tokens),
+                    seconds_to_reset,
+                )
+                self._budget_logged = True
+            time.sleep(delay)
+
+    HEARTBEAT_INTERVAL = 60.0  # Log every 60s during exhaustion sleep
+
+    def _sleep_with_heartbeat(self, total_seconds: float, count_attr: str) -> None:
+        """Sleep for *total_seconds* but log progress periodically so the
+        platform heartbeat stays alive and operators can see the connector
+        is not stuck."""
+        remaining = total_seconds
+        while remaining > 0:
+            chunk = min(remaining, self.HEARTBEAT_INTERVAL)
+            time.sleep(chunk)
+            remaining -= chunk
+            if remaining > 0:
+                self._logger.info(
+                    "Rate limit exhausted (%s). Waiting for reset — %.0fs remaining.",
+                    count_attr,
+                    remaining,
+                )
 
     def process_token(self, current_token, count_attr, reset_attr):
         if getattr(current_token, count_attr) > 0:
+            self._apply_budget_throttle(current_token, count_attr, reset_attr)
             setattr(current_token, count_attr, getattr(current_token, count_attr) - 1)
             return True
         elif all(getattr(x, count_attr) == 0 for x in self._tokens.values()):
             min_time_to_wait = min((getattr(x, reset_attr) - ab_datetime_now()).total_seconds() for x in self._tokens.values())
             if min_time_to_wait < self.max_time:
-                time.sleep(min_time_to_wait if min_time_to_wait > 0 else 0)
+                self._logger.info(
+                    "All tokens exhausted (%s). Sleeping %.0fs until rate limit resets.",
+                    count_attr,
+                    max(min_time_to_wait, 0),
+                )
+                self._sleep_with_heartbeat(max(min_time_to_wait, 0), count_attr)
                 self.check_all_tokens()
             else:
                 raise GitHubAPILimitException(f"Rate limits for all tokens ({count_attr}) were reached")