airbyte-source-github 1.5.7__py3-none-any.whl

source_github/utils.py

@@ -0,0 +1,152 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import time
+from dataclasses import dataclass
+from itertools import cycle
+from typing import Any, List, Mapping
+
+import pendulum
+import requests
+from airbyte_cdk.models import SyncMode
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
+from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+
+
+def getter(D: dict, key_or_keys, strict=True):
+    if not isinstance(key_or_keys, list):
+        key_or_keys = [key_or_keys]
+    for k in key_or_keys:
+        if strict:
+            D = D[k]
+        else:
+            D = D.get(k, {})
+    return D
+
+
+def read_full_refresh(stream_instance: Stream):
+    slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh)
+    for _slice in slices:
+        records = stream_instance.read_records(stream_slice=_slice, sync_mode=SyncMode.full_refresh)
+        for record in records:
+            yield record
+
+
+class GitHubAPILimitException(Exception):
+    """General class for Rate Limits errors"""
+
+
+@dataclass
+class Token:
+    count_rest: int = 5000
+    count_graphql: int = 5000
+    reset_at_rest: pendulum.DateTime = pendulum.now()
+    reset_at_graphql: pendulum.DateTime = pendulum.now()
+
+
+class MultipleTokenAuthenticatorWithRateLimiter(AbstractHeaderAuthenticator):
+    """
+    Each token in the cycle is checked against the rate limiter.
+    If a token exceeds the capacity limit, the system switches to another token.
+    If all tokens are exhausted, the system will enter a sleep state until
+    the first token becomes available again.
+    """
+
+    DURATION = pendulum.duration(seconds=3600)  # Duration at which the current rate limit window resets
+
+    def __init__(self, tokens: List[str], auth_method: str = "token", auth_header: str = "Authorization"):
+        self._auth_method = auth_method
+        self._auth_header = auth_header
+        self._tokens = {t: Token() for t in tokens}
+        self.check_all_tokens()
+        self._tokens_iter = cycle(self._tokens)
+        self._active_token = next(self._tokens_iter)
+        self._max_time = 60 * 10  # 10 minutes as default
+
+    @property
+    def auth_header(self) -> str:
+        return self._auth_header
+
+    def get_auth_header(self) -> Mapping[str, Any]:
+        """The header to set on outgoing HTTP requests"""
+        if self.auth_header:
+            return {self.auth_header: self.token}
+        return {}
+
+    def __call__(self, request):
+        """Attach the HTTP headers required to authenticate on the HTTP request"""
+        while True:
+            current_token = self._tokens[self.current_active_token]
+            if "graphql" in request.path_url:
+                if self.process_token(current_token, "count_graphql", "reset_at_graphql"):
+                    break
+            else:
+                if self.process_token(current_token, "count_rest", "reset_at_rest"):
+                    break
+
+        request.headers.update(self.get_auth_header())
+
+        return request
+
+    @property
+    def current_active_token(self) -> str:
+        return self._active_token
+
+    def update_token(self) -> None:
+        self._active_token = next(self._tokens_iter)
+
+    @property
+    def token(self) -> str:
+
+        token = self.current_active_token
+        return f"{self._auth_method} {token}"
+
+    @property
+    def max_time(self) -> int:
+        return self._max_time
+
+    @max_time.setter
+    def max_time(self, value: int) -> None:
+        self._max_time = value
+
+    def _check_token_limits(self, token: str):
+        """check that token is not limited"""
+        headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}
+        rate_limit_info = (
+            requests.get(
+                "https://api.github.com/rate_limit", headers=headers, auth=TokenAuthenticator(token, auth_method=self._auth_method)
+            )
+            .json()
+            .get("resources")
+        )
+        token_info = self._tokens[token]
+        remaining_info_core = rate_limit_info.get("core")
+        token_info.count_rest, token_info.reset_at_rest = remaining_info_core.get("remaining"), pendulum.from_timestamp(
+            remaining_info_core.get("reset")
+        )
+
+        remaining_info_graphql = rate_limit_info.get("graphql")
+        token_info.count_graphql, token_info.reset_at_graphql = remaining_info_graphql.get("remaining"), pendulum.from_timestamp(
+            remaining_info_graphql.get("reset")
+        )
+
+    def check_all_tokens(self):
+        for token in self._tokens:
+            self._check_token_limits(token)
+
+    def process_token(self, current_token, count_attr, reset_attr):
+        if getattr(current_token, count_attr) > 0:
+            setattr(current_token, count_attr, getattr(current_token, count_attr) - 1)
+            return True
+        elif all(getattr(x, count_attr) == 0 for x in self._tokens.values()):
+            min_time_to_wait = min((getattr(x, reset_attr) - pendulum.now()).in_seconds() for x in self._tokens.values())
+            if min_time_to_wait < self.max_time:
+                time.sleep(min_time_to_wait if min_time_to_wait > 0 else 0)
+                self.check_all_tokens()
+            else:
+                raise GitHubAPILimitException(f"Rate limits for all tokens ({count_attr}) were reached")
+        else:
+            self.update_token()
+        return False

unit_tests/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#

unit_tests/conftest.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+import os
+
+import pytest
+import responses
+
+os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH"
+
+
+@pytest.fixture(name="rate_limit_mock_response")
+def rate_limit_mock_response():
+    rate_limit_response = {
+        "resources": {
+            "core": {
+                "limit": 5000,
+                "used": 0,
+                "remaining": 5000,
+                "reset": 4070908800
+            },
+            "graphql": {
+                "limit": 5000,
+                "used": 0,
+                "remaining": 5000,
+                "reset": 4070908800
+            }
+        }
+    }
+    responses.add(responses.GET, "https://api.github.com/rate_limit", json=rate_limit_response)

unit_tests/projects_v2_pull_requests_query.json

@@ -0,0 +1,3 @@
+{
+  "query": "query {\n  repository(owner: \"airbytehq\", name: \"airbyte\") {\n    name\n    owner {\n      login\n    }\n    projectsV2(first: 100, orderBy: {field: UPDATED_AT, direction: ASC}) {\n      nodes {\n        closed\n        created_at: createdAt\n        closed_at: closedAt\n        updated_at: updatedAt\n        creator: creator {\n          avatarUrl\n          login\n          resourcePath\n          url\n        }\n        node_id: id\n        id: databaseId\n        number\n        public\n        readme: readme\n        short_description: shortDescription\n        template\n        title: title\n        url: url\n        viewerCanClose\n        viewerCanReopen\n        viewerCanUpdate\n        owner {\n          id: id\n        }\n      }\n      pageInfo {\n        hasNextPage\n        endCursor\n      }\n    }\n  }\n}"
+}

unit_tests/pull_request_stats_query.json

@@ -0,0 +1,3 @@
+{
+  "query": "query {\n  repository(owner: \"airbytehq\", name: \"airbyte\") {\n    name\n    owner {\n      login\n    }\n    pullRequests(first: 10, orderBy: {field: UPDATED_AT, direction: ASC}) {\n      nodes {\n        node_id: id\n        id: databaseId\n        number\n        updated_at: updatedAt\n        changed_files: changedFiles\n        deletions\n        additions\n        merged\n        mergeable\n        can_be_rebased: canBeRebased\n        maintainer_can_modify: maintainerCanModify\n        merge_state_status: mergeStateStatus\n        comments {\n          totalCount\n        }\n        commits {\n          totalCount\n        }\n        review_comments: reviews(first: 100) {\n          totalCount\n          nodes {\n            comments {\n              totalCount\n            }\n          }\n        }\n        merged_by: mergedBy {\n          __typename\n          ... on User {\n            node_id: id\n            id: databaseId\n            login\n            avatar_url: avatarUrl\n            html_url: url\n            site_admin: isSiteAdmin\n          }\n        }\n      }\n      pageInfo {\n        hasNextPage\n        endCursor\n      }\n    }\n  }\n}"
+}

unit_tests/responses/contributor_activity_response.json

@@ -0,0 +1,33 @@
+[
+  {
+    "author": {
+      "login": "octocat",
+      "id": 1,
+      "node_id": "MDQ6VXNlcjE=",
+      "avatar_url": "https://github.com/images/error/octocat_happy.gif",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/octocat",
+      "html_url": "https://github.com/octocat",
+      "followers_url": "https://api.github.com/users/octocat/followers",
+      "following_url": "https://api.github.com/users/octocat/following{/other_user}",
+      "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/octocat/subscriptions",
+      "organizations_url": "https://api.github.com/users/octocat/orgs",
+      "repos_url": "https://api.github.com/users/octocat/repos",
+      "events_url": "https://api.github.com/users/octocat/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/octocat/received_events",
+      "type": "User",
+      "site_admin": false
+    },
+    "total": 135,
+    "weeks": [
+      {
+        "w": 1367712000,
+        "a": 6898,
+        "d": 77,
+        "c": 10
+      }
+    ]
+  }
+]

unit_tests/responses/graphql_reviews_responses.json

@@ -0,0 +1,405 @@
+[
+  {
+    "data": {
+      "repository": {
+        "owner": {
+          "login": "airbytehq"
+        },
+        "name": "airbyte",
+        "pullRequests": {
+          "nodes": [
+            {
+              "number": 1,
+              "url": "https://github.com/airbytehq/airbyte/pull/1",
+              "reviews": {
+                "nodes": [
+                  {
+                    "id": 1000,
+                    "updated_at": "2000-01-01T00:00:01Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/1#pullrequestreview-1000",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  },
+                  {
+                    "id": 1001,
+                    "updated_at": "2000-01-01T00:00:01Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/1#pullrequestreview-1001",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  }
+                ],
+                "pageInfo": {
+                  "endCursor": "cursor",
+                  "hasNextPage": true
+                }
+              }
+            },
+            {
+              "number": 2,
+              "url": "https://github.com/airbytehq/airbyte/pull/2",
+              "reviews": {
+                "nodes": [
+                  {
+                    "id": 1002,
+                    "updated_at": "2000-01-01T00:00:01Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/2#pullrequestreview-1002",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  },
+                  {
+                    "id": 1003,
+                    "updated_at": "2000-01-01T00:00:01Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/2#pullrequestreview-1003",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  }
+                ],
+                "pageInfo": {
+                  "endCursor": "cursor",
+                  "hasNextPage": true
+                }
+              }
+            }
+          ],
+          "pageInfo": {
+            "endCursor": "cursor",
+            "hasNextPage": true
+          }
+        }
+      }
+    }
+  },
+  {
+    "data": {
+      "repository": {
+        "owner": {
+          "login": "airbytehq"
+        },
+        "name": "airbyte",
+        "pullRequest": {
+          "number": 1,
+          "url": "https://github.com/airbytehq/airbyte/pull/1",
+          "reviews": {
+            "nodes": [
+              {
+                "id": 1004,
+                "updated_at": "2000-01-01T00:00:01Z",
+                "html_url": "https://github.com/airbytehq/airbyte/pull/1#pullrequestreview-1004",
+                "commit": null,
+                "user": {
+                  "__typename": "User"
+                }
+              },
+              {
+                "id": 1005,
+                "updated_at": "2000-01-01T00:00:01Z",
+                "html_url": "https://github.com/airbytehq/airbyte/pull/1#pullrequestreview-1005",
+                "commit": null,
+                "user": {
+                  "__typename": "User"
+                }
+              }
+            ],
+            "pageInfo": {
+              "endCursor": null,
+              "hasNextPage": false
+            }
+          }
+        }
+      }
+    }
+  },
+  {
+    "data": {
+      "repository": {
+        "owner": {
+          "login": "airbytehq"
+        },
+        "name": "airbyte",
+        "pullRequest": {
+          "number": 2,
+          "url": "https://github.com/airbytehq/airbyte/pull/2",
+          "reviews": {
+            "nodes": [
+              {
+                "id": 1006,
+                "updated_at": "2000-01-01T00:00:01Z",
+                "html_url": "https://github.com/airbytehq/airbyte/pull/2#pullrequestreview-1006",
+                "commit": null,
+                "user": {
+                  "__typename": "User"
+                }
+              },
+              {
+                "id": 1007,
+                "updated_at": "2000-01-01T00:00:01Z",
+                "html_url": "https://github.com/airbytehq/airbyte/pull/2#pullrequestreview-1007",
+                "commit": null,
+                "user": {
+                  "__typename": "User"
+                }
+              }
+            ],
+            "pageInfo": {
+              "endCursor": null,
+              "hasNextPage": false
+            }
+          }
+        }
+      }
+    }
+  },
+  {
+    "data": {
+      "repository": {
+        "owner": {
+          "login": "airbytehq"
+        },
+        "name": "airbyte",
+        "pullRequests": {
+          "nodes": [
+            {
+              "number": 3,
+              "url": "https://github.com/airbytehq/airbyte/pull/3",
+              "reviews": {
+                "nodes": [
+                  {
+                    "id": 1008,
+                    "updated_at": "2000-01-01T00:00:01Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/3#pullrequestreview-1008",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  }
+                ],
+                "pageInfo": {
+                  "endCursor": null,
+                  "hasNextPage": false
+                }
+              }
+            }
+          ],
+          "pageInfo": {
+            "endCursor": null,
+            "hasNextPage": false
+          }
+        }
+      }
+    }
+  },
+  {
+    "data": {
+      "repository": {
+        "owner": {
+          "login": "airbytehq"
+        },
+        "name": "airbyte",
+        "pullRequests": {
+          "nodes": [
+            {
+              "number": 1,
+              "url": "https://github.com/airbytehq/airbyte/pull/1",
+              "reviews": {
+                "nodes": [
+                  {
+                    "id": 1000,
+                    "updated_at": "2000-01-01T00:00:02Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/1#pullrequestreview-1000",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  },
+                  {
+                    "id": 1001,
+                    "updated_at": "2000-01-01T00:00:01Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/1#pullrequestreview-1001",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  }
+                ],
+                "pageInfo": {
+                  "endCursor": "cursor",
+                  "hasNextPage": true
+                }
+              }
+            },
+            {
+              "number": 2,
+              "url": "https://github.com/airbytehq/airbyte/pull/2",
+              "reviews": {
+                "nodes": [
+                  {
+                    "id": 1002,
+                    "updated_at": "2000-01-01T00:00:01Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/2#pullrequestreview-1002",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  },
+                  {
+                    "id": 1003,
+                    "updated_at": "2000-01-01T00:00:01Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/2#pullrequestreview-1003",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  }
+                ],
+                "pageInfo": {
+                  "endCursor": "cursor",
+                  "hasNextPage": true
+                }
+              }
+            }
+          ],
+          "pageInfo": {
+            "endCursor": "cursor",
+            "hasNextPage": true
+          }
+        }
+      }
+    }
+  },
+  {
+    "data": {
+      "repository": {
+        "owner": {
+          "login": "airbytehq"
+        },
+        "name": "airbyte",
+        "pullRequest": {
+          "number": 1,
+          "url": "https://github.com/airbytehq/airbyte/pull/1",
+          "reviews": {
+            "nodes": [
+              {
+                "id": 1004,
+                "updated_at": "2000-01-01T00:00:01Z",
+                "html_url": "https://github.com/airbytehq/airbyte/pull/1#pullrequestreview-1004",
+                "commit": null,
+                "user": {
+                  "__typename": "User"
+                }
+              },
+              {
+                "id": 1005,
+                "updated_at": "2000-01-01T00:00:01Z",
+                "html_url": "https://github.com/airbytehq/airbyte/pull/1#pullrequestreview-1005",
+                "commit": null,
+                "user": {
+                  "__typename": "User"
+                }
+              }
+            ],
+            "pageInfo": {
+              "endCursor": null,
+              "hasNextPage": false
+            }
+          }
+        }
+      }
+    }
+  },
+  {
+    "data": {
+      "repository": {
+        "owner": {
+          "login": "airbytehq"
+        },
+        "name": "airbyte",
+        "pullRequest": {
+          "number": 2,
+          "url": "https://github.com/airbytehq/airbyte/pull/2",
+          "reviews": {
+            "nodes": [
+              {
+                "id": 1006,
+                "updated_at": "2000-01-01T00:00:01Z",
+                "html_url": "https://github.com/airbytehq/airbyte/pull/2#pullrequestreview-1006",
+                "commit": null,
+                "user": {
+                  "__typename": "User"
+                }
+              },
+              {
+                "id": 1007,
+                "updated_at": "2000-01-01T00:00:02Z",
+                "html_url": "https://github.com/airbytehq/airbyte/pull/2#pullrequestreview-1007",
+                "commit": null,
+                "user": {
+                  "__typename": "User"
+                }
+              }
+            ],
+            "pageInfo": {
+              "endCursor": null,
+              "hasNextPage": false
+            }
+          }
+        }
+      }
+    }
+  },
+  {
+    "data": {
+      "repository": {
+        "owner": {
+          "login": "airbytehq"
+        },
+        "name": "airbyte",
+        "pullRequests": {
+          "nodes": [
+            {
+              "number": 3,
+              "url": "https://github.com/airbytehq/airbyte/pull/3",
+              "reviews": {
+                "nodes": [
+                  {
+                    "id": 1008,
+                    "updated_at": "2000-01-01T00:00:01Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/3#pullrequestreview-1008",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  },
+                  {
+                    "id": 1009,
+                    "updated_at": "2000-01-01T00:00:02Z",
+                    "html_url": "https://github.com/airbytehq/airbyte/pull/3#pullrequestreview-1009",
+                    "commit": null,
+                    "user": {
+                      "__typename": "User"
+                    }
+                  }
+                ],
+                "pageInfo": {
+                  "endCursor": null,
+                  "hasNextPage": false
+                }
+              }
+            }
+          ],
+          "pageInfo": {
+            "endCursor": null,
+            "hasNextPage": false
+          }
+        }
+      }
+    }
+  }
+]