airbyte-source-github 1.5.7__py3-none-any.whl

unit_tests/responses/pull_request_stats_response.json

@@ -0,0 +1,317 @@
+{
+  "data": {
+    "repository": {
+      "name": "integration-test",
+      "owner": {
+        "login": "airbytehq"
+      },
+      "pullRequests": {
+        "nodes": [
+          {
+            "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2",
+            "id": 721435506,
+            "number": 5,
+            "updated_at": "2021-08-27T15:53:14Z",
+            "changed_files": 5,
+            "deletions": 0,
+            "additions": 5,
+            "merged": false,
+            "mergeable": "MERGEABLE",
+            "can_be_rebased": true,
+            "maintainer_can_modify": false,
+            "merge_state_status": "BLOCKED",
+            "comments": {
+              "totalCount": 0
+            },
+            "commits": {
+              "totalCount": 5
+            },
+            "review_comments": {
+              "totalCount": 1,
+              "nodes": [
+                {
+                  "comments": {
+                    "totalCount": 0
+                  }
+                }
+              ]
+            },
+            "merged_by": null
+          },
+          {
+            "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NDA3",
+            "id": 721435407,
+            "number": 2,
+            "updated_at": "2021-08-27T15:53:27Z",
+            "changed_files": 5,
+            "deletions": 0,
+            "additions": 5,
+            "merged": false,
+            "mergeable": "MERGEABLE",
+            "can_be_rebased": true,
+            "maintainer_can_modify": false,
+            "merge_state_status": "BLOCKED",
+            "comments": {
+              "totalCount": 0
+            },
+            "commits": {
+              "totalCount": 5
+            },
+            "review_comments": {
+              "totalCount": 1,
+              "nodes": [
+                {
+                  "comments": {
+                    "totalCount": 0
+                  }
+                }
+              ]
+            },
+            "merged_by": null
+          },
+          {
+            "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NDQw",
+            "id": 721435440,
+            "number": 3,
+            "updated_at": "2021-08-27T16:02:49Z",
+            "changed_files": 5,
+            "deletions": 0,
+            "additions": 5,
+            "merged": true,
+            "mergeable": "UNKNOWN",
+            "can_be_rebased": false,
+            "maintainer_can_modify": false,
+            "merge_state_status": "UNKNOWN",
+            "comments": {
+              "totalCount": 0
+            },
+            "commits": {
+              "totalCount": 5
+            },
+            "review_comments": {
+              "totalCount": 1,
+              "nodes": [
+                {
+                  "comments": {
+                    "totalCount": 0
+                  }
+                }
+              ]
+            },
+            "merged_by": {
+              "__typename": "User",
+              "node_id": "MDQ6VXNlcjc0MzkwMQ==",
+              "id": 743901,
+              "login": "gaart",
+              "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4",
+              "html_url": "https://github.com/gaart",
+              "site_admin": false
+            }
+          },
+          {
+            "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NDY2",
+            "id": 721435466,
+            "number": 4,
+            "updated_at": "2021-08-31T12:01:15Z",
+            "changed_files": 5,
+            "deletions": 0,
+            "additions": 5,
+            "merged": false,
+            "mergeable": "MERGEABLE",
+            "can_be_rebased": true,
+            "maintainer_can_modify": false,
+            "merge_state_status": "BLOCKED",
+            "comments": {
+              "totalCount": 0
+            },
+            "commits": {
+              "totalCount": 5
+            },
+            "review_comments": {
+              "totalCount": 3,
+              "nodes": [
+                {
+                  "comments": {
+                    "totalCount": 0
+                  }
+                },
+                {
+                  "comments": {
+                    "totalCount": 0
+                  }
+                },
+                {
+                  "comments": {
+                    "totalCount": 1
+                  }
+                }
+              ]
+            },
+            "merged_by": null
+          },
+          {
+            "node_id": "PR_kwDOF9hP9c4xmEi6",
+            "id": 832063674,
+            "number": 12,
+            "updated_at": "2022-01-26T03:46:56Z",
+            "changed_files": 1,
+            "deletions": 0,
+            "additions": 2,
+            "merged": true,
+            "mergeable": "UNKNOWN",
+            "can_be_rebased": false,
+            "maintainer_can_modify": false,
+            "merge_state_status": "UNKNOWN",
+            "comments": {
+              "totalCount": 0
+            },
+            "commits": {
+              "totalCount": 1
+            },
+            "review_comments": {
+              "totalCount": 1,
+              "nodes": [
+                {
+                  "comments": {
+                    "totalCount": 0
+                  }
+                }
+              ]
+            },
+            "merged_by": {
+              "__typename": "User",
+              "node_id": "MDQ6VXNlcjUxNTQzMjI=",
+              "id": 5154322,
+              "login": "marcosmarxm",
+              "avatar_url": "https://avatars.githubusercontent.com/u/5154322?u=92c89b82271d48f41fad03923b0a24083e049038&v=4",
+              "html_url": "https://github.com/marcosmarxm",
+              "site_admin": false
+            }
+          },
+          {
+            "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1Mzcz",
+            "id": 721435373,
+            "number": 1,
+            "updated_at": "2022-03-31T11:06:06Z",
+            "changed_files": 5,
+            "deletions": 0,
+            "additions": 5,
+            "merged": false,
+            "mergeable": "MERGEABLE",
+            "can_be_rebased": true,
+            "maintainer_can_modify": false,
+            "merge_state_status": "BLOCKED",
+            "comments": {
+              "totalCount": 0
+            },
+            "commits": {
+              "totalCount": 5
+            },
+            "review_comments": {
+              "totalCount": 1,
+              "nodes": [
+                {
+                  "comments": {
+                    "totalCount": 0
+                  }
+                }
+              ]
+            },
+            "merged_by": null
+          },
+          {
+            "node_id": "PR_kwDOF9hP9c46s2Qa",
+            "id": 984835098,
+            "number": 14,
+            "updated_at": "2022-10-04T17:41:29Z",
+            "changed_files": 1,
+            "deletions": 0,
+            "additions": 1,
+            "merged": false,
+            "mergeable": "MERGEABLE",
+            "can_be_rebased": true,
+            "maintainer_can_modify": false,
+            "merge_state_status": "BLOCKED",
+            "comments": {
+              "totalCount": 0
+            },
+            "commits": {
+              "totalCount": 2
+            },
+            "review_comments": {
+              "totalCount": 1,
+              "nodes": [
+                {
+                  "comments": {
+                    "totalCount": 0
+                  }
+                }
+              ]
+            },
+            "merged_by": null
+          },
+          {
+            "node_id": "PR_kwDOF9hP9c41Vftv",
+            "id": 894827375,
+            "number": 13,
+            "updated_at": "2023-05-03T06:50:23Z",
+            "changed_files": 1,
+            "deletions": 1,
+            "additions": 1,
+            "merged": false,
+            "mergeable": "MERGEABLE",
+            "can_be_rebased": true,
+            "maintainer_can_modify": false,
+            "merge_state_status": "BLOCKED",
+            "comments": {
+              "totalCount": 0
+            },
+            "commits": {
+              "totalCount": 1
+            },
+            "review_comments": {
+              "totalCount": 0,
+              "nodes": []
+            },
+            "merged_by": null
+          },
+          {
+            "node_id": "PR_kwDOF9hP9c5PqZhG",
+            "id": 1336514630,
+            "number": 15,
+            "updated_at": "2023-05-04T10:07:46Z",
+            "changed_files": 1,
+            "deletions": 0,
+            "additions": 1,
+            "merged": false,
+            "mergeable": "MERGEABLE",
+            "can_be_rebased": false,
+            "maintainer_can_modify": false,
+            "merge_state_status": "BLOCKED",
+            "comments": {
+              "totalCount": 1
+            },
+            "commits": {
+              "totalCount": 1
+            },
+            "review_comments": {
+              "totalCount": 1,
+              "nodes": [
+                {
+                  "comments": {
+                    "totalCount": 0
+                  }
+                }
+              ]
+            },
+            "merged_by": null
+          }
+        ],
+        "pageInfo": {
+          "hasNextPage": false,
+          "endCursor": "Y3Vyc29yOnYyOpK5MjAyMy0wNS0wNFQxMzowNzo0NiswMzowMM5PqZhG"
+        }
+      }
+    }
+  }
+}

unit_tests/test_migrations/test_config.json

@@ -0,0 +1,8 @@
+{
+  "credentials": {
+    "personal_access_token": "personal_access_token"
+  },
+  "repository": "airbytehq/airbyte airbytehq/airbyte-platform",
+  "start_date": "2000-01-01T00:00:00Z",
+  "branch": "airbytehq/airbyte/master airbytehq/airbyte-platform/main"
+}

unit_tests/test_migrations/test_new_config.json

@@ -0,0 +1,8 @@
+{
+  "credentials": {
+    "personal_access_token": "personal_access_token"
+  },
+  "repositories": ["airbytehq/airbyte", "airbytehq/airbyte-platform"],
+  "start_date": "2000-01-01T00:00:00Z",
+  "branches": ["airbytehq/airbyte/master", "airbytehq/airbyte-platform/main"]
+}

unit_tests/test_multiple_token_authenticator.py

@@ -0,0 +1,160 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import json
+from unittest.mock import patch
+
+import pendulum
+import responses
+from freezegun import freeze_time
+from source_github import SourceGithub
+from source_github.streams import Organizations
+from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter, read_full_refresh
+
+
+@responses.activate
+def test_multiple_tokens(rate_limit_mock_response):
+    authenticator = SourceGithub()._get_authenticator({"access_token": "token_1, token_2, token_3"})
+    assert isinstance(authenticator, MultipleTokenAuthenticatorWithRateLimiter)
+    assert ["token_1", "token_2", "token_3"] == list(authenticator._tokens)
+
+
+@responses.activate
+def test_authenticator_counter(rate_limit_mock_response):
+    """
+    This test ensures that the rate limiter:
+     1. correctly handles the available limits from GitHub API and saves it.
+     2. correctly counts the number of requests made.
+    """
+    authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2", "token3"])
+
+    assert [(x.count_rest, x.count_graphql) for x in authenticator._tokens.values()] == [(5000, 5000), (5000, 5000), (5000, 5000)]
+    organization_args = {"organizations": ["org1", "org2"], "authenticator": authenticator}
+    stream = Organizations(**organization_args)
+    responses.add("GET", "https://api.github.com/orgs/org1", json={"id": 1})
+    responses.add("GET", "https://api.github.com/orgs/org2", json={"id": 2})
+    list(read_full_refresh(stream))
+    assert authenticator._tokens["token1"].count_rest == 4998
+
+
+@responses.activate
+def test_multiple_token_authenticator_with_rate_limiter(caplog):
+    """
+    This test ensures that:
+     1. The rate limiter iterates over all tokens one-by-one after the previous is fully drained.
+     2. Counter is set to zero after 1500 requests were made. (500 available requests per key were set as default)
+     3. Exception is handled and log warning message could be found in output. Connector does not raise AirbyteTracedException because there might be GraphQL streams with remaining request we still can read.
+    """
+
+    counter_rate_limits = 0
+    counter_orgs = 0
+
+    def request_callback_rate_limits(request):
+        nonlocal counter_rate_limits
+        while counter_rate_limits < 3:
+            counter_rate_limits += 1
+            resp_body = {
+                "resources": {
+                    "core": {
+                        "limit": 500,
+                        "used": 0,
+                        "remaining": 500,
+                        "reset": 4070908800
+                    },
+                    "graphql": {
+                        "limit": 500,
+                        "used": 0,
+                        "remaining": 500,
+                        "reset": 4070908800
+                    }
+                }
+            }
+            return (200, {}, json.dumps(resp_body))
+
+    responses.add_callback(responses.GET, "https://api.github.com/rate_limit", callback=request_callback_rate_limits)
+    authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2", "token3"])
+    organization_args = {"organizations": ["org1"], "authenticator": authenticator}
+    stream = Organizations(**organization_args)
+
+    def request_callback_orgs(request):
+        nonlocal counter_orgs
+        while counter_orgs < 1_501:
+            counter_orgs += 1
+            resp_body = {"id": 1}
+            headers = {"Link": '<https://api.github.com/orgs/org1?page=2>; rel="next"'}
+            return (200, headers, json.dumps(resp_body))
+
+    responses.add_callback(
+        responses.GET,
+        "https://api.github.com/orgs/org1",
+        callback=request_callback_orgs,
+        content_type="application/json",
+    )
+
+    list(read_full_refresh(stream))
+    assert [(x.count_rest, x.count_graphql) for x in authenticator._tokens.values()] == [(0, 500), (0, 500), (0, 500)]
+    assert "Stream: `organizations`, slice: `{'organization': 'org1'}`. Limits for all provided tokens are reached, please try again later" in caplog.messages
+
+
+@freeze_time("2021-01-01 12:00:00")
+@responses.activate
+@patch("time.sleep")
+def test_multiple_token_authenticator_with_rate_limiter_and_sleep(sleep_mock, caplog):
+    """
+    This test ensures that:
+     1. The rate limiter will only wait (sleep) for token availability if the nearest available token appears within 600 seconds (see max_time).
+     2. Token Counter is reset to new values after 1500 requests were made and last token is still in use.
+    """
+
+    counter_rate_limits = 0
+    counter_orgs = 0
+    ACCEPTED_WAITING_TIME_IN_SECONDS = 595
+    reset_time = (pendulum.now() + pendulum.duration(seconds=ACCEPTED_WAITING_TIME_IN_SECONDS)).int_timestamp
+
+    def request_callback_rate_limits(request):
+        nonlocal counter_rate_limits
+        while counter_rate_limits < 6:
+            counter_rate_limits += 1
+            resp_body = {
+                "resources": {
+                    "core": {
+                        "limit": 500,
+                        "used": 0,
+                        "remaining": 500,
+                        "reset": reset_time
+                    },
+                    "graphql": {
+                        "limit": 500,
+                        "used": 0,
+                        "remaining": 500,
+                        "reset": reset_time
+                    }
+                }
+            }
+            return (200, {}, json.dumps(resp_body))
+
+    responses.add_callback(responses.GET, "https://api.github.com/rate_limit", callback=request_callback_rate_limits)
+    authenticator = MultipleTokenAuthenticatorWithRateLimiter(tokens=["token1", "token2", "token3"])
+    organization_args = {"organizations": ["org1"], "authenticator": authenticator}
+    stream = Organizations(**organization_args)
+
+    def request_callback_orgs(request):
+        nonlocal counter_orgs
+        while counter_orgs < 1_501:
+            counter_orgs += 1
+            resp_body = {"id": 1}
+            headers = {"Link": '<https://api.github.com/orgs/org1?page=2>; rel="next"'}
+            return (200, headers, json.dumps(resp_body))
+        return (200, {}, json.dumps({"id": 2}))
+
+    responses.add_callback(
+        responses.GET,
+        "https://api.github.com/orgs/org1",
+        callback=request_callback_orgs,
+        content_type="application/json",
+    )
+
+    list(read_full_refresh(stream))
+    sleep_mock.assert_called_once_with(ACCEPTED_WAITING_TIME_IN_SECONDS)
+    assert [(x.count_rest, x.count_graphql) for x in authenticator._tokens.values()] == [(500, 500), (500, 500), (498, 500)]