airbyte-source-github 1.6.6.dev202403201925__py3-none-any.whl → 1.7.1__py3-none-any.whl

{airbyte_source_github-1.6.6.dev202403201925.dist-info → airbyte_source_github-1.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-github
-Version: 1.6.6.dev202403201925
+Version: 1.7.1
 Summary: Source implementation for GitHub.
 Home-page: https://airbyte.com
 License: MIT
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=0.68.4,<0.69.0)
+Requires-Dist: airbyte-cdk (>=0,<1)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/github
 Project-URL: Repository, https://github.com/airbytehq/airbyte

{airbyte_source_github-1.6.6.dev202403201925.dist-info → airbyte_source_github-1.7.1.dist-info}/RECORD

@@ -52,11 +52,11 @@ source_github/schemas/users.json,sha256=xASJmm56AqLYxSCfn5qlPy0xUVJOW8K3gWlwRr4J
 source_github/schemas/workflow_jobs.json,sha256=ORowQYqvJhJE2EEV1jXyQSPCFmtO6NyhJZGTgpXte1Q,2089
 source_github/schemas/workflow_runs.json,sha256=JWK1p1HQI2dDnutF4rd7gPG7Nx1_RJL2VXIka4KQwMQ,10171
 source_github/schemas/workflows.json,sha256=zvtOslS-veNo5_iXmMxMNlY8OOt8DdvTZ3hjtdJbdvY,753
-source_github/source.py,sha256=MSAwf6DCNMhu6Pu6ADPR7EfOIYYDnYf8bnx1LHG6LY4,15612
-source_github/spec.json,sha256=_L8sFYPPMeUTRqXOza0IISyR3c2E5u4aHThKMQA2r4s,7096
-source_github/streams.py,sha256=Vu9tuyeaEseJII7YjZDF0Q3l6cFkq_aMp6d8_kID3rE,75297
+source_github/source.py,sha256=jOGHJLL6ys4NRAjGs-Lw1RJxK25NxA5XS8n-uH2Gico,13879
+source_github/spec.json,sha256=tsRjwqInYQjvqhm-Yzdn7_VC5QyInCGAToFJnUrCnOU,7074
+source_github/streams.py,sha256=vcP2P0vCvShaa3z8M9dnmbY6awKR8vAIw3XgR8PlZtk,77006
 source_github/utils.py,sha256=DfAHFjsF8hzDXeSCR6qtfs7W_av6o2BkkEVhtHpWbis,5462
-airbyte_source_github-1.6.6.dev202403201925.dist-info/METADATA,sha256=p7UQci9K2xTFRcsg4eLmOOSLVlyEskU_CQ1t7WaJH5A,5254
-airbyte_source_github-1.6.6.dev202403201925.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-airbyte_source_github-1.6.6.dev202403201925.dist-info/entry_points.txt,sha256=gYhqVrTAZvMwuYByg0b_-o115yUFLLcfNxMrLZmiW9k,55
-airbyte_source_github-1.6.6.dev202403201925.dist-info/RECORD,,
+airbyte_source_github-1.7.1.dist-info/METADATA,sha256=fr12eeFOb6XmyXnxebMc5CnwuBKsQZieK9wmbGt_aEo,5228
+airbyte_source_github-1.7.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+airbyte_source_github-1.7.1.dist-info/entry_points.txt,sha256=gYhqVrTAZvMwuYByg0b_-o115yUFLLcfNxMrLZmiW9k,55
+airbyte_source_github-1.7.1.dist-info/RECORD,,

source_github/source.py

@@ -3,7 +3,7 @@
 #
 
 from os import getenv
-from typing import Any, Dict, List, Mapping, MutableMapping, Tuple
+from typing import Any, List, Mapping, MutableMapping, Optional, Tuple
 from urllib.parse import urlparse
 
 from airbyte_cdk import AirbyteLogger
@@ -65,7 +65,9 @@ class SourceGithub(AbstractSource):
     continue_sync_on_stream_failure = True
 
     @staticmethod
-    def _get_org_repositories(config: Mapping[str, Any], authenticator: MultipleTokenAuthenticator) -> Tuple[List[str], List[str]]:
+    def _get_org_repositories(
+        config: Mapping[str, Any], authenticator: MultipleTokenAuthenticator
+    ) -> Tuple[List[str], List[str], Optional[str]]:
         """
         Parse config/repositories and produce two lists: organizations, repositories.
         Args:
@@ -78,16 +80,19 @@ class SourceGithub(AbstractSource):
         organizations = set()
         unchecked_repos = set()
         unchecked_orgs = set()
+        pattern = None
 
         for org_repos in config_repositories:
-            org, _, repos = org_repos.partition("/")
-            if repos == "*":
-                unchecked_orgs.add(org)
+            _, _, repos = org_repos.partition("/")
+            if "*" in repos:
+                unchecked_orgs.add(org_repos)
             else:
                 unchecked_repos.add(org_repos)
 
         if unchecked_orgs:
-            stream = Repositories(authenticator=authenticator, organizations=unchecked_orgs, api_url=config.get("api_url"))
+            org_names = [org.split("/")[0] for org in unchecked_orgs]
+            pattern = "|".join([f"({org.replace('*', '.*')})" for org in unchecked_orgs])
+            stream = Repositories(authenticator=authenticator, organizations=org_names, api_url=config.get("api_url"), pattern=pattern)
             for record in read_full_refresh(stream):
                 repositories.add(record["full_name"])
                 organizations.add(record["organization"])
@@ -96,7 +101,7 @@ class SourceGithub(AbstractSource):
         if unchecked_repos:
             stream = RepositoryStats(
                 authenticator=authenticator,
-                repositories=unchecked_repos,
+                repositories=list(unchecked_repos),
                 api_url=config.get("api_url"),
                 # This parameter is deprecated and in future will be used sane default, page_size: 10
                 page_size_for_large_streams=config.get("page_size_for_large_streams", constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM),
@@ -107,7 +112,7 @@ class SourceGithub(AbstractSource):
                 if organization:
                     organizations.add(organization)
 
-        return list(organizations), list(repositories)
+        return list(organizations), list(repositories), pattern
 
     @staticmethod
     def get_access_token(config: Mapping[str, Any]):
@@ -169,45 +174,6 @@ class SourceGithub(AbstractSource):
     def _is_http_allowed() -> bool:
         return getenv("DEPLOYMENT_MODE", "").upper() != "CLOUD"
 
-    @staticmethod
-    def _get_branches_data(
-        selected_branches: List, full_refresh_args: Dict[str, Any] = None
-    ) -> Tuple[Dict[str, str], Dict[str, List[str]]]:
-        selected_branches = set(selected_branches)
-
-        # Get the default branch for each repository
-        default_branches = {}
-        repository_stats_stream = RepositoryStats(**full_refresh_args)
-        for stream_slice in repository_stats_stream.stream_slices(sync_mode=SyncMode.full_refresh):
-            default_branches.update(
-                {
-                    repo_stats["full_name"]: repo_stats["default_branch"]
-                    for repo_stats in repository_stats_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)
-                }
-            )
-
-        all_branches = []
-        branches_stream = Branches(**full_refresh_args)
-        for stream_slice in branches_stream.stream_slices(sync_mode=SyncMode.full_refresh):
-            for branch in branches_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
-                all_branches.append(f"{branch['repository']}/{branch['name']}")
-
-        # Create mapping of repository to list of branches to pull commits for
-        # If no branches are specified for a repo, use its default branch
-        branches_to_pull: Dict[str, List[str]] = {}
-        for repo in full_refresh_args["repositories"]:
-            repo_branches = []
-            for branch in selected_branches:
-                branch_parts = branch.split("/", 2)
-                if "/".join(branch_parts[:2]) == repo and branch in all_branches:
-                    repo_branches.append(branch_parts[-1])
-            if not repo_branches:
-                repo_branches = [default_branches[repo]]
-
-            branches_to_pull[repo] = repo_branches
-
-        return default_branches, branches_to_pull
-
     def user_friendly_error_message(self, message: str) -> str:
         user_message = ""
         if "404 Client Error: Not Found for url: https://api.github.com/repos/" in message:
@@ -229,7 +195,7 @@ class SourceGithub(AbstractSource):
         config = self._validate_and_transform_config(config)
         try:
             authenticator = self._get_authenticator(config)
-            _, repositories = self._get_org_repositories(config=config, authenticator=authenticator)
+            _, repositories, _ = self._get_org_repositories(config=config, authenticator=authenticator)
             if not repositories:
                 return (
                     False,
@@ -246,7 +212,7 @@ class SourceGithub(AbstractSource):
         authenticator = self._get_authenticator(config)
         config = self._validate_and_transform_config(config)
         try:
-            organizations, repositories = self._get_org_repositories(config=config, authenticator=authenticator)
+            organizations, repositories, pattern = self._get_org_repositories(config=config, authenticator=authenticator)
         except Exception as e:
             message = repr(e)
             user_message = self.user_friendly_error_message(message)
@@ -291,7 +257,6 @@ class SourceGithub(AbstractSource):
         }
         repository_args_with_start_date = {**repository_args, "start_date": start_date}
 
-        default_branches, branches_to_pull = self._get_branches_data(config.get("branch", []), repository_args)
         pull_requests_stream = PullRequests(**repository_args_with_start_date)
         projects_stream = Projects(**repository_args_with_start_date)
         project_columns_stream = ProjectColumns(projects_stream, **repository_args_with_start_date)
@@ -307,7 +272,7 @@ class SourceGithub(AbstractSource):
             Comments(**repository_args_with_start_date),
             CommitCommentReactions(**repository_args_with_start_date),
             CommitComments(**repository_args_with_start_date),
-            Commits(**repository_args_with_start_date, branches_to_pull=branches_to_pull, default_branches=default_branches),
+            Commits(**repository_args_with_start_date, branches_to_pull=config.get("branches", [])),
             ContributorActivity(**repository_args),
             Deployments(**repository_args_with_start_date),
             Events(**repository_args_with_start_date),
@@ -327,7 +292,7 @@ class SourceGithub(AbstractSource):
             ProjectsV2(**repository_args_with_start_date),
             pull_requests_stream,
             Releases(**repository_args_with_start_date),
-            Repositories(**organization_args_with_start_date),
+            Repositories(**organization_args_with_start_date, pattern=pattern),
             ReviewComments(**repository_args_with_start_date),
             Reviews(**repository_args_with_start_date),
             Stargazers(**repository_args_with_start_date),

source_github/spec.json

@@ -81,18 +81,19 @@
         "type": "array",
         "items": {
           "type": "string",
-          "pattern": "^([\\w.-]+/(\\*|[\\w.-]+(?<!\\.git))\\s+)*[\\w.-]+/(\\*|[\\w.-]+(?<!\\.git))$"
+          "pattern": "^[\\w.-]+/(([\\w.-]*\\*)|[\\w.-]+(?<!\\.git))$"
         },
         "minItems": 1,
         "examples": [
-          "airbytehq/airbyte airbytehq/another-repo",
+          "airbytehq/airbyte",
+          "airbytehq/another-repo",
           "airbytehq/*",
-          "airbytehq/airbyte"
+          "airbytehq/a*"
         ],
         "title": "GitHub Repositories",
-        "description": "List of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get all repositories from organization and `airbytehq/airbyte airbytehq/another-repo` for multiple repositories.",
+        "description": "List of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get all repositories from organization and `airbytehq/a* for matching multiple repositories by pattern.",
         "order": 1,
-        "pattern_descriptor": "org/repo org/another-repo org/*"
+        "pattern_descriptor": "org/repo org/another-repo org/* org/a*"
       },
       "start_date": {
         "type": "string",
@@ -126,7 +127,7 @@
           "type": "string"
         },
         "title": "Branches",
-        "examples": ["airbytehq/airbyte/master airbytehq/airbyte/my-branch"],
+        "examples": ["airbytehq/airbyte/master", "airbytehq/airbyte/my-branch"],
         "description": "List of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.",
         "order": 4,
         "pattern_descriptor": "org/repo/branch1 org/repo/branch2"

source_github/streams.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+import re
 import time
 from abc import ABC, abstractmethod
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional
@@ -41,8 +42,7 @@ class GithubStreamABC(HttpStream, ABC):
 
     def __init__(self, api_url: str = "https://api.github.com", access_token_type: str = "", **kwargs):
         if kwargs.get("authenticator"):
-            # TODO: get max time from env variable via Retool?
-            kwargs["authenticator"].max_time = 60 * 60  # Set to 60 minutes to bypass limitation of CDK
+            kwargs["authenticator"].max_time = self.max_time
         super().__init__(**kwargs)
 
         self.access_token_type = access_token_type
@@ -442,12 +442,18 @@ class Repositories(SemiIncrementalMixin, Organizations):
         "direction": "desc",
     }
 
+    def __init__(self, *args, pattern: Optional[str] = None, **kwargs):
+        self._pattern = re.compile(pattern) if pattern else pattern
+        super().__init__(*args, **kwargs)
+
     def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         return f"orgs/{stream_slice['organization']}/repos"
 
     def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]:
         for record in response.json():  # GitHub puts records in an array.
-            yield self.transform(record=record, stream_slice=stream_slice)
+            record = self.transform(record=record, stream_slice=stream_slice)
+            if not self._pattern or self._pattern.match(record["full_name"]):
+                yield record
 
 
 class Tags(GithubStream):
@@ -677,10 +683,13 @@ class Commits(IncrementalMixin, GithubStream):
     cursor_field = "created_at"
     slice_keys = ["repository", "branch"]
 
-    def __init__(self, branches_to_pull: Mapping[str, List[str]], default_branches: Mapping[str, str], **kwargs):
+    def __init__(self, branches_to_pull: List[str], **kwargs):
         super().__init__(**kwargs)
-        self.branches_to_pull = branches_to_pull
-        self.default_branches = default_branches
+        kwargs.pop("start_date")
+        self.branches_to_repos = {}
+        self.branches_to_pull = set(branches_to_pull)
+        self.branches_stream = Branches(**kwargs)
+        self.repositories_stream = RepositoryStats(**kwargs)
 
     def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]:
         params = super(IncrementalMixin, self).request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs)
@@ -691,9 +700,10 @@ class Commits(IncrementalMixin, GithubStream):
         return params
 
     def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
+        self._validate_branches_to_pull()
         for stream_slice in super().stream_slices(**kwargs):
             repository = stream_slice["repository"]
-            for branch in self.branches_to_pull.get(repository, []):
+            for branch in self.branches_to_repos.get(repository, []):
                 yield {"branch": branch, "repository": repository}
 
     def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
@@ -719,6 +729,30 @@ class Commits(IncrementalMixin, GithubStream):
         current_stream_state.setdefault(repository, {}).setdefault(branch, {})[self.cursor_field] = updated_state
         return current_stream_state
 
+    def _validate_branches_to_pull(self):
+        # Get the default branch for each repository
+        default_branches = {}
+        for stream_slice in self.repositories_stream.stream_slices(sync_mode=SyncMode.full_refresh):
+            for repo_stats in self.repositories_stream.read_records(stream_slice=stream_slice, sync_mode=SyncMode.full_refresh):
+                default_branches[repo_stats["full_name"]] = repo_stats["default_branch"]
+
+        all_branches = []
+        for stream_slice in self.branches_stream.stream_slices(sync_mode=SyncMode.full_refresh):
+            for branch in self.branches_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
+                all_branches.append(f"{branch['repository']}/{branch['name']}")
+
+        # Create mapping of repository to list of branches to pull commits for
+        # If no branches are specified for a repo, use its default branch
+        for repo in self.repositories:
+            repo_branches = []
+            for branch in self.branches_to_pull:
+                branch_parts = branch.split("/", 2)
+                if "/".join(branch_parts[:2]) == repo and branch in all_branches:
+                    repo_branches.append(branch_parts[-1])
+            if not repo_branches:
+                repo_branches = [default_branches[repo]]
+            self.branches_to_repos[repo] = repo_branches
+
 
 class Issues(IncrementalMixin, GithubStream):
     """