airbyte-source-github 1.5.7__py3-none-any.whl

source_github/source.py

@@ -0,0 +1,339 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from os import getenv
+from typing import Any, Dict, List, Mapping, MutableMapping, Tuple
+from urllib.parse import urlparse
+
+from airbyte_cdk import AirbyteLogger
+from airbyte_cdk.models import FailureType, SyncMode
+from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http.auth import MultipleTokenAuthenticator
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter
+
+from . import constants
+from .streams import (
+    Assignees,
+    Branches,
+    Collaborators,
+    Comments,
+    CommitCommentReactions,
+    CommitComments,
+    Commits,
+    ContributorActivity,
+    Deployments,
+    Events,
+    IssueCommentReactions,
+    IssueEvents,
+    IssueLabels,
+    IssueMilestones,
+    IssueReactions,
+    Issues,
+    IssueTimelineEvents,
+    Organizations,
+    ProjectCards,
+    ProjectColumns,
+    Projects,
+    ProjectsV2,
+    PullRequestCommentReactions,
+    PullRequestCommits,
+    PullRequests,
+    PullRequestStats,
+    Releases,
+    Repositories,
+    RepositoryStats,
+    ReviewComments,
+    Reviews,
+    Stargazers,
+    Tags,
+    TeamMembers,
+    TeamMemberships,
+    Teams,
+    Users,
+    WorkflowJobs,
+    WorkflowRuns,
+    Workflows,
+)
+from .utils import read_full_refresh
+
+
+class SourceGithub(AbstractSource):
+    @staticmethod
+    def _get_org_repositories(config: Mapping[str, Any], authenticator: MultipleTokenAuthenticator) -> Tuple[List[str], List[str]]:
+        """
+        Parse config/repositories and produce two lists: organizations, repositories.
+        Args:
+            config (dict): Dict representing connector's config
+            authenticator(MultipleTokenAuthenticator): authenticator object
+        """
+        config_repositories = set(config.get("repositories"))
+
+        repositories = set()
+        organizations = set()
+        unchecked_repos = set()
+        unchecked_orgs = set()
+
+        for org_repos in config_repositories:
+            org, _, repos = org_repos.partition("/")
+            if repos == "*":
+                unchecked_orgs.add(org)
+            else:
+                unchecked_repos.add(org_repos)
+
+        if unchecked_orgs:
+            stream = Repositories(authenticator=authenticator, organizations=unchecked_orgs, api_url=config.get("api_url"))
+            for record in read_full_refresh(stream):
+                repositories.add(record["full_name"])
+                organizations.add(record["organization"])
+
+        unchecked_repos = unchecked_repos - repositories
+        if unchecked_repos:
+            stream = RepositoryStats(
+                authenticator=authenticator,
+                repositories=unchecked_repos,
+                api_url=config.get("api_url"),
+                # This parameter is deprecated and in future will be used sane default, page_size: 10
+                page_size_for_large_streams=config.get("page_size_for_large_streams", constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM),
+            )
+            for record in read_full_refresh(stream):
+                repositories.add(record["full_name"])
+                organization = record.get("organization", {}).get("login")
+                if organization:
+                    organizations.add(organization)
+
+        return list(organizations), list(repositories)
+
+    @staticmethod
+    def get_access_token(config: Mapping[str, Any]):
+        # Before we supported oauth, personal_access_token was called `access_token` and it lived at the
+        # config root. So we first check to make sure any backwards compatbility is handled.
+        if "access_token" in config:
+            return constants.PERSONAL_ACCESS_TOKEN_TITLE, config["access_token"]
+
+        credentials = config.get("credentials", {})
+        if "access_token" in credentials:
+            return constants.ACCESS_TOKEN_TITLE, credentials["access_token"]
+        if "personal_access_token" in credentials:
+            return constants.PERSONAL_ACCESS_TOKEN_TITLE, credentials["personal_access_token"]
+        raise Exception("Invalid config format")
+
+    def _get_authenticator(self, config: Mapping[str, Any]):
+        _, token = self.get_access_token(config)
+        tokens = [t.strip() for t in token.split(constants.TOKEN_SEPARATOR)]
+        return MultipleTokenAuthenticatorWithRateLimiter(tokens=tokens)
+
+    def _validate_and_transform_config(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+        config = self._ensure_default_values(config)
+        config = self._validate_repositories(config)
+        config = self._validate_branches(config)
+        return config
+
+    def _ensure_default_values(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+        config.setdefault("api_url", "https://api.github.com")
+        api_url_parsed = urlparse(config["api_url"])
+
+        if not api_url_parsed.scheme.startswith("http"):
+            message = "Please enter a full url for `API URL` field starting with `http`"
+        elif api_url_parsed.scheme == "http" and not self._is_http_allowed():
+            message = "HTTP connection is insecure and is not allowed in this environment. Please use `https` instead."
+        elif not api_url_parsed.netloc:
+            message = "Please provide a correct API URL."
+        else:
+            return config
+
+        raise AirbyteTracedException(message=message, failure_type=FailureType.config_error)
+
+    def _validate_repositories(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+        if config.get("repositories"):
+            pass
+        elif config.get("repository"):
+            config["repositories"] = set(filter(None, config["repository"].split(" ")))
+
+        return config
+
+    def _validate_branches(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+        if config.get("branches"):
+            pass
+        elif config.get("branch"):
+            config["branches"] = set(filter(None, config["branch"].split(" ")))
+
+        return config
+
+    @staticmethod
+    def _is_http_allowed() -> bool:
+        return getenv("DEPLOYMENT_MODE", "").upper() != "CLOUD"
+
+    @staticmethod
+    def _get_branches_data(
+        selected_branches: List, full_refresh_args: Dict[str, Any] = None
+    ) -> Tuple[Dict[str, str], Dict[str, List[str]]]:
+        selected_branches = set(selected_branches)
+
+        # Get the default branch for each repository
+        default_branches = {}
+        repository_stats_stream = RepositoryStats(**full_refresh_args)
+        for stream_slice in repository_stats_stream.stream_slices(sync_mode=SyncMode.full_refresh):
+            default_branches.update(
+                {
+                    repo_stats["full_name"]: repo_stats["default_branch"]
+                    for repo_stats in repository_stats_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)
+                }
+            )
+
+        all_branches = []
+        branches_stream = Branches(**full_refresh_args)
+        for stream_slice in branches_stream.stream_slices(sync_mode=SyncMode.full_refresh):
+            for branch in branches_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
+                all_branches.append(f"{branch['repository']}/{branch['name']}")
+
+        # Create mapping of repository to list of branches to pull commits for
+        # If no branches are specified for a repo, use its default branch
+        branches_to_pull: Dict[str, List[str]] = {}
+        for repo in full_refresh_args["repositories"]:
+            repo_branches = []
+            for branch in selected_branches:
+                branch_parts = branch.split("/", 2)
+                if "/".join(branch_parts[:2]) == repo and branch in all_branches:
+                    repo_branches.append(branch_parts[-1])
+            if not repo_branches:
+                repo_branches = [default_branches[repo]]
+
+            branches_to_pull[repo] = repo_branches
+
+        return default_branches, branches_to_pull
+
+    def user_friendly_error_message(self, message: str) -> str:
+        user_message = ""
+        if "404 Client Error: Not Found for url: https://api.github.com/repos/" in message:
+            # 404 Client Error: Not Found for url: https://api.github.com/repos/airbytehq/airbyte3?per_page=100
+            full_repo_name = message.split("https://api.github.com/repos/")[1].split("?")[0]
+            user_message = f'Repo name: "{full_repo_name}" is unknown, "repository" config option should use existing full repo name <organization>/<repository>'
+        elif "404 Client Error: Not Found for url: https://api.github.com/orgs/" in message:
+            # 404 Client Error: Not Found for url: https://api.github.com/orgs/airbytehqBLA/repos?per_page=100
+            org_name = message.split("https://api.github.com/orgs/")[1].split("/")[0]
+            user_message = f'Organization name: "{org_name}" is unknown, "repository" config option should be updated. Please validate your repository config.'
+        elif "401 Client Error: Unauthorized for url" in message:
+            # 401 Client Error: Unauthorized for url: https://api.github.com/orgs/datarootsio/repos?per_page=100&sort=updated&direction=desc
+            user_message = (
+                "Github credentials have expired or changed, please review your credentials and re-authenticate or renew your access token."
+            )
+        return user_message
+
+    def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
+        config = self._validate_and_transform_config(config)
+        try:
+            authenticator = self._get_authenticator(config)
+            _, repositories = self._get_org_repositories(config=config, authenticator=authenticator)
+            if not repositories:
+                return (
+                    False,
+                    "Some of the provided repositories couldn't be found. Please verify if every entered repository has a valid name and it matches the following format: airbytehq/airbyte airbytehq/another-repo airbytehq/* airbytehq/airbyte.",
+                )
+            return True, None
+
+        except Exception as e:
+            message = repr(e)
+            user_message = self.user_friendly_error_message(message)
+            return False, user_message or message
+
+    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
+        authenticator = self._get_authenticator(config)
+        config = self._validate_and_transform_config(config)
+        try:
+            organizations, repositories = self._get_org_repositories(config=config, authenticator=authenticator)
+        except Exception as e:
+            message = repr(e)
+            user_message = self.user_friendly_error_message(message)
+            if user_message:
+                raise AirbyteTracedException(
+                    internal_message=message, message=user_message, failure_type=FailureType.config_error, exception=e
+                )
+            else:
+                raise e
+
+        if not any((organizations, repositories)):
+            user_message = (
+                "No streams available. Looks like your config for repositories or organizations is not valid."
+                " Please, check your permissions, names of repositories and organizations."
+                " Needed scopes: repo, read:org, read:repo_hook, read:user, read:discussion, workflow."
+            )
+            raise AirbyteTracedException(
+                internal_message="No streams available. Please check permissions",
+                message=user_message,
+                failure_type=FailureType.config_error,
+            )
+
+        # This parameter is deprecated and in future will be used sane default, page_size: 10
+        page_size = config.get("page_size_for_large_streams", constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM)
+        access_token_type, _ = self.get_access_token(config)
+
+        organization_args = {
+            "authenticator": authenticator,
+            "organizations": organizations,
+            "api_url": config.get("api_url"),
+            "access_token_type": access_token_type,
+        }
+        start_date = config.get("start_date")
+        organization_args_with_start_date = {**organization_args, "start_date": start_date}
+
+        repository_args = {
+            "authenticator": authenticator,
+            "api_url": config.get("api_url"),
+            "repositories": repositories,
+            "page_size_for_large_streams": page_size,
+            "access_token_type": access_token_type,
+        }
+        repository_args_with_start_date = {**repository_args, "start_date": start_date}
+
+        default_branches, branches_to_pull = self._get_branches_data(config.get("branch", []), repository_args)
+        pull_requests_stream = PullRequests(**repository_args_with_start_date)
+        projects_stream = Projects(**repository_args_with_start_date)
+        project_columns_stream = ProjectColumns(projects_stream, **repository_args_with_start_date)
+        teams_stream = Teams(**organization_args)
+        team_members_stream = TeamMembers(parent=teams_stream, **repository_args)
+        workflow_runs_stream = WorkflowRuns(**repository_args_with_start_date)
+
+        return [
+            IssueTimelineEvents(**repository_args),
+            Assignees(**repository_args),
+            Branches(**repository_args),
+            Collaborators(**repository_args),
+            Comments(**repository_args_with_start_date),
+            CommitCommentReactions(**repository_args_with_start_date),
+            CommitComments(**repository_args_with_start_date),
+            Commits(**repository_args_with_start_date, branches_to_pull=branches_to_pull, default_branches=default_branches),
+            ContributorActivity(**repository_args),
+            Deployments(**repository_args_with_start_date),
+            Events(**repository_args_with_start_date),
+            IssueCommentReactions(**repository_args_with_start_date),
+            IssueEvents(**repository_args_with_start_date),
+            IssueLabels(**repository_args),
+            IssueMilestones(**repository_args_with_start_date),
+            IssueReactions(**repository_args_with_start_date),
+            Issues(**repository_args_with_start_date),
+            Organizations(**organization_args),
+            ProjectCards(project_columns_stream, **repository_args_with_start_date),
+            project_columns_stream,
+            projects_stream,
+            PullRequestCommentReactions(**repository_args_with_start_date),
+            PullRequestCommits(parent=pull_requests_stream, **repository_args),
+            PullRequestStats(**repository_args_with_start_date),
+            ProjectsV2(**repository_args_with_start_date),
+            pull_requests_stream,
+            Releases(**repository_args_with_start_date),
+            Repositories(**organization_args_with_start_date),
+            ReviewComments(**repository_args_with_start_date),
+            Reviews(**repository_args_with_start_date),
+            Stargazers(**repository_args_with_start_date),
+            Tags(**repository_args),
+            teams_stream,
+            team_members_stream,
+            Users(**organization_args),
+            Workflows(**repository_args_with_start_date),
+            workflow_runs_stream,
+            WorkflowJobs(parent=workflow_runs_stream, **repository_args_with_start_date),
+            TeamMemberships(parent=team_members_stream, **repository_args),
+        ]

source_github/spec.json

@@ -0,0 +1,179 @@
+{
+  "documentationUrl": "https://docs.airbyte.com/integrations/sources/github",
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "GitHub Source Spec",
+    "type": "object",
+    "required": ["credentials", "repositories"],
+    "additionalProperties": true,
+    "properties": {
+      "credentials": {
+        "title": "Authentication",
+        "description": "Choose how to authenticate to GitHub",
+        "type": "object",
+        "order": 0,
+        "group": "auth",
+        "oneOf": [
+          {
+            "type": "object",
+            "title": "OAuth",
+            "required": ["access_token"],
+            "properties": {
+              "option_title": {
+                "type": "string",
+                "const": "OAuth Credentials",
+                "order": 0
+              },
+              "access_token": {
+                "type": "string",
+                "title": "Access Token",
+                "description": "OAuth access token",
+                "airbyte_secret": true
+              },
+              "client_id": {
+                "type": "string",
+                "title": "Client Id",
+                "description": "OAuth Client Id",
+                "airbyte_secret": true
+              },
+              "client_secret": {
+                "type": "string",
+                "title": "Client ssecret",
+                "description": "OAuth Client secret",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "type": "object",
+            "title": "Personal Access Token",
+            "required": ["personal_access_token"],
+            "properties": {
+              "option_title": {
+                "type": "string",
+                "const": "PAT Credentials",
+                "order": 0
+              },
+              "personal_access_token": {
+                "type": "string",
+                "title": "Personal Access Tokens",
+                "description": "Log into GitHub and then generate a <a href=\"https://github.com/settings/tokens\">personal access token</a>. To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with \",\"",
+                "airbyte_secret": true
+              }
+            }
+          }
+        ]
+      },
+      "repository": {
+        "type": "string",
+        "examples": [
+          "airbytehq/airbyte airbytehq/another-repo",
+          "airbytehq/*",
+          "airbytehq/airbyte"
+        ],
+        "title": "GitHub Repositories",
+        "description": "(DEPRCATED) Space-delimited list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get all repositories from organization and `airbytehq/airbyte airbytehq/another-repo` for multiple repositories.",
+        "airbyte_hidden": true,
+        "pattern": "^([\\w.-]+/(\\*|[\\w.-]+(?<!\\.git))\\s+)*[\\w.-]+/(\\*|[\\w.-]+(?<!\\.git))$",
+        "pattern_descriptor": "org/repo org/another-repo org/*"
+      },
+      "repositories": {
+        "type": "array",
+        "items": {
+          "type": "string",
+          "pattern": "^([\\w.-]+/(\\*|[\\w.-]+(?<!\\.git))\\s+)*[\\w.-]+/(\\*|[\\w.-]+(?<!\\.git))$"
+        },
+        "minItems": 1,
+        "examples": [
+          "airbytehq/airbyte airbytehq/another-repo",
+          "airbytehq/*",
+          "airbytehq/airbyte"
+        ],
+        "title": "GitHub Repositories",
+        "description": "List of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get all repositories from organization and `airbytehq/airbyte airbytehq/another-repo` for multiple repositories.",
+        "order": 1,
+        "pattern_descriptor": "org/repo org/another-repo org/*"
+      },
+      "start_date": {
+        "type": "string",
+        "title": "Start date",
+        "description": "The date from which you'd like to replicate data from GitHub in the format YYYY-MM-DDT00:00:00Z. If the date is not set, all data will be replicated.  For the streams which support this configuration, only data generated on or after the start date will be replicated. This field doesn't apply to all streams, see the <a href=\"https://docs.airbyte.com/integrations/sources/github\">docs</a> for more info",
+        "examples": ["2021-03-01T00:00:00Z"],
+        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
+        "pattern_descriptor": "YYYY-MM-DDTHH:mm:ssZ",
+        "order": 2,
+        "format": "date-time"
+      },
+      "api_url": {
+        "type": "string",
+        "examples": ["https://github.com", "https://github.company.org"],
+        "title": "API URL",
+        "default": "https://api.github.com/",
+        "description": "Please enter your basic URL from self-hosted GitHub instance or leave it empty to use GitHub.",
+        "order": 3
+      },
+      "branch": {
+        "type": "string",
+        "title": "Branch",
+        "examples": ["airbytehq/airbyte/master airbytehq/airbyte/my-branch"],
+        "description": "(DEPRCATED) Space-delimited list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.",
+        "airbyte_hidden": true,
+        "pattern_descriptor": "org/repo/branch1 org/repo/branch2"
+      },
+      "branches": {
+        "type": "array",
+        "items": {
+          "type": "string"
+        },
+        "title": "Branches",
+        "examples": ["airbytehq/airbyte/master airbytehq/airbyte/my-branch"],
+        "description": "List of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.",
+        "order": 4,
+        "pattern_descriptor": "org/repo/branch1 org/repo/branch2"
+      }
+    }
+  },
+  "advanced_auth": {
+    "auth_flow_type": "oauth2.0",
+    "predicate_key": ["credentials", "option_title"],
+    "predicate_value": "OAuth Credentials",
+    "oauth_config_specification": {
+      "complete_oauth_output_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "access_token": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "access_token"]
+          }
+        }
+      },
+      "complete_oauth_server_input_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "client_id": {
+            "type": "string"
+          },
+          "client_secret": {
+            "type": "string"
+          }
+        }
+      },
+      "complete_oauth_server_output_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "client_id": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_id"]
+          },
+          "client_secret": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_secret"]
+          }
+        }
+      }
+    }
+  }
+}