github-org-manager 0.5.4__tar.gz → 0.5.6__tar.gz

{github_org_manager-0.5.4 → github_org_manager-0.5.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: github-org-manager
-Version: 0.5.4
+Version: 0.5.6
 Summary: Manage a GitHub Organization, its teams, repository permissions, and more
 Home-page: https://github.com/OpenRailAssociation/github-org-manager
 License: Apache-2.0

{github_org_manager-0.5.4 → github_org_manager-0.5.6}/gh_org_mgr/_gh_org.py

@@ -40,6 +40,7 @@ class GHorg:  # pylint: disable=too-many-instance-attributes, too-many-lines
     configured_teams: dict[str, dict | None] = field(default_factory=dict)
     newly_added_users: list[NamedUser] = field(default_factory=list)
     current_repos_teams: dict[Repository, dict[Team, str]] = field(default_factory=dict)
+    graphql_repos_collaborators: dict[str, list[dict]] = field(default_factory=dict)
     current_repos_collaborators: dict[Repository, dict[str, str]] = field(default_factory=dict)
     configured_repos_collaborators: dict[str, dict[str, str]] = field(default_factory=dict)
     archived_repos: list[Repository] = field(default_factory=list)
@@ -77,11 +78,15 @@ class GHorg:  # pylint: disable=too-many-instance-attributes, too-many-lines
 
         # Decide how to login. If app set, prefer this
         if self.gh_app_id and self.gh_app_private_key:
-            logging.debug("Logged in via app %s", self.gh_app_id)
+            logging.debug("Logging in via app %s", self.gh_app_id)
             auth = Auth.AppAuth(app_id=self.gh_app_id, private_key=self.gh_app_private_key)
             app = GithubIntegration(auth=auth)
-            installation = app.get_installations()[0]
+            installation = app.get_org_installation(org=orgname)
             self.gh = installation.get_github_for_installation()
+            logging.debug("Logged in via app installation %s", installation.id)
+
+            logging.debug("Getting access token for installation %s", installation.id)
+            self.gh_token = app.get_access_token(installation_id=installation.id).token
         elif self.gh_token:
             logging.debug("Logging in as user with PAT")
             self.gh = Github(auth=Auth.Token(self.gh_token))
@@ -90,6 +95,7 @@ class GHorg:  # pylint: disable=too-many-instance-attributes, too-many-lines
             logging.error("No GitHub token or App ID+private key provided")
             sys.exit(1)
 
+        logging.debug("Gathering data from organization '%s'", orgname)
         self.org = self.gh.get_organization(orgname)
         logging.debug("Gathered data from organization '%s' (%s)", self.org.login, self.org.name)
 
@@ -110,7 +116,7 @@ class GHorg:  # pylint: disable=too-many-instance-attributes, too-many-lines
             half2 = len(string) - half1
             return string[:half1] + "*" * (half2)
 
-        sensible_keys = ["gh_token"]
+        sensible_keys = ["gh_token", "gh_app_private_key"]
         for key in sensible_keys:
             if value := dictionary.get(key, ""):
                 dictionary[key] = censor_half_string(value)
@@ -702,7 +708,25 @@ class GHorg:  # pylint: disable=too-many-instance-attributes, too-many-lines
                 continue
 
             # Convert team name to Team object
-            team = self.org.get_team_by_slug(self._sluggify_teamname(team_name))
+            try:
+                team = self.org.get_team_by_slug(self._sluggify_teamname(team_name))
+            # Team not found, probably because a new team should be created, but it's a dry-run
+            except UnknownObjectException:
+                logging.debug(
+                    "Team %s not found, probably because it should be created but it's a dry-run",
+                    team_name,
+                )
+                # Initialise a new Team() object with the name, manually
+                team = Team(
+                    requester=None,  # type: ignore
+                    headers={},  # No headers required
+                    attributes={
+                        "id": 0,
+                        "name": team_name,
+                        "slug": self._sluggify_teamname(team_name),
+                    },
+                    completed=True,  # Mark as fully initialized
+                )
 
             # Get configured repo permissions
             for repo, perm in team_attrs.get("repos", {}).items():
@@ -941,59 +965,220 @@ class GHorg:  # pylint: disable=too-many-instance-attributes, too-many-lines
 
         return permission
 
-    def _fetch_collaborators_of_repo(self, repo: Repository):
-        """Get all collaborators (individuals) of a GitHub repo with their
-        permissions using the GraphQL API"""
-        # TODO: Consider doing this for all repositories at once, but calculate
-        # costs beforehand
+    def _fetch_collaborators_of_all_organization_repos(self) -> None:
+        """Get all collaborators (individuals) of all repos of a GitHub
+        organization with their permissions using the GraphQL API"""
+
         graphql_query = """
-            query($owner: String!, $name: String!, $cursor: String) {
-                repository(owner: $owner, name: $name) {
-                    collaborators(first: 100, after: $cursor) {
+            query($owner: String!, $cursor: String) {
+                organization(login: $owner) {
+                    repositories(first: 100, after: $cursor) {
                         edges {
                             node {
-                                login
+                                name
+                                collaborators(first: 100) {
+                                    edges {
+                                        node {
+                                            login
+                                        }
+                                        permission
+                                    }
+                                    pageInfo {
+                                        endCursor
+                                        hasNextPage
+                                    }
+                                }
                             }
-                            permission
                         }
                         pageInfo {
                             endCursor
                             hasNextPage
                         }
+                    }
                 }
             }
-        }
         """
 
-        # Initial query parameters
-        variables = {"owner": self.org.login, "name": repo.name, "cursor": None}
+        # Initial query parameters for org-level request
+        variables = {"owner": self.org.login, "cursor": None}
+
+        # dict in which we store repos for which there are more than 100
+        # collaborators, and their respective end cursors
+        next_page_cursors_for_repos: dict[str, str] = {}
+
+        more_repos_in_org = True
+        while more_repos_in_org:
+            logging.debug("Requesting collaborators for %s", self.org.login)
+            org_result = run_graphql_query(graphql_query, variables, self.gh_token)
+            more_repos_in_org, variables["cursor"] = self._extract_data_from_graphql_response(
+                graphql_response=org_result, next_page_cursors_for_repos=next_page_cursors_for_repos
+            )
+
+        # If there are more than 100 collaborators in a repo, we need to fetch
+        # rest via individual GraphQL queries
+        if next_page_cursors_for_repos:
+            logging.debug(
+                "Not all collaborators of all repos have been fetched. Missing data: %s",
+                next_page_cursors_for_repos,
+            )
+            for repo_name, end_cursor in next_page_cursors_for_repos.items():
+                more_collaborators_in_repo = True
+                while more_collaborators_in_repo:
+                    logging.debug("Requesting additional collaborators for repo %s", repo_name)
+                    # Initial query parameters for repo-level request
+                    repo_variables = {
+                        "owner": self.org.login,
+                        "repo": repo_name,
+                        "cursor": end_cursor,
+                    }
+                    repo_query = """
+                        query($owner: String!, $repo: String!, $cursor: String) {
+                            repository(owner: $owner, name: $repo) {
+                                collaborators(first: 100, after: $cursor) {
+                                    edges {
+                                        node {
+                                            login
+                                        }
+                                        permission
+                                    }
+                                    pageInfo {
+                                        endCursor
+                                        hasNextPage
+                                    }
+                                }
+                            }
+                        }
+                    """
+                    repo_result = run_graphql_query(repo_query, repo_variables, self.gh_token)
+                    more_collaborators_in_repo, end_cursor = (
+                        self._extract_data_from_graphql_response(
+                            graphql_response=repo_result,
+                            next_page_cursors_for_repos=next_page_cursors_for_repos,
+                            single_repo_name=repo_name,
+                        )
+                    )
 
-        collaborators = []
-        has_next_page = True
+        # All collaborators from all repos have been fetched, now populate the
+        # actual dictionary
+        self._populate_current_repos_collaborators()
+
+    def _extract_data_from_graphql_response(
+        self,
+        graphql_response: dict,
+        next_page_cursors_for_repos: dict[str, str],
+        single_repo_name: str = "",
+    ) -> tuple[bool, str]:
+        """
+        Extracts collaborator data from a GraphQL response for either an
+        organization or a single repository.
+
+        Args:
+            graphql_response (dict): The GraphQL response containing the data.
+            next_page_cursors_for_repos (dict[str, str]): A dictionary to store
+            the next page cursors for repositories.
+            single_repo_name (str, optional): The name of a single repository to
+            extract data for. Defaults to "".
 
-        while has_next_page:
-            logging.debug("Requesting collaborators for %s", repo.name)
-            result = run_graphql_query(graphql_query, variables, self.gh_token)
+        Returns:
+            tuple[bool, str]: A tuple containing a boolean indicating if there
+            is a next page and a string for the cursor.
+                - For organization level extraction:
+                    - bool: Indicates if there is a next page of repositories.
+                    - str: The cursor for the next page of repositories.
+                - For single repository extraction:
+                    - bool: Indicates if there is a next page of collaborators.
+                    - str: The cursor for the next page of collaborators.
+
+        Raises:
+            SystemExit: If a repository name is not found in the GraphQL
+            response at the organization level.
+
+        This method processes the GraphQL response to extract information about
+        repositories and their collaborators. It handles pagination by
+        identifying if there are more pages of repositories or collaborators to
+        be fetched.
+        """
+        if not single_repo_name and "organization" in graphql_response["data"]:
+            logging.debug("Extracting collaborators for organization from GraphQL response")
+
+            # Initialise returns
+            org_has_next_page = graphql_response["data"]["organization"]["repositories"][
+                "pageInfo"
+            ]["hasNextPage"]
+            org_cursor = graphql_response["data"]["organization"]["repositories"]["pageInfo"][
+                "endCursor"
+            ]
+
+            for repo_edges in graphql_response["data"]["organization"]["repositories"]["edges"]:
+                try:
+                    repo_name: str = repo_edges["node"]["name"]
+                    logging.debug(
+                        "Extracting collaborators for %s from GraphQL response", repo_name
+                    )
+                except KeyError:
+                    logging.error(
+                        "Did not find a repo name in the GraphQL response "
+                        "(organization level) which seems to hint to a bug: %s",
+                        repo_edges,
+                    )
+                    sys.exit(1)
+
+                # fill in collaborators of repo
+                try:
+                    repo_collaborators = repo_edges["node"]["collaborators"]["edges"]
+                    self.graphql_repos_collaborators[repo_name] = repo_collaborators
+                except (TypeError, KeyError):
+                    logging.debug("Repo %s does not seem to have any collaborators", repo_name)
+
+                # Find out if there are more than 100 collaborators in the
+                # GraphQL response for this repo
+                if repo_edges["node"]["collaborators"]["pageInfo"]["hasNextPage"]:
+                    next_page_cursors_for_repos[repo_name] = repo_edges["node"]["collaborators"][
+                        "pageInfo"
+                    ]["endCursor"]
+
+            return org_has_next_page, org_cursor
+
+        if single_repo_name and "repository" in graphql_response["data"]:
+            logging.debug(
+                "Extracting collaborators for repository %s from GraphQL response", single_repo_name
+            )
+
+            # Initialise returns
+            repo_has_next_page = graphql_response["data"]["repository"]["collaborators"][
+                "pageInfo"
+            ]["hasNextPage"]
+            repo_cursor = graphql_response["data"]["repository"]["collaborators"]["pageInfo"][
+                "endCursor"
+            ]
+
+            # fill in collaborators of repo
             try:
-                collaborators.extend(result["data"]["repository"]["collaborators"]["edges"])
-                has_next_page = result["data"]["repository"]["collaborators"]["pageInfo"][
-                    "hasNextPage"
-                ]
-                variables["cursor"] = result["data"]["repository"]["collaborators"]["pageInfo"][
-                    "endCursor"
+                repo_collaborators = graphql_response["data"]["repository"]["collaborators"][
+                    "edges"
                 ]
+                self.graphql_repos_collaborators[single_repo_name].extend(repo_collaborators)
             except (TypeError, KeyError):
-                logging.debug("Repo %s does not seem to have any collaborators", repo.name)
-                continue
-
-        # Extract relevant data
-        for collaborator in collaborators:
-            login: str = collaborator["node"]["login"]
-            # Skip entry if collaborator is org owner, which is "admin" anyway
-            if login.lower() in [user.login.lower() for user in self.current_org_owners]:
-                continue
-            permission = self._convert_graphql_perm_to_rest(collaborator["permission"])
-            self.current_repos_collaborators[repo][login.lower()] = permission
+                logging.debug("Repo %s does not seem to have any collaborators", single_repo_name)
+
+            return repo_has_next_page, repo_cursor
+
+        logging.warning("No relevant data found in GraphQL response")
+        logging.debug("GraphQL response: %s", graphql_response)
+        return False, ""
+
+    def _populate_current_repos_collaborators(self) -> None:
+        """Populate self.current_repos_collaborators with data from repo_collaborators"""
+        for repo, collaborators in self.current_repos_collaborators.items():
+            if repo.name in self.graphql_repos_collaborators:
+                # Extract each collaborator from the GraphQL response for this repo
+                for collaborator in self.graphql_repos_collaborators[repo.name]:
+                    login: str = collaborator["node"]["login"]
+                    # Skip entry if collaborator is org owner, which is "admin" anyway
+                    if login.lower() in [user.login.lower() for user in self.current_org_owners]:
+                        continue
+                    permission = self._convert_graphql_perm_to_rest(collaborator["permission"])
+                    collaborators[login.lower()] = permission
 
     def _get_current_repos_and_user_perms(self):
         """Get all repos, their current collaborators and their permissions"""
@@ -1001,9 +1186,7 @@ class GHorg:  # pylint: disable=too-many-instance-attributes, too-many-lines
         for repo in self.current_repos_teams:
             self.current_repos_collaborators[repo] = {}
 
-        for repo in self.current_repos_collaborators:
-            # Get users for this repo
-            self._fetch_collaborators_of_repo(repo)
+        self._fetch_collaborators_of_all_organization_repos()
 
     def _get_default_repository_permission(self):
         """Get the default repository permission for all users. Convert to

{github_org_manager-0.5.4 → github_org_manager-0.5.6}/pyproject.toml

@@ -4,7 +4,7 @@
 
 [tool.poetry]
 name = "github-org-manager"
-version = "0.5.4"
+version = "0.5.6"
 description = "Manage a GitHub Organization, its teams, repository permissions, and more"
 authors = ["Max Mehl <max.mehl@deutschebahn.com>"]
 readme = "README.md"