qontract-reconcile 0.10.2.dev195__tar.gz → 0.10.2.dev196__tar.gz

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev195
+Version: 0.10.2.dev196
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/pyproject.toml

@@ -427,7 +427,6 @@ module = [
     "reconcile.test.utils.test_expiration",
     "reconcile.test.utils.test_external_resource_spec",
     "reconcile.test.utils.test_external_resources",
-    "reconcile.test.utils.test_github_api",
     "reconcile.test.utils.test_gpg",
     "reconcile.test.utils.test_gql",
     "reconcile.test.utils.test_grouping",

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/dashdotdb_dora.py

@@ -402,7 +402,7 @@ class DashdotdbDORA(DashdotdbBase):
             if not saas_file_yaml:
                 LOG.info(f"failed to fetch saas file {saastarget.path} at {sha}")
                 return (None, None)
-            saas_file = yaml.safe_load(saas_file_yaml.decode())
+            saas_file = yaml.safe_load(saas_file_yaml)
         except Exception as e:
             LOG.info(f"failed to decode saas file {saastarget.path} with error: {e}")
             return (None, None)

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/github_api.py

@@ -1,13 +1,21 @@
+import base64
 import os
 from pathlib import Path
 from types import TracebackType
 from urllib.parse import urlparse
 
 from github import Commit, Github, GithubException, UnknownObjectException
+from github.Repository import Repository
 from sretoolbox.utils import retry
 
 GH_BASE_URL = os.environ.get("GITHUB_API", "https://api.github.com")
 
+MAX_FILE_CONTENT_SIZE = 1024**2  # 1MB
+
+
+class UnsupportedDirectoryError(Exception):
+    pass
+
 
 class GithubRepositoryApi:
     """
@@ -66,16 +74,36 @@ class GithubRepositoryApi:
             tree_items.append(tree_item)
         return tree_items
 
+    @staticmethod
+    def get_raw_file(
+        repo: Repository,
+        path: str,
+        ref: str,
+    ) -> bytes:
+        content = repo.get_contents(path=path, ref=ref)
+        if isinstance(content, list):
+            raise UnsupportedDirectoryError(
+                f"Path {path} of ref {ref} in repo {repo.full_name} is a directory!"
+            )
+        if content.size < MAX_FILE_CONTENT_SIZE:
+            return content.decoded_content
+        blob = repo.get_git_blob(content.sha)
+        return base64.b64decode(blob.content)
+
     @retry()
-    def get_file(self, path: str, ref: str = "master") -> bytes | None:
+    def get_file(
+        self,
+        path: str,
+        ref: str = "master",
+    ) -> bytes | None:
         try:
-            content = self._repo.get_contents(path=path, ref=ref)
-            if isinstance(content, list):
-                # TODO: we should probably raise an exception here
-                # or handle this properly
-                # -> for now staying backwards compatible
-                return None
-            return content.decoded_content
+            return self.get_raw_file(
+                repo=self._repo,
+                path=path,
+                ref=ref,
+            )
+        except UnsupportedDirectoryError:
+            return None
         except GithubException as e:
             # handling a bug in the upstream GH library
             # https://github.com/PyGithub/PyGithub/issues/3179

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/gitlab_api.py

@@ -41,7 +41,6 @@ from gitlab.v4.objects import (
     GroupMember,
     PersonalAccessToken,
     Project,
-    ProjectFile,
     ProjectHook,
     ProjectIssue,
     ProjectIssueManager,
@@ -666,25 +665,19 @@ class GitLabApi:
         *,
         ref: str = "master",
         recursive: bool = False,
-        project: Project | None = None,
-        path: str = "",
     ) -> list[dict]:
         """
         Get a list of repository files and directories in a project.
 
         :param ref: The name of a repository branch or tag or, if not given, the default branch.
         :param recursive: Boolean value used to get a recursive tree. Default is false.
-        :param project: The project to get the tree from, if None, use the current project
-        :param path: The path inside the repository. Used to get content of subdirectories.
 
         :return: list of tree objects
         """
-        target_project = self.project if project is None else project
         return cast(
             list[dict],
-            target_project.repository_tree(
+            self.project.repository_tree(
                 ref=ref,
-                path=path,
                 recursive=recursive,
                 pagination="keyset",
                 per_page=MAX_PER_PAGE,
@@ -692,13 +685,36 @@ class GitLabApi:
             ),
         )
 
-    def get_file(self, path: str, ref: str = "master") -> ProjectFile | None:
+    @staticmethod
+    def get_raw_file(
+        project: Project,
+        path: str,
+        ref: str,
+    ) -> bytes:
+        file_path = path.lstrip("/")
+        return project.files.raw(
+            file_path=file_path,
+            ref=ref,
+        )
+
+    def get_file(
+        self,
+        path: str,
+        ref: str = "master",
+    ) -> bytes | None:
         """
-        Wrapper around Gitlab.files.get() with exception handling.
+        Get the raw content of a file in a project.
+
+        :param path: The path to the file in the repository.
+        :param ref: The name of branch, tag or commit.
+        :return: The content of the file as bytes, or None if the file does not exist.
         """
         try:
-            path = path.lstrip("/")
-            return self.project.files.get(file_path=path, ref=ref)
+            return self.get_raw_file(
+                project=self.project,
+                path=path,
+                ref=ref,
+            )
         except GitlabGetError:
             return None
 

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/jinja2/utils.py

@@ -12,6 +12,7 @@ from reconcile.checkpoint import url_makes_sense
 from reconcile.github_users import init_github
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.helpers import flatten
 from reconcile.utils.jinja2.extensions import B64EncodeExtension, RaiseErrorExtension
 from reconcile.utils.jinja2.filters import (
@@ -116,10 +117,12 @@ def lookup_github_file_content(
         )
 
     gh = init_github()
-    content = gh.get_repo(repo).get_contents(path, ref)
-    if isinstance(content, list):
-        raise Exception(f"multiple files found for {repo}/{path}/{ref}")
-    return content.decoded_content.decode("utf-8")
+    content = GithubRepositoryApi.get_raw_file(
+        repo=gh.get_repo(repo),
+        path=path,
+        ref=ref,
+    )
+    return content.decode("utf-8")
 
 
 def lookup_graphql_query_results(query: str, **kwargs: dict[str, Any]) -> list[Any]:

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/mr/aws_access.py

@@ -37,10 +37,12 @@ class CreateDeleteAwsAccessKey(MergeRequestBase):
 
     def process(self, gitlab_cli: GitLabApi) -> None:
         # add key to deleteKeys list to be picked up by aws-iam-keys
-        raw_file = gitlab_cli.project.files.get(
-            file_path=self.path, ref=gitlab_cli.main_branch
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=self.path,
+            ref=gitlab_cli.main_branch,
         )
-        content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+        content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
 
         content.setdefault("deleteKeys", [])
         content["deleteKeys"].append(self.key)

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/mr/clusters_updates.py

@@ -37,10 +37,12 @@ class CreateClustersUpdates(MergeRequestBase):
                 continue
 
             cluster_path = cluster_updates.pop("path")
-            raw_file = gitlab_cli.project.files.get(
-                file_path=cluster_path, ref=gitlab_cli.main_branch
+            raw_file = gitlab_cli.get_raw_file(
+                project=gitlab_cli.project,
+                path=cluster_path,
+                ref=gitlab_cli.main_branch,
             )
-            content = yaml.load(raw_file.decode())
+            content = yaml.load(raw_file)
             if "spec" not in content:
                 self.cancel("Spec missing. Nothing to do.")
 

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/mr/ocm_update_recommended_version.py

@@ -28,10 +28,12 @@ class CreateOCMUpdateRecommendedVersion(MergeRequestBase):
         return f"ocm update recommended version for {self.ocm_name}"
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        raw_file = gitlab_cli.project.files.get(
-            file_path=self.path, ref=gitlab_cli.main_branch
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=self.path,
+            ref=gitlab_cli.main_branch,
         )
-        content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+        content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
 
         content["recommendedVersions"] = self.recommended_versions
 

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/mr/ocm_upgrade_scheduler_org_updates.py

@@ -28,10 +28,12 @@ class CreateOCMUpgradeSchedulerOrgUpdates(MergeRequestBase):
         ocm_path = self.updates_info["path"]
         ocm_name = self.updates_info["name"]
 
-        raw_file = gitlab_cli.project.files.get(
-            file_path=ocm_path, ref=gitlab_cli.main_branch
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=ocm_path,
+            ref=gitlab_cli.main_branch,
         )
-        content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+        content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
         upgrade_policy_clusters = content["upgradePolicyClusters"]
 
         for update in self.updates_info["updates"]:

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/mr/promote_qontract.py

@@ -37,10 +37,12 @@ class PromoteQontractSchemas(MergeRequestBase):
         return f"promote qontract-schemas to version {self.version}"
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        raw_file = gitlab_cli.project.files.get(
-            file_path=self.path, ref=gitlab_cli.main_branch
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=self.path,
+            ref=gitlab_cli.main_branch,
         )
-        content = raw_file.decode().decode("utf-8")
+        content = raw_file.decode("utf-8")
         lines = content.splitlines()
         for index, text in enumerate(lines):
             if text.startswith("export SCHEMAS_IMAGE_TAG="):
@@ -121,9 +123,11 @@ class PromoteQontractReconcileCommercial(MergeRequestBase):
         search_text: str,
         replace_text: str,
     ) -> None:
-        raw_file = gitlab_cli.project.files.get(
-            file_path=path, ref=gitlab_cli.main_branch
-        ).decode()
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=path,
+            ref=gitlab_cli.main_branch,
+        )
         match method:
             case "line_search":
                 new_content = self._process_by_line_search(

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/mr/update_access_report_base.py

@@ -100,10 +100,12 @@ class UpdateAccessReportBase(MergeRequestBase):
         return new_workbook_md
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        workbook_file = gitlab_cli.project.files.get(
-            file_path=self._workbook_file_name, ref=self.branch
+        workbook_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=self._workbook_file_name,
+            ref=self.branch,
         )
-        workbook_md = self._update_workbook(workbook_file.decode().decode("utf-8"))
+        workbook_md = self._update_workbook(workbook_file.decode("utf-8"))
 
         if not self._dry_run:
             logging.info(

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/mr/user_maintenance.py

@@ -56,8 +56,12 @@ class CreateDeleteUserAppInterface(MergeRequestBase):
                     branch_name=self.branch, file_path=path, commit_message=self.title
                 )
             elif path_type == PathTypes.GABI:
-                raw_file = gitlab_cli.project.files.get(file_path=path, ref=self.branch)
-                content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+                raw_file = gitlab_cli.get_raw_file(
+                    project=gitlab_cli.project,
+                    path=path,
+                    ref=self.branch,
+                )
+                content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
                 for gabi_user in content["users"][:]:
                     if self.username in gabi_user["$ref"]:
                         content["users"].remove(gabi_user)
@@ -70,8 +74,12 @@ class CreateDeleteUserAppInterface(MergeRequestBase):
                     content=new_content,
                 )
             elif path_type == PathTypes.AWS_ACCOUNTS:
-                raw_file = gitlab_cli.project.files.get(file_path=path, ref=self.branch)
-                content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+                raw_file = gitlab_cli.get_raw_file(
+                    project=gitlab_cli.project,
+                    path=path,
+                    ref=self.branch,
+                )
+                content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
                 for reset_record in content["resetPasswords"]:
                     if self.username in reset_record["user"]["$ref"]:
                         content["resetPasswords"].remove(reset_record)
@@ -84,8 +92,12 @@ class CreateDeleteUserAppInterface(MergeRequestBase):
                             content=new_content,
                         )
             elif path_type == PathTypes.SCHEDULE:
-                raw_file = gitlab_cli.project.files.get(file_path=path, ref=self.branch)
-                content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+                raw_file = gitlab_cli.get_raw_file(
+                    project=gitlab_cli.project,
+                    path=path,
+                    ref=self.branch,
+                )
+                content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
                 delete_indexes: list[tuple[int, int]] = []
                 for schedule_index, schedule_record in enumerate(content["schedule"]):
                     for user_index, user in enumerate(schedule_record["users"]):
@@ -124,10 +136,12 @@ class CreateDeleteUserInfra(MergeRequestBase):
         return "delete user(s)"
 
     def process(self, gitlab_cli):
-        raw_file = gitlab_cli.project.files.get(
-            file_path=self.PLAYBOOK, ref=self.branch
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=self.PLAYBOOK,
+            ref=self.branch,
         )
-        content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+        content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
 
         new_list = []
         for user in content[0]["vars"]["users"]:

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/repo_owners.py

@@ -131,7 +131,7 @@ class RepoOwners:
                 _LOG.warning(f"{self._git_cli!s}:{owner_file['path']} not found")
                 continue
             try:
-                owners = yaml.safe_load(raw_owners.decode())
+                owners = yaml.safe_load(raw_owners)
             except yaml.parser.ParserError:
                 owners = None
             if owners is None:
@@ -184,7 +184,7 @@ class RepoOwners:
         if raw_aliases is None:
             return {}
 
-        aliases = yaml.safe_load(raw_aliases.decode())
+        aliases = yaml.safe_load(raw_aliases)
         if aliases is None:
             return {}
 

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/saasherder/saasherder.py

@@ -27,7 +27,6 @@ from github import (
     GithubException,
 )
 from github.ContentFile import ContentFile
-from github.Repository import Repository
 from gitlab.exceptions import GitlabError
 from requests import exceptions as rqexc
 from sretoolbox.container import Image
@@ -39,6 +38,7 @@ from sretoolbox.utils import (
 from reconcile.github_org import get_default_config
 from reconcile.status import RunningState
 from reconcile.utils import helm
+from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.jenkins_api import JenkinsApi
 from reconcile.utils.jjb_client import JJB
@@ -715,24 +715,6 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     def _collect_image_patterns(self) -> set[str]:
         return {p for sf in self.saas_files for p in sf.image_patterns}
 
-    @staticmethod
-    def _get_file_contents_github(repo: Repository, path: str, commit_sha: str) -> str:
-        f = repo.get_contents(path, commit_sha)
-        if isinstance(f, list):
-            raise Exception(f"Path {path} and sha {commit_sha} is a directory!")
-
-        if f.size < 1024**2:  # 1 MB
-            return f.decoded_content.decode("utf8")
-
-        tree = repo.get_git_tree(commit_sha, recursive="/" in path).tree
-        for x in tree:
-            if x.path != path.lstrip("/"):
-                continue
-            blob = repo.get_git_blob(x.sha)
-            return base64.b64decode(blob.content).decode("utf8")
-
-        return ""
-
     @retry(max_attempts=20)
     def get_archive_info(
         self,
@@ -768,13 +750,20 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         match repo_info.platform:
             case "github":
                 repo = github.get_repo(repo_info.name)
-                content = self._get_file_contents_github(repo, path, commit_sha)
+                content = GithubRepositoryApi.get_raw_file(
+                    repo=repo,
+                    path=path,
+                    ref=commit_sha,
+                )
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
                 project = self.gitlab.get_project(url)
-                f = project.files.get(file_path=path.lstrip("/"), ref=commit_sha)
-                content = f.decode()
+                content = self.gitlab.get_raw_file(
+                    project=project,
+                    path=path,
+                    ref=commit_sha,
+                )
             case _:
                 raise Exception(f"Only GitHub and GitLab are supported: {url}")
 
@@ -795,10 +784,12 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                     raise Exception(f"Path {path} and sha {commit_sha} is a file!")
                 for f in directory:
                     file_path = os.path.join(path, f.name)
-                    file_contents_decoded = self._get_file_contents_github(
-                        repo, file_path, commit_sha
+                    raw_file = GithubRepositoryApi.get_raw_file(
+                        repo=repo,
+                        path=file_path,
+                        ref=commit_sha,
                     )
-                    result_resources = yaml.safe_load_all(file_contents_decoded)
+                    result_resources = yaml.safe_load_all(raw_file)
                     resources.extend(result_resources)
             case "gitlab":
                 if not self.gitlab:

{qontract_reconcile-0.10.2.dev195 → qontract_reconcile-0.10.2.dev196}/reconcile/utils/vcs.py

@@ -266,11 +266,12 @@ class VCS:
                 if not file_path.startswith("data")
                 else file_path
             )
-        return (
-            self._app_interface_api.project.files.get(file_path=file_path, ref=ref)
-            .decode()
-            .decode("utf-8")
+        file = self._app_interface_api.get_raw_file(
+            project=self._app_interface_api.project,
+            path=file_path,
+            ref=ref,
         )
+        return file.decode("utf-8")
 
     def get_open_app_interface_merge_requests(self) -> list[ProjectMergeRequest]:
         return self._app_interface_api.get_merge_requests(state=MRState.OPENED)