qontract-reconcile 0.10.2.dev195__py3-none-any.whl → 0.10.2.dev196__py3-none-any.whl

{qontract_reconcile-0.10.2.dev195.dist-info → qontract_reconcile-0.10.2.dev196.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev195
+Version: 0.10.2.dev196
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev195.dist-info → qontract_reconcile-0.10.2.dev196.dist-info}/RECORD

@@ -14,7 +14,7 @@ reconcile/closedbox_endpoint_monitoring_base.py,sha256=al7m8EgnnYx90rY1REryW3byN
 reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
 reconcile/dashdotdb_base.py,sha256=83ZWIf5JJk3P_D69y2TmXRcQr6ELJGlv10OM0h7fJVs,4767
 reconcile/dashdotdb_cso.py,sha256=QRK0YfIqO4rehs8btD3l_GXIO2ZIycTQEKEthBdB0xA,3639
-reconcile/dashdotdb_dora.py,sha256=juZvr7R45f3TLbIZy290Oa1OtsSnqX-Ez71ppYAUjXw,17780
+reconcile/dashdotdb_dora.py,sha256=UI83R8VsrX3vd2ocBakQDKPNE5Ym2a8pnAGUhfkDeR0,17771
 reconcile/dashdotdb_dvo.py,sha256=lCkZ0iby6HrNQb-3kYb6xrt8wCjVUZYxKzz9SiStfHU,8946
 reconcile/dashdotdb_slo.py,sha256=TvKdMOtUZcZP9QydcUJMKh0zURHgOMN_RTpQpCkD1Z8,3960
 reconcile/database_access_manager.py,sha256=Z3aAmw2LsmMIIor-bOGzziVZdVNC82Gmw8oHBUAFf-8,25577
@@ -618,8 +618,8 @@ reconcile/utils/external_resource_spec.py,sha256=qeupz4t4trd2uPjlHjf_AFA9Y-EKrMn
 reconcile/utils/external_resources.py,sha256=YzTb0xAcNdmKO326mGQy7BmST56CZcdru4lX7ai_7kw,7579
 reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,402
 reconcile/utils/git.py,sha256=o4p9m8jlzCJDcutl2HErvGLhL6sZ1NB4Aw3zGcQIzso,2427
-reconcile/utils/github_api.py,sha256=o4J0ZU1ZSr9808uoorKHv19iae-eLo85yrCZX67p2kw,2822
-reconcile/utils/gitlab_api.py,sha256=U7l26fM1lQ4g-O06D0ZawvOXF_yzzH5NdR5iDx2ZJFA,28616
+reconcile/utils/github_api.py,sha256=S1vO-hvYPzm5BIychVIHSYibMns0HBmLgS78MkPfunE,3402
+reconcile/utils/gitlab_api.py,sha256=-wf1praBpU9h4TratCnAPBHEYgYJZLib6dl6Nszzrrw,28761
 reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
 reconcile/utils/gql.py,sha256=C0thIm_k9MBldfqwHzyqtYZk9sIvMdm9IbbnXLGwjD8,14158
 reconcile/utils/grouping.py,sha256=vr9SFHZ7bqmHYrvYcEZt-Er3-yQYfAAdq5sHLZVmXPY,456
@@ -656,7 +656,7 @@ reconcile/utils/promtool.py,sha256=xmPBWEApkk0L2qZBAvTxakNXxfTz-tVLPFxGnpsxXnM,2
 reconcile/utils/quay_api.py,sha256=uE_jxcdy3ViHtYFAfwDQuFDaO7Pr6AAPoVnmORbyHio,7822
 reconcile/utils/quay_mirror.py,sha256=dpWCNv5lITwIk6Q9RkmqaQKHNk_JPy27UQEribJ7E-U,1324
 reconcile/utils/raw_github_api.py,sha256=2WKtE8ABYYB9UGOAh9N_kLkksBWL3320Z2_scteZddI,2805
-reconcile/utils/repo_owners.py,sha256=Xwe1HOcMZe7Pknk47GLZHg5LDpDElmGfmc_x6pAdzsg,6589
+reconcile/utils/repo_owners.py,sha256=P0QX6F0oB8wYA08yiyzhYUiBtU57iIK_PsxbzKENbKM,6571
 reconcile/utils/rest_api_base.py,sha256=MT7tp6CQO2S5aKfVOzw_hipWg7wAGoOqkm4qurI1hEU,4342
 reconcile/utils/ruamel.py,sha256=FzL4_L0FnMOUZmgThrZSMJs5MTdXwiy-E9MZWfk8bh8,397
 reconcile/utils/secret_reader.py,sha256=MaP56KZaAE35EyYbgAitdm6fUSxdzWeGFSOym9qiZkw,10206
@@ -675,7 +675,7 @@ reconcile/utils/three_way_diff_strategy.py,sha256=oQcHXd9LVhirJfoaOBoHUYuZVGfyL2
 reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
 reconcile/utils/vault.py,sha256=aSA8l9cJlPUHpChFGl27nSY-Mpq9FMjBo7Dcgb1BVfM,15036
 reconcile/utils/vaultsecretref.py,sha256=0KUSzuvTRxPyKY919TO3-B_eYg4_76fzKvMF8j5s1G0,911
-reconcile/utils/vcs.py,sha256=AK35vIjx9bXYclKmvNekpaG_OETt-ZybibwV-m123xc,10186
+reconcile/utils/vcs.py,sha256=X9wCe0aWrDxP9kuOABLjkex0fYW-6vuYwjboYYlNMZM,10225
 reconcile/utils/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/acs/base.py,sha256=4UsDrCpAOuddL3PKNuIQYoJP1BtZQNNB8_KEX0lXneg,2532
 reconcile/utils/acs/policies.py,sha256=jpbi3qpGkBD_X6MfzsX12dPajUbmACmhIOz_0rDvYzs,5489
@@ -706,7 +706,7 @@ reconcile/utils/internal_groups/models.py,sha256=y_IqBVqfGqNXiu0VudvBWFrm_-uafVm
 reconcile/utils/jinja2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/jinja2/extensions.py,sha256=7K-uo6G2eCWa98MHT8fRPYIKCLQB_5D2keqQ_LyAfHM,1293
 reconcile/utils/jinja2/filters.py,sha256=JfO_14APySBPidsMvHXG-8dULNPddZCE15Umjk_aSBk,4830
-reconcile/utils/jinja2/utils.py,sha256=AQYEhrA4eEl7-tjf8KyVheSGWS3n-56K7zoJIH9IMF4,8819
+reconcile/utils/jinja2/utils.py,sha256=UTA0oOdhUgUBY57lWxgJS9WIAzP1ONm7mgGhY_DrczM,8821
 reconcile/utils/jobcontroller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/jobcontroller/controller.py,sha256=Vh08lZuCSIIceWGSDhBB00iFwTI9eeKZW1sfHlkAvSo,15373
 reconcile/utils/jobcontroller/models.py,sha256=x9YIvWfYOOvXNKToFVx1H7qDrZb0Sa1KI_4Y0gl7rMM,6336
@@ -721,17 +721,17 @@ reconcile/utils/mr/README.md,sha256=i9sCLkDFhSxAUtpa_I1_TxhR5vPOLcowuwn2VEWO41w,
 reconcile/utils/mr/__init__.py,sha256=hcfHDIIIsJT4C0BnzDnyeZEfZdamrqHzMLcBzIT1ibI,2578
 reconcile/utils/mr/app_interface_reporter.py,sha256=6Kpg93V9FvcOke9Jimkva359MQ-ZyBIkUpf8QIA6-to,1793
 reconcile/utils/mr/app_sre_tekton_access_report.py,sha256=i7JeTkyaq0HyVGbAFKCNbiLy3NVE05IXqJzwArsGre4,1513
-reconcile/utils/mr/aws_access.py,sha256=9MMpYD24j2lLr_hLeMSh_OsJ07waalrlNpz-JlOsKAM,2575
+reconcile/utils/mr/aws_access.py,sha256=_nQyRkNbz_UVnRv5YkGkc6o8NMlRJEHQ91BVscrRQag,2609
 reconcile/utils/mr/base.py,sha256=pHRmyxJzlBy9ldjYKOdHg9-9KbsK8bdRScntKZa5Tew,8130
-reconcile/utils/mr/clusters_updates.py,sha256=pcusPAwRUkvyk_-bixsRNTzSvpTLypJ1kflq5UEVgcM,2271
+reconcile/utils/mr/clusters_updates.py,sha256=T9eONmyTshEcd6UbYNDAslC6bw-Klql-Vr0scQBZnJE,2313
 reconcile/utils/mr/glitchtip_access_reporter.py,sha256=cTkOtzdgeKPaqro0VS2hDuAClQiN4nZATh-mplQC-AI,1369
 reconcile/utils/mr/labels.py,sha256=9QRTRjZAtq45zELd9SwavaraczMjwjn5no3RK1YxFTg,825
 reconcile/utils/mr/notificator.py,sha256=f8IcGQ1_iBsXJFnhPsWQ7UE3NfigaOrXcVieJPplYrY,2955
-reconcile/utils/mr/ocm_update_recommended_version.py,sha256=p_aVP0TGrlKk9WBwgQnYWqUDsED_Hg6G5Bqj0UvtRwA,1536
-reconcile/utils/mr/ocm_upgrade_scheduler_org_updates.py,sha256=5EncHGr4QRnZgHedRfCwMYZ9CaijYzHGj7-M6lhtQRo,3004
-reconcile/utils/mr/promote_qontract.py,sha256=wgvX2CBlcZaihKJSXJ0zcEK8NGaEP2_DUQDz0STzGes,7158
-reconcile/utils/mr/update_access_report_base.py,sha256=4Grohtp44v1sSHZyIAYOwClxH8SLj_nnOrCcHPKp9p0,4361
-reconcile/utils/mr/user_maintenance.py,sha256=KSfl4i0k1CqCa9mj93bvFxHeBHPRMn3sBWdGS1nNzYY,5371
+reconcile/utils/mr/ocm_update_recommended_version.py,sha256=KGOBURLE3XMynjAFej5D5PO9MJRDqMeoL1C8jLJMhD8,1570
+reconcile/utils/mr/ocm_upgrade_scheduler_org_updates.py,sha256=0myPruETuT6V2bjSSVPVsQu0ZHE9Jf3kRx5htt_fhKM,3038
+reconcile/utils/mr/promote_qontract.py,sha256=oOjfxkDAFhMDWCT9xaRyLSYHcc67jCc34s_tENOuHGw,7226
+reconcile/utils/mr/update_access_report_base.py,sha256=LOXTIonpfxXrvZI9nPqkszW3OnuIgbFM7pFpqHxTMp0,4395
+reconcile/utils/mr/user_maintenance.py,sha256=OC9-3YkpXsYKZGsdOdx-NLq7AkjgrhDR2JL7hov7BV8,5669
 reconcile/utils/ocm/__init__.py,sha256=Y-bp8GomMpyCo0tFW6kJ78-ZG1UIupYRtBzbMWU0kwM,798
 reconcile/utils/ocm/addons.py,sha256=_LDdJ-gapM3s5exKlIUt-MlXZTAUoHezbYBU0QmvfWQ,7335
 reconcile/utils/ocm/base.py,sha256=8rZ8WilNeAfq7HRNI8kNOLB4VcYzQpqQ5gvWbS54MuM,14576
@@ -763,7 +763,7 @@ reconcile/utils/runtime/sharding.py,sha256=r0ieUtNed7NvknSw6qQrCkKpVXE1shuHGnfFc
 reconcile/utils/saasherder/__init__.py,sha256=3U8plqMAPRE1kjwZ5YnIsYsggTf4_gS7flRUEuXVBAs,343
 reconcile/utils/saasherder/interfaces.py,sha256=NEYQspYfyWQhBeJyNCqSFbixi1A4wRVGB7FeNM5BDCk,9141
 reconcile/utils/saasherder/models.py,sha256=JaOz_DEtudJZhiDe90kaBlJkppFufn81V92oK9PHYx0,10208
-reconcile/utils/saasherder/saasherder.py,sha256=V8DOaMvY1POYd_EquIlU2sI62l64C-PjtaTCuzwVbn4,87306
+reconcile/utils/saasherder/saasherder.py,sha256=uQovEpcnZXpJlG_tBUY0X1v5obYMH55vc5cSvi3eZjU,86862
 reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
 reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
 reconcile/utils/terraform/config_client.py,sha256=gRL1rQ0AqvShei_rcGqC3HDYGskOFKE1nPrJyJE9yno,4676
@@ -809,7 +809,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev195.dist-info/METADATA,sha256=MHzXq3sy6lXSH9U2kaPoAkQ9w9LDLvg6vK7zNg4CbwQ,24555
-qontract_reconcile-0.10.2.dev195.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev195.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev195.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev196.dist-info/METADATA,sha256=uRgPJGf2hvsWk3jhJUQz581Qv8WkL4Q4R70sX5tYxSo,24555
+qontract_reconcile-0.10.2.dev196.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev196.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev196.dist-info/RECORD,,

reconcile/dashdotdb_dora.py

@@ -402,7 +402,7 @@ class DashdotdbDORA(DashdotdbBase):
             if not saas_file_yaml:
                 LOG.info(f"failed to fetch saas file {saastarget.path} at {sha}")
                 return (None, None)
-            saas_file = yaml.safe_load(saas_file_yaml.decode())
+            saas_file = yaml.safe_load(saas_file_yaml)
         except Exception as e:
             LOG.info(f"failed to decode saas file {saastarget.path} with error: {e}")
             return (None, None)

reconcile/utils/github_api.py

@@ -1,13 +1,21 @@
+import base64
 import os
 from pathlib import Path
 from types import TracebackType
 from urllib.parse import urlparse
 
 from github import Commit, Github, GithubException, UnknownObjectException
+from github.Repository import Repository
 from sretoolbox.utils import retry
 
 GH_BASE_URL = os.environ.get("GITHUB_API", "https://api.github.com")
 
+MAX_FILE_CONTENT_SIZE = 1024**2  # 1MB
+
+
+class UnsupportedDirectoryError(Exception):
+    pass
+
 
 class GithubRepositoryApi:
     """
@@ -66,16 +74,36 @@ class GithubRepositoryApi:
             tree_items.append(tree_item)
         return tree_items
 
+    @staticmethod
+    def get_raw_file(
+        repo: Repository,
+        path: str,
+        ref: str,
+    ) -> bytes:
+        content = repo.get_contents(path=path, ref=ref)
+        if isinstance(content, list):
+            raise UnsupportedDirectoryError(
+                f"Path {path} of ref {ref} in repo {repo.full_name} is a directory!"
+            )
+        if content.size < MAX_FILE_CONTENT_SIZE:
+            return content.decoded_content
+        blob = repo.get_git_blob(content.sha)
+        return base64.b64decode(blob.content)
+
     @retry()
-    def get_file(self, path: str, ref: str = "master") -> bytes | None:
+    def get_file(
+        self,
+        path: str,
+        ref: str = "master",
+    ) -> bytes | None:
         try:
-            content = self._repo.get_contents(path=path, ref=ref)
-            if isinstance(content, list):
-                # TODO: we should probably raise an exception here
-                # or handle this properly
-                # -> for now staying backwards compatible
-                return None
-            return content.decoded_content
+            return self.get_raw_file(
+                repo=self._repo,
+                path=path,
+                ref=ref,
+            )
+        except UnsupportedDirectoryError:
+            return None
         except GithubException as e:
             # handling a bug in the upstream GH library
             # https://github.com/PyGithub/PyGithub/issues/3179

reconcile/utils/gitlab_api.py

@@ -41,7 +41,6 @@ from gitlab.v4.objects import (
     GroupMember,
     PersonalAccessToken,
     Project,
-    ProjectFile,
     ProjectHook,
     ProjectIssue,
     ProjectIssueManager,
@@ -666,25 +665,19 @@ class GitLabApi:
         *,
         ref: str = "master",
         recursive: bool = False,
-        project: Project | None = None,
-        path: str = "",
     ) -> list[dict]:
         """
         Get a list of repository files and directories in a project.
 
         :param ref: The name of a repository branch or tag or, if not given, the default branch.
         :param recursive: Boolean value used to get a recursive tree. Default is false.
-        :param project: The project to get the tree from, if None, use the current project
-        :param path: The path inside the repository. Used to get content of subdirectories.
 
         :return: list of tree objects
         """
-        target_project = self.project if project is None else project
         return cast(
             list[dict],
-            target_project.repository_tree(
+            self.project.repository_tree(
                 ref=ref,
-                path=path,
                 recursive=recursive,
                 pagination="keyset",
                 per_page=MAX_PER_PAGE,
@@ -692,13 +685,36 @@ class GitLabApi:
             ),
         )
 
-    def get_file(self, path: str, ref: str = "master") -> ProjectFile | None:
+    @staticmethod
+    def get_raw_file(
+        project: Project,
+        path: str,
+        ref: str,
+    ) -> bytes:
+        file_path = path.lstrip("/")
+        return project.files.raw(
+            file_path=file_path,
+            ref=ref,
+        )
+
+    def get_file(
+        self,
+        path: str,
+        ref: str = "master",
+    ) -> bytes | None:
         """
-        Wrapper around Gitlab.files.get() with exception handling.
+        Get the raw content of a file in a project.
+
+        :param path: The path to the file in the repository.
+        :param ref: The name of branch, tag or commit.
+        :return: The content of the file as bytes, or None if the file does not exist.
         """
         try:
-            path = path.lstrip("/")
-            return self.project.files.get(file_path=path, ref=ref)
+            return self.get_raw_file(
+                project=self.project,
+                path=path,
+                ref=ref,
+            )
         except GitlabGetError:
             return None
 

reconcile/utils/jinja2/utils.py

@@ -12,6 +12,7 @@ from reconcile.checkpoint import url_makes_sense
 from reconcile.github_users import init_github
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.helpers import flatten
 from reconcile.utils.jinja2.extensions import B64EncodeExtension, RaiseErrorExtension
 from reconcile.utils.jinja2.filters import (
@@ -116,10 +117,12 @@ def lookup_github_file_content(
         )
 
     gh = init_github()
-    content = gh.get_repo(repo).get_contents(path, ref)
-    if isinstance(content, list):
-        raise Exception(f"multiple files found for {repo}/{path}/{ref}")
-    return content.decoded_content.decode("utf-8")
+    content = GithubRepositoryApi.get_raw_file(
+        repo=gh.get_repo(repo),
+        path=path,
+        ref=ref,
+    )
+    return content.decode("utf-8")
 
 
 def lookup_graphql_query_results(query: str, **kwargs: dict[str, Any]) -> list[Any]:

reconcile/utils/mr/aws_access.py

@@ -37,10 +37,12 @@ class CreateDeleteAwsAccessKey(MergeRequestBase):
 
     def process(self, gitlab_cli: GitLabApi) -> None:
         # add key to deleteKeys list to be picked up by aws-iam-keys
-        raw_file = gitlab_cli.project.files.get(
-            file_path=self.path, ref=gitlab_cli.main_branch
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=self.path,
+            ref=gitlab_cli.main_branch,
         )
-        content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+        content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
 
         content.setdefault("deleteKeys", [])
         content["deleteKeys"].append(self.key)

reconcile/utils/mr/clusters_updates.py

@@ -37,10 +37,12 @@ class CreateClustersUpdates(MergeRequestBase):
                 continue
 
             cluster_path = cluster_updates.pop("path")
-            raw_file = gitlab_cli.project.files.get(
-                file_path=cluster_path, ref=gitlab_cli.main_branch
+            raw_file = gitlab_cli.get_raw_file(
+                project=gitlab_cli.project,
+                path=cluster_path,
+                ref=gitlab_cli.main_branch,
             )
-            content = yaml.load(raw_file.decode())
+            content = yaml.load(raw_file)
             if "spec" not in content:
                 self.cancel("Spec missing. Nothing to do.")
 

reconcile/utils/mr/ocm_update_recommended_version.py

@@ -28,10 +28,12 @@ class CreateOCMUpdateRecommendedVersion(MergeRequestBase):
         return f"ocm update recommended version for {self.ocm_name}"
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        raw_file = gitlab_cli.project.files.get(
-            file_path=self.path, ref=gitlab_cli.main_branch
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=self.path,
+            ref=gitlab_cli.main_branch,
         )
-        content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+        content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
 
         content["recommendedVersions"] = self.recommended_versions
 

reconcile/utils/mr/ocm_upgrade_scheduler_org_updates.py

@@ -28,10 +28,12 @@ class CreateOCMUpgradeSchedulerOrgUpdates(MergeRequestBase):
         ocm_path = self.updates_info["path"]
         ocm_name = self.updates_info["name"]
 
-        raw_file = gitlab_cli.project.files.get(
-            file_path=ocm_path, ref=gitlab_cli.main_branch
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=ocm_path,
+            ref=gitlab_cli.main_branch,
         )
-        content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+        content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
         upgrade_policy_clusters = content["upgradePolicyClusters"]
 
         for update in self.updates_info["updates"]:

reconcile/utils/mr/promote_qontract.py

@@ -37,10 +37,12 @@ class PromoteQontractSchemas(MergeRequestBase):
         return f"promote qontract-schemas to version {self.version}"
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        raw_file = gitlab_cli.project.files.get(
-            file_path=self.path, ref=gitlab_cli.main_branch
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=self.path,
+            ref=gitlab_cli.main_branch,
         )
-        content = raw_file.decode().decode("utf-8")
+        content = raw_file.decode("utf-8")
         lines = content.splitlines()
         for index, text in enumerate(lines):
             if text.startswith("export SCHEMAS_IMAGE_TAG="):
@@ -121,9 +123,11 @@ class PromoteQontractReconcileCommercial(MergeRequestBase):
         search_text: str,
         replace_text: str,
     ) -> None:
-        raw_file = gitlab_cli.project.files.get(
-            file_path=path, ref=gitlab_cli.main_branch
-        ).decode()
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=path,
+            ref=gitlab_cli.main_branch,
+        )
         match method:
             case "line_search":
                 new_content = self._process_by_line_search(

reconcile/utils/mr/update_access_report_base.py

@@ -100,10 +100,12 @@ class UpdateAccessReportBase(MergeRequestBase):
         return new_workbook_md
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        workbook_file = gitlab_cli.project.files.get(
-            file_path=self._workbook_file_name, ref=self.branch
+        workbook_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=self._workbook_file_name,
+            ref=self.branch,
         )
-        workbook_md = self._update_workbook(workbook_file.decode().decode("utf-8"))
+        workbook_md = self._update_workbook(workbook_file.decode("utf-8"))
 
         if not self._dry_run:
             logging.info(

reconcile/utils/mr/user_maintenance.py

@@ -56,8 +56,12 @@ class CreateDeleteUserAppInterface(MergeRequestBase):
                     branch_name=self.branch, file_path=path, commit_message=self.title
                 )
             elif path_type == PathTypes.GABI:
-                raw_file = gitlab_cli.project.files.get(file_path=path, ref=self.branch)
-                content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+                raw_file = gitlab_cli.get_raw_file(
+                    project=gitlab_cli.project,
+                    path=path,
+                    ref=self.branch,
+                )
+                content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
                 for gabi_user in content["users"][:]:
                     if self.username in gabi_user["$ref"]:
                         content["users"].remove(gabi_user)
@@ -70,8 +74,12 @@ class CreateDeleteUserAppInterface(MergeRequestBase):
                     content=new_content,
                 )
             elif path_type == PathTypes.AWS_ACCOUNTS:
-                raw_file = gitlab_cli.project.files.get(file_path=path, ref=self.branch)
-                content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+                raw_file = gitlab_cli.get_raw_file(
+                    project=gitlab_cli.project,
+                    path=path,
+                    ref=self.branch,
+                )
+                content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
                 for reset_record in content["resetPasswords"]:
                     if self.username in reset_record["user"]["$ref"]:
                         content["resetPasswords"].remove(reset_record)
@@ -84,8 +92,12 @@ class CreateDeleteUserAppInterface(MergeRequestBase):
                             content=new_content,
                         )
             elif path_type == PathTypes.SCHEDULE:
-                raw_file = gitlab_cli.project.files.get(file_path=path, ref=self.branch)
-                content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+                raw_file = gitlab_cli.get_raw_file(
+                    project=gitlab_cli.project,
+                    path=path,
+                    ref=self.branch,
+                )
+                content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
                 delete_indexes: list[tuple[int, int]] = []
                 for schedule_index, schedule_record in enumerate(content["schedule"]):
                     for user_index, user in enumerate(schedule_record["users"]):
@@ -124,10 +136,12 @@ class CreateDeleteUserInfra(MergeRequestBase):
         return "delete user(s)"
 
     def process(self, gitlab_cli):
-        raw_file = gitlab_cli.project.files.get(
-            file_path=self.PLAYBOOK, ref=self.branch
+        raw_file = gitlab_cli.get_raw_file(
+            project=gitlab_cli.project,
+            path=self.PLAYBOOK,
+            ref=self.branch,
         )
-        content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
+        content = yaml.load(raw_file, Loader=yaml.RoundTripLoader)
 
         new_list = []
         for user in content[0]["vars"]["users"]:

reconcile/utils/repo_owners.py

@@ -131,7 +131,7 @@ class RepoOwners:
                 _LOG.warning(f"{self._git_cli!s}:{owner_file['path']} not found")
                 continue
             try:
-                owners = yaml.safe_load(raw_owners.decode())
+                owners = yaml.safe_load(raw_owners)
             except yaml.parser.ParserError:
                 owners = None
             if owners is None:
@@ -184,7 +184,7 @@ class RepoOwners:
         if raw_aliases is None:
             return {}
 
-        aliases = yaml.safe_load(raw_aliases.decode())
+        aliases = yaml.safe_load(raw_aliases)
         if aliases is None:
             return {}
 

reconcile/utils/saasherder/saasherder.py

@@ -27,7 +27,6 @@ from github import (
     GithubException,
 )
 from github.ContentFile import ContentFile
-from github.Repository import Repository
 from gitlab.exceptions import GitlabError
 from requests import exceptions as rqexc
 from sretoolbox.container import Image
@@ -39,6 +38,7 @@ from sretoolbox.utils import (
 from reconcile.github_org import get_default_config
 from reconcile.status import RunningState
 from reconcile.utils import helm
+from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.jenkins_api import JenkinsApi
 from reconcile.utils.jjb_client import JJB
@@ -715,24 +715,6 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     def _collect_image_patterns(self) -> set[str]:
         return {p for sf in self.saas_files for p in sf.image_patterns}
 
-    @staticmethod
-    def _get_file_contents_github(repo: Repository, path: str, commit_sha: str) -> str:
-        f = repo.get_contents(path, commit_sha)
-        if isinstance(f, list):
-            raise Exception(f"Path {path} and sha {commit_sha} is a directory!")
-
-        if f.size < 1024**2:  # 1 MB
-            return f.decoded_content.decode("utf8")
-
-        tree = repo.get_git_tree(commit_sha, recursive="/" in path).tree
-        for x in tree:
-            if x.path != path.lstrip("/"):
-                continue
-            blob = repo.get_git_blob(x.sha)
-            return base64.b64decode(blob.content).decode("utf8")
-
-        return ""
-
     @retry(max_attempts=20)
     def get_archive_info(
         self,
@@ -768,13 +750,20 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         match repo_info.platform:
             case "github":
                 repo = github.get_repo(repo_info.name)
-                content = self._get_file_contents_github(repo, path, commit_sha)
+                content = GithubRepositoryApi.get_raw_file(
+                    repo=repo,
+                    path=path,
+                    ref=commit_sha,
+                )
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
                 project = self.gitlab.get_project(url)
-                f = project.files.get(file_path=path.lstrip("/"), ref=commit_sha)
-                content = f.decode()
+                content = self.gitlab.get_raw_file(
+                    project=project,
+                    path=path,
+                    ref=commit_sha,
+                )
             case _:
                 raise Exception(f"Only GitHub and GitLab are supported: {url}")
 
@@ -795,10 +784,12 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                     raise Exception(f"Path {path} and sha {commit_sha} is a file!")
                 for f in directory:
                     file_path = os.path.join(path, f.name)
-                    file_contents_decoded = self._get_file_contents_github(
-                        repo, file_path, commit_sha
+                    raw_file = GithubRepositoryApi.get_raw_file(
+                        repo=repo,
+                        path=file_path,
+                        ref=commit_sha,
                     )
-                    result_resources = yaml.safe_load_all(file_contents_decoded)
+                    result_resources = yaml.safe_load_all(raw_file)
                     resources.extend(result_resources)
             case "gitlab":
                 if not self.gitlab:

reconcile/utils/vcs.py

@@ -266,11 +266,12 @@ class VCS:
                 if not file_path.startswith("data")
                 else file_path
             )
-        return (
-            self._app_interface_api.project.files.get(file_path=file_path, ref=ref)
-            .decode()
-            .decode("utf-8")
+        file = self._app_interface_api.get_raw_file(
+            project=self._app_interface_api.project,
+            path=file_path,
+            ref=ref,
         )
+        return file.decode("utf-8")
 
     def get_open_app_interface_merge_requests(self) -> list[ProjectMergeRequest]:
         return self._app_interface_api.get_merge_requests(state=MRState.OPENED)