qontract-reconcile 0.10.2.dev181__tar.gz → 0.10.2.dev183__tar.gz

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev181
+Version: 0.10.2.dev183
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/reconcile/dashdotdb_dora.py

@@ -34,6 +34,7 @@ from reconcile.typed_queries.saas_files import get_saas_files
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 QONTRACT_INTEGRATION = "dashdotdb-dora"
 
@@ -421,7 +422,7 @@ class DashdotdbDORA(DashdotdbBase):
             return rc, []
 
         LOG.info("Fetching commits %s", rc)
-        if rc.repo_url.startswith("https://github.com"):
+        if rc.repo_url.startswith(GITHUB_BASE_URL):
             try:
                 commits = self._github_compare_commits(rc)
             except GithubException as e:
@@ -476,8 +477,7 @@ class DashdotdbDORA(DashdotdbBase):
         if not rc.repo_url:
             return []
 
-        prefix = "https://github.com/"
-        repo = rc.repo_url[rc.repo_url.startswith(prefix) and len(prefix) :]
+        repo = rc.repo_url.removeprefix(GITHUB_BASE_URL).rstrip("/")
 
         return [
             Commit(rc.repo_url, commit.sha, commit.commit.committer.date)

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/reconcile/github_repo_permissions_validator.py

@@ -11,6 +11,7 @@ from reconcile.jenkins_job_builder import init_jjb
 from reconcile.utils.jjb_client import JJB
 from reconcile.utils.secret_reader import SecretReader
 from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 QONTRACT_INTEGRATION = "github-repo-permissions-validator"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
@@ -45,7 +46,7 @@ def run(dry_run: bool, instance_name: str) -> None:
     error = False
     for job in pr_check_jobs:
         repo_url = jjb.get_repo_url(job)
-        repo_name = repo_url.rstrip("/").replace("https://github.com/", "")
+        repo_name = repo_url.removeprefix(GITHUB_BASE_URL).rstrip("/")
         repo = gh.get_repo(repo_name)
         permissions = repo.permissions
         if not permissions.push and repo_url not in invitations:

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/reconcile/integrations_manager.py

@@ -47,6 +47,7 @@ from reconcile.utils.runtime.sharding import (
     StaticShardingStrategy,
 )
 from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 QONTRACT_INTEGRATION = "integrations-manager"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
@@ -58,8 +59,7 @@ INTEGRATION_UPSTREAM_REPOS_PARAM = "INTEGRATION_UPSTREAM_REPOS"
 
 
 def get_image_tag_from_ref(ref: str, upstream: str) -> str:
-    gh_prefix = "https://github.com/"
-    upstream = upstream.removeprefix(gh_prefix)
+    upstream = upstream.removeprefix(GITHUB_BASE_URL)
     settings = queries.get_app_interface_settings()
     gh_token = get_default_config()["token"]
     github = Github(gh_token, base_url=GH_BASE_URL)

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/reconcile/service_dependencies.py

@@ -16,6 +16,7 @@ from reconcile.gql_definitions.service_dependencies.service_dependencies import
     SaasResourceTemplateV2,
 )
 from reconcile.utils import gql
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 QONTRACT_INTEGRATION = "service-dependencies"
 
@@ -35,7 +36,9 @@ def get_desired_dependency_names(
         gitlab_urls = [cc for cc in code_components if "gitlab" in cc.url]
         if gitlab_urls:
             required_dep_names.update(get_dependency_names(dependency_map, "gitlab"))
-        github_urls = [cc for cc in code_components if "github.com" in cc.url]
+        github_urls = [
+            cc for cc in code_components if cc.url.startswith(GITHUB_BASE_URL)
+        ]
         if github_urls:
             required_dep_names.update(get_dependency_names(dependency_map, "github"))
 

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/reconcile/terraform_tgw_attachments.py

@@ -550,7 +550,7 @@ def run(
             integration_version=QONTRACT_INTEGRATION_VERSION,
             dry_run=dry_run,
             cache_source=cache_source,
-            shard="_".join(account_name) if account_name else "",
+            shard=account_name or "",
             ttl_seconds=extended_early_exit_cache_ttl_seconds,
             logger=logging.getLogger(),
             runner=runner,

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/reconcile/utils/gitlab_api.py

@@ -1,6 +1,8 @@
+import io
 import logging
 import os
 import re
+import tarfile
 from collections.abc import (
     Iterable,
     Mapping,
@@ -828,3 +830,29 @@ class GitLabApi:
             list[PersonalAccessToken],
             self.gl.personal_access_tokens.list(get_all=True),
         )
+
+    @staticmethod
+    def get_directory_contents(
+        project: Project,
+        ref: str | None = None,
+        path: str | None = None,
+    ) -> dict[str, bytes]:
+        """
+        Get the contents of a directory in a project.
+
+        :param project: The project to get the contents from.
+        :param ref: The commit SHA to download. A tag, branch reference, or SHA can be used. If not specified, defaults to the tip of the default branch.
+        :param path: The subpath of the repository to download. If an empty string, defaults to the whole repository.
+        :return: A dictionary with the file path as keys and the file content bytes as values.
+        """
+        archive = project.repository_archive(format="tar.gz", sha=ref, path=path)
+        tar_bytes = io.BytesIO(archive)
+        with tarfile.open(fileobj=tar_bytes, mode="r:gz") as tar:
+            return {
+                file_path: file.read()
+                for member in tar.getmembers()
+                if member.isfile()
+                # skip leading prefix xxx/
+                and (file_path := member.name.split("/", 1)[-1])
+                and (file := tar.extractfile(member))
+            }

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/reconcile/utils/jjb_client.py

@@ -25,6 +25,7 @@ from sretoolbox.utils import retry
 
 from reconcile.utils import throughput
 from reconcile.utils.helpers import toggle_logger
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
 
@@ -290,7 +291,7 @@ class JJB:  # pylint: disable=too-many-public-methods
             for job in jobs:
                 try:
                     project_url_raw = job["properties"][0]["github"]["url"]
-                    if "https://github.com" in project_url_raw:
+                    if project_url_raw.startswith(GITHUB_BASE_URL):
                         continue
                     if str(job.get("disabled")).lower() == "true":
                         continue

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/reconcile/utils/saasherder/saasherder.py

@@ -83,6 +83,7 @@ from reconcile.utils.saasherder.models import (
 )
 from reconcile.utils.secret_reader import SecretReaderBase
 from reconcile.utils.state import State
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 TARGET_CONFIG_HASH = "target_config_hash"
 
@@ -742,7 +743,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         [url, sha] = trigger_reason.split(" ")[0].split("/commit/")
         repo_name = urlparse(url).path.strip("/")
         file_name = f"{repo_name.replace('/', '-')}-{sha}.tar.gz"
-        if "github" in url:
+        if url.startswith(GITHUB_BASE_URL):
             github = self._initiate_github(saas_file, base_url="https://api.github.com")
             repo = github.get_repo(repo_name)
             # get_archive_link get redirect url form header, it does not work with github-mirror
@@ -760,8 +761,8 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     ) -> tuple[Any, str]:
         commit_sha = self._get_commit_sha(url, ref, github)
 
-        if "github" in url:
-            repo_name = url.rstrip("/").replace("https://github.com/", "")
+        if url.startswith(GITHUB_BASE_URL):
+            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
             repo = github.get_repo(repo_name)
             content = self._get_file_contents_github(repo, path, commit_sha)
         elif "gitlab" in url:
@@ -781,8 +782,8 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     ) -> tuple[list[Any], str]:
         commit_sha = self._get_commit_sha(url, ref, github)
         resources: list[Any] = []
-        if "github" in url:
-            repo_name = url.rstrip("/").replace("https://github.com/", "")
+        if url.startswith(GITHUB_BASE_URL):
+            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
             repo = github.get_repo(repo_name)
             directory = repo.get_contents(path, commit_sha)
             if isinstance(directory, ContentFile):
@@ -798,17 +799,14 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
             if not self.gitlab:
                 raise Exception("gitlab is not initialized")
             project = self.gitlab.get_project(url)
-            for item in self.gitlab.get_repository_tree(
-                project=project,
-                path=path.lstrip("/"),
+            dir_contents = self.gitlab.get_directory_contents(
+                project,
                 ref=commit_sha,
-                recursive=False,
-            ):
-                file_contents = project.files.get(
-                    file_path=item["path"], ref=commit_sha
-                )
-                resource = yaml.safe_load(file_contents.decode())
-                resources.append(resource)
+                path=path,
+            )
+            for content in dir_contents.values():
+                result_resources = yaml.safe_load_all(content)
+                resources.extend(result_resources)
         else:
             raise Exception(f"Only GitHub and GitLab are supported: {url}")
 
@@ -817,8 +815,8 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     @retry()
     def _get_commit_sha(self, url: str, ref: str, github: Github) -> str:
         commit_sha = ""
-        if "github" in url:
-            repo_name = url.rstrip("/").replace("https://github.com/", "")
+        if url.startswith(GITHUB_BASE_URL):
+            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
             repo = github.get_repo(repo_name)
             commit = repo.get_commit(sha=ref)
             commit_sha = commit.sha

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/reconcile/utils/terrascript_aws_client.py

@@ -186,6 +186,7 @@ from reconcile.utils.password_validator import (
 )
 from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
 from reconcile.utils.terraform import safe_resource_id
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 GH_BASE_URL = os.environ.get("GITHUB_API", "https://api.github.com")
 LOGTOES_RELEASE = "repos/app-sre/logs-to-elasticsearch-lambda/releases/latest"
@@ -5685,9 +5686,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             return ref
 
         # get commit_sha from branch
-        if "github" in url:
+        if url.startswith(GITHUB_BASE_URL):
             github = self.init_github()
-            repo_name = url.rstrip("/").replace("https://github.com/", "")
+            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
             repo = github.get_repo(repo_name)
             commit = repo.get_commit(sha=ref)
             return commit.sha

{qontract_reconcile-0.10.2.dev181 → qontract_reconcile-0.10.2.dev183}/reconcile/utils/vcs.py

@@ -22,6 +22,8 @@ from reconcile.utils.secret_reader import (
     SecretReaderBase,
 )
 
+GITHUB_BASE_URL = "https://github.com/"
+
 
 class MRCheckStatus(Enum):
     NONE = 0
@@ -158,7 +160,7 @@ class VCS:
     ) -> str:
         if bool(self._is_commit_sha_regex.search(ref)):
             return ref
-        if repo_url.startswith("https://github.com/"):
+        if repo_url.startswith(GITHUB_BASE_URL):
             github = self._init_github(repo_url=repo_url, auth_code=auth_code)
             return github.get_commit_sha(ref=ref)
         # assume gitlab by default
@@ -175,7 +177,7 @@ class VCS:
         Return a list of commits between two commits.
         Note, that the commit_to is included in the result list, whereas commit_from is not included.
         """
-        if repo_url.startswith("https://github.com/"):
+        if repo_url.startswith(GITHUB_BASE_URL):
             github = self._init_github(repo_url=repo_url, auth_code=auth_code)
             data = github.compare(commit_from=commit_from, commit_to=commit_to)
             return [