qontract-reconcile 0.10.2.dev182__tar.gz → 0.10.2.dev184__tar.gz

{qontract_reconcile-0.10.2.dev182 → qontract_reconcile-0.10.2.dev184}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev182
+Version: 0.10.2.dev184
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev182 → qontract_reconcile-0.10.2.dev184}/reconcile/dashdotdb_dora.py

@@ -34,7 +34,7 @@ from reconcile.typed_queries.saas_files import get_saas_files
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.secret_reader import create_secret_reader
-from reconcile.utils.vcs import GITHUB_BASE_URL
+from reconcile.utils.vcs import VCS
 
 QONTRACT_INTEGRATION = "dashdotdb-dora"
 
@@ -422,19 +422,21 @@ class DashdotdbDORA(DashdotdbBase):
             return rc, []
 
         LOG.info("Fetching commits %s", rc)
-        if rc.repo_url.startswith(GITHUB_BASE_URL):
-            try:
-                commits = self._github_compare_commits(rc)
-            except GithubException as e:
-                if e.status == 404:
-                    LOG.info(
-                        f"Ignoring RepoChanges for {rc} because could not calculate them: {e.data['message']}"
-                    )
-                    return rc, []
-        elif rc.repo_url.startswith(self.gl.server):
-            commits = self._gitlab_compare_commits(rc)
-        else:
-            raise Exception(f"Unknown git hosting {rc.repo_url}")
+        repo_info = VCS.parse_repo_url(rc.repo_url)
+        match repo_info.platform:
+            case "github":
+                try:
+                    commits = self._github_compare_commits(rc, repo_info.name)
+                except GithubException as e:
+                    if e.status == 404:
+                        LOG.info(
+                            f"Ignoring RepoChanges for {rc} because could not calculate them: {e.data['message']}"
+                        )
+                        return rc, []
+            case "gitlab":
+                commits = self._gitlab_compare_commits(rc, repo_info.name)
+            case _:
+                raise Exception(f"Unknown git hosting {rc.repo_url}")
 
         return rc, commits
 
@@ -455,13 +457,10 @@ class DashdotdbDORA(DashdotdbBase):
 
         return app_env, datetime.fromisoformat(response.json()["finish_timestamp"])
 
-    def _gitlab_compare_commits(self, rc: RepoChanges) -> list[Commit]:
+    def _gitlab_compare_commits(self, rc: RepoChanges, repo: str) -> list[Commit]:
         if not rc.repo_url or not rc.ref_from or not rc.ref_to:
             return []
 
-        prefix = "https://gitlab.cee.redhat.com/"
-        repo = rc.repo_url[rc.repo_url.startswith(prefix) and len(prefix) :]
-
         commits = self.gl.repository_compare(repo, rc.ref_from, rc.ref_to)
 
         return [
@@ -473,12 +472,10 @@ class DashdotdbDORA(DashdotdbBase):
             for commit in commits
         ]
 
-    def _github_compare_commits(self, rc: RepoChanges) -> list[Commit]:
+    def _github_compare_commits(self, rc: RepoChanges, repo: str) -> list[Commit]:
         if not rc.repo_url:
             return []
 
-        repo = rc.repo_url.removeprefix(GITHUB_BASE_URL).rstrip("/")
-
         return [
             Commit(rc.repo_url, commit.sha, commit.commit.committer.date)
             for commit in self.gh_api(repo).compare(rc.ref_from, rc.ref_to)

{qontract_reconcile-0.10.2.dev182 → qontract_reconcile-0.10.2.dev184}/reconcile/github_repo_permissions_validator.py

@@ -11,7 +11,7 @@ from reconcile.jenkins_job_builder import init_jjb
 from reconcile.utils.jjb_client import JJB
 from reconcile.utils.secret_reader import SecretReader
 from reconcile.utils.semver_helper import make_semver
-from reconcile.utils.vcs import GITHUB_BASE_URL
+from reconcile.utils.vcs import VCS
 
 QONTRACT_INTEGRATION = "github-repo-permissions-validator"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
@@ -46,8 +46,8 @@ def run(dry_run: bool, instance_name: str) -> None:
     error = False
     for job in pr_check_jobs:
         repo_url = jjb.get_repo_url(job)
-        repo_name = repo_url.removeprefix(GITHUB_BASE_URL).rstrip("/")
-        repo = gh.get_repo(repo_name)
+        repo_info = VCS.parse_repo_url(repo_url)
+        repo = gh.get_repo(repo_info.name)
         permissions = repo.permissions
         if not permissions.push and repo_url not in invitations:
             logging.error(f"missing write permissions for bot in repo {repo_url}")

{qontract_reconcile-0.10.2.dev182 → qontract_reconcile-0.10.2.dev184}/reconcile/service_dependencies.py

@@ -16,7 +16,7 @@ from reconcile.gql_definitions.service_dependencies.service_dependencies import
     SaasResourceTemplateV2,
 )
 from reconcile.utils import gql
-from reconcile.utils.vcs import GITHUB_BASE_URL
+from reconcile.utils.vcs import VCS
 
 QONTRACT_INTEGRATION = "service-dependencies"
 
@@ -32,15 +32,13 @@ def get_desired_dependency_names(
     required_dep_names = set()
 
     code_components: list[AppCodeComponentsV1] = app.code_components or []
-    if code_components:
-        gitlab_urls = [cc for cc in code_components if "gitlab" in cc.url]
-        if gitlab_urls:
-            required_dep_names.update(get_dependency_names(dependency_map, "gitlab"))
-        github_urls = [
-            cc for cc in code_components if cc.url.startswith(GITHUB_BASE_URL)
-        ]
-        if github_urls:
-            required_dep_names.update(get_dependency_names(dependency_map, "github"))
+    code_component_platforms = {
+        platform
+        for cc in code_components
+        if (platform := VCS.parse_repo_url(cc.url).platform)
+    }
+    for platform in code_component_platforms:
+        required_dep_names.update(get_dependency_names(dependency_map, platform))
 
     jenkins_configs: list[JenkinsConfigV1] = app.jenkins_configs or []
     if jenkins_configs:

{qontract_reconcile-0.10.2.dev182 → qontract_reconcile-0.10.2.dev184}/reconcile/slack_usergroups.py

@@ -10,7 +10,6 @@ from typing import (
     Any,
     TypedDict,
 )
-from urllib.parse import urlparse
 
 from github.GithubException import UnknownObjectException
 from pydantic import BaseModel
@@ -67,6 +66,7 @@ from reconcile.utils.slack_api import (
     SlackApiError,
     UsergroupNotFoundException,
 )
+from reconcile.utils.vcs import VCS
 
 DATE_FORMAT = "%Y-%m-%d %H:%M"
 QONTRACT_INTEGRATION = "slack-usergroups"
@@ -77,25 +77,23 @@ error_occurred = False
 
 def get_git_api(url: str) -> GithubRepositoryApi | GitLabApi:
     """Return GitHub/GitLab API based on url."""
-    parsed_url = urlparse(url)
-
-    if parsed_url.hostname:
-        if "github" in parsed_url.hostname:
+    repo_info = VCS.parse_repo_url(url)
+    match repo_info.platform:
+        case "github":
             vault_settings = get_app_interface_vault_settings()
             secret_reader = create_secret_reader(use_vault=vault_settings.vault)
             instance = queries.get_github_instance()
             token = secret_reader.read(instance["token"])
-
             return GithubRepositoryApi(
                 repo_url=url,
                 token=token,
             )
-        if "gitlab" in parsed_url.hostname:
+        case "gitlab":
             settings = queries.get_app_interface_settings()
             instance = queries.get_gitlab_instance()
             return GitLabApi(instance=instance, project_url=url, settings=settings)
-
-    raise ValueError(f"Unable to handle URL: {url}")
+        case _:
+            raise ValueError(f"Unable to handle URL: {url}")
 
 
 class SlackObject(BaseModel):

{qontract_reconcile-0.10.2.dev182 → qontract_reconcile-0.10.2.dev184}/reconcile/terraform_tgw_attachments.py

@@ -550,7 +550,7 @@ def run(
             integration_version=QONTRACT_INTEGRATION_VERSION,
             dry_run=dry_run,
             cache_source=cache_source,
-            shard="_".join(account_name) if account_name else "",
+            shard=account_name or "",
             ttl_seconds=extended_early_exit_cache_ttl_seconds,
             logger=logging.getLogger(),
             runner=runner,

{qontract_reconcile-0.10.2.dev182 → qontract_reconcile-0.10.2.dev184}/reconcile/utils/gitlab_api.py

@@ -388,7 +388,7 @@ class GitLabApi:
 
     @retry()
     def get_project(self, repo_url: str) -> Project | None:
-        repo = repo_url.replace(self.server + "/", "")
+        repo = repo_url.removeprefix(self.server).strip("/")
         try:
             project = self.gl.projects.get(repo)
         except GitlabGetError:

{qontract_reconcile-0.10.2.dev182 → qontract_reconcile-0.10.2.dev184}/reconcile/utils/saasherder/saasherder.py

@@ -20,7 +20,6 @@ from contextlib import suppress
 from datetime import UTC, datetime, timedelta
 from types import TracebackType
 from typing import Any
-from urllib.parse import urlparse
 
 import yaml
 from github import (
@@ -83,7 +82,7 @@ from reconcile.utils.saasherder.models import (
 )
 from reconcile.utils.secret_reader import SecretReaderBase
 from reconcile.utils.state import State
-from reconcile.utils.vcs import GITHUB_BASE_URL
+from reconcile.utils.vcs import VCS
 
 TARGET_CONFIG_HASH = "target_config_hash"
 
@@ -741,17 +740,21 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         trigger_reason: str,
     ) -> tuple[str, str]:
         [url, sha] = trigger_reason.split(" ")[0].split("/commit/")
-        repo_name = urlparse(url).path.strip("/")
+        repo_info = VCS.parse_repo_url(url)
+        repo_name = repo_info.name
         file_name = f"{repo_name.replace('/', '-')}-{sha}.tar.gz"
-        if url.startswith(GITHUB_BASE_URL):
-            github = self._initiate_github(saas_file, base_url="https://api.github.com")
-            repo = github.get_repo(repo_name)
-            # get_archive_link get redirect url form header, it does not work with github-mirror
-            archive_url = repo.get_archive_link("tarball", ref=sha)
-        elif "gitlab" in url:
-            archive_url = f"{url}/-/archive/{sha}/{file_name}"
-        else:
-            raise Exception(f"Only GitHub and GitLab are supported: {url}")
+        match repo_info.platform:
+            case "github":
+                github = self._initiate_github(
+                    saas_file, base_url="https://api.github.com"
+                )
+                repo = github.get_repo(repo_name)
+                # get_archive_link get redirect url form header, it does not work with github-mirror
+                archive_url = repo.get_archive_link("tarball", ref=sha)
+            case "gitlab":
+                archive_url = f"{url}/-/archive/{sha}/{file_name}"
+            case _:
+                raise Exception(f"Only GitHub and GitLab are supported: {url}")
 
         return file_name, archive_url
 
@@ -761,18 +764,19 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     ) -> tuple[Any, str]:
         commit_sha = self._get_commit_sha(url, ref, github)
 
-        if url.startswith(GITHUB_BASE_URL):
-            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
-            repo = github.get_repo(repo_name)
-            content = self._get_file_contents_github(repo, path, commit_sha)
-        elif "gitlab" in url:
-            if not self.gitlab:
-                raise Exception("gitlab is not initialized")
-            project = self.gitlab.get_project(url)
-            f = project.files.get(file_path=path.lstrip("/"), ref=commit_sha)
-            content = f.decode()
-        else:
-            raise Exception(f"Only GitHub and GitLab are supported: {url}")
+        repo_info = VCS.parse_repo_url(url)
+        match repo_info.platform:
+            case "github":
+                repo = github.get_repo(repo_info.name)
+                content = self._get_file_contents_github(repo, path, commit_sha)
+            case "gitlab":
+                if not self.gitlab:
+                    raise Exception("gitlab is not initialized")
+                project = self.gitlab.get_project(url)
+                f = project.files.get(file_path=path.lstrip("/"), ref=commit_sha)
+                content = f.decode()
+            case _:
+                raise Exception(f"Only GitHub and GitLab are supported: {url}")
 
         return yaml.safe_load(content), commit_sha
 
@@ -782,52 +786,53 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     ) -> tuple[list[Any], str]:
         commit_sha = self._get_commit_sha(url, ref, github)
         resources: list[Any] = []
-        if url.startswith(GITHUB_BASE_URL):
-            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
-            repo = github.get_repo(repo_name)
-            directory = repo.get_contents(path, commit_sha)
-            if isinstance(directory, ContentFile):
-                raise Exception(f"Path {path} and sha {commit_sha} is a file!")
-            for f in directory:
-                file_path = os.path.join(path, f.name)
-                file_contents_decoded = self._get_file_contents_github(
-                    repo, file_path, commit_sha
+        repo_info = VCS.parse_repo_url(url)
+        match repo_info.platform:
+            case "github":
+                repo = github.get_repo(repo_info.name)
+                directory = repo.get_contents(path, commit_sha)
+                if isinstance(directory, ContentFile):
+                    raise Exception(f"Path {path} and sha {commit_sha} is a file!")
+                for f in directory:
+                    file_path = os.path.join(path, f.name)
+                    file_contents_decoded = self._get_file_contents_github(
+                        repo, file_path, commit_sha
+                    )
+                    result_resources = yaml.safe_load_all(file_contents_decoded)
+                    resources.extend(result_resources)
+            case "gitlab":
+                if not self.gitlab:
+                    raise Exception("gitlab is not initialized")
+                project = self.gitlab.get_project(url)
+                dir_contents = self.gitlab.get_directory_contents(
+                    project,
+                    ref=commit_sha,
+                    path=path,
                 )
-                result_resources = yaml.safe_load_all(file_contents_decoded)
-                resources.extend(result_resources)
-        elif "gitlab" in url:
-            if not self.gitlab:
-                raise Exception("gitlab is not initialized")
-            project = self.gitlab.get_project(url)
-            dir_contents = self.gitlab.get_directory_contents(
-                project,
-                ref=commit_sha,
-                path=path,
-            )
-            for content in dir_contents.values():
-                result_resources = yaml.safe_load_all(content)
-                resources.extend(result_resources)
-        else:
-            raise Exception(f"Only GitHub and GitLab are supported: {url}")
+                for content in dir_contents.values():
+                    result_resources = yaml.safe_load_all(content)
+                    resources.extend(result_resources)
+            case _:
+                raise Exception(f"Only GitHub and GitLab are supported: {url}")
 
         return resources, commit_sha
 
     @retry()
     def _get_commit_sha(self, url: str, ref: str, github: Github) -> str:
-        commit_sha = ""
-        if url.startswith(GITHUB_BASE_URL):
-            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
-            repo = github.get_repo(repo_name)
-            commit = repo.get_commit(sha=ref)
-            commit_sha = commit.sha
-        elif "gitlab" in url:
-            if not self.gitlab:
-                raise Exception("gitlab is not initialized")
-            project = self.gitlab.get_project(url)
-            commits = project.commits.list(ref_name=ref, per_page=1, page=1)
-            commit_sha = commits[0].id
-
-        return commit_sha
+        repo_info = VCS.parse_repo_url(url)
+        match repo_info.platform:
+            case "github":
+                repo = github.get_repo(repo_info.name)
+                commit = repo.get_commit(sha=ref)
+                return commit.sha
+            case "gitlab":
+                if not self.gitlab:
+                    raise Exception("gitlab is not initialized")
+                project = self.gitlab.get_project(url)
+                commits = project.commits.list(ref_name=ref, per_page=1, page=1)
+                return commits[0].id
+            case _:
+                return ""
 
     @staticmethod
     def _additional_resource_process(resources: Resources, html_url: str) -> None:

{qontract_reconcile-0.10.2.dev182 → qontract_reconcile-0.10.2.dev184}/reconcile/utils/terrascript_aws_client.py

@@ -186,7 +186,7 @@ from reconcile.utils.password_validator import (
 )
 from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
 from reconcile.utils.terraform import safe_resource_id
-from reconcile.utils.vcs import GITHUB_BASE_URL
+from reconcile.utils.vcs import VCS
 
 GH_BASE_URL = os.environ.get("GITHUB_API", "https://api.github.com")
 LOGTOES_RELEASE = "repos/app-sre/logs-to-elasticsearch-lambda/releases/latest"
@@ -5686,20 +5686,20 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             return ref
 
         # get commit_sha from branch
-        if url.startswith(GITHUB_BASE_URL):
-            github = self.init_github()
-            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
-            repo = github.get_repo(repo_name)
-            commit = repo.get_commit(sha=ref)
-            return commit.sha
-
-        if "gitlab" in url:
-            gitlab = self.init_gitlab()
-            project = gitlab.get_project(url)
-            commits = project.commits.list(ref_name=ref, per_page=1, page=1)
-            return commits[0].id
-
-        return ""
+        repo_url_info = VCS.parse_repo_url(url)
+        match repo_url_info.platform:
+            case "github":
+                github = self.init_github()
+                repo = github.get_repo(repo_url_info.name)
+                commit = repo.get_commit(sha=ref)
+                return commit.sha
+            case "gitlab":
+                gitlab = self.init_gitlab()
+                project = gitlab.get_project(url)
+                commits = project.commits.list(ref_name=ref, per_page=1, page=1)
+                return commits[0].id
+            case _:
+                return ""
 
     def get_asg_image_id(
         self, filters: Iterable[Mapping[str, Any]], account: str, region: str

{qontract_reconcile-0.10.2.dev182 → qontract_reconcile-0.10.2.dev184}/reconcile/utils/vcs.py

@@ -6,6 +6,8 @@ from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
+from typing import Literal
+from urllib.parse import urlparse
 
 from gitlab.v4.objects import ProjectMergeRequest
 
@@ -24,6 +26,16 @@ from reconcile.utils.secret_reader import (
 
 GITHUB_BASE_URL = "https://github.com/"
 
+Platform = Literal["github", "gitlab"]
+
+SUPPORTED_PLATFORMS: list[Platform] = ["github", "gitlab"]
+
+
+@dataclass(frozen=True)
+class RepoInfo:
+    platform: Platform | None
+    name: str
+
 
 class MRCheckStatus(Enum):
     NONE = 0
@@ -155,16 +167,40 @@ class VCS:
                 # Lets assume all other states as non-present
                 return MRCheckStatus.NONE
 
+    @staticmethod
+    def parse_repo_url(url: str) -> RepoInfo:
+        """
+        Parse a repository URL and return a RepoInfo object.
+        `platform` can be 'github', 'gitlab' or None if not recognized,
+        it's inferred from the URL host.
+        `name` is the path part of the URL, stripped of leading and trailing slashes.
+        """
+        parsed_url = urlparse(url)
+        platform = next(
+            (
+                p
+                for p in SUPPORTED_PLATFORMS
+                if (hostname := parsed_url.hostname) and p in hostname
+            ),
+            None,
+        )
+        name = parsed_url.path.strip("/").removesuffix(".git")
+        return RepoInfo(platform=platform, name=name)
+
     def get_commit_sha(
         self, repo_url: str, ref: str, auth_code: HasSecret | None
     ) -> str:
         if bool(self._is_commit_sha_regex.search(ref)):
             return ref
-        if repo_url.startswith(GITHUB_BASE_URL):
-            github = self._init_github(repo_url=repo_url, auth_code=auth_code)
-            return github.get_commit_sha(ref=ref)
-        # assume gitlab by default
-        return self._gitlab_instance.get_commit_sha(ref=ref, repo_url=repo_url)
+        repo_info = self.parse_repo_url(repo_url)
+        match repo_info.platform:
+            case "github":
+                github = self._init_github(repo_url=repo_url, auth_code=auth_code)
+                return github.get_commit_sha(ref=ref)
+            case "gitlab":
+                return self._gitlab_instance.get_commit_sha(ref=ref, repo_url=repo_url)
+            case _:
+                raise ValueError(f"Unsupported repository URL: {repo_url}")
 
     def get_commits_between(
         self,
@@ -177,30 +213,33 @@ class VCS:
         Return a list of commits between two commits.
         Note, that the commit_to is included in the result list, whereas commit_from is not included.
         """
-        if repo_url.startswith(GITHUB_BASE_URL):
-            github = self._init_github(repo_url=repo_url, auth_code=auth_code)
-            data = github.compare(commit_from=commit_from, commit_to=commit_to)
-            return [
-                Commit(
-                    repo=repo_url,
-                    sha=gh_commit.sha,
-                    date=gh_commit.commit.committer.date,
+        repo_info = self.parse_repo_url(repo_url)
+        match repo_info.platform:
+            case "github":
+                github = self._init_github(repo_url=repo_url, auth_code=auth_code)
+                data = github.compare(commit_from=commit_from, commit_to=commit_to)
+                return [
+                    Commit(
+                        repo=repo_url,
+                        sha=gh_commit.sha,
+                        date=gh_commit.commit.committer.date,
+                    )
+                    for gh_commit in data
+                ]
+            case "gitlab":
+                data = self._gitlab_instance.repository_compare(
+                    repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
                 )
-                for gh_commit in data
-            ]
-        # assume gitlab by default
-        else:
-            data = self._gitlab_instance.repository_compare(
-                repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
-            )
-            return [
-                Commit(
-                    repo=repo_url,
-                    sha=gl_commit["id"],
-                    date=datetime.fromisoformat(gl_commit["committed_date"]),
-                )
-                for gl_commit in data
-            ]
+                return [
+                    Commit(
+                        repo=repo_url,
+                        sha=gl_commit["id"],
+                        date=datetime.fromisoformat(gl_commit["committed_date"]),
+                    )
+                    for gl_commit in data
+                ]
+            case _:
+                raise ValueError(f"Unsupported repository URL: {repo_url}")
 
     def close_app_interface_mr(self, mr: ProjectMergeRequest, comment: str) -> None:
         if not self._allow_deleting_mrs: