qontract-reconcile 0.10.2.dev180__tar.gz → 0.10.2.dev182__tar.gz

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev180
+Version: 0.10.2.dev182
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/dashdotdb_dora.py

@@ -34,6 +34,7 @@ from reconcile.typed_queries.saas_files import get_saas_files
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 QONTRACT_INTEGRATION = "dashdotdb-dora"
 
@@ -421,7 +422,7 @@ class DashdotdbDORA(DashdotdbBase):
             return rc, []
 
         LOG.info("Fetching commits %s", rc)
-        if rc.repo_url.startswith("https://github.com"):
+        if rc.repo_url.startswith(GITHUB_BASE_URL):
             try:
                 commits = self._github_compare_commits(rc)
             except GithubException as e:
@@ -476,8 +477,7 @@ class DashdotdbDORA(DashdotdbBase):
         if not rc.repo_url:
             return []
 
-        prefix = "https://github.com/"
-        repo = rc.repo_url[rc.repo_url.startswith(prefix) and len(prefix) :]
+        repo = rc.repo_url.removeprefix(GITHUB_BASE_URL).rstrip("/")
 
         return [
             Commit(rc.repo_url, commit.sha, commit.commit.committer.date)

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/github_repo_permissions_validator.py

@@ -11,6 +11,7 @@ from reconcile.jenkins_job_builder import init_jjb
 from reconcile.utils.jjb_client import JJB
 from reconcile.utils.secret_reader import SecretReader
 from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 QONTRACT_INTEGRATION = "github-repo-permissions-validator"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
@@ -45,7 +46,7 @@ def run(dry_run: bool, instance_name: str) -> None:
     error = False
     for job in pr_check_jobs:
         repo_url = jjb.get_repo_url(job)
-        repo_name = repo_url.rstrip("/").replace("https://github.com/", "")
+        repo_name = repo_url.removeprefix(GITHUB_BASE_URL).rstrip("/")
         repo = gh.get_repo(repo_name)
         permissions = repo.permissions
         if not permissions.push and repo_url not in invitations:

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/integrations_manager.py

@@ -47,6 +47,7 @@ from reconcile.utils.runtime.sharding import (
     StaticShardingStrategy,
 )
 from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 QONTRACT_INTEGRATION = "integrations-manager"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
@@ -58,8 +59,7 @@ INTEGRATION_UPSTREAM_REPOS_PARAM = "INTEGRATION_UPSTREAM_REPOS"
 
 
 def get_image_tag_from_ref(ref: str, upstream: str) -> str:
-    gh_prefix = "https://github.com/"
-    upstream = upstream.removeprefix(gh_prefix)
+    upstream = upstream.removeprefix(GITHUB_BASE_URL)
     settings = queries.get_app_interface_settings()
     gh_token = get_default_config()["token"]
     github = Github(gh_token, base_url=GH_BASE_URL)

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/saas_auto_promotions_manager/integration.py

@@ -94,6 +94,7 @@ class SaasAutoPromotionsManager:
 
 def init_external_dependencies(
     dry_run: bool,
+    thread_pool_size: int,
     env_name: str | None = None,
     app_name: str | None = None,
 ) -> tuple[
@@ -139,7 +140,11 @@ def init_external_dependencies(
         renderer=Renderer(),
     )
     saas_files = get_saas_files(env_name=env_name, app_name=app_name)
-    saas_inventory = SaasFilesInventory(saas_files=saas_files)
+    saas_inventory = SaasFilesInventory(
+        saas_files=saas_files,
+        secret_reader=secret_reader,
+        thread_pool_size=thread_pool_size,
+    )
     saas_deploy_state = init_state(
         integration=OPENSHIFT_SAAS_DEPLOY, secret_reader=secret_reader
     )
@@ -181,7 +186,10 @@ def run(
         saas_deploy_state,
         sapm_state,
     ) = init_external_dependencies(
-        dry_run=dry_run, env_name=env_name, app_name=app_name
+        dry_run=dry_run,
+        env_name=env_name,
+        app_name=app_name,
+        thread_pool_size=thread_pool_size,
     )
     if defer:
         defer(vcs.cleanup)

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/saas_auto_promotions_manager/subscriber.py

@@ -13,6 +13,7 @@ from reconcile.saas_auto_promotions_manager.publisher import (
     DeploymentInfo,
     Publisher,
 )
+from reconcile.utils.slo_document_manager import SLODocumentManager
 
 CONTENT_HASH_LENGTH = 32
 
@@ -47,7 +48,9 @@ class Subscriber:
         uid: str,
         soak_days: int,
         blocked_versions: set[str],
+        hotfix_versions: set[str],
         schedule: str,
+        slo_document_manager: SLODocumentManager | None = None,
     ):
         self.saas_name = saas_name
         self.template_name = template_name
@@ -64,6 +67,8 @@ class Subscriber:
         self._content_hash = ""
         self._use_target_config_hash = use_target_config_hash
         self._blocked_versions = blocked_versions
+        self._hotfix_versions = hotfix_versions
+        self.slo_document_manager = slo_document_manager
 
     def has_diff(self) -> bool:
         current_hashes = {
@@ -192,6 +197,10 @@ class Subscriber:
             return
 
         desired_ref = next(iter(publisher_refs))
+        # validate slo gatekeeping
+        if self._has_breached_slos(desired_ref=desired_ref):
+            return
+
         if desired_ref in self._blocked_versions:
             logging.info(
                 "Subscriber at path %s promotion stopped because of blocked ref: %s",
@@ -203,6 +212,25 @@ class Subscriber:
         # Passed all gates -> lets promote desired ref
         self.desired_ref = desired_ref
 
+    def _has_breached_slos(self, desired_ref: str) -> bool:
+        if self.slo_document_manager and desired_ref not in self._hotfix_versions:
+            breached_slos = self.slo_document_manager.get_breached_slos()
+            if breached_slos:
+                logging.info(
+                    "Subscriber at path %s promotion stopped because following breached SLOs",
+                    self.target_file_path,
+                )
+                for slo in breached_slos:
+                    logging.info(
+                        "SLO:%s of document %s is breached. Current value:%f Expected:%f ",
+                        slo.slo.name,
+                        slo.slo_document_name,
+                        slo.current_slo_value,
+                        slo.slo.slo_target,
+                    )
+                return True
+        return False
+
     def _compute_desired_config_hashes(self) -> None:
         """
         Compute the desired config hashes for this subscriber.

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/saas_auto_promotions_manager/utils/saas_files_inventory.py

@@ -1,4 +1,5 @@
 import logging
+from collections import defaultdict
 from collections.abc import Iterable
 
 from reconcile.gql_definitions.common.saas_files import ParentSaasPromotionV1
@@ -8,7 +9,9 @@ from reconcile.saas_auto_promotions_manager.subscriber import (
     ConfigHash,
     Subscriber,
 )
-from reconcile.typed_queries.saas_files import SaasFile
+from reconcile.typed_queries.saas_files import SaasFile, SaasResourceTemplateTarget
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.slo_document_manager import SLODocumentManager
 
 
 class SaasFileInventoryError(Exception):
@@ -25,8 +28,15 @@ class SaasFilesInventory:
     This basically spans a directed graph, with subscribers as the root.
     """
 
-    def __init__(self, saas_files: Iterable[SaasFile]):
+    def __init__(
+        self,
+        saas_files: Iterable[SaasFile],
+        secret_reader: SecretReaderBase,
+        thread_pool_size: int,
+    ):
         self._saas_files = saas_files
+        self.secret_reader = secret_reader
+        self.thread_pool_size = thread_pool_size
         self._channels_by_name: dict[str, Channel] = {}
         self.subscribers: list[Subscriber] = []
         self.publishers: list[Publisher] = []
@@ -86,10 +96,13 @@ class SaasFilesInventory:
 
     def _assemble_subscribers_with_auto_promotions(self) -> None:
         for saas_file in self._saas_files:
-            blocked_versions: dict[str, set[str]] = {}
+            blocked_versions: dict[str, set[str]] = defaultdict(set[str])
+            hotfix_versions: dict[str, set[str]] = defaultdict(set[str])
             for code_component in saas_file.app.code_components or []:
                 for version in code_component.blocked_versions or []:
-                    blocked_versions.setdefault(code_component.url, set()).add(version)
+                    blocked_versions[code_component.url].add(version)
+                for hf_version in code_component.hotfix_versions or []:
+                    hotfix_versions[code_component.url].add(hf_version)
             for resource_template in saas_file.resource_templates:
                 for target in resource_template.targets:
                     file_path = target.path or saas_file.path
@@ -101,6 +114,7 @@ class SaasFilesInventory:
                         continue
                     soak_days = target.promotion.soak_days or 0
                     schedule = target.promotion.schedule or "* * * * *"
+
                     subscriber = Subscriber(
                         uid=target.uid(
                             parent_saas_file_name=saas_file.name,
@@ -112,10 +126,10 @@ class SaasFilesInventory:
                         ref=target.ref,
                         target_namespace=target.namespace,
                         soak_days=soak_days,
+                        slo_document_manager=self._build_slo_document_manager(target),
                         schedule=schedule,
-                        blocked_versions=blocked_versions.get(
-                            resource_template.url, set()
-                        ),
+                        hotfix_versions=hotfix_versions[resource_template.url],
+                        blocked_versions=blocked_versions[resource_template.url],
                         use_target_config_hash=bool(
                             target.promotion.redeploy_on_publisher_config_change
                         ),
@@ -158,6 +172,17 @@ class SaasFilesInventory:
                             self._channels_by_name[subscribe_channel]
                         )
 
+    def _build_slo_document_manager(
+        self, target: SaasResourceTemplateTarget
+    ) -> SLODocumentManager | None:
+        if target.slos:
+            return SLODocumentManager(
+                slo_documents=target.slos,
+                secret_reader=self.secret_reader,
+                thread_pool_size=self.thread_pool_size,
+            )
+        return None
+
     def _remove_unsupported(self) -> None:
         """
         Lets remove subscribers from which we know we do not support them and log an error.

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/service_dependencies.py

@@ -16,6 +16,7 @@ from reconcile.gql_definitions.service_dependencies.service_dependencies import
     SaasResourceTemplateV2,
 )
 from reconcile.utils import gql
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 QONTRACT_INTEGRATION = "service-dependencies"
 
@@ -35,7 +36,9 @@ def get_desired_dependency_names(
         gitlab_urls = [cc for cc in code_components if "gitlab" in cc.url]
         if gitlab_urls:
             required_dep_names.update(get_dependency_names(dependency_map, "gitlab"))
-        github_urls = [cc for cc in code_components if "github.com" in cc.url]
+        github_urls = [
+            cc for cc in code_components if cc.url.startswith(GITHUB_BASE_URL)
+        ]
         if github_urls:
             required_dep_names.update(get_dependency_names(dependency_map, "github"))
 

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/typed_queries/saas_files.py

@@ -39,6 +39,7 @@ from reconcile.gql_definitions.common.saasherder_settings import AppInterfaceSet
 from reconcile.gql_definitions.common.saasherder_settings import (
     query as saasherder_settings_query,
 )
+from reconcile.gql_definitions.fragments.saas_slo_document import SLODocument
 from reconcile.gql_definitions.fragments.saas_target_namespace import (
     SaasTargetNamespace,
 )
@@ -63,6 +64,7 @@ class SaasResourceTemplateTarget(ConfiguredBaseModel):
     secret_parameters: (
         list[SaasResourceTemplateTargetV2_SaasSecretParametersV1] | None
     ) = Field(..., alias="secretParameters")
+    slos: list[SLODocument] | None = Field(..., alias="slos")
     upstream: SaasResourceTemplateTargetUpstreamV1 | None = Field(..., alias="upstream")
     images: list[SaasResourceTemplateTargetImageV1] | None = Field(..., alias="images")
     disable: bool | None = Field(..., alias="disable")

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/utils/gitlab_api.py

@@ -1,6 +1,8 @@
+import io
 import logging
 import os
 import re
+import tarfile
 from collections.abc import (
     Iterable,
     Mapping,
@@ -828,3 +830,29 @@ class GitLabApi:
             list[PersonalAccessToken],
             self.gl.personal_access_tokens.list(get_all=True),
         )
+
+    @staticmethod
+    def get_directory_contents(
+        project: Project,
+        ref: str | None = None,
+        path: str | None = None,
+    ) -> dict[str, bytes]:
+        """
+        Get the contents of a directory in a project.
+
+        :param project: The project to get the contents from.
+        :param ref: The commit SHA to download. A tag, branch reference, or SHA can be used. If not specified, defaults to the tip of the default branch.
+        :param path: The subpath of the repository to download. If an empty string, defaults to the whole repository.
+        :return: A dictionary with the file path as keys and the file content bytes as values.
+        """
+        archive = project.repository_archive(format="tar.gz", sha=ref, path=path)
+        tar_bytes = io.BytesIO(archive)
+        with tarfile.open(fileobj=tar_bytes, mode="r:gz") as tar:
+            return {
+                file_path: file.read()
+                for member in tar.getmembers()
+                if member.isfile()
+                # skip leading prefix xxx/
+                and (file_path := member.name.split("/", 1)[-1])
+                and (file := tar.extractfile(member))
+            }

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/utils/jjb_client.py

@@ -25,6 +25,7 @@ from sretoolbox.utils import retry
 
 from reconcile.utils import throughput
 from reconcile.utils.helpers import toggle_logger
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
 
@@ -290,7 +291,7 @@ class JJB:  # pylint: disable=too-many-public-methods
             for job in jobs:
                 try:
                     project_url_raw = job["properties"][0]["github"]["url"]
-                    if "https://github.com" in project_url_raw:
+                    if project_url_raw.startswith(GITHUB_BASE_URL):
                         continue
                     if str(job.get("disabled")).lower() == "true":
                         continue

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/utils/saasherder/saasherder.py

@@ -83,6 +83,7 @@ from reconcile.utils.saasherder.models import (
 )
 from reconcile.utils.secret_reader import SecretReaderBase
 from reconcile.utils.state import State
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 TARGET_CONFIG_HASH = "target_config_hash"
 
@@ -742,7 +743,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         [url, sha] = trigger_reason.split(" ")[0].split("/commit/")
         repo_name = urlparse(url).path.strip("/")
         file_name = f"{repo_name.replace('/', '-')}-{sha}.tar.gz"
-        if "github" in url:
+        if url.startswith(GITHUB_BASE_URL):
             github = self._initiate_github(saas_file, base_url="https://api.github.com")
             repo = github.get_repo(repo_name)
             # get_archive_link get redirect url form header, it does not work with github-mirror
@@ -760,8 +761,8 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     ) -> tuple[Any, str]:
         commit_sha = self._get_commit_sha(url, ref, github)
 
-        if "github" in url:
-            repo_name = url.rstrip("/").replace("https://github.com/", "")
+        if url.startswith(GITHUB_BASE_URL):
+            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
             repo = github.get_repo(repo_name)
             content = self._get_file_contents_github(repo, path, commit_sha)
         elif "gitlab" in url:
@@ -781,8 +782,8 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     ) -> tuple[list[Any], str]:
         commit_sha = self._get_commit_sha(url, ref, github)
         resources: list[Any] = []
-        if "github" in url:
-            repo_name = url.rstrip("/").replace("https://github.com/", "")
+        if url.startswith(GITHUB_BASE_URL):
+            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
             repo = github.get_repo(repo_name)
             directory = repo.get_contents(path, commit_sha)
             if isinstance(directory, ContentFile):
@@ -798,17 +799,14 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
             if not self.gitlab:
                 raise Exception("gitlab is not initialized")
             project = self.gitlab.get_project(url)
-            for item in self.gitlab.get_repository_tree(
-                project=project,
-                path=path.lstrip("/"),
+            dir_contents = self.gitlab.get_directory_contents(
+                project,
                 ref=commit_sha,
-                recursive=False,
-            ):
-                file_contents = project.files.get(
-                    file_path=item["path"], ref=commit_sha
-                )
-                resource = yaml.safe_load(file_contents.decode())
-                resources.append(resource)
+                path=path,
+            )
+            for content in dir_contents.values():
+                result_resources = yaml.safe_load_all(content)
+                resources.extend(result_resources)
         else:
             raise Exception(f"Only GitHub and GitLab are supported: {url}")
 
@@ -817,8 +815,8 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     @retry()
     def _get_commit_sha(self, url: str, ref: str, github: Github) -> str:
         commit_sha = ""
-        if "github" in url:
-            repo_name = url.rstrip("/").replace("https://github.com/", "")
+        if url.startswith(GITHUB_BASE_URL):
+            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
             repo = github.get_repo(repo_name)
             commit = repo.get_commit(sha=ref)
             commit_sha = commit.sha

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/utils/terrascript_aws_client.py

@@ -186,6 +186,7 @@ from reconcile.utils.password_validator import (
 )
 from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
 from reconcile.utils.terraform import safe_resource_id
+from reconcile.utils.vcs import GITHUB_BASE_URL
 
 GH_BASE_URL = os.environ.get("GITHUB_API", "https://api.github.com")
 LOGTOES_RELEASE = "repos/app-sre/logs-to-elasticsearch-lambda/releases/latest"
@@ -5685,9 +5686,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             return ref
 
         # get commit_sha from branch
-        if "github" in url:
+        if url.startswith(GITHUB_BASE_URL):
             github = self.init_github()
-            repo_name = url.rstrip("/").replace("https://github.com/", "")
+            repo_name = url.removeprefix(GITHUB_BASE_URL).rstrip("/")
             repo = github.get_repo(repo_name)
             commit = repo.get_commit(sha=ref)
             return commit.sha

{qontract_reconcile-0.10.2.dev180 → qontract_reconcile-0.10.2.dev182}/reconcile/utils/vcs.py

@@ -22,6 +22,8 @@ from reconcile.utils.secret_reader import (
     SecretReaderBase,
 )
 
+GITHUB_BASE_URL = "https://github.com/"
+
 
 class MRCheckStatus(Enum):
     NONE = 0
@@ -158,7 +160,7 @@ class VCS:
     ) -> str:
         if bool(self._is_commit_sha_regex.search(ref)):
             return ref
-        if repo_url.startswith("https://github.com/"):
+        if repo_url.startswith(GITHUB_BASE_URL):
             github = self._init_github(repo_url=repo_url, auth_code=auth_code)
             return github.get_commit_sha(ref=ref)
         # assume gitlab by default
@@ -175,7 +177,7 @@ class VCS:
         Return a list of commits between two commits.
         Note, that the commit_to is included in the result list, whereas commit_from is not included.
         """
-        if repo_url.startswith("https://github.com/"):
+        if repo_url.startswith(GITHUB_BASE_URL):
             github = self._init_github(repo_url=repo_url, auth_code=auth_code)
             data = github.compare(commit_from=commit_from, commit_to=commit_to)
             return [