qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/templating/renderer.py

@@ -0,0 +1,410 @@
+import json
+import logging
+import os
+import tempfile
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Self
+
+from ruamel import yaml
+
+from reconcile.gql_definitions.templating.template_collection import (
+    TemplateCollectionV1,
+    TemplateCollectionVariablesV1,
+    TemplateV1,
+    query,
+)
+from reconcile.templating.lib.merge_request_manager import (
+    MergeRequestManager,
+    MrData,
+    create_parser,
+)
+from reconcile.templating.lib.model import TemplateOutput, TemplateResult
+from reconcile.templating.lib.rendering import (
+    Renderer,
+    TemplateData,
+    create_renderer,
+)
+from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.github_orgs import get_github_orgs
+from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
+from reconcile.utils import gql
+from reconcile.utils.git import clone
+from reconcile.utils.gql import GqlApi, init_from_config
+from reconcile.utils.jinja2.utils import TemplateRenderOptions, process_jinja2_template
+from reconcile.utils.ruamel import create_ruamel_instance
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.vcs import VCS
+
+QONTRACT_INTEGRATION = "template-renderer"
+
+APP_INTERFACE_PATH_SEPERATOR = "/"
+
+
+def get_template_collections(
+    query_func: Callable | None = None, name: str | None = None
+) -> list[TemplateCollectionV1]:
+    variables = {}
+    if name:
+        variables["name"] = name
+    if not query_func:
+        query_func = gql.get_api().query
+    return query(query_func, variables=variables).template_collection_v1 or []
+
+
+class FilePersistence(ABC):
+    def __init__(self, dry_run: bool) -> None:
+        self.dry_run = dry_run
+        self.outputs: list[TemplateOutput] = []
+        self.result: TemplateResult | None = None
+
+    def __enter__(self) -> Self:
+        return self
+
+    @abstractmethod
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        pass
+
+    def write(self, output: TemplateOutput) -> None:
+        self.outputs.append(output)
+
+    @abstractmethod
+    def read(self, path: str) -> str | None:
+        pass
+
+    @staticmethod
+    def _read_local_file(path: str) -> str | None:
+        try:
+            with open(
+                path,
+                encoding="utf-8",
+            ) as f:
+                return f.read()
+        except FileNotFoundError:
+            logging.debug(f"File not found: {path}, need to create it")
+        return None
+
+
+class LocalFilePersistence(FilePersistence):
+    """
+    This class provides a simple file persistence implementation for local files.
+    """
+
+    def __init__(self, dry_run: bool, app_interface_data_path: str) -> None:
+        super().__init__(dry_run)
+        if not app_interface_data_path.endswith("/data"):
+            raise ValueError("app_interface_data_path should end with /data")
+        self.app_interface_data_path = app_interface_data_path
+
+    def read(self, path: str) -> str | None:
+        return self._read_local_file(join_path(self.app_interface_data_path, path))
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        if self.dry_run:
+            return
+        for output in self.outputs:
+            filepath = Path(join_path(self.app_interface_data_path, output.path))
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+            filepath.write_text(output.content, encoding="utf-8")
+
+
+class PersistenceTransaction(FilePersistence):
+    """
+    This class provides a context manager to make read/write operations
+    consistent. Reads/writes are beeing cached and writes are being delayed
+    until the context is left.
+    """
+
+    def __init__(self, persistence: FilePersistence) -> None:
+        super().__init__(persistence.dry_run)
+        self.persistence = persistence
+        self.content_cache: dict[str, str | None] = {}
+        self.output_cache: dict[str, TemplateOutput] = {}
+
+    def write(self, output: TemplateOutput) -> None:
+        self.content_cache[output.path] = output.content
+        self.output_cache[output.path] = output
+
+    def read(self, path: str) -> str | None:
+        if path not in self.content_cache:
+            self.content_cache[path] = self.persistence.read(path)
+        return self.content_cache[path]
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        if not self.dry_run and self.output_cache:
+            for output in self.output_cache.values():
+                self.persistence.write(output)
+
+
+class ClonedRepoGitlabPersistence(FilePersistence):
+    """
+    This class is used to persist the rendered templates in a cloned gitlab repo
+    Reads are from the local filesystem, writes are done via utils.VCS abstraction
+
+    One MR is created per template-collection. Auto-approval MRs are prefered.
+    """
+
+    def __init__(
+        self, dry_run: bool, local_path: str, vcs: VCS, mr_manager: MergeRequestManager
+    ):
+        super().__init__(dry_run)
+        self.local_path = join_path(local_path, "data")
+        self.vcs = vcs
+        self.mr_manager = mr_manager
+
+    def read(self, path: str) -> str | None:
+        return self._read_local_file(join_path(self.local_path, path))
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        if self.result is None:
+            raise ValueError("ClonedRepoGitlabPersistence.result not set!")
+        self.result.outputs = self.outputs
+
+        if self.dry_run or not self.outputs:
+            return
+
+        self.mr_manager.housekeeping()
+
+        if self.result.enable_auto_approval:
+            if auto_approved_outputs := [o for o in self.outputs if o.auto_approved]:
+                # create an MR with auto-approved templates only
+                self.mr_manager.create_merge_request(
+                    MrData(
+                        result=TemplateResult(
+                            collection=f"{self.result.collection}-auto-approved",
+                            enable_auto_approval=self.result.enable_auto_approval,
+                            labels=self.result.labels,
+                            outputs=auto_approved_outputs,
+                        ),
+                        auto_approved=True,
+                    )
+                )
+            if not_auto_approved_outputs := [
+                o for o in self.outputs if not o.auto_approved
+            ]:
+                # create an MR with not auto-approved templates only
+                self.mr_manager.create_merge_request(
+                    MrData(
+                        result=TemplateResult(
+                            collection=f"{self.result.collection}-not-auto-approved",
+                            enable_auto_approval=self.result.enable_auto_approval,
+                            labels=self.result.labels,
+                            outputs=not_auto_approved_outputs,
+                        ),
+                        auto_approved=False,
+                    )
+                )
+        else:
+            # create an MR with all templates
+            self.mr_manager.create_merge_request(
+                MrData(result=self.result, auto_approved=False)
+            )
+
+
+def unpack_static_variables(
+    collection_variables: TemplateCollectionVariablesV1,
+    each: dict[str, Any],
+) -> dict:
+    return {
+        k: json.loads(process_jinja2_template(body=json.dumps(v), vars={"each": each}))
+        for k, v in (collection_variables.static or {}).items()
+    }
+
+
+def unpack_dynamic_variables(
+    collection_variables: TemplateCollectionVariablesV1,
+    each: dict[str, Any],
+    gql: gql.GqlApi,
+) -> dict[str, dict[str, Any]]:
+    static = collection_variables.static or {}
+    dynamic: dict[str, dict[str, Any]] = {}
+    for dv in collection_variables.dynamic or []:
+        query = process_jinja2_template(
+            body=dv.query,
+            vars={"static": static, "dynamic": dynamic, "each": each},
+        )
+        dynamic[dv.name] = gql.query(query) or {}
+    return dynamic
+
+
+class TemplateRendererIntegrationParams(PydanticRunParams):
+    clone_repo: bool = False
+    app_interface_data_path: str | None
+    template_collection_name: str | None
+
+
+def join_path(base: str, sub: str) -> str:
+    return os.path.join(base, sub.lstrip(APP_INTERFACE_PATH_SEPERATOR))
+
+
+class TemplateRendererIntegration(QontractReconcileIntegration):
+    def __init__(self, params: TemplateRendererIntegrationParams) -> None:
+        super().__init__(params)
+
+    @staticmethod
+    def _create_renderer(
+        template: TemplateV1,
+        variables: dict,
+        secret_reader: SecretReaderBase | None = None,
+    ) -> Renderer:
+        return create_renderer(
+            template,
+            TemplateData(
+                variables=variables,
+            ),
+            secret_reader=secret_reader,
+            template_render_options=TemplateRenderOptions.create(
+                trim_blocks=template.template_render_options.trim_blocks
+                if template.template_render_options
+                else None,
+                lstrip_blocks=template.template_render_options.lstrip_blocks
+                if template.template_render_options
+                else None,
+                keep_trailing_newline=template.template_render_options.keep_trailing_newline
+                if template.template_render_options
+                else None,
+            ),
+        )
+
+    def process_template(
+        self,
+        template: TemplateV1,
+        variables: dict,
+        persistence: FilePersistence,
+        ruaml_instance: yaml.YAML,
+    ) -> TemplateOutput | None:
+        r = TemplateRendererIntegration._create_renderer(
+            template, variables, secret_reader=self.secret_reader
+        )
+        target_path = r.render_target_path()
+        current_file = persistence.read(target_path)
+        if current_file is None and template.patch:
+            raise ValueError(f"Can not patch non-existing file {target_path}")
+
+        if current_file:
+            r.data.current = ruaml_instance.load(current_file)
+
+        if r.render_condition():
+            output = r.render_output()
+
+            if current_file != output:
+                logging.info(
+                    f"diff for template {template.name} in target path {target_path}"
+                )
+                return TemplateOutput(
+                    path=target_path,
+                    content=output,
+                    is_new=current_file is None,
+                    auto_approved=template.auto_approved or False,
+                )
+        return None
+
+    def reconcile_template_collection(
+        self,
+        collection: TemplateCollectionV1,
+        gql_api: GqlApi,
+        persistence: FilePersistence,
+        ruamel_instance: yaml.YAML,
+        each: dict[str, Any],
+    ) -> None:
+        variables = {}
+        if collection.variables:
+            variables = {
+                "dynamic": unpack_dynamic_variables(
+                    collection.variables, each, gql_api
+                ),
+                "static": unpack_static_variables(collection.variables, each),
+            }
+        with PersistenceTransaction(persistence) as persistence_transaction:
+            for template in collection.templates:
+                if output := self.process_template(
+                    template, variables, persistence_transaction, ruamel_instance
+                ):
+                    persistence_transaction.write(output)
+
+    def reconcile(
+        self, persistence: FilePersistence, ruamel_instance: yaml.YAML
+    ) -> None:
+        gql_api = init_from_config(validate_schemas=False)
+        for collection in get_template_collections(
+            name=self.params.template_collection_name
+        ):
+            for_each_items: list[dict[str, Any]] = [{}]
+            if collection.for_each and collection.for_each.items:
+                for_each_items = collection.for_each.items
+            result = TemplateResult(
+                collection=collection.name,
+                enable_auto_approval=collection.enable_auto_approval or False,
+                labels=collection.additional_mr_labels or [],
+            )
+            with persistence as p:
+                p.result = result
+                p.outputs = []
+                for item in for_each_items:
+                    self.reconcile_template_collection(
+                        collection=collection,
+                        gql_api=gql_api,
+                        persistence=p,
+                        ruamel_instance=ruamel_instance,
+                        each=item,
+                    )
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def run(self, dry_run: bool) -> None:
+        persistence: FilePersistence
+        ruaml_instance = create_ruamel_instance(explicit_start=True)
+
+        if not self.params.clone_repo and self.params.app_interface_data_path:
+            persistence = LocalFilePersistence(
+                dry_run=dry_run,
+                app_interface_data_path=self.params.app_interface_data_path,
+            )
+            self.reconcile(persistence, ruaml_instance)
+
+        elif self.params.clone_repo:
+            gitlab_instances = get_gitlab_instances()
+            vcs = VCS(
+                secret_reader=self.secret_reader,
+                github_orgs=get_github_orgs(),
+                gitlab_instances=gitlab_instances,
+                app_interface_repo_url=get_app_interface_repo_url(),
+                dry_run=dry_run,
+                allow_deleting_mrs=True,
+                allow_opening_mrs=True,
+            )
+            merge_request_manager = MergeRequestManager(
+                vcs=vcs,
+                parser=create_parser(),
+            )
+            url = get_app_interface_repo_url()
+
+            ssl_verify = next(
+                g.ssl_verify for g in gitlab_instances if url.startswith(g.url)
+            )
+
+            with tempfile.TemporaryDirectory(
+                prefix=f"{QONTRACT_INTEGRATION}-",
+            ) as temp_dir:
+                logging.debug(f"Cloning {url} to {temp_dir}")
+                clone(url, temp_dir, depth=1, verify=ssl_verify)
+                persistence = ClonedRepoGitlabPersistence(
+                    dry_run=dry_run,
+                    local_path=temp_dir,
+                    vcs=vcs,
+                    mr_manager=merge_request_manager,
+                )
+                self.reconcile(persistence, ruaml_instance)
+
+        else:
+            raise ValueError("App-interface-data-path must be set")

reconcile/templating/validator.py

@@ -0,0 +1,153 @@
+import logging
+from collections.abc import Callable
+from difflib import context_diff
+
+from pydantic import BaseModel
+from ruamel import yaml
+
+from reconcile.gql_definitions.templating.templates import (
+    TemplateTestV1,
+    TemplateV1,
+    query,
+)
+from reconcile.templating.lib.rendering import Renderer, TemplateData, create_renderer
+from reconcile.utils import gql
+from reconcile.utils.jinja2.utils import TemplateRenderOptions
+from reconcile.utils.ruamel import create_ruamel_instance
+from reconcile.utils.runtime.integration import (
+    QontractReconcileIntegration,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+
+QONTRACT_INTEGRATION = "template-validator"
+
+
+def get_templates(
+    query_func: Callable | None = None,
+) -> list[TemplateV1]:
+    if not query_func:
+        query_func = gql.get_api().query
+    return query(query_func).template_v1 or []
+
+
+class TemplateDiff(BaseModel):
+    template: str
+    test: str
+    diff: str
+
+
+class TemplateValidatorIntegration(QontractReconcileIntegration):
+    @staticmethod
+    def _create_renderer(
+        template: TemplateV1,
+        template_test: TemplateTestV1,
+        ruaml_instance: yaml.YAML,
+        secret_reader: SecretReaderBase | None = None,
+    ) -> Renderer:
+        return create_renderer(
+            template,
+            TemplateData(
+                variables=template_test.variables or {},
+                current=ruaml_instance.load(template_test.current or ""),
+            ),
+            secret_reader=secret_reader,
+            template_render_options=TemplateRenderOptions.create(
+                trim_blocks=template.template_render_options.trim_blocks
+                if template.template_render_options
+                else None,
+                lstrip_blocks=template.template_render_options.lstrip_blocks
+                if template.template_render_options
+                else None,
+                keep_trailing_newline=template.template_render_options.keep_trailing_newline
+                if template.template_render_options
+                else None,
+            ),
+        )
+
+    @staticmethod
+    def validate_template(
+        template: TemplateV1,
+        template_test: TemplateTestV1,
+        ruaml_instance: yaml.YAML,
+        secret_reader: SecretReaderBase | None = None,
+    ) -> list[TemplateDiff]:
+        diffs: list[TemplateDiff] = []
+
+        r = TemplateValidatorIntegration._create_renderer(
+            template, template_test, ruaml_instance, secret_reader=secret_reader
+        )
+
+        # Check target path
+        if template_test.expected_target_path:
+            rendered_target_path = r.render_target_path().strip()
+            if rendered_target_path != template_test.expected_target_path.strip():
+                diffs.append(
+                    TemplateDiff(
+                        template=template.name,
+                        test=template_test.name,
+                        diff=f"Target path mismatch, got: {rendered_target_path}, expected: {template_test.expected_target_path}",
+                    )
+                )
+
+        # Check condition
+        should_render = r.render_condition()
+        expected_to_render = (
+            template_test.expected_to_render
+            if template_test.expected_to_render is not None
+            else True
+        )
+        if expected_to_render != should_render:
+            diffs.append(
+                TemplateDiff(
+                    template=template.name,
+                    test=template_test.name,
+                    diff=f"Condition mismatch for expectedToRender, got: {should_render}, expected: {expected_to_render}",
+                )
+            )
+
+        # Check template output
+        if should_render:
+            output = r.render_output()
+            diff = list(
+                context_diff(
+                    output.splitlines(keepends=True),
+                    template_test.expected_output.splitlines(keepends=True),
+                )
+            )
+            if diff:
+                diffs.append(
+                    TemplateDiff(
+                        template=template.name,
+                        test=template_test.name,
+                        diff="".join(diff),
+                    )
+                )
+
+        return diffs
+
+    def run(self, dry_run: bool) -> None:
+        diffs: list[TemplateDiff] = []
+        ruaml_instance = create_ruamel_instance(explicit_start=True)
+
+        for template in get_templates():
+            for test in template.template_test:
+                logging.info(f"Running test {test.name} for template {template.name}")
+                diffs.extend(
+                    self.validate_template(
+                        template, test, ruaml_instance, self.secret_reader
+                    )
+                )
+
+        if diffs:
+            for diff in diffs:
+                logging.error(f"template: {diff.template}, test: {diff.test}")
+                # This log should never be added except for local debugging.
+                # Credentials could be leaked, i.e. creating an MR with a diff,
+                # using a template, that uses the vault function.
+                # Use template-validator CLI instead.
+                # logging.debug(diff.diff)
+            raise ValueError("Template validation failed")
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION

reconcile/terraform_aws_route53.py

@@ -4,10 +4,7 @@ from collections.abc import (
     Iterable,
     Mapping,
 )
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 from reconcile import queries
 from reconcile.status import ExitCodes
@@ -87,7 +84,7 @@ def build_desired_state(
                 # get_a_record is used here to validate the record and reused later
                 target_cluster_elb_value = dnsutils.get_a_records(target_cluster_elb)
 
-                if target_cluster_elb is None or target_cluster_elb == "":
+                if not target_cluster_elb:
                     msg = (
                         f"{zone_name}: field `_target_cluster` for record "
                         f"{record_name} of type {record_type} points to a "
@@ -150,9 +147,9 @@ def build_desired_state(
                     sys.exit(ExitCodes.ERROR)
                 tf_zone_spec = tf_zone_specs[0]
                 tf_zone_account_name = tf_zone_spec.provisioner_name
-                zone_account = [
+                zone_account = next(
                     a for a in all_accounts if a["name"] == tf_zone_account_name
-                ][0]
+                )
                 tf_zone_region = (
                     tf_zone_spec.resource.get("region")
                     or zone_account["resourcesDefaultRegion"]
@@ -206,21 +203,27 @@ def build_desired_state(
 @defer
 def run(
     dry_run: bool = False,
-    print_to_file: Optional[str] = None,
+    print_to_file: str | None = None,
     enable_deletion: bool = True,
     thread_pool_size: int = 10,
-    account_name: Optional[str] = None,
+    account_name: str | None = None,
     defer=None,
 ):
     settings = queries.get_app_interface_settings()
     zones = queries.get_dns_zones(account_name=account_name)
 
     all_accounts = queries.get_aws_accounts(terraform_state=True)
-    participating_account_names = [z["account"]["name"] for z in zones]
+    participating_account_names = {z["account"]["name"] for z in zones}
     participating_accounts = [
         a for a in all_accounts if a["name"] in participating_account_names
     ]
 
+    if not participating_accounts:
+        logging.warning(
+            f"No participating AWS accounts found, consider disabling this integration, account name: {account_name}"
+        )
+        return
+
     ts = Terrascript(
         QONTRACT_INTEGRATION,
         "",