qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/templates/rosa-classic-cluster-creation.sh.j2

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+rosa init
+rosa create ocm-role --admin -y -m auto
+rosa create account-roles --classic -y -m auto
+rosa create user-role -y -m auto
+
+# OIDC config
+{% if cluster.spec.oidc_endpoint_url %}
+OIDC_CONFIG_ID="{{ (cluster.spec.oidc_endpoint_url | split("/")) | last }}"
+{% else %}
+OIDC_CONFIG_ID=$(rosa list oidc-provider -o json | jq '.[0].arn // "/" | split("/") | .[-1]' -r)
+if [[ -z "${OIDC_CONFIG_ID}" ]]; then
+    rosa create oidc-config -m auto -y
+    OIDC_CONFIG_ID=$(rosa list oidc-provider -o json | jq '.[0].arn // "/" | split("/") | .[-1]' -r)
+else
+    echo "reuse existing OIDC config ${OIDC_CONFIG_ID}"
+fi
+{% endif %}
+
+# operator roles
+INSTALLER_ROLE_ARN=$(rosa list account-roles --region us-east-1 -o json | jq '.[] | select(.RoleType == "Installer") | .RoleARN' -r)
+rosa create operator-roles --prefix {{ cluster_name }} --oidc-config-id ${OIDC_CONFIG_ID} --installer-role-arn ${INSTALLER_ROLE_ARN} -m auto -y
+
+# cluster creation
+rosa create cluster -y --cluster-name={{ cluster_name }} \
+    {% if dry_run -%}
+    --dry-run \
+    {% endif -%}
+    --sts \
+    {% if cluster.spec.private -%}
+    --private \
+    {% endif -%}
+    {% if cluster.spec.private -%}
+    --private-link \
+    {% endif -%}
+    {% if cluster.spec.multi_az -%}
+    --multi-az \
+    {% endif -%}
+    --oidc-config-id ${OIDC_CONFIG_ID} \
+    --operator-roles-prefix {{ cluster_name }} \
+    --subnet-ids {{ cluster.spec.subnet_ids | join(",") }} \
+    --region {{ cluster.spec.region }} \
+    --version {{ cluster.spec.initial_version }} \
+    --machine-cidr {{ cluster.network.vpc }} \
+    --service-cidr {{ cluster.network.service }} \
+    --pod-cidr {{ cluster.network.pod }} \
+    --host-prefix 23 \
+    --replicas {{ cluster.machine_pools | length }} \
+    --compute-machine-type {{ cluster.machine_pools[0].instance_type }} \
+    {% if cluster.spec.disable_user_workload_monitoring -%}
+    --disable-workload-monitoring \
+    {% endif -%}
+    {% if cluster.spec.provision_shard_id -%}
+    --properties provision_shard_id:{{ cluster.spec.provision_shard_id }} \
+    {% endif -%}
+    --channel-group {{ cluster.spec.channel }}

reconcile/templates/rosa-hcp-cluster-creation.sh.j2

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+rosa init
+rosa create ocm-role --admin -y -m auto
+rosa create account-roles --hosted-cp -y -m auto
+rosa create user-role -y -m auto
+
+# OIDC config
+{% if cluster.spec.oidc_endpoint_url %}
+OIDC_CONFIG_ID="{{ (cluster.spec.oidc_endpoint_url | split("/")) | last }}"
+{% else %}
+OIDC_CONFIG_ID=$(rosa list oidc-provider -o json | jq '.[0].arn // "/" | split("/") | .[-1]' -r)
+if [[ -z "${OIDC_CONFIG_ID}" ]]; then
+    rosa create oidc-config -m auto -y
+    OIDC_CONFIG_ID=$(rosa list oidc-provider -o json | jq '.[0].arn // "/" | split("/") | .[-1]' -r)
+else
+    echo "reuse existing OIDC config ${OIDC_CONFIG_ID}"
+fi
+{% endif %}
+
+# operator roles
+INSTALLER_ROLE_ARN=$(rosa list account-roles --region us-east-1 -o json | jq '.[] | select(.RoleType == "Installer") | .RoleARN' -r)
+rosa create operator-roles --prefix {{ cluster_name }} --oidc-config-id ${OIDC_CONFIG_ID} --hosted-cp --installer-role-arn ${INSTALLER_ROLE_ARN} -m auto -y
+
+# cluster creation
+{% if cluster.spec.account.billing_account_id %}
+BILLING_ACCOUNT_ID="{{ cluster.spec.account.billing_account_id }}"
+{% else %}
+BILLING_ACCOUNT_ID=$(aws organizations describe-organization | jq .Organization.MasterAccountId -r)
+{% endif %}
+rosa create cluster --cluster-name={{ cluster_name }} \
+    --billing-account ${BILLING_ACCOUNT_ID} \
+    {% if dry_run -%}
+    --dry-run \
+    {% endif -%}
+    --sts \
+    --hosted-cp \
+    --oidc-config-id ${OIDC_CONFIG_ID} \
+    --operator-roles-prefix {{ cluster_name }} \
+    --subnet-ids {{ cluster.spec.subnet_ids | join(",") }} \
+    --region {{ cluster.spec.region }} \
+    --version {{ cluster.spec.initial_version }} \
+    --machine-cidr {{ cluster.network.vpc }} \
+    --service-cidr {{ cluster.network.service }} \
+    --pod-cidr {{ cluster.network.pod }} \
+    --host-prefix 23 \
+    --replicas {{ cluster.machine_pools | length }} \
+    --compute-machine-type {{ cluster.machine_pools[0].instance_type }} \
+    {% if cluster.spec.private -%}
+    --private \
+    {% endif -%}
+    {% if cluster.spec.disable_user_workload_monitoring -%}
+    --disable-workload-monitoring \
+    {% endif -%}
+    {% if cluster.spec.provision_shard_id -%}
+    --properties provision_shard_id:{{ cluster.spec.provision_shard_id }} \
+    {% endif -%}
+    --channel-group {{ cluster.spec.channel }}

reconcile/templating/lib/merge_request_manager.py

@@ -0,0 +1,180 @@
+import logging
+import re
+import string
+
+from pydantic import BaseModel
+
+from reconcile.templating.lib.model import TemplateOutput, TemplateResult
+from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.merge_request_manager.merge_request_manager import (
+    MergeRequestManagerBase,
+)
+from reconcile.utils.merge_request_manager.parser import (
+    Parser,
+)
+from reconcile.utils.mr import MergeRequestBase
+from reconcile.utils.mr.labels import AUTO_MERGE
+from reconcile.utils.vcs import VCS
+
+DATA_SEPARATOR = (
+    "**TEMPLATE RENDERING DATA - DO NOT MANUALLY CHANGE ANYTHING BELOW THIS LINE**"
+)
+TR_VERSION = "1.0.0"
+TR_LABEL = "template-output"
+
+VERSION_REF = "tr_version"
+COLLECTION_REF = "collection"
+TEMPLATE_RESULT_HASH_REF = "result_hash"
+
+COMPILED_REGEXES = {
+    i: re.compile(rf".*{i}: (.*)$", re.MULTILINE)
+    for i in [
+        VERSION_REF,
+        COLLECTION_REF,
+        TEMPLATE_RESULT_HASH_REF,
+    ]
+}
+
+MR_DESC = string.Template(
+    f"""
+This MR is triggered by app-interface's [template-rendering](https://github.com/app-sre/qontract-reconcile/blob/master/reconcile/templating/renderer.py).
+
+Please **do not remove the {TR_LABEL} label** from this MR!
+
+Parts of this description are used by the Template Renderer to manage the MR.
+
+{DATA_SEPARATOR}
+
+* {VERSION_REF}: $version
+* {COLLECTION_REF}: $collection
+* {TEMPLATE_RESULT_HASH_REF}: $result_hash
+"""
+)
+
+
+def create_parser() -> Parser:
+    return Parser[TemplateInfo](
+        klass=TemplateInfo,
+        compiled_regexes=COMPILED_REGEXES,
+        version_ref=VERSION_REF,
+        expected_version=TR_VERSION,
+        data_separator=DATA_SEPARATOR,
+    )
+
+
+def render_description(
+    collection: str, result_hash: str, version: str = TR_VERSION
+) -> str:
+    return MR_DESC.substitute(
+        collection=collection, result_hash=result_hash, version=version
+    )
+
+
+def render_title(collection: str) -> str:
+    return f'[auto] Rendered Templates for collection "{collection}"'
+
+
+class TemplateInfo(BaseModel):
+    collection: str
+    result_hash: str
+
+
+class TemplateRenderingMR(MergeRequestBase):
+    name = "TemplateRendering"
+
+    def __init__(
+        self,
+        title: str,
+        description: str,
+        content: list[TemplateOutput],
+        labels: list[str],
+    ):
+        super().__init__()
+        self._title = title
+        self._description = description
+        self._content = content
+        self.labels = labels
+
+    @property
+    def title(self) -> str:
+        return self._title
+
+    @property
+    def description(self) -> str:
+        return self._description
+
+    def process(self, gitlab_cli: GitLabApi) -> None:
+        for content in self._content:
+            if content.is_new:
+                gitlab_cli.create_file(
+                    branch_name=self.branch,
+                    file_path=f"data{content.path}",
+                    commit_message="termplate rendering output",
+                    content=content.content,
+                )
+            else:
+                gitlab_cli.update_file(
+                    branch_name=self.branch,
+                    file_path=f"data{content.path}",
+                    commit_message="termplate rendering output",
+                    content=content.content,
+                )
+
+
+class MrData(BaseModel):
+    result: TemplateResult
+    auto_approved: bool
+
+
+class MergeRequestManager(MergeRequestManagerBase[TemplateInfo]):
+    def __init__(self, vcs: VCS, parser: Parser):
+        super().__init__(vcs, parser, TR_LABEL)
+
+    def create_merge_request(self, data: MrData) -> None:
+        if not self._housekeeping_ran:
+            self.housekeeping()
+
+        result = data.result
+        output = result.outputs
+        collection = result.collection
+        result_hash = result.calc_result_hash()
+        additional_labels = result.labels
+
+        """Create a new MR with the rendered template."""
+        if mr := self._merge_request_already_exists({"collection": collection}):
+            if mr.mr_info.result_hash == result_hash:
+                logging.info(
+                    "MR already exists and has the same result hash. Skipping: %s",
+                    mr.raw.attributes.get("web_url", "NO_WEBURL"),
+                )
+                return None
+            else:
+                logging.info(
+                    "Collection Hash changed. Closing: %s",
+                    mr.raw.attributes.get("web_url", "NO_WEBURL"),
+                )
+                self._vcs.close_app_interface_mr(
+                    mr.raw,
+                    "Closing this MR because the result hash has changed.",
+                )
+
+        description = render_description(collection, result_hash)
+        title = render_title(collection)
+
+        logging.info("Opening MR for %s with hash (%s)", collection, result_hash)
+        mr_labels = [TR_LABEL]
+
+        if data.auto_approved:
+            mr_labels.append(AUTO_MERGE)
+
+        if additional_labels:
+            mr_labels.extend(additional_labels)
+
+        self._vcs.open_app_interface_merge_request(
+            mr=TemplateRenderingMR(
+                title=title,
+                description=description,
+                content=output,
+                labels=mr_labels,
+            )
+        )

reconcile/templating/lib/model.py

@@ -0,0 +1,20 @@
+from deepdiff import DeepHash
+from pydantic import BaseModel
+
+
+class TemplateOutput(BaseModel):
+    is_new: bool = False
+    path: str
+    content: str
+    auto_approved: bool = False
+
+
+class TemplateResult(BaseModel):
+    collection: str
+    enable_auto_approval: bool = False
+    labels: list[str] = []
+    outputs: list[TemplateOutput] = []
+
+    def calc_result_hash(self) -> str:
+        hashable = {o.path: o for o in self.outputs}
+        return DeepHash(hashable)[hashable]

reconcile/templating/lib/rendering.py

@@ -0,0 +1,191 @@
+import logging
+from abc import ABC, abstractmethod
+from io import StringIO
+from typing import Any, Protocol
+
+from pydantic import BaseModel
+
+from reconcile.utils.jinja2.utils import (
+    Jinja2TemplateError,
+    TemplateRenderOptions,
+    process_jinja2_template,
+)
+from reconcile.utils.jsonpath import parse_jsonpath
+from reconcile.utils.ruamel import create_ruamel_instance
+from reconcile.utils.secret_reader import SecretReaderBase
+
+
+class TemplateData(BaseModel):
+    variables: dict[str, Any]
+    current: dict[str, Any] | None
+    current_with_explicit_start: bool | None = False
+
+
+class TemplatePatch(Protocol):
+    path: str
+    identifier: str | None
+
+    def dict(self) -> dict[str, str]: ...
+
+
+class Template(Protocol):
+    name: str
+    condition: str | None
+    target_path: str
+    template: str
+
+    def dict(self) -> dict[str, str]: ...
+
+    @property
+    def patch(self) -> TemplatePatch | None:
+        pass
+
+
+class Renderer(ABC):
+    def __init__(
+        self,
+        template: Template,
+        data: TemplateData,
+        secret_reader: SecretReaderBase | None = None,
+        template_render_options: TemplateRenderOptions | None = None,
+    ):
+        self.template = template
+        self.data = data
+        self.secret_reader = secret_reader
+        self.ruamel_instance = create_ruamel_instance(explicit_start=True)
+        self.template_render_options = template_render_options
+
+    def _jinja2_render_kwargs(self) -> dict[str, Any]:
+        return {**self.data.variables, "current": self.data.current}
+
+    def _render_template(self, template: str) -> str:
+        try:
+            return process_jinja2_template(
+                body=template,
+                vars=self._jinja2_render_kwargs(),
+                secret_reader=self.secret_reader,
+                template_render_options=self.template_render_options,
+            )
+        except Jinja2TemplateError as e:
+            logging.error(f"Error rendering template {self.template.name}: {e}")
+            raise e
+
+    @abstractmethod
+    def render_output(self) -> str:
+        """
+        Implementation of a renderer is required and should return the entire rendered file as a string.
+        """
+        pass
+
+    def render_target_path(self) -> str:
+        return self._render_template(self.template.target_path).strip()
+
+    def render_condition(self) -> bool:
+        return self._render_template(self.template.condition or "True") == "True"
+
+
+class FullRenderer(Renderer):
+    def render_output(self) -> str:
+        """
+        Take the variables from Template Data and render the template with it.
+
+        This method returns the entire file as a string.
+        """
+        return self._render_template(self.template.template)
+
+
+class PatchRenderer(Renderer):
+    def render_output(self) -> str:
+        """
+        Takes the variables from Template Data and render the template with it.
+
+        This method partially updates the current data with the rendered template.
+        It checks the existence of the path in the current data and updates it if it exists.
+        If the path is not a list, it will update the value with the rendered template.
+
+        This method returns the entire file as a string.
+        """
+        if self.template.patch is None:  # here to satisfy mypy
+            raise ValueError("PatchRenderer requires a patch")
+
+        p = parse_jsonpath(self._render_template(self.template.patch.path))
+
+        matched_values = [match.value for match in p.find(self.data.current)]
+
+        if len(matched_values) != 1:
+            raise ValueError(
+                f"Expected exactly one match for {self.template.patch.path}, got {len(matched_values)}"
+            )
+        matched_value = matched_values[0]
+
+        data_to_add = self.ruamel_instance.load(
+            self._render_template(self.template.template)
+        )
+
+        if isinstance(matched_value, list):
+
+            def get_identifier(data: dict[str, Any]) -> Any:
+                assert self.template.patch is not None  # mypy
+                if not self.template.patch.identifier:
+                    raise ValueError(
+                        f"Expected identifier in patch for list at {self.template}"
+                    )
+                if self.template.patch.identifier.startswith(
+                    "$"
+                ) and not self.template.patch.identifier.startswith("$ref"):
+                    # jsonpath and list of strings support
+                    if matches := [
+                        match.value
+                        for match in parse_jsonpath(
+                            self.template.patch.identifier
+                        ).find(data)
+                    ]:
+                        return matches[0]
+                    return None
+                return data.get(self.template.patch.identifier)
+
+            dta_identifier = get_identifier(data_to_add)
+            if not dta_identifier:
+                raise ValueError(
+                    f"Expected identifier {self.template.patch.identifier} in data to add"
+                )
+
+            index = next(
+                (
+                    index
+                    for index, data in enumerate(matched_value)
+                    if get_identifier(data) == dta_identifier
+                ),
+                None,
+            )
+            if index is None:
+                matched_value.append(data_to_add)
+            else:
+                matched_value[index] = data_to_add
+        else:
+            matched_value.update(data_to_add)
+
+        with StringIO() as s:
+            self.ruamel_instance.dump(self.data.current, s)
+            return s.getvalue()
+
+
+def create_renderer(
+    template: Template,
+    data: TemplateData,
+    secret_reader: SecretReaderBase | None = None,
+    template_render_options: TemplateRenderOptions | None = None,
+) -> Renderer:
+    if template.patch:
+        return PatchRenderer(
+            template=template,
+            data=data,
+            secret_reader=secret_reader,
+            template_render_options=template_render_options,
+        )
+    return FullRenderer(
+        template=template,
+        data=data,
+        secret_reader=secret_reader,
+        template_render_options=template_render_options,
+    )