qontract-reconcile 0.10.1rc558__py3-none-any.whl → 0.10.1rc560__py3-none-any.whl

{qontract_reconcile-0.10.1rc558.dist-info → qontract_reconcile-0.10.1rc560.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc558
+Version: 0.10.1rc560
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc558.dist-info → qontract_reconcile-0.10.1rc560.dist-info}/RECORD

@@ -9,7 +9,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=R2WFXUXLTB4sWMi4GeA4eegsuf_1-Q4vH8M0Toh3Ij4,5036
-reconcile/cli.py,sha256=R8qe7Wn89TUlbVhW1a5A-dwu2o-ekIF_o4mdv9Yfz3I,88594
+reconcile/cli.py,sha256=mHp_Jym5aUlvRmijHeL32bEIpx1gzvilHurYyS7WUOs,88893
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=SMhkcQqprWvThrIJa3U_3uh5w1h-alleW1QnCJFY4Qw,4909
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=a5aPLVxyqPSbjdB0Ty-uliOtxwvEbbEljHJKxdK3-Zk,4813
@@ -300,6 +300,8 @@ reconcile/gql_definitions/status_board/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 reconcile/gql_definitions/status_board/status_board.py,sha256=vHEzncabujkqbjJ-ibMYNJTODgTc4DMf4y6TW3I_7II,4700
 reconcile/gql_definitions/statuspage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/statuspage/statuspages.py,sha256=gxDb42H93nwtBg7oFRb6Gk9pbAZpsWk_y4Y0s3_g3nE,3520
+reconcile/gql_definitions/templating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/gql_definitions/templating/templates.py,sha256=ujPPFZm9BbkRrxkcR7ZyReDYfFem4uxONYoPuzgiBTc,2735
 reconcile/gql_definitions/terraform_cloudflare_dns/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/terraform_cloudflare_dns/app_interface_cloudflare_dns_settings.py,sha256=eyGX9HcTF6MZbOYZ6Kl6Mg3k6nJTUtwqs9gDxBP_8Dk,1920
 reconcile/gql_definitions/terraform_cloudflare_dns/terraform_cloudflare_zones.py,sha256=uVZYu5EUcvdAQYBK5YKD0mjoMKDb5inSuCJrrOD5KpE,5704
@@ -382,6 +384,9 @@ reconcile/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 reconcile/templates/aws_access_key_email.j2,sha256=2MUr1ERmyISzKgHqsWYLd-1Wbl-peUa-FsGUS-JLUFc,238
 reconcile/templates/email.yml.j2,sha256=OZgczNRgXPj2gVYTgwQyHAQrMGu7xp-e4W1rX19GcrU,690
 reconcile/templates/jira-checkpoint-missinginfo.j2,sha256=c_Vvg-lEENsB3tgxm9B6Y9igCUQhCnFDYh6xw-zcIbU,570
+reconcile/templating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/templating/rendering.py,sha256=3kHB9rBzndWLqW36LWyqCo0ysV4_xdAucsJW1vnwV2A,3904
+reconcile/templating/validator.py,sha256=0bWxJOQL5UQ9iQT6XAnQuSIS5MJ7uE1Yf8yZJVy-EHo,3381
 reconcile/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/test/conftest.py,sha256=rQousYrxUz-EwAIbsYO6bIwR1B4CrOz9y_zaUVo2lfI,4466
 reconcile/test/fixtures.py,sha256=9SDWAUlSd1rCx7z3GhULHcpr-I6FyCsXxaFAZIqYQsQ,591
@@ -651,7 +656,7 @@ reconcile/utils/runtime/sharding.py,sha256=roCdbnBklhTK_g34zbgQYqzpKPaNQ8J6Xd9XL
 reconcile/utils/saasherder/__init__.py,sha256=J3MBZBFa5YmhqYm08QsjBXz8mFcVOCiOCkyIcw41t7E,343
 reconcile/utils/saasherder/interfaces.py,sha256=XXY35h8VWQ66z3LBPxaoUAMkIW50264DQiecrzyV6oA,9076
 reconcile/utils/saasherder/models.py,sha256=PBv8DuAb6KUw_ayn5Ufiya20cCAelBv6Iv--x7hbpa4,5449
-reconcile/utils/saasherder/saasherder.py,sha256=HuJQWnugPFWIVMPpH-5Hw4KLhRY3OVlCMGED74Pu9Xc,85737
+reconcile/utils/saasherder/saasherder.py,sha256=vZ0S_7cT2FP1al2zOwLNuOedx8M7MIvokXiCsdm-5W4,85739
 reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
 reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
 reconcile/utils/terraform/config_client.py,sha256=py-Ree-QUYD6Hvng6bM40VgSuttteehIKNgwOSoJO1o,4706
@@ -679,8 +684,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=se-YG_YVCWRFrnCPvBVHDBT_59CkbIoEni-4SJa8_MU,2755
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc558.dist-info/METADATA,sha256=zuvGj1oR22yLBw_2XFSDHCFjcnnyXTawqr-DjPlqyLY,2349
-qontract_reconcile-0.10.1rc558.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-qontract_reconcile-0.10.1rc558.dist-info/entry_points.txt,sha256=rTjAv28I_CHLM8ID3OPqMI_suoQ9s7tFbim4aYjn9kk,376
-qontract_reconcile-0.10.1rc558.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc558.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc560.dist-info/METADATA,sha256=ehv0dWxZP3LGPKg6XVOI_-eH7A_-cZxdJfuZz3bFTwg,2349
+qontract_reconcile-0.10.1rc560.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+qontract_reconcile-0.10.1rc560.dist-info/entry_points.txt,sha256=rTjAv28I_CHLM8ID3OPqMI_suoQ9s7tFbim4aYjn9kk,376
+qontract_reconcile-0.10.1rc560.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc560.dist-info/RECORD,,

reconcile/cli.py

@@ -1858,6 +1858,17 @@ def terraform_repo(ctx, output_file, gitlab_project_id, gitlab_merge_request_id)
     )
 
 
+@integration.command(short_help="Test app-interface templates.")
+@click.pass_context
+def template_validator(ctx):
+    from reconcile.templating import validator
+
+    run_class_integration(
+        integration=validator.TemplateValidatorIntegration(PydanticRunParams()),
+        ctx=ctx.obj,
+    )
+
+
 @integration.command(short_help="Manage AWS Resources using Terraform.")
 @print_to_file
 @vault_output_path

reconcile/gql_definitions/templating/templates.py

@@ -0,0 +1,94 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+DEFINITION = """
+query Templatev1 {
+  template_v1 {
+    name
+    condition
+    patch {
+      path
+      identifier
+    }
+    targetPath
+    template
+    templateTest {
+      name
+      variables
+      current
+      expectedOutput
+      expectedTargetPath
+      expectedToRender
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class TemplatePatchV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+    identifier: Optional[str] = Field(..., alias="identifier")
+
+
+class TemplateTestV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    variables: Optional[Json] = Field(..., alias="variables")
+    current: Optional[str] = Field(..., alias="current")
+    expected_output: str = Field(..., alias="expectedOutput")
+    expected_target_path: Optional[str] = Field(..., alias="expectedTargetPath")
+    expected_to_render: Optional[bool] = Field(..., alias="expectedToRender")
+
+
+class TemplateV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    condition: Optional[str] = Field(..., alias="condition")
+    patch: Optional[TemplatePatchV1] = Field(..., alias="patch")
+    target_path: str = Field(..., alias="targetPath")
+    template: str = Field(..., alias="template")
+    template_test: list[TemplateTestV1] = Field(..., alias="templateTest")
+
+
+class Templatev1QueryData(ConfiguredBaseModel):
+    template_v1: Optional[list[TemplateV1]] = Field(..., alias="template_v1")
+
+
+def query(query_func: Callable, **kwargs: Any) -> Templatev1QueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        Templatev1QueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return Templatev1QueryData(**raw_data)

reconcile/templating/rendering.py

@@ -0,0 +1,113 @@
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+from jinja2.sandbox import SandboxedEnvironment
+from pydantic import BaseModel
+from ruamel import yaml
+
+from reconcile.gql_definitions.templating.templates import TemplateV1
+from reconcile.utils.jsonpath import parse_jsonpath
+
+
+class TemplateData(BaseModel):
+    variables: dict[str, Any]
+    current: Optional[dict[str, Any]]
+
+
+class Renderer(ABC):
+    def __init__(self, template: TemplateV1, data: TemplateData):
+        self.template = template
+        self.data = data
+        self.jinja_env = SandboxedEnvironment()
+
+    def _jinja2_render_kwargs(self) -> dict[str, Any]:
+        return {**self.data.variables, "current": self.data.current}
+
+    def _render_template(self, template: str) -> str:
+        return self.jinja_env.from_string(template).render(
+            **self._jinja2_render_kwargs()
+        )
+
+    @abstractmethod
+    def render_output(self) -> str:
+        """
+        Implementation of a renderer is required and should return the entire rendered file as a string.
+        """
+        pass
+
+    def render_target_path(self) -> str:
+        return self._render_template(self.template.target_path)
+
+    def render_condition(self) -> bool:
+        return self._render_template(self.template.condition or "True") == "True"
+
+
+class FullRenderer(Renderer):
+    def render_output(self) -> str:
+        """
+        Take the variables from Template Data and render the template with it.
+
+        This method returns the entire file as a string.
+        """
+        return self._render_template(self.template.template)
+
+
+class PatchRenderer(Renderer):
+    def render_output(self) -> str:
+        """
+        Takes the variables from Template Data and render the template with it.
+
+        This method partially updates the current data with the rendered template.
+        It checks the existence of the path in the current data and updates it if it exists.
+        If the path is not a list, it will update the value with the rendered template.
+
+        This method returns the entire file as a string.
+        """
+        if self.template.patch is None:  # here to satisfy mypy
+            raise ValueError("PatchRenderer requires a patch")
+
+        p = parse_jsonpath(self.template.patch.path)
+
+        matched_values = [match.value for match in p.find(self.data.current)]
+
+        if len(matched_values) != 1:
+            raise ValueError(
+                f"Expected exactly one match for {self.template.patch.path}, got {len(matched_values)}"
+            )
+        matched_value = matched_values[0]
+
+        data_to_add = yaml.safe_load(self._render_template(self.template.template))
+
+        if isinstance(matched_value, list):
+            if not self.template.patch.identifier:
+                raise ValueError(
+                    f"Expected identifier in patch for list at {self.template}"
+                )
+            dta_identifier = data_to_add.get(self.template.patch.identifier)
+            if not dta_identifier:
+                raise ValueError(
+                    f"Expected identifier {self.template.patch.identifier} in data to add"
+                )
+
+            data = next(
+                (
+                    data
+                    for data in matched_value
+                    if data.get(self.template.patch.identifier) == dta_identifier
+                ),
+                None,
+            )
+            if data is None:
+                matched_value.append(data_to_add)
+            else:
+                data.update(data_to_add)
+        else:
+            matched_value.update(data_to_add)
+
+        return yaml.dump(self.data.current, width=4096, Dumper=yaml.RoundTripDumper)
+
+
+def create_renderer(template: TemplateV1, data: TemplateData) -> Renderer:
+    if template.patch:
+        return PatchRenderer(template=template, data=data)
+    return FullRenderer(template=template, data=data)

reconcile/templating/validator.py

@@ -0,0 +1,101 @@
+import logging
+from difflib import context_diff
+from typing import Callable, Optional
+
+from pydantic import BaseModel
+from ruamel import yaml
+
+from reconcile.gql_definitions.templating.templates import TemplateV1, query
+from reconcile.templating.rendering import TemplateData, create_renderer
+from reconcile.utils import gql
+from reconcile.utils.runtime.integration import (
+    QontractReconcileIntegration,
+    RunParamsTypeVar,
+)
+
+QONTRACT_INTEGRATION = "template-validator"
+
+
+def get_templates(
+    query_func: Optional[Callable] = None,
+) -> list[TemplateV1]:
+    if not query_func:
+        query_func = gql.get_api().query
+    return query(query_func).template_v1 or []
+
+
+class TemplateDiff(BaseModel):
+    template: str
+    test: str
+    diff: str
+
+
+class TemplateValidatorIntegration(QontractReconcileIntegration):
+    def __init__(self, params: RunParamsTypeVar) -> None:
+        super().__init__(params)
+        self.diffs: list[TemplateDiff] = []
+
+    def diff_result(
+        self, template_name: str, test_name: str, output: str, expected: str
+    ) -> None:
+        diff = list(
+            context_diff(
+                output.splitlines(keepends=True), expected.splitlines(keepends=True)
+            )
+        )
+        if diff:
+            self.diffs.append(
+                TemplateDiff(template=template_name, test=test_name, diff="".join(diff))
+            )
+
+    def run(self, dry_run: bool) -> None:
+        for template in get_templates():
+            for test in template.template_test:
+                logging.info(f"Running test {test.name} for template {template.name}")
+
+                r = create_renderer(
+                    template,
+                    TemplateData(
+                        variables=test.variables or {},
+                        current=yaml.load(
+                            test.current or "", Loader=yaml.RoundTripLoader
+                        ),
+                    ),
+                )
+                if test.expected_target_path:
+                    self.diff_result(
+                        template.name,
+                        test.name,
+                        r.render_target_path().strip(),
+                        test.expected_target_path.strip(),
+                    )
+                should_render = r.render_condition()
+                if (
+                    test.expected_to_render is not None
+                    and test.expected_to_render != should_render
+                ):
+                    self.diffs.append(
+                        TemplateDiff(
+                            template=template.name,
+                            test=test.name,
+                            diff=f"Condition mismatch, got: {should_render}, expected: {test.expected_to_render}",
+                        )
+                    )
+                if should_render:
+                    self.diff_result(
+                        template.name,
+                        test.name,
+                        r.render_output().strip(),
+                        test.expected_output.strip(),
+                    )
+
+        if self.diffs:
+            for diff in self.diffs:
+                logging.error(
+                    f"template: {diff.template}, test: {diff.test}: {diff.diff}"
+                )
+            raise ValueError("Template validation")
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION

reconcile/utils/saasherder/saasherder.py

@@ -147,7 +147,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         self.state = state
         self._promotion_state = PromotionState(state=state) if state else None
         self._channel_map = self._assemble_channels(saas_files=all_saas_files)
-        self.images: list[str] = []
+        self.images: set[str] = set()
 
         # each namespace is in fact a target,
         # so we can use it to calculate.
@@ -1172,7 +1172,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
             self._collect_images, resources, self.available_thread_pool_size
         )
         images = set(itertools.chain.from_iterable(images_list))
-        self.images.extend(images)
+        self.images.update(images)
         if not images:
             return False  # no errors
         errors = threaded.run(