qontract-reconcile 0.10.1rc950__py3-none-any.whl → 0.10.1rc952__py3-none-any.whl

{qontract_reconcile-0.10.1rc950.dist-info → qontract_reconcile-0.10.1rc952.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc950
+Version: 0.10.1rc952
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc950.dist-info → qontract_reconcile-0.10.1rc952.dist-info}/RECORD

@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=q96mwr2KeEQ5bpNniGlgIMZTxiuLSodcYfX-t
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=O1wFp52EyF538c6txaWBs8eMtUIy19gyHZ6VzJ6QXS8,3512
 reconcile/checkpoint.py,sha256=_JhMxrye5BgkRMxWYuf7Upli6XayPINKSsuo3ynHTRc,5010
-reconcile/cli.py,sha256=rcy5rudCBXog_WSWGnHoTo8xZDeP-VHVXzF4sdpdrbU,105520
+reconcile/cli.py,sha256=1uK79dvw3lF7p2oyOwK4552uywXqZ8MxRbv4WYvpoto,105531
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=rLh16BOlBOxTmJ8Si3wWyyEpmMlhh4Znx1Gc36qsmOc,4865
 reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
 reconcile/dashdotdb_base.py,sha256=l34QDu1G96_Ctnh7ZXdxXgSeCE93GQMdLAkWxmN6vDA,4775
@@ -463,12 +463,12 @@ reconcile/templates/jira-checkpoint-missinginfo.j2,sha256=c_Vvg-lEENsB3tgxm9B6Y9
 reconcile/templates/rosa-classic-cluster-creation.sh.j2,sha256=zqMMlKWV-aUDMtA-Xu5kl5pYLilewDClnKM0c807nNA,2146
 reconcile/templates/rosa-hcp-cluster-creation.sh.j2,sha256=hQ-WapiBNV3oEKQ813W3bD4HwQ4gADRPy80z9OLtW3Y,2317
 reconcile/templating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/templating/renderer.py,sha256=rcaGQ8sQjsAyWJKjsOFgKhLEUnPvJAh7eDAJBHHSQMs,12800
+reconcile/templating/renderer.py,sha256=6qFeXNqAE8t85eewfm8Pcj9i_Fp2UJ1lad_zTqYmF7M,14753
 reconcile/templating/validator.py,sha256=5f9f35PCHOOdjb7KZquL2YdabyuAUokPDa4xutSEHIQ,5360
 reconcile/templating/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/templating/lib/merge_request_manager.py,sha256=XwpOR4rVS9ZiJ_Mn8qfCXPZ7CRLMGSJgeIkqD-8Jhgc,5228
 reconcile/templating/lib/model.py,sha256=YVUIXuPny3_kpFgBMSud8q_ndY5o882wKiX0l0A14L4,481
-reconcile/templating/lib/rendering.py,sha256=pbZv8uYlNtgzTjYMl5qSIW94vDS4CfE47aSjX4HpL-k,6210
+reconcile/templating/lib/rendering.py,sha256=IzTbXJ5cO0c9mV6P9HrstOo79ovOcoNVtmyc6RgfMe0,6241
 reconcile/terraform_init/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/terraform_init/integration.py,sha256=glQ9uy8Kj2aTQXCAupwSFeih7reX_xMX_UuWW_ywBMU,6100
 reconcile/terraform_init/merge_request.py,sha256=3CYtgSd7Q9zjKg4wsDz437EPCRfGeZZ8fZ0Y-ChKXJY,1475
@@ -657,7 +657,7 @@ reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,
 reconcile/utils/git.py,sha256=actOWI2HiNpMIV6nHCzinhRa6b04Y9plWOCcPQa8lNA,1437
 reconcile/utils/git_secrets.py,sha256=y1rEhwA8DyDpBSAEuhMS7Y2X3mpxT2zQ4zyDFkhLe_g,1936
 reconcile/utils/github_api.py,sha256=R8OvqyPdnRqvP-Efnv9RvIcbBlb4M0KC4RlbnJMD0Tg,2426
-reconcile/utils/gitlab_api.py,sha256=ZPU6e94DQ1T7pNO1VPoSq_iS0HfELN-gwYUyot42PdM,30334
+reconcile/utils/gitlab_api.py,sha256=EUjo2IBmiz37tmmBmjadzyh6_Dh991G-QKD_NnftX6Q,30347
 reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
 reconcile/utils/gql.py,sha256=IGhxzBcuebbapDKLseevEThSsxa_eDCPNpo3A4VnOS4,14066
 reconcile/utils/grouping.py,sha256=vr9SFHZ7bqmHYrvYcEZt-Er3-yQYfAAdq5sHLZVmXPY,456
@@ -846,8 +846,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc950.dist-info/METADATA,sha256=68rjYO_yguToaqfByY1BPGufGV1SFwkBdqZy5bt2OtU,2262
-qontract_reconcile-0.10.1rc950.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc950.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc950.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc950.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc952.dist-info/METADATA,sha256=2-vhkaZkD_8sOM7nxeu08c69Q6nZCMcK-Qi2MBcUVRo,2262
+qontract_reconcile-0.10.1rc952.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc952.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc952.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc952.dist-info/RECORD,,

reconcile/cli.py

@@ -2084,7 +2084,8 @@ def template_validator(ctx):
 )
 @click.option(
     "--clone-repo",
-    help="Path to a folder app-interface repo should be cloned to. Use this for regular integration run.",
+    is_flag=True,
+    help="Flag to enable cloning of the app-interface repo. Use this for regular integration run.",
     default=False,
 )
 @click.option(

reconcile/templating/lib/rendering.py

@@ -78,7 +78,7 @@ class Renderer(ABC):
         pass
 
     def render_target_path(self) -> str:
-        return self._render_template(self.template.target_path)
+        return self._render_template(self.template.target_path).strip()
 
     def render_condition(self) -> bool:
         return self._render_template(self.template.condition or "True") == "True"
@@ -108,7 +108,7 @@ class PatchRenderer(Renderer):
         if self.template.patch is None:  # here to satisfy mypy
             raise ValueError("PatchRenderer requires a patch")
 
-        p = parse_jsonpath(self.template.patch.path)
+        p = parse_jsonpath(self._render_template(self.template.patch.path))
 
         matched_values = [match.value for match in p.find(self.data.current)]
 

reconcile/templating/renderer.py

@@ -58,10 +58,21 @@ def get_template_collections(
 
 
 class FilePersistence(ABC):
+    def __init__(self, dry_run: bool) -> None:
+        self.dry_run = dry_run
+        self.outputs: list[TemplateOutput] = []
+        self.result: TemplateResult | None = None
+
+    def __enter__(self) -> Self:
+        return self
+
     @abstractmethod
-    def write(self, result: TemplateResult) -> None:
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
         pass
 
+    def write(self, output: TemplateOutput) -> None:
+        self.outputs.append(output)
+
     @abstractmethod
     def read(self, path: str) -> str | None:
         pass
@@ -84,40 +95,40 @@ class LocalFilePersistence(FilePersistence):
     This class provides a simple file persistence implementation for local files.
     """
 
-    def __init__(self, app_interface_data_path: str) -> None:
+    def __init__(self, dry_run: bool, app_interface_data_path: str) -> None:
+        super().__init__(dry_run)
         if not app_interface_data_path.endswith("/data"):
             raise ValueError("app_interface_data_path should end with /data")
         self.app_interface_data_path = app_interface_data_path
 
-    def write(self, result: TemplateResult) -> None:
-        for output in result.outputs:
+    def read(self, path: str) -> str | None:
+        return self._read_local_file(join_path(self.app_interface_data_path, path))
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        if self.dry_run:
+            return
+        for output in self.outputs:
             filepath = Path(join_path(self.app_interface_data_path, output.path))
             filepath.parent.mkdir(parents=True, exist_ok=True)
             filepath.write_text(output.content, encoding="utf-8")
 
-    def read(self, path: str) -> str | None:
-        return self._read_local_file(join_path(self.app_interface_data_path, path))
-
 
 class PersistenceTransaction(FilePersistence):
     """
     This class provides a context manager to make read/write operations
-    consistent. Reads/writes are beeing cached and writes are beeing delayed
+    consistent. Reads/writes are beeing cached and writes are being delayed
     until the context is left.
     """
 
-    def __init__(self, persistence: FilePersistence, dry_run: bool) -> None:
+    def __init__(self, persistence: FilePersistence) -> None:
+        super().__init__(persistence.dry_run)
         self.persistence = persistence
-        self.dry_run = dry_run
         self.content_cache: dict[str, str | None] = {}
         self.output_cache: dict[str, TemplateOutput] = {}
-        self.result: TemplateResult
 
-    def write(self, result: TemplateResult) -> None:
-        self.result = result
-        for output in result.outputs:
-            self.content_cache[output.path] = output.content
-            self.output_cache[output.path] = output
+    def write(self, output: TemplateOutput) -> None:
+        self.content_cache[output.path] = output.content
+        self.output_cache[output.path] = output
 
     def read(self, path: str) -> str | None:
         if path not in self.content_cache:
@@ -129,8 +140,8 @@ class PersistenceTransaction(FilePersistence):
 
     def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
         if not self.dry_run and self.output_cache:
-            self.result.outputs = list(self.output_cache.values())
-            self.persistence.write(self.result)
+            for output in self.output_cache.values():
+                self.persistence.write(output)
 
 
 class ClonedRepoGitlabPersistence(FilePersistence):
@@ -138,30 +149,64 @@ class ClonedRepoGitlabPersistence(FilePersistence):
     This class is used to persist the rendered templates in a cloned gitlab repo
     Reads are from the local filesystem, writes are done via utils.VCS abstraction
 
-    Only one MR is created per run. Auto-approval MRs are prefered.
+    One MR is created per template-collection. Auto-approval MRs are prefered.
     """
 
-    def __init__(self, local_path: str, vcs: VCS, mr_manager: MergeRequestManager):
+    def __init__(
+        self, dry_run: bool, local_path: str, vcs: VCS, mr_manager: MergeRequestManager
+    ):
+        super().__init__(dry_run)
         self.local_path = join_path(local_path, "data")
         self.vcs = vcs
         self.mr_manager = mr_manager
 
-    def write(self, result: TemplateResult) -> None:
+    def read(self, path: str) -> str | None:
+        return self._read_local_file(join_path(self.local_path, path))
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        if self.result is None:
+            raise ValueError("ClonedRepoGitlabPersistence.result not set!")
+        self.result.outputs = self.outputs
+
+        if self.dry_run or not self.outputs:
+            return
+
         self.mr_manager.housekeeping()
 
-        if result.enable_auto_approval:
-            auto_approved = [o for o in result.outputs if o.auto_approved]
-            if auto_approved:
-                result.outputs = auto_approved
+        if self.result.enable_auto_approval:
+            if auto_approved_outputs := [o for o in self.outputs if o.auto_approved]:
+                # create an MR with auto-approved templates only
                 self.mr_manager.create_merge_request(
-                    MrData(result=result, auto_approved=True)
+                    MrData(
+                        result=TemplateResult(
+                            collection=f"{self.result.collection}-auto-approved",
+                            enable_auto_approval=self.result.enable_auto_approval,
+                            labels=self.result.labels,
+                            outputs=auto_approved_outputs,
+                        ),
+                        auto_approved=True,
+                    )
                 )
-                return
-
-        self.mr_manager.create_merge_request(MrData(result=result, auto_approved=False))
-
-    def read(self, path: str) -> str | None:
-        return self._read_local_file(join_path(self.local_path, path))
+            if not_auto_approved_outputs := [
+                o for o in self.outputs if not o.auto_approved
+            ]:
+                # create an MR with not auto-approved templates only
+                self.mr_manager.create_merge_request(
+                    MrData(
+                        result=TemplateResult(
+                            collection=f"{self.result.collection}-not-auto-approved",
+                            enable_auto_approval=self.result.enable_auto_approval,
+                            labels=self.result.labels,
+                            outputs=not_auto_approved_outputs,
+                        ),
+                        auto_approved=False,
+                    )
+                )
+        else:
+            # create an MR with all templates
+            self.mr_manager.create_merge_request(
+                MrData(result=self.result, auto_approved=False)
+            )
 
 
 def unpack_static_variables(
@@ -240,27 +285,24 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
             template, variables, secret_reader=self.secret_reader
         )
         target_path = r.render_target_path()
-
-        current_str = persistence.read(
-            target_path,
-        )
-        if current_str is None and template.patch:
+        current_file = persistence.read(target_path)
+        if current_file is None and template.patch:
             raise ValueError(f"Can not patch non-existing file {target_path}")
 
-        if current_str:
-            r.data.current = ruaml_instance.load(current_str)
+        if current_file:
+            r.data.current = ruaml_instance.load(current_file)
 
         if r.render_condition():
             output = r.render_output()
 
-            if current_str != output:
+            if current_file != output:
                 logging.info(
                     f"diff for template {template.name} in target path {target_path}"
                 )
                 return TemplateOutput(
                     path=target_path,
                     content=output,
-                    is_new=current_str is None,
+                    is_new=current_file is None,
                     auto_approved=template.auto_approved or False,
                 )
         return None
@@ -272,7 +314,7 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
         persistence: FilePersistence,
         ruamel_instance: yaml.YAML,
         each: dict[str, Any],
-    ) -> list[TemplateOutput]:
+    ) -> None:
         variables = {}
         if collection.variables:
             variables = {
@@ -281,46 +323,39 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
                 ),
                 "static": unpack_static_variables(collection.variables, each),
             }
-
-        outputs: list[TemplateOutput] = []
-        for template in collection.templates:
-            output = self.process_template(
-                template, variables, persistence, ruamel_instance
-            )
-            if output:
-                outputs.append(output)
-
-        return outputs
+        with PersistenceTransaction(persistence) as persistence_transaction:
+            for template in collection.templates:
+                if output := self.process_template(
+                    template, variables, persistence_transaction, ruamel_instance
+                ):
+                    persistence_transaction.write(output)
 
     def reconcile(
-        self,
-        dry_run: bool,
-        persistence: FilePersistence,
-        ruamel_instance: yaml.YAML,
+        self, persistence: FilePersistence, ruamel_instance: yaml.YAML
     ) -> None:
-        gql_no_validation = init_from_config(validate_schemas=False)
-        for c in get_template_collections(name=self.params.template_collection_name):
+        gql_api = init_from_config(validate_schemas=False)
+        for collection in get_template_collections(
+            name=self.params.template_collection_name
+        ):
             for_each_items: list[dict[str, Any]] = [{}]
-            if c.for_each and c.for_each.items:
-                for_each_items = c.for_each.items
+            if collection.for_each and collection.for_each.items:
+                for_each_items = collection.for_each.items
             result = TemplateResult(
-                collection=c.name,
-                enable_auto_approval=c.enable_auto_approval or False,
-                labels=c.additional_mr_labels or [],
+                collection=collection.name,
+                enable_auto_approval=collection.enable_auto_approval or False,
+                labels=collection.additional_mr_labels or [],
             )
-            with PersistenceTransaction(persistence, dry_run) as p:
+            with persistence as p:
+                p.result = result
+                p.outputs = []
                 for item in for_each_items:
-                    result.outputs.extend(
-                        self.reconcile_template_collection(
-                            c,
-                            gql_no_validation,
-                            p,
-                            ruamel_instance,
-                            item,
-                        )
+                    self.reconcile_template_collection(
+                        collection=collection,
+                        gql_api=gql_api,
+                        persistence=p,
+                        ruamel_instance=ruamel_instance,
+                        each=item,
                     )
-                if not dry_run and result.outputs:
-                    p.write(result)
 
     @property
     def name(self) -> str:
@@ -331,8 +366,11 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
         ruaml_instance = create_ruamel_instance(explicit_start=True)
 
         if not self.params.clone_repo and self.params.app_interface_data_path:
-            persistence = LocalFilePersistence(self.params.app_interface_data_path)
-            self.reconcile(dry_run, persistence, ruaml_instance)
+            persistence = LocalFilePersistence(
+                dry_run=dry_run,
+                app_interface_data_path=self.params.app_interface_data_path,
+            )
+            self.reconcile(persistence, ruaml_instance)
 
         elif self.params.clone_repo:
             gitlab_instances = get_gitlab_instances()
@@ -359,14 +397,14 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
                 prefix=f"{QONTRACT_INTEGRATION}-",
             ) as temp_dir:
                 logging.debug(f"Cloning {url} to {temp_dir}")
-
                 clone(url, temp_dir, depth=1, verify=ssl_verify)
-
                 persistence = ClonedRepoGitlabPersistence(
-                    temp_dir, vcs, merge_request_manager
+                    dry_run=dry_run,
+                    local_path=temp_dir,
+                    vcs=vcs,
+                    mr_manager=merge_request_manager,
                 )
-
-                self.reconcile(dry_run, persistence, ruaml_instance)
+                self.reconcile(persistence, ruaml_instance)
 
         else:
             raise ValueError("App-interface-data-path must be set")

reconcile/utils/gitlab_api.py

@@ -130,7 +130,7 @@ class GitLabApi:  # pylint: disable=too-many-public-methods
     @cached_property
     def project_main_branch(self) -> str:
         return next(
-            (b.name for b in self.project.branches.list() if b.default),
+            (b.name for b in self.project.branches.list(iterator=True) if b.default),
             DEFAULT_MAIN_BRANCH,
         )