qontract-reconcile 0.10.1rc718__py3-none-any.whl → 0.10.1rc719__py3-none-any.whl

{qontract_reconcile-0.10.1rc718.dist-info → qontract_reconcile-0.10.1rc719.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc718
+Version: 0.10.1rc719
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc718.dist-info → qontract_reconcile-0.10.1rc719.dist-info}/RECORD

@@ -9,7 +9,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=R2WFXUXLTB4sWMi4GeA4eegsuf_1-Q4vH8M0Toh3Ij4,5036
-reconcile/cli.py,sha256=Xci_L0O3nPUPK3HA1NhIhbrzcPLZkXT65fCaJA66Tvg,96354
+reconcile/cli.py,sha256=HLfkpQXhNA9YDwNWJUYg8zjm03sCEUIVB6-Ly4FOh48,96572
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=SMhkcQqprWvThrIJa3U_3uh5w1h-alleW1QnCJFY4Qw,4909
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=a5aPLVxyqPSbjdB0Ty-uliOtxwvEbbEljHJKxdK3-Zk,4813
@@ -416,7 +416,7 @@ reconcile/templates/jira-checkpoint-missinginfo.j2,sha256=c_Vvg-lEENsB3tgxm9B6Y9
 reconcile/templates/rosa-classic-cluster-creation.sh.j2,sha256=0UHfYtXRVJqP07VJQx456cRI6EbZNBgamtP_8nb4WPY,2353
 reconcile/templates/rosa-hcp-cluster-creation.sh.j2,sha256=O7Bf3WQIJhsZoEqaYA0wRktUO4yXXCb4BQkuvvp-C80,2385
 reconcile/templating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/templating/renderer.py,sha256=xXCzRuhkDOCPELRKzjkAARTgs4iOoNcdDN_hlKJxyjg,8516
+reconcile/templating/renderer.py,sha256=wWsuCUtmC3vdUHP9ZYZgafbQxuAxPGpXiMYWuewPEyg,10156
 reconcile/templating/validator.py,sha256=QGH2VSk7sVBuojhk9quRAbj_XBykHN-KZ53DbEneUJs,4391
 reconcile/templating/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/templating/lib/merge_request_manager.py,sha256=El3ufdVjHP4EW-LztX7zPiI9syJBJ6ETGRmiM9K4Nqw,5112
@@ -595,7 +595,7 @@ reconcile/utils/extended_early_exit.py,sha256=QSktrmfw37zSRMNk930tDbQsVeKxaPPPD4
 reconcile/utils/external_resource_spec.py,sha256=IRY8MCsyWKzt-Qj_hXiFKgkCvZu6VVyj6IYtqEb05BA,6618
 reconcile/utils/external_resources.py,sha256=a2CkJ3KLociYBnc_9F2VWfZGWMhzDl6fDNhwo2U-MWU,7501
 reconcile/utils/filtering.py,sha256=zZnHH0u0SaTDyzuFXZ_mREURGLvjEqQIQy4z-7QBVlc,419
-reconcile/utils/git.py,sha256=Qad7mfPuS9s7eKODeWSewehwSGgJPCbQuLda1qg_6GA,1522
+reconcile/utils/git.py,sha256=BdxXFgQ1XOZpS-4qb3qMsKTCFDG8MlE26rv1jAhvCkM,1560
 reconcile/utils/git_secrets.py,sha256=0wGNL5mvDtVPRuu3vEQgld1Am64gIDJHtmu1_ZKxMAI,1973
 reconcile/utils/github_api.py,sha256=_bttNxYKeam_tLVe27L7O4gKqSn6CeyuFnJn8tSaUVY,2488
 reconcile/utils/gitlab_api.py,sha256=UMqWIHR_GkxEn5lgvxHLrjcwprRvRuKowxkwvu1xvvM,27241
@@ -764,8 +764,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=UEwAW7PA_GIrbqzaLxpkCxbuVjEFLNvnVG-6VyoCGIc,4147
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc718.dist-info/METADATA,sha256=QlMpHi8afUGz4ATxa8HHjMuCcJGKPjK9I3tFFFfCPXI,2382
-qontract_reconcile-0.10.1rc718.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc718.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
-qontract_reconcile-0.10.1rc718.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc718.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc719.dist-info/METADATA,sha256=c4dAaDQmMj1M0mTRWNFxv-fhaoYqat8QpFMThH9GqUw,2382
+qontract_reconcile-0.10.1rc719.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc719.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
+qontract_reconcile-0.10.1rc719.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc719.dist-info/RECORD,,

reconcile/cli.py

@@ -1968,12 +1968,17 @@ def template_validator(ctx):
 @integration.command(short_help="Render datafile templates in app-interface.")
 @click.option(
     "--app-interface-data-path",
-    help="Path to app-interface dictory, used to write output and calculate diff in dry-run.",
+    help="Path to data dir in app-interface repo. Use this for local rendering or in MR checks.",
     required=False,
     envvar="APP_INTERFACE_DATA_PATH",
 )
+@click.option(
+    "--clone-repo",
+    help="Path to a folder app-interface repo should be cloned to. Use this for regular integration run.",
+    default=False,
+)
 @click.pass_context
-def template_renderer(ctx, app_interface_data_path):
+def template_renderer(ctx, app_interface_data_path, clone_repo):
     from reconcile.templating.renderer import (
         TemplateRendererIntegration,
         TemplateRendererIntegrationParams,
@@ -1982,7 +1987,8 @@ def template_renderer(ctx, app_interface_data_path):
     run_class_integration(
         integration=TemplateRendererIntegration(
             TemplateRendererIntegrationParams(
-                app_interface_data_path=app_interface_data_path
+                app_interface_data_path=app_interface_data_path,
+                clone_repo=clone_repo,
             )
         ),
         ctx=ctx.obj,

reconcile/templating/renderer.py

@@ -2,12 +2,11 @@ import hashlib
 import json
 import logging
 import os
+import tempfile
 from abc import ABC, abstractmethod
 from collections.abc import Callable
-from datetime import datetime, timedelta
-from typing import Any, Optional
+from typing import Any, Optional, Self
 
-import gitlab
 from ruamel import yaml
 
 from reconcile.gql_definitions.templating.template_collection import (
@@ -29,20 +28,17 @@ from reconcile.typed_queries.app_interface_repo_url import get_app_interface_rep
 from reconcile.typed_queries.github_orgs import get_github_orgs
 from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
 from reconcile.utils import gql
+from reconcile.utils.git import clone
 from reconcile.utils.gql import init_from_config
 from reconcile.utils.ruamel import create_ruamel_instance
 from reconcile.utils.runtime.integration import (
     PydanticRunParams,
     QontractReconcileIntegration,
 )
-from reconcile.utils.state import State, init_state
 from reconcile.utils.vcs import VCS
 
 QONTRACT_INTEGRATION = "template-renderer"
 
-# Renders templates again to detect drift after one day
-CACHE_TTL_MINUTES = 24 * 60
-
 APP_INTERFACE_PATH_SEPERATOR = "/"
 
 
@@ -65,7 +61,13 @@ class FilePersistence(ABC):
 
 
 class LocalFilePersistence(FilePersistence):
+    """
+    This class provides a simple file persistence implementation for local files.
+    """
+
     def __init__(self, app_interface_data_path: str) -> None:
+        if not app_interface_data_path.endswith("/data"):
+            raise ValueError("app_interface_data_path should end with /data")
         self.app_interface_data_path = app_interface_data_path
 
     def write(self, outputs: list[TemplateOutput]) -> None:
@@ -90,8 +92,45 @@ class LocalFilePersistence(FilePersistence):
         return None
 
 
-class GitlabFilePersistence(FilePersistence):
-    def __init__(self, vcs: VCS, mr_manager: MergeRequestManager) -> None:
+class PersistenceTransaction(FilePersistence):
+    """
+    This class provides a context manager to make read/write operations
+    consistent. Reads/writes are beeing cached and writes are beeing delayed
+    until the context is left.
+    """
+
+    def __init__(self, persistence: FilePersistence, dry_run: bool) -> None:
+        self.persistence = persistence
+        self.dry_run = dry_run
+        self.content_cache: dict[str, Optional[str]] = {}
+        self.output_cache: dict[str, TemplateOutput] = {}
+
+    def write(self, outputs: list[TemplateOutput]) -> None:
+        for output in outputs:
+            self.content_cache[output.path] = output.content
+            self.output_cache[output.path] = output
+
+    def read(self, path: str) -> Optional[str]:
+        if path not in self.content_cache:
+            self.content_cache[path] = self.persistence.read(path)
+        return self.content_cache[path]
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        if not self.dry_run:
+            self.persistence.write(list(self.output_cache.values()))
+
+
+class ClonedRepoGitlabPersistence(FilePersistence):
+    """
+    This class is used to persist the rendered templates in a cloned gitlab repo
+    Reads are from the local filesystem, writes are done via utils.VCS abstraction
+    """
+
+    def __init__(self, local_path: str, vcs: VCS, mr_manager: MergeRequestManager):
+        self.local_path = join_path(local_path, "data")
         self.vcs = vcs
         self.mr_manager = mr_manager
 
@@ -101,10 +140,15 @@ class GitlabFilePersistence(FilePersistence):
 
     def read(self, path: str) -> Optional[str]:
         try:
-            current = self.vcs.get_file_content_from_app_interface_master(path)
-        except gitlab.exceptions.GitlabGetError:
-            return None
-        return current
+            with open(
+                f"{join_path(self.local_path, path)}",
+                "r",
+                encoding="utf-8",
+            ) as f:
+                return f.read()
+        except FileNotFoundError:
+            logging.debug(f"File not found: {path}, need to create it")
+        return None
 
 
 def unpack_static_variables(
@@ -125,6 +169,7 @@ def unpack_dynamic_variables(
 
 
 class TemplateRendererIntegrationParams(PydanticRunParams):
+    clone_repo: bool = False
     app_interface_data_path: Optional[str]
 
 
@@ -180,7 +225,6 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
         dry_run: bool,
         persistence: FilePersistence,
         ruamel_instance: yaml.YAML,
-        state: Optional[State] = None,
     ) -> None:
         gql_no_validation = init_from_config(validate_schemas=False)
 
@@ -199,39 +243,23 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
                 ).encode("utf-8")
             ).hexdigest()
 
-            if state and state.exists(c.name):
-                val = state.get(c.name)
-                if val["hash"] == template_hash and datetime.fromisoformat(
-                    val["timestamp"]
-                ) > (datetime.utcnow() - timedelta(minutes=CACHE_TTL_MINUTES)):
-                    logging.debug(f"Skipping {c.name} because it hasn't changed")
-                    break
-
-            for template in c.templates:
-                output = self.process_template(
-                    template,
-                    variables,
-                    persistence,
-                    ruamel_instance,
-                )
-                if output:
-                    output.input = TemplateInput(
-                        collection=c.name,
-                        collection_hash=template_hash,
-                    )
-                    outputs.append(output)
-
-            if not dry_run:
-                persistence.write(outputs)
-                if state:
-                    state.add(
-                        c.name,
-                        {
-                            "hash": template_hash,
-                            "timestamp": datetime.utcnow().isoformat(),
-                        },
-                        force=True,
+            with PersistenceTransaction(persistence, dry_run) as p:
+                for template in c.templates:
+                    output = self.process_template(
+                        template,
+                        variables,
+                        p,
+                        ruamel_instance,
                     )
+                    if output:
+                        output.input = TemplateInput(
+                            collection=c.name,
+                            collection_hash=template_hash,
+                        )
+                        outputs.append(output)
+
+                if not dry_run:
+                    p.write(outputs)
 
     @property
     def name(self) -> str:
@@ -239,14 +267,18 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
 
     def run(self, dry_run: bool) -> None:
         persistence: FilePersistence
-        state: Optional[State] = None
-        if self.params.app_interface_data_path:
+        ruaml_instance = create_ruamel_instance(explicit_start=True)
+
+        if not self.params.clone_repo and self.params.app_interface_data_path:
             persistence = LocalFilePersistence(self.params.app_interface_data_path)
-        else:
+            self.reconcile(dry_run, persistence, ruaml_instance)
+
+        elif self.params.clone_repo:
+            gitlab_instances = get_gitlab_instances()
             vcs = VCS(
                 secret_reader=self.secret_reader,
                 github_orgs=get_github_orgs(),
-                gitlab_instances=get_gitlab_instances(),
+                gitlab_instances=gitlab_instances,
                 app_interface_repo_url=get_app_interface_repo_url(),
                 dry_run=dry_run,
                 allow_deleting_mrs=True,
@@ -256,10 +288,24 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
                 vcs=vcs,
                 parser=create_parser(),
             )
-            persistence = GitlabFilePersistence(vcs, merge_request_manager)
+            url = get_app_interface_repo_url()
+
+            ssl_verify = next(
+                g.ssl_verify for g in gitlab_instances if url.startswith(g.url)
+            )
 
-            state = init_state(QONTRACT_INTEGRATION, self.secret_reader)
+            with tempfile.TemporaryDirectory(
+                prefix=f"{QONTRACT_INTEGRATION}-",
+            ) as temp_dir:
+                logging.debug(f"Cloning {url} to {temp_dir}")
 
-        ruaml_instance = create_ruamel_instance(explicit_start=True)
+                clone(url, temp_dir, depth=1, verify=ssl_verify)
 
-        self.reconcile(dry_run, persistence, ruaml_instance, state=state)
+                persistence = ClonedRepoGitlabPersistence(
+                    temp_dir, vcs, merge_request_manager
+                )
+
+                self.reconcile(dry_run, persistence, ruaml_instance)
+
+        else:
+            raise ValueError("App-interface-data-path must be set")

reconcile/utils/git.py

@@ -6,9 +6,14 @@ class GitError(Exception):
     pass
 
 
-def clone(repo_url, wd):
-    # pylint: disable=subprocess-run-check
-    cmd = ["git", "clone", repo_url, wd]
+def clone(repo_url, wd, depth=None, verify=True):
+    cmd = ["git"]
+    if not verify:
+        cmd += ["-c", "http.sslVerify=false"]
+    cmd += ["clone"]
+    if depth:
+        cmd += ["--depth", str(depth)]
+    cmd += [repo_url, wd]
     result = subprocess.run(
         cmd, cwd=wd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False
     )
@@ -17,7 +22,6 @@ def clone(repo_url, wd):
 
 
 def checkout(commit, wd):
-    # pylint: disable=subprocess-run-check
     cmd = ["git", "checkout", commit]
     result = subprocess.run(
         cmd, cwd=wd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False
@@ -29,7 +33,6 @@ def checkout(commit, wd):
 def is_file_in_git_repo(file_path):
     real_path = os.path.realpath(file_path)
     dir_path = os.path.dirname(real_path)
-    # pylint: disable=subprocess-run-check
     cmd = ["git", "rev-parse", "--is-inside-work-tree"]
     result = subprocess.run(
         cmd, cwd=dir_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False