qontract-reconcile 0.10.1rc354__py3-none-any.whl → 0.10.1rc355__py3-none-any.whl

{qontract_reconcile-0.10.1rc354.dist-info → qontract_reconcile-0.10.1rc355.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc354
+Version: 0.10.1rc355
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc354.dist-info → qontract_reconcile-0.10.1rc355.dist-info}/RECORD

@@ -69,8 +69,8 @@ reconcile/openshift_resources.py,sha256=kwsY5cko7udEKNlhL2oKiKv_5wzEw9wmmwROE016
 reconcile/openshift_resources_base.py,sha256=K56KXgxxnAsVRvqSMVTW47_ebCsxYSc0BsViwZcRS6k,44409
 reconcile/openshift_rolebindings.py,sha256=1k0o3hb3ZhhlbUjc8cP7IjKFux0oZApT8kLT8Y-pvqI,6579
 reconcile/openshift_routes.py,sha256=fXvuPSjcjVw1X3j2EQvUAdbOepmIFdKk-M3qP8QzPiw,1075
-reconcile/openshift_saas_deploy.py,sha256=2V3uPZib7KqG3dZ29Yl2BPV1Ao9oRoBV1XLDVBj7lbw,10699
-reconcile/openshift_saas_deploy_change_tester.py,sha256=zosvOIY50MHwLvq_NBX2K4eKcpNx-ENKZf1tSLg09Po,8893
+reconcile/openshift_saas_deploy.py,sha256=In3hQfSCni3T5P1vP_j3bzEO2thLoQuxz4HrMSPjyIc,10761
+reconcile/openshift_saas_deploy_change_tester.py,sha256=spWjxapC-u4TrCAsz1Q6_297QwfrIRx19oqz2bRPQn0,8907
 reconcile/openshift_saas_deploy_trigger_base.py,sha256=UEKWAJo6cN3Nml89tzJzbnpkJ7efOnFDf9Wfz9_tBdg,14325
 reconcile/openshift_saas_deploy_trigger_cleaner.py,sha256=tcvziJdw5lgJbbogk0-wKT2aYCFP99sL4qTSfau4otY,2971
 reconcile/openshift_saas_deploy_trigger_configs.py,sha256=uWzUV5D5CW0frdi1ys7BObNg-rA-VZKlefd4TD_Z-pY,959
@@ -456,7 +456,7 @@ reconcile/typed_queries/namespaces.py,sha256=vItPrn7sfcHOix-VvkzQkf54_ljzI_ymyxh
 reconcile/typed_queries/namespaces_minimal.py,sha256=rUtqNQ0ORXXUTQfnpsMURymAJ4gYtE77V-Lb3LiJFEY,278
 reconcile/typed_queries/pagerduty_instances.py,sha256=QCHqEAakiH6eSob0Pnnn3IBd8Ga0zpEp1Z6Qu3v2uH4,733
 reconcile/typed_queries/repos.py,sha256=RKBsf7IDS6NsXTtXxJ9Ol9G3bxG9sr3vW9QQ2bahEHo,512
-reconcile/typed_queries/saas_files.py,sha256=bqokMGJIbIMV6EZbio8uIecuqCrk9-MS8jBzN3q3HL4,12209
+reconcile/typed_queries/saas_files.py,sha256=BuDZf83hv6ItJMqBEWBOF14XNyeEwr4qMkEWsA1fTK8,13990
 reconcile/typed_queries/smtp.py,sha256=aSLglYa5bHKmlGwKkxq2RZqyMWuAf0a4S_mOuhDa084,542
 reconcile/typed_queries/status_board.py,sha256=jsisR46kuHJM0VOPRQinnZigFNo51lHyvy-Q6Fdhs94,1673
 reconcile/typed_queries/tekton_pipeline_providers.py,sha256=2mpHBdsNPQB94tw0H9aenGuqj8EEjYolQ03YEq1CpiY,546
@@ -607,8 +607,8 @@ tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/test/test_qontract_cli.py,sha256=awwTHEc2DWlykuqGIYM0WOBoSL0KRnOraCLk3C7izis,1401
 tools/test/test_sd_app_sre_alert_report.py,sha256=JeLhgzpKCPgLvptwg_4ZvJHLVWKNG1T5845HXTkMBxA,1826
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc354.dist-info/METADATA,sha256=GO-ppoBN3oRal8GaVSD77eqjxDpLNKwg8QjiCuoBIuI,2347
-qontract_reconcile-0.10.1rc354.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-qontract_reconcile-0.10.1rc354.dist-info/entry_points.txt,sha256=ErVY2Jp-0Rtuq5KOtMlW5yvna4nIEuc_1YbEdEdcy9o,301
-qontract_reconcile-0.10.1rc354.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc354.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc355.dist-info/METADATA,sha256=OpYBE-4PcUg9vnQAfecLU0U3GXQITpJb672__LKMp4Q,2347
+qontract_reconcile-0.10.1rc355.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+qontract_reconcile-0.10.1rc355.dist-info/entry_points.txt,sha256=ErVY2Jp-0Rtuq5KOtMlW5yvna4nIEuc_1YbEdEdcy9o,301
+qontract_reconcile-0.10.1rc355.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc355.dist-info/RECORD,,

reconcile/openshift_saas_deploy.py

@@ -20,7 +20,7 @@ from reconcile.typed_queries.app_interface_vault_settings import (
 )
 from reconcile.typed_queries.saas_files import (
     SaasFile,
-    get_saas_files,
+    SaasFileList,
     get_saasherder_settings,
 )
 from reconcile.utils.defer import defer
@@ -100,15 +100,17 @@ def run(
     env_name: Optional[str] = None,
     trigger_integration: Optional[str] = None,
     trigger_reason: Optional[str] = None,
-    all_saas_files: Optional[list[SaasFile]] = None,
+    saas_file_list: Optional[SaasFileList] = None,
     defer: Optional[Callable] = None,
 ) -> None:
     vault_settings = get_app_interface_vault_settings()
     secret_reader = create_secret_reader(use_vault=vault_settings.vault)
 
-    if not all_saas_files:
-        all_saas_files = get_saas_files()
-    saas_files = get_saas_files(saas_file_name, env_name)
+    if not saas_file_list:
+        saas_file_list = SaasFileList()
+    all_saas_files = saas_file_list.saas_files
+    saas_files = saas_file_list.where(name=saas_file_name, env_name=env_name)
+
     if not saas_files:
         logging.error("no saas files found")
         raise RuntimeError("no saas files found")

reconcile/openshift_saas_deploy_change_tester.py

@@ -18,7 +18,7 @@ from reconcile.gql_definitions.common.saas_files import (
 from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
 from reconcile.typed_queries.saas_files import (
     SaasFile,
-    get_saas_files,
+    SaasFileList,
 )
 from reconcile.utils import gql
 from reconcile.utils.gitlab_api import GitLabApi
@@ -58,7 +58,7 @@ def osd_run_wrapper(
     dry_run: bool,
     available_thread_pool_size: int,
     use_jump_host: bool,
-    all_saas_files: Optional[list[SaasFile]],
+    saas_file_list: Optional[SaasFileList],
 ) -> int:
     saas_file_name, env_name = spec
     exit_code = 0
@@ -69,7 +69,7 @@ def osd_run_wrapper(
             use_jump_host=use_jump_host,
             saas_file_name=saas_file_name,
             env_name=env_name,
-            all_saas_files=all_saas_files,
+            saas_file_list=saas_file_list,
         )
     except SystemExit as e:
         exit_code = e.code if isinstance(e.code, int) else 1
@@ -214,10 +214,10 @@ def run(
     )
     # find the differences in saas-file state
     comparison_saas_file_state = collect_state(
-        get_saas_files(query_func=comparison_gql_api.query)
+        SaasFileList(query_func=comparison_gql_api.query).saas_files
     )
-    all_saas_files = get_saas_files()
-    desired_saas_file_state = collect_state(all_saas_files)
+    saas_file_list = SaasFileList()
+    desired_saas_file_state = collect_state(saas_file_list.saas_files)
     # compare dicts against dicts which is much faster than comparing BaseModel objects
     comparison_saas_file_state_dicts = [s.dict() for s in comparison_saas_file_state]
     saas_file_state_diffs = [
@@ -249,7 +249,7 @@ def run(
         dry_run=dry_run,
         available_thread_pool_size=available_thread_pool_size,
         use_jump_host=use_jump_host,
-        all_saas_files=all_saas_files,
+        saas_file_list=saas_file_list,
     )
 
     if [ec for ec in exit_codes if ec]:

reconcile/typed_queries/saas_files.py

@@ -1,6 +1,7 @@
 import hashlib
 import json
 from collections.abc import Callable
+from threading import Lock
 from typing import (
     Any,
     Optional,
@@ -8,7 +9,6 @@ from typing import (
 )
 
 from jsonpath_ng.exceptions import JsonPathParserError
-from jsonpath_ng.ext import parser
 from pydantic import (
     BaseModel,
     Extra,
@@ -51,6 +51,7 @@ from reconcile.utils.exceptions import (
     AppInterfaceSettingsError,
     ParameterError,
 )
+from reconcile.utils.jsonpath import parse_jsonpath
 
 
 class SaasResourceTemplateTarget(ConfiguredBaseModel):
@@ -141,51 +142,164 @@ class SaasFile(ConfiguredBaseModel):
     self_service_roles: Optional[list[RoleV1]] = Field(..., alias="selfServiceRoles")
 
 
-def get_namespaces_by_selector(
-    namespaces: list[SaasTargetNamespace],
-    namespace_selector: SaasResourceTemplateTargetNamespaceSelectorV1,
-) -> list[SaasTargetNamespace]:
-    # json representation of all the namespaces to filter on
-    # remove all the None values to simplify the jsonpath expressions
-    namespaces_as_dict = {
-        "namespace": [ns.dict(by_alias=True, exclude_none=True) for ns in namespaces]
-    }
-
-    def _get_namespace_by_cluster_and_name(
-        cluster_name: str, name: str
-    ) -> SaasTargetNamespace:
-        for ns in namespaces:
-            if ns.cluster.name == cluster_name and ns.name == name:
-                return ns
-        # this should never ever happen - just make mypy happy
-        raise RuntimeError(f"namespace '{name}' not found in cluster '{cluster_name}'")
-
-    filtered_namespaces: dict[str, Any] = {}
-
-    try:
+class SaasFileList:
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        query_func: Optional[Callable] = None,
+        namespaces: Optional[list[SaasTargetNamespace]] = None,
+    ) -> None:
+        # query_func and namespaces are optional args mostly used in tests
+        if not query_func:
+            query_func = gql.get_api().query
+        if not namespaces:
+            namespaces = namespaces_query(query_func).namespaces or []
+        self.namespaces = namespaces
+        self.cluster_namespaces = {
+            (ns.cluster.name, ns.name): ns for ns in self.namespaces
+        }
+
+        self._init_caches()
+
+        self.saas_files_v2 = saas_files_query(query_func).saas_files or []
+        if name:
+            self.saas_files_v2 = [sf for sf in self.saas_files_v2 if sf.name == name]
+        self.saas_files = self._resolve_namespace_selectors()
+
+    def _init_caches(self) -> None:
+        self._namespaces_as_dict_cache: Optional[dict[str, list[Any]]] = None
+        self._namespaces_as_dict_lock = Lock()
+        self._matching_namespaces_cache: dict[str, Any] = {}
+        self._matching_namespaces_lock = Lock()
+
+    def _resolve_namespace_selectors(self) -> list[SaasFile]:
+        saas_files: list[SaasFile] = []
+        # resolve namespaceSelectors to real namespaces
+        for sfv2 in self.saas_files_v2:
+            for rt_gql in sfv2.resource_templates:
+                for target_gql in rt_gql.targets[:]:
+                    # either namespace or namespaceSelector must be set
+                    if target_gql.namespace and target_gql.namespace_selector:
+                        raise ParameterError(
+                            f"SaasFile {sfv2.name}: namespace and namespaceSelector are mutually exclusive"
+                        )
+                    if not target_gql.provider:
+                        target_gql.provider = "static"
+
+                    if (
+                        target_gql.namespace_selector
+                        and target_gql.provider != "dynamic"
+                    ):
+                        raise ParameterError(
+                            f"SaasFile {sfv2.name}: namespaceSelector can only be used with 'provider: dynamic'"
+                        )
+                    if (
+                        target_gql.namespace_selector
+                        and target_gql.provider == "dynamic"
+                    ):
+                        rt_gql.targets.remove(target_gql)
+                        rt_gql.targets += self.create_targets_for_namespace_selector(
+                            target_gql, target_gql.namespace_selector
+                        )
+            # convert SaasFileV2 (with optional resource_templates.targets.namespace field)
+            # to SaasFile (with required resource_templates.targets.namespace field)
+            saas_files.append(SaasFile(**export_model(sfv2)))
+        return saas_files
+
+    def create_targets_for_namespace_selector(
+        self,
+        target: SaasResourceTemplateTargetV2,
+        namespace_selector: SaasResourceTemplateTargetNamespaceSelectorV1,
+    ) -> list[SaasResourceTemplateTargetV2]:
+        targets = []
+        for namespace in self.get_namespaces_by_selector(namespace_selector):
+            target_dict = export_model(target)
+            target_dict["namespace"] = export_model(namespace)
+            targets.append(SaasResourceTemplateTargetV2(**target_dict))
+        return targets
+
+    def _get_namespaces_as_dict(self) -> dict[str, list[Any]]:
+        # json representation of all the namespaces to filter on
+        # remove all the None values to simplify the jsonpath expressions
+        if self._namespaces_as_dict_cache is None:
+            with self._namespaces_as_dict_lock:
+                self._namespaces_as_dict_cache = {
+                    "namespace": [
+                        ns.dict(by_alias=True, exclude_none=True)
+                        for ns in self.namespaces
+                    ]
+                }
+        return self._namespaces_as_dict_cache
+
+    def _matching_namespaces(self, selector: str) -> Any:
+        if selector not in self._matching_namespaces_cache:
+            with self._matching_namespaces_lock:
+                namespaces_as_dict = self._get_namespaces_as_dict()
+                try:
+                    self._matching_namespaces_cache[selector] = parse_jsonpath(
+                        selector
+                    ).find(namespaces_as_dict)
+                except JsonPathParserError as e:
+                    raise ParameterError(
+                        f"Invalid jsonpath expression in namespaceSelector '{selector}' :{e}"
+                    )
+
+        return self._matching_namespaces_cache[selector]
+
+    def get_namespaces_by_selector(
+        self, namespace_selector: SaasResourceTemplateTargetNamespaceSelectorV1
+    ) -> list[SaasTargetNamespace]:
+        filtered_namespaces: dict[tuple[str, str], Any] = {}
+
         for include in namespace_selector.json_path_selectors.include:
-            for match in parser.parse(include).find(namespaces_as_dict):
+            for match in self._matching_namespaces(include):
                 cluster_name = match.value["cluster"]["name"]
                 ns_name = match.value["name"]
-                filtered_namespaces[
-                    f"{cluster_name}-{ns_name}"
-                ] = _get_namespace_by_cluster_and_name(cluster_name, ns_name)
-    except JsonPathParserError as e:
-        raise ParameterError(
-            f"Invalid jsonpath expression in namespaceSelector '{include}' :{e}"
-        )
+                filtered_namespaces[(cluster_name, ns_name)] = self.cluster_namespaces[
+                    (cluster_name, ns_name)
+                ]
 
-    try:
         for exclude in namespace_selector.json_path_selectors.exclude or []:
-            for match in parser.parse(exclude).find(namespaces_as_dict):
+            for match in self._matching_namespaces(exclude):
                 cluster_name = match.value["cluster"]["name"]
                 ns_name = match.value["name"]
-                filtered_namespaces.pop(f"{cluster_name}-{ns_name}", None)
-    except JsonPathParserError as e:
-        raise ParameterError(
-            f"Invalid jsonpath expression in namespaceSelector '{exclude}' :{e}"
-        )
-    return list(filtered_namespaces.values())
+                filtered_namespaces.pop((cluster_name, ns_name), None)
+
+        return list(filtered_namespaces.values())
+
+    def where(
+        self,
+        name: Optional[str] = None,
+        env_name: Optional[str] = None,
+        app_name: Optional[str] = None,
+    ) -> list[SaasFile]:
+        if name is None and env_name is None and app_name is None:
+            return self.saas_files
+
+        if name == "" or env_name == "" or app_name == "":
+            return []
+
+        filtered: list[SaasFile] = []
+        for saas_file in self.saas_files[:]:
+            if name and saas_file.name != name:
+                continue
+
+            if app_name and saas_file.app.name != app_name:
+                continue
+
+            sf = saas_file.copy(deep=True)
+            if env_name:
+                for rt in sf.resource_templates[:]:
+                    for target in rt.targets[:]:
+                        if target.namespace.environment.name != env_name:
+                            rt.targets.remove(target)
+                    if not rt.targets:
+                        sf.resource_templates.remove(rt)
+                if not sf.resource_templates:
+                    continue
+            filtered.append(sf)
+
+        return filtered
 
 
 def convert_parameters_to_json_string(root: dict[str, Any]) -> dict[str, Any]:
@@ -207,92 +321,19 @@ def export_model(model: BaseModel) -> dict[str, Any]:
     return convert_parameters_to_json_string(model.dict(by_alias=True))
 
 
-def create_targets_for_namespace_selector(
-    target: SaasResourceTemplateTargetV2,
-    namespaces: list[SaasTargetNamespace],
-    namespace_selector: SaasResourceTemplateTargetNamespaceSelectorV1,
-) -> list[SaasResourceTemplateTargetV2]:
-    targets = []
-    for namespace in get_namespaces_by_selector(namespaces, namespace_selector):
-        target_dict = export_model(target)
-        target_dict["namespace"] = export_model(namespace)
-        targets.append(SaasResourceTemplateTargetV2(**target_dict))
-    return targets
-
-
 def get_saas_files(
     name: Optional[str] = None,
     env_name: Optional[str] = None,
     app_name: Optional[str] = None,
     query_func: Optional[Callable] = None,
     namespaces: Optional[list[SaasTargetNamespace]] = None,
+    saas_file_list: Optional[SaasFileList] = None,
 ) -> list[SaasFile]:
-    if not query_func:
-        query_func = gql.get_api().query
-    data = saas_files_query(query_func)
-    saas_files: list[SaasFile] = []
-    if not namespaces:
-        namespaces = namespaces_query(query_func).namespaces or []
-
-    data_saas_files = list(data.saas_files or [])
-    if name:
-        data_saas_files = [sf for sf in data_saas_files if sf.name == name]
-    # resolve namespaceSelectors to real namespaces
-    for saas_file_gql in data_saas_files:
-        for rt_gql in saas_file_gql.resource_templates:
-            for target_gql in rt_gql.targets[:]:
-                # either namespace or namespaceSelector must be set
-                if target_gql.namespace and target_gql.namespace_selector:
-                    raise ParameterError(
-                        f"SaasFile {saas_file_gql.name}: namespace and namespaceSelector are mutually exclusive"
-                    )
-                if not target_gql.provider:
-                    target_gql.provider = "static"
-
-                if (
-                    target_gql.namespace_selector
-                    and not target_gql.provider == "dynamic"
-                ):
-                    raise ParameterError(
-                        f"SaasFile {saas_file_gql.name}: namespaceSelector can only be used with 'provider: dynamic'"
-                    )
-                if target_gql.namespace_selector and target_gql.provider == "dynamic":
-                    rt_gql.targets.remove(target_gql)
-                    rt_gql.targets += create_targets_for_namespace_selector(
-                        target_gql, namespaces, target_gql.namespace_selector
-                    )
-        # convert SaasFileV2 (with optional resource_templates.targets.namespace field)
-        # to SaasFile (with required resource_templates.targets.namespace field)
-        saas_files.append(SaasFile(**export_model(saas_file_gql)))
-
-    if name is None and env_name is None and app_name is None:
-        return saas_files
-    if name == "" or env_name == "" or app_name == "":
-        return []
-
-    for saas_file in saas_files[:]:
-        if name:
-            if saas_file.name != name:
-                saas_files.remove(saas_file)
-                continue
-
-        if env_name:
-            for rt in saas_file.resource_templates[:]:
-                for target in rt.targets[:]:
-                    if target.namespace.environment.name != env_name:
-                        rt.targets.remove(target)
-                if not rt.targets:
-                    saas_file.resource_templates.remove(rt)
-            if not saas_file.resource_templates:
-                saas_files.remove(saas_file)
-                continue
-
-        if app_name:
-            if saas_file.app.name != app_name:
-                saas_files.remove(saas_file)
-                continue
-
-    return saas_files
+    if not saas_file_list:
+        saas_file_list = SaasFileList(
+            name=name, query_func=query_func, namespaces=namespaces
+        )
+    return saas_file_list.where(env_name=env_name, app_name=app_name)
 
 
 def get_saasherder_settings(