qontract-reconcile 0.9.1rc235__py3-none-any.whl → 0.9.1rc236__py3-none-any.whl

reconcile/test/test_saasherder.py

@@ -25,13 +25,11 @@ from reconcile.gql_definitions.common.saas_files import (
     SaasFileV2,
     SaasResourceTemplateTargetImageV1,
     SaasResourceTemplateTargetPromotionV1,
-    SaasResourceTemplateTargetV2_SaasSecretParametersV1,
     SaasResourceTemplateV2,
 )
 from reconcile.utils.jjb_client import JJB
 from reconcile.utils.openshift_resource import ResourceInventory
 from reconcile.utils.saasherder import SaasHerder
-from reconcile.utils.saasherder.interfaces import SaasFile
 from reconcile.utils.saasherder.models import TriggerSpecMovingCommit
 from reconcile.utils.secret_reader import SecretReaderBase
 
@@ -939,119 +937,3 @@ class TestRemoveNoneAttributes(TestCase):
         expected: dict[Any, Any] = {}
         res = SaasHerder.remove_none_values(input)
         self.assertEqual(res, expected)
-
-
-def test_render_templated_parameters(
-    gql_class_factory: Callable[..., SaasFile]
-) -> None:
-    saas_file = gql_class_factory(
-        SaasFileV2,
-        Fixtures("saasherder").get_anymarkup("saas-templated-params.gql.yml"),
-    )
-    SaasHerder.resolve_templated_parameters([saas_file])
-    assert saas_file.resource_templates[0].targets[0].parameters == {
-        "no-template": "v1",
-        "ignore-go-template": "{{ .GO_PARAM }}-go",
-        "template-param-1": "test-namespace-ns",
-        "template-param-2": "appsres03ue1-cluster",
-    }
-    assert saas_file.resource_templates[0].targets[0].secret_parameters == [
-        SaasResourceTemplateTargetV2_SaasSecretParametersV1(
-            name="no-template",
-            secret=dict(
-                path="path/to/secret",
-                field="secret_key",
-                version=1,
-                format=None,
-            ),
-        ),
-        SaasResourceTemplateTargetV2_SaasSecretParametersV1(
-            name="ignore-go-template",
-            secret=dict(
-                path="path/{{ .GO_PARAM }}/secret",
-                field="{{ .GO_PARAM }}-secret_key",
-                version=1,
-                format=None,
-            ),
-        ),
-        SaasResourceTemplateTargetV2_SaasSecretParametersV1(
-            name="template-param-1",
-            secret=dict(
-                path="path/appsres03ue1/test-namespace/secret",
-                field="secret_key",
-                version=1,
-                format=None,
-            ),
-        ),
-        SaasResourceTemplateTargetV2_SaasSecretParametersV1(
-            name="template-param-2",
-            secret=dict(
-                path="path/appsres03ue1/test-namespace/secret",
-                field="App-SRE-stage-secret_key",
-                version=1,
-                format=None,
-            ),
-        ),
-    ]
-
-
-def test_render_templated_parameters_in_init(
-    gql_class_factory: Callable[..., SaasFile]
-) -> None:
-    saas_file = gql_class_factory(
-        SaasFileV2,
-        Fixtures("saasherder").get_anymarkup("saas-templated-params.gql.yml"),
-    )
-    SaasHerder(
-        [saas_file],
-        secret_reader=MockSecretReader(),
-        thread_pool_size=1,
-        integration="",
-        integration_version="",
-        hash_length=24,
-        repo_url="https://repo-url.com",
-    )
-    assert saas_file.resource_templates[0].targets[0].parameters == {
-        "no-template": "v1",
-        "ignore-go-template": "{{ .GO_PARAM }}-go",
-        "template-param-1": "test-namespace-ns",
-        "template-param-2": "appsres03ue1-cluster",
-    }
-    assert saas_file.resource_templates[0].targets[0].secret_parameters == [
-        SaasResourceTemplateTargetV2_SaasSecretParametersV1(
-            name="no-template",
-            secret=dict(
-                path="path/to/secret",
-                field="secret_key",
-                version=1,
-                format=None,
-            ),
-        ),
-        SaasResourceTemplateTargetV2_SaasSecretParametersV1(
-            name="ignore-go-template",
-            secret=dict(
-                path="path/{{ .GO_PARAM }}/secret",
-                field="{{ .GO_PARAM }}-secret_key",
-                version=1,
-                format=None,
-            ),
-        ),
-        SaasResourceTemplateTargetV2_SaasSecretParametersV1(
-            name="template-param-1",
-            secret=dict(
-                path="path/appsres03ue1/test-namespace/secret",
-                field="secret_key",
-                version=1,
-                format=None,
-            ),
-        ),
-        SaasResourceTemplateTargetV2_SaasSecretParametersV1(
-            name="template-param-2",
-            secret=dict(
-                path="path/appsres03ue1/test-namespace/secret",
-                field="App-SRE-stage-secret_key",
-                version=1,
-                format=None,
-            ),
-        ),
-    ]

reconcile/test/test_saasherder_allowed_secret_paths.py

@@ -2,7 +2,7 @@ from collections.abc import Callable
 
 import pytest
 
-from reconcile.typed_queries.saas_files import SaasFile
+from reconcile.gql_definitions.common.saas_files import SaasFileV2
 from reconcile.utils.saasherder import SaasHerder
 from reconcile.utils.secret_reader import SecretReader
 
@@ -27,7 +27,7 @@ def test_saasherder_allowed_secret_paths(
     secret_reader: SecretReader,
     gql_class_factory: Callable[
         ...,
-        SaasFile,
+        SaasFileV2,
     ],
 ):
     """
@@ -35,7 +35,7 @@ def test_saasherder_allowed_secret_paths(
     """
     saas_files = [
         gql_class_factory(
-            SaasFile,
+            SaasFileV2,
             {
                 "path": "path1",
                 "name": "a1",

reconcile/test/test_typed_queries.py

@@ -0,0 +1,35 @@
+from typing import Any
+
+import pytest
+
+from reconcile.gql_definitions.common.pagerduty_instances import PagerDutyInstanceV1
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+from reconcile.test.fixtures import Fixtures
+from reconcile.typed_queries.pagerduty_instances import get_pagerduty_instances
+
+
+@pytest.fixture
+def fxt() -> Fixtures:
+    return Fixtures("typed_queries")
+
+
+def test_get_pagerduty_instances(fxt: Fixtures) -> None:
+    def q(*args: Any, **kwargs: Any) -> dict[Any, Any]:
+        return fxt.get_anymarkup("pagerduty_instances.yml")
+
+    items = get_pagerduty_instances(q)
+    assert len(items) == 2
+    assert items == [
+        PagerDutyInstanceV1(
+            name="instance-1",
+            token=VaultSecret(
+                path="vault-path", field="token", version=None, format=None
+            ),
+        ),
+        PagerDutyInstanceV1(
+            name="instance-2",
+            token=VaultSecret(
+                path="vault-path2", field="token2", version=2, format="format"
+            ),
+        ),
+    ]

reconcile/typed_queries/saas_files.py

@@ -1,189 +1,14 @@
-import json
 from collections.abc import Callable
-from typing import (
-    Any,
-    Optional,
-    Union,
-)
-
-from jsonpath_ng.exceptions import JsonPathParserError
-from jsonpath_ng.ext import parser
-from pydantic import (
-    BaseModel,
-    Extra,
-    Field,
-    Json,
-)
+from typing import Optional
 
-from reconcile.gql_definitions.common.saas_files import (
-    AppV1,
-    ConfiguredBaseModel,
-    DeployResourcesV1,
-    PipelinesProviderTektonV1,
-    PipelinesProviderV1,
-    RoleV1,
-    SaasFileAuthenticationV1,
-    SaasResourceTemplateTargetImageV1,
-    SaasResourceTemplateTargetNamespaceSelectorV1,
-    SaasResourceTemplateTargetPromotionV1,
-    SaasResourceTemplateTargetUpstreamV1,
-    SaasResourceTemplateTargetV2,
-    SaasResourceTemplateTargetV2_SaasSecretParametersV1,
-    SaasResourceTemplateV2_SaasSecretParametersV1,
-    SaasSecretParametersV1,
-    SlackOutputV1,
-)
+from reconcile.gql_definitions.common.saas_files import SaasFileV2
 from reconcile.gql_definitions.common.saas_files import query as saas_files_query
-from reconcile.gql_definitions.common.saas_target_namespaces import (
-    query as namespaces_query,
-)
 from reconcile.gql_definitions.common.saasherder_settings import AppInterfaceSettingsV1
 from reconcile.gql_definitions.common.saasherder_settings import (
     query as saasherder_settings_query,
 )
-from reconcile.gql_definitions.fragments.saas_target_namespace import (
-    SaasTargetNamespace,
-)
 from reconcile.utils import gql
-from reconcile.utils.exceptions import (
-    AppInterfaceSettingsError,
-    ParameterError,
-)
-
-
-class SaasResourceTemplateTarget(ConfiguredBaseModel):
-    path: Optional[str] = Field(..., alias="path")
-    name: Optional[str] = Field(..., alias="name")
-    # the namespace must be required to fulfill the saas file schema (utils.saasherder.interface.SaasFile)
-    namespace: SaasTargetNamespace = Field(..., alias="namespace")
-    ref: str = Field(..., alias="ref")
-    promotion: Optional[SaasResourceTemplateTargetPromotionV1] = Field(
-        ..., alias="promotion"
-    )
-    parameters: Optional[Json] = Field(..., alias="parameters")
-    secret_parameters: Optional[
-        list[SaasResourceTemplateTargetV2_SaasSecretParametersV1]
-    ] = Field(..., alias="secretParameters")
-    upstream: Optional[SaasResourceTemplateTargetUpstreamV1] = Field(
-        ..., alias="upstream"
-    )
-    image: Optional[SaasResourceTemplateTargetImageV1] = Field(..., alias="image")
-    disable: Optional[bool] = Field(..., alias="disable")
-    delete: Optional[bool] = Field(..., alias="delete")
-
-    class Config:
-        # ignore `namespaceSelector` and 'provider' fields from the GQL schema
-        extra = Extra.ignore
-
-
-class SaasResourceTemplate(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    url: str = Field(..., alias="url")
-    path: str = Field(..., alias="path")
-    provider: Optional[str] = Field(..., alias="provider")
-    hash_length: Optional[int] = Field(..., alias="hash_length")
-    parameters: Optional[Json] = Field(..., alias="parameters")
-    secret_parameters: Optional[
-        list[SaasResourceTemplateV2_SaasSecretParametersV1]
-    ] = Field(..., alias="secretParameters")
-    targets: list[SaasResourceTemplateTarget] = Field(..., alias="targets")
-
-
-class SaasFile(ConfiguredBaseModel):
-    path: str = Field(..., alias="path")
-    name: str = Field(..., alias="name")
-    app: AppV1 = Field(..., alias="app")
-    pipelines_provider: Union[PipelinesProviderTektonV1, PipelinesProviderV1] = Field(
-        ..., alias="pipelinesProvider"
-    )
-    deploy_resources: Optional[DeployResourcesV1] = Field(..., alias="deployResources")
-    slack: Optional[SlackOutputV1] = Field(..., alias="slack")
-    managed_resource_types: list[str] = Field(..., alias="managedResourceTypes")
-    takeover: Optional[bool] = Field(..., alias="takeover")
-    deprecated: Optional[bool] = Field(..., alias="deprecated")
-    compare: Optional[bool] = Field(..., alias="compare")
-    timeout: Optional[str] = Field(..., alias="timeout")
-    publish_job_logs: Optional[bool] = Field(..., alias="publishJobLogs")
-    cluster_admin: Optional[bool] = Field(..., alias="clusterAdmin")
-    image_patterns: list[str] = Field(..., alias="imagePatterns")
-    allowed_secret_parameter_paths: Optional[list[str]] = Field(
-        ..., alias="allowedSecretParameterPaths"
-    )
-    use_channel_in_image_tag: Optional[bool] = Field(
-        ..., alias="use_channel_in_image_tag"
-    )
-    authentication: Optional[SaasFileAuthenticationV1] = Field(
-        ..., alias="authentication"
-    )
-    parameters: Optional[Json] = Field(..., alias="parameters")
-    secret_parameters: Optional[list[SaasSecretParametersV1]] = Field(
-        ..., alias="secretParameters"
-    )
-    resource_templates: list[SaasResourceTemplate] = Field(
-        ..., alias="resourceTemplates"
-    )
-    self_service_roles: Optional[list[RoleV1]] = Field(..., alias="selfServiceRoles")
-
-
-def get_namespaces_by_selector(
-    namespaces: list[SaasTargetNamespace],
-    namespace_selector: SaasResourceTemplateTargetNamespaceSelectorV1,
-) -> list[SaasTargetNamespace]:
-    namespaces_as_dict = {"namespace": [export_model(n) for n in namespaces]}
-    filtered_namespaces: dict[str, Any] = {}
-
-    try:
-        for include in namespace_selector.json_path_selectors.include:
-            for match in parser.parse(include).find(namespaces_as_dict):
-                ns = SaasTargetNamespace(**match.value)
-                filtered_namespaces[f"{ns.cluster.name}-{ns.name}"] = ns
-    except JsonPathParserError as e:
-        raise ParameterError(
-            f"Invalid jsonpath expression in namespaceSelector '{include}' :{e}"
-        )
-
-    try:
-        for exclude in namespace_selector.json_path_selectors.exclude or []:
-            for match in parser.parse(exclude).find(namespaces_as_dict):
-                ns = SaasTargetNamespace(**match.value)
-                filtered_namespaces.pop(f"{ns.cluster.name}-{ns.name}", None)
-    except JsonPathParserError as e:
-        raise ParameterError(
-            f"Invalid jsonpath expression in namespaceSelector '{exclude}' :{e}"
-        )
-    return list(filtered_namespaces.values())
-
-
-def convert_parameters_to_json_string(root: dict[str, Any]) -> dict[str, Any]:
-    """Find all parameter occurrences and convert them to a json string."""
-    for key, value in root.items():
-        if key == "parameters":
-            root[key] = json.dumps(value)
-        elif isinstance(value, dict):
-            root[key] = convert_parameters_to_json_string(value)
-        elif isinstance(value, list):
-            root[key] = [
-                convert_parameters_to_json_string(v) if isinstance(v, dict) else v
-                for v in value
-            ]
-    return root
-
-
-def export_model(model: BaseModel) -> dict[str, Any]:
-    return convert_parameters_to_json_string(model.dict(by_alias=True))
-
-
-def create_targets_for_namespace_selector(
-    target: SaasResourceTemplateTargetV2,
-    namespaces: list[SaasTargetNamespace],
-    namespace_selector: SaasResourceTemplateTargetNamespaceSelectorV1,
-) -> list[SaasResourceTemplateTargetV2]:
-    targets = []
-    for namespace in get_namespaces_by_selector(namespaces, namespace_selector):
-        target_dict = export_model(target)
-        target_dict["namespace"] = export_model(namespace)
-        targets.append(SaasResourceTemplateTargetV2(**target_dict))
-    return targets
+from reconcile.utils.exceptions import AppInterfaceSettingsError
 
 
 def get_saas_files(
@@ -191,43 +16,11 @@ def get_saas_files(
     env_name: Optional[str] = None,
     app_name: Optional[str] = None,
     query_func: Optional[Callable] = None,
-    namespaces: Optional[list[SaasTargetNamespace]] = None,
-) -> list[SaasFile]:
+) -> list[SaasFileV2]:
     if not query_func:
         query_func = gql.get_api().query
     data = saas_files_query(query_func)
-    saas_files: list[SaasFile] = []
-    if not namespaces:
-        namespaces = namespaces_query(query_func).namespaces or []
-
-    # resolve namespaceSelectors to real namespaces
-    for saas_file_gql in list(data.saas_files or []):
-        for rt_gql in saas_file_gql.resource_templates:
-            for target_gql in rt_gql.targets[:]:
-                # either namespace or namespaceSelector must be set
-                if target_gql.namespace and target_gql.namespace_selector:
-                    raise ParameterError(
-                        f"SaasFile {saas_file_gql.name}: namespace and namespaceSelector are mutually exclusive"
-                    )
-                if not target_gql.provider:
-                    target_gql.provider = "static"
-
-                if (
-                    target_gql.namespace_selector
-                    and not target_gql.provider == "dynamic"
-                ):
-                    raise ParameterError(
-                        f"SaasFile {saas_file_gql.name}: namespaceSelector can only be used with 'provider: dynamic'"
-                    )
-                if target_gql.namespace_selector and target_gql.provider == "dynamic":
-                    rt_gql.targets.remove(target_gql)
-                    rt_gql.targets += create_targets_for_namespace_selector(
-                        target_gql, namespaces, target_gql.namespace_selector
-                    )
-        # convert SaasFileV2 (with optional resource_templates.targets.namespace field)
-        # to SaasFile (with required resource_templates.targets.namespace field)
-        saas_files.append(SaasFile(**export_model(saas_file_gql)))
-
+    saas_files = list(data.saas_files or [])
     if name is None and env_name is None and app_name is None:
         return saas_files
     if name == "" or env_name == "" or app_name == "":

reconcile/utils/saasherder/interfaces.py

@@ -204,7 +204,7 @@ class SaasResourceTemplateTargetNamespace(Protocol):
         self,
         *,
         by_alias: bool = False,
-        include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None,
+        include: Union[AbstractSetIntStr, MappingIntStrAny],
     ) -> dict[str, Any]:
         ...
 

reconcile/utils/saasherder/saasherder.py

@@ -117,7 +117,6 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
         self.error_registered = False
         self.saas_files = saas_files
         self.repo_urls = self._collect_repo_urls()
-        self.resolve_templated_parameters(self.saas_files)
         if validate:
             self._validate_saas_files()
             if not self.valid:
@@ -1875,32 +1874,3 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                             subscribe_target_path_map[channel].add(target.path)
 
         return subscribe_saas_file_path_map, subscribe_target_path_map
-
-    @staticmethod
-    def resolve_templated_parameters(saas_files: Iterable[SaasFile]) -> None:
-        """Resolve templated target parameters in saas files."""
-        from reconcile.openshift_resources_base import (
-            compile_jinja2_template,  # avoid circular import
-        )
-
-        for saas_file in saas_files:
-            for rt in saas_file.resource_templates:
-                for target in rt.targets:
-                    template_vars = {
-                        "resource": {"namespace": target.namespace.dict(by_alias=True)}
-                    }
-                    if target.parameters:
-                        for param in target.parameters:
-                            if not isinstance(target.parameters[param], str):
-                                continue
-                            target.parameters[param] = compile_jinja2_template(
-                                target.parameters[param], extra_curly=True
-                            ).render(template_vars)
-                    if target.secret_parameters:
-                        for secret_param in target.secret_parameters:
-                            secret_param.secret.field = compile_jinja2_template(
-                                secret_param.secret.field, extra_curly=True
-                            ).render(template_vars)
-                            secret_param.secret.path = compile_jinja2_template(
-                                secret_param.secret.path, extra_curly=True
-                            ).render(template_vars)

tools/app_interface_reporter.py

@@ -29,10 +29,6 @@ from reconcile.jenkins_job_builder import (
     get_openshift_saas_deploy_job_name,
     init_jjb,
 )
-from reconcile.typed_queries.saas_files import (
-    export_model,
-    get_saas_files,
-)
 from reconcile.utils.jjb_client import JJB
 from reconcile.utils.mr import CreateAppInterfaceReporter
 from reconcile.utils.runtime.environment import init_env
@@ -231,7 +227,7 @@ def get_apps_data(date, month_delta=1, thread_pool_size=10):
     secret_reader = SecretReader(settings)
 
     apps = queries.get_apps()
-    saas_files = [export_model(saas_file) for saas_file in get_saas_files()]
+    saas_files = queries.get_saas_files()
     jjb: JJB = init_jjb(secret_reader)
     jenkins_map = jenkins_base.get_jenkins_map()
     time_limit = date - relativedelta(months=month_delta)