qontract-reconcile 0.10.2.dev394__py3-none-any.whl → 0.10.2.dev427__py3-none-any.whl

reconcile/utils/saasherder/saasherder.py

@@ -92,8 +92,7 @@ from reconcile.utils.state import State
 from reconcile.utils.vcs import VCS
 
 TARGET_CONFIG_HASH = "target_config_hash"
-
-
+TEMPLATE_API_VERSION = "template.openshift.io/v1"
 UNIQUE_SAAS_FILE_ENV_COMBO_LEN = 56
 REQUEST_TIMEOUT = 60
 
@@ -765,7 +764,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 content = self.gitlab.get_raw_file(
                     project=project,
                     path=path,
@@ -801,7 +801,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 dir_contents = self.gitlab.get_directory_contents(
                     project,
                     ref=commit_sha,
@@ -826,7 +827,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 commits = project.commits.list(ref_name=ref, per_page=1, page=1)
                 return commits[0].id
             case _:
@@ -871,10 +873,23 @@ class SaasHerder:
         """
         if parameter_name in consolidated_parameters:
             return False
-        for template_parameter in template.get("parameters", {}):
-            if template_parameter["name"] == parameter_name:
-                return True
-        return False
+        return any(
+            template_parameter["name"] == parameter_name
+            for template_parameter in template.get("parameters") or []
+        )
+
+    @staticmethod
+    def _pre_process_template(template: dict[str, Any]) -> dict[str, Any]:
+        """
+        The only supported apiVersion for OpenShift Template is "template.openshift.io/v1".
+        There are examples of templates using "v1", it can't pass validation on 4.19+ oc versions.
+
+        Args:
+            template (dict): The OpenShift template dictionary.
+        Returns:
+            dict: The OpenShift template dictionary with the correct apiVersion.
+        """
+        return template | {"apiVersion": TEMPLATE_API_VERSION}
 
     def _process_template(
         self, spec: TargetSpec
@@ -964,7 +979,8 @@ class SaasHerder:
             oc = OCLocal("cluster", None, None, local=True)
             try:
                 resources: Iterable[Mapping[str, Any]] = oc.process(
-                    template, consolidated_parameters
+                    template=self._pre_process_template(template),
+                    parameters=consolidated_parameters,
                 )
             except StatusCodeError as e:
                 logging.error(f"{error_prefix} error processing template: {e!s}")
@@ -1179,13 +1195,13 @@ class SaasHerder:
         images_list = threaded.run(
             self._collect_images, resources, self.available_thread_pool_size
         )
-        images = set(itertools.chain.from_iterable(images_list))
-        self.images.update(images)
-        if not images:
+        images_set = set(itertools.chain.from_iterable(images_list))
+        self.images.update(images_set)
+        if not images_set:
             return False  # no errors
         images = threaded.run(
             self._get_image,
-            images,
+            images_set,
             self.available_thread_pool_size,
             image_patterns=spec.image_patterns,
             image_auth=spec.image_auth,
@@ -1250,7 +1266,9 @@ class SaasHerder:
             self.saas_files,
             self.thread_pool_size,
         )
-        desired_state_specs = list(itertools.chain.from_iterable(results))
+        desired_state_specs: list[TargetSpec] = list(
+            itertools.chain.from_iterable(results)
+        )
         promotions = threaded.run(
             self.populate_desired_state_saas_file,
             desired_state_specs,
@@ -1898,21 +1916,23 @@ class SaasHerder:
             name=target.name,
             ref=target.ref,
             promotion=(
-                target.promotion.dict(by_alias=True) if target.promotion else None
+                target.promotion.model_dump(by_alias=True) if target.promotion else None
             ),
             secretParameters=(
-                [p.dict(by_alias=True) for p in target.secret_parameters]
+                [p.model_dump(by_alias=True) for p in target.secret_parameters]
                 if target.secret_parameters
                 else None
             ),
             slos=(
-                [slo.dict(by_alias=True) for slo in target.slos]
+                [slo.model_dump(by_alias=True) for slo in target.slos]
                 if target.slos
                 else None
             ),
-            upstream=(target.upstream.dict(by_alias=True) if target.upstream else None),
+            upstream=(
+                target.upstream.model_dump(by_alias=True) if target.upstream else None
+            ),
             images=(
-                [i.dict(by_alias=True) for i in target.images]
+                [i.model_dump(by_alias=True) for i in target.images]
                 if target.images
                 else None
             ),
@@ -1948,16 +1968,16 @@ class SaasHerder:
         )
         if saas_file.managed_resource_names:
             state_content["saas_file_managed_resource_names"] = [
-                m.dict() for m in saas_file.managed_resource_names
+                m.model_dump() for m in saas_file.managed_resource_names
             ]
         # include secret parameters from resource template and saas file
         if resource_template.secret_parameters:
             state_content["rt_secretparameters"] = [
-                p.dict() for p in resource_template.secret_parameters
+                p.model_dump() for p in resource_template.secret_parameters
             ]
         if saas_file.secret_parameters:
             state_content["saas_file_secretparameters"] = [
-                p.dict() for p in saas_file.secret_parameters
+                p.model_dump() for p in saas_file.secret_parameters
             ]
         return state_content
 
@@ -2240,7 +2260,9 @@ class SaasHerder:
             for rt in saas_file.resource_templates:
                 for target in rt.targets:
                     template_vars = {
-                        "resource": {"namespace": target.namespace.dict(by_alias=True)}
+                        "resource": {
+                            "namespace": target.namespace.model_dump(by_alias=True)
+                        }
                     }
                     if target.parameters:
                         for param in target.parameters:

reconcile/utils/slack_api.py

@@ -71,14 +71,14 @@ class HasClientGlobalConfig(Protocol):
     max_retries: int | None
     timeout: int | None
 
-    def dict(self) -> dict[str, int | None]: ...
+    def model_dump(self) -> dict[str, int | None]: ...
 
 
 class HasClientMethodConfig(Protocol):
     name: str
     args: Any
 
-    def dict(self) -> dict[str, str]: ...
+    def model_dump(self) -> dict[str, str]: ...
 
 
 class HasClientConfig(Protocol):

reconcile/utils/structs.py

@@ -1,4 +1,4 @@
-from pydantic.dataclasses import dataclass
+from dataclasses import dataclass
 
 
 @dataclass

reconcile/utils/terraform_client.py

@@ -223,7 +223,7 @@ class TerraformClient:
         if disable_deletions_detected:
             raise RuntimeError("Terraform plan has disabled deletions detected")
 
-    @retry(no_retry_exceptions=RdsUpgradeValidationError)
+    @retry(no_retry_exceptions=(RdsUpgradeValidationError,))
     def terraform_plan(
         self, spec: TerraformSpec, enable_deletion: bool
     ) -> tuple[bool, list[AccountUser], bool]:

reconcile/utils/terrascript_aws_client.py

@@ -1058,7 +1058,9 @@ class TerrascriptClient:
             ignore_changes = (
                 "all" if "all" in lifecycle.ignore_changes else lifecycle.ignore_changes
             )
-            return lifecycle.dict(by_alias=True) | {"ignore_changes": ignore_changes}
+            return lifecycle.model_dump(by_alias=True) | {
+                "ignore_changes": ignore_changes
+            }
         return None
 
     def populate_additional_providers(
@@ -1424,7 +1426,7 @@ class TerrascriptClient:
             req_account_name = req_account.name
             # Accepter's side of the connection - the cluster's account
             acc_account = accepter.account
-            acc_alias = self.get_provider_alias(acc_account.dict(by_alias=True))
+            acc_alias = self.get_provider_alias(acc_account.model_dump(by_alias=True))
             acc_uid = acc_account.uid
             if acc_account.assume_role:
                 acc_uid = awsh.get_account_uid_from_arn(acc_account.assume_role)
@@ -2210,6 +2212,43 @@ class TerrascriptClient:
         letters_and_digits = string.ascii_letters + string.digits
         return "".join(random.choice(letters_and_digits) for i in range(string_length))
 
+    @staticmethod
+    def _build_tf_resource_s3_lifecycle_rules(
+        versioning: bool,
+        common_values: Mapping[str, Any],
+    ) -> list[dict]:
+        lifecycle_rules = common_values.get("lifecycle_rules") or []
+        if versioning and not any(
+            "noncurrent_version_expiration" in lr for lr in lifecycle_rules
+        ):
+            # Add a default noncurrent object expiration rule
+            # if one isn't already set
+            rule = {
+                "id": "expire_noncurrent_versions",
+                "enabled": True,
+                "noncurrent_version_expiration": {"days": 30},
+                "expiration": {"expired_object_delete_marker": True},
+                "abort_incomplete_multipart_upload_days": 3,
+            }
+            lifecycle_rules.append(rule)
+
+        if storage_class := common_values.get("storage_class"):
+            sc = storage_class.upper()
+            days = "1"
+            if sc.endswith("_IA"):
+                # Infrequent Access storage class has minimum 30 days
+                # before transition
+                days = "30"
+            rule = {
+                "id": sc + "_storage_class",
+                "enabled": True,
+                "transition": {"days": days, "storage_class": sc},
+                "noncurrent_version_transition": {"days": days, "storage_class": sc},
+            }
+            lifecycle_rules.append(rule)
+
+        return lifecycle_rules
+
     def populate_tf_resource_s3(self, spec: ExternalResourceSpec) -> aws_s3_bucket:
         account = spec.provisioner_name
         identifier = spec.identifier
@@ -2249,47 +2288,11 @@ class TerrascriptClient:
         request_payer = common_values.get("request_payer")
         if request_payer:
             values["request_payer"] = request_payer
-        lifecycle_rules = common_values.get("lifecycle_rules")
-        if lifecycle_rules:
-            # common_values['lifecycle_rules'] is a list of lifecycle_rules
+        if lifecycle_rules := self._build_tf_resource_s3_lifecycle_rules(
+            versioning=versioning,
+            common_values=common_values,
+        ):
             values["lifecycle_rule"] = lifecycle_rules
-        if versioning:
-            lrs = values.get("lifecycle_rule", [])
-            expiration_rule = False
-            for lr in lrs:
-                if "noncurrent_version_expiration" in lr:
-                    expiration_rule = True
-                    break
-            if not expiration_rule:
-                # Add a default noncurrent object expiration rule if
-                # if one isn't already set
-                rule = {
-                    "id": "expire_noncurrent_versions",
-                    "enabled": "true",
-                    "noncurrent_version_expiration": {"days": 30},
-                }
-                if len(lrs) > 0:
-                    lrs.append(rule)
-                else:
-                    lrs = rule
-        sc = common_values.get("storage_class")
-        if sc:
-            sc = sc.upper()
-            days = "1"
-            if sc.endswith("_IA"):
-                # Infrequent Access storage class has minimum 30 days
-                # before transition
-                days = "30"
-            rule = {
-                "id": sc + "_storage_class",
-                "enabled": "true",
-                "transition": {"days": days, "storage_class": sc},
-                "noncurrent_version_transition": {"days": days, "storage_class": sc},
-            }
-            if values.get("lifecycle_rule"):
-                values["lifecycle_rule"].append(rule)
-            else:
-                values["lifecycle_rule"] = rule
         cors_rules = common_values.get("cors_rules")
         if cors_rules:
             # common_values['cors_rules'] is a list of cors_rules
@@ -5907,7 +5910,8 @@ class TerrascriptClient:
                 return commit.sha
             case "gitlab":
                 gitlab = self.init_gitlab()
-                project = gitlab.get_project(url)
+                if not (project := gitlab.get_project(url)):
+                    raise ValueError(f"could not find gitlab project for url {url}")
                 commits = project.commits.list(ref_name=ref, per_page=1, page=1)
                 return commits[0].id
             case _:

reconcile/utils/unleash/server.py

@@ -24,30 +24,24 @@ class Environment(BaseModel):
         return self.name == other
 
 
-class FeatureToggle(BaseModel):
+class FeatureToggle(BaseModel, validate_by_name=True, validate_by_alias=True):
     name: str
     type: FeatureToggleType = FeatureToggleType.release
     description: str | None = None
     impression_data: bool = Field(False, alias="impressionData")
     environments: list[Environment]
 
-    class Config:
-        allow_population_by_field_name = True
-
     def __eq__(self, other: object) -> bool:
         if isinstance(other, FeatureToggle):
             return self.name == other.name
         return self.name == other
 
 
-class Project(BaseModel):
+class Project(BaseModel, validate_by_name=True, validate_by_alias=True):
     pk: str = Field(alias="id")
     name: str
     feature_toggles: list[FeatureToggle] = []
 
-    class Config:
-        allow_population_by_field_name = True
-
 
 class TokenAuth(BearerTokenAuth):
     def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:

reconcile/utils/vault.py

@@ -6,7 +6,7 @@ import threading
 import time
 from collections.abc import Mapping
 from functools import lru_cache
-from typing import Any, Self, TypedDict
+from typing import Any, Self
 
 import hvac
 import requests
@@ -48,13 +48,6 @@ class VaultConnectionError(Exception):
     pass
 
 
-class Secret(TypedDict):
-    path: str
-    field: str
-    format: str | None
-    version: str | None
-
-
 SECRET_VERSION_LATEST = "LATEST"
 
 
@@ -197,7 +190,7 @@ class VaultClient:
             self._client.auth_approle(self.role_id, self.secret_id)
 
     @retry()
-    def read_all_with_version(self, secret: Mapping) -> tuple[Mapping, str | None]:
+    def read_all_with_version(self, secret: Mapping) -> tuple[dict, int | None]:
         """Returns a dictionary of keys and values in a Vault secret and the
         version of the secret, for V1 secrets, version will be None.
 
@@ -207,7 +200,7 @@ class VaultClient:
                                a v2 KV engine)
         """
         secret_path = secret["path"]
-        secret_version = secret.get("version", SECRET_VERSION_LATEST)
+        secret_version = secret.get("version") or SECRET_VERSION_LATEST
 
         kv_version = self._get_mount_version_by_secret_path(secret_path)
 
@@ -250,7 +243,7 @@ class VaultClient:
 
     def __read_all_v2(
         self, path: str, version: str | None
-    ) -> tuple[dict[str, Any], str | None]:
+    ) -> tuple[dict[str, Any], int]:
         path_split = path.split("/")
         mount_point = path_split[0]
         read_path = "/".join(path_split[1:])
@@ -294,7 +287,7 @@ class VaultClient:
         return secret["data"]
 
     @retry()
-    def read(self, secret: Secret) -> Any:
+    def read(self, secret: Mapping[str, Any]) -> Any:
         """Returns a value of a key in a Vault secret.
 
         The input secret is a dictionary which contains the following fields:

reconcile/utils/vcs.py

@@ -140,7 +140,7 @@ class VCS:
         gitlab_instances: Iterable[GitlabInstanceV1],
     ) -> GitLabApi:
         return GitLabApi(
-            next(iter(gitlab_instances)).dict(by_alias=True),
+            next(iter(gitlab_instances)).model_dump(by_alias=True),
             secret_reader=self._secret_reader,
         )
 
@@ -150,7 +150,7 @@ class VCS:
         app_interface_repo_url: str,
     ) -> GitLabApi:
         return GitLabApi(
-            next(iter(gitlab_instances)).dict(by_alias=True),
+            next(iter(gitlab_instances)).model_dump(by_alias=True),
             secret_reader=self._secret_reader,
             project_url=app_interface_repo_url,
         )
@@ -221,26 +221,26 @@ class VCS:
         match repo_info.platform:
             case "github":
                 github = self._init_github(repo_url=repo_url, auth_code=auth_code)
-                data = github.compare(commit_from=commit_from, commit_to=commit_to)
                 return [
                     Commit(
                         repo=repo_url,
                         sha=gh_commit.sha,
                         date=gh_commit.commit.committer.date,
                     )
-                    for gh_commit in data
+                    for gh_commit in github.compare(
+                        commit_from=commit_from, commit_to=commit_to
+                    )
                 ]
             case "gitlab":
-                data = self._gitlab_instance.repository_compare(
-                    repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
-                )
                 return [
                     Commit(
                         repo=repo_url,
                         sha=gl_commit["id"],
                         date=datetime.fromisoformat(gl_commit["committed_date"]),
                     )
-                    for gl_commit in data
+                    for gl_commit in self._gitlab_instance.repository_compare(
+                        repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
+                    )
                 ]
             case _:
                 raise ValueError(f"Unsupported repository URL: {repo_url}")

reconcile/vault_replication.py

@@ -217,7 +217,7 @@ def copy_vault_secret(
         return
 
     # If we reach here, we successfully read the destination secret
-    if dest_version is None and version is None:
+    if dest_version is None or version is None:
         # v1 secrets don't have version
         if source_data == dest_data:
             # If the secret is the same in both vaults, we don't need

tools/cli_commands/cost_report/cost_management_api.py

@@ -74,7 +74,7 @@ class CostManagementApi(ApiBase):
             timeout=self.read_timeout,
         )
         response.raise_for_status()
-        return AwsReportCostResponse.parse_obj(response.json())
+        return AwsReportCostResponse.model_validate(response.json())
 
     def get_openshift_costs_report(
         self,
@@ -97,7 +97,7 @@ class CostManagementApi(ApiBase):
             timeout=self.read_timeout,
         )
         response.raise_for_status()
-        return OpenShiftReportCostResponse.parse_obj(response.json())
+        return OpenShiftReportCostResponse.model_validate(response.json())
 
     def get_openshift_cost_optimization_report(
         self,
@@ -120,7 +120,7 @@ class CostManagementApi(ApiBase):
         response.raise_for_status()
 
         data = self._get_paginated(response)
-        return OpenShiftCostOptimizationReportResponse.parse_obj(data)
+        return OpenShiftCostOptimizationReportResponse.model_validate(data)
 
     def _get_paginated(
         self,

tools/cli_commands/cost_report/view.py

@@ -4,6 +4,7 @@ from typing import Any
 
 from pydantic import BaseModel
 
+from reconcile.utils.json import json_dumps
 from tools.cli_commands.cost_report.model import OptimizationReport, Report
 
 LAYOUT = """\
@@ -244,7 +245,7 @@ def render_summary(
     return template.format(
         date=get_date(reports),
         total_cost=format_cost_value(total_cost),
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 
@@ -274,7 +275,7 @@ def render_month_over_month_change(reports: Mapping[str, Report]) -> str:
     )
     return MONTH_OVER_MONTH_CHANGE.format(
         date=get_date(reports),
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 
@@ -304,7 +305,7 @@ def render_aws_services_cost(
         items_total=format_cost_value(report.items_total),
         items_delta_value=format_delta_value(report.items_delta_value),
         items_delta_percent=format_delta_percent(report.items_delta_percent),
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 
@@ -316,7 +317,7 @@ def render_openshift_workloads_cost(
         items_total=format_cost_value(report.items_total),
         items_delta_value=format_delta_value(report.items_delta_value),
         items_delta_percent=format_delta_percent(report.items_delta_percent),
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 
@@ -362,7 +363,7 @@ def render_child_apps_cost(report: Report) -> str:
     )
     return CHILD_APPS_COST.format(
         child_apps_total=format_cost_value(report.child_apps_total),
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 
@@ -509,7 +510,7 @@ def render_optimization(
     )
     return OPTIMIZATION.format(
         app_name=report.app_name,
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 

tools/cli_commands/erv2.py

@@ -133,7 +133,7 @@ class Erv2Cli:
 
     @property
     def input_data(self) -> str:
-        return self._resource.json(exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2}})
+        return self._resource.export(exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2}})
 
     @property
     def image(self) -> str:

tools/qontract_cli.py

@@ -141,6 +141,7 @@ from reconcile.utils.gitlab_api import (
 )
 from reconcile.utils.glitchtip.client import GlitchtipClient
 from reconcile.utils.gql import GqlApiSingleton
+from reconcile.utils.json import json_dumps
 from reconcile.utils.keycloak import (
     KeycloakAPI,
     SSOClient,
@@ -1566,7 +1567,7 @@ def rosa_create_cluster_command(ctx: click.Context, cluster_name: str) -> None:
         billing_account = account.billing_account.uid
     else:
         with AWSApi(
-            1, [account.dict(by_alias=True)], settings=settings, init_users=False
+            1, [account.model_dump(by_alias=True)], settings=settings, init_users=False
         ) as aws_api:
             billing_account = aws_api.get_organization_billing_account(account.name)
 
@@ -1750,7 +1751,7 @@ def aws_terraform_resources(ctx: click.Context) -> None:
     for ns_info in namespaces:
         specs = (
             get_external_resource_specs(
-                ns_info.dict(by_alias=True), provision_provider=PROVIDER_AWS
+                ns_info.model_dump(by_alias=True), provision_provider=PROVIDER_AWS
             )
             or []
         )
@@ -1808,7 +1809,7 @@ def rds(ctx: click.Context) -> None:
         specs = [
             s
             for s in get_external_resource_specs(
-                namespace.dict(by_alias=True), provision_provider=PROVIDER_AWS
+                namespace.model_dump(by_alias=True), provision_provider=PROVIDER_AWS
             )
             if s.provider == "rds"
         ]
@@ -2961,7 +2962,7 @@ def maintenances(ctx: click.Context) -> None:
     maintenances = maintenances_gql.query(gql.get_api().query).maintenances or []
     data = [
         {
-            **m.dict(),
+            **m.model_dump(),
             "services": ", ".join(a.name for a in m.affected_services),
         }
         for m in maintenances
@@ -4298,7 +4299,7 @@ def create(
         bg="red",
         fg="white",
     )
-    print(sso_client.json(by_alias=True, indent=2))
+    print(json_dumps(sso_client, indent=2))
 
 
 @sso_client.command()

tools/template_validation.py

@@ -67,7 +67,9 @@ def main(templates: tuple[str]) -> None:
             tests.append(test_yaml)
 
         template_raw["templateTest"] = tests
-        template: TemplateV1 = TemplateV1(**data_default_none(TemplateV1, template_raw))
+        data = data_default_none(TemplateV1, template_raw)
+        assert isinstance(data, dict)
+        template: TemplateV1 = TemplateV1(**data)
 
         # templates_to_validate = {}
         for test in template.template_test: