qontract-reconcile 0.10.2.dev394__py3-none-any.whl → 0.10.2.dev414__py3-none-any.whl

reconcile/utils/raw_github_api.py

@@ -76,7 +76,7 @@ class RawGithubApi:
             if login is not None
         ]
 
-    def team_invitations(self, org_id: str, team_id: str) -> list[str]:
+    def team_invitations(self, org_id: str | int, team_id: str | int) -> list[str]:
         invitations = self.query(f"/organizations/{org_id}/team/{team_id}/invitations")
 
         return [

reconcile/utils/rhcsv2_certs.py

@@ -9,15 +9,12 @@ from cryptography.x509.oid import NameOID
 from pydantic import BaseModel, Field
 
 
-class RhcsV2Cert(BaseModel):
+class RhcsV2Cert(BaseModel, validate_by_name=True, validate_by_alias=True):
     certificate: str = Field(alias="tls.crt")
     private_key: str = Field(alias="tls.key")
     ca_cert: str = Field(alias="ca.crt")
     expiration_timestamp: int
 
-    class Config:
-        allow_population_by_field_name = True
-
 
 def extract_cert(text: str) -> re.Match:
     # The CA webform returns an HTML page with inline JS that builds an array of “outputList”

reconcile/utils/runtime/integration.py

@@ -119,7 +119,7 @@ class PydanticRunParams(RunParams, BaseModel):
     def copy_and_update(
         self: PydanticRunParamsSelfTypeVar, update: dict[str, Any]
     ) -> PydanticRunParamsSelfTypeVar:
-        return self.copy(update=update)
+        return self.model_copy(update=update)
 
     def get(self, field: str) -> Any:
         return getattr(self, field)

reconcile/utils/saasherder/interfaces.py

@@ -1,7 +1,7 @@
 # ruff: noqa: N801
 from __future__ import annotations
 
-from collections.abc import Mapping, Sequence, Set
+from collections.abc import Sequence
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -12,6 +12,8 @@ from typing import (
 from reconcile.utils import oc_connection_parameters
 
 if TYPE_CHECKING:
+    from pydantic.main import IncEx
+
     from reconcile.gql_definitions.fragments.saas_slo_document import SLODocument
     from reconcile.utils.secret_reader import HasSecret
 
@@ -27,18 +29,12 @@ class SaasFileSecretParameters(Protocol):
     @property
     def secret(self) -> HasSecret: ...
 
-    def dict(
-        self,
-        *,
-        by_alias: bool = False,
-        include: AbstractSetIntStr | MappingIntStrAny | None = None,
+    def model_dump(
+        self, *, by_alias: bool = False, include: IncEx | None = None
     ) -> dict[str, Any]: ...
 
 
 SaasSecretParameters = Sequence[SaasFileSecretParameters] | None
-# Taken from pydantic.typing
-AbstractSetIntStr = Set[int | str]
-MappingIntStrAny = Mapping[int | str, Any]
 
 
 @runtime_checkable
@@ -212,11 +208,8 @@ class SaasResourceTemplateTargetNamespace(Protocol):
     @property
     def cluster(self) -> oc_connection_parameters.Cluster: ...
 
-    def dict(
-        self,
-        *,
-        by_alias: bool = False,
-        include: AbstractSetIntStr | MappingIntStrAny | None = None,
+    def model_dump(
+        self, *, by_alias: bool = False, include: IncEx | None = None
     ) -> dict[str, Any]: ...
 
 
@@ -249,7 +242,7 @@ class SaasResourceTemplateTargetPromotion(Protocol):
     @property
     def promotion_data(self) -> Sequence[SaasPromotionData] | None: ...
 
-    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+    def model_dump(self, *, by_alias: bool = False) -> dict[str, Any]: ...
 
 
 class Channel(Protocol):
@@ -274,7 +267,7 @@ class SaasPromotion(Protocol):
     @property
     def subscribe(self) -> list[Channel] | None: ...
 
-    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+    def model_dump(self, *, by_alias: bool = False) -> dict[str, Any]: ...
 
 
 class SaasResourceTemplateTarget_SaasSecretParameters(Protocol):
@@ -295,7 +288,7 @@ class SaasResourceTemplateTargetUpstream(Protocol):
     @property
     def instance(self) -> SaasJenkinsInstance: ...
 
-    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+    def model_dump(self, *, by_alias: bool = False) -> dict[str, Any]: ...
 
 
 class SaasQuayInstance(Protocol):
@@ -315,7 +308,7 @@ class SaasResourceTemplateTargetImage(Protocol):
     @property
     def org(self) -> SaasQuayOrg: ...
 
-    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+    def model_dump(self, *, by_alias: bool = False) -> dict[str, Any]: ...
 
 
 class SaasResourceTemplateTarget(HasParameters, HasSecretParameters, Protocol):
@@ -344,7 +337,7 @@ class SaasResourceTemplateTarget(HasParameters, HasSecretParameters, Protocol):
         self, parent_saas_file_name: str, parent_resource_template_name: str
     ) -> str: ...
 
-    def dict(self, *, by_alias: bool = False) -> dict[str, Any]: ...
+    def model_dump(self, *, by_alias: bool = False) -> dict[str, Any]: ...
 
 
 class SaasResourceTemplate(HasParameters, HasSecretParameters, Protocol):
@@ -381,7 +374,7 @@ class ManagedResourceName(Protocol):
     resource: str
     resource_names: list[str]
 
-    def dict(self) -> dict[str, str]: ...
+    def model_dump(self) -> dict[str, str]: ...
 
 
 class SaasFile(HasParameters, HasSecretParameters, Protocol):

reconcile/utils/saasherder/models.py

@@ -7,10 +7,7 @@ from enum import Enum
 from typing import Any, NotRequired, TypedDict
 
 from github import Github
-from pydantic import (
-    BaseModel,
-    Field,
-)
+from pydantic import BaseModel, Field, model_validator
 
 from reconcile.gql_definitions.fragments.saas_slo_document import (
     SLODocument,
@@ -207,7 +204,12 @@ TriggerSpecUnion = (
 )
 
 
-class Namespace(BaseModel):
+class Namespace(
+    BaseModel,
+    validate_by_name=True,
+    validate_by_alias=True,
+    arbitrary_types_allowed=True,
+):
     name: str
     environment: SaasEnvironment
     app: SaasApp
@@ -217,29 +219,19 @@ class Namespace(BaseModel):
         ..., alias="managedResourceNames"
     )
 
-    class Config:
-        arbitrary_types_allowed = True
-        allow_population_by_field_name = True
-
 
-class PromotionChannelData(BaseModel):
+class PromotionChannelData(BaseModel, validate_by_name=True, validate_by_alias=True):
     q_type: str = Field(..., alias="type")
 
-    class Config:
-        allow_population_by_field_name = True
 
-
-class ParentSaasPromotion(BaseModel):
+class ParentSaasPromotion(BaseModel, validate_by_name=True, validate_by_alias=True):
     q_type: str = Field(..., alias="type")
-    parent_saas: str | None
-    target_config_hash: str | None
-
-    class Config:
-        allow_population_by_field_name = True
+    parent_saas: str | None = None
+    target_config_hash: str | None = None
 
 
 class PromotionData(BaseModel):
-    channel: str | None
+    channel: str | None = None
     data: list[ParentSaasPromotion | PromotionChannelData] | None = None
 
 
@@ -264,6 +256,17 @@ class Promotion(BaseModel):
     saas_file_paths: list[str] | None = None
     target_paths: list[str] | None = None
 
+    @model_validator(mode="before")
+    @classmethod
+    def handle_gql_classes(cls, data: Any) -> Any:
+        if data.get("promotion_data"):
+            data["promotion_data"] = [
+                # convert a GQL class to a dict
+                item.model_dump() if hasattr(item, "model_dump") else item
+                for item in data["promotion_data"]
+            ]
+        return data
+
 
 @dataclass
 class ImageAuth:

reconcile/utils/saasherder/saasherder.py

@@ -765,7 +765,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 content = self.gitlab.get_raw_file(
                     project=project,
                     path=path,
@@ -801,7 +802,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 dir_contents = self.gitlab.get_directory_contents(
                     project,
                     ref=commit_sha,
@@ -826,7 +828,8 @@ class SaasHerder:
             case "gitlab":
                 if not self.gitlab:
                     raise Exception("gitlab is not initialized")
-                project = self.gitlab.get_project(url)
+                if not (project := self.gitlab.get_project(url)):
+                    raise Exception(f"Could not find gitlab project for {url}")
                 commits = project.commits.list(ref_name=ref, per_page=1, page=1)
                 return commits[0].id
             case _:
@@ -1179,13 +1182,13 @@ class SaasHerder:
         images_list = threaded.run(
             self._collect_images, resources, self.available_thread_pool_size
         )
-        images = set(itertools.chain.from_iterable(images_list))
-        self.images.update(images)
-        if not images:
+        images_set = set(itertools.chain.from_iterable(images_list))
+        self.images.update(images_set)
+        if not images_set:
             return False  # no errors
         images = threaded.run(
             self._get_image,
-            images,
+            images_set,
             self.available_thread_pool_size,
             image_patterns=spec.image_patterns,
             image_auth=spec.image_auth,
@@ -1250,7 +1253,9 @@ class SaasHerder:
             self.saas_files,
             self.thread_pool_size,
         )
-        desired_state_specs = list(itertools.chain.from_iterable(results))
+        desired_state_specs: list[TargetSpec] = list(
+            itertools.chain.from_iterable(results)
+        )
         promotions = threaded.run(
             self.populate_desired_state_saas_file,
             desired_state_specs,
@@ -1898,21 +1903,23 @@ class SaasHerder:
             name=target.name,
             ref=target.ref,
             promotion=(
-                target.promotion.dict(by_alias=True) if target.promotion else None
+                target.promotion.model_dump(by_alias=True) if target.promotion else None
             ),
             secretParameters=(
-                [p.dict(by_alias=True) for p in target.secret_parameters]
+                [p.model_dump(by_alias=True) for p in target.secret_parameters]
                 if target.secret_parameters
                 else None
             ),
             slos=(
-                [slo.dict(by_alias=True) for slo in target.slos]
+                [slo.model_dump(by_alias=True) for slo in target.slos]
                 if target.slos
                 else None
             ),
-            upstream=(target.upstream.dict(by_alias=True) if target.upstream else None),
+            upstream=(
+                target.upstream.model_dump(by_alias=True) if target.upstream else None
+            ),
             images=(
-                [i.dict(by_alias=True) for i in target.images]
+                [i.model_dump(by_alias=True) for i in target.images]
                 if target.images
                 else None
             ),
@@ -1948,16 +1955,16 @@ class SaasHerder:
         )
         if saas_file.managed_resource_names:
             state_content["saas_file_managed_resource_names"] = [
-                m.dict() for m in saas_file.managed_resource_names
+                m.model_dump() for m in saas_file.managed_resource_names
             ]
         # include secret parameters from resource template and saas file
         if resource_template.secret_parameters:
             state_content["rt_secretparameters"] = [
-                p.dict() for p in resource_template.secret_parameters
+                p.model_dump() for p in resource_template.secret_parameters
             ]
         if saas_file.secret_parameters:
             state_content["saas_file_secretparameters"] = [
-                p.dict() for p in saas_file.secret_parameters
+                p.model_dump() for p in saas_file.secret_parameters
             ]
         return state_content
 
@@ -2240,7 +2247,9 @@ class SaasHerder:
             for rt in saas_file.resource_templates:
                 for target in rt.targets:
                     template_vars = {
-                        "resource": {"namespace": target.namespace.dict(by_alias=True)}
+                        "resource": {
+                            "namespace": target.namespace.model_dump(by_alias=True)
+                        }
                     }
                     if target.parameters:
                         for param in target.parameters:

reconcile/utils/slack_api.py

@@ -71,14 +71,14 @@ class HasClientGlobalConfig(Protocol):
     max_retries: int | None
     timeout: int | None
 
-    def dict(self) -> dict[str, int | None]: ...
+    def model_dump(self) -> dict[str, int | None]: ...
 
 
 class HasClientMethodConfig(Protocol):
     name: str
     args: Any
 
-    def dict(self) -> dict[str, str]: ...
+    def model_dump(self) -> dict[str, str]: ...
 
 
 class HasClientConfig(Protocol):

reconcile/utils/structs.py

@@ -1,4 +1,4 @@
-from pydantic.dataclasses import dataclass
+from dataclasses import dataclass
 
 
 @dataclass

reconcile/utils/terraform_client.py

@@ -223,7 +223,7 @@ class TerraformClient:
         if disable_deletions_detected:
             raise RuntimeError("Terraform plan has disabled deletions detected")
 
-    @retry(no_retry_exceptions=RdsUpgradeValidationError)
+    @retry(no_retry_exceptions=(RdsUpgradeValidationError,))
     def terraform_plan(
         self, spec: TerraformSpec, enable_deletion: bool
     ) -> tuple[bool, list[AccountUser], bool]:

reconcile/utils/terrascript_aws_client.py

@@ -1058,7 +1058,9 @@ class TerrascriptClient:
             ignore_changes = (
                 "all" if "all" in lifecycle.ignore_changes else lifecycle.ignore_changes
             )
-            return lifecycle.dict(by_alias=True) | {"ignore_changes": ignore_changes}
+            return lifecycle.model_dump(by_alias=True) | {
+                "ignore_changes": ignore_changes
+            }
         return None
 
     def populate_additional_providers(
@@ -1424,7 +1426,7 @@ class TerrascriptClient:
             req_account_name = req_account.name
             # Accepter's side of the connection - the cluster's account
             acc_account = accepter.account
-            acc_alias = self.get_provider_alias(acc_account.dict(by_alias=True))
+            acc_alias = self.get_provider_alias(acc_account.model_dump(by_alias=True))
             acc_uid = acc_account.uid
             if acc_account.assume_role:
                 acc_uid = awsh.get_account_uid_from_arn(acc_account.assume_role)
@@ -2210,6 +2212,43 @@ class TerrascriptClient:
         letters_and_digits = string.ascii_letters + string.digits
         return "".join(random.choice(letters_and_digits) for i in range(string_length))
 
+    @staticmethod
+    def _build_tf_resource_s3_lifecycle_rules(
+        versioning: bool,
+        common_values: Mapping[str, Any],
+    ) -> list[dict]:
+        lifecycle_rules = common_values.get("lifecycle_rules") or []
+        if versioning and not any(
+            "noncurrent_version_expiration" in lr for lr in lifecycle_rules
+        ):
+            # Add a default noncurrent object expiration rule
+            # if one isn't already set
+            rule = {
+                "id": "expire_noncurrent_versions",
+                "enabled": True,
+                "noncurrent_version_expiration": {"days": 30},
+                "expiration": {"expired_object_delete_marker": True},
+                "abort_incomplete_multipart_upload_days": 3,
+            }
+            lifecycle_rules.append(rule)
+
+        if storage_class := common_values.get("storage_class"):
+            sc = storage_class.upper()
+            days = "1"
+            if sc.endswith("_IA"):
+                # Infrequent Access storage class has minimum 30 days
+                # before transition
+                days = "30"
+            rule = {
+                "id": sc + "_storage_class",
+                "enabled": True,
+                "transition": {"days": days, "storage_class": sc},
+                "noncurrent_version_transition": {"days": days, "storage_class": sc},
+            }
+            lifecycle_rules.append(rule)
+
+        return lifecycle_rules
+
     def populate_tf_resource_s3(self, spec: ExternalResourceSpec) -> aws_s3_bucket:
         account = spec.provisioner_name
         identifier = spec.identifier
@@ -2249,47 +2288,11 @@ class TerrascriptClient:
         request_payer = common_values.get("request_payer")
         if request_payer:
             values["request_payer"] = request_payer
-        lifecycle_rules = common_values.get("lifecycle_rules")
-        if lifecycle_rules:
-            # common_values['lifecycle_rules'] is a list of lifecycle_rules
+        if lifecycle_rules := self._build_tf_resource_s3_lifecycle_rules(
+            versioning=versioning,
+            common_values=common_values,
+        ):
             values["lifecycle_rule"] = lifecycle_rules
-        if versioning:
-            lrs = values.get("lifecycle_rule", [])
-            expiration_rule = False
-            for lr in lrs:
-                if "noncurrent_version_expiration" in lr:
-                    expiration_rule = True
-                    break
-            if not expiration_rule:
-                # Add a default noncurrent object expiration rule if
-                # if one isn't already set
-                rule = {
-                    "id": "expire_noncurrent_versions",
-                    "enabled": "true",
-                    "noncurrent_version_expiration": {"days": 30},
-                }
-                if len(lrs) > 0:
-                    lrs.append(rule)
-                else:
-                    lrs = rule
-        sc = common_values.get("storage_class")
-        if sc:
-            sc = sc.upper()
-            days = "1"
-            if sc.endswith("_IA"):
-                # Infrequent Access storage class has minimum 30 days
-                # before transition
-                days = "30"
-            rule = {
-                "id": sc + "_storage_class",
-                "enabled": "true",
-                "transition": {"days": days, "storage_class": sc},
-                "noncurrent_version_transition": {"days": days, "storage_class": sc},
-            }
-            if values.get("lifecycle_rule"):
-                values["lifecycle_rule"].append(rule)
-            else:
-                values["lifecycle_rule"] = rule
         cors_rules = common_values.get("cors_rules")
         if cors_rules:
             # common_values['cors_rules'] is a list of cors_rules
@@ -5907,7 +5910,8 @@ class TerrascriptClient:
                 return commit.sha
             case "gitlab":
                 gitlab = self.init_gitlab()
-                project = gitlab.get_project(url)
+                if not (project := gitlab.get_project(url)):
+                    raise ValueError(f"could not find gitlab project for url {url}")
                 commits = project.commits.list(ref_name=ref, per_page=1, page=1)
                 return commits[0].id
             case _:

reconcile/utils/unleash/server.py

@@ -24,30 +24,24 @@ class Environment(BaseModel):
         return self.name == other
 
 
-class FeatureToggle(BaseModel):
+class FeatureToggle(BaseModel, validate_by_name=True, validate_by_alias=True):
     name: str
     type: FeatureToggleType = FeatureToggleType.release
     description: str | None = None
     impression_data: bool = Field(False, alias="impressionData")
     environments: list[Environment]
 
-    class Config:
-        allow_population_by_field_name = True
-
     def __eq__(self, other: object) -> bool:
         if isinstance(other, FeatureToggle):
             return self.name == other.name
         return self.name == other
 
 
-class Project(BaseModel):
+class Project(BaseModel, validate_by_name=True, validate_by_alias=True):
     pk: str = Field(alias="id")
     name: str
     feature_toggles: list[FeatureToggle] = []
 
-    class Config:
-        allow_population_by_field_name = True
-
 
 class TokenAuth(BearerTokenAuth):
     def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:

reconcile/utils/vault.py

@@ -6,7 +6,7 @@ import threading
 import time
 from collections.abc import Mapping
 from functools import lru_cache
-from typing import Any, Self, TypedDict
+from typing import Any, Self
 
 import hvac
 import requests
@@ -48,13 +48,6 @@ class VaultConnectionError(Exception):
     pass
 
 
-class Secret(TypedDict):
-    path: str
-    field: str
-    format: str | None
-    version: str | None
-
-
 SECRET_VERSION_LATEST = "LATEST"
 
 
@@ -197,7 +190,7 @@ class VaultClient:
             self._client.auth_approle(self.role_id, self.secret_id)
 
     @retry()
-    def read_all_with_version(self, secret: Mapping) -> tuple[Mapping, str | None]:
+    def read_all_with_version(self, secret: Mapping) -> tuple[dict, int | None]:
         """Returns a dictionary of keys and values in a Vault secret and the
         version of the secret, for V1 secrets, version will be None.
 
@@ -250,7 +243,7 @@ class VaultClient:
 
     def __read_all_v2(
         self, path: str, version: str | None
-    ) -> tuple[dict[str, Any], str | None]:
+    ) -> tuple[dict[str, Any], int]:
         path_split = path.split("/")
         mount_point = path_split[0]
         read_path = "/".join(path_split[1:])
@@ -294,7 +287,7 @@ class VaultClient:
         return secret["data"]
 
     @retry()
-    def read(self, secret: Secret) -> Any:
+    def read(self, secret: Mapping[str, Any]) -> Any:
         """Returns a value of a key in a Vault secret.
 
         The input secret is a dictionary which contains the following fields:

reconcile/utils/vcs.py

@@ -140,7 +140,7 @@ class VCS:
         gitlab_instances: Iterable[GitlabInstanceV1],
     ) -> GitLabApi:
         return GitLabApi(
-            next(iter(gitlab_instances)).dict(by_alias=True),
+            next(iter(gitlab_instances)).model_dump(by_alias=True),
             secret_reader=self._secret_reader,
         )
 
@@ -150,7 +150,7 @@ class VCS:
         app_interface_repo_url: str,
     ) -> GitLabApi:
         return GitLabApi(
-            next(iter(gitlab_instances)).dict(by_alias=True),
+            next(iter(gitlab_instances)).model_dump(by_alias=True),
             secret_reader=self._secret_reader,
             project_url=app_interface_repo_url,
         )
@@ -221,26 +221,26 @@ class VCS:
         match repo_info.platform:
             case "github":
                 github = self._init_github(repo_url=repo_url, auth_code=auth_code)
-                data = github.compare(commit_from=commit_from, commit_to=commit_to)
                 return [
                     Commit(
                         repo=repo_url,
                         sha=gh_commit.sha,
                         date=gh_commit.commit.committer.date,
                     )
-                    for gh_commit in data
+                    for gh_commit in github.compare(
+                        commit_from=commit_from, commit_to=commit_to
+                    )
                 ]
             case "gitlab":
-                data = self._gitlab_instance.repository_compare(
-                    repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
-                )
                 return [
                     Commit(
                         repo=repo_url,
                         sha=gl_commit["id"],
                         date=datetime.fromisoformat(gl_commit["committed_date"]),
                     )
-                    for gl_commit in data
+                    for gl_commit in self._gitlab_instance.repository_compare(
+                        repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
+                    )
                 ]
             case _:
                 raise ValueError(f"Unsupported repository URL: {repo_url}")

reconcile/vault_replication.py

@@ -217,7 +217,7 @@ def copy_vault_secret(
         return
 
     # If we reach here, we successfully read the destination secret
-    if dest_version is None and version is None:
+    if dest_version is None or version is None:
         # v1 secrets don't have version
         if source_data == dest_data:
             # If the secret is the same in both vaults, we don't need

tools/cli_commands/cost_report/cost_management_api.py

@@ -74,7 +74,7 @@ class CostManagementApi(ApiBase):
             timeout=self.read_timeout,
         )
         response.raise_for_status()
-        return AwsReportCostResponse.parse_obj(response.json())
+        return AwsReportCostResponse.model_validate(response.json())
 
     def get_openshift_costs_report(
         self,
@@ -97,7 +97,7 @@ class CostManagementApi(ApiBase):
             timeout=self.read_timeout,
         )
         response.raise_for_status()
-        return OpenShiftReportCostResponse.parse_obj(response.json())
+        return OpenShiftReportCostResponse.model_validate(response.json())
 
     def get_openshift_cost_optimization_report(
         self,
@@ -120,7 +120,7 @@ class CostManagementApi(ApiBase):
         response.raise_for_status()
 
         data = self._get_paginated(response)
-        return OpenShiftCostOptimizationReportResponse.parse_obj(data)
+        return OpenShiftCostOptimizationReportResponse.model_validate(data)
 
     def _get_paginated(
         self,