qontract-reconcile 0.10.2.dev349__py3-none-any.whl → 0.10.2.dev414__py3-none-any.whl

reconcile/aws_saml_roles/integration.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import sys
 from collections.abc import (
@@ -7,10 +6,11 @@ from collections.abc import (
 )
 from typing import (
     Any,
+    Self,
     TypedDict,
 )
 
-from pydantic import BaseModel, root_validator, validator
+from pydantic import BaseModel, field_validator, model_validator
 
 from reconcile.gql_definitions.aws_saml_roles.aws_accounts import (
     AWSAccountV1,
@@ -22,6 +22,7 @@ from reconcile.gql_definitions.aws_saml_roles.roles import (
     query as roles_query,
 )
 from reconcile.status import ExitCodes
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.aws_helper import unique_sso_aws_accounts
@@ -32,6 +33,7 @@ from reconcile.utils.extended_early_exit import (
     ExtendedEarlyExitRunnerResult,
     extended_early_exit_run,
 )
+from reconcile.utils.json import json_dumps
 from reconcile.utils.runtime.integration import (
     PydanticRunParams,
     QontractReconcileIntegration,
@@ -58,7 +60,8 @@ class AwsSamlRolesIntegrationParams(PydanticRunParams):
     extended_early_exit_cache_ttl_seconds: int = 3600
     log_cached_log_output: bool = False
 
-    @validator("max_session_duration_hours")
+    @field_validator("max_session_duration_hours")
+    @classmethod
     def max_session_duration_range(cls, v: str | int) -> int:
         if 1 <= int(v) <= 12:
             return int(v)
@@ -69,7 +72,8 @@ class CustomPolicy(BaseModel):
     name: str
     policy: dict[str, Any]
 
-    @validator("name")
+    @field_validator("name")
+    @classmethod
     def name_size(cls, v: str) -> str:
         """Check the policy name size.
 
@@ -81,13 +85,14 @@ class CustomPolicy(BaseModel):
             )
         return v
 
-    @validator("policy")
+    @field_validator("policy")
+    @classmethod
     def policy_size(cls, v: dict[str, Any]) -> dict[str, Any]:
         """Check the policy size.
 
         See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html
         """
-        if len(json.dumps(v, separators=(",", ":"))) > 6144:
+        if len(json_dumps(v, compact=True)) > 6144:
             raise ValueError(
                 f"The policy document '{v}' is too large. AWS policy documents must be 6144 characters or less (w/o white spaces)."
             )
@@ -104,7 +109,8 @@ class AwsRole(BaseModel):
     custom_policies: list[CustomPolicy]
     managed_policies: list[ManagedPolicy]
 
-    @validator("name")
+    @field_validator("name")
+    @classmethod
     def name_size(cls, v: str) -> str:
         """Check the role name size.
 
@@ -116,29 +122,23 @@ class AwsRole(BaseModel):
             )
         return v
 
-    @root_validator
-    def validate_policies(cls, values: dict[str, Any]) -> dict[str, Any]:
+    @model_validator(mode="after")
+    def validate_policies(self) -> Self:
         """Check the policies.
 
         See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html
         """
-        custom_policies = values.get("custom_policies", [])
-        managed_policies = values.get("managed_policies", [])
-        if len(custom_policies) + len(managed_policies) > 20:
+        if len(self.custom_policies) + len(self.managed_policies) > 20:
             raise ValueError(
-                f"The role '{values['name']}' has too many policies. AWS roles can have at most 20 policies (via quota increase). Please consider consolidating the policies."
+                f"The role '{self.name}' has too many policies. AWS roles can have at most 20 policies (via quota increase). Please consider consolidating the policies."
             )
-        cp_names = [cp.name for cp in custom_policies]
+        cp_names = [cp.name for cp in self.custom_policies]
         if len(set(cp_names)) != len(cp_names):
-            raise ValueError(
-                f"The role '{values['name']}' has duplicate custom policies."
-            )
-        mp_names = [mp.name for mp in managed_policies]
+            raise ValueError(f"The role '{self.name}' has duplicate custom policies.")
+        mp_names = [mp.name for mp in self.managed_policies]
         if len(set(mp_names)) != len(mp_names):
-            raise ValueError(
-                f"The role '{values['name']}' has duplicate managed policies."
-            )
-        return values
+            raise ValueError(f"The role '{self.name}' has duplicate managed policies.")
+        return self
 
 
 class RunnerParams(TypedDict):
@@ -163,7 +163,7 @@ class AwsSamlRolesIntegration(
         if not query_func:
             query_func = gql.get_api().query
         return {
-            "roles": [c.dict() for c in self.get_roles(query_func)],
+            "roles": [c.model_dump() for c in self.get_roles(query_func)],
         }
 
     def get_aws_accounts(
@@ -251,15 +251,22 @@ class AwsSamlRolesIntegration(
         aws_accounts = self.get_aws_accounts(
             gql_api.query, account_name=self.params.account_name
         )
-        aws_accounts_dict = [account.dict(by_alias=True) for account in aws_accounts]
+        aws_accounts_dict = [
+            account.model_dump(by_alias=True) for account in aws_accounts
+        ]
         aws_roles = self.get_roles(gql_api.query, account_name=self.params.account_name)
-
+        try:
+            default_tags = get_settings().default_tags
+        except ValueError:
+            # no external resources settings found
+            default_tags = None
         ts = TerrascriptClient(
             self.name.replace("-", "_"),
             "",
             self.params.thread_pool_size,
             aws_accounts_dict,
             secret_reader=self.secret_reader,
+            default_tags=default_tags,
         )
         self.populate_saml_iam_roles(ts, aws_roles)
         working_dirs = ts.dump(print_to_file=self.params.print_to_file)

reconcile/aws_version_sync/integration.py

@@ -7,12 +7,7 @@ from enum import StrEnum
 from typing import Any
 
 import semver
-from pydantic import (
-    BaseModel,
-    ValidationError,
-    root_validator,
-    validator,
-)
+from pydantic import BaseModel, ValidationError, field_validator, model_validator
 
 from reconcile.aws_version_sync.merge_request_manager.merge_request import (
     Renderer,
@@ -81,7 +76,7 @@ class SupportedResourceProvider(StrEnum):
     ELASTICACHE = "elasticache"
 
 
-class ExternalResource(BaseModel):
+class ExternalResource(BaseModel, arbitrary_types_allowed=True):
     namespace_file: str | None = None
     provider: str = "aws"
     provisioner: ExternalResourceProvisioner
@@ -94,9 +89,6 @@ class ExternalResource(BaseModel):
     # used to map AWS cache name to resource_identifier
     redis_replication_group_id: str | None = None
 
-    class Config:
-        arbitrary_types_allowed = True
-
     @property
     def key(self) -> tuple:
         return (
@@ -106,7 +98,8 @@ class ExternalResource(BaseModel):
             self.resource_identifier,
         )
 
-    @validator("resource_engine_version", pre=True)
+    @field_validator("resource_engine_version", mode="before")
+    @classmethod
     def parse_resource_engine_version(
         cls, v: str | semver.VersionInfo
     ) -> semver.VersionInfo:
@@ -114,7 +107,8 @@ class ExternalResource(BaseModel):
             return v
         return parse_semver(str(v), optional_minor_and_patch=True)
 
-    @root_validator(pre=True)
+    @model_validator(mode="before")
+    @classmethod
     def set_resource_engine_version_format(cls, values: dict) -> dict:
         resource_engine_version, resource_engine_version_format = (
             str(values.get("resource_engine_version")),

reconcile/change_owners/bundle.py

@@ -62,8 +62,8 @@ class QontractServerDatafileDiff(BaseModel):
 
     datafilepath: str
     datafileschema: str
-    old: dict[str, Any] | None
-    new: dict[str, Any] | None
+    old: dict[str, Any] | None = None
+    new: dict[str, Any] | None = None
 
     @property
     def old_datafilepath(self) -> str | None:
@@ -119,7 +119,7 @@ class QontractServerResourcefileDiffState(BaseModel):
     content: str
     resourcefileschema: str | None = Field(..., alias="$schema")
     sha256sum: str
-    backrefs: list[QontractServerResourcefileBackref] | None
+    backrefs: list[QontractServerResourcefileBackref] | None = None
 
 
 class QontractServerResourcefileDiff(BaseModel):

reconcile/change_owners/change_log_tracking.py

@@ -155,7 +155,8 @@ class ChangeLogIntegration(QontractReconcileIntegration[ChangeLogIntegrationPara
             changes = aggregate_resource_changes(
                 bundle_changes=aggregate_file_moves(parse_bundle_changes(diff)),
                 content_store={
-                    c.path: c.dict(by_alias=True) for c in namespaces + jenkins_configs
+                    c.path: c.model_dump(by_alias=True)
+                    for c in namespaces + jenkins_configs
                 },
                 supported_schemas={
                     "/openshift/namespace-1.yml",
@@ -239,4 +240,4 @@ class ChangeLogIntegration(QontractReconcileIntegration[ChangeLogIntegrationPara
             change_log.items, key=lambda i: i.merged_at, reverse=True
         )
         if not dry_run:
-            integration_state.add(BUNDLE_DIFFS_OBJ, change_log.dict(), force=True)
+            integration_state.add(BUNDLE_DIFFS_OBJ, change_log.model_dump(), force=True)

reconcile/change_owners/change_owners.py

@@ -140,7 +140,7 @@ def write_coverage_report_to_mr(
     approver_reachability = set()
     for d in change_decisions:
         approvers = [
-            f"{cr.context} - {' '.join([f'@{a.org_username}' if a.tag_on_merge_requests else a.org_username for a in cr.approvers])}"
+            f"{cr.context} - {' '.join([f'@{a.org_username}' if (a.tag_on_merge_requests or len(cr.approvers) == 1) else a.org_username for a in cr.approvers])}"
             for cr in d.change_responsibles
         ]
         if d.coverable_by_fragment_decisions:

reconcile/change_owners/diff.py

@@ -1,5 +1,4 @@
 import copy
-import json
 from dataclasses import dataclass
 from enum import Enum
 from functools import reduce
@@ -11,6 +10,7 @@ from deepdiff.helper import CannotCompare
 from deepdiff.model import DiffLevel
 from deepdiff.path import parse_path
 
+from reconcile.utils.json import json_dumps
 from reconcile.utils.jsonpath import parse_jsonpath
 
 
@@ -75,7 +75,7 @@ class Diff:
     def _value_repr(self, value: Any | None) -> str | None:
         if value:
             if isinstance(value, dict | list):
-                return json.dumps(value, indent=2)
+                return json_dumps(value, indent=2)
             return str(value)
         return value
 
@@ -251,8 +251,6 @@ def deepdiff_path_to_jsonpath(deep_diff_path: str) -> jsonpath_ng.JSONPath:
             case int():
                 return jsonpath_ng.Index(element)
             case str():
-                if "." in element:
-                    return jsonpath_ng.Fields(f"'{element}'")
                 return jsonpath_ng.Fields(element)
 
     path_parts = [build_jsonpath_part(p) for p in parse_path(deep_diff_path)]

reconcile/checkpoint.py

@@ -26,6 +26,7 @@ from jira import Issue
 
 from reconcile.utils.constants import PROJ_ROOT
 from reconcile.utils.jira_client import JiraClient
+from reconcile.utils.secret_reader import SecretReaderBase
 
 DEFAULT_CHECKPOINT_LABELS = ("sre-checkpoint",)
 
@@ -118,8 +119,8 @@ def file_ticket(
 def report_invalid_metadata(
     app: Mapping[str, Any],
     path: str,
-    board: Mapping[str, str | Mapping],
-    settings: Mapping[str, Any],
+    board: Mapping[str, Any],
+    secret_reader: SecretReaderBase,
     parent: str,
     dry_run: bool = False,
 ) -> None:
@@ -150,7 +151,14 @@ def report_invalid_metadata(
             path=path,
         )
     else:
-        jira = JiraClient(board, settings)
+        jira = JiraClient.create(
+            project_name=board["name"],
+            token=secret_reader.read_secret(board["server"]["token"]),
+            email=secret_reader.read_secret(board["server"]["email"])
+            if board["server"]["email"]
+            else None,
+            server_url=board["server"]["server_url"],
+        )
         do_cut = partial(
             file_ticket,  # type: ignore
             jira=jira,

reconcile/cli.py

@@ -1,6 +1,5 @@
 # ruff: noqa: PLC0415 - `import` should be at the top-level of a file
 import faulthandler
-import json
 import logging
 import os
 import re
@@ -31,6 +30,7 @@ from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
 from reconcile.utils.exceptions import PrintToFileInGitRepositoryError
 from reconcile.utils.git import is_file_in_git_repo
 from reconcile.utils.gql import GqlApiSingleton
+from reconcile.utils.json import json_dumps
 from reconcile.utils.promtool import PROMTOOL_VERSION, PROMTOOL_VERSION_REGEX
 from reconcile.utils.runtime.environment import init_env
 from reconcile.utils.runtime.integration import (
@@ -608,7 +608,7 @@ def run_class_integration(
         if dump_schemas_file:
             gqlapi = gql.get_api()
             with open(dump_schemas_file, "w", encoding="locale") as f:
-                f.write(json.dumps(gqlapi.get_queried_schemas()))
+                f.write(json_dumps(gqlapi.get_queried_schemas()))
 
 
 @click.group()
@@ -1028,7 +1028,7 @@ def aws_account_manager(
     "--state-tmpl-resource",
     help="Resource name of the state template-collection template in the app-interface.",
     required=True,
-    default="/terraform-init/terraform-state.yml",
+    default="/terraform-init/terraform-state.yml.j2",
 )
 @click.option(
     "--template-collection-root-path",
@@ -1036,12 +1036,26 @@ def aws_account_manager(
     required=True,
     default="data/templating/collections/terraform-init",
 )
+@click.option(
+    "--cloudformation-template-resource",
+    help="Resource name of the CloudFormation template to create the S3 bucket",
+    required=True,
+    default="/terraform-init/terraform-state-s3-bucket.yaml",
+)
+@click.option(
+    "--cloudformation-import-template-resource",
+    help="Resource name of the CloudFormation template to import existing S3 bucket",
+    required=True,
+    default="/terraform-init/terraform-state-s3-bucket-import.yaml",
+)
 @click.pass_context
 def terraform_init(
     ctx: click.Context,
     account_name: str | None,
     state_tmpl_resource: str,
     template_collection_root_path: str,
+    cloudformation_template_resource: str,
+    cloudformation_import_template_resource: str,
 ) -> None:
     from reconcile.terraform_init.integration import (
         TerraformInitIntegration,
@@ -1054,6 +1068,8 @@ def terraform_init(
                 account_name=account_name,
                 state_tmpl_resource=state_tmpl_resource,
                 template_collection_root_path=template_collection_root_path,
+                cloudformation_template_resource=cloudformation_template_resource,
+                cloudformation_import_template_resource=cloudformation_import_template_resource,
             )
         ),
         ctx=ctx,
@@ -1135,9 +1151,17 @@ def jenkins_webhooks_cleaner(ctx: click.Context) -> None:
     "--jira-board-name", help="The Jira board to act on.", default=None, multiple=True
 )
 @click.option("--board-check-interval", help="Check interval in minutes", default=120)
+@click.option(
+    "--use-cache/--no-use-cache",
+    default=True,
+    help="Use cached results for validation.",
+)
 @click.pass_context
 def jira_permissions_validator(
-    ctx: click.Context, jira_board_name: Iterable[str] | None, board_check_interval: int
+    ctx: click.Context,
+    jira_board_name: Iterable[str] | None,
+    board_check_interval: int,
+    use_cache: bool,
 ) -> None:
     import reconcile.jira_permissions_validator
 
@@ -1146,6 +1170,7 @@ def jira_permissions_validator(
         ctx,
         jira_board_name=jira_board_name,
         board_check_interval_sec=board_check_interval * 60,
+        use_cache=use_cache,
     )
 
 
@@ -1270,14 +1295,14 @@ def aws_ami_cleanup(ctx: click.Context, thread_pool_size: int) -> None:
     run_integration(reconcile.aws_ami_cleanup.integration, ctx, thread_pool_size)
 
 
-@integration.command(short_help="Set up retention period for Cloudwatch logs.")
-@threaded()
+@integration.command(short_help="Set up retention period and tags for Cloudwatch logs.")
 @click.pass_context
-def aws_cloudwatch_log_retention(ctx: click.Context, thread_pool_size: int) -> None:
+def aws_cloudwatch_log_retention(ctx: click.Context) -> None:
     import reconcile.aws_cloudwatch_log_retention.integration
 
     run_integration(
-        reconcile.aws_cloudwatch_log_retention.integration, ctx, thread_pool_size
+        reconcile.aws_cloudwatch_log_retention.integration,
+        ctx,
     )
 
 

reconcile/dashdotdb_dora.py

@@ -4,7 +4,6 @@ from collections import defaultdict
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
 from datetime import (
-    UTC,
     datetime,
     timedelta,
 )
@@ -31,6 +30,7 @@ from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
 from reconcile.typed_queries.saas_files import get_saas_files
+from reconcile.utils.datetime_util import ensure_utc, utc_now
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.secret_reader import create_secret_reader
@@ -159,15 +159,8 @@ class Commit:
     date: datetime
 
     def lttc(self, finish_timestamp: datetime) -> int:
-        commit_date_tzaware = self.date
-        finish_timestamp_tzaware = finish_timestamp
-
-        if commit_date_tzaware.tzinfo is None:
-            commit_date_tzaware = commit_date_tzaware.replace(tzinfo=UTC)
-
-        if finish_timestamp_tzaware.tzinfo is None:
-            finish_timestamp_tzaware = finish_timestamp_tzaware.replace(tzinfo=UTC)
-
+        commit_date_tzaware = ensure_utc(self.date)
+        finish_timestamp_tzaware = ensure_utc(finish_timestamp)
         return int((finish_timestamp_tzaware - commit_date_tzaware).total_seconds())
 
 
@@ -277,7 +270,7 @@ class DashdotdbDORA(DashdotdbBase):
         # from the DB for a unique (app_name, env_name) multiple times.
         app_envs = {s.app_env for s in saastargets}
 
-        since_default = datetime.now() - timedelta(days=90)
+        since_default = utc_now() - timedelta(days=90)
         app_env_since_list: list[tuple[AppEnv, datetime]] = threaded.run(
             func=functools.partial(self.get_latest_with_default, since_default),
             iterable=app_envs,
@@ -473,7 +466,7 @@ class DashdotdbDORA(DashdotdbBase):
         ]
 
     def _github_compare_commits(self, rc: RepoChanges, repo: str) -> list[Commit]:
-        if not rc.repo_url:
+        if not rc.repo_url or not rc.ref_from or not rc.ref_to:
             return []
 
         return [

reconcile/dashdotdb_slo.py

@@ -119,4 +119,4 @@ def run(
 
 
 def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
-    return {doc.name: doc.dict() for doc in get_slo_documents()}
+    return {doc.name: doc.model_dump() for doc in get_slo_documents()}