qontract-reconcile 0.10.2.dev349__py3-none-any.whl → 0.10.2.dev414__py3-none-any.whl

reconcile/utils/aws_api_typed/cloudformation.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from reconcile.utils.json import json_dumps
+
+if TYPE_CHECKING:
+    from mypy_boto3_cloudformation import CloudFormationClient
+    from mypy_boto3_cloudformation.type_defs import (
+        ParameterTypeDef,
+        StackTypeDef,
+        TagTypeDef,
+    )
+
+
+class AWSApiCloudFormation:
+    def __init__(self, client: CloudFormationClient) -> None:
+        self.client = client
+
+    def create_stack(
+        self,
+        stack_name: str,
+        change_set_name: str,
+        template_body: str,
+        parameters: dict[str, str] | None = None,
+        tags: dict[str, str] | None = None,
+    ) -> str:
+        """
+        Create a CloudFormation stack using a change set with import existing resources.
+        This method creates a change set of type "CREATE" with the provided template and parameters,
+        waits for the change set to be created, executes it, and then waits for the stack creation to complete.
+        It returns the StackId of the created stack.
+
+        Args:
+            stack_name (str): The name of the stack to create.
+            change_set_name (str): The name of the change set to create.
+            template_body (str): The CloudFormation template body as a string.
+            parameters (dict[str, str] | None): A dictionary of parameter key-value pairs for
+                the stack. Defaults to None.
+            tags (dict[str, str] | None): A dictionary of tag key-value pairs to
+                associate with the stack. Defaults to None.
+
+        Returns:
+            str: The StackId of the created stack.
+        """
+        response = self.client.create_change_set(
+            StackName=stack_name,
+            ChangeSetName=change_set_name,
+            TemplateBody=template_body,
+            ChangeSetType="CREATE",
+            Parameters=self._build_parameters(parameters or {}),
+            Tags=self._build_tags(tags or {}),
+            ImportExistingResources=True,
+        )
+        change_set_arn = response["Id"]
+        self.client.get_waiter("change_set_create_complete").wait(
+            ChangeSetName=change_set_arn
+        )
+        self.client.execute_change_set(ChangeSetName=change_set_arn)
+        self.client.get_waiter("stack_create_complete").wait(StackName=stack_name)
+        return response["StackId"]
+
+    def update_stack(
+        self,
+        stack_name: str,
+        template_body: str,
+        parameters: dict[str, str] | None = None,
+        tags: dict[str, str] | None = None,
+    ) -> str:
+        """
+        Update a CloudFormation stack with the provided template and parameters.
+        This method updates the specified stack, waits for the update to complete,
+        and returns the StackId of the updated stack.
+
+        Args:
+            stack_name (str): The name of the stack to update.
+            template_body (str): The CloudFormation template body as a string.
+            parameters (dict[str, str] | None): A dictionary of parameter key-value pairs for
+                the stack. Defaults to None.
+            tags (dict[str, str] | None): A dictionary of tag key-value pairs to
+                associate with the stack. Defaults to None.
+
+        Returns:
+            str: The StackId of the updated stack.
+        """
+        response = self.client.update_stack(
+            StackName=stack_name,
+            TemplateBody=template_body,
+            Parameters=self._build_parameters(parameters or {}),
+            Tags=self._build_tags(tags or {}),
+        )
+        self.client.get_waiter("stack_update_complete").wait(StackName=stack_name)
+        return response["StackId"]
+
+    def get_stack(self, stack_name: str) -> StackTypeDef | None:
+        """
+        Retrieve information about a CloudFormation stack by its name.
+        If the stack exists, it returns the stack details as a dictionary.
+        If the stack does not exist, it returns None.
+
+        Args:
+            stack_name (str): The name of the stack to retrieve.
+
+        Returns:
+            StackTypeDef | None: The stack details if found, otherwise None.
+        """
+        try:
+            response = self.client.describe_stacks(StackName=stack_name)
+            return response["Stacks"][0]
+        except self.client.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] == "ValidationError":
+                return None
+            raise
+
+    def get_template_body(self, stack_name: str) -> str:
+        """
+        Retrieve the CloudFormation template body for a specified stack.
+
+        Args:
+            stack_name (str): The name of the stack whose template is to be retrieved.
+
+        Returns:
+            str: The CloudFormation template body as a string.
+        """
+        response = self.client.get_template(StackName=stack_name)
+        # TemplateBody is str when using yaml
+        if isinstance(response["TemplateBody"], str):
+            return response["TemplateBody"]
+        return json_dumps(response["TemplateBody"])
+
+    @staticmethod
+    def _build_parameters(parameters: dict[str, str]) -> list[ParameterTypeDef]:
+        return [
+            {
+                "ParameterKey": key,
+                "ParameterValue": value,
+            }
+            for key, value in sorted(parameters.items())
+        ]
+
+    @staticmethod
+    def _build_tags(tags: dict[str, str]) -> list[TagTypeDef]:
+        return [
+            {
+                "Key": key,
+                "Value": value,
+            }
+            for key, value in sorted(tags.items())
+        ]

reconcile/utils/aws_api_typed/logs.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Iterator
+
+    from mypy_boto3_logs import CloudWatchLogsClient
+    from mypy_boto3_logs.type_defs import LogGroupTypeDef
+
+
+class AWSApiLogs:
+    def __init__(self, client: CloudWatchLogsClient) -> None:
+        self.client = client
+
+    def get_log_groups(self) -> Iterator[LogGroupTypeDef]:
+        paginator = self.client.get_paginator("describe_log_groups")
+        for page in paginator.paginate():
+            yield from page["logGroups"]
+
+    def delete_log_group(self, log_group_name: str) -> None:
+        self.client.delete_log_group(logGroupName=log_group_name)
+
+    def put_retention_policy(
+        self,
+        log_group_name: str,
+        retention_in_days: int,
+    ) -> None:
+        self.client.put_retention_policy(
+            logGroupName=log_group_name,
+            retentionInDays=retention_in_days,
+        )
+
+    def get_tags(self, arn: str) -> dict[str, str]:
+        tags = self.client.list_tags_for_resource(
+            resourceArn=self._normalize_log_group_arn(arn),
+        )
+        return tags.get("tags") or {}
+
+    def set_tags(
+        self,
+        arn: str,
+        tags: dict[str, str],
+    ) -> None:
+        self.client.tag_resource(
+            resourceArn=self._normalize_log_group_arn(arn),
+            tags=tags,
+        )
+
+    def delete_tags(
+        self,
+        arn: str,
+        tag_keys: Iterable[str],
+    ) -> None:
+        self.client.untag_resource(
+            resourceArn=self._normalize_log_group_arn(arn),
+            tagKeys=list(tag_keys),
+        )
+
+    @staticmethod
+    def _normalize_log_group_arn(arn: str) -> str:
+        """
+        Normalize a log group ARN by removing any trailing ":*".
+
+        DescribeLogGroups response arn has additional :* at the end.
+
+        Args:
+            arn: The ARN of the log group.
+
+        Returns:
+            The normalized ARN without the trailing ":*".
+        """
+        return arn.rstrip(":*")

reconcile/utils/aws_api_typed/organization.py

@@ -52,9 +52,11 @@ class AwsOrganizationOU(BaseModel):
 class AWSAccountStatus(BaseModel):
     id: str = Field(..., alias="Id")
     name: str = Field(..., alias="AccountName")
-    uid: str | None = Field(alias="AccountId")
+    uid: str | None = Field(None, alias="AccountId")
     state: str = Field(..., alias="State")
-    failure_reason: CreateAccountFailureReasonType | None = Field(alias="FailureReason")
+    failure_reason: CreateAccountFailureReasonType | None = Field(
+        None, alias="FailureReason"
+    )
 
 
 class AWSAccount(BaseModel):

reconcile/utils/datetime_util.py

@@ -0,0 +1,67 @@
+from datetime import UTC, datetime
+
+ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
+ISO8601_MICRO = "%Y-%m-%dT%H:%M:%S.%fZ"
+
+
+def utc_now() -> datetime:
+    """
+    Get the current UTC datetime.
+
+    Returns:
+        A datetime object representing the current time in UTC.
+    """
+    return datetime.now(tz=UTC)
+
+
+def ensure_utc(dt: datetime) -> datetime:
+    """
+    Ensure the provided datetime is in UTC timezone.
+
+    Args:
+        dt: A datetime object.
+
+    Returns:
+        A datetime object in UTC timezone.
+    """
+    if dt.tzinfo is None:
+        return dt.replace(tzinfo=UTC)
+    return dt.astimezone(UTC)
+
+
+def to_utc_seconds_iso_format(dt: datetime) -> str:
+    """
+    Convert a datetime object to ISO 8601 format YYYY-MM-DDTHH:MM:SSZ.
+
+    Args:
+        dt: A datetime object.
+
+    Returns:
+        A string representing the datetime in ISO 8601 format.
+    """
+    return ensure_utc(dt).strftime(ISO8601)
+
+
+def to_utc_microseconds_iso_format(dt: datetime) -> str:
+    """
+    Convert a datetime object to ISO 8601 format with microseconds YYYY-MM-DDTHH:MM:SS.mmmmmmZ.
+
+    Args:
+        dt: A datetime object.
+
+    Returns:
+        A string representing the datetime in ISO 8601 format with microseconds.
+    """
+    return ensure_utc(dt).strftime(ISO8601_MICRO)
+
+
+def from_utc_iso_format(dt_str: str) -> datetime:
+    """
+    Parse a datetime string in ISO 8601 format to a datetime object.
+
+    Args:
+        dt_str: A string representing the datetime in ISO 8601 format.
+    Returns:
+        A datetime object in UTC timezone.
+    """
+    return ensure_utc(datetime.fromisoformat(dt_str))

reconcile/utils/deadmanssnitch_api.py

@@ -26,7 +26,7 @@ class Snitch(BaseModel):
     interval: str
     alert_type: str
     alert_email: list[str]
-    vault_data: str | None
+    vault_data: str | None = None
 
     def needs_vault_update(self) -> bool:
         return self.vault_data is not None and self.check_in_url != self.vault_data

reconcile/utils/differ.py

@@ -7,7 +7,6 @@ from collections.abc import (
 from dataclasses import dataclass
 from typing import (
     Any,
-    Generic,
     TypeVar,
 )
 
@@ -18,13 +17,13 @@ Key = TypeVar("Key")
 
 
 @dataclass(frozen=True, eq=True)
-class DiffPair(Generic[Current, Desired]):
+class DiffPair[Current, Desired]:
     current: Current
     desired: Desired
 
 
 @dataclass(frozen=True, eq=True)
-class DiffResult(Generic[Current, Desired, Key]):
+class DiffResult[Current, Desired, Key]:
     add: dict[Key, Desired]
     delete: dict[Key, Current]
     change: dict[Key, DiffPair[Current, Desired]]

reconcile/utils/early_exit_cache.py

@@ -5,8 +5,9 @@ from functools import cached_property
 from typing import Any, Self
 
 from deepdiff import DeepHash
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.secret_reader import SecretReaderBase
 from reconcile.utils.state import State, init_state
 
@@ -16,7 +17,7 @@ CACHE_SOURCE_DIGEST_METADATA_KEY = "cache-source-digest"
 LATEST_CACHE_SOURCE_DIGEST_METADATA_KEY = "latest-cache-source-digest"
 
 
-class CacheKeyWithDigest(BaseModel):
+class CacheKeyWithDigest(BaseModel, frozen=True):
     integration: str
     integration_version: str
     dry_run: bool
@@ -69,9 +70,6 @@ class CacheKeyWithDigest(BaseModel):
             args.append(f"--shard {self.shard}")
         return " ".join(args)
 
-    class Config:
-        frozen = True
-
 
 class CacheKey(BaseModel):
     integration: str
@@ -121,9 +119,10 @@ class CacheKey(BaseModel):
         """
         return self.cache_key_with_digest.build_cli_delete_args()
 
-    class Config:
-        frozen = True
-        keep_untouched = (cached_property,)
+    model_config = ConfigDict(
+        frozen=True,
+        ignored_types=(cached_property,),
+    )
 
 
 class CacheValue(BaseModel):
@@ -167,7 +166,7 @@ class EarlyExitCache:
 
     def get(self, key: CacheKey) -> CacheValue:
         value = self.state.get(str(key))
-        return CacheValue.parse_obj(value)
+        return CacheValue.model_validate(value)
 
     def set(
         self,
@@ -185,7 +184,7 @@ class EarlyExitCache:
         :param latest_cache_source_digest: latest cache source digest, used to check stale for dry run cache
         :return: None
         """
-        expire_at = datetime.now(tz=UTC) + timedelta(seconds=ttl_seconds)
+        expire_at = utc_now() + timedelta(seconds=ttl_seconds)
         metadata = {
             EXPIRE_AT_METADATA_KEY: str(int(expire_at.timestamp())),
             CACHE_SOURCE_DIGEST_METADATA_KEY: key.cache_source_digest,
@@ -193,7 +192,7 @@ class EarlyExitCache:
         }
         self.state.add(
             str(key),
-            value.dict(),
+            value.model_dump(),
             metadata=metadata,
             force=True,
         )
@@ -233,7 +232,7 @@ class EarlyExitCache:
             int(metadata[EXPIRE_AT_METADATA_KEY]),
             tz=UTC,
         )
-        now = datetime.now(UTC)
+        now = utc_now()
         return now >= expire_at
 
     def _head_dry_run_status(

reconcile/utils/expiration.py

@@ -6,6 +6,8 @@ from typing import (
     cast,
 )
 
+from reconcile.utils.datetime_util import ensure_utc, utc_now
+
 DATE_FORMAT = "%Y-%m-%d"
 
 
@@ -17,12 +19,14 @@ DictsOrRoles = TypeVar("DictsOrRoles", bound=Iterable[FilterableRole] | Iterable
 
 
 def date_expired(date: str) -> bool:
-    exp_date = datetime.datetime.strptime(date, DATE_FORMAT).date()
-    current_date = datetime.datetime.utcnow().date()
+    exp_date = ensure_utc(datetime.datetime.strptime(date, DATE_FORMAT)).date()  # noqa: DTZ007
+    current_date = utc_now().date()
     return current_date >= exp_date
 
 
-def filter(roles: DictsOrRoles | None) -> DictsOrRoles:
+def filter[DictsOrRoles: Iterable[FilterableRole] | Iterable[dict]](
+    roles: DictsOrRoles | None,
+) -> DictsOrRoles:
     """Filters roles and returns the ones which are not yet expired."""
     filtered = []
     for r in roles or []:

reconcile/utils/external_resource_spec.py

@@ -144,13 +144,36 @@ class ExternalResourceSpec:
         return {}
 
     def tags(self, integration: str) -> dict[str, str]:
-        return {
+        tags = {
             "managed_by_integration": integration,
             "cluster": self.cluster_name,
             "namespace": self.namespace_name,
             "environment": self.namespace["environment"]["name"],
             "app": self.namespace["app"]["name"],
         }
+        if app_code := self.namespace["app"].get("appCode"):
+            tags["app-code"] = app_code
+        if cost_center := self.namespace["app"].get("costCenter"):
+            tags["cost-center"] = cost_center
+        if service_phase := self.namespace["environment"].get("servicePhase"):
+            tags["service-phase"] = service_phase
+
+        resource_tags_str = self.resource.get("tags")
+        if resource_tags_str:
+            resource_tags = json.loads(resource_tags_str)
+            # normalize camelCase keys to kebab-case
+            key_mapping = {
+                "appCode": "app-code",
+                "costCenter": "cost-center",
+                "servicePhase": "service-phase",
+            }
+            normalized_tags = {}
+            for key, value in resource_tags.items():
+                normalized_key = key_mapping.get(key, key)
+                normalized_tags[normalized_key] = value
+            tags.update(normalized_tags)
+
+        return tags
 
     def get_secret_field(self, field: str) -> str | None:
         return self.secret.get(field)

reconcile/utils/filtering.py

@@ -6,7 +6,7 @@ KeyType = TypeVar("KeyType")
 ValueType = TypeVar("ValueType")
 
 
-def remove_none_values_from_dict(
+def remove_none_values_from_dict[KeyType, ValueType](
     a_dict: dict[KeyType, ValueType | None],
 ) -> dict[KeyType, ValueType]:
     """

reconcile/utils/gitlab_api.py

@@ -263,13 +263,13 @@ class GitLabApi:
         # we can determine if a pending MR exists based on the title
         return any(mr.title == title for mr in mrs)
 
-    @retry()
+    @retry(no_retry_exceptions=(RuntimeError,))
     def get_project_maintainers(
         self, repo_url: str | None = None, query: dict | None = None
-    ) -> list[str] | None:
+    ) -> list[str]:
         project = self.project if repo_url is None else self.get_project(repo_url)
         if project is None:
-            return None
+            raise RuntimeError("project not found")
         members = project.members_all.list(iterator=True, query_parameters=query or {})
         return [m.username for m in members if m.access_level >= 40]
 
@@ -826,14 +826,16 @@ class GitLabApi:
         )
 
     def get_commit_sha(self, ref: str, repo_url: str) -> str:
-        project = self.get_project(repo_url)
+        if not (project := self.get_project(repo_url)):
+            raise ValueError(f"Project not found for repo_url: {repo_url}")
         commits = project.commits.list(ref_name=ref, per_page=1, page=1)
         return commits[0].id
 
     def repository_compare(
         self, repo_url: str, ref_from: str, ref_to: str
     ) -> list[dict[str, Any]]:
-        project = self.get_project(repo_url)
+        if not (project := self.get_project(repo_url)):
+            raise ValueError(f"Project not found for repo_url: {repo_url}")
         response: Any = project.repository_compare(ref_from, ref_to)
         return response.get("commits", [])
 

reconcile/utils/glitchtip/client.py

@@ -164,7 +164,9 @@ class GlitchtipClient(ApiBase):
         return ProjectAlert(
             **self._post(
                 f"/api/0/projects/{organization_slug}/{project_slug}/alerts/",
-                data=alert.dict(by_alias=True, exclude_unset=True, exclude_none=True),
+                data=alert.model_dump(
+                    by_alias=True, exclude_unset=True, exclude_none=True
+                ),
             )
         )
 
@@ -183,7 +185,9 @@ class GlitchtipClient(ApiBase):
         return ProjectAlert(
             **self._put(
                 f"/api/0/projects/{organization_slug}/{project_slug}/alerts/{alert.pk}/",
-                data=alert.dict(by_alias=True, exclude_unset=True, exclude_none=True),
+                data=alert.model_dump(
+                    by_alias=True, exclude_unset=True, exclude_none=True
+                ),
             )
         )
 

reconcile/utils/glitchtip/models.py

@@ -3,13 +3,13 @@ from __future__ import annotations
 import re
 from datetime import datetime
 from enum import Enum
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Self
 
 from pydantic import (
     BaseModel,
     Field,
-    root_validator,
-    validator,
+    field_validator,
+    model_validator,
 )
 
 if TYPE_CHECKING:
@@ -49,7 +49,8 @@ class Team(BaseModel):
     slug: str = ""
     users: list[User] = []
 
-    @root_validator(pre=True)
+    @model_validator(mode="before")
+    @classmethod
     def name_xor_slug_must_be_set(
         cls, values: MutableMapping[str, Any]
     ) -> MutableMapping[str, Any]:
@@ -58,11 +59,11 @@ class Team(BaseModel):
         ), "name xor slug must be set!"
         return values
 
-    @root_validator
-    def slugify(cls, values: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
-        values["slug"] = values.get("slug") or slugify(values.get("name", ""))
-        values["name"] = slugify(values.get("name", "")) or values.get("slug")
-        return values
+    @model_validator(mode="after")
+    def slugify(self) -> Self:
+        self.slug = self.slug or slugify(self.name)
+        self.name = slugify(self.name) or self.slug
+        return self
 
     def __lt__(self, other: Team) -> bool:
         return self.slug < other.slug
@@ -86,16 +87,15 @@ class RecipientType(Enum):
     WEBHOOK = "webhook"
 
 
-class ProjectAlertRecipient(BaseModel):
+class ProjectAlertRecipient(
+    BaseModel, validate_by_name=True, validate_by_alias=True, use_enum_values=True
+):
     pk: int | None = Field(None, alias="id")
     recipient_type: RecipientType = Field(..., alias="recipientType")
     url: str = ""
 
-    class Config:
-        allow_population_by_field_name = True
-        use_enum_values = True
-
-    @validator("recipient_type")
+    @field_validator("recipient_type")
+    @classmethod
     def recipient_type_enforce_enum_type(cls, v: str | RecipientType) -> RecipientType:
         if isinstance(v, RecipientType):
             return v
@@ -113,17 +113,15 @@ class ProjectAlertRecipient(BaseModel):
         return hash((self.recipient_type, self.url))
 
 
-class ProjectAlert(BaseModel):
+class ProjectAlert(BaseModel, validate_by_name=True, validate_by_alias=True):
     pk: int | None = Field(None, alias="id")
     name: str
     timespan_minutes: int = Field(..., alias="timespanMinutes")
     quantity: int
     recipients: list[ProjectAlertRecipient] = Field([], alias="alertRecipients")
 
-    class Config:
-        allow_population_by_field_name = True
-
-    @root_validator
+    @model_validator(mode="before")
+    @classmethod
     def empty_name(cls, values: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
         # name is an empty string if the alert was created manually because it can't be set via UI
         # use the pk instead.
@@ -141,20 +139,18 @@ class ProjectAlert(BaseModel):
         )
 
 
-class Project(BaseModel):
+class Project(BaseModel, validate_by_name=True, validate_by_alias=True):
     pk: int | None = Field(None, alias="id")
     name: str
     slug: str = ""
-    platform: str | None
+    platform: str | None = None
     teams: list[Team] = []
     alerts: list[ProjectAlert] = []
     event_throttle_rate: int = Field(0, alias="eventThrottleRate")
     organization: Organization | None = None
 
-    class Config:
-        allow_population_by_field_name = True
-
-    @root_validator
+    @model_validator(mode="before")
+    @classmethod
     def slugify(cls, values: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
         values["slug"] = values.get("slug") or slugify(values["name"])
         return values
@@ -195,7 +191,8 @@ class Organization(BaseModel):
     teams: list[Team] = []
     users: list[User] = []
 
-    @root_validator
+    @model_validator(mode="before")
+    @classmethod
     def slugify(cls, values: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
         values["slug"] = values.get("slug") or slugify(values["name"])
         return values
@@ -212,4 +209,4 @@ class Organization(BaseModel):
         return hash(self.name)
 
 
-Project.update_forward_refs()
+Project.model_rebuild()