qontract-reconcile 0.10.2.dev310__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl

reconcile/slack_usergroups.py

@@ -3,18 +3,19 @@ import sys
 from collections.abc import (
     Callable,
     Iterable,
+    MutableMapping,
     Sequence,
 )
 from datetime import datetime
 from typing import (
     Any,
     TypedDict,
+    TypeVar,
 )
 
 from github.GithubException import UnknownObjectException
 from pydantic import BaseModel
-from pydantic.utils import deep_update
-from sretoolbox.utils import retry
+from sretoolbox.utils import datatransformation, retry
 
 from reconcile import (
     openshift_users,
@@ -40,6 +41,7 @@ from reconcile.typed_queries.app_interface_vault_settings import (
 )
 from reconcile.typed_queries.pagerduty_instances import get_pagerduty_instances
 from reconcile.utils import gql
+from reconcile.utils.datetime_util import ensure_utc, utc_now
 from reconcile.utils.disabled_integrations import integration_is_enabled
 from reconcile.utils.exceptions import (
     AppInterfaceSettingsError,
@@ -74,6 +76,26 @@ INTEGRATION_VERSION = "0.1.0"
 
 error_occurred = False
 
+KeyType = TypeVar("KeyType")
+
+
+def deep_update(
+    mapping: dict[KeyType, Any],
+    *updating_mappings: MutableMapping[KeyType, Any],
+) -> dict[KeyType, Any]:
+    updated_mapping = mapping.copy()
+    for updating_mapping in updating_mappings:
+        for k, v in updating_mapping.items():
+            if (
+                k in updated_mapping
+                and isinstance(updated_mapping[k], dict)
+                and isinstance(v, dict)
+            ):
+                updated_mapping[k] = deep_update(updated_mapping[k], v)
+            else:
+                updated_mapping[k] = v
+    return updated_mapping
+
 
 def get_git_api(url: str) -> GithubRepositoryApi | GitLabApi:
     """Return GitHub/GitLab API based on url."""
@@ -122,15 +144,12 @@ class State(BaseModel):
 SlackState = dict[str, dict[str, State]]
 
 
-class WorkspaceSpec(BaseModel):
+class WorkspaceSpec(BaseModel, arbitrary_types_allowed=True):
     """Slack workspace spec."""
 
     slack: SlackApi
     managed_usergroups: list[str] = []
 
-    class Config:
-        arbitrary_types_allowed = True
-
 
 SlackMap = dict[str, WorkspaceSpec]
 
@@ -357,11 +376,11 @@ def get_slack_usernames_from_owners(
 
 def get_slack_usernames_from_schedule(schedule: Iterable[ScheduleEntryV1]) -> list[str]:
     """Return list of usernames from all schedules."""
-    now = datetime.utcnow()
+    now = utc_now()
     all_slack_usernames: list[str] = []
     for entry in schedule:
-        start = datetime.strptime(entry.start, DATE_FORMAT)
-        end = datetime.strptime(entry.end, DATE_FORMAT)
+        start = ensure_utc(datetime.strptime(entry.start, DATE_FORMAT))  # noqa: DTZ007
+        end = ensure_utc(datetime.strptime(entry.end, DATE_FORMAT))  # noqa: DTZ007
         if start <= now <= end:
             all_slack_usernames.extend(get_slack_username(u) for u in entry.users)
     return all_slack_usernames
@@ -819,7 +838,9 @@ def run(
         desired_usergroup_name=usergroup_name,
     )
     # merge the two desired states recursively
-    desired_state = deep_update(desired_state, desired_state_cluster_usergroups)
+    desired_state = datatransformation.deep_merge(
+        desired_state, desired_state_cluster_usergroups
+    )
 
     runner_params: RunnerParams = {
         "dry_run": dry_run,
@@ -891,10 +912,10 @@ def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
             if role.tag_on_cluster_updates is not False
         ]
     return {
-        "permissions": [p.dict() for p in get_permissions(gqlapi.query)],
+        "permissions": [p.model_dump() for p in get_permissions(gqlapi.query)],
         "pagerduty_instances": [
-            p.dict() for p in get_pagerduty_instances(gqlapi.query)
+            p.model_dump() for p in get_pagerduty_instances(gqlapi.query)
         ],
-        "users": [u.dict() for u in users],
-        "clusters": [c.dict() for c in get_clusters(gqlapi.query)],
+        "users": [u.model_dump() for u in users],
+        "clusters": [c.model_dump() for c in get_clusters(gqlapi.query)],
     }

reconcile/sql_query.py

@@ -229,6 +229,7 @@ def collect_queries(
         accounts=[],
         prefetch_resources_by_schemas=["/aws/rds-defaults-1.yml"],
         secret_reader=secret_reader,
+        default_tags=None,
     )
 
     for sql_query in sql_queries:

reconcile/status.py

@@ -11,7 +11,7 @@ class ExitCodes:
 class _RunningState:
     _state: dict[Any, Any] = {}
 
-    def __init__(self):
+    def __init__(self) -> None:
         self.__dict__ = self._state
 
 
@@ -22,7 +22,7 @@ class RunningState(_RunningState):
     by the callers.
     """
 
-    def __getattr__(self, item):
+    def __getattr__(self, item: str) -> None:
         """
         Default value for attributes not explicitly created is None.
         """

reconcile/status_board.py

@@ -52,7 +52,7 @@ class AbstractStatusBoard(ABC, BaseModel):
     """Abstract class for upgrade policies
     Used to create and delete upgrade policies in OCM."""
 
-    id: str | None
+    id: str | None = None
     name: str
     fullname: str
 
@@ -88,7 +88,7 @@ class AbstractStatusBoard(ABC, BaseModel):
 
 
 class Product(AbstractStatusBoard):
-    applications: list["Application"] | None
+    applications: list["Application"] | None = None
 
     def create(self, ocm: OCMBaseClient) -> None:
         spec = self.to_ocm_spec()
@@ -121,7 +121,7 @@ class Product(AbstractStatusBoard):
 
 class Application(AbstractStatusBoard):
     product: Product
-    services: list["Service"] | None
+    services: list["Service"] | None = None
 
     def create(self, ocm: OCMBaseClient) -> None:
         if self.product.id:
@@ -214,9 +214,9 @@ class Service(AbstractStatusBoard):
 
 
 # Resolve forward references after class definitions
-Product.update_forward_refs()
-Application.update_forward_refs()
-Service.update_forward_refs()
+Product.model_rebuild()
+Application.model_rebuild()
+Service.model_rebuild()
 
 
 class UpdateNotSupportedError(Exception):

reconcile/statuspage/atlassian.py

@@ -29,12 +29,12 @@ class AtlassianRawComponent(BaseModel):
 
     id: str
     name: str
-    description: str | None
+    description: str | None = None
     position: int
     status: str
-    automation_email: str | None
-    group_id: str | None
-    group: bool | None
+    automation_email: str | None = None
+    group_id: str | None = None
+    group: bool | None = None
 
 
 class AtlassianRawMaintenanceUpdate(BaseModel):
@@ -56,9 +56,9 @@ class AtlassianRawMaintenance(BaseModel):
     scheduled_until: str
     incident_updates: list[AtlassianRawMaintenanceUpdate]
     components: list[AtlassianRawComponent]
-    auto_transition_deliver_notifications_at_end: bool | None
-    auto_transition_deliver_notifications_at_start: bool | None
-    scheduled_remind_prior: bool | None
+    auto_transition_deliver_notifications_at_end: bool | None = None
+    auto_transition_deliver_notifications_at_start: bool | None = None
+    scheduled_remind_prior: bool | None = None
 
 
 class AtlassianAPI:

reconcile/statuspage/integrations/maintenances.py

@@ -1,12 +1,13 @@
 import logging
 import sys
-from datetime import UTC, datetime, timedelta
+from datetime import datetime, timedelta
 
 from reconcile.slack_base import slackapi_from_queries
 from reconcile.statuspage.atlassian import AtlassianStatusPageProvider
 from reconcile.statuspage.integration import get_binding_state, get_status_pages
 from reconcile.statuspage.page import StatusMaintenance
 from reconcile.statuspage.state import S3ComponentBindingState
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.differ import diff_iterables
 from reconcile.utils.runtime.integration import (
     NoParams,
@@ -52,7 +53,7 @@ class StatusPageMaintenancesIntegration(QontractReconcileIntegration[NoParams]):
         desired_state: list[StatusMaintenance],
         binding_state: S3ComponentBindingState,
     ) -> None:
-        now = datetime.now(UTC)
+        now = utc_now()
         slack = slackapi_from_queries(QONTRACT_INTEGRATION, init_usergroups=False)
         for m in desired_state:
             scheduled_start = m.schedule_start
@@ -68,7 +69,7 @@ class StatusPageMaintenancesIntegration(QontractReconcileIntegration[NoParams]):
     def run(self, dry_run: bool = False) -> None:
         binding_state = get_binding_state(self.name, self.secret_reader)
         pages = get_status_pages()
-        now = datetime.now(UTC)
+        now = utc_now()
 
         error = False
         for p in pages:

reconcile/statuspage/page.py

@@ -19,19 +19,17 @@ from reconcile.statuspage.status import (
 PROVIDER_NAME = "statuspage"
 
 
-class StatusComponent(BaseModel):
+class StatusComponent(BaseModel, arbitrary_types_allowed=True):
     """
     Represents a status page component from the desired state.
     """
 
     name: str
     display_name: str
-    description: str | None
-    group_name: str | None
+    description: str | None = None
+    group_name: str | None = None
+    # Status provider configs hold different ways for a component to determine its status
     status_provider_configs: list[StatusProvider]
-    """
-    Status provider configs hold different ways for a component to determine its status
-    """
 
     def status_management_enabled(self) -> bool:
         """
@@ -49,9 +47,6 @@ class StatusComponent(BaseModel):
             return "operational"
         return None
 
-    class Config:
-        arbitrary_types_allowed = True
-
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, StatusComponent):
             raise NotImplementedError("Cannot compare to non StatusComponent objects.")

reconcile/statuspage/status.py

@@ -2,18 +2,15 @@ from abc import (
     ABC,
     abstractmethod,
 )
-from datetime import (
-    UTC,
-    datetime,
-)
+from datetime import datetime
 
-from dateutil.parser import isoparse
 from pydantic import BaseModel
 
 from reconcile.gql_definitions.statuspage.statuspages import (
     ManualStatusProviderV1,
     StatusProviderV1,
 )
+from reconcile.utils.datetime_util import from_utc_iso_format, utc_now
 
 # This module defines the interface for status providers for components on status
 # pages. A status provider is responsible for determining the status of a component.
@@ -70,7 +67,7 @@ class ManualStatusProvider(StatusProvider, BaseModel):
             raise ValueError(
                 "manual component status time window is invalid: end before start"
             )
-        now = datetime.now(UTC)
+        now = utc_now()
         if self.start and now < self.start:
             return False
         return not (self.end and self.end < now)
@@ -84,8 +81,8 @@ def build_status_provider_config(
     provider specific implementation that provides the status resolution logic.
     """
     if isinstance(cfg, ManualStatusProviderV1):
-        start = isoparse(cfg.manual.q_from) if cfg.manual.q_from else None
-        end = isoparse(cfg.manual.until) if cfg.manual.until else None
+        start = from_utc_iso_format(cfg.manual.q_from) if cfg.manual.q_from else None
+        end = from_utc_iso_format(cfg.manual.until) if cfg.manual.until else None
         return ManualStatusProvider(
             component_status=cfg.manual.component_status,
             start=start,

reconcile/templates/rosa-classic-cluster-creation.sh.j2

@@ -47,7 +47,7 @@ rosa create cluster -y --cluster-name={{ cluster_name }} \
     --service-cidr {{ cluster.network.service }} \
     --pod-cidr {{ cluster.network.pod }} \
     --host-prefix 23 \
-    --replicas {{ cluster.machine_pools | length }} \
+    --replicas 3 \
     --compute-machine-type {{ cluster.machine_pools[0].instance_type }} \
     {% if cluster.spec.disable_user_workload_monitoring -%}
     --disable-workload-monitoring \
@@ -55,4 +55,8 @@ rosa create cluster -y --cluster-name={{ cluster_name }} \
     {% if cluster.spec.provision_shard_id -%}
     --properties provision_shard_id:{{ cluster.spec.provision_shard_id }} \
     {% endif -%}
+    {% if cluster.spec.fips -%}
+    --fips \
+    {% endif -%}
     --channel-group {{ cluster.spec.channel }}
+

reconcile/templates/rosa-hcp-cluster-creation.sh.j2

@@ -47,7 +47,7 @@ rosa create cluster --cluster-name={{ cluster_name }} \
     --service-cidr {{ cluster.network.service }} \
     --pod-cidr {{ cluster.network.pod }} \
     --host-prefix 23 \
-    --replicas {{ cluster.machine_pools | length }} \
+    --replicas 3 \
     --compute-machine-type {{ cluster.machine_pools[0].instance_type }} \
     {% if cluster.spec.private -%}
     --private \
@@ -59,4 +59,7 @@ rosa create cluster --cluster-name={{ cluster_name }} \
     {% if cluster.spec.provision_shard_id -%}
     --properties provision_shard_id:{{ cluster.spec.provision_shard_id }} \
     {% endif -%}
+    {% if cluster.spec.fips -%}
+    --fips \
+    {% endif -%}
     --channel-group {{ cluster.spec.channel }}

reconcile/templating/lib/merge_request_manager.py

@@ -108,14 +108,14 @@ class TemplateRenderingMR(MergeRequestBase):
             if content.is_new:
                 gitlab_cli.create_file(
                     branch_name=self.branch,
-                    file_path=f"data{content.path}",
+                    file_path=content.path.lstrip("/"),
                     commit_message="termplate rendering output",
                     content=content.content,
                 )
             else:
                 gitlab_cli.update_file(
                     branch_name=self.branch,
-                    file_path=f"data{content.path}",
+                    file_path=content.path.lstrip("/"),
                     commit_message="termplate rendering output",
                     content=content.content,
                 )

reconcile/templating/lib/rendering.py

@@ -18,7 +18,7 @@ from reconcile.utils.secret_reader import SecretReaderBase
 
 class TemplateData(BaseModel):
     variables: dict[str, Any]
-    current: dict[str, Any] | None
+    current: dict[str, Any] | None = None
     current_with_explicit_start: bool | None = False
 
 
@@ -26,7 +26,7 @@ class TemplatePatch(Protocol):
     path: str
     identifier: str | None
 
-    def dict(self) -> dict[str, str]: ...
+    def model_dump(self) -> dict[str, str]: ...
 
 
 class Template(Protocol):
@@ -36,7 +36,7 @@ class Template(Protocol):
     template: str
     overwrite: bool | None
 
-    def dict(self) -> dict[str, str]: ...
+    def model_dump(self) -> dict[str, str]: ...
 
     @property
     def patch(self) -> TemplatePatch | None:

reconcile/templating/renderer.py

@@ -33,6 +33,7 @@ from reconcile.utils import gql
 from reconcile.utils.git import checkout, clone
 from reconcile.utils.gql import GqlApi
 from reconcile.utils.jinja2.utils import TemplateRenderOptions, process_jinja2_template
+from reconcile.utils.json import json_dumps
 from reconcile.utils.ruamel import create_ruamel_instance
 from reconcile.utils.runtime.integration import (
     PydanticRunParams,
@@ -95,18 +96,16 @@ class LocalFilePersistence(FilePersistence):
     This class provides a simple file persistence implementation for local files.
     """
 
-    def __init__(self, dry_run: bool, app_interface_data_path: str) -> None:
+    def __init__(self, dry_run: bool, app_interface_root_path: str) -> None:
         super().__init__(dry_run)
-        if not app_interface_data_path.endswith("/data"):
-            raise ValueError("app_interface_data_path should end with /data")
-        self.app_interface_data_path = app_interface_data_path
+        self.app_interface_root_path = app_interface_root_path
 
     def read(self, path: str) -> str | None:
-        return self._read_local_file(join_path(self.app_interface_data_path, path))
+        return self._read_local_file(join_path(self.app_interface_root_path, path))
 
     def flush(self) -> None:
         for output in self.outputs:
-            filepath = Path(join_path(self.app_interface_data_path, output.path))
+            filepath = Path(join_path(self.app_interface_root_path, output.path))
             filepath.parent.mkdir(parents=True, exist_ok=True)
             filepath.write_text(output.content, encoding="utf-8")
 
@@ -159,7 +158,7 @@ class ClonedRepoGitlabPersistence(FilePersistence):
         self, dry_run: bool, local_path: str, vcs: VCS, mr_manager: MergeRequestManager
     ):
         super().__init__(dry_run)
-        self.local_path = join_path(local_path, "data")
+        self.local_path = local_path
         self.vcs = vcs
         self.mr_manager = mr_manager
 
@@ -217,7 +216,7 @@ def unpack_static_variables(
     each: dict[str, Any],
 ) -> dict:
     return {
-        k: json.loads(process_jinja2_template(body=json.dumps(v), vars={"each": each}))
+        k: json.loads(process_jinja2_template(body=json_dumps(v), vars={"each": each}))
         for k, v in (collection_variables.static or {}).items()
     }
 
@@ -240,8 +239,8 @@ def unpack_dynamic_variables(
 
 class TemplateRendererIntegrationParams(PydanticRunParams):
     clone_repo: bool = False
-    app_interface_data_path: str | None
-    template_collection_name: str | None
+    app_interface_root_path: str | None = None
+    template_collection_name: str | None = None
 
 
 def join_path(base: str, sub: str) -> str:
@@ -367,10 +366,10 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
         persistence: FilePersistence
         ruaml_instance = create_ruamel_instance(explicit_start=True)
 
-        if not self.params.clone_repo and self.params.app_interface_data_path:
+        if not self.params.clone_repo and self.params.app_interface_root_path:
             persistence = LocalFilePersistence(
                 dry_run=dry_run,
-                app_interface_data_path=self.params.app_interface_data_path,
+                app_interface_root_path=self.params.app_interface_root_path,
             )
             self.reconcile(persistence, ruaml_instance)
 
@@ -411,4 +410,4 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
                 self.reconcile(persistence, ruaml_instance)
 
         else:
-            raise ValueError("App-interface-data-path must be set")
+            raise ValueError("App-interface-root-path must be set")

reconcile/terraform_aws_route53.py

@@ -1,6 +1,7 @@
 import logging
 import sys
 from collections.abc import (
+    Callable,
     Iterable,
     Mapping,
 )
@@ -8,6 +9,7 @@ from typing import Any
 
 from reconcile import queries
 from reconcile.status import ExitCodes
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.utils import dnsutils
 from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
@@ -25,8 +27,10 @@ QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
 
 
 def build_desired_state(
-    zones: Iterable[Mapping], all_accounts: Iterable[Mapping], settings: Mapping
-) -> list[dict]:
+    zones: Iterable[Mapping[str, Any]],
+    all_accounts: Iterable[Mapping[str, Any]],
+    settings: Mapping[str, Any],
+) -> list[dict[str, Any]]:
     """
     Build the desired state from the app-interface resources
 
@@ -36,7 +40,7 @@ def build_desired_state(
     :rtype: list of dict
     """
 
-    desired_state = []
+    desired_state: list[dict[str, Any]] = []
     for zone in zones:
         account = zone["account"]
         account_name = account["name"]
@@ -208,8 +212,8 @@ def run(
     enable_deletion: bool = True,
     thread_pool_size: int = DEFAULT_THREAD_POOL_SIZE,
     account_name: str | None = None,
-    defer=None,
-):
+    defer: Callable | None = None,
+) -> None:
     settings = queries.get_app_interface_settings()
     zones = queries.get_dns_zones(account_name=account_name)
 
@@ -224,13 +228,18 @@ def run(
             f"No participating AWS accounts found, consider disabling this integration, account name: {account_name}"
         )
         return
-
+    try:
+        default_tags = get_settings().default_tags
+    except ValueError:
+        # no external resources settings found
+        default_tags = None
     ts = Terrascript(
         QONTRACT_INTEGRATION,
         "",
         thread_pool_size,
         participating_accounts,
         settings=settings,
+        default_tags=default_tags,
     )
 
     desired_state = build_desired_state(zones, all_accounts, settings)
@@ -255,7 +264,8 @@ def run(
     if tf is None:
         sys.exit(ExitCodes.ERROR)
 
-    defer(tf.cleanup)
+    if defer:
+        defer(tf.cleanup)
 
     _, err = tf.plan(enable_deletion)
     if err:
@@ -269,7 +279,7 @@ def run(
         sys.exit(ExitCodes.ERROR)
 
 
-def early_exit_desired_state(*args, **kwargs) -> dict[str, Any]:
+def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
     return {
         "zones": queries.get_dns_zones(),
     }

reconcile/terraform_cloudflare_dns.py

@@ -155,7 +155,7 @@ class TerraformCloudflareDNSIntegration(
 
         accts_per_zone = []
         for zone in query_zones.zones or []:
-            acct = zone.account.dict(by_alias=True)
+            acct = zone.account.model_dump(by_alias=True)
             acct["name"] = f"{zone.account.name}-{zone.identifier}"
             accts_per_zone.append(acct)
 
@@ -369,11 +369,11 @@ def cloudflare_dns_zone_to_external_resource(
             provision_provider=DEFAULT_PROVISIONER_PROVIDER,
             provisioner={"name": f"{zone.account.name}-{zone.identifier}"},
             namespace=DEFAULT_NAMESPACE,
-            resource=zone.dict(by_alias=True, exclude=DEFAULT_EXCLUDE_KEY),
+            resource=zone.model_dump(by_alias=True, exclude=DEFAULT_EXCLUDE_KEY),
         )
         external_resource_spec.resource["provider"] = DEFAULT_PROVIDER
         external_resource_spec.resource["records"] = [
-            record.dict(by_alias=True) for record in zone.records or []
+            record.model_dump(by_alias=True) for record in zone.records or []
         ]
         external_resource_specs.append(external_resource_spec)
     return external_resource_specs