qontract-reconcile 0.10.2.dev334__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl

reconcile/utils/unleash/server.py

@@ -24,30 +24,24 @@ class Environment(BaseModel):
         return self.name == other
 
 
-class FeatureToggle(BaseModel):
+class FeatureToggle(BaseModel, validate_by_name=True, validate_by_alias=True):
     name: str
     type: FeatureToggleType = FeatureToggleType.release
     description: str | None = None
     impression_data: bool = Field(False, alias="impressionData")
     environments: list[Environment]
 
-    class Config:
-        allow_population_by_field_name = True
-
     def __eq__(self, other: object) -> bool:
         if isinstance(other, FeatureToggle):
             return self.name == other.name
         return self.name == other
 
 
-class Project(BaseModel):
+class Project(BaseModel, validate_by_name=True, validate_by_alias=True):
     pk: str = Field(alias="id")
     name: str
     feature_toggles: list[FeatureToggle] = []
 
-    class Config:
-        allow_population_by_field_name = True
-
 
 class TokenAuth(BearerTokenAuth):
     def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:

reconcile/utils/vault.py

@@ -6,7 +6,7 @@ import threading
 import time
 from collections.abc import Mapping
 from functools import lru_cache
-from typing import Any, Self, TypedDict
+from typing import Any, Self
 
 import hvac
 import requests
@@ -48,13 +48,6 @@ class VaultConnectionError(Exception):
     pass
 
 
-class Secret(TypedDict):
-    path: str
-    field: str
-    format: str | None
-    version: str | None
-
-
 SECRET_VERSION_LATEST = "LATEST"
 
 
@@ -197,7 +190,7 @@ class VaultClient:
             self._client.auth_approle(self.role_id, self.secret_id)
 
     @retry()
-    def read_all_with_version(self, secret: Mapping) -> tuple[Mapping, str | None]:
+    def read_all_with_version(self, secret: Mapping) -> tuple[dict, int | None]:
         """Returns a dictionary of keys and values in a Vault secret and the
         version of the secret, for V1 secrets, version will be None.
 
@@ -207,7 +200,7 @@ class VaultClient:
                                a v2 KV engine)
         """
         secret_path = secret["path"]
-        secret_version = secret.get("version")
+        secret_version = secret.get("version") or SECRET_VERSION_LATEST
 
         kv_version = self._get_mount_version_by_secret_path(secret_path)
 
@@ -250,7 +243,7 @@ class VaultClient:
 
     def __read_all_v2(
         self, path: str, version: str | None
-    ) -> tuple[dict[str, Any], str | None]:
+    ) -> tuple[dict[str, Any], int]:
         path_split = path.split("/")
         mount_point = path_split[0]
         read_path = "/".join(path_split[1:])
@@ -294,7 +287,7 @@ class VaultClient:
         return secret["data"]
 
     @retry()
-    def read(self, secret: Secret) -> Any:
+    def read(self, secret: Mapping[str, Any]) -> Any:
         """Returns a value of a key in a Vault secret.
 
         The input secret is a dictionary which contains the following fields:

reconcile/utils/vcs.py

@@ -140,7 +140,7 @@ class VCS:
         gitlab_instances: Iterable[GitlabInstanceV1],
     ) -> GitLabApi:
         return GitLabApi(
-            next(iter(gitlab_instances)).dict(by_alias=True),
+            next(iter(gitlab_instances)).model_dump(by_alias=True),
             secret_reader=self._secret_reader,
         )
 
@@ -150,7 +150,7 @@ class VCS:
         app_interface_repo_url: str,
     ) -> GitLabApi:
         return GitLabApi(
-            next(iter(gitlab_instances)).dict(by_alias=True),
+            next(iter(gitlab_instances)).model_dump(by_alias=True),
             secret_reader=self._secret_reader,
             project_url=app_interface_repo_url,
         )
@@ -221,26 +221,26 @@ class VCS:
         match repo_info.platform:
             case "github":
                 github = self._init_github(repo_url=repo_url, auth_code=auth_code)
-                data = github.compare(commit_from=commit_from, commit_to=commit_to)
                 return [
                     Commit(
                         repo=repo_url,
                         sha=gh_commit.sha,
                         date=gh_commit.commit.committer.date,
                     )
-                    for gh_commit in data
+                    for gh_commit in github.compare(
+                        commit_from=commit_from, commit_to=commit_to
+                    )
                 ]
             case "gitlab":
-                data = self._gitlab_instance.repository_compare(
-                    repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
-                )
                 return [
                     Commit(
                         repo=repo_url,
                         sha=gl_commit["id"],
                         date=datetime.fromisoformat(gl_commit["committed_date"]),
                     )
-                    for gl_commit in data
+                    for gl_commit in self._gitlab_instance.repository_compare(
+                        repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
+                    )
                 ]
             case _:
                 raise ValueError(f"Unsupported repository URL: {repo_url}")

reconcile/vault_replication.py

@@ -84,6 +84,54 @@ def deep_copy_versions(
             dest_vault.write(secret=write_dict, decode_base64=False, force=True)
 
 
+def _handle_missing_destination_secret(
+    dry_run: bool,
+    source_vault: VaultClient,
+    dest_vault: VaultClient,
+    source_data: dict,
+    source_version: int | None,
+    path: str,
+) -> None:
+    """Handles replication when destination secret is missing or has no accessible versions.
+
+    This covers two scenarios:
+    1. Secret doesn't exist at all in destination vault (SecretNotFoundError)
+    2. Secret exists but all versions are deleted in KV v2 (SecretVersionNotFoundError)
+
+    For both cases, we replicate from source starting from version 0 (or copy directly for v1).
+
+    Args:
+        dry_run: Whether this is a dry run
+        source_vault: Source vault client (needed for v2 deep copy)
+        dest_vault: Destination vault client
+        source_data: Already retrieved source secret data
+        source_version: Source secret version (None for v1 secrets)
+        path: Secret path
+    """
+    if source_version is None:
+        # v1 secret - just copy it over using the already-retrieved source data
+        logging.info(["replicate_vault_secret", "Copying v1 secret", path])
+        if not dry_run:
+            write_dict = {"path": path, "data": source_data}
+            dest_vault.write(secret=write_dict, decode_base64=False, force=True)
+    else:
+        # v2 secret - deep copy all versions starting from 0
+        # Note: deep_copy_versions will read individual versions from source as needed
+        logging.info([
+            "replicate_vault_secret",
+            "Deep copying v2 secret versions",
+            path,
+        ])
+        deep_copy_versions(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            current_dest_version=0,
+            current_source_version=source_version,
+            path=path,
+        )
+
+
 def write_dummy_versions(
     dry_run: bool,
     dest_vault: VaultClient,
@@ -133,48 +181,65 @@ def copy_vault_secret(
 
     try:
         dest_data, dest_version = dest_vault.read_all_with_version(secret_dict)
-        if dest_version is None and version is None:
-            # v1 secrets don't have version
-            if source_data == dest_data:
-                # If the secret is the same in both vaults, we don't need
-                # to copy it again
-                return
-
-            secret, _ = source_vault.read_all_with_version(secret_dict)
-            write_dict = {"path": path, "data": secret}
-            logging.info(["replicate_vault_secret", path])
-            if not dry_run:
-                # Using force=True to write the secret to force the vault client even
-                # if the data is the same as the previous version. This happens in
-                # some secrets even tho the library does not create it
-                dest_vault.write(secret=write_dict, decode_base64=False, force=True)
-        elif dest_version < version:
-            deep_copy_versions(
-                dry_run=dry_run,
-                source_vault=source_vault,
-                dest_vault=dest_vault,
-                current_dest_version=dest_version,
-                current_source_version=version,
-                path=path,
-            )
-    except (SecretVersionNotFoundError, SecretNotFoundError):
-        logging.info(["replicate_vault_secret", "Secret not found", path])
-        # Handle v1 secrets where version is None and we don't need to deep sync.
-        if version is None:
-            logging.info(["replicate_vault_secret", path])
-            if not dry_run:
-                secret, _ = source_vault.read_all_with_version(secret_dict)
-                write_dict = {"path": path, "data": secret}
-                dest_vault.write(secret=write_dict, decode_base64=False, force=True)
-        else:
-            deep_copy_versions(
-                dry_run=dry_run,
-                source_vault=source_vault,
-                dest_vault=dest_vault,
-                current_dest_version=0,
-                current_source_version=version,
-                path=path,
-            )
+    except SecretVersionNotFoundError:
+        # Handle KV v2 case where secret metadata exists but latest version is deleted
+        # This occurs when someone manually deletes the latest version but the secret
+        # metadata still exists in Vault. This should only happen for v2 secrets.
+        logging.info([
+            "replicate_vault_secret",
+            "KV v2 latest version deleted, replicating all versions",
+            path,
+        ])
+        _handle_missing_destination_secret(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            source_data=source_data,
+            source_version=version,
+            path=path,
+        )
+        return
+    except SecretNotFoundError:
+        # Handle case where secret doesn't exist at all in destination vault
+        logging.info([
+            "replicate_vault_secret",
+            "Secret not found in destination",
+            path,
+        ])
+        _handle_missing_destination_secret(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            source_data=source_data,
+            source_version=version,
+            path=path,
+        )
+        return
+
+    # If we reach here, we successfully read the destination secret
+    if dest_version is None or version is None:
+        # v1 secrets don't have version
+        if source_data == dest_data:
+            # If the secret is the same in both vaults, we don't need
+            # to copy it again
+            return
+
+        write_dict = {"path": path, "data": source_data}
+        logging.info(["replicate_vault_secret", path])
+        if not dry_run:
+            # Using force=True to write the secret to force the vault client even
+            # if the data is the same as the previous version. This happens in
+            # some secrets even tho the library does not create it
+            dest_vault.write(secret=write_dict, decode_base64=False, force=True)
+    elif dest_version < version:
+        deep_copy_versions(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            current_dest_version=dest_version,
+            current_source_version=version,
+            path=path,
+        )
 
 
 def check_invalid_paths(

tools/app_interface_reporter.py

@@ -4,7 +4,6 @@ import os
 import textwrap
 from collections.abc import Mapping, MutableMapping
 from datetime import (
-    UTC,
     datetime,
 )
 
@@ -29,6 +28,7 @@ from reconcile.cli import (
 )
 from reconcile.jenkins_job_builder import init_jjb
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
+from reconcile.utils.datetime_util import ensure_utc, utc_now
 from reconcile.utils.mr import CreateAppInterfaceReporter
 from reconcile.utils.runtime.environment import init_env
 from reconcile.utils.secret_reader import SecretReader
@@ -189,8 +189,8 @@ def get_apps_data(
     apps = queries.get_apps()
     jjb = init_jjb(secret_reader)
     jenkins_map = jenkins_base.get_jenkins_map()
-    time_limit = date - relativedelta(months=month_delta)
-    timestamp_limit = int(time_limit.replace(tzinfo=UTC).timestamp())
+    time_limit = ensure_utc(date) - relativedelta(months=month_delta)
+    timestamp_limit = int(time_limit.timestamp())
 
     secret_content = secret_reader.read_all({"path": DASHDOTDB_SECRET})
     dashdotdb_url = secret_content["url"]
@@ -411,7 +411,7 @@ def main(
 ) -> None:
     init_env(log_level=log_level, config_file=configfile)
 
-    now = datetime.now()
+    now = utc_now()
     apps = get_apps_data(now, thread_pool_size=thread_pool_size)
 
     reports = [Report(app, now).to_message() for app in apps]

tools/cli_commands/cost_report/cost_management_api.py

@@ -74,7 +74,7 @@ class CostManagementApi(ApiBase):
             timeout=self.read_timeout,
         )
         response.raise_for_status()
-        return AwsReportCostResponse.parse_obj(response.json())
+        return AwsReportCostResponse.model_validate(response.json())
 
     def get_openshift_costs_report(
         self,
@@ -97,7 +97,7 @@ class CostManagementApi(ApiBase):
             timeout=self.read_timeout,
         )
         response.raise_for_status()
-        return OpenShiftReportCostResponse.parse_obj(response.json())
+        return OpenShiftReportCostResponse.model_validate(response.json())
 
     def get_openshift_cost_optimization_report(
         self,
@@ -120,7 +120,7 @@ class CostManagementApi(ApiBase):
         response.raise_for_status()
 
         data = self._get_paginated(response)
-        return OpenShiftCostOptimizationReportResponse.parse_obj(data)
+        return OpenShiftCostOptimizationReportResponse.model_validate(data)
 
     def _get_paginated(
         self,

tools/cli_commands/cost_report/view.py

@@ -4,6 +4,7 @@ from typing import Any
 
 from pydantic import BaseModel
 
+from reconcile.utils.json import json_dumps
 from tools.cli_commands.cost_report.model import OptimizationReport, Report
 
 LAYOUT = """\
@@ -244,7 +245,7 @@ def render_summary(
     return template.format(
         date=get_date(reports),
         total_cost=format_cost_value(total_cost),
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 
@@ -274,7 +275,7 @@ def render_month_over_month_change(reports: Mapping[str, Report]) -> str:
     )
     return MONTH_OVER_MONTH_CHANGE.format(
         date=get_date(reports),
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 
@@ -304,7 +305,7 @@ def render_aws_services_cost(
         items_total=format_cost_value(report.items_total),
         items_delta_value=format_delta_value(report.items_delta_value),
         items_delta_percent=format_delta_percent(report.items_delta_percent),
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 
@@ -316,7 +317,7 @@ def render_openshift_workloads_cost(
         items_total=format_cost_value(report.items_total),
         items_delta_value=format_delta_value(report.items_delta_value),
         items_delta_percent=format_delta_percent(report.items_delta_percent),
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 
@@ -362,7 +363,7 @@ def render_child_apps_cost(report: Report) -> str:
     )
     return CHILD_APPS_COST.format(
         child_apps_total=format_cost_value(report.child_apps_total),
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 
@@ -509,7 +510,7 @@ def render_optimization(
     )
     return OPTIMIZATION.format(
         app_name=report.app_name,
-        json_table=json_table.json(indent=2),
+        json_table=json_dumps(json_table, indent=2, mode="python"),
     )
 
 

tools/cli_commands/erv2.py

@@ -133,7 +133,7 @@ class Erv2Cli:
 
     @property
     def input_data(self) -> str:
-        return self._resource.json(exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2}})
+        return self._resource.export(exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2}})
 
     @property
     def image(self) -> str:

tools/cli_commands/systems_and_tools.py

@@ -132,6 +132,7 @@ from reconcile.typed_queries.vault import get_vault_instances
 from reconcile.utils import (
     gql,
 )
+from reconcile.utils.slack_api import is_gov_slack_workspace
 
 
 class SystemTool(BaseModel):
@@ -322,11 +323,14 @@ class SystemTool(BaseModel):
 
     @classmethod
     def init_from_slack_workspace(cls, s: SlackWorkspaceV1, enumeration: Any) -> Self:
+        # Automatically determine the correct Slack domain based on GOV_SLACK environment variable
+        domain = "slack-gov.com" if is_gov_slack_workspace() else "slack.com"
+
         return cls(
             system_type="slack",
             system_id=s.name,
             name=s.name,
-            url=f"https://{s.name}.slack.com",
+            url=f"https://{s.name}.{domain}",
             description=s.description,
             enumeration=enumeration,
         )

tools/qontract_cli.py

@@ -13,7 +13,6 @@ import tempfile
 import textwrap
 from collections import defaultdict
 from datetime import (
-    UTC,
     datetime,
     timedelta,
 )
@@ -122,6 +121,7 @@ from reconcile.utils.binary import (
     binary,
     binary_version,
 )
+from reconcile.utils.datetime_util import from_utc_iso_format, utc_now
 from reconcile.utils.early_exit_cache import (
     CacheKey,
     CacheKeyWithDigest,
@@ -141,6 +141,7 @@ from reconcile.utils.gitlab_api import (
 )
 from reconcile.utils.glitchtip.client import GlitchtipClient
 from reconcile.utils.gql import GqlApiSingleton
+from reconcile.utils.json import json_dumps
 from reconcile.utils.keycloak import (
     KeycloakAPI,
     SSOClient,
@@ -417,8 +418,8 @@ def get_upgrade_policies_data(
             upgrade_next_run = None
         upgrade_emoji = "💫"
         if upgrade_next_run:
-            dt = datetime.strptime(upgrade_next_run, "%Y-%m-%dT%H:%M:%SZ")
-            now = datetime.utcnow()
+            dt = from_utc_iso_format(upgrade_next_run)
+            now = utc_now()
             if dt > now:
                 upgrade_emoji = "⏰"
             hours_ago = (now - dt).total_seconds() / 3600
@@ -841,7 +842,7 @@ def alert_report(
             )
             sys.exit(1)
 
-        now = datetime.utcnow()
+        now = utc_now()
         from_timestamp = int((now - timedelta(days=days)).timestamp())
         to_timestamp = int(now.timestamp())
 
@@ -887,7 +888,9 @@ def alert_report(
             "Triggered": str(data.triggered_alerts),
             "Resolved": str(data.resolved_alerts),
             "Median time to resolve (h:mm:ss)": median_elapsed,
-            "Response Rate": f"{data.responsed_alerts / data.triggered_alerts * 100:.2f}%",
+            "Response Rate": f"{data.responsed_alerts / data.triggered_alerts * 100:.2f}%"
+            if data.triggered_alerts != 0
+            else "0.00%",
         })
 
     # TODO(mafriedm, rporres): Fix this
@@ -1564,7 +1567,7 @@ def rosa_create_cluster_command(ctx: click.Context, cluster_name: str) -> None:
         billing_account = account.billing_account.uid
     else:
         with AWSApi(
-            1, [account.dict(by_alias=True)], settings=settings, init_users=False
+            1, [account.model_dump(by_alias=True)], settings=settings, init_users=False
         ) as aws_api:
             billing_account = aws_api.get_organization_billing_account(account.name)
 
@@ -1748,7 +1751,7 @@ def aws_terraform_resources(ctx: click.Context) -> None:
     for ns_info in namespaces:
         specs = (
             get_external_resource_specs(
-                ns_info.dict(by_alias=True), provision_provider=PROVIDER_AWS
+                ns_info.model_dump(by_alias=True), provision_provider=PROVIDER_AWS
             )
             or []
         )
@@ -1806,7 +1809,7 @@ def rds(ctx: click.Context) -> None:
         specs = [
             s
             for s in get_external_resource_specs(
-                namespace.dict(by_alias=True), provision_provider=PROVIDER_AWS
+                namespace.model_dump(by_alias=True), provision_provider=PROVIDER_AWS
             )
             if s.provider == "rds"
         ]
@@ -2272,7 +2275,7 @@ def app_interface_merge_queue(ctx: click.Context) -> None:
         "labels",
     ]
     merge_queue_data = []
-    now = datetime.utcnow()
+    now = utc_now()
     for mr in merge_requests:
         item = {
             "id": f"[{mr['mr'].iid}]({mr['mr'].web_url})",
@@ -2281,7 +2284,7 @@ def app_interface_merge_queue(ctx: click.Context) -> None:
             + 1,  # adding 1 for human readability
             "approved_at": mr["approved_at"],
             "approved_span_minutes": (
-                now - datetime.strptime(mr["approved_at"], glhk.DATE_FORMAT)
+                now - from_utc_iso_format(mr["approved_at"])
             ).total_seconds()
             / 60,
             "approved_by": mr["approved_by"],
@@ -2695,7 +2698,7 @@ def ec2_jenkins_workers(
     client = boto3.client("autoscaling")
     ec2 = boto3.resource("ec2")
     results = []
-    now = datetime.now(UTC)
+    now = utc_now()
     columns = [
         "type",
         "id",
@@ -2955,11 +2958,11 @@ def osd_component_versions(ctx: click.Context) -> None:
 @get.command()
 @click.pass_context
 def maintenances(ctx: click.Context) -> None:
-    now = datetime.now(UTC)
+    now = utc_now()
     maintenances = maintenances_gql.query(gql.get_api().query).maintenances or []
     data = [
         {
-            **m.dict(),
+            **m.model_dump(),
             "services": ", ".join(a.name for a in m.affected_services),
         }
         for m in maintenances
@@ -4097,7 +4100,9 @@ def sre_checkpoint_metadata(
 ) -> None:
     """Check an app path for checkpoint-related metadata."""
     data = queries.get_app_metadata(app_path)
-    settings = queries.get_app_interface_settings()
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+
     app = data[0]
 
     if jiradef:
@@ -4110,7 +4115,14 @@ def sre_checkpoint_metadata(
     # Overrides for easier testing
     if jiraboard:
         board["name"] = jiraboard
-    report_invalid_metadata(app, app_path, board, settings, parent_ticket, dry_run)
+    report_invalid_metadata(
+        app=app,
+        path=app_path,
+        board=board,
+        secret_reader=secret_reader,
+        parent=parent_ticket,
+        dry_run=dry_run,
+    )
 
 
 @root.command()
@@ -4287,7 +4299,7 @@ def create(
         bg="red",
         fg="white",
     )
-    print(sso_client.json(by_alias=True, indent=2))
+    print(json_dumps(sso_client, indent=2))
 
 
 @sso_client.command()
@@ -4839,11 +4851,12 @@ def top_talkers(ctx: click.Context, top: int) -> None:
             assert project.organization  # make mypy happy
             assert project.pk  # make mypy happy
 
+            now = utc_now()
             stat = client.project_statistics(
                 organization_slug=project.organization.slug,
                 project_pk=project.pk,
-                start=datetime.now(tz=UTC) - timedelta(hours=24),
-                end=datetime.now(tz=UTC),
+                start=now - timedelta(hours=24),
+                end=now,
             )
             stats.append((project, stat))
 

tools/template_validation.py

@@ -67,7 +67,9 @@ def main(templates: tuple[str]) -> None:
             tests.append(test_yaml)
 
         template_raw["templateTest"] = tests
-        template: TemplateV1 = TemplateV1(**data_default_none(TemplateV1, template_raw))
+        data = data_default_none(TemplateV1, template_raw)
+        assert isinstance(data, dict)
+        template: TemplateV1 = TemplateV1(**data)
 
         # templates_to_validate = {}
         for test in template.template_test: