qontract-reconcile 0.10.2.dev310__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl

reconcile/utils/three_way_diff_strategy.py

@@ -61,7 +61,7 @@ def normalize_object(item: OR) -> OR:
         validate_k8s_object=False,
     )
 
-    annotations = n.body.get("metadata").get("annotations") or {}
+    annotations = n.body.get("metadata", {}).get("annotations") or {}
     metadata["annotations"] = {
         k: v for k, v in annotations.items() if k not in NORMALIZE_IGNORE_ANNOTATIONS
     }

reconcile/utils/unleash/server.py

@@ -24,30 +24,24 @@ class Environment(BaseModel):
         return self.name == other
 
 
-class FeatureToggle(BaseModel):
+class FeatureToggle(BaseModel, validate_by_name=True, validate_by_alias=True):
     name: str
     type: FeatureToggleType = FeatureToggleType.release
     description: str | None = None
     impression_data: bool = Field(False, alias="impressionData")
     environments: list[Environment]
 
-    class Config:
-        allow_population_by_field_name = True
-
     def __eq__(self, other: object) -> bool:
         if isinstance(other, FeatureToggle):
             return self.name == other.name
         return self.name == other
 
 
-class Project(BaseModel):
+class Project(BaseModel, validate_by_name=True, validate_by_alias=True):
     pk: str = Field(alias="id")
     name: str
     feature_toggles: list[FeatureToggle] = []
 
-    class Config:
-        allow_population_by_field_name = True
-
 
 class TokenAuth(BearerTokenAuth):
     def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:

reconcile/utils/vault.py

@@ -1,10 +1,12 @@
 import base64
+import builtins
 import logging
 import os
 import threading
 import time
 from collections.abc import Mapping
 from functools import lru_cache
+from typing import Any, Self
 
 import hvac
 import requests
@@ -49,7 +51,7 @@ class VaultConnectionError(Exception):
 SECRET_VERSION_LATEST = "LATEST"
 
 
-class _VaultClient:
+class VaultClient:
     """
     A class representing a Vault client. Allows read/write operations.
     The client caches read requests in-memory if the request is made
@@ -57,6 +59,28 @@ class _VaultClient:
     and a version (no invalidation required).
     """
 
+    _instance = None
+    _lock = threading.Lock()
+
+    @classmethod
+    def get_instance(cls) -> Self:
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = cls()
+                return cls._instance
+
+            try:
+                is_authenticated = cls._instance._client.is_authenticated()
+            except requests.exceptions.ConnectionError:
+                is_authenticated = False
+
+            if is_authenticated:
+                return cls._instance
+
+            cls._instance.close()
+            cls._instance = cls()
+            return cls._instance
+
     def __init__(
         self,
         server: str | None = None,
@@ -122,13 +146,13 @@ class _VaultClient:
             t = threading.Thread(target=self._auto_refresh_client_auth, daemon=True)
             t.start()
 
-    def __enter__(self):
+    def __enter__(self) -> Self:
         return self
 
-    def __exit__(self, *exc):
+    def __exit__(self, *exc: Any) -> None:
         self.close()
 
-    def close(self):
+    def close(self) -> None:
         """
         Close the client and release any resources associated with it.
         """
@@ -137,7 +161,7 @@ class _VaultClient:
                 self._client.adapter.close()
                 self._closed = True
 
-    def _auto_refresh_client_auth(self):
+    def _auto_refresh_client_auth(self) -> None:
         """
         Thread that periodically refreshes the vault token
         """
@@ -150,7 +174,7 @@ class _VaultClient:
                 LOG.debug("auto refresh client auth")
                 self._refresh_client_auth()
 
-    def _refresh_client_auth(self):
+    def _refresh_client_auth(self) -> None:
         if self.kube_auth_enabled:
             # must read each time to account for sa token refresh
             with open(self.kube_sa_token_path, encoding="locale") as f:
@@ -166,7 +190,7 @@ class _VaultClient:
             self._client.auth_approle(self.role_id, self.secret_id)
 
     @retry()
-    def read_all_with_version(self, secret: Mapping) -> tuple[Mapping, str | None]:
+    def read_all_with_version(self, secret: Mapping) -> tuple[dict, int | None]:
         """Returns a dictionary of keys and values in a Vault secret and the
         version of the secret, for V1 secrets, version will be None.
 
@@ -176,7 +200,7 @@ class _VaultClient:
                                a v2 KV engine)
         """
         secret_path = secret["path"]
-        secret_version = secret.get("version")
+        secret_version = secret.get("version") or SECRET_VERSION_LATEST
 
         kv_version = self._get_mount_version_by_secret_path(secret_path)
 
@@ -203,12 +227,12 @@ class _VaultClient:
         """
         return self.read_all_with_version(secret)[0]
 
-    def _get_mount_version_by_secret_path(self, path):
+    def _get_mount_version_by_secret_path(self, path: str) -> int:
         path_split = path.split("/")
         mount_point = path_split[0]
         return self._get_mount_version(mount_point)
 
-    def __get_mount_version(self, mount_point):
+    def __get_mount_version(self, mount_point: str) -> int:
         try:
             self._client.secrets.kv.v2.read_configuration(mount_point)
             version = 2
@@ -217,7 +241,9 @@ class _VaultClient:
 
         return version
 
-    def __read_all_v2(self, path: str, version: str | None) -> tuple[dict, str | None]:
+    def __read_all_v2(
+        self, path: str, version: str | None
+    ) -> tuple[dict[str, Any], int]:
         path_split = path.split("/")
         mount_point = path_split[0]
         read_path = "/".join(path_split[1:])
@@ -248,7 +274,7 @@ class _VaultClient:
         secret_version = secret["data"]["metadata"]["version"]
         return data, secret_version
 
-    def _read_all_v1(self, path):
+    def _read_all_v1(self, path: str) -> Any:
         try:
             secret = self._client.read(path)
         except hvac.exceptions.Forbidden:
@@ -261,7 +287,7 @@ class _VaultClient:
         return secret["data"]
 
     @retry()
-    def read(self, secret):
+    def read(self, secret: Mapping[str, Any]) -> Any:
         """Returns a value of a key in a Vault secret.
 
         The input secret is a dictionary which contains the following fields:
@@ -293,7 +319,7 @@ class _VaultClient:
             else data
         )
 
-    def _read_v2(self, path, field, version):
+    def _read_v2(self, path: str, field: str, version: str | None) -> Any:
         data, _ = self._read_all_v2(path, version)
         try:
             secret_field = data[field]
@@ -301,7 +327,7 @@ class _VaultClient:
             raise SecretFieldNotFoundError(f"{path}/{field} ({version})") from None
         return secret_field
 
-    def _read_v1(self, path, field):
+    def _read_v1(self, path: str, field: str) -> Any:
         data = self._read_all_v1(path)
         try:
             secret_field = data[field]
@@ -360,7 +386,7 @@ class _VaultClient:
             msg = f"permission denied accessing secret '{path}'"
             raise SecretAccessForbiddenError(msg) from None
 
-    def _write_v1(self, path, data):
+    def _write_v1(self, path: str, data: dict[str, Any]) -> None:
         try:
             self._client.write(path, **data)
         except hvac.exceptions.Forbidden:
@@ -395,7 +421,7 @@ class _VaultClient:
             return []
         return path_list["data"]["keys"] or []
 
-    def list_all(self, path):
+    def list_all(self, path: str) -> builtins.list[str]:
         """Returns a list of secrets in a given path and
         all its subpaths."""
         secrets = []
@@ -415,32 +441,9 @@ class _VaultClient:
             raise ValueError("deleting V2 secrets is not supported yet")
         self._delete_v1(path)
 
-    def _delete_v1(self, path):
+    def _delete_v1(self, path: str) -> None:
         try:
             self._client.delete(path)
         except hvac.exceptions.Forbidden:
             msg = f"permission denied accessing secret '{path}'"
             raise SecretAccessForbiddenError(msg) from None
-
-
-class VaultClient:
-    _instance_lock = threading.Lock()
-    _instance = None
-
-    def __new__(cls, *args, **kwargs):
-        with cls._instance_lock:
-            if cls._instance is None:
-                cls._instance = _VaultClient(*args, **kwargs)
-                return cls._instance
-
-            try:
-                is_authenticated = cls._instance._client.is_authenticated()
-            except requests.exceptions.ConnectionError:
-                is_authenticated = False
-
-            if not is_authenticated:
-                cls._instance.close()
-                cls._instance = _VaultClient(*args, **kwargs)
-                return cls._instance
-
-            return cls._instance

reconcile/utils/vcs.py

@@ -140,7 +140,7 @@ class VCS:
         gitlab_instances: Iterable[GitlabInstanceV1],
     ) -> GitLabApi:
         return GitLabApi(
-            next(iter(gitlab_instances)).dict(by_alias=True),
+            next(iter(gitlab_instances)).model_dump(by_alias=True),
             secret_reader=self._secret_reader,
         )
 
@@ -150,7 +150,7 @@ class VCS:
         app_interface_repo_url: str,
     ) -> GitLabApi:
         return GitLabApi(
-            next(iter(gitlab_instances)).dict(by_alias=True),
+            next(iter(gitlab_instances)).model_dump(by_alias=True),
             secret_reader=self._secret_reader,
             project_url=app_interface_repo_url,
         )
@@ -221,26 +221,26 @@ class VCS:
         match repo_info.platform:
             case "github":
                 github = self._init_github(repo_url=repo_url, auth_code=auth_code)
-                data = github.compare(commit_from=commit_from, commit_to=commit_to)
                 return [
                     Commit(
                         repo=repo_url,
                         sha=gh_commit.sha,
                         date=gh_commit.commit.committer.date,
                     )
-                    for gh_commit in data
+                    for gh_commit in github.compare(
+                        commit_from=commit_from, commit_to=commit_to
+                    )
                 ]
             case "gitlab":
-                data = self._gitlab_instance.repository_compare(
-                    repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
-                )
                 return [
                     Commit(
                         repo=repo_url,
                         sha=gl_commit["id"],
                         date=datetime.fromisoformat(gl_commit["committed_date"]),
                     )
-                    for gl_commit in data
+                    for gl_commit in self._gitlab_instance.repository_compare(
+                        repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
+                    )
                 ]
             case _:
                 raise ValueError(f"Unsupported repository URL: {repo_url}")

reconcile/vault_replication.py

@@ -1,9 +1,6 @@
 import logging
 import re
 from collections.abc import Iterable
-from typing import (
-    cast,
-)
 
 from reconcile.gql_definitions.jenkins_configs import jenkins_configs
 from reconcile.gql_definitions.jenkins_configs.jenkins_configs import (
@@ -30,7 +27,6 @@ from reconcile.utils.vault import (
     SecretNotFoundError,
     SecretVersionNotFoundError,
     VaultClient,
-    _VaultClient,
 )
 
 QONTRACT_INTEGRATION = "vault-replication"
@@ -51,8 +47,8 @@ class VaultInvalidPolicyError(Exception):
 
 def deep_copy_versions(
     dry_run: bool,
-    source_vault: _VaultClient,
-    dest_vault: _VaultClient,
+    source_vault: VaultClient,
+    dest_vault: VaultClient,
     current_dest_version: int,
     current_source_version: int,
     path: str,
@@ -88,9 +84,57 @@ def deep_copy_versions(
             dest_vault.write(secret=write_dict, decode_base64=False, force=True)
 
 
+def _handle_missing_destination_secret(
+    dry_run: bool,
+    source_vault: VaultClient,
+    dest_vault: VaultClient,
+    source_data: dict,
+    source_version: int | None,
+    path: str,
+) -> None:
+    """Handles replication when destination secret is missing or has no accessible versions.
+
+    This covers two scenarios:
+    1. Secret doesn't exist at all in destination vault (SecretNotFoundError)
+    2. Secret exists but all versions are deleted in KV v2 (SecretVersionNotFoundError)
+
+    For both cases, we replicate from source starting from version 0 (or copy directly for v1).
+
+    Args:
+        dry_run: Whether this is a dry run
+        source_vault: Source vault client (needed for v2 deep copy)
+        dest_vault: Destination vault client
+        source_data: Already retrieved source secret data
+        source_version: Source secret version (None for v1 secrets)
+        path: Secret path
+    """
+    if source_version is None:
+        # v1 secret - just copy it over using the already-retrieved source data
+        logging.info(["replicate_vault_secret", "Copying v1 secret", path])
+        if not dry_run:
+            write_dict = {"path": path, "data": source_data}
+            dest_vault.write(secret=write_dict, decode_base64=False, force=True)
+    else:
+        # v2 secret - deep copy all versions starting from 0
+        # Note: deep_copy_versions will read individual versions from source as needed
+        logging.info([
+            "replicate_vault_secret",
+            "Deep copying v2 secret versions",
+            path,
+        ])
+        deep_copy_versions(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            current_dest_version=0,
+            current_source_version=source_version,
+            path=path,
+        )
+
+
 def write_dummy_versions(
     dry_run: bool,
-    dest_vault: _VaultClient,
+    dest_vault: VaultClient,
     secret_version: int,
     path: str,
 ) -> None:
@@ -112,7 +156,7 @@ def write_dummy_versions(
 
 
 def copy_vault_secret(
-    dry_run: bool, source_vault: _VaultClient, dest_vault: _VaultClient, path: str
+    dry_run: bool, source_vault: VaultClient, dest_vault: VaultClient, path: str
 ) -> None:
     """Copies a secret from the source vault to the destination vault"""
     secret_dict = {"path": path, "version": "LATEST"}
@@ -137,48 +181,65 @@ def copy_vault_secret(
 
     try:
         dest_data, dest_version = dest_vault.read_all_with_version(secret_dict)
-        if dest_version is None and version is None:
-            # v1 secrets don't have version
-            if source_data == dest_data:
-                # If the secret is the same in both vaults, we don't need
-                # to copy it again
-                return
-
-            secret, _ = source_vault.read_all_with_version(secret_dict)
-            write_dict = {"path": path, "data": secret}
-            logging.info(["replicate_vault_secret", path])
-            if not dry_run:
-                # Using force=True to write the secret to force the vault client even
-                # if the data is the same as the previous version. This happens in
-                # some secrets even tho the library does not create it
-                dest_vault.write(secret=write_dict, decode_base64=False, force=True)
-        elif dest_version < version:
-            deep_copy_versions(
-                dry_run=dry_run,
-                source_vault=source_vault,
-                dest_vault=dest_vault,
-                current_dest_version=dest_version,
-                current_source_version=version,
-                path=path,
-            )
-    except (SecretVersionNotFoundError, SecretNotFoundError):
-        logging.info(["replicate_vault_secret", "Secret not found", path])
-        # Handle v1 secrets where version is None and we don't need to deep sync.
-        if version is None:
-            logging.info(["replicate_vault_secret", path])
-            if not dry_run:
-                secret, _ = source_vault.read_all_with_version(secret_dict)
-                write_dict = {"path": path, "data": secret}
-                dest_vault.write(secret=write_dict, decode_base64=False, force=True)
-        else:
-            deep_copy_versions(
-                dry_run=dry_run,
-                source_vault=source_vault,
-                dest_vault=dest_vault,
-                current_dest_version=0,
-                current_source_version=version,
-                path=path,
-            )
+    except SecretVersionNotFoundError:
+        # Handle KV v2 case where secret metadata exists but latest version is deleted
+        # This occurs when someone manually deletes the latest version but the secret
+        # metadata still exists in Vault. This should only happen for v2 secrets.
+        logging.info([
+            "replicate_vault_secret",
+            "KV v2 latest version deleted, replicating all versions",
+            path,
+        ])
+        _handle_missing_destination_secret(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            source_data=source_data,
+            source_version=version,
+            path=path,
+        )
+        return
+    except SecretNotFoundError:
+        # Handle case where secret doesn't exist at all in destination vault
+        logging.info([
+            "replicate_vault_secret",
+            "Secret not found in destination",
+            path,
+        ])
+        _handle_missing_destination_secret(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            source_data=source_data,
+            source_version=version,
+            path=path,
+        )
+        return
+
+    # If we reach here, we successfully read the destination secret
+    if dest_version is None or version is None:
+        # v1 secrets don't have version
+        if source_data == dest_data:
+            # If the secret is the same in both vaults, we don't need
+            # to copy it again
+            return
+
+        write_dict = {"path": path, "data": source_data}
+        logging.info(["replicate_vault_secret", path])
+        if not dry_run:
+            # Using force=True to write the secret to force the vault client even
+            # if the data is the same as the previous version. This happens in
+            # some secrets even tho the library does not create it
+            dest_vault.write(secret=write_dict, decode_base64=False, force=True)
+    elif dest_version < version:
+        deep_copy_versions(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            current_dest_version=dest_version,
+            current_source_version=version,
+            path=path,
+        )
 
 
 def check_invalid_paths(
@@ -228,7 +289,7 @@ def get_policy_paths(
 
 
 def get_policy_secret_list(
-    vault_instance: _VaultClient, policy_paths: Iterable[str]
+    vault_instance: VaultClient, policy_paths: Iterable[str]
 ) -> list[str]:
     """Returns a list of secrets to be copied from the given policy"""
     secrets = set()
@@ -249,7 +310,7 @@ def get_policy_secret_list(
 
 
 def get_jenkins_secret_list(
-    vault_instance: _VaultClient,
+    vault_instance: VaultClient,
     jenkins_instance: str,
     query_data: JenkinsConfigsQueryData,
 ) -> list[str]:
@@ -294,7 +355,7 @@ def get_vault_credentials(
     """Returns a dictionary with the credentials used to authenticate with Vault,
     retrieved from the values present on AppInterface and comming from Vault itself."""
     vault_creds = {}
-    vault = cast("_VaultClient", VaultClient())
+    vault = VaultClient.get_instance()
 
     if not isinstance(
         vault_auth,
@@ -324,8 +385,8 @@ def get_vault_credentials(
 
 def replicate_paths(
     dry_run: bool,
-    source_vault: _VaultClient,
-    dest_vault: _VaultClient,
+    source_vault: VaultClient,
+    dest_vault: VaultClient,
     replications: VaultReplicationConfigV1,
 ) -> None:
     """For each path present in the definition of the vault instance, replicate
@@ -435,16 +496,16 @@ def run(dry_run: bool) -> None:
                         replication.dest_auth, replication.vault_instance.address
                     )
 
-                    # Private class _VaultClient is used because the public class is
+                    # Private class VaultClient is used because the public class is
                     # defined as a singleton, and we need to create multiple instances
                     # as the source vault is different than the replication.
                     with (
-                        _VaultClient(
+                        VaultClient(
                             server=source_creds["server"],
                             role_id=source_creds["role_id"],
                             secret_id=source_creds["secret_id"],
                         ) as source_vault,
-                        _VaultClient(
+                        VaultClient(
                             server=dest_creds["server"],
                             role_id=dest_creds["role_id"],
                             secret_id=dest_creds["secret_id"],

tools/app_interface_reporter.py

@@ -4,7 +4,6 @@ import os
 import textwrap
 from collections.abc import Mapping, MutableMapping
 from datetime import (
-    UTC,
     datetime,
 )
 
@@ -29,6 +28,7 @@ from reconcile.cli import (
 )
 from reconcile.jenkins_job_builder import init_jjb
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
+from reconcile.utils.datetime_util import ensure_utc, utc_now
 from reconcile.utils.mr import CreateAppInterfaceReporter
 from reconcile.utils.runtime.environment import init_env
 from reconcile.utils.secret_reader import SecretReader
@@ -189,8 +189,8 @@ def get_apps_data(
     apps = queries.get_apps()
     jjb = init_jjb(secret_reader)
     jenkins_map = jenkins_base.get_jenkins_map()
-    time_limit = date - relativedelta(months=month_delta)
-    timestamp_limit = int(time_limit.replace(tzinfo=UTC).timestamp())
+    time_limit = ensure_utc(date) - relativedelta(months=month_delta)
+    timestamp_limit = int(time_limit.timestamp())
 
     secret_content = secret_reader.read_all({"path": DASHDOTDB_SECRET})
     dashdotdb_url = secret_content["url"]
@@ -411,7 +411,7 @@ def main(
 ) -> None:
     init_env(log_level=log_level, config_file=configfile)
 
-    now = datetime.now()
+    now = utc_now()
     apps = get_apps_data(now, thread_pool_size=thread_pool_size)
 
     reports = [Report(app, now).to_message() for app in apps]

tools/cli_commands/cost_report/cost_management_api.py

@@ -74,7 +74,7 @@ class CostManagementApi(ApiBase):
             timeout=self.read_timeout,
         )
         response.raise_for_status()
-        return AwsReportCostResponse.parse_obj(response.json())
+        return AwsReportCostResponse.model_validate(response.json())
 
     def get_openshift_costs_report(
         self,
@@ -97,7 +97,7 @@ class CostManagementApi(ApiBase):
             timeout=self.read_timeout,
         )
         response.raise_for_status()
-        return OpenShiftReportCostResponse.parse_obj(response.json())
+        return OpenShiftReportCostResponse.model_validate(response.json())
 
     def get_openshift_cost_optimization_report(
         self,
@@ -120,7 +120,7 @@ class CostManagementApi(ApiBase):
         response.raise_for_status()
 
         data = self._get_paginated(response)
-        return OpenShiftCostOptimizationReportResponse.parse_obj(data)
+        return OpenShiftCostOptimizationReportResponse.model_validate(data)
 
     def _get_paginated(
         self,