qontract-reconcile 0.10.2.dev345__py3-none-any.whl → 0.10.2.dev408__py3-none-any.whl

reconcile/vault_replication.py

@@ -84,6 +84,54 @@ def deep_copy_versions(
             dest_vault.write(secret=write_dict, decode_base64=False, force=True)
 
 
+def _handle_missing_destination_secret(
+    dry_run: bool,
+    source_vault: VaultClient,
+    dest_vault: VaultClient,
+    source_data: dict,
+    source_version: int | None,
+    path: str,
+) -> None:
+    """Handles replication when destination secret is missing or has no accessible versions.
+
+    This covers two scenarios:
+    1. Secret doesn't exist at all in destination vault (SecretNotFoundError)
+    2. Secret exists but all versions are deleted in KV v2 (SecretVersionNotFoundError)
+
+    For both cases, we replicate from source starting from version 0 (or copy directly for v1).
+
+    Args:
+        dry_run: Whether this is a dry run
+        source_vault: Source vault client (needed for v2 deep copy)
+        dest_vault: Destination vault client
+        source_data: Already retrieved source secret data
+        source_version: Source secret version (None for v1 secrets)
+        path: Secret path
+    """
+    if source_version is None:
+        # v1 secret - just copy it over using the already-retrieved source data
+        logging.info(["replicate_vault_secret", "Copying v1 secret", path])
+        if not dry_run:
+            write_dict = {"path": path, "data": source_data}
+            dest_vault.write(secret=write_dict, decode_base64=False, force=True)
+    else:
+        # v2 secret - deep copy all versions starting from 0
+        # Note: deep_copy_versions will read individual versions from source as needed
+        logging.info([
+            "replicate_vault_secret",
+            "Deep copying v2 secret versions",
+            path,
+        ])
+        deep_copy_versions(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            current_dest_version=0,
+            current_source_version=source_version,
+            path=path,
+        )
+
+
 def write_dummy_versions(
     dry_run: bool,
     dest_vault: VaultClient,
@@ -133,48 +181,65 @@ def copy_vault_secret(
 
     try:
         dest_data, dest_version = dest_vault.read_all_with_version(secret_dict)
-        if dest_version is None and version is None:
-            # v1 secrets don't have version
-            if source_data == dest_data:
-                # If the secret is the same in both vaults, we don't need
-                # to copy it again
-                return
-
-            secret, _ = source_vault.read_all_with_version(secret_dict)
-            write_dict = {"path": path, "data": secret}
-            logging.info(["replicate_vault_secret", path])
-            if not dry_run:
-                # Using force=True to write the secret to force the vault client even
-                # if the data is the same as the previous version. This happens in
-                # some secrets even tho the library does not create it
-                dest_vault.write(secret=write_dict, decode_base64=False, force=True)
-        elif dest_version < version:
-            deep_copy_versions(
-                dry_run=dry_run,
-                source_vault=source_vault,
-                dest_vault=dest_vault,
-                current_dest_version=dest_version,
-                current_source_version=version,
-                path=path,
-            )
-    except (SecretVersionNotFoundError, SecretNotFoundError):
-        logging.info(["replicate_vault_secret", "Secret not found", path])
-        # Handle v1 secrets where version is None and we don't need to deep sync.
-        if version is None:
-            logging.info(["replicate_vault_secret", path])
-            if not dry_run:
-                secret, _ = source_vault.read_all_with_version(secret_dict)
-                write_dict = {"path": path, "data": secret}
-                dest_vault.write(secret=write_dict, decode_base64=False, force=True)
-        else:
-            deep_copy_versions(
-                dry_run=dry_run,
-                source_vault=source_vault,
-                dest_vault=dest_vault,
-                current_dest_version=0,
-                current_source_version=version,
-                path=path,
-            )
+    except SecretVersionNotFoundError:
+        # Handle KV v2 case where secret metadata exists but latest version is deleted
+        # This occurs when someone manually deletes the latest version but the secret
+        # metadata still exists in Vault. This should only happen for v2 secrets.
+        logging.info([
+            "replicate_vault_secret",
+            "KV v2 latest version deleted, replicating all versions",
+            path,
+        ])
+        _handle_missing_destination_secret(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            source_data=source_data,
+            source_version=version,
+            path=path,
+        )
+        return
+    except SecretNotFoundError:
+        # Handle case where secret doesn't exist at all in destination vault
+        logging.info([
+            "replicate_vault_secret",
+            "Secret not found in destination",
+            path,
+        ])
+        _handle_missing_destination_secret(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            source_data=source_data,
+            source_version=version,
+            path=path,
+        )
+        return
+
+    # If we reach here, we successfully read the destination secret
+    if dest_version is None and version is None:
+        # v1 secrets don't have version
+        if source_data == dest_data:
+            # If the secret is the same in both vaults, we don't need
+            # to copy it again
+            return
+
+        write_dict = {"path": path, "data": source_data}
+        logging.info(["replicate_vault_secret", path])
+        if not dry_run:
+            # Using force=True to write the secret to force the vault client even
+            # if the data is the same as the previous version. This happens in
+            # some secrets even tho the library does not create it
+            dest_vault.write(secret=write_dict, decode_base64=False, force=True)
+    elif dest_version < version:
+        deep_copy_versions(
+            dry_run=dry_run,
+            source_vault=source_vault,
+            dest_vault=dest_vault,
+            current_dest_version=dest_version,
+            current_source_version=version,
+            path=path,
+        )
 
 
 def check_invalid_paths(

tools/app_interface_reporter.py

@@ -4,7 +4,6 @@ import os
 import textwrap
 from collections.abc import Mapping, MutableMapping
 from datetime import (
-    UTC,
     datetime,
 )
 
@@ -29,6 +28,7 @@ from reconcile.cli import (
 )
 from reconcile.jenkins_job_builder import init_jjb
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
+from reconcile.utils.datetime_util import ensure_utc, utc_now
 from reconcile.utils.mr import CreateAppInterfaceReporter
 from reconcile.utils.runtime.environment import init_env
 from reconcile.utils.secret_reader import SecretReader
@@ -189,8 +189,8 @@ def get_apps_data(
     apps = queries.get_apps()
     jjb = init_jjb(secret_reader)
     jenkins_map = jenkins_base.get_jenkins_map()
-    time_limit = date - relativedelta(months=month_delta)
-    timestamp_limit = int(time_limit.replace(tzinfo=UTC).timestamp())
+    time_limit = ensure_utc(date) - relativedelta(months=month_delta)
+    timestamp_limit = int(time_limit.timestamp())
 
     secret_content = secret_reader.read_all({"path": DASHDOTDB_SECRET})
     dashdotdb_url = secret_content["url"]
@@ -411,7 +411,7 @@ def main(
 ) -> None:
     init_env(log_level=log_level, config_file=configfile)
 
-    now = datetime.now()
+    now = utc_now()
     apps = get_apps_data(now, thread_pool_size=thread_pool_size)
 
     reports = [Report(app, now).to_message() for app in apps]

tools/cli_commands/systems_and_tools.py

@@ -132,6 +132,7 @@ from reconcile.typed_queries.vault import get_vault_instances
 from reconcile.utils import (
     gql,
 )
+from reconcile.utils.slack_api import is_gov_slack_workspace
 
 
 class SystemTool(BaseModel):
@@ -322,11 +323,14 @@ class SystemTool(BaseModel):
 
     @classmethod
     def init_from_slack_workspace(cls, s: SlackWorkspaceV1, enumeration: Any) -> Self:
+        # Automatically determine the correct Slack domain based on GOV_SLACK environment variable
+        domain = "slack-gov.com" if is_gov_slack_workspace() else "slack.com"
+
         return cls(
             system_type="slack",
             system_id=s.name,
             name=s.name,
-            url=f"https://{s.name}.slack.com",
+            url=f"https://{s.name}.{domain}",
             description=s.description,
             enumeration=enumeration,
         )

tools/qontract_cli.py

@@ -13,7 +13,6 @@ import tempfile
 import textwrap
 from collections import defaultdict
 from datetime import (
-    UTC,
     datetime,
     timedelta,
 )
@@ -122,6 +121,7 @@ from reconcile.utils.binary import (
     binary,
     binary_version,
 )
+from reconcile.utils.datetime_util import from_utc_iso_format, utc_now
 from reconcile.utils.early_exit_cache import (
     CacheKey,
     CacheKeyWithDigest,
@@ -417,8 +417,8 @@ def get_upgrade_policies_data(
             upgrade_next_run = None
         upgrade_emoji = "💫"
         if upgrade_next_run:
-            dt = datetime.strptime(upgrade_next_run, "%Y-%m-%dT%H:%M:%SZ")
-            now = datetime.utcnow()
+            dt = from_utc_iso_format(upgrade_next_run)
+            now = utc_now()
             if dt > now:
                 upgrade_emoji = "⏰"
             hours_ago = (now - dt).total_seconds() / 3600
@@ -841,7 +841,7 @@ def alert_report(
             )
             sys.exit(1)
 
-        now = datetime.utcnow()
+        now = utc_now()
         from_timestamp = int((now - timedelta(days=days)).timestamp())
         to_timestamp = int(now.timestamp())
 
@@ -887,7 +887,9 @@ def alert_report(
             "Triggered": str(data.triggered_alerts),
             "Resolved": str(data.resolved_alerts),
             "Median time to resolve (h:mm:ss)": median_elapsed,
-            "Response Rate": f"{data.responsed_alerts / data.triggered_alerts * 100:.2f}%",
+            "Response Rate": f"{data.responsed_alerts / data.triggered_alerts * 100:.2f}%"
+            if data.triggered_alerts != 0
+            else "0.00%",
         })
 
     # TODO(mafriedm, rporres): Fix this
@@ -2272,7 +2274,7 @@ def app_interface_merge_queue(ctx: click.Context) -> None:
         "labels",
     ]
     merge_queue_data = []
-    now = datetime.utcnow()
+    now = utc_now()
     for mr in merge_requests:
         item = {
             "id": f"[{mr['mr'].iid}]({mr['mr'].web_url})",
@@ -2281,7 +2283,7 @@ def app_interface_merge_queue(ctx: click.Context) -> None:
             + 1,  # adding 1 for human readability
             "approved_at": mr["approved_at"],
             "approved_span_minutes": (
-                now - datetime.strptime(mr["approved_at"], glhk.DATE_FORMAT)
+                now - from_utc_iso_format(mr["approved_at"])
             ).total_seconds()
             / 60,
             "approved_by": mr["approved_by"],
@@ -2695,7 +2697,7 @@ def ec2_jenkins_workers(
     client = boto3.client("autoscaling")
     ec2 = boto3.resource("ec2")
     results = []
-    now = datetime.now(UTC)
+    now = utc_now()
     columns = [
         "type",
         "id",
@@ -2955,7 +2957,7 @@ def osd_component_versions(ctx: click.Context) -> None:
 @get.command()
 @click.pass_context
 def maintenances(ctx: click.Context) -> None:
-    now = datetime.now(UTC)
+    now = utc_now()
     maintenances = maintenances_gql.query(gql.get_api().query).maintenances or []
     data = [
         {
@@ -4097,7 +4099,9 @@ def sre_checkpoint_metadata(
 ) -> None:
     """Check an app path for checkpoint-related metadata."""
     data = queries.get_app_metadata(app_path)
-    settings = queries.get_app_interface_settings()
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+
     app = data[0]
 
     if jiradef:
@@ -4110,7 +4114,14 @@ def sre_checkpoint_metadata(
     # Overrides for easier testing
     if jiraboard:
         board["name"] = jiraboard
-    report_invalid_metadata(app, app_path, board, settings, parent_ticket, dry_run)
+    report_invalid_metadata(
+        app=app,
+        path=app_path,
+        board=board,
+        secret_reader=secret_reader,
+        parent=parent_ticket,
+        dry_run=dry_run,
+    )
 
 
 @root.command()
@@ -4839,11 +4850,12 @@ def top_talkers(ctx: click.Context, top: int) -> None:
             assert project.organization  # make mypy happy
             assert project.pk  # make mypy happy
 
+            now = utc_now()
             stat = client.project_statistics(
                 organization_slug=project.organization.slug,
                 project_pk=project.pk,
-                start=datetime.now(tz=UTC) - timedelta(hours=24),
-                end=datetime.now(tz=UTC),
+                start=now - timedelta(hours=24),
+                end=now,
             )
             stats.append((project, stat))