qontract-reconcile 0.10.2.dev345__py3-none-any.whl → 0.10.2.dev408__py3-none-any.whl

reconcile/openshift_base.py

@@ -30,9 +30,11 @@ from reconcile.utils import (
 )
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
 from reconcile.utils.oc import (
+    AmbiguousResourceTypeError,
     DeploymentFieldIsImmutableError,
     FieldIsImmutableError,
     InvalidValueApplyError,
+    KindNotFoundError,
     MayNotChangeOnceSetError,
     MetaDataAnnotationsTooLongApplyError,
     OC_Map,
@@ -128,6 +130,29 @@ class ClusterMap(Protocol):
     ) -> list[str]: ...
 
 
+def validate_managed_resource_types(
+    oc: OCCli,
+    managed_resource_types: Iterable[str],
+    managed_resource_names: Iterable[Mapping[str, Any]],
+    cluster_scope_resource_validation: bool,
+) -> None:
+    """Validate the managed resource types."""
+    managed_resources = [
+        managed_resource_name["resource"]
+        for managed_resource_name in managed_resource_names
+    ]
+    for managed_resource_type in managed_resource_types:
+        # The k8s kind must be supported by the cluster
+        resource = oc.get_api_resource(managed_resource_type)
+
+        if cluster_scope_resource_validation and not resource.namespaced:
+            # cluster-scoped resources must be use managedResourceNames!
+            if managed_resource_type not in managed_resources:
+                raise ValidationError(
+                    f"Cluster-scoped resource {managed_resource_type} must be managed by name only. Please use 'managedResourceNames' field to specify the names of the resources to manage."
+                )
+
+
 def init_specs_to_fetch(
     ri: ResourceInventory,
     oc_map: ClusterMap,
@@ -136,6 +161,7 @@ def init_specs_to_fetch(
     override_managed_types: Iterable[str] | None = None,
     managed_types_key: str = "managedResourceTypes",
     cluster_admin: bool = False,
+    cluster_scope_resource_validation: bool = False,
 ) -> list[StateSpec]:
     state_specs: list[StateSpec] = []
 
@@ -163,9 +189,27 @@ def init_specs_to_fetch(
                 logging.log(level=ex.log_level, msg=ex.message)
                 continue
 
+            managed_resource_names = namespace_info.get("managedResourceNames") or []
+            try:
+                validate_managed_resource_types(
+                    oc,
+                    managed_types,
+                    managed_resource_names,
+                    cluster_scope_resource_validation=cluster_scope_resource_validation,
+                )
+            except KindNotFoundError:
+                # We must allow kinds that are not supported by the cluster because:
+                # 1. We install CRD with an operator in the same MR
+                # 2. SAAS files initialize the namespace objects with managedResourceTypes from the SAAS file
+                #    and we can't expect that all of those are valid for all clusters
+                pass
+            except (AmbiguousResourceTypeError, ValidationError) as e:
+                ri.register_error()
+                logging.error(f"[{cluster}/{namespace_info['name']}] {e}")
+                continue
+
             namespace = namespace_info["name"]
             # These may exit but have a value of None
-            managed_resource_names = namespace_info.get("managedResourceNames") or []
             managed_resource_type_overrides = (
                 namespace_info.get("managedResourceTypeOverrides") or []
             )
@@ -340,6 +384,7 @@ def fetch_current_state(
     cluster_admin: bool = False,
     caller: str | None = None,
     init_projects: bool = False,
+    cluster_scope_resource_validation: bool = False,
 ) -> tuple[ResourceInventory, OC_Map]:
     ri = ResourceInventory()
     settings = queries.get_app_interface_settings()
@@ -362,6 +407,7 @@ def fetch_current_state(
         clusters=clusters,
         override_managed_types=override_managed_types,
         cluster_admin=cluster_admin,
+        cluster_scope_resource_validation=cluster_scope_resource_validation,
     )
     threaded.run(
         populate_current_state,

reconcile/openshift_cluster_bots.py

@@ -16,6 +16,7 @@ from reconcile.gql_definitions.openshift_cluster_bots.clusters import ClusterV1
 from reconcile.status import ExitCodes
 from reconcile.utils import gql
 from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.json import json_dumps
 from reconcile.utils.mr import clusters_updates
 from reconcile.utils.ocm import OCM, OCMMap
 from reconcile.utils.openshift_resource import (
@@ -102,7 +103,7 @@ def oc(
 
 def oc_apply(kubeconfig: str, namespace: str, items: list[dict]) -> None:
     for item in items:
-        stdin = json.dumps(item).encode()
+        stdin = json_dumps(item).encode()
         oc(kubeconfig, namespace, ["apply", "-f", "-"], stdin)
 
 

reconcile/openshift_resources_base.py

@@ -806,7 +806,11 @@ def fetch_data(
         init_api_resources=init_api_resources,
     )
     state_specs = ob.init_specs_to_fetch(
-        ri, oc_map, namespaces=namespaces, override_managed_types=overrides
+        ri,
+        oc_map,
+        namespaces=namespaces,
+        override_managed_types=overrides,
+        cluster_scope_resource_validation=True,
     )
     threaded.run(fetch_states, state_specs, thread_pool_size, ri=ri, settings=settings)
 
@@ -861,7 +865,7 @@ def canonicalize_namespaces(
             elif providers[0] == "route":
                 override = ["Route"]
             elif providers[0] == "prometheus-rule":
-                override = ["PrometheusRule"]
+                override = ["PrometheusRule.monitoring.coreos.com"]
 
             namespace_info["openshiftResources"] = ors
             canonicalized_namespaces.append(namespace_info)

reconcile/openshift_saas_deploy.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import os
 import sys
@@ -28,6 +27,7 @@ from reconcile.typed_queries.saas_files import (
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
 from reconcile.utils.defer import defer
 from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.json import json_dumps
 from reconcile.utils.openshift_resource import ResourceInventory
 from reconcile.utils.saasherder import SaasHerder
 from reconcile.utils.secret_reader import create_secret_reader
@@ -346,4 +346,4 @@ def run(
         if image_auth.auth_server:
             json_file = os.path.join(io_dir, "dockerconfigjson")
             with open(json_file, "w", encoding="locale") as f:
-                f.write(json.dumps(image_auth.get_docker_config_json(), indent=2))
+                f.write(json_dumps(image_auth.get_docker_config_json(), indent=2))

reconcile/openshift_saas_deploy_trigger_cleaner.py

@@ -1,14 +1,11 @@
 import logging
 from collections.abc import Callable
 from datetime import (
-    UTC,
     datetime,
     timedelta,
 )
 from typing import Any
 
-from dateutil import parser
-
 from reconcile.gql_definitions.fragments.pipeline_provider_retention import (
     PipelineProviderRetention,
 )
@@ -19,6 +16,7 @@ from reconcile.typed_queries.tekton_pipeline_providers import (
     get_tekton_pipeline_providers,
 )
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
+from reconcile.utils.datetime_util import from_utc_iso_format, utc_now
 from reconcile.utils.defer import defer
 from reconcile.utils.oc_map import (
     OCLogMsg,
@@ -35,7 +33,7 @@ def within_retention_days(
     resource: dict[str, Any], days: int, now_date: datetime
 ) -> bool:
     metadata = resource["metadata"]
-    creation_date = parser.parse(metadata["creationTimestamp"])
+    creation_date = from_utc_iso_format(metadata["creationTimestamp"])
     interval = now_date.timestamp() - creation_date.timestamp()
 
     return interval < timedelta(days=days).total_seconds()
@@ -69,7 +67,7 @@ def run(
     use_jump_host: bool = True,
     defer: Callable | None = None,
 ) -> None:
-    now_date = datetime.now(UTC)
+    now_date = utc_now()
     vault_settings = get_app_interface_vault_settings()
     secret_reader = create_secret_reader(use_vault=vault_settings.vault)
     pipeline_providers = get_tekton_pipeline_providers()

reconcile/openshift_upgrade_watcher.py

@@ -3,7 +3,6 @@ from collections.abc import (
     Callable,
     Iterable,
 )
-from datetime import datetime
 
 from reconcile import queries
 from reconcile.gql_definitions.common.clusters import ClusterV1
@@ -13,6 +12,7 @@ from reconcile.typed_queries.app_interface_vault_settings import (
 )
 from reconcile.typed_queries.clusters import get_clusters
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
+from reconcile.utils.datetime_util import from_utc_iso_format, utc_now
 from reconcile.utils.defer import defer
 from reconcile.utils.oc_map import (
     OCLogMsg,
@@ -101,7 +101,7 @@ def notify_upgrades_start(
     state: State,
     slack: SlackApi | None,
 ) -> None:
-    now = datetime.utcnow()
+    now = utc_now()
     for cluster in clusters:
         if cluster.spec and not cluster.spec.hypershift:
             upgrade_at, version = _get_start_osd(oc_map, cluster.name)
@@ -113,7 +113,7 @@ def notify_upgrades_start(
             continue
 
         if upgrade_at and version:
-            upgrade_at_obj = datetime.strptime(upgrade_at, "%Y-%m-%dT%H:%M:%SZ")
+            upgrade_at_obj = from_utc_iso_format(upgrade_at)
             state_key = f"{cluster.name}-{upgrade_at}1"
             # if this is the first iteration in which 'now' had passed
             # the upgrade at date time, we send a notification

reconcile/queries.py

@@ -477,6 +477,12 @@ AWS_ACCOUNTS_QUERY = """
       integrations
     }
     deleteKeys
+    organization {
+      payerAccount {
+        organizationAccountTags
+      }
+      tags
+    }
     {% if reset_passwords %}
     resetPasswords {
       user {
@@ -499,6 +505,12 @@ AWS_ACCOUNTS_QUERY = """
         name
         uid
         supportedDeploymentRegions
+        organization {
+          payerAccount {
+            organizationAccountTags
+          }
+          tags
+        }
       }
       ... on AWSAccountSharingOptionAMI_v1 {
         regex
@@ -606,6 +618,12 @@ awsInfrastructureManagementAccounts {
       version
       format
     }
+    organization {
+      payerAccount {
+        organizationAccountTags
+      }
+      tags
+    }
   }
   accessLevel
   default
@@ -636,6 +654,12 @@ awsInfrastructureAccess {
         version
         format
       }
+      organization {
+        payerAccount {
+          organizationAccountTags
+        }
+        tags
+      }
     }
     roles {
       users {
@@ -745,6 +769,12 @@ CLUSTERS_QUERY = """
             version
             format
           }
+          organization {
+            payerAccount {
+                organizationAccountTags
+            }
+            tags
+          }
           rosa {
             ocm_environments {
               ocm {
@@ -773,6 +803,7 @@ CLUSTERS_QUERY = """
       private
       provision_shard_id
       disable_user_workload_monitoring
+      fips
     }
     externalConfiguration {
       labels
@@ -829,6 +860,12 @@ CLUSTERS_QUERY = """
                 version
                 format
               }
+              organization {
+                payerAccount {
+                  organizationAccountTags
+                }
+                tags
+              }
             }
             vpc_id
             cidr_block
@@ -847,6 +884,12 @@ CLUSTERS_QUERY = """
               version
               format
             }
+            organization {
+              payerAccount {
+                organizationAccountTags
+              }
+              tags
+            }
           }
           tags
         }
@@ -861,6 +904,12 @@ CLUSTERS_QUERY = """
               version
               format
             }
+            organization {
+              payerAccount {
+                organizationAccountTags
+              }
+              tags
+            }
           }
           tags
           cidrBlock
@@ -889,6 +938,12 @@ CLUSTERS_QUERY = """
                     version
                     format
                   }
+                  organization {
+                    payerAccount {
+                      organizationAccountTags
+                    }
+                    tags
+                  }
                 }
               }
               accessLevel
@@ -914,6 +969,12 @@ CLUSTERS_QUERY = """
                       version
                       format
                     }
+                    organization {
+                      payerAccount {
+                        organizationAccountTags
+                      }
+                      tags
+                    }
                   }
                 }
               }
@@ -1067,6 +1128,12 @@ CLUSTER_PEERING_QUERY = """
           version
           format
         }
+        organization {
+          payerAccount {
+            organizationAccountTags
+          }
+          tags
+        }
       }
       accessLevel
       default
@@ -1088,6 +1155,12 @@ CLUSTER_PEERING_QUERY = """
             version
             format
           }
+          organization {
+            payerAccount {
+              organizationAccountTags
+            }
+            tags
+          }
         }
       }
     }
@@ -1112,6 +1185,12 @@ CLUSTER_PEERING_QUERY = """
                 version
                 format
               }
+              organization {
+                payerAccount {
+                  organizationAccountTags
+                }
+                tags
+              }
             }
             vpc_id
             cidr_block
@@ -1131,6 +1210,12 @@ CLUSTER_PEERING_QUERY = """
               version
               format
             }
+            organization {
+              payerAccount {
+                organizationAccountTags
+              }
+              tags
+            }
           }
           tags
           assumeRole
@@ -1146,6 +1231,12 @@ CLUSTER_PEERING_QUERY = """
               version
               format
             }
+            organization {
+              payerAccount {
+                organizationAccountTags
+              }
+              tags
+            }
           }
           tags
           cidrBlock
@@ -1175,6 +1266,12 @@ CLUSTER_PEERING_QUERY = """
                     version
                     format
                   }
+                  organization {
+                    payerAccount {
+                      organizationAccountTags
+                    }
+                    tags
+                  }
                 }
               }
             }
@@ -1190,6 +1287,12 @@ CLUSTER_PEERING_QUERY = """
                   version
                   format
                 }
+                organization {
+                  payerAccount {
+                    organizationAccountTags
+                  }
+                  tags
+                }
               }
               accessLevel
               default
@@ -1216,6 +1319,12 @@ CLUSTER_PEERING_QUERY = """
                             version
                             format
                           }
+                          organization {
+                            payerAccount {
+                              organizationAccountTags
+                            }
+                            tags
+                          }
                         }
                       }
                     }
@@ -1231,6 +1340,12 @@ CLUSTER_PEERING_QUERY = """
                       version
                       format
                     }
+                    organization {
+                      payerAccount {
+                        organizationAccountTags
+                      }
+                      tags
+                    }
                   }
                 }
               }
@@ -2230,6 +2345,12 @@ JIRA_BOARDS_QUICK_QUERY = """
         version
         format
       }
+      email {
+        path
+        field
+        version
+        format
+      }
     }
   }
 }
@@ -2336,6 +2457,12 @@ DNS_ZONES_QUERY = """
         version
         format
       }
+      organization {
+        payerAccount {
+          organizationAccountTags
+        }
+        tags
+      }
     }
     vpc {
       vpc_id
@@ -2697,6 +2824,10 @@ APP_METADATA = """
               path
               field
             }
+            email {
+              path
+              field
+            }
           }
         }
         slackUserGroup {

reconcile/saas_auto_promotions_manager/subscriber.py

@@ -2,7 +2,7 @@ import hashlib
 import logging
 from collections.abc import Iterable
 from dataclasses import dataclass
-from datetime import UTC, datetime, timedelta
+from datetime import timedelta
 
 from croniter import croniter
 
@@ -13,6 +13,7 @@ from reconcile.saas_auto_promotions_manager.publisher import (
     DeploymentInfo,
     Publisher,
 )
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.slo_document_manager import SLODocumentManager
 
 CONTENT_HASH_LENGTH = 32
@@ -113,7 +114,7 @@ class Subscriber:
         We accumulate the time a ref is running on all publishers for this subscriber.
         We compare that accumulated time with the soak_days setting of the subscriber.
         """
-        now = datetime.now(UTC)
+        now = utc_now()
         delta = timedelta(days=0)
         for channel in self.channels:
             for publisher in channel.publishers:
@@ -136,7 +137,7 @@ class Subscriber:
                 self.schedule,
             )
             return False
-        return croniter.match(self.schedule, datetime.now(UTC), day_or=False)
+        return croniter.match(self.schedule, utc_now(), day_or=False)
 
     def _compute_desired_ref(self) -> None:
         """

reconcile/slack_usergroups.py

@@ -40,6 +40,7 @@ from reconcile.typed_queries.app_interface_vault_settings import (
 )
 from reconcile.typed_queries.pagerduty_instances import get_pagerduty_instances
 from reconcile.utils import gql
+from reconcile.utils.datetime_util import ensure_utc, utc_now
 from reconcile.utils.disabled_integrations import integration_is_enabled
 from reconcile.utils.exceptions import (
     AppInterfaceSettingsError,
@@ -357,11 +358,11 @@ def get_slack_usernames_from_owners(
 
 def get_slack_usernames_from_schedule(schedule: Iterable[ScheduleEntryV1]) -> list[str]:
     """Return list of usernames from all schedules."""
-    now = datetime.utcnow()
+    now = utc_now()
     all_slack_usernames: list[str] = []
     for entry in schedule:
-        start = datetime.strptime(entry.start, DATE_FORMAT)
-        end = datetime.strptime(entry.end, DATE_FORMAT)
+        start = ensure_utc(datetime.strptime(entry.start, DATE_FORMAT))  # noqa: DTZ007
+        end = ensure_utc(datetime.strptime(entry.end, DATE_FORMAT))  # noqa: DTZ007
         if start <= now <= end:
             all_slack_usernames.extend(get_slack_username(u) for u in entry.users)
     return all_slack_usernames

reconcile/sql_query.py

@@ -229,6 +229,7 @@ def collect_queries(
         accounts=[],
         prefetch_resources_by_schemas=["/aws/rds-defaults-1.yml"],
         secret_reader=secret_reader,
+        default_tags=None,
     )
 
     for sql_query in sql_queries:

reconcile/statuspage/integrations/maintenances.py

@@ -1,12 +1,13 @@
 import logging
 import sys
-from datetime import UTC, datetime, timedelta
+from datetime import datetime, timedelta
 
 from reconcile.slack_base import slackapi_from_queries
 from reconcile.statuspage.atlassian import AtlassianStatusPageProvider
 from reconcile.statuspage.integration import get_binding_state, get_status_pages
 from reconcile.statuspage.page import StatusMaintenance
 from reconcile.statuspage.state import S3ComponentBindingState
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.differ import diff_iterables
 from reconcile.utils.runtime.integration import (
     NoParams,
@@ -52,7 +53,7 @@ class StatusPageMaintenancesIntegration(QontractReconcileIntegration[NoParams]):
         desired_state: list[StatusMaintenance],
         binding_state: S3ComponentBindingState,
     ) -> None:
-        now = datetime.now(UTC)
+        now = utc_now()
         slack = slackapi_from_queries(QONTRACT_INTEGRATION, init_usergroups=False)
         for m in desired_state:
             scheduled_start = m.schedule_start
@@ -68,7 +69,7 @@ class StatusPageMaintenancesIntegration(QontractReconcileIntegration[NoParams]):
     def run(self, dry_run: bool = False) -> None:
         binding_state = get_binding_state(self.name, self.secret_reader)
         pages = get_status_pages()
-        now = datetime.now(UTC)
+        now = utc_now()
 
         error = False
         for p in pages:

reconcile/statuspage/status.py

@@ -2,18 +2,15 @@ from abc import (
     ABC,
     abstractmethod,
 )
-from datetime import (
-    UTC,
-    datetime,
-)
+from datetime import datetime
 
-from dateutil.parser import isoparse
 from pydantic import BaseModel
 
 from reconcile.gql_definitions.statuspage.statuspages import (
     ManualStatusProviderV1,
     StatusProviderV1,
 )
+from reconcile.utils.datetime_util import from_utc_iso_format, utc_now
 
 # This module defines the interface for status providers for components on status
 # pages. A status provider is responsible for determining the status of a component.
@@ -70,7 +67,7 @@ class ManualStatusProvider(StatusProvider, BaseModel):
             raise ValueError(
                 "manual component status time window is invalid: end before start"
             )
-        now = datetime.now(UTC)
+        now = utc_now()
         if self.start and now < self.start:
             return False
         return not (self.end and self.end < now)
@@ -84,8 +81,8 @@ def build_status_provider_config(
     provider specific implementation that provides the status resolution logic.
     """
     if isinstance(cfg, ManualStatusProviderV1):
-        start = isoparse(cfg.manual.q_from) if cfg.manual.q_from else None
-        end = isoparse(cfg.manual.until) if cfg.manual.until else None
+        start = from_utc_iso_format(cfg.manual.q_from) if cfg.manual.q_from else None
+        end = from_utc_iso_format(cfg.manual.until) if cfg.manual.until else None
         return ManualStatusProvider(
             component_status=cfg.manual.component_status,
             start=start,

reconcile/templates/rosa-classic-cluster-creation.sh.j2

@@ -55,4 +55,8 @@ rosa create cluster -y --cluster-name={{ cluster_name }} \
     {% if cluster.spec.provision_shard_id -%}
     --properties provision_shard_id:{{ cluster.spec.provision_shard_id }} \
     {% endif -%}
+    {% if cluster.spec.fips -%}
+    --fips \
+    {% endif -%}
     --channel-group {{ cluster.spec.channel }}
+

reconcile/templates/rosa-hcp-cluster-creation.sh.j2

@@ -59,4 +59,7 @@ rosa create cluster --cluster-name={{ cluster_name }} \
     {% if cluster.spec.provision_shard_id -%}
     --properties provision_shard_id:{{ cluster.spec.provision_shard_id }} \
     {% endif -%}
+    {% if cluster.spec.fips -%}
+    --fips \
+    {% endif -%}
     --channel-group {{ cluster.spec.channel }}

reconcile/templating/renderer.py

@@ -33,6 +33,7 @@ from reconcile.utils import gql
 from reconcile.utils.git import checkout, clone
 from reconcile.utils.gql import GqlApi
 from reconcile.utils.jinja2.utils import TemplateRenderOptions, process_jinja2_template
+from reconcile.utils.json import json_dumps
 from reconcile.utils.ruamel import create_ruamel_instance
 from reconcile.utils.runtime.integration import (
     PydanticRunParams,
@@ -215,7 +216,7 @@ def unpack_static_variables(
     each: dict[str, Any],
 ) -> dict:
     return {
-        k: json.loads(process_jinja2_template(body=json.dumps(v), vars={"each": each}))
+        k: json.loads(process_jinja2_template(body=json_dumps(v), vars={"each": each}))
         for k, v in (collection_variables.static or {}).items()
     }