qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/saas_auto_promotions_manager/subscriber.py

@@ -2,8 +2,13 @@ import hashlib
 import logging
 from collections.abc import Iterable
 from dataclasses import dataclass
-from typing import Optional
+from datetime import UTC, datetime, timedelta
 
+from croniter import croniter
+
+from reconcile.gql_definitions.fragments.saas_target_namespace import (
+    SaasTargetNamespace,
+)
 from reconcile.saas_auto_promotions_manager.publisher import (
     DeploymentInfo,
     Publisher,
@@ -37,8 +42,12 @@ class Subscriber:
         template_name: str,
         ref: str,
         target_file_path: str,
-        namespace_file_path: str,
+        target_namespace: SaasTargetNamespace,
         use_target_config_hash: bool,
+        uid: str,
+        soak_days: int,
+        blocked_versions: set[str],
+        schedule: str,
     ):
         self.saas_name = saas_name
         self.template_name = template_name
@@ -48,9 +57,13 @@ class Subscriber:
         self.channels: list[Channel] = []
         self.desired_ref = ""
         self.desired_hashes: list[ConfigHash] = []
-        self.namespace_file_path = namespace_file_path
+        self.target_namespace = target_namespace
+        self.uid = uid
+        self.soak_days = soak_days
+        self.schedule = schedule
         self._content_hash = ""
         self._use_target_config_hash = use_target_config_hash
+        self._blocked_versions = blocked_versions
 
     def has_diff(self) -> bool:
         current_hashes = {
@@ -72,7 +85,7 @@ class Subscriber:
 
     def _validate_deployment(
         self, publisher: Publisher, channel: Channel
-    ) -> Optional[DeploymentInfo]:
+    ) -> DeploymentInfo | None:
         deployment_info = publisher.deployment_info_by_channel.get(channel.name)
         if not deployment_info:
             logging.info(
@@ -90,6 +103,36 @@ class Subscriber:
             return None
         return deployment_info
 
+    def _passed_accumulated_soak_days(self) -> bool:
+        """
+        We accumulate the time a ref is running on all publishers for this subscriber.
+        We compare that accumulated time with the soak_days setting of the subscriber.
+        """
+        now = datetime.now(UTC)
+        delta = timedelta(days=0)
+        for channel in self.channels:
+            for publisher in channel.publishers:
+                deploy_info = publisher.deployment_info_by_channel.get(channel.name)
+                if not deploy_info:
+                    # At this stage we always expect a deploy_info to be present
+                    return False
+                deployed_at = deploy_info.check_in
+                if not deployed_at:
+                    continue
+                delta += now - deployed_at
+        return delta >= timedelta(days=self.soak_days)
+
+    def _is_valid_deployment_window(self) -> bool:
+        # Ideally we would catch that at schema validation time
+        if not croniter.is_valid(self.schedule):
+            logging.error(
+                "Subscriber at %s has an invalid schedule declaration %s. We will block any promotion for that target until this is fixed.",
+                self.target_file_path,
+                self.schedule,
+            )
+            return False
+        return croniter.match(self.schedule, datetime.now(UTC), day_or=False)
+
     def _compute_desired_ref(self) -> None:
         """
         Compute the desired reference for this subscriber.
@@ -98,27 +141,67 @@ class Subscriber:
         successfully deployed, then this subscriber is not ready for promotion and we keep
         the current ref.
         """
-        new_ref = ""
+        publisher_refs: set[str] = set()
+        any_bad_deployment = False
         for channel in self.channels:
-            publisher = channel.publishers[0]
-            valid_deployment = self._validate_deployment(
-                publisher=publisher, channel=channel
-            )
-            if not valid_deployment:
-                new_ref = ""
-                break
-            if self.ref != publisher.commit_sha:
-                if new_ref and new_ref != publisher.commit_sha:
+            for publisher in channel.publishers:
+                valid_deployment = self._validate_deployment(
+                    publisher=publisher, channel=channel
+                )
+                if not valid_deployment:
+                    any_bad_deployment = True
                     logging.info(
-                        "[%s] mismatching commit shas in different subscribed channels (%s != %s) -> not ready for promotion",
-                        channel.name,
-                        publisher.commit_sha,
-                        new_ref,
+                        "[%s] publisher with uid %s has unsuccessful deployment",
+                        channel,
+                        publisher.uid,
                     )
-                    new_ref = ""
                     break
-                new_ref = publisher.commit_sha
-        self.desired_ref = new_ref if new_ref else self.ref
+                publisher_refs.add(publisher.commit_sha)
+
+        # By default we keep current state
+        self.desired_ref = self.ref
+
+        if any_bad_deployment:
+            logging.info(
+                "Subscriber at path %s promotion stopped because of bad publisher deployment",
+                self.target_file_path,
+            )
+            return
+
+        if len(publisher_refs) != 1:
+            logging.info(
+                "Publishers for subscriber at path %s have mismatching refs: %s",
+                self.target_file_path,
+                publisher_refs,
+            )
+            return
+
+        if not self._passed_accumulated_soak_days():
+            logging.debug(
+                "Subscriber at path %s promotion stopped because of soak days",
+                self.target_file_path,
+            )
+            return
+
+        if not self._is_valid_deployment_window():
+            logging.debug(
+                "Subscriber at path %s promotion stopped because we are not in the deployment window %s",
+                self.target_file_path,
+                self.schedule,
+            )
+            return
+
+        desired_ref = next(iter(publisher_refs))
+        if desired_ref in self._blocked_versions:
+            logging.info(
+                "Subscriber at path %s promotion stopped because of blocked ref: %s",
+                self.target_file_path,
+                desired_ref,
+            )
+            return
+
+        # Passed all gates -> lets promote desired ref
+        self.desired_ref = desired_ref
 
     def _compute_desired_config_hashes(self) -> None:
         """
@@ -168,7 +251,11 @@ class Subscriber:
         """
         sorted_subs = sorted(
             subscribers,
-            key=lambda s: (s.target_file_path, s.template_name, s.namespace_file_path),
+            key=lambda s: (
+                s.target_file_path,
+                s.template_name,
+                s.target_namespace.path,
+            ),
         )
         m = hashlib.sha256()
         msg = ""
@@ -179,7 +266,7 @@ class Subscriber:
             )
             msg += f"""
             target_file_path: {sub.target_file_path}
-            namespace_file_path: {sub.namespace_file_path}
+            namespace_file_path: {sub.target_namespace.path}
             desired_ref: {sub.desired_ref}
             desired_hashes: {[(h.channel, h.parent_saas, h.target_config_hash) for h in sorted_hashes]}
             """

reconcile/saas_auto_promotions_manager/utils/saas_files_inventory.py

@@ -34,29 +34,16 @@ class SaasFilesInventory:
         self._assemble_publishers()
         self._remove_unsupported()
 
-    def _assemble_channels(self) -> None:
-        for saas_file in self._saas_files:
-            for resource_template in saas_file.resource_templates:
-                for target in resource_template.targets:
-                    if not target.promotion:
-                        continue
-                    for publish_channel in target.promotion.publish or []:
-                        if publish_channel not in self._channels_by_name:
-                            self._channels_by_name[publish_channel] = Channel(
-                                name=publish_channel,
-                                publishers=[],
-                            )
-                    for subscribe_channel in target.promotion.subscribe or []:
-                        if subscribe_channel not in self._channels_by_name:
-                            self._channels_by_name[subscribe_channel] = Channel(
-                                name=subscribe_channel,
-                                publishers=[],
-                            )
+    @property
+    def publishers_with_subscribers(self) -> list[Publisher]:
+        return [p for p in self.publishers if p.has_subscriber]
 
     def _assemble_publishers(self) -> None:
         for saas_file in self._saas_files:
             for resource_template in saas_file.resource_templates:
                 for target in resource_template.targets:
+                    if target.disable or target.delete:
+                        continue
                     if not target.promotion:
                         continue
                     auth_code = (
@@ -66,9 +53,22 @@ class SaasFilesInventory:
                     )
                     publisher = Publisher(
                         ref=target.ref,
+                        uid=target.uid(
+                            parent_saas_file_name=saas_file.name,
+                            parent_resource_template_name=resource_template.name,
+                        ),
                         repo_url=resource_template.url,
+                        saas_file_path=saas_file.path,
+                        saas_name=saas_file.name,
+                        app_name=saas_file.app.name,
+                        namespace_name=target.namespace.name,
+                        cluster_name=target.namespace.cluster.name,
+                        resource_template_name=resource_template.name,
+                        target_name=target.name,
                         auth_code=auth_code,
+                        redeploy_on_config_change=target.promotion.redeploy_on_publisher_config_change,
                     )
+
                     has_subscriber = False
                     for publish_channel in target.promotion.publish or []:
                         if publish_channel not in self._channels_by_name:
@@ -80,27 +80,51 @@ class SaasFilesInventory:
                         self._channels_by_name[publish_channel].publishers.append(
                             publisher
                         )
-                    if has_subscriber:
-                        self.publishers.append(publisher)
+
+                    publisher.has_subscriber = has_subscriber
+                    self.publishers.append(publisher)
 
     def _assemble_subscribers_with_auto_promotions(self) -> None:
         for saas_file in self._saas_files:
+            blocked_versions: dict[str, set[str]] = {}
+            for code_component in saas_file.app.code_components or []:
+                for version in code_component.blocked_versions or []:
+                    blocked_versions.setdefault(code_component.url, set()).add(version)
             for resource_template in saas_file.resource_templates:
                 for target in resource_template.targets:
                     file_path = target.path if target.path else saas_file.path
+                    if target.disable or target.delete:
+                        continue
                     if not target.promotion:
                         continue
                     if not target.promotion.auto:
                         continue
+                    soak_days = (
+                        target.promotion.soak_days if target.promotion.soak_days else 0
+                    )
+                    schedule = (
+                        target.promotion.schedule
+                        if target.promotion.schedule
+                        else "* * * * *"
+                    )
                     subscriber = Subscriber(
+                        uid=target.uid(
+                            parent_saas_file_name=saas_file.name,
+                            parent_resource_template_name=resource_template.name,
+                        ),
                         saas_name=saas_file.name,
                         template_name=resource_template.name,
                         target_file_path=file_path,
                         ref=target.ref,
-                        namespace_file_path=target.namespace.path,
-                        # Note: this will be refactored at a later point.
-                        # https://issues.redhat.com/browse/APPSRE-7516
-                        use_target_config_hash=bool(saas_file.publish_job_logs),
+                        target_namespace=target.namespace,
+                        soak_days=soak_days,
+                        schedule=schedule,
+                        blocked_versions=blocked_versions.get(
+                            resource_template.url, set()
+                        ),
+                        use_target_config_hash=bool(
+                            target.promotion.redeploy_on_publisher_config_change
+                        ),
                     )
                     self.subscribers.append(subscriber)
                     for prom_data in target.promotion.promotion_data or []:
@@ -149,13 +173,6 @@ class SaasFilesInventory:
         for subscriber in self.subscribers:
             is_supported = True
             for channel in subscriber.channels:
-                if len(channel.publishers) > 1:
-                    logging.error(
-                        "[%s] We do not support multiple publishers for a single channel - blocked by https://issues.redhat.com/browse/APPSRE-7414",
-                        channel.name,
-                    )
-                    is_supported = False
-                    break
                 if not channel.publishers:
                     logging.error(
                         "[%s] There must be at least one publisher per channel.",
@@ -163,16 +180,6 @@ class SaasFilesInventory:
                     )
                     is_supported = False
                     break
-                if (
-                    len(subscriber.config_hashes_by_channel_name.get(channel.name, []))
-                    > 1
-                ):
-                    logging.error(
-                        "[%s] We do not support multiple publishers for a single channel - blocked by https://issues.redhat.com/browse/APPSRE-7414",
-                        channel.name,
-                    )
-                    is_supported = False
-                    break
             if is_supported:
                 supported_subscribers.append(subscriber)
         self.subscribers = supported_subscribers

reconcile/saas_file_validator.py

@@ -1,15 +1,13 @@
 import logging
 import sys
-from typing import (
-    Callable,
-    Optional,
-)
+from collections.abc import Callable
 
 from reconcile.jenkins_job_builder import init_jjb
 from reconcile.status import ExitCodes
 from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
+from reconcile.typed_queries.quay import get_quay_instances, get_quay_orgs
 from reconcile.typed_queries.repos import get_repos
 from reconcile.typed_queries.saas_files import (
     get_saas_files,
@@ -26,7 +24,7 @@ QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
 
 
 @defer
-def run(dry_run: bool, defer: Optional[Callable] = None) -> None:
+def run(dry_run: bool, defer: Callable | None = None) -> None:
     vault_settings = get_app_interface_vault_settings()
     saasherder_settings = get_saasherder_settings()
     secret_reader = create_secret_reader(use_vault=vault_settings.vault)
@@ -51,7 +49,19 @@ def run(dry_run: bool, defer: Optional[Callable] = None) -> None:
     missing_repos = [r for r in saasherder.repo_urls if r not in app_int_repos]
     for r in missing_repos:
         logging.error(f"repo is missing from codeComponents: {r}")
+    app_int_quay_instances = {i.url for i in get_quay_instances()}
+    app_int_quay_orgs = {(o.instance.url, o.name) for o in get_quay_orgs()}
+    missing_image_patterns = [
+        p
+        for p in saasherder.image_patterns
+        if (parts := p.split("/"))
+        and parts[0] in app_int_quay_instances
+        and len(parts) >= 2
+        and (parts[0], parts[1]) not in app_int_quay_orgs
+    ]
+    for p in missing_image_patterns:
+        logging.error(f"image pattern is missing from quayOrgs: {p}")
     jjb: JJB = init_jjb(secret_reader)
     saasherder.validate_upstream_jobs(jjb)
-    if not saasherder.valid or missing_repos:
+    if not saasherder.valid or missing_repos or missing_image_patterns:
         sys.exit(ExitCodes.ERROR)

reconcile/sendgrid_teammates.py

@@ -44,22 +44,37 @@ def fetch_desired_state(users):
 @retry()
 def fetch_current_state(sg_client):
     state = []
+    limit = 100
 
     # pending invites
-    invites = sg_client.teammates.pending.get().to_dict["result"]
-    for invite in invites:
-        t = Teammate(invite["email"], pending_token=invite["token"])
-        state.append(t)
+    offset = 0
+    while True:
+        invites = sg_client.teammates.pending.get(
+            query_params={"limit": limit, "offset": offset}
+        ).to_dict["result"]
+        if not invites:
+            break
+        for invite in invites:
+            t = Teammate(invite["email"], pending_token=invite["token"])
+            state.append(t)
+        offset += limit
 
     # current teammates
-    teammates = sg_client.teammates.get().to_dict["result"]
-    for teammate in teammates:
-        if teammate["user_type"] == "owner":
-            # we want to ignore the root account (owner account)
-            continue
-
-        t = Teammate(teammate["email"], username=teammate["username"])
-        state.append(t)
+    offset = 0
+    while True:
+        teammates = sg_client.teammates.get(
+            query_params={"limit": limit, "offset": offset}
+        ).to_dict["result"]
+        if not teammates:
+            break
+        for teammate in teammates:
+            if teammate["user_type"] == "owner":
+                # we want to ignore the root account (owner account)
+                continue
+
+            t = Teammate(teammate["email"], username=teammate["username"])
+            state.append(t)
+        offset += limit
 
     return state
 

reconcile/service_dependencies.py

@@ -87,9 +87,6 @@ def get_desired_dependency_names(
                 providers = {res.provider for res in ern.external_resources}
             for p in providers:
                 required_dep_names.update(get_dependency_names(dependency_map, p))
-        kafka_namespaces = [n for n in namespaces if n.kafka_cluster]
-        if kafka_namespaces:
-            required_dep_names.update(get_dependency_names(dependency_map, "kafka"))
 
     return required_dep_names
 

reconcile/signalfx_endpoint_monitoring.py

@@ -1,9 +1,6 @@
 import logging
 import sys
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 from reconcile.closedbox_endpoint_monitoring_base import (
     Endpoint,
@@ -43,7 +40,7 @@ def run(
 
 def build_probe(
     provider: EndpointMonitoringProvider, endpoints: list[Endpoint]
-) -> Optional[OpenshiftResource]:
+) -> OpenshiftResource | None:
     signalfx = provider.signalFx
     if not signalfx:
         return None

reconcile/skupper_network/integration.py

@@ -4,10 +4,7 @@ from collections.abc import (
     Callable,
     Iterable,
 )
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 import jinja2
 import yaml
@@ -58,7 +55,9 @@ def load_site_controller_template(
             resource["content"], undefined=jinja2.StrictUndefined
         ).render(variables)
     except jinja2.exceptions.UndefinedError as e:
-        raise SkupperNetworkExcpetion(f"Failed to render template {path}: {e.message}")
+        raise SkupperNetworkExcpetion(
+            f"Failed to render template {path}: {e.message}"
+        ) from None
     return yaml.safe_load(body)
 
 
@@ -145,7 +144,7 @@ def fetch_current_state(
             ri.initialize_resource_type(
                 cluster=site.cluster.name,
                 namespace=site.namespace.name,
-                resource_type=openshift_resource.kind,
+                resource_type=openshift_resource.kind_and_group,
             )
             ri.add_current(
                 cluster=site.cluster.name,
@@ -169,18 +168,18 @@ def fetch_desired_state(
                 integration=QONTRACT_INTEGRATION,
                 integration_version=QONTRACT_INTEGRATION_VERSION,
             )
-            integration_managed_kinds.add(openshift_resource.kind)
+            integration_managed_kinds.add(openshift_resource.kind_and_group)
             # only add desired state if not deleting
             if not site.delete:
                 ri.initialize_resource_type(
                     cluster=site.cluster.name,
                     namespace=site.namespace.name,
-                    resource_type=openshift_resource.kind,
+                    resource_type=openshift_resource.kind_and_group,
                 )
                 ri.add_desired(
                     cluster=site.cluster.name,
                     namespace=site.namespace.name,
-                    resource_type=openshift_resource.kind,
+                    resource_type=openshift_resource.kind_and_group,
                     name=openshift_resource.name,
                     value=openshift_resource,
                 )
@@ -248,9 +247,9 @@ def get_skupper_networks(query_func: Callable) -> list[SkupperNetworkV1]:
 def run(
     dry_run: bool,
     thread_pool_size: int = 10,
-    internal: Optional[bool] = None,
+    internal: bool | None = None,
     use_jump_host: bool = True,
-    defer: Optional[Callable] = None,
+    defer: Callable | None = None,
 ) -> None:
     vault_settings = get_app_interface_vault_settings()
     secret_reader = create_secret_reader(use_vault=vault_settings.vault)

reconcile/skupper_network/models.py

@@ -77,10 +77,8 @@ class SkupperSite(BaseModel):
             if (
                 isinstance(
                     c,
-                    (
-                        ClusterPeeringConnectionClusterRequesterV1,
-                        ClusterPeeringConnectionClusterAccepterV1,
-                    ),
+                    ClusterPeeringConnectionClusterRequesterV1
+                    | ClusterPeeringConnectionClusterAccepterV1,
                 )
             ) and c.cluster.name == other.cluster.name:
                 return True
@@ -168,7 +166,7 @@ class SkupperSite(BaseModel):
 
     def unique_token_name(self, other: SkupperSite) -> str:
         """Generate a unique token name for a site connection."""
-        return hashlib.sha256(f"{other}-{self}".encode("UTF-8")).hexdigest()
+        return hashlib.sha256(f"{other}-{self}".encode()).hexdigest()
 
     def token_name(self, other: SkupperSite) -> str:
         """Get the token name for a site connection."""

reconcile/skupper_network/reconciler.py

@@ -30,24 +30,20 @@ def delete_skupper_site(
 
     for kind in integration_managed_kinds:
         # delete everything labeled by us
-        to_delete.update(
-            {
-                f'{item["kind"]}-{item["metadata"]["name"]}': item
-                for item in oc.get_items(
-                    kind=kind,
-                    namespace=site.namespace.name,
-                    labels=labels,
-                )
-            }
-        )
+        to_delete.update({
+            f'{item["kind"]}-{item["metadata"]["name"]}': item
+            for item in oc.get_items(
+                kind=kind,
+                namespace=site.namespace.name,
+                labels=labels,
+            )
+        })
         # delete everything else that starts with 'skupper-'
-        to_delete.update(
-            {
-                f'{item["kind"]}-{item["metadata"]["name"]}': item
-                for item in oc.get_items(kind=kind, namespace=site.namespace.name)
-                if item["metadata"]["name"].startswith("skupper-")
-            }
-        )
+        to_delete.update({
+            f'{item["kind"]}-{item["metadata"]["name"]}': item
+            for item in oc.get_items(kind=kind, namespace=site.namespace.name)
+            if item["metadata"]["name"].startswith("skupper-")
+        })
 
     for item in to_delete.values():
         qontract_integration = (
@@ -57,15 +53,13 @@ def delete_skupper_site(
             # don't delete resources managed by other integrations
             continue
 
-        logging.info(
-            [
-                "delete",
-                site.cluster.name,
-                site.namespace.name,
-                item["kind"],
-                item["metadata"]["name"],
-            ]
-        )
+        logging.info([
+            "delete",
+            site.cluster.name,
+            site.namespace.name,
+            item["kind"],
+            item["metadata"]["name"],
+        ])
         if not dry_run:
             oc.delete(site.namespace.name, item["kind"], item["metadata"]["name"])
 
@@ -140,15 +134,13 @@ def _transfer_token(
         integration_version=integration_version,
     ).annotate()
     if not dry_run:
-        logging.info(
-            [
-                "apply",
-                site.cluster.name,
-                site.namespace.name,
-                "Secret",
-                connection_token.name,
-            ]
-        )
+        logging.info([
+            "apply",
+            site.cluster.name,
+            site.namespace.name,
+            "Secret",
+            connection_token.name,
+        ])
         oc.apply(site.namespace.name, connection_token)
 
 
@@ -251,6 +243,7 @@ def reconcile(
 ) -> None:
     """Realize all skupper resources and create skupper site connections."""
 
+    ob.publish_metrics(ri, integration)
     # create/update/delete all skupper site resources
     ob.realize_data(dry_run, oc_map, ri, thread_pool_size)
 

reconcile/skupper_network/site_controller.py

@@ -30,14 +30,14 @@ class SiteController:
         """Skupper site token secret."""
         _labels = copy.deepcopy(labels)
         _labels["skupper.io/type"] = "connection-token-request"
-        return dict(
-            apiVersion="v1",
-            kind="Secret",
-            metadata=dict(
-                name=name,
-                labels=_labels,
-            ),
-        )
+        return {
+            "apiVersion": "v1",
+            "kind": "Secret",
+            "metadata": {
+                "name": name,
+                "labels": _labels,
+            },
+        }
 
 
 def get_site_controller(site: SkupperSite) -> SiteController: