krkn-lib 5.1.10__py3-none-any.whl → 6.0.0__py3-none-any.whl

krkn_lib/k8s/pod_monitor/pod_monitor.py

@@ -1,15 +1,19 @@
+import logging
 import re
+import time
+import traceback
 from concurrent.futures import Future
 from concurrent.futures.thread import ThreadPoolExecutor
 from functools import partial
 
 from kubernetes import watch
-from kubernetes.client import V1Pod, CoreV1Api
+from kubernetes.client import CoreV1Api, V1Pod
+from urllib3.exceptions import ProtocolError
 
 from krkn_lib.models.pod_monitor.models import (
-    PodsSnapshot,
     MonitoredPod,
     PodEvent,
+    PodsSnapshot,
     PodStatus,
 )
 
@@ -47,67 +51,153 @@ def _monitor_pods(
     max_timeout: int,
     name_pattern: str = None,
     namespace_pattern: str = None,
+    max_retries: int = 3,
 ) -> PodsSnapshot:
-    w = watch.Watch(return_type=V1Pod)
+    """
+    Monitor pods with automatic retry on watch stream disconnection.
+
+    :param monitor_partial: Partial function for monitoring pods
+    :param snapshot: Snapshot to populate with pod events
+    :param max_timeout: Maximum time to monitor (seconds)
+    :param name_pattern: Regex pattern for pod names
+    :param namespace_pattern: Regex pattern for namespaces
+    :param max_retries: Maximum number of retries on connection error
+        (default: 3)
+    :return: PodsSnapshot with collected pod events
+    """
+
+    start_time = time.time()
+    retry_count = 0
     deleted_parent_pods = []
     restored_pods = []
     cluster_restored = False
-    for event in w.stream(monitor_partial, timeout_seconds=max_timeout):
-        match_name = True
-        match_namespace = True
-        event_type = event["type"]
-        pod = event["object"]
 
-        if namespace_pattern:
-            match = re.match(namespace_pattern, pod.metadata.namespace)
-            match_namespace = match is not None
-        if name_pattern:
-            match = re.match(name_pattern, pod.metadata.name)
-            match_name = match is not None
-
-        if match_name and match_namespace:
-            pod_event = PodEvent()
-            if event_type == "MODIFIED":
-                if pod.metadata.deletion_timestamp is not None:
-                    pod_event.status = PodStatus.DELETION_SCHEDULED
-                    deleted_parent_pods.append(pod.metadata.name)
-                elif _is_pod_ready(pod):
-                    pod_event.status = PodStatus.READY
-                    # if there are at least the same number of ready
-                    # pods as the snapshot.initial_pods set we assume that
-                    # the cluster is restored to the initial condition
-                    restored_pods.append(pod.metadata.name)
-                    if len(restored_pods) >= len(snapshot.initial_pods):
-                        cluster_restored = True
-                else:
-                    pod_event.status = PodStatus.NOT_READY
-
-            elif event_type == "DELETED":
-                pod_event.status = PodStatus.DELETED
-            elif event_type == "ADDED":
-                pod_event.status = PodStatus.ADDED
-
-            if pod_event.status == PodStatus.ADDED:
-                snapshot.added_pods.append(pod.metadata.name)
-                # in case a pod is respawn with the same name
-                # the dictionary must not be reinitialized
-                if pod.metadata.name not in snapshot.pods:
-                    snapshot.pods[pod.metadata.name] = MonitoredPod()
-                    snapshot.pods[pod.metadata.name].name = pod.metadata.name
-                    snapshot.pods[pod.metadata.name].namespace = (
-                        pod.metadata.namespace
+    while retry_count <= max_retries:
+        try:
+            # Calculate remaining timeout if retrying
+            if retry_count > 0:
+                elapsed = time.time() - start_time
+                remain_timeout = max(1, int(max_timeout - elapsed))
+                logging.info("remain timeout " + str(remain_timeout))
+                if remain_timeout <= 0:
+                    logging.info(
+                        "Maximum timeout reached, stopping monitoring"
                     )
-            # skips events out of the snapshot
-            if pod.metadata.name in snapshot.pods:
-                snapshot.pods[pod.metadata.name].status_changes.append(
-                    pod_event
+                    break
+                logging.info(
+                    "Reconnecting watch stream"
+                    f"(attempt {retry_count}/{max_retries}),"
+                    f"remaining timeout: {remain_timeout}s"
                 )
-            # this flag is set when all the pods
-            # that has been deleted or not ready
-            # have been restored, if True the
-            # monitoring is stopeed earlier
-            if cluster_restored:
-                w.stop()
+            else:
+                remain_timeout = max_timeout
+
+            w = watch.Watch(return_type=V1Pod)
+
+            for e in w.stream(monitor_partial, timeout_seconds=remain_timeout):
+                match_name = True
+                match_namespace = True
+                event_type = e["type"]
+                pod = e["object"]
+
+                if namespace_pattern:
+                    match = re.match(namespace_pattern, pod.metadata.namespace)
+                    match_namespace = match is not None
+                if name_pattern:
+                    match = re.match(name_pattern, pod.metadata.name)
+                    match_name = match is not None
+
+                if match_name and match_namespace:
+                    pod_event = PodEvent()
+                    pod_name = pod.metadata.name
+                    if event_type == "MODIFIED":
+                        if pod.metadata.deletion_timestamp is not None:
+                            pod_event.status = PodStatus.DELETION_SCHEDULED
+                            if pod_name not in deleted_parent_pods:
+                                deleted_parent_pods.append(pod_name)
+                        elif _is_pod_ready(pod):
+                            pod_event.status = PodStatus.READY
+                            # if there are at least the same number of ready
+                            # pods as the snapshot.initial_pods set we assume
+                            # the cluster is restored to the initial condition
+                            if pod_name not in restored_pods:
+                                restored_pods.append(pod_name)
+                            inital_pod_len = len(snapshot.initial_pods)
+                            if len(restored_pods) >= inital_pod_len:
+                                cluster_restored = True
+                        else:
+                            pod_event.status = PodStatus.NOT_READY
+
+                    elif event_type == "DELETED":
+                        pod_event.status = PodStatus.DELETED
+                    elif event_type == "ADDED":
+                        pod_event.status = PodStatus.ADDED
+
+                    if pod_event.status == PodStatus.ADDED:
+
+                        if pod_name not in snapshot.added_pods:
+                            snapshot.added_pods.append(pod_name)
+                        # in case a pod is respawn with the same name
+                        # the dictionary must not be reinitialized
+                        if pod_name not in snapshot.pods:
+                            snapshot.pods[pod_name] = MonitoredPod()
+                            snapshot.pods[pod_name].name = pod_name
+                            snapshot.pods[pod_name].namespace = (
+                                pod.metadata.namespace
+                            )
+                    # skips events out of the snapshot
+                    if pod_name in snapshot.pods:
+                        snapshot.pods[pod_name].status_changes.append(
+                            pod_event
+                        )
+                    # this flag is set when all the pods
+                    # that has been deleted or not ready
+                    # have been restored, if True the
+                    # monitoring is stopped earlier
+                    if cluster_restored:
+                        logging.info("Cluster restored, stopping monitoring")
+                        w.stop()
+                        return snapshot
+
+            # If we exit the loop normally (timeout reached), we're done
+            logging.info("Watch stream completed normally")
+            break
+
+        except ProtocolError as e:
+
+            if retry_count > max_retries:
+                logging.warning(
+                    f"Watch stream connection broken after {max_retries}"
+                    f"retries. ProtocolError: {e}. Returning snapshot "
+                    "with data collected so far."
+                )
+                break
+
+            # Log retry attempt
+            logging.info(
+                f"Watch stream connection broken (ProtocolError): {e}. "
+                f"Retry {retry_count}/{max_retries} in progress..."
+            )
+            backoff_time = 1
+
+            # Check if we have time for backoff
+            elapsed = time.time() - start_time
+            if elapsed + backoff_time >= max_timeout:
+                logging.info(
+                    "Not enough time remaining for backoff, "
+                    "returning snapshot with data collected."
+                )
+                break
+
+            logging.debug(f"Waiting {backoff_time}s before retry...")
+            time.sleep(backoff_time)
+
+        except Exception as e:
+            logging.error("Error in monitor pods: " + str(e))
+            logging.error("Stack trace:\n%s", traceback.format_exc())
+            raise Exception(e)
+
+        retry_count += 1
 
     return snapshot
 

krkn_lib/k8s/templates/snapshot.j2

@@ -0,0 +1,10 @@
+apiVersion: snapshot.kubevirt.io/v1beta1
+kind: VirtualMachineSnapshot
+metadata:
+  name: {{name}}
+  namespace: {{namespace}}
+spec:
+  source:
+    apiGroup: kubevirt.io
+    kind: VirtualMachine
+    name: {{vm_name}}

krkn_lib/models/elastic/models.py

@@ -1,6 +1,7 @@
 import datetime
 
 from elasticsearch_dsl import (
+    Boolean,
     Date,
     Document,
     Float,
@@ -9,7 +10,6 @@ from elasticsearch_dsl import (
     Keyword,
     Nested,
     Text,
-    Boolean,
 )
 
 from krkn_lib.models.telemetry import ChaosRunTelemetry
@@ -144,6 +144,7 @@ class ElasticChaosRunTelemetry(Document):
     run_uuid = Text(fields={"keyword": Keyword()})
     health_checks = Nested(ElasticHealthChecks, multi=True)
     virt_checks = Nested(ElasticVirtChecks, multi=True)
+    post_virt_checks = Nested(ElasticVirtChecks, multi=True)
 
     class Index:
         name = "chaos_run_telemetry"
@@ -261,6 +262,28 @@ class ElasticChaosRunTelemetry(Document):
         else:
             self.virt_checks = None
 
+        if chaos_run_telemetry.post_virt_checks:
+            self.post_virt_checks = [
+                ElasticVirtChecks(
+                    vm_name=post_info.vm_name,
+                    ip_address=post_info.ip_address,
+                    new_ip_address=post_info.new_ip_address,
+                    namespace=post_info.namespace,
+                    node_name=post_info.node_name,
+                    status=post_info.status,
+                    start_timestamp=datetime.datetime.fromisoformat(
+                        str(post_info.start_timestamp)
+                    ),
+                    end_timestamp=datetime.datetime.fromisoformat(
+                        str(post_info.end_timestamp)
+                    ),
+                    duration=post_info.duration,
+                )
+                for post_info in chaos_run_telemetry.post_virt_checks
+            ]
+        else:
+            self.post_virt_checks = None
+
         self.timestamp = chaos_run_telemetry.timestamp
         self.total_node_count = chaos_run_telemetry.total_node_count
         self.cloud_infrastructure = chaos_run_telemetry.cloud_infrastructure

krkn_lib/models/k8s/models.py

@@ -326,7 +326,7 @@ class AffectedNodeStatus:
         match_found = []
         for affected_node in self.affected_nodes:
             counter2 = counter + 1
-            for aff_node2 in self.affected_nodes[counter + 1:]:  # fmt: skip
+            for aff_node2 in self.affected_nodes[counter + 1 :]:  # noqa: E203
                 if affected_node.node_name == aff_node2.node_name:
                     match_found.append(counter2)
                     cur_node = self.affected_nodes[counter]

krkn_lib/models/pod_monitor/models.py

@@ -2,9 +2,9 @@ import json
 import time
 from dataclasses import dataclass
 from enum import Enum
-from typing import Optional, Any
+from typing import Any, Optional
 
-from krkn_lib.models.k8s import PodsStatus, AffectedPod
+from krkn_lib.models.k8s import AffectedPod, PodsStatus
 
 
 class PodStatus(Enum):

krkn_lib/models/telemetry/models.py

@@ -545,6 +545,10 @@ class ChaosRunTelemetry:
     """
     Virt checks of VMIs
     """
+    post_virt_checks: list[VirtCheck] = None
+    """
+    Post Scenario Virt checks of VMIs
+    """
     job_status: bool = True
     """
     Overall job status, will take all scenario's exit status
@@ -597,6 +601,11 @@ class ChaosRunTelemetry:
                 if json_dict.get("virt_checks")
                 else None
             )
+            self.post_virt_checks = (
+                [VirtCheck(k) for k in json_dict.get("post_virt_checks")]
+                if json_dict.get("post_virt_checks")
+                else None
+            )
             self.job_status = json_dict.get("job_status")
             self.build_url = json_dict.get("build_url")
 

krkn_lib/ocp/krkn_openshift.py

@@ -65,7 +65,7 @@ class KrknOpenshift(KrknKubernetes):
                 )
 
                 path = "/apis/config.openshift.io/v1/infrastructures/cluster"
-                (data) = api_client.call_api(
+                data = api_client.call_api(
                     path,
                     "GET",
                     path_params,
@@ -111,7 +111,7 @@ class KrknOpenshift(KrknKubernetes):
                 )
 
                 path = "/apis/config.openshift.io/v1/infrastructures/cluster"
-                (data) = api_client.call_api(
+                data = api_client.call_api(
                     path,
                     "GET",
                     path_params,
@@ -147,7 +147,7 @@ class KrknOpenshift(KrknKubernetes):
                 )
 
                 path = "/apis/config.openshift.io/v1/networks"
-                (data) = api_client.call_api(
+                data = api_client.call_api(
                     path,
                     "GET",
                     path_params,
@@ -440,7 +440,7 @@ class KrknOpenshift(KrknKubernetes):
                 ["application/json"]
             )
 
-            (data) = self.api_client.call_api(
+            data = self.api_client.call_api(
                 path,
                 "GET",
                 path_params,

krkn_lib/prometheus/krkn_prometheus.py

@@ -50,7 +50,7 @@ class KrknPrometheus:
         query: str,
         start_time: datetime = None,
         end_time: datetime = None,
-        granularity: int = 10
+        granularity: int = 10,
     ) -> list[dict[str:any]]:
         """
         Executes a query to the Prometheus API in PromQL languag,

krkn_lib/telemetry/k8s/krkn_telemetry_kubernetes.py

@@ -425,7 +425,7 @@ class KrknTelemetryKubernetes:
         :param uploaded_file_list: uploaded file list shared between threads
         :param max_retries: maximum number of retries from config.yaml.
             If 0 will retry indefinitely.
-        :param remote_file_prefix: the prefix that will given to the file
+        :param remote_file_prefix: the prefix that will be given to the file
             in the S3 bucket along with the progressive number
             (if is a multiple file archive)
         :param remote_file_extension: the extension of the remote

krkn_lib/telemetry/ocp/krkn_telemetry_openshift.py

@@ -244,7 +244,7 @@ class KrknTelemetryOpenshift(KrknTelemetryKubernetes):
                 )
 
                 path = "/apis/kubevirt.io/v1/virtualmachineinstances"
-                (data) = api_client.call_api(
+                data = api_client.call_api(
                     path,
                     "GET",
                     path_params,

krkn_lib/tests/base_test.py

@@ -583,7 +583,7 @@ class BaseTest(unittest.TestCase):
                     "start_timestamp": "2025-03-12T14:57:34.555878",
                     "end_timestamp": "2025-03-12T14:57:54.904352",
                     "duration": 20.348474,
-                    "new_ip_address": ""
+                    "new_ip_address": "",
                 },
                 {
                     "node_name": "h27-r660",
@@ -594,7 +594,7 @@ class BaseTest(unittest.TestCase):
                     "start_timestamp": "2025-03-12T14:57:34.759105",
                     "end_timestamp": "2025-03-12T14:57:54.904352",
                     "duration": 20.145247,
-                    "new_ip_address": ""
+                    "new_ip_address": "",
                 },
                 {
                     "node_name": "h10-r660",
@@ -605,9 +605,22 @@ class BaseTest(unittest.TestCase):
                     "start_timestamp": "2025-03-12T14:57:35.308957",
                     "end_timestamp": "2025-03-12T14:57:54.904352",
                     "duration": 19.595395,
-                    "new_ip_address": "0.0.0.3"
+                    "new_ip_address": "0.0.0.3",
                 },
             ],
+            "post_virt_checks": [
+                {
+                    "node_name": "h10-r660",
+                    "namespace": "benchmark-runner",
+                    "ip_address": "0.0.0.0",
+                    "vm_name": "windows-vm-52",
+                    "status": False,
+                    "start_timestamp": "2025-03-12T14:57:55.904352",
+                    "end_timestamp": "2025-03-12T14:57:55.904352",
+                    "duration": 0.00,
+                    "new_ip_address": "",
+                }
+            ],
             "total_node_count": 3,
             "cloud_infrastructure": "AWS",
             "cloud_type": "EC2",

krkn_lib/tests/test_krkn_elastic_models.py

@@ -1,9 +1,9 @@
+import datetime
 import uuid
 
 from krkn_lib.models.elastic.models import ElasticChaosRunTelemetry
 from krkn_lib.models.telemetry import ChaosRunTelemetry
 from krkn_lib.tests import BaseTest
-import datetime
 
 
 class TestKrknElasticModels(BaseTest):
@@ -222,9 +222,7 @@ class TestKrknElasticModels(BaseTest):
         self.assertEqual(
             elastic_telemetry.virt_checks[0].ip_address, "0.0.0.0"
         )
-        self.assertEqual(
-            elastic_telemetry.virt_checks[0].new_ip_address, ""
-        )
+        self.assertEqual(elastic_telemetry.virt_checks[0].new_ip_address, "")
         self.assertEqual(
             elastic_telemetry.virt_checks[2].new_ip_address, "0.0.0.3"
         )
@@ -246,6 +244,27 @@ class TestKrknElasticModels(BaseTest):
         )
         self.assertEqual(elastic_telemetry.virt_checks[0].duration, 20.348474)
 
+        # post_virt_checks
+        self.assertEqual(len(elastic_telemetry.post_virt_checks), 1)
+        self.assertEqual(
+            elastic_telemetry.post_virt_checks[0].vm_name, "windows-vm-52"
+        )
+        self.assertEqual(
+            elastic_telemetry.post_virt_checks[0].ip_address, "0.0.0.0"
+        )
+        self.assertEqual(
+            elastic_telemetry.post_virt_checks[0].new_ip_address, ""
+        )
+
+        self.assertEqual(
+            elastic_telemetry.post_virt_checks[0].namespace, "benchmark-runner"
+        )
+        self.assertEqual(
+            elastic_telemetry.post_virt_checks[0].node_name, "h10-r660"
+        )
+
+        self.assertEqual(elastic_telemetry.post_virt_checks[0].status, False)
+
         self.assertEqual(elastic_telemetry.total_node_count, 3)
         self.assertEqual(elastic_telemetry.cloud_infrastructure, "AWS")
         self.assertEqual(elastic_telemetry.cloud_type, "EC2")

krkn_lib/tests/test_krkn_kubernetes_check.py

@@ -2,10 +2,11 @@ import logging
 import time
 import unittest
 
-from krkn_lib.tests import BaseTest
-from krkn_lib.k8s import ApiRequestException
 from kubernetes.client import ApiException
 
+from krkn_lib.k8s import ApiRequestException
+from krkn_lib.tests import BaseTest
+
 
 class KrknKubernetesTestsCheck(BaseTest):
     def test_check_namespaces(self):

krkn_lib/tests/test_krkn_kubernetes_create.py

@@ -1,10 +1,12 @@
 import logging
+import tempfile
 import unittest
+
 import yaml
-import tempfile
-from krkn_lib.tests import BaseTest
-from kubernetes.client import ApiException
 from jinja2 import Environment, FileSystemLoader
+from kubernetes.client import ApiException
+
+from krkn_lib.tests import BaseTest
 
 
 class KrknKubernetesTestsCreate(BaseTest):

krkn_lib/tests/test_krkn_kubernetes_delete.py

@@ -2,10 +2,11 @@ import logging
 import time
 import unittest
 
-from krkn_lib.tests import BaseTest
-from krkn_lib.k8s import ApiRequestException
 from kubernetes.client import ApiException
 
+from krkn_lib.k8s import ApiRequestException
+from krkn_lib.tests import BaseTest
+
 
 class KrknKubernetesTestsDelete(BaseTest):
     def test_delete_namespace(self):

krkn_lib/tests/test_krkn_kubernetes_get.py

@@ -1,16 +1,17 @@
 import logging
+import os
 import random
 import re
 import time
-import os
 import unittest
 
-from krkn_lib.models.telemetry import ChaosRunTelemetry
-from krkn_lib.tests import BaseTest
 from kubernetes import config
-from krkn_lib.k8s import ApiRequestException, KrknKubernetes
 from kubernetes.client import ApiException
 
+from krkn_lib.k8s import ApiRequestException, KrknKubernetes
+from krkn_lib.models.telemetry import ChaosRunTelemetry
+from krkn_lib.tests import BaseTest
+
 
 class KrknKubernetesTestsGet(BaseTest):
     def test_get_version(self):

krkn_lib/tests/test_krkn_kubernetes_misc.py

@@ -6,11 +6,11 @@ import time
 import unittest
 
 import yaml
+from kubernetes.client import ApiException
+from tzlocal import get_localzone
 
 from krkn_lib.models.krkn import HogConfig, HogType
 from krkn_lib.tests import BaseTest
-from tzlocal import get_localzone
-from kubernetes.client import ApiException
 
 
 class KrknKubernetesTestsMisc(BaseTest):
@@ -232,7 +232,7 @@ class KrknKubernetesTestsMisc(BaseTest):
             ["application/json"]
         )
         path = f"/api/v1/nodes/{node_name}/proxy/stats/summary"
-        (data) = self.lib_k8s.api_client.call_api(
+        data = self.lib_k8s.api_client.call_api(
             path,
             "GET",
             path_params,

krkn_lib/tests/test_krkn_kubernetes_models.py

@@ -1,10 +1,10 @@
-from krkn_lib.models.krkn import HogConfig, HogType
 from krkn_lib.models.k8s import (
     AffectedNode,
     AffectedNodeStatus,
     AffectedPod,
     PodsStatus,
 )
+from krkn_lib.models.krkn import HogConfig, HogType
 from krkn_lib.tests import BaseTest
 
 

krkn_lib/tests/test_krkn_kubernetes_pods_monitor_models.py

@@ -1,13 +1,12 @@
 import time
 import unittest
 
-
-from krkn_lib.models.k8s import PodsStatus, AffectedPod
+from krkn_lib.models.k8s import AffectedPod, PodsStatus
 from krkn_lib.models.pod_monitor.models import (
-    PodEvent,
-    PodStatus,
     MonitoredPod,
+    PodEvent,
     PodsSnapshot,
+    PodStatus,
 )