Kea2-python 1.1.0b1__py3-none-any.whl

kea2/report/utils.py

@@ -0,0 +1,10 @@
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+
+@contextmanager
+def thread_pool(max_workers=128, wait=True, name_prefix="worker"):
+    executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix=name_prefix)
+    try:
+        yield executor
+    finally:
+        executor.shutdown(wait=wait)

kea2/result.py

@@ -0,0 +1,257 @@
+from dataclasses import asdict
+from enum import Enum
+import json
+from typing import Deque, Dict
+from unittest import TestCase, TextTestResult
+from collections import deque
+
+from .state import INVARIANT_MARKER
+from .typedefs import PBTTestResult, PropertyExecutionInfo, PropStatistic
+from .utils import getLogger, getFullPropName
+
+
+logger = getLogger(__name__)
+
+
+class CheckKind(Enum):
+    PROPERTY = "property"
+    INVARIANT = "invariant"
+
+
+def get_check_kind(test: TestCase) -> CheckKind:
+    if hasattr(test, INVARIANT_MARKER):
+        return CheckKind.INVARIANT
+    return CheckKind.PROPERTY
+
+
+class KeaJsonResult(TextTestResult):
+
+    # +------------------------------+
+    # | Setup utils                  |
+    # +------------------------------+
+    res: PBTTestResult = dict()
+    lastPropertyInfo: PropertyExecutionInfo
+    lastInvariantInfo: PropertyExecutionInfo
+    executionInfoBuffer: Deque["PropertyExecutionInfo"] = deque()
+    currentStepsCount: int
+    result_file: str
+    property_exection_result_file: str
+
+    @classmethod
+    def setProperties(cls, allProperties: Dict):
+        for testCase in allProperties.values():
+            cls.res[getFullPropName(testCase)] = PropStatistic(kind=CheckKind.PROPERTY.value)
+    
+    @classmethod
+    def setInvariants(cls, allInvariants: Dict):
+        for testCase in allInvariants.values():
+            cls.res[getFullPropName(testCase)] = PropStatistic(kind=CheckKind.INVARIANT.value)
+    
+    @classmethod
+    def setOutputFile(cls, result_file, property_exec_result_file):
+        cls.result_file = result_file
+        cls.property_exection_result_file = property_exec_result_file
+
+    def __init__(self, stream, descriptions, verbosity):
+        super().__init__(stream, descriptions, verbosity)
+        self.showAll = True
+        self.currentStepsCount = 0
+        self.lastInvariantInfo = None
+    
+    def startTest(self, test):
+        if get_check_kind(test) is CheckKind.INVARIANT:
+            self.lastInvariantInfo = PropertyExecutionInfo(
+                propName=getFullPropName(test),
+                kind=CheckKind.INVARIANT.value,
+                state="start",
+                tb="",
+                startStepsCount=self.currentStepsCount
+            )
+        super(TextTestResult, self).startTest(test)
+        if self.showAll:
+            self.stream.write(" - ")
+            self.stream.write(str(test))
+            self.stream.write(" ... ")
+            self.stream.flush()
+            self._newline = False
+
+    def setCurrentStepsCount(self, stepsCount: int):
+        self.currentStepsCount = stepsCount
+
+    # +------------------------------+
+    # | Property-specific execution  |
+    # +------------------------------+
+    def addExcutedProperty(self, test: TestCase, stepsCount: int):
+        self.res[getFullPropName(test)].executed += 1
+
+        self.lastPropertyInfo = PropertyExecutionInfo(
+            propName=getFullPropName(test),
+            kind=CheckKind.PROPERTY.value,
+            state="start",
+            tb="",
+            startStepsCount=stepsCount
+        )
+
+    def addPropertyPrecondSatisfied(self, test: TestCase):
+        self.res[getFullPropName(test)].precond_satisfied += 1
+
+    def updateExecutionInfo(self, test):
+        # if the test is a property, and it is still in "start" state, set it to "pass"
+        # then record the last executed property info
+        if get_check_kind(test) is CheckKind.PROPERTY:
+            if self.lastPropertyInfo.state == "start":
+                self.lastPropertyInfo.state = "pass"
+            self.executionInfoBuffer.append(self.lastPropertyInfo)
+        # if the test is an invariant, and the last invariant failed or errored, record it
+        # (only record failed/errored invariants)
+        if get_check_kind(test) is CheckKind.INVARIANT:
+            if self.lastInvariantInfo.state in {"fail", "error"}:
+                self.executionInfoBuffer.append(self.lastInvariantInfo)
+
+    def getExcutedProperty(self, test: TestCase):
+        return self.res[getFullPropName(test)].executed
+
+    # +------------------------------+
+    # | Shared result updates        |
+    # +------------------------------+
+    def addFailure(self, test, err):
+        super().addFailure(test, err)
+        self.res[getFullPropName(test)].fail += 1
+        self._record_exception(test, err, "fail")
+
+    def addError(self, test, err):
+        super().addError(test, err)
+        self.res[getFullPropName(test)].error += 1
+        self._record_exception(test, err, "error")
+
+    # +------------------------------+
+    # | Result utils                 |
+    # +------------------------------+
+    def printError(self, test):
+        self._print_property_error(test)
+        self._print_invariant_error(test)
+
+    def logSummary(self):
+        property_fails = sum(_.fail for _ in self.res.values() if _.kind == "property")
+        property_errors = sum(_.error for _ in self.res.values() if _.kind == "property")
+        invariant_fails = sum(_.fail for _ in self.res.values() if _.kind == "invariant")
+        invariant_errors = sum(_.error for _ in self.res.values() if _.kind == "invariant")
+        logger.info(f"[Property Execution Summary] Errors:{property_errors}, Fails:{property_fails}")
+        if invariant_fails > 0 or invariant_errors > 0:
+            logger.info(f"[Invariant Execution Summary] Errors:{invariant_errors}, Fails:{invariant_fails}")
+    
+    def flushResult(self):
+        json_res = dict()
+        for propName, propStatitic in self.res.items():
+            json_res[propName] = asdict(propStatitic)
+        with open(self.result_file, "w", encoding="utf-8") as fp:
+            json.dump(json_res, fp, indent=4)
+
+        with open(self.property_exection_result_file, "a", encoding="utf-8") as fp:
+            while self.executionInfoBuffer:
+                execInfo = self.executionInfoBuffer.popleft()
+                fp.write(f"{json.dumps(asdict(execInfo))}\n")
+
+    # +------------------------------+
+    # | Property/invariant helpers   |
+    # +------------------------------+
+    def _record_exception(self, test, err, state):
+        if get_check_kind(test) is CheckKind.PROPERTY:
+            self.lastPropertyInfo.state = state
+            self.lastPropertyInfo.tb = self._exc_info_to_string(err, test)
+            return
+        if get_check_kind(test) is CheckKind.INVARIANT:
+            self.lastInvariantInfo = PropertyExecutionInfo(
+                propName=getFullPropName(test),
+                kind=CheckKind.INVARIANT.value,
+                state=state,
+                tb=self._exc_info_to_string(err, test),
+                startStepsCount=self.currentStepsCount,
+            )
+
+    def _print_property_error(self, test):
+        if get_check_kind(test) is not CheckKind.PROPERTY:
+            return
+        if self.lastPropertyInfo.state not in ["fail", "error"]:
+            return
+        flavour = self.lastPropertyInfo.state.upper()
+        self.stream.writeln("")
+        self.stream.writeln(self.separator1)
+        self.stream.writeln("%s: %s" % (flavour, self.getDescription(test)))
+        self.stream.writeln(self.separator2)
+        self.stream.writeln("%s" % self.lastPropertyInfo.tb)
+        self.stream.writeln(self.separator1)
+        self.stream.flush()
+
+    def _print_invariant_error(self, test):
+        if get_check_kind(test) is not CheckKind.INVARIANT:
+            return
+        if not self.lastInvariantInfo or self.lastInvariantInfo.state not in {"fail", "error"}:
+            return
+        self.stream.writeln("")
+        self.stream.writeln(self.separator1)
+        self.stream.writeln("%s: %s" % (self.lastInvariantInfo.state.upper(), self.lastInvariantInfo.tb))
+        self.stream.writeln(self.separator1)
+        self.stream.flush()
+
+
+class KeaTextTestResult(TextTestResult):
+
+    # +------------------------------+
+    # | Setup and output formatting  |
+    # +------------------------------+
+    def __init__(self, stream, descriptions, verbosity):
+        super().__init__(stream, descriptions, verbosity)
+        self.showAll = True
+    
+    def getDescription(self: "KeaJsonResult", test: "TestCase"):
+        doc_first_line = test.shortDescription()
+        if get_check_kind(test) is CheckKind.INVARIANT:
+            return getFullPropName(test)
+        if self.descriptions and doc_first_line:
+            doc_first_line = "# " + doc_first_line
+            return '\n'.join((str(test), doc_first_line))
+        else:
+            return str(test)
+    
+    def startTest(self: "KeaJsonResult", test):
+        if get_check_kind(test) is CheckKind.INVARIANT:
+            self.stream.writeln(f"[INFO] Invariant: {getFullPropName(test)}")
+            self.stream.flush()
+            self._newline = True
+        else:
+            self.stream.write("[INFO] Start executing property: ")
+            self.stream.writeln(self.getDescription(test))
+            self.stream.flush()
+            self._newline = True
+    
+    # +------------------------------+
+    # | Shared status updates        |
+    # +------------------------------+
+    @property
+    def wasFail(self):
+        return self._wasFail
+    
+    def addError(self, test, err):
+        self._wasFail = True
+        return super().addError(test, err)
+    
+    def addFailure(self, test, err):
+        self._wasFail = True
+        return super().addFailure(test, err)
+    
+    def addSuccess(self, test):
+        self._wasFail = False
+        return super().addSuccess(test)
+
+    def addSkip(self, test, reason):
+        self._wasFail = False
+        return super().addSkip(test, reason)
+    
+    def addExpectedFailure(self, test, err):
+        self._wasFail = False
+        return super().addExpectedFailure(test, err)
+    
+    def addUnexpectedSuccess(self, test):
+        self._wasFail = False
+        return super().addUnexpectedSuccess(test)

kea2/resultSyncer.py

@@ -0,0 +1,65 @@
+import threading
+
+from pathlib import Path
+
+from .adbUtils import ADBDevice
+from .utils import getLogger, catchException, timer
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from .keaUtils import Options
+
+logger = getLogger(__name__)
+
+
+class ResultSyncer:
+
+    def __init__(self, device_output_dir, options: "Options"):
+        self.device_output_dir = device_output_dir
+        self.output_dir = options.output_dir / Path(device_output_dir).name
+        self.running = False
+        self.thread = None
+        self.sync_event = threading.Event()
+
+        ADBDevice.setDevice(serial=options.serial, transport_id=options.transport_id)
+        self.dev = ADBDevice()
+
+    def run(self):
+        """Start a background thread to sync device data when triggered"""
+        self.running = True
+        self.thread = threading.Thread(target=self._sync_thread, daemon=True)
+        self.thread.start()
+
+    def _sync_thread(self):
+        """Thread function that waits for sync event and then syncs data"""
+        while self.running:
+            # Wait for sync event with a timeout to periodically check if still running
+            if self.sync_event.wait(timeout=1):
+                self._sync_device_data()
+                self.sync_event.clear()
+
+    @timer("Data Sync cost %cost_time seconds")
+    def close(self):
+        self.running = False
+        self.sync_event.set()
+        if self.thread and self.thread.is_alive():
+            logger.info("Syncing result data from device. Please wait...")
+            self.thread.join(timeout=10)
+        self._sync_device_data()
+        try:
+            logger.debug(f"Removing device output directory: {self.device_output_dir}")
+            remove_device_dir = ["rm", "-rf", self.device_output_dir]
+            self.dev.shell(remove_device_dir)
+        except Exception as e:
+            logger.error(f"Error removing device output directory: {e}", flush=True)
+
+    @catchException("Error during device data sync.")    
+    def _sync_device_data(self):
+        """
+        Sync the device data to the local directory.
+        """
+        logger.debug("Syncing data")
+        self.dev.sync.pull_dir(self.device_output_dir, self.output_dir, exist_ok=True)
+
+        remove_pulled_screenshots = ["find", self.device_output_dir, "-name", '"*.png"', "-delete"]
+        self.dev.shell(remove_pulled_screenshots)

kea2/state.py

@@ -0,0 +1,22 @@
+# state.py is used for stateful testing
+#
+# Supports defining complex state that can be shared across multiple test cases 
+# to control test flow and dependencies
+INVARIANT_MARKER = "_is_invariant"
+
+
+class State(dict):
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+
+state = State()
+
+
+def invariant(f):
+    setattr(f, INVARIANT_MARKER, True)
+    return f

kea2/typedefs.py

@@ -0,0 +1,32 @@
+# Class Typing
+from dataclasses import dataclass
+from typing import Deque, Dict, Literal, NewType
+from unittest import TestCase
+
+PRECONDITIONS_MARKER = "preconds"
+PROB_MARKER = "prob"
+MAX_TRIES_MARKER = "max_tries"
+INTERRUPTABLE_MARKER = "interruptable"
+
+@dataclass
+class PropStatistic:
+    kind: Literal["property", "invariant"] = "unknown"
+    precond_satisfied: int = 0
+    executed: int = 0
+    fail: int = 0
+    error: int = 0
+
+
+@dataclass
+class PropertyExecutionInfo:
+    startStepsCount: int
+    propName: "PropName"
+    kind: Literal["property", "invariant"]
+    state: Literal["start", "pass", "fail", "error"]
+    tb: str
+
+
+PropName = NewType("PropName", str)
+PropertyStore = NewType("PropertyStore", Dict[PropName, TestCase])
+
+PBTTestResult = NewType("PBTTestResult", Dict[PropName, PropStatistic])