Kea2-python 0.3.6__py3-none-any.whl → 1.0.0__py3-none-any.whl

kea2/kea_launcher.py

@@ -2,6 +2,9 @@ import sys
 import argparse
 import unittest
 from typing import List
+import os
+
+
 
 def _set_runner_parser(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]"):
     parser = subparsers.add_parser("run", help="run kea2")
@@ -122,6 +125,15 @@ def _set_runner_parser(subparsers: "argparse._SubParsersAction[argparse.Argument
         help="Take screenshots for every step.",
     )
 
+    parser.add_argument(
+        "--pre-failure-screenshots",
+        dest="pre_failure_screenshots",
+        type=int,
+        required=False,
+        default=0,
+        help="Dump n screenshots before failure. 0 means take screenshots for every step.",
+    )
+
     parser.add_argument(
         "--act-whitelist-file",
         dest="act_whitelist_file",
@@ -141,15 +153,19 @@ def _set_runner_parser(subparsers: "argparse._SubParsersAction[argparse.Argument
     parser.add_argument(
         "extra",
         nargs=argparse.REMAINDER,
-        help="Extra args for unittest <args>",
+        help="Extra args (e.g. propertytest & --). See docs (https://github.com/ecnusse/Kea2/blob/main/docs/manual_en.md) for details.",
     )
 
 
-def unittest_info_logger(args):
+def extra_args_info_logger(args):
     if args.agent == "native":
         print("[Warning] Property not availble in native agent.", flush=True)
     if args.unittest_args:
         print("Captured unittest args:", args.unittest_args, flush=True)
+    if args.propertytest_args:
+        print("Captured propertytest args:", args.propertytest_args, flush=True)
+    if args.extra:
+        print("Captured extra args (Will be appended to fastbot launcher):", args.extra, flush=True)
 
 
 def driver_info_logger(args):
@@ -170,6 +186,8 @@ def driver_info_logger(args):
         print("  log_stamp:", args.log_stamp, flush=True)
     if args.take_screenshots:
         print("  take_screenshots:", args.take_screenshots, flush=True)
+        if args.pre_failure_screenshots:
+            print("  pre_failure_screenshots:", args.pre_failure_screenshots, flush=True)
     if args.max_step:
         print("  max_step:", args.max_step, flush=True)
 
@@ -184,30 +202,44 @@ def parse_args(argv: List):
 
 
 def _sanitize_args(args):
+    args.mode = None
+    args.propertytest_args = None
     if args.agent == "u2" and not args.driver_name:
         if args.extra == []:
             args.driver_name = "d"
         else:
             raise ValueError("--driver-name should be specified when customizing script in --agent u2")
-    if args.extra:
-        args.extra = args.extra[1:] if args.extra[0] == "--" else args.extra
-        unittest_index = args.extra.index("unittest") if "unittest" in args.extra else -1
-        if unittest_index != -1:
-            unittest_args = args.extra[unittest_index+1:]
-            setattr(args, "unittest_args", unittest_args)
-            args.extra = args.extra[:unittest_index]
+    
+    extra_args = {
+        "unittest": [],
+        "propertytest": [],
+        "extra": []
+    }    
+
+    for i in range(len(args.extra)):
+        if args.extra[i] == "unittest":
+            current = "unittest"
+        elif args.extra[i] == "propertytest":
+            current = "propertytest"
+        elif args.extra[i] == "--":
+            current = "extra"
+        else:
+            extra_args[current].append(args.extra[i])
+    setattr(args, "unittest_args", [])
+    setattr(args, "propertytest_args", [])
+    args.unittest_args = extra_args["unittest"]
+    args.propertytest_args = extra_args["propertytest"]
+    args.extra = extra_args["extra"]
+
 
 def run(args=None):
     if args is None:
         args = parse_args(sys.argv[1:])
     _sanitize_args(args)
     driver_info_logger(args)
-    unittest_info_logger(args)
-    if args.extra:
-        print("[Warning] Captured extra args:", args.extra, flush=True)
-        print("The extra args will be passed into fastbot launcher.", flush=True)
+    extra_args_info_logger(args)
 
-    from kea2 import KeaTestRunner, Options
+    from kea2 import KeaTestRunner, Options, HybridTestRunner
     from kea2.u2Driver import U2Driver
     options = Options(
         agent=args.agent,
@@ -219,20 +251,26 @@ def run(args=None):
         running_mins=args.running_minutes,
         maxStep=args.max_step,
         throttle=args.throttle_ms,
+        output_dir=args.output_dir,
         log_stamp=args.log_stamp,
         profile_period=args.profile_period,
         take_screenshots=args.take_screenshots,
+        pre_failure_screenshots=args.pre_failure_screenshots,
         device_output_root=args.device_output_root,
         act_whitelist_file=args.act_whitelist_file,
         act_blacklist_file=args.act_blacklist_file,
+        propertytest_args=args.propertytest_args,
+        unittest_args=args.unittest_args,
         extra_args=args.extra,
     )
-
-    KeaTestRunner.setOptions(options)
-    sys.argv = ["python3 -m unittest"] + args.unittest_args
-
-    unittest.main(module=None, testRunner=KeaTestRunner)
-
-
-if __name__ == "__main__":
-    run()
+    
+    is_hybrid_test = True if options.unittest_args else False
+    if is_hybrid_test:
+        HybridTestRunner.setOptions(options)
+        testRunner = HybridTestRunner
+        argv = ["python3 -m unittest"] + options.unittest_args
+    if not is_hybrid_test or options.agent == "u2":
+        KeaTestRunner.setOptions(options)
+        testRunner = KeaTestRunner
+        argv = ["python3 -m unittest"] + options.propertytest_args
+    unittest.main(module=None, argv=argv, testRunner=testRunner)

kea2/mixin.py

@@ -0,0 +1,22 @@
+from unittest import TextTestResult, TestCase
+
+
+class BetterConsoleLogExtensionMixin:
+    def __init__(self, stream, descriptions, verbosity):
+        super().__init__(stream, descriptions, verbosity)
+        self.showAll = True
+    
+    def getDescription(self: "TextTestResult", test: "TestCase"):
+        doc_first_line = test.shortDescription()
+        if self.descriptions and doc_first_line:
+            doc_first_line = "# " + doc_first_line
+            return '\n'.join((str(test), doc_first_line))
+        else:
+            return str(test)
+    
+    def startTest(self: "TextTestResult", test):
+        if self.showAll:
+            self.stream.write("[INFO] Start executing property: ")
+            self.stream.writeln(self.getDescription(test))
+            self.stream.flush()
+            self._newline = True

kea2/report_merger.py

@@ -35,7 +35,6 @@ class TestReportMerger:
         try:
             # Convert paths and validate
             self.result_dirs = [Path(p).resolve() for p in result_paths]
-            self._validate_result_dirs()
             
             # Setup output directory
             timestamp = datetime.now().strftime("%Y%m%d%H_%M%S")
@@ -45,13 +44,13 @@ class TestReportMerger:
                 output_dir = Path(output_dir).resolve() / f"merged_report_{timestamp}"
             
             output_dir.mkdir(parents=True, exist_ok=True)
-            
+
             logger.debug(f"Merging {len(self.result_dirs)} test result directories...")
-            
+
             # Merge different types of data
-            merged_property_stats, property_source_mapping = self._merge_property_results()
+            merged_property_stats, property_source_mapping = self._merge_property_results(output_dir)
             merged_coverage_data = self._merge_coverage_data()
-            merged_crash_anr_data = self._merge_crash_dump_data()
+            merged_crash_anr_data = self._merge_crash_dump_data(output_dir)
 
             # Calculate final statistics
             final_data = self._calculate_final_statistics(merged_property_stats, merged_coverage_data, merged_crash_anr_data, property_source_mapping)
@@ -73,27 +72,18 @@ class TestReportMerger:
             logger.error(f"Error merging test reports: {e}")
             raise
     
-    def _validate_result_dirs(self):
-        """Validate that all result directories exist and contain required files"""
-        for result_dir in self.result_dirs:
-            if not result_dir.exists():
-                raise FileNotFoundError(f"Result directory does not exist: {result_dir}")
-            
-            # Check for required files pattern
-            result_files = list(result_dir.glob("result_*.json"))
-            if not result_files:
-                raise FileNotFoundError(f"No result_*.json file found in: {result_dir}")
-            
-            logger.debug(f"Validated result directory: {result_dir}")
-    
-    def _merge_property_results(self) -> Tuple[Dict[str, Dict], Dict[str, List[str]]]:
+    def _merge_property_results(self, output_dir: Path = None) -> Tuple[Dict[str, Dict], Dict[str, List[Dict]]]:
         """
         Merge property test results from all directories
 
+        Args:
+            output_dir: The output directory where the merged report will be saved (for calculating relative paths)
+
         Returns:
             Tuple of (merged_property_results, property_source_mapping)
             - merged_property_results: Merged property execution results
-            - property_source_mapping: Maps property names to list of source directories with fail/error
+            - property_source_mapping: Maps property names to list of source directory info with fail/error
+              Each entry contains: {'dir_name': str, 'report_path': str}
         """
         merged_results = defaultdict(lambda: {
             "precond_satisfied": 0,
@@ -107,32 +97,47 @@ class TestReportMerger:
 
         for result_dir in self.result_dirs:
             result_files = list(result_dir.glob("result_*.json"))
+            html_files = list(result_dir.glob("*.html"))
             if not result_files:
                 logger.warning(f"No result file found in {result_dir}")
                 continue
+            if not html_files:
+                logger.warning(f"No html file found in {result_dir}")
+                continue
 
             result_file = result_files[0]  # Take the first (should be only one)
+            html_file = html_files[0]
             dir_name = result_dir.name  # Get the directory name (e.g., res_2025072011_5048015228)
 
+            # Find the HTML report file in the result directory
+            html_report_path = None
+            
+            # Calculate relative path from output_dir to the HTML file
             try:
-                with open(result_file, 'r', encoding='utf-8') as f:
-                    test_results = json.load(f)
+                html_report_path = os.path.relpath(html_file.resolve(), output_dir.resolve())
+            except ValueError:
+                # If on different drives (Windows), use absolute path as fallback
+                html_report_path = str(html_file.resolve())
 
-                # Merge results for each property
-                for prop_name, prop_result in test_results.items():
-                    for key in ["precond_satisfied", "executed", "fail", "error"]:
-                        merged_results[prop_name][key] += prop_result.get(key, 0)
+            with open(result_file, 'r', encoding='utf-8') as f:
+                test_results = json.load(f)
 
-                    # Track source directories for properties with fail/error
-                    if prop_result.get('fail', 0) > 0 or prop_result.get('error', 0) > 0:
-                        if dir_name not in property_source_mapping[prop_name]:
-                            property_source_mapping[prop_name].append(dir_name)
+            # Merge results for each property
+            for prop_name, prop_result in test_results.items():
+                for key in ["precond_satisfied", "executed", "fail", "error"]:
+                    merged_results[prop_name][key] += prop_result.get(key, 0)
 
-                logger.debug(f"Merged results from: {result_file}")
+                # Track source directories for properties with fail/error
+                if prop_result.get('fail', 0) > 0 or prop_result.get('error', 0) > 0:
+                    # Check if this directory is already in the mapping
+                    existing_dirs = [item['dir_name'] for item in property_source_mapping[prop_name]]
+                    if dir_name not in existing_dirs:
+                        property_source_mapping[prop_name].append({
+                            'dir_name': dir_name,
+                            'report_path': html_report_path
+                        })
 
-            except Exception as e:
-                logger.error(f"Error reading result file {result_file}: {e}")
-                continue
+            logger.debug(f"Merged results from: {result_file}")
 
         return dict(merged_results), dict(property_source_mapping)
     
@@ -194,11 +199,12 @@ class TestReportMerger:
             "total_activities": list(all_activities),
             "tested_activities": list(tested_activities),
             "total_activities_count": len(all_activities),
+            "tested_activities_count": len(tested_activities),
             "activity_count_history": dict(activity_counts),
             "total_steps": total_steps
         }
 
-    def _merge_crash_dump_data(self) -> Dict:
+    def _merge_crash_dump_data(self, output_dir: Path = None) -> Dict:
         """
         Merge crash and ANR data from all directories
 
@@ -209,10 +215,22 @@ class TestReportMerger:
         all_anr_events = []
 
         for result_dir in self.result_dirs:
+            dir_name = result_dir.name
+
+            # Locate corresponding HTML report for hyperlinking
+            html_report_path = None
+            html_files = list(result_dir.glob("*.html"))
+            if not html_files:
+                continue
+            html_file = html_files[0]
+            try:
+                html_report_path = os.path.relpath(html_file.resolve(), output_dir.resolve())
+            except ValueError:
+                html_report_path = str(html_file.resolve())
+
             # Find crash dump log file
             output_dirs = list(result_dir.glob("output_*"))
             if not output_dirs:
-                logger.warning(f"No output directory found in {result_dir}")
                 continue
 
             crash_dump_file = output_dirs[0] / "crash-dump.log"
@@ -223,6 +241,15 @@ class TestReportMerger:
             try:
                 # Parse crash and ANR events from this file
                 crash_events, anr_events = self._parse_crash_dump_file(crash_dump_file)
+
+                for crash in crash_events:
+                    crash["source_directory"] = dir_name
+                    crash["report_path"] = html_report_path
+
+                for anr in anr_events:
+                    anr["source_directory"] = dir_name
+                    anr["report_path"] = html_report_path
+
                 all_crash_events.extend(crash_events)
                 all_anr_events.extend(anr_events)
 
@@ -507,7 +534,11 @@ class TestReportMerger:
 
             # Use first 3 lines of stack trace for deduplication
             stack_lines = stack_trace.split('\n')[:3]
-            crash_key = (exception_type, '\n'.join(stack_lines))
+            crash_key = (
+                exception_type,
+                '\n'.join(stack_lines),
+                crash.get("source_directory", "")
+            )
 
             if crash_key not in seen_crashes:
                 seen_crashes.add(crash_key)
@@ -532,7 +563,7 @@ class TestReportMerger:
             # Create a hash key based on reason and process
             reason = anr.get("reason", "")
             process = anr.get("process", "")
-            anr_key = (reason, process)
+            anr_key = (reason, process, anr.get("source_directory", ""))
 
             if anr_key not in seen_anrs:
                 seen_anrs.add(anr_key)
@@ -580,6 +611,42 @@ class TestReportMerger:
         # Calculate total bugs found (only property bugs, not including crashes/ANRs)
         total_bugs_found = property_bugs_found
 
+        # Prepare enhanced property statistics with derived metrics
+        processed_property_stats = {}
+        property_stats_summary = {
+            "total_properties": 0,
+            "total_precond_satisfied": 0,
+            "total_executed": 0,
+            "total_passes": 0,
+            "total_fails": 0,
+            "total_errors": 0,
+            "total_not_executed": 0,
+        }
+
+        for prop_name, stats in property_stats.items():
+            precond_satisfied = stats.get("precond_satisfied", 0)
+            total_executions = stats.get("executed", 0)
+            fail_count = stats.get("fail", 0)
+            error_count = stats.get("error", 0)
+
+            pass_count = max(total_executions - fail_count - error_count, 0)
+            not_executed_count = max(precond_satisfied - total_executions, 0)
+
+            processed_property_stats[prop_name] = {
+                **stats,
+                "executed_total": total_executions,
+                "pass_count": pass_count,
+                "not_executed": not_executed_count,
+            }
+
+            property_stats_summary["total_properties"] += 1
+            property_stats_summary["total_precond_satisfied"] += precond_satisfied
+            property_stats_summary["total_executed"] += total_executions
+            property_stats_summary["total_passes"] += pass_count
+            property_stats_summary["total_fails"] += fail_count
+            property_stats_summary["total_errors"] += error_count
+            property_stats_summary["total_not_executed"] += not_executed_count
+
         # Prepare final data
         final_data = {
             'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
@@ -587,7 +654,8 @@ class TestReportMerger:
             'property_bugs_found': property_bugs_found,
             'all_properties_count': all_properties_count,
             'executed_properties_count': executed_properties_count,
-            'property_stats': property_stats,
+            'property_stats': processed_property_stats,
+            'property_stats_summary': property_stats_summary,
             'property_source_mapping': property_source_mapping or {},
             'crash_events': crash_events,
             'anr_events': anr_events,
@@ -659,6 +727,3 @@ class TestReportMerger:
         except Exception as e:
             logger.error(f"Error generating HTML report: {e}")
             raise
-
-
-

kea2/resultSyncer.py

@@ -13,7 +13,7 @@ class ResultSyncer:
 
     def __init__(self, device_output_dir, options: "Options"):
         self.device_output_dir = device_output_dir
-        self.output_dir = Path(options.output_dir) / Path(device_output_dir).name
+        self.output_dir = options.output_dir / Path(device_output_dir).name
         self.running = False
         self.thread = None
         self.sync_event = threading.Event()