Kea2-python 0.3.6__py3-none-any.whl → 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of Kea2-python might be problematic. Click here for more details.
- kea2/__init__.py +1 -1
- kea2/assets/config_version.json +16 -0
- kea2/assets/fastbot_configs/teardown.py +18 -0
- kea2/assets/monkeyq.jar +0 -0
- kea2/assets/quicktest.py +21 -2
- kea2/bug_report_generator.py +116 -34
- kea2/cli.py +19 -9
- kea2/fastbotManager.py +20 -4
- kea2/keaUtils.py +360 -111
- kea2/kea_launcher.py +61 -23
- kea2/mixin.py +22 -0
- kea2/report_merger.py +107 -42
- kea2/resultSyncer.py +1 -1
- kea2/templates/bug_report_template.html +187 -19
- kea2/templates/merged_bug_report_template.html +3293 -3213
- kea2/u2Driver.py +18 -8
- kea2/utils.py +60 -14
- kea2/version_manager.py +101 -0
- {kea2_python-0.3.6.dist-info → kea2_python-1.0.1.dist-info}/METADATA +63 -15
- {kea2_python-0.3.6.dist-info → kea2_python-1.0.1.dist-info}/RECORD +24 -20
- {kea2_python-0.3.6.dist-info → kea2_python-1.0.1.dist-info}/WHEEL +0 -0
- {kea2_python-0.3.6.dist-info → kea2_python-1.0.1.dist-info}/entry_points.txt +0 -0
- {kea2_python-0.3.6.dist-info → kea2_python-1.0.1.dist-info}/licenses/LICENSE +0 -0
- {kea2_python-0.3.6.dist-info → kea2_python-1.0.1.dist-info}/top_level.txt +0 -0
kea2/keaUtils.py
CHANGED
|
@@ -1,29 +1,36 @@
|
|
|
1
1
|
from collections import deque
|
|
2
|
+
from copy import deepcopy
|
|
2
3
|
import json
|
|
3
4
|
import os
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
import traceback
|
|
6
|
-
import
|
|
7
|
-
from
|
|
8
|
-
from unittest import TextTestRunner, registerResult, TestSuite, TestCase, TextTestResult
|
|
7
|
+
from typing import Callable, Any, Deque, Dict, List, Literal, NewType, Tuple, Union
|
|
8
|
+
from contextvars import ContextVar
|
|
9
|
+
from unittest import TextTestRunner, registerResult, TestSuite, TestCase, TextTestResult, defaultTestLoader, SkipTest
|
|
10
|
+
from unittest import main as unittest_main
|
|
9
11
|
import random
|
|
10
12
|
import warnings
|
|
11
13
|
from dataclasses import dataclass, asdict
|
|
12
14
|
from kea2.absDriver import AbstractDriver
|
|
13
|
-
from functools import wraps
|
|
14
15
|
from kea2.bug_report_generator import BugReportGenerator
|
|
15
16
|
from kea2.resultSyncer import ResultSyncer
|
|
16
17
|
from kea2.logWatcher import LogWatcher
|
|
17
|
-
from kea2.utils import TimeStamp, catchException, getProjectRoot, getLogger, timer
|
|
18
|
-
from kea2.u2Driver import StaticU2UiObject, StaticXpathUiObject
|
|
18
|
+
from kea2.utils import TimeStamp, catchException, getProjectRoot, getLogger, loadFuncsFromFile, timer
|
|
19
|
+
from kea2.u2Driver import StaticU2UiObject, StaticXpathUiObject, U2Driver
|
|
19
20
|
from kea2.fastbotManager import FastbotManager
|
|
20
21
|
from kea2.adbUtils import ADBDevice
|
|
22
|
+
from kea2.mixin import BetterConsoleLogExtensionMixin
|
|
21
23
|
import uiautomator2 as u2
|
|
22
24
|
import types
|
|
23
25
|
|
|
26
|
+
|
|
27
|
+
hybrid_mode = ContextVar("hybrid_mode", default=False)
|
|
28
|
+
|
|
29
|
+
|
|
24
30
|
PRECONDITIONS_MARKER = "preconds"
|
|
25
|
-
|
|
31
|
+
PROB_MARKER = "prob"
|
|
26
32
|
MAX_TRIES_MARKER = "max_tries"
|
|
33
|
+
INTERRUPTABLE_MARKER = "interruptable"
|
|
27
34
|
|
|
28
35
|
logger = getLogger(__name__)
|
|
29
36
|
|
|
@@ -38,6 +45,7 @@ LOGFILE: str
|
|
|
38
45
|
RESFILE: str
|
|
39
46
|
PROP_EXEC_RESFILE: str
|
|
40
47
|
|
|
48
|
+
|
|
41
49
|
def precondition(precond: Callable[[Any], bool]) -> Callable:
|
|
42
50
|
"""the decorator @precondition
|
|
43
51
|
|
|
@@ -45,18 +53,13 @@ def precondition(precond: Callable[[Any], bool]) -> Callable:
|
|
|
45
53
|
A property could have multiple preconditions, each of which is specified by @precondition.
|
|
46
54
|
"""
|
|
47
55
|
def accept(f):
|
|
48
|
-
@wraps(f)
|
|
49
|
-
def precondition_wrapper(*args, **kwargs):
|
|
50
|
-
return f(*args, **kwargs)
|
|
51
|
-
|
|
52
56
|
preconds = getattr(f, PRECONDITIONS_MARKER, tuple())
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
return precondition_wrapper
|
|
57
|
+
setattr(f, PRECONDITIONS_MARKER, preconds + (precond,))
|
|
58
|
+
return f
|
|
57
59
|
|
|
58
60
|
return accept
|
|
59
61
|
|
|
62
|
+
|
|
60
63
|
def prob(p: float):
|
|
61
64
|
"""the decorator @prob
|
|
62
65
|
|
|
@@ -65,14 +68,10 @@ def prob(p: float):
|
|
|
65
68
|
p = float(p)
|
|
66
69
|
if not 0 < p <= 1.0:
|
|
67
70
|
raise ValueError("The propbability should between 0 and 1")
|
|
68
|
-
def accept(f):
|
|
69
|
-
@wraps(f)
|
|
70
|
-
def precondition_wrapper(*args, **kwargs):
|
|
71
|
-
return f(*args, **kwargs)
|
|
72
71
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
return
|
|
72
|
+
def accept(f):
|
|
73
|
+
setattr(f, PROB_MARKER, p)
|
|
74
|
+
return f
|
|
76
75
|
|
|
77
76
|
return accept
|
|
78
77
|
|
|
@@ -85,16 +84,25 @@ def max_tries(n: int):
|
|
|
85
84
|
n = int(n)
|
|
86
85
|
if not n > 0:
|
|
87
86
|
raise ValueError("The maxium tries should be a positive integer.")
|
|
87
|
+
|
|
88
88
|
def accept(f):
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
return f(*args, **kwargs)
|
|
89
|
+
setattr(f, MAX_TRIES_MARKER, n)
|
|
90
|
+
return f
|
|
92
91
|
|
|
93
|
-
|
|
92
|
+
return accept
|
|
94
93
|
|
|
95
|
-
return precondition_wrapper
|
|
96
94
|
|
|
97
|
-
|
|
95
|
+
def interruptable(strategy='default'):
|
|
96
|
+
"""the decorator @interruptable
|
|
97
|
+
|
|
98
|
+
@interruptable specify the propbability of **fuzzing** when calling every line of code in a property.
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
def decorator(func):
|
|
102
|
+
setattr(func, INTERRUPTABLE_MARKER, True)
|
|
103
|
+
setattr(func, 'strategy', strategy)
|
|
104
|
+
return func
|
|
105
|
+
return decorator
|
|
98
106
|
|
|
99
107
|
|
|
100
108
|
@dataclass
|
|
@@ -103,11 +111,11 @@ class Options:
|
|
|
103
111
|
Kea and Fastbot configurations
|
|
104
112
|
"""
|
|
105
113
|
# the driver_name in script (if self.d, then d.)
|
|
106
|
-
driverName: str
|
|
114
|
+
driverName: str = None
|
|
107
115
|
# the driver (only U2Driver available now)
|
|
108
|
-
Driver: AbstractDriver
|
|
116
|
+
Driver: AbstractDriver = None
|
|
109
117
|
# list of package names. Specify the apps under test
|
|
110
|
-
packageNames: List[str]
|
|
118
|
+
packageNames: List[str] = None
|
|
111
119
|
# target device
|
|
112
120
|
serial: str = None
|
|
113
121
|
# target device with transport_id
|
|
@@ -128,6 +136,8 @@ class Options:
|
|
|
128
136
|
profile_period: int = 25
|
|
129
137
|
# take screenshots for every step
|
|
130
138
|
take_screenshots: bool = False
|
|
139
|
+
# Screenshots before failure (Dump n screenshots before failure. 0 means take screenshots for every step)
|
|
140
|
+
pre_failure_screenshots: int = 0
|
|
131
141
|
# The root of output dir on device
|
|
132
142
|
device_output_root: str = "/sdcard"
|
|
133
143
|
# the debug mode
|
|
@@ -136,6 +146,10 @@ class Options:
|
|
|
136
146
|
act_whitelist_file: str = None
|
|
137
147
|
# Activity BlackList File
|
|
138
148
|
act_blacklist_file: str = None
|
|
149
|
+
# Feat4. propertytest args(eg. discover -s xxx -p xxx)
|
|
150
|
+
propertytest_args: str = None
|
|
151
|
+
# Feat4. unittest args(eg. -v -s xxx -p xxx)
|
|
152
|
+
unittest_args: List[str] = None
|
|
139
153
|
# Extra args
|
|
140
154
|
extra_args: List[str] = None
|
|
141
155
|
|
|
@@ -143,37 +157,49 @@ class Options:
|
|
|
143
157
|
if value is None:
|
|
144
158
|
return
|
|
145
159
|
super().__setattr__(name, value)
|
|
146
|
-
|
|
160
|
+
|
|
147
161
|
def __post_init__(self):
|
|
148
162
|
import logging
|
|
149
163
|
logging.basicConfig(level=logging.DEBUG if self.debug else logging.INFO)
|
|
150
|
-
|
|
164
|
+
|
|
151
165
|
if self.Driver:
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
target_device["serial"] = self.serial
|
|
155
|
-
if self.transport_id:
|
|
156
|
-
target_device["transport_id"] = self.transport_id
|
|
157
|
-
self.Driver.setDevice(target_device)
|
|
158
|
-
ADBDevice.setDevice(self.serial, self.transport_id)
|
|
159
|
-
|
|
160
|
-
global LOGFILE, RESFILE, PROP_EXEC_RESFILE, STAMP
|
|
166
|
+
self._set_driver()
|
|
167
|
+
|
|
161
168
|
if self.log_stamp:
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
raise ValueError(
|
|
166
|
-
f"char: `{char}` is illegal in --log-stamp. current stamp: {self.log_stamp}"
|
|
167
|
-
)
|
|
168
|
-
STAMP = self.log_stamp
|
|
169
|
-
|
|
170
|
-
self.log_stamp = STAMP
|
|
171
|
-
|
|
169
|
+
self._sanitize_custom_stamp()
|
|
170
|
+
|
|
171
|
+
global STAMP
|
|
172
172
|
self.output_dir = Path(self.output_dir).absolute() / f"res_{STAMP}"
|
|
173
|
+
self.set_stamp()
|
|
174
|
+
|
|
175
|
+
self._sanitize_args()
|
|
176
|
+
|
|
177
|
+
_check_package_installation(self.packageNames)
|
|
178
|
+
_save_bug_report_configs(self)
|
|
179
|
+
|
|
180
|
+
def set_stamp(self, stamp: str = None):
|
|
181
|
+
global STAMP, LOGFILE, RESFILE, PROP_EXEC_RESFILE
|
|
182
|
+
if stamp:
|
|
183
|
+
STAMP = stamp
|
|
184
|
+
|
|
173
185
|
LOGFILE = f"fastbot_{STAMP}.log"
|
|
174
186
|
RESFILE = f"result_{STAMP}.json"
|
|
175
187
|
PROP_EXEC_RESFILE = f"property_exec_info_{STAMP}.json"
|
|
176
188
|
|
|
189
|
+
def _sanitize_custom_stamp(self):
|
|
190
|
+
global STAMP
|
|
191
|
+
illegal_chars = ['/', '\\', ':', '*', '?', '"', '<', '>', '|', '\n', '\r', '\t', '\0']
|
|
192
|
+
for char in illegal_chars:
|
|
193
|
+
if char in self.log_stamp:
|
|
194
|
+
raise ValueError(
|
|
195
|
+
f"char: `{char}` is illegal in --log-stamp. current stamp: {self.log_stamp}"
|
|
196
|
+
)
|
|
197
|
+
STAMP = self.log_stamp
|
|
198
|
+
|
|
199
|
+
def _sanitize_args(self):
|
|
200
|
+
if not self.take_screenshots and self.pre_failure_screenshots > 0:
|
|
201
|
+
raise ValueError("--screenshots-before-error should be 0 when --take-screenshots is not set.")
|
|
202
|
+
|
|
177
203
|
self.profile_period = int(self.profile_period)
|
|
178
204
|
if self.profile_period < 1:
|
|
179
205
|
raise ValueError("--profile-period should be greater than 0")
|
|
@@ -182,7 +208,35 @@ class Options:
|
|
|
182
208
|
if self.throttle < 0:
|
|
183
209
|
raise ValueError("--throttle should be greater than or equal to 0")
|
|
184
210
|
|
|
185
|
-
|
|
211
|
+
if self.agent == 'u2' and self.driverName == None:
|
|
212
|
+
raise ValueError("--driver-name should be specified when customizing script in --agent u2")
|
|
213
|
+
|
|
214
|
+
def _set_driver(self):
|
|
215
|
+
target_device = dict()
|
|
216
|
+
if self.serial:
|
|
217
|
+
target_device["serial"] = self.serial
|
|
218
|
+
if self.transport_id:
|
|
219
|
+
target_device["transport_id"] = self.transport_id
|
|
220
|
+
self.Driver.setDevice(target_device)
|
|
221
|
+
ADBDevice.setDevice(self.serial, self.transport_id)
|
|
222
|
+
|
|
223
|
+
def getKeaTestOptions(self, hybrid_test_count: int) -> "Options":
|
|
224
|
+
""" Get the KeaTestOptions for hybrid test run when switching from unittest to kea2 test.
|
|
225
|
+
hybrid_test_count: the count of hybrid test runs
|
|
226
|
+
"""
|
|
227
|
+
if not self.unittest_args:
|
|
228
|
+
raise RuntimeError("unittest_args is None. Cannot get KeaTestOptions from it")
|
|
229
|
+
|
|
230
|
+
opts = deepcopy(self)
|
|
231
|
+
|
|
232
|
+
time_stamp = TimeStamp().getTimeStamp()
|
|
233
|
+
hybrid_test_stamp = f"{time_stamp}_hybrid_{hybrid_test_count}"
|
|
234
|
+
|
|
235
|
+
opts.output_dir = self.output_dir / f"res_{hybrid_test_stamp}"
|
|
236
|
+
|
|
237
|
+
opts.set_stamp(hybrid_test_stamp)
|
|
238
|
+
opts.unittest_args = []
|
|
239
|
+
return opts
|
|
186
240
|
|
|
187
241
|
|
|
188
242
|
def _check_package_installation(packageNames):
|
|
@@ -194,6 +248,20 @@ def _check_package_installation(packageNames):
|
|
|
194
248
|
raise ValueError("package not installed")
|
|
195
249
|
|
|
196
250
|
|
|
251
|
+
def _save_bug_report_configs(options: Options):
|
|
252
|
+
output_dir = options.output_dir
|
|
253
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
254
|
+
configs = {
|
|
255
|
+
"driverName": options.driverName,
|
|
256
|
+
"packageNames": options.packageNames,
|
|
257
|
+
"take_screenshots": options.take_screenshots,
|
|
258
|
+
"pre_failure_screenshots": options.pre_failure_screenshots,
|
|
259
|
+
"device_output_root": options.device_output_root,
|
|
260
|
+
}
|
|
261
|
+
with open(output_dir / "bug_report_config.json", "w", encoding="utf-8") as fp:
|
|
262
|
+
json.dump(configs, fp, indent=4)
|
|
263
|
+
|
|
264
|
+
|
|
197
265
|
@dataclass
|
|
198
266
|
class PropStatistic:
|
|
199
267
|
precond_satisfied: int = 0
|
|
@@ -222,12 +290,16 @@ def getFullPropName(testCase: TestCase):
|
|
|
222
290
|
])
|
|
223
291
|
|
|
224
292
|
|
|
225
|
-
class JsonResult(TextTestResult):
|
|
293
|
+
class JsonResult(BetterConsoleLogExtensionMixin, TextTestResult):
|
|
226
294
|
|
|
227
295
|
res: PBTTestResult
|
|
228
296
|
lastExecutedInfo: PropertyExecutionInfo
|
|
229
297
|
executionInfoStore: PropertyExecutionInfoStore = deque()
|
|
230
298
|
|
|
299
|
+
def __init__(self, stream, descriptions, verbosity):
|
|
300
|
+
super().__init__(stream, descriptions, verbosity)
|
|
301
|
+
self.showAll = True
|
|
302
|
+
|
|
231
303
|
@classmethod
|
|
232
304
|
def setProperties(cls, allProperties: Dict):
|
|
233
305
|
cls.res = dict()
|
|
@@ -281,6 +353,17 @@ class JsonResult(TextTestResult):
|
|
|
281
353
|
def getExcuted(self, test: TestCase):
|
|
282
354
|
return self.res[getFullPropName(test)].executed
|
|
283
355
|
|
|
356
|
+
def printError(self, test):
|
|
357
|
+
if self.lastExecutedInfo.state in ["fail", "error"]:
|
|
358
|
+
flavour = self.lastExecutedInfo.state.upper()
|
|
359
|
+
self.stream.writeln("")
|
|
360
|
+
self.stream.writeln(self.separator1)
|
|
361
|
+
self.stream.writeln("%s: %s" % (flavour, self.getDescription(test)))
|
|
362
|
+
self.stream.writeln(self.separator2)
|
|
363
|
+
self.stream.writeln("%s" % self.lastExecutedInfo.tb)
|
|
364
|
+
self.stream.writeln(self.separator1)
|
|
365
|
+
self.stream.flush()
|
|
366
|
+
|
|
284
367
|
def logSummary(self):
|
|
285
368
|
fails = sum(_.fail for _ in self.res.values())
|
|
286
369
|
errors = sum(_.error for _ in self.res.values())
|
|
@@ -288,12 +371,8 @@ class JsonResult(TextTestResult):
|
|
|
288
371
|
logger.info(f"[Property Exectution Summary] Errors:{errors}, Fails:{fails}")
|
|
289
372
|
|
|
290
373
|
|
|
291
|
-
class
|
|
292
|
-
|
|
293
|
-
resultclass: JsonResult
|
|
294
|
-
allProperties: PropertyStore
|
|
374
|
+
class KeaOptionSetter:
|
|
295
375
|
options: Options = None
|
|
296
|
-
_block_funcs: Dict[Literal["widgets", "trees"], List[Callable]] = None
|
|
297
376
|
|
|
298
377
|
@classmethod
|
|
299
378
|
def setOptions(cls, options: Options):
|
|
@@ -303,9 +382,16 @@ class KeaTestRunner(TextTestRunner):
|
|
|
303
382
|
logger.warning("[Warning] Can not use any Driver when runing native mode.")
|
|
304
383
|
options.Driver = None
|
|
305
384
|
cls.options = options
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
class KeaTestRunner(TextTestRunner, KeaOptionSetter):
|
|
388
|
+
|
|
389
|
+
resultclass: JsonResult
|
|
390
|
+
allProperties: PropertyStore
|
|
391
|
+
_block_funcs: Dict[Literal["widgets", "trees"], List[Callable]] = None
|
|
306
392
|
|
|
307
393
|
def _setOuputDir(self):
|
|
308
|
-
output_dir =
|
|
394
|
+
output_dir = self.options.output_dir
|
|
309
395
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
310
396
|
global LOGFILE, RESFILE, PROP_EXEC_RESFILE
|
|
311
397
|
LOGFILE = output_dir / Path(LOGFILE)
|
|
@@ -359,7 +445,7 @@ class KeaTestRunner(TextTestRunner):
|
|
|
359
445
|
# initialize the result.json file
|
|
360
446
|
result.flushResult()
|
|
361
447
|
# setUp for the u2 driver
|
|
362
|
-
self.scriptDriver =
|
|
448
|
+
self.scriptDriver = U2Driver.getScriptDriver(mode="proxy")
|
|
363
449
|
fb.check_alive()
|
|
364
450
|
|
|
365
451
|
fb.init(options=self.options, stamp=STAMP)
|
|
@@ -371,15 +457,18 @@ class KeaTestRunner(TextTestRunner):
|
|
|
371
457
|
self.stepsCount = 0
|
|
372
458
|
while self.stepsCount < self.options.maxStep:
|
|
373
459
|
|
|
374
|
-
self.stepsCount += 1
|
|
375
|
-
logger.info("Sending monkeyEvent {}".format(
|
|
376
|
-
f"({self.stepsCount} / {self.options.maxStep})" if self.options.maxStep != float("inf")
|
|
377
|
-
else f"({self.stepsCount})"
|
|
378
|
-
)
|
|
379
|
-
)
|
|
380
|
-
|
|
381
460
|
try:
|
|
382
|
-
|
|
461
|
+
if fb.executed_prop:
|
|
462
|
+
fb.executed_prop = False
|
|
463
|
+
xml_raw = fb.dumpHierarchy()
|
|
464
|
+
else:
|
|
465
|
+
self.stepsCount += 1
|
|
466
|
+
logger.info("Sending monkeyEvent {}".format(
|
|
467
|
+
f"({self.stepsCount} / {self.options.maxStep})" if self.options.maxStep != float("inf")
|
|
468
|
+
else f"({self.stepsCount})"
|
|
469
|
+
)
|
|
470
|
+
)
|
|
471
|
+
xml_raw = fb.stepMonkey(self._monkeyStepInfo)
|
|
383
472
|
propsSatisfiedPrecond = self.getValidProperties(xml_raw, result)
|
|
384
473
|
except u2.HTTPError:
|
|
385
474
|
logger.info("Connection refused by remote.")
|
|
@@ -402,7 +491,7 @@ class KeaTestRunner(TextTestRunner):
|
|
|
402
491
|
# filter the properties according to the given p
|
|
403
492
|
for propName, test in propsSatisfiedPrecond.items():
|
|
404
493
|
result.addPrecondSatisfied(test)
|
|
405
|
-
if getattr(test,
|
|
494
|
+
if getattr(test, PROB_MARKER, 1) >= p:
|
|
406
495
|
propsNameFilteredByP.append(propName)
|
|
407
496
|
|
|
408
497
|
if len(propsNameFilteredByP) == 0:
|
|
@@ -412,19 +501,20 @@ class KeaTestRunner(TextTestRunner):
|
|
|
412
501
|
execPropName = random.choice(propsNameFilteredByP)
|
|
413
502
|
test = propsSatisfiedPrecond[execPropName]
|
|
414
503
|
# Dependency Injection. driver when doing scripts
|
|
415
|
-
self.scriptDriver =
|
|
504
|
+
self.scriptDriver = U2Driver.getScriptDriver(mode="proxy")
|
|
505
|
+
|
|
416
506
|
setattr(test, self.options.driverName, self.scriptDriver)
|
|
417
|
-
print("execute property %s." % execPropName, flush=True)
|
|
418
507
|
|
|
419
508
|
result.addExcuted(test, self.stepsCount)
|
|
420
509
|
fb.logScript(result.lastExecutedInfo)
|
|
421
510
|
try:
|
|
422
511
|
test(result)
|
|
423
512
|
finally:
|
|
424
|
-
result.
|
|
513
|
+
result.printError(test)
|
|
425
514
|
|
|
426
515
|
result.updateExectedInfo()
|
|
427
516
|
fb.logScript(result.lastExecutedInfo)
|
|
517
|
+
fb.executed_prop = True
|
|
428
518
|
result.flushResult()
|
|
429
519
|
|
|
430
520
|
if not end_by_remote:
|
|
@@ -435,41 +525,6 @@ class KeaTestRunner(TextTestRunner):
|
|
|
435
525
|
fb.join()
|
|
436
526
|
print(f"Finish sending monkey events.", flush=True)
|
|
437
527
|
log_watcher.close()
|
|
438
|
-
|
|
439
|
-
# Source code from unittest Runner
|
|
440
|
-
# process the result
|
|
441
|
-
expectedFails = unexpectedSuccesses = skipped = 0
|
|
442
|
-
try:
|
|
443
|
-
results = map(
|
|
444
|
-
len,
|
|
445
|
-
(result.expectedFailures, result.unexpectedSuccesses, result.skipped),
|
|
446
|
-
)
|
|
447
|
-
except AttributeError:
|
|
448
|
-
pass
|
|
449
|
-
else:
|
|
450
|
-
expectedFails, unexpectedSuccesses, skipped = results
|
|
451
|
-
|
|
452
|
-
infos = []
|
|
453
|
-
if not result.wasSuccessful():
|
|
454
|
-
self.stream.write("FAILED")
|
|
455
|
-
failed, errored = len(result.failures), len(result.errors)
|
|
456
|
-
if failed:
|
|
457
|
-
infos.append("failures=%d" % failed)
|
|
458
|
-
if errored:
|
|
459
|
-
infos.append("errors=%d" % errored)
|
|
460
|
-
else:
|
|
461
|
-
self.stream.write("OK")
|
|
462
|
-
if skipped:
|
|
463
|
-
infos.append("skipped=%d" % skipped)
|
|
464
|
-
if expectedFails:
|
|
465
|
-
infos.append("expected failures=%d" % expectedFails)
|
|
466
|
-
if unexpectedSuccesses:
|
|
467
|
-
infos.append("unexpected successes=%d" % unexpectedSuccesses)
|
|
468
|
-
if infos:
|
|
469
|
-
self.stream.writeln(" (%s)" % (", ".join(infos),))
|
|
470
|
-
else:
|
|
471
|
-
self.stream.write("\n")
|
|
472
|
-
self.stream.flush()
|
|
473
528
|
|
|
474
529
|
result.logSummary()
|
|
475
530
|
return result
|
|
@@ -493,12 +548,14 @@ class KeaTestRunner(TextTestRunner):
|
|
|
493
548
|
|
|
494
549
|
def getValidProperties(self, xml_raw: str, result: JsonResult) -> PropertyStore:
|
|
495
550
|
|
|
496
|
-
staticCheckerDriver =
|
|
551
|
+
staticCheckerDriver = U2Driver.getStaticChecker(hierarchy=xml_raw)
|
|
497
552
|
|
|
498
553
|
validProps: PropertyStore = dict()
|
|
499
554
|
for propName, test in self.allProperties.items():
|
|
500
555
|
valid = True
|
|
501
556
|
prop = getattr(test, propName)
|
|
557
|
+
p = getattr(prop, PROB_MARKER, 1)
|
|
558
|
+
setattr(test, PROB_MARKER, p)
|
|
502
559
|
# check if all preconds passed
|
|
503
560
|
for precond in prop.preconds:
|
|
504
561
|
# Dependency injection. Static driver checker for precond
|
|
@@ -553,7 +610,12 @@ class KeaTestRunner(TextTestRunner):
|
|
|
553
610
|
yield test
|
|
554
611
|
|
|
555
612
|
# Traverse the TestCase to get all properties
|
|
613
|
+
_result = TextTestResult(self.stream, self.descriptions, self.verbosity)
|
|
556
614
|
for t in iter_tests(test):
|
|
615
|
+
# Find all the _FailedTest (Caused by ImportError) and directly run it to report errors
|
|
616
|
+
if type(t).__name__ == "_FailedTest":
|
|
617
|
+
t(_result)
|
|
618
|
+
continue
|
|
557
619
|
testMethodName = t._testMethodName
|
|
558
620
|
# get the test method name and check if it's a property
|
|
559
621
|
testMethod = getattr(t, testMethodName)
|
|
@@ -564,6 +626,8 @@ class KeaTestRunner(TextTestRunner):
|
|
|
564
626
|
# save it into allProperties for PBT
|
|
565
627
|
self.allProperties[testMethodName] = t
|
|
566
628
|
print(f"[INFO] Load property: {getFullPropName(t)}", flush=True)
|
|
629
|
+
# Print errors caused by ImportError
|
|
630
|
+
_result.printErrors()
|
|
567
631
|
|
|
568
632
|
@property
|
|
569
633
|
def _blockWidgetFuncs(self):
|
|
@@ -644,7 +708,7 @@ class KeaTestRunner(TextTestRunner):
|
|
|
644
708
|
|
|
645
709
|
if preconds_pass(preconds):
|
|
646
710
|
try:
|
|
647
|
-
_widgets = func(
|
|
711
|
+
_widgets = func(U2Driver.getStaticChecker())
|
|
648
712
|
_widgets = _widgets if isinstance(_widgets, list) else [_widgets]
|
|
649
713
|
for w in _widgets:
|
|
650
714
|
if isinstance(w, (StaticU2UiObject, StaticXpathUiObject)):
|
|
@@ -690,4 +754,189 @@ class KeaTestRunner(TextTestRunner):
|
|
|
690
754
|
if self.options.Driver:
|
|
691
755
|
self.options.Driver.tearDown()
|
|
692
756
|
|
|
693
|
-
self.
|
|
757
|
+
if self.options.agent == "u2":
|
|
758
|
+
self._generate_bug_report()
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
class KeaTextTestResult(BetterConsoleLogExtensionMixin, TextTestResult):
|
|
762
|
+
|
|
763
|
+
@property
|
|
764
|
+
def wasFail(self):
|
|
765
|
+
return self._wasFail
|
|
766
|
+
|
|
767
|
+
def addError(self, test, err):
|
|
768
|
+
self._wasFail = True
|
|
769
|
+
return super().addError(test, err)
|
|
770
|
+
|
|
771
|
+
def addFailure(self, test, err):
|
|
772
|
+
self._wasFail = True
|
|
773
|
+
return super().addFailure(test, err)
|
|
774
|
+
|
|
775
|
+
def addSuccess(self, test):
|
|
776
|
+
self._wasFail = False
|
|
777
|
+
return super().addSuccess(test)
|
|
778
|
+
|
|
779
|
+
def addSkip(self, test, reason):
|
|
780
|
+
self._wasFail = False
|
|
781
|
+
return super().addSkip(test, reason)
|
|
782
|
+
|
|
783
|
+
def addExpectedFailure(self, test, err):
|
|
784
|
+
self._wasFail = False
|
|
785
|
+
return super().addExpectedFailure(test, err)
|
|
786
|
+
|
|
787
|
+
def addUnexpectedSuccess(self, test):
|
|
788
|
+
self._wasFail = False
|
|
789
|
+
return super().addUnexpectedSuccess(test)
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
class HybridTestRunner(TextTestRunner, KeaOptionSetter):
|
|
793
|
+
|
|
794
|
+
allTestCases: Dict[str, Tuple[TestCase, bool]]
|
|
795
|
+
_common_teardown_func = None
|
|
796
|
+
resultclass = KeaTextTestResult
|
|
797
|
+
|
|
798
|
+
def __init__(self, stream = None, descriptions = True, verbosity = 1, failfast = False, buffer = False, resultclass = None, warnings = None, *, tb_locals = False):
|
|
799
|
+
super().__init__(stream, descriptions, verbosity, failfast, buffer, resultclass, warnings, tb_locals=tb_locals)
|
|
800
|
+
hybrid_mode.set(True)
|
|
801
|
+
self.hybrid_report_dirs = []
|
|
802
|
+
|
|
803
|
+
def run(self, test):
|
|
804
|
+
|
|
805
|
+
self.allTestCases = dict()
|
|
806
|
+
self.collectAllTestCases(test)
|
|
807
|
+
if len(self.allTestCases) == 0:
|
|
808
|
+
logger.warning("[Warning] No test case has been found.")
|
|
809
|
+
|
|
810
|
+
result: KeaTextTestResult = self._makeResult()
|
|
811
|
+
registerResult(result)
|
|
812
|
+
result.failfast = self.failfast
|
|
813
|
+
result.buffer = self.buffer
|
|
814
|
+
result.tb_locals = self.tb_locals
|
|
815
|
+
with warnings.catch_warnings():
|
|
816
|
+
if self.warnings:
|
|
817
|
+
# if self.warnings is set, use it to filter all the warnings
|
|
818
|
+
warnings.simplefilter(self.warnings)
|
|
819
|
+
# if the filter is 'default' or 'always', special-case the
|
|
820
|
+
# warnings from the deprecated unittest methods to show them
|
|
821
|
+
# no more than once per module, because they can be fairly
|
|
822
|
+
# noisy. The -Wd and -Wa flags can be used to bypass this
|
|
823
|
+
# only when self.warnings is None.
|
|
824
|
+
if self.warnings in ["default", "always"]:
|
|
825
|
+
warnings.filterwarnings(
|
|
826
|
+
"module",
|
|
827
|
+
category=DeprecationWarning,
|
|
828
|
+
message=r"Please use assert\w+ instead.",
|
|
829
|
+
)
|
|
830
|
+
|
|
831
|
+
hybrid_test_count = 0
|
|
832
|
+
for testCaseName, test in self.allTestCases.items():
|
|
833
|
+
test, isInterruptable = test, getattr(test, "isInterruptable", False)
|
|
834
|
+
|
|
835
|
+
# Dependency Injection. driver when doing scripts
|
|
836
|
+
self.scriptDriver = U2Driver.getScriptDriver(mode="direct")
|
|
837
|
+
setattr(test, self.options.driverName, self.scriptDriver)
|
|
838
|
+
logger.info("Executing unittest testCase %s." % testCaseName)
|
|
839
|
+
|
|
840
|
+
try:
|
|
841
|
+
test._common_setUp()
|
|
842
|
+
ret: KeaTextTestResult = test(result)
|
|
843
|
+
if ret.wasFail:
|
|
844
|
+
logger.error(f"Fail when running test.")
|
|
845
|
+
if isInterruptable and not ret.wasFail:
|
|
846
|
+
logger.info(f"Launch fastbot after interruptable script.")
|
|
847
|
+
hybrid_test_count += 1
|
|
848
|
+
hybrid_test_options = self.options.getKeaTestOptions(hybrid_test_count)
|
|
849
|
+
|
|
850
|
+
# Track the sub-report directory for later merging
|
|
851
|
+
self.hybrid_report_dirs.append(hybrid_test_options.output_dir)
|
|
852
|
+
|
|
853
|
+
argv = ["python3 -m unittest"] + hybrid_test_options.propertytest_args
|
|
854
|
+
KeaTestRunner.setOptions(hybrid_test_options)
|
|
855
|
+
unittest_main(module=None, argv=argv, testRunner=KeaTestRunner, exit=False)
|
|
856
|
+
|
|
857
|
+
finally:
|
|
858
|
+
test._common_tearDown()
|
|
859
|
+
result.printErrors()
|
|
860
|
+
|
|
861
|
+
# Auto-merge all hybrid test reports after all tests complete
|
|
862
|
+
if len(self.hybrid_report_dirs) > 0:
|
|
863
|
+
self._merge_hybrid_reports()
|
|
864
|
+
|
|
865
|
+
return result
|
|
866
|
+
|
|
867
|
+
def _merge_hybrid_reports(self):
|
|
868
|
+
"""
|
|
869
|
+
Merge all hybrid test reports into a single merged report
|
|
870
|
+
"""
|
|
871
|
+
try:
|
|
872
|
+
from kea2.report_merger import TestReportMerger
|
|
873
|
+
|
|
874
|
+
if len(self.hybrid_report_dirs) < 2:
|
|
875
|
+
logger.info("Only one hybrid test report generated, skipping merge.")
|
|
876
|
+
return
|
|
877
|
+
|
|
878
|
+
main_output_dir = self.options.output_dir
|
|
879
|
+
|
|
880
|
+
merger = TestReportMerger()
|
|
881
|
+
merged_dir = merger.merge_reports(
|
|
882
|
+
result_paths=self.hybrid_report_dirs,
|
|
883
|
+
output_dir=main_output_dir
|
|
884
|
+
)
|
|
885
|
+
|
|
886
|
+
merge_summary = merger.get_merge_summary()
|
|
887
|
+
except Exception as e:
|
|
888
|
+
logger.error(f"Error merging hybrid test reports: {e}")
|
|
889
|
+
|
|
890
|
+
def collectAllTestCases(self, test: TestSuite):
|
|
891
|
+
"""collect all the properties to prepare for PBT
|
|
892
|
+
"""
|
|
893
|
+
|
|
894
|
+
def iter_tests(suite):
|
|
895
|
+
for test in suite:
|
|
896
|
+
if isinstance(test, TestSuite):
|
|
897
|
+
yield from iter_tests(test)
|
|
898
|
+
else:
|
|
899
|
+
yield test
|
|
900
|
+
|
|
901
|
+
funcs = loadFuncsFromFile(getProjectRoot() / "configs" / "teardown.py")
|
|
902
|
+
setUp = funcs.get("setUp", None)
|
|
903
|
+
tearDown = funcs.get("tearDown", None)
|
|
904
|
+
if setUp is None:
|
|
905
|
+
raise ValueError("setUp function not found in teardown.py.")
|
|
906
|
+
if tearDown is None:
|
|
907
|
+
raise ValueError("tearDown function not found in teardown.py.")
|
|
908
|
+
|
|
909
|
+
# Traverse the TestCase to get all properties
|
|
910
|
+
for t in iter_tests(test):
|
|
911
|
+
|
|
912
|
+
def dummy(self): ...
|
|
913
|
+
# remove the hook func in its TestCase
|
|
914
|
+
t.setUp = types.MethodType(dummy, t)
|
|
915
|
+
t.tearDown = types.MethodType(dummy, t)
|
|
916
|
+
t._common_setUp = types.MethodType(setUp, t)
|
|
917
|
+
t._common_tearDown = types.MethodType(tearDown, t)
|
|
918
|
+
|
|
919
|
+
# check if it's interruptable (reflection)
|
|
920
|
+
testMethodName = t._testMethodName
|
|
921
|
+
testMethod = getattr(t, testMethodName)
|
|
922
|
+
isInterruptable = hasattr(testMethod, INTERRUPTABLE_MARKER)
|
|
923
|
+
|
|
924
|
+
# save it into allTestCases, if interruptable, mark as true
|
|
925
|
+
setattr(t, "isInterruptable", isInterruptable)
|
|
926
|
+
self.allTestCases[testMethodName] = t
|
|
927
|
+
logger.info(f"Load TestCase: {getFullPropName(t)} , interruptable: {t.isInterruptable}")
|
|
928
|
+
|
|
929
|
+
def __del__(self):
|
|
930
|
+
"""tearDown method. Cleanup the env.
|
|
931
|
+
"""
|
|
932
|
+
if self.options.Driver:
|
|
933
|
+
self.options.Driver.tearDown()
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def kea2_breakpoint():
|
|
937
|
+
"""kea2 entrance. Call this function in TestCase.
|
|
938
|
+
Kea2 will automatically switch to Kea2 Test in kea2_breakpoint in HybridTest mode.
|
|
939
|
+
The normal launch in unittest will not be affected.
|
|
940
|
+
"""
|
|
941
|
+
if hybrid_mode.get():
|
|
942
|
+
raise SkipTest("Skip the test after the breakpoint and run kea2 in hybrid mode.")
|