codecraftai-testplus 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ccai/testplus/__init__.py +57 -0
- ccai/testplus/cli/__init__.py +13 -0
- ccai/testplus/cli/cmdtree/__init__.py +13 -0
- ccai/testplus/cli/cmdtree/testing/__init__.py +33 -0
- ccai/testplus/cli/cmdtree/testing/constants.py +62 -0
- ccai/testplus/cli/cmdtree/testing/jobs.py +195 -0
- ccai/testplus/cli/cmdtree/testing/query.py +165 -0
- ccai/testplus/cli/cmdtree/testing/run.py +371 -0
- ccai/testplus/cli/testplus_command.py +79 -0
- ccai/testplus/distributedtestjob.py +90 -0
- ccai/testplus/distributedtestsequencer.py +111 -0
- ccai/testplus/entrypoints.py +140 -0
- ccai/testplus/initialize.py +103 -0
- ccai/testplus/py.typed +0 -0
- ccai/testplus/testplusdefaulttypefactory.py +47 -0
- codecraftai_testplus-1.2.0.dist-info/METADATA +100 -0
- codecraftai_testplus-1.2.0.dist-info/RECORD +20 -0
- codecraftai_testplus-1.2.0.dist-info/WHEEL +4 -0
- codecraftai_testplus-1.2.0.dist-info/entry_points.txt +3 -0
- codecraftai_testplus-1.2.0.dist-info/licenses/LICENSE.md +204 -0
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
"""
|
|
2
|
+
.. module:: run
|
|
3
|
+
:platform: Darwin, Linux, Unix, Windows
|
|
4
|
+
:synopsis: CLI command used to execute collections of tests via the
|
|
5
|
+
TestPlus framework.
|
|
6
|
+
|
|
7
|
+
.. moduleauthor:: Myron Walker <myron.walker@gmail.com>
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__author__ = "Myron Walker"
|
|
11
|
+
__copyright__ = "Copyright 2025, Code Craft AI LLC"
|
|
12
|
+
__credits__ = []
|
|
13
|
+
|
|
14
|
+
from typing import List
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import sys
|
|
18
|
+
|
|
19
|
+
import click
|
|
20
|
+
|
|
21
|
+
from ccai.interfaces.itestjob import ITestJob
|
|
22
|
+
from ccai.xmodules.xlogging.levels import LOG_LEVEL_NAMES
|
|
23
|
+
from ccai.xmodules.xlogging.enhancedloggers import getEnhancedLogger
|
|
24
|
+
|
|
25
|
+
from ccai.testplus.cli.cmdtree.testing.constants import (
|
|
26
|
+
HELP_ROOT,
|
|
27
|
+
HELP_EXCLUDES,
|
|
28
|
+
HELP_INCLUDES,
|
|
29
|
+
HELP_OUTPUT,
|
|
30
|
+
HELP_START,
|
|
31
|
+
HELP_BRANCH,
|
|
32
|
+
HELP_BUILD,
|
|
33
|
+
HELP_FLAVOR,
|
|
34
|
+
HELP_JOB_ID,
|
|
35
|
+
HELP_JOB_INITIATOR,
|
|
36
|
+
HELP_JOB_LABEL,
|
|
37
|
+
HELP_JOB_NAME,
|
|
38
|
+
HELP_JOB_OWNER,
|
|
39
|
+
HELP_RUNID,
|
|
40
|
+
HELP_CONSOLE_LOG_LEVEL,
|
|
41
|
+
HELP_FILE_LOG_LEVEL,
|
|
42
|
+
HELP_DEBUGGER,
|
|
43
|
+
HELP_BREAKPOINT,
|
|
44
|
+
HELP_PRERUN_DIAGNOSTIC,
|
|
45
|
+
HELP_POSTRUN_DIAGNOSTIC,
|
|
46
|
+
HELP_INCLUDE_MARKER_EXP,
|
|
47
|
+
HELP_EXCLUDE_MARKER_EXP
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
from ccai.testplus.cli.cmdtree.testing.constants import (
|
|
51
|
+
HELP_CONFIG_NAME,
|
|
52
|
+
HELP_CREDENTIAL,
|
|
53
|
+
HELP_CREDENTIAL_NAMES,
|
|
54
|
+
HELP_CREDENTIAL_SOURCES,
|
|
55
|
+
HELP_DEFAULT_CONFIGS,
|
|
56
|
+
HELP_LANDSCAPE,
|
|
57
|
+
HELP_LANDSCAPE_NAMES,
|
|
58
|
+
HELP_LANDSCAPE_SOURCES,
|
|
59
|
+
HELP_RUNTIME,
|
|
60
|
+
HELP_RUNTIME_NAMES,
|
|
61
|
+
HELP_RUNTIME_SOURCES,
|
|
62
|
+
HELP_TOPOLOGY,
|
|
63
|
+
HELP_TOPOLOGY_NAMES,
|
|
64
|
+
HELP_TOPOLOGY_SOURCES,
|
|
65
|
+
HELP_PRERUN_DIAGNOSTIC,
|
|
66
|
+
HELP_POSTRUN_DIAGNOSTIC,
|
|
67
|
+
CONFIGURATION_CHOICES
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
@click.command("run", help="Command used to run collections of tests.")
|
|
71
|
+
@click.option("--root", default=None, type=str, required=False, help=HELP_ROOT)
|
|
72
|
+
@click.option("--excludes", "-x", multiple=True, required=False, help=HELP_EXCLUDES)
|
|
73
|
+
@click.option("--includes", "-i", multiple=True, help=HELP_INCLUDES)
|
|
74
|
+
@click.option("--output", "-o", default=None, required=False, help=HELP_OUTPUT)
|
|
75
|
+
@click.option("--start", default=None, required=False, help=HELP_START)
|
|
76
|
+
@click.option("--runid", default=None, required=False, help=HELP_RUNID)
|
|
77
|
+
@click.option("--branch", default=None, required=False, help=HELP_BRANCH)
|
|
78
|
+
@click.option("--build", default=None, required=False, help=HELP_BUILD)
|
|
79
|
+
@click.option("--flavor", default=None, required=False, help=HELP_FLAVOR)
|
|
80
|
+
@click.option("--job-id", default=None, required=False, help=HELP_JOB_ID)
|
|
81
|
+
@click.option("--job-initiator", default=None, required=False, help=HELP_JOB_INITIATOR)
|
|
82
|
+
@click.option("--job-label", default=None, required=False, help=HELP_JOB_LABEL)
|
|
83
|
+
@click.option("--job-name", default=None, required=False, help=HELP_JOB_NAME)
|
|
84
|
+
@click.option("--job-owner", default=None, required=False, help=HELP_JOB_OWNER)
|
|
85
|
+
@click.option("--default-configs", "default_configs", multiple=True, default=None, required=False, type=click.Choice(CONFIGURATION_CHOICES), help=HELP_DEFAULT_CONFIGS)
|
|
86
|
+
@click.option("--config-name", "config_name", multiple=False, default=None, required=False, help=HELP_CONFIG_NAME)
|
|
87
|
+
@click.option("--credential", "credential_files", multiple=True, default=None, required=False, help=HELP_CREDENTIAL)
|
|
88
|
+
@click.option("--credential-name", "credential_names", multiple=True, default=None, required=False, help=HELP_CREDENTIAL_NAMES)
|
|
89
|
+
@click.option("--credential-source", "credential_sources", multiple=True, default=None, required=False, help=HELP_CREDENTIAL_SOURCES)
|
|
90
|
+
@click.option("--landscape", "landscape_files", multiple=True, default=None, required=False, help=HELP_LANDSCAPE)
|
|
91
|
+
@click.option("--landscape-name", "landscape_names", multiple=True, default=None, required=False, help=HELP_LANDSCAPE_NAMES)
|
|
92
|
+
@click.option("--landscape-source", "landscape_sources", multiple=True, default=None, required=False, help=HELP_LANDSCAPE_SOURCES)
|
|
93
|
+
@click.option("--runtime", "runtime_files", multiple=True, default=None, required=False, help=HELP_RUNTIME)
|
|
94
|
+
@click.option("--runtime-name", "runtime_names", multiple=True, default=None, required=False, help=HELP_RUNTIME_NAMES)
|
|
95
|
+
@click.option("--runtime-source", "runtime_sources", multiple=True, default=None, required=False, help=HELP_RUNTIME_SOURCES)
|
|
96
|
+
@click.option("--topology", "topology_files", multiple=True, default=None, required=False, help=HELP_TOPOLOGY)
|
|
97
|
+
@click.option("--topology-name", "topology_names", multiple=True, default=None, required=False, help=HELP_TOPOLOGY_NAMES)
|
|
98
|
+
@click.option("--topology-source", "topology_sources", multiple=True, default=None, required=False, help=HELP_TOPOLOGY_SOURCES)
|
|
99
|
+
@click.option("--console-level", default=None, required=False, type=click.Choice(LOG_LEVEL_NAMES, case_sensitive=False), help=HELP_CONSOLE_LOG_LEVEL)
|
|
100
|
+
@click.option("--logfile-level", default=None, required=False, type=click.Choice(LOG_LEVEL_NAMES, case_sensitive=False), help=HELP_FILE_LOG_LEVEL)
|
|
101
|
+
@click.option("--debugger", default=None, required=False, type=click.Choice(['pdb', 'debugpy']), help=HELP_DEBUGGER)
|
|
102
|
+
@click.option("--breakpoint", "breakpoints", default=None, required=False, multiple=True, type=click.Choice(['test-discovery', 'testrun-start']), help=HELP_BREAKPOINT)
|
|
103
|
+
@click.option("--prerun-diagnostic", is_flag=True, default=False, required=False, help=HELP_PRERUN_DIAGNOSTIC)
|
|
104
|
+
@click.option("--postrun-diagnostic", is_flag=True, default=False, required=False, help=HELP_POSTRUN_DIAGNOSTIC)
|
|
105
|
+
@click.option("--include-marker-exp", required=False, multiple=True, help=HELP_INCLUDE_MARKER_EXP)
|
|
106
|
+
@click.option("--exclude-marker-exp", required=False, multiple=True, help=HELP_EXCLUDE_MARKER_EXP)
|
|
107
|
+
def command_testplus_testing_run(root, includes, excludes, output, start, runid, branch, build, flavor, job_id, job_initiator,
|
|
108
|
+
job_label, job_name, job_owner, default_configs, config_name, credential_files, credential_names, credential_sources,
|
|
109
|
+
landscape_files, landscape_names, landscape_sources, runtime_files, runtime_names, runtime_sources,
|
|
110
|
+
topology_files, topology_names, topology_sources, console_level, logfile_level,
|
|
111
|
+
debugger, breakpoints, prerun_diagnostic, postrun_diagnostic, include_marker_exp,
|
|
112
|
+
exclude_marker_exp):
|
|
113
|
+
"""
|
|
114
|
+
Executes a collection of tests using the TestPlus framework.
|
|
115
|
+
|
|
116
|
+
:param root: Root directory used when scanning for tests.
|
|
117
|
+
:param includes: Test inclusion expressions.
|
|
118
|
+
:param excludes: Test exclusion expressions.
|
|
119
|
+
|
|
120
|
+
:returns: ``None``
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
# The click options below are declared multiple=True, so these arrive as
|
|
124
|
+
# immutable tuples; copy them into lists before appending the config name.
|
|
125
|
+
landscape_names = list(landscape_names)
|
|
126
|
+
topology_names = list(topology_names)
|
|
127
|
+
|
|
128
|
+
if config_name is not None:
|
|
129
|
+
if config_name not in landscape_names:
|
|
130
|
+
landscape_names.append(config_name)
|
|
131
|
+
if config_name not in topology_names:
|
|
132
|
+
topology_names.append(config_name)
|
|
133
|
+
|
|
134
|
+
# pylint: disable=unused-import,import-outside-toplevel
|
|
135
|
+
|
|
136
|
+
# We do the imports of the automation framework code inside the action functions because
|
|
137
|
+
# we don't want to startup loggin and the processing of inputs and environment variables
|
|
138
|
+
# until we have entered an action method. Thats way we know how to setup the environment.
|
|
139
|
+
|
|
140
|
+
# IMPORTANT: We need to load the context first because it will trigger the loading
|
|
141
|
+
# of the default user configuration
|
|
142
|
+
|
|
143
|
+
from ccai.thisapp.contextpaths import ContextPaths
|
|
144
|
+
from ccai.thisapp.wellknown import ThisApplicationSingleton
|
|
145
|
+
|
|
146
|
+
from ccai.xmodules.xpython import prepend_python_path
|
|
147
|
+
|
|
148
|
+
from ccai.thisapp.runtimevariables import RuntimeVariables, RuntimeVariablesDesc
|
|
149
|
+
|
|
150
|
+
from ccai.runtime.optionoverrides import AM_RUNTIME_OPTION_OVERRIDES
|
|
151
|
+
from ccai.configuration.optionoverrides import ConfigOptionOverrides
|
|
152
|
+
|
|
153
|
+
# STEP 1 - Initialize the Runtime (Performed by the root command)
|
|
154
|
+
|
|
155
|
+
# STEP 2 - Resolve the configuration variables (Performed by the root command)
|
|
156
|
+
|
|
157
|
+
# STEP 3 - Apply any 'Configuration' related option overrides
|
|
158
|
+
|
|
159
|
+
this_app = ThisApplicationSingleton()
|
|
160
|
+
|
|
161
|
+
# We need to default these to 'None' so they do not effect the runtime behavior unless
|
|
162
|
+
# this command explicitly sets them later.
|
|
163
|
+
use_credentials = None
|
|
164
|
+
use_landscape = None
|
|
165
|
+
use_runtime = None
|
|
166
|
+
use_topology = None
|
|
167
|
+
|
|
168
|
+
if "all" in default_configs:
|
|
169
|
+
use_credentials = True
|
|
170
|
+
use_landscape = True
|
|
171
|
+
use_runtime = True
|
|
172
|
+
use_topology = True
|
|
173
|
+
else:
|
|
174
|
+
if "credentials" in default_configs:
|
|
175
|
+
use_credentials = True
|
|
176
|
+
if "landscape" in default_configs:
|
|
177
|
+
use_landscape = True
|
|
178
|
+
if "runtime" in default_configs:
|
|
179
|
+
use_runtime = True
|
|
180
|
+
if "topology" in default_configs:
|
|
181
|
+
use_topology = True
|
|
182
|
+
|
|
183
|
+
from ccai.testbase.markers import MetaFilter, parse_marker_expression
|
|
184
|
+
|
|
185
|
+
if len(credential_files) > 0:
|
|
186
|
+
use_credentials = True
|
|
187
|
+
ConfigOptionOverrides.override_config_credentials_files(credential_files)
|
|
188
|
+
|
|
189
|
+
if len(credential_names) > 0:
|
|
190
|
+
use_credentials = True
|
|
191
|
+
ConfigOptionOverrides.override_config_credentials_names(credential_names)
|
|
192
|
+
|
|
193
|
+
if len(credential_sources) > 0:
|
|
194
|
+
use_credentials = True
|
|
195
|
+
ConfigOptionOverrides.override_config_credentials_sources(credential_sources)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
if len(landscape_files) > 0:
|
|
199
|
+
use_landscape = True
|
|
200
|
+
ConfigOptionOverrides.override_config_landscape_files(landscape_files)
|
|
201
|
+
|
|
202
|
+
if len(landscape_names) > 0:
|
|
203
|
+
use_landscape = True
|
|
204
|
+
ConfigOptionOverrides.override_config_landscape_names(landscape_names)
|
|
205
|
+
|
|
206
|
+
if len(landscape_sources) > 0:
|
|
207
|
+
use_landscape = True
|
|
208
|
+
ConfigOptionOverrides.override_config_landscape_sources(landscape_sources)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
if len(runtime_files) > 0:
|
|
212
|
+
use_runtime = True
|
|
213
|
+
ConfigOptionOverrides.override_config_runtime_files(runtime_files)
|
|
214
|
+
|
|
215
|
+
if len(runtime_names) > 0:
|
|
216
|
+
use_runtime = True
|
|
217
|
+
ConfigOptionOverrides.override_config_runtime_names(runtime_names)
|
|
218
|
+
|
|
219
|
+
if len(runtime_sources) > 0:
|
|
220
|
+
use_runtime = True
|
|
221
|
+
ConfigOptionOverrides.override_config_runtime_sources(runtime_sources)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
if len(topology_files) > 0:
|
|
225
|
+
use_topology = True
|
|
226
|
+
ConfigOptionOverrides.override_config_topology_files(topology_files)
|
|
227
|
+
|
|
228
|
+
if len(topology_names) > 0:
|
|
229
|
+
use_topology = True
|
|
230
|
+
ConfigOptionOverrides.override_config_topology_names(topology_names)
|
|
231
|
+
|
|
232
|
+
if len(topology_sources) > 0:
|
|
233
|
+
use_topology = True
|
|
234
|
+
ConfigOptionOverrides.override_config_topology_sources(topology_sources)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
# STEP 4 - Resolve the Configuration Maps
|
|
238
|
+
from ccai.configuration.configurationmaps import resolve_configuration_maps
|
|
239
|
+
resolve_configuration_maps(use_credentials=use_credentials, use_landscape=use_landscape,
|
|
240
|
+
use_runtime=use_runtime, use_topology=use_topology)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
# STEP 5 - The application of the remaining options must happen after we have resolved configuration
|
|
244
|
+
# maps because some of those settings will get overridden by the options. So the runtime
|
|
245
|
+
# needs to be loaded and set first before we apply runtime options
|
|
246
|
+
|
|
247
|
+
if prerun_diagnostic:
|
|
248
|
+
this_app.override_variable_value(RuntimeVariablesDesc.AM_DIAGNOSTICS_PRERUN, {})
|
|
249
|
+
|
|
250
|
+
if postrun_diagnostic:
|
|
251
|
+
this_app.override_variable_value(RuntimeVariablesDesc.AM_DIAGNOSTICS_POSTRUN, {})
|
|
252
|
+
|
|
253
|
+
if branch is not None:
|
|
254
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_build_branch(branch)
|
|
255
|
+
|
|
256
|
+
if build is not None:
|
|
257
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_build_name(build)
|
|
258
|
+
|
|
259
|
+
if flavor is not None:
|
|
260
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_build_flavor(flavor)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
if job_id is not None:
|
|
264
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_job_id(job_id)
|
|
265
|
+
|
|
266
|
+
if job_initiator is not None:
|
|
267
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_job_initiator(job_initiator)
|
|
268
|
+
|
|
269
|
+
if job_label is not None:
|
|
270
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_job_label(job_label)
|
|
271
|
+
|
|
272
|
+
if job_name is not None:
|
|
273
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_job_name(job_name)
|
|
274
|
+
|
|
275
|
+
if job_owner is not None:
|
|
276
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_job_owner(job_owner)
|
|
277
|
+
|
|
278
|
+
if console_level is not None:
|
|
279
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_loglevel_console(console_level)
|
|
280
|
+
|
|
281
|
+
if logfile_level is not None:
|
|
282
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_loglevel_file(logfile_level)
|
|
283
|
+
|
|
284
|
+
if output is not None:
|
|
285
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_output_directory(output)
|
|
286
|
+
|
|
287
|
+
if start is not None:
|
|
288
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_starttime(start)
|
|
289
|
+
|
|
290
|
+
if runid is not None:
|
|
291
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_run_id(runid)
|
|
292
|
+
|
|
293
|
+
# The '--breakpoint' option is declared multiple=True, so click delivers an
|
|
294
|
+
# empty tuple rather than None when it is not given. Only a non-empty
|
|
295
|
+
# selection is a request to break, so only then override the breakpoints
|
|
296
|
+
# and fall back to a default debugger.
|
|
297
|
+
if breakpoints is not None and len(breakpoints) > 0:
|
|
298
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_debug_breakpoints(breakpoints)
|
|
299
|
+
|
|
300
|
+
# If a breakpoint was passed but the debugger was not, fall back to
|
|
301
|
+
# 'debugpy' as the default debugger.
|
|
302
|
+
if debugger is None:
|
|
303
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_debug_debugger('debugpy')
|
|
304
|
+
|
|
305
|
+
if debugger is not None:
|
|
306
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_debug_debugger(debugger)
|
|
307
|
+
|
|
308
|
+
# TODO: Handle 'root' parameter better
|
|
309
|
+
if root is None:
|
|
310
|
+
if RuntimeVariables.AM_TESTROOT is not None:
|
|
311
|
+
root = RuntimeVariables.AM_TESTROOT
|
|
312
|
+
else:
|
|
313
|
+
root = "."
|
|
314
|
+
|
|
315
|
+
test_root = os.path.abspath(os.path.expandvars(os.path.expanduser(root)))
|
|
316
|
+
if not os.path.isdir(test_root):
|
|
317
|
+
errmsg = "The specified root folder does not exist. root=%s" % root
|
|
318
|
+
if test_root != root:
|
|
319
|
+
errmsg += " expanded=%s" % test_root
|
|
320
|
+
raise click.BadParameter(errmsg)
|
|
321
|
+
|
|
322
|
+
AM_RUNTIME_OPTION_OVERRIDES.override_testroot(root)
|
|
323
|
+
|
|
324
|
+
# Make sure we extend PATH to include the test roots parent folder so imports will
|
|
325
|
+
# work properly.
|
|
326
|
+
test_root_parent = os.path.dirname(test_root)
|
|
327
|
+
prepend_python_path(test_root_parent)
|
|
328
|
+
|
|
329
|
+
metafilters: List[MetaFilter] = []
|
|
330
|
+
for imexp in include_marker_exp:
|
|
331
|
+
imexp = imexp.strip("\"")
|
|
332
|
+
imexp = imexp.strip("'")
|
|
333
|
+
mfilter = parse_marker_expression("+" + imexp)
|
|
334
|
+
metafilters.append(mfilter)
|
|
335
|
+
|
|
336
|
+
for exmexp in exclude_marker_exp:
|
|
337
|
+
exmexp = exmexp.strip("\"")
|
|
338
|
+
exmexp = exmexp.strip("'")
|
|
339
|
+
mfilter = parse_marker_expression("-" + exmexp)
|
|
340
|
+
metafilters.append(mfilter)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
# STEP 6 - Initialize the Results Output
|
|
344
|
+
|
|
345
|
+
# Initialize testplus results and logging
|
|
346
|
+
from ccai.testplus.initialize import initialize_testplus_results
|
|
347
|
+
|
|
348
|
+
initialize_testplus_results()
|
|
349
|
+
|
|
350
|
+
logger = getEnhancedLogger()
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
# STEP 7 - Create and kick off the job
|
|
354
|
+
|
|
355
|
+
from ccai.thisapp.wellknown import ConfiguredSuperFactorySingleton
|
|
356
|
+
from ccai.testbase.factoryprotocols import TestBaseFactoryProtocol
|
|
357
|
+
|
|
358
|
+
# At this point in the code, we either lookup an existing test job or we create a test job
|
|
359
|
+
# from the includes, excludes or test_module
|
|
360
|
+
sfactory = ConfiguredSuperFactorySingleton()
|
|
361
|
+
TestJobType = sfactory.get_override_types_by_order(TestBaseFactoryProtocol.get_testbase_default_job_type)
|
|
362
|
+
|
|
363
|
+
tjob: ITestJob
|
|
364
|
+
|
|
365
|
+
result_code = 0
|
|
366
|
+
with TestJobType(logger, test_root, includes=includes, excludes=excludes, metafilters=metafilters) as tjob:
|
|
367
|
+
result_code = tjob.execute()
|
|
368
|
+
|
|
369
|
+
sys.exit(result_code)
|
|
370
|
+
|
|
371
|
+
return
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
.. module:: testplus_command
|
|
4
|
+
:platform: Darwin, Linux, Unix, Windows
|
|
5
|
+
:synopsis: The script entrypoint for the 'testplus' command.
|
|
6
|
+
|
|
7
|
+
.. moduleauthor:: Myron Walker <myron.walker@gmail.com>
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__author__ = "Myron Walker"
|
|
11
|
+
__copyright__ = "Copyright 2025, Code Craft AI LLC"
|
|
12
|
+
__credits__ = []
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
import click
|
|
16
|
+
|
|
17
|
+
from ccai.thisapp.wellknown import ThisApplicationSingleton
|
|
18
|
+
|
|
19
|
+
from ccai.thisapp.runtimevariables import RuntimeVariablesDesc
|
|
20
|
+
|
|
21
|
+
from ccai.testplus.cli.cmdtree.testing import group_testplus_testing
|
|
22
|
+
|
|
23
|
+
from ccai.testplus.initialize import initialize_testplus_runtime
|
|
24
|
+
|
|
25
|
+
@click.group("testplus", help="Run and inspect distributed, multi-host tests.")
|
|
26
|
+
@click.option('-v', '--verbose', count=True, help="Set the verbosity level.")
|
|
27
|
+
def testplus_root_command(verbose):
|
|
28
|
+
"""
|
|
29
|
+
Entry point for the ``testplus`` command line interface.
|
|
30
|
+
|
|
31
|
+
:param int verbose: Verbosity level specified by ``-v`` or ``--verbose``.
|
|
32
|
+
|
|
33
|
+
:returns: ``None``
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
initialize_testplus_runtime()
|
|
37
|
+
|
|
38
|
+
interactive_console = False
|
|
39
|
+
log_level_console = "WARNING"
|
|
40
|
+
|
|
41
|
+
if verbose == 0:
|
|
42
|
+
interactive_console = True
|
|
43
|
+
else:
|
|
44
|
+
interactive_console = False
|
|
45
|
+
|
|
46
|
+
if verbose == 1:
|
|
47
|
+
log_level_console = "INFO"
|
|
48
|
+
elif verbose == 2:
|
|
49
|
+
log_level_console = "DEBUG"
|
|
50
|
+
elif verbose > 2:
|
|
51
|
+
log_level_console = "NOTSET"
|
|
52
|
+
|
|
53
|
+
this_app = ThisApplicationSingleton()
|
|
54
|
+
|
|
55
|
+
this_app.override_variable_value(RuntimeVariablesDesc.AM_INTERACTIVE_CONSOLE, interactive_console)
|
|
56
|
+
this_app.override_variable_value(RuntimeVariablesDesc.AM_LOG_LEVEL_CONSOLE, log_level_console)
|
|
57
|
+
|
|
58
|
+
return
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# NOTE: testbase's 'services' and 'utilities' groups are deliberately NOT
|
|
62
|
+
# mounted here. This root command is still a click group, while testbase's
|
|
63
|
+
# command tree has been migrated to arcli, and click cannot dispatch to an
|
|
64
|
+
# arcli command: click's Group.invoke calls cmd.make_context() and its help
|
|
65
|
+
# formatter reads cmd.hidden, neither of which an arcli GroupingCommand has.
|
|
66
|
+
#
|
|
67
|
+
# Mounting them raised AttributeError on EVERY use -- 'testplus services',
|
|
68
|
+
# 'testplus utilities', any subcommand of either, and 'testplus --help'
|
|
69
|
+
# itself, because formatting the root help walks every registered command.
|
|
70
|
+
# So they contributed no working functionality while breaking root help.
|
|
71
|
+
#
|
|
72
|
+
# Those commands remain available through the 'testbase' entry point, which
|
|
73
|
+
# is installed alongside this one (testplus depends on testbase). Mount them
|
|
74
|
+
# here again only once this root command is itself migrated to arcli.
|
|
75
|
+
testplus_root_command.add_command(group_testplus_testing)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
if __name__ == '__main__':
|
|
79
|
+
testplus_root_command()
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""
|
|
2
|
+
.. module:: distributedtestjob
|
|
3
|
+
:platform: Darwin, Linux, Unix, Windows
|
|
4
|
+
:synopsis: Module containing the :class:`DistributedTestJob` class which is the
|
|
5
|
+
TestPlus parent container for distributed (multi-host) test runs.
|
|
6
|
+
|
|
7
|
+
.. moduleauthor:: Myron Walker <myron.walker@gmail.com>
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__author__ = "Myron Walker"
|
|
11
|
+
__copyright__ = "Copyright 2025, Code Craft AI LLC"
|
|
12
|
+
__credits__ = []
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from ccai.landscaping.testing.testjob import LandscapeTestJob
|
|
16
|
+
|
|
17
|
+
from ccai.testplus.distributedtestsequencer import DistributedTestSequencer
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class DistributedTestJob(LandscapeTestJob):
|
|
21
|
+
"""
|
|
22
|
+
TestPlus parent container for a distributed test run.
|
|
23
|
+
|
|
24
|
+
Layered inheritance:
|
|
25
|
+
|
|
26
|
+
* :class:`ccai.testbase.testjob.TestJob` — single-host base
|
|
27
|
+
(lifecycle, recorder, sequencer factory, properties, ``execute``);
|
|
28
|
+
standalone-default seams (stdout recorder).
|
|
29
|
+
* :class:`ccai.integrationruntime.integratedtestjob.IntegratedTestJob` —
|
|
30
|
+
integration tier: JSOS/HTML/presence-folder result recording and run
|
|
31
|
+
metadata sourced from ``RuntimeVariables``.
|
|
32
|
+
* :class:`ccai.landscaping.testing.testjob.LandscapeTestJob` —
|
|
33
|
+
the landscaping tier: supplies the landscape singleton and carries
|
|
34
|
+
the landscape-activation lifecycle on the two attachment hooks
|
|
35
|
+
(``activate_configuration`` / ``activate_integration`` /
|
|
36
|
+
``activate_operations``, ``collect_resources``,
|
|
37
|
+
``establish_presence``).
|
|
38
|
+
* :class:`DistributedTestJob` (this class) — constructs a
|
|
39
|
+
:class:`DistributedTestSequencer` so the sequencer's
|
|
40
|
+
``attach_to_environment`` / ``attach_to_framework`` methods accept
|
|
41
|
+
the landscape and so distributed-test scheduling primitives become
|
|
42
|
+
available.
|
|
43
|
+
|
|
44
|
+
Single Instance. Setup Test Landscape. Jobs have an expected number
|
|
45
|
+
of tests based on includes / excludes. Collects and organises results
|
|
46
|
+
across hosts. Can be subclassed to customise the sequencing of test
|
|
47
|
+
packs across distributed infrastructure.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
# Derived jobs set these to indicate that the test framework must load the
|
|
51
|
+
# corresponding configuration maps before the run (opt-in; default False).
|
|
52
|
+
# ``distributed_test_entrypoint`` reads them to call ``resolve_configuration_maps``.
|
|
53
|
+
requires_credentials = False
|
|
54
|
+
requires_landscape = False
|
|
55
|
+
requires_runtime = False
|
|
56
|
+
requires_topology = False
|
|
57
|
+
|
|
58
|
+
def _create_sequencer(self) -> DistributedTestSequencer:
|
|
59
|
+
"""
|
|
60
|
+
Hook used by :meth:`TestJob.execute` to construct the test
|
|
61
|
+
sequencer for this job. Returns a
|
|
62
|
+
:class:`DistributedTestSequencer` so the sequencer's
|
|
63
|
+
``attach_to_environment`` / ``attach_to_framework`` methods
|
|
64
|
+
accept the landscape supplied by
|
|
65
|
+
:class:`LandscapeTestJob`.
|
|
66
|
+
|
|
67
|
+
:returns: A :class:`DistributedTestSequencer` configured with
|
|
68
|
+
this job's title, test root, includes / excludes,
|
|
69
|
+
metafilters, and ``job_seed`` for reproducible
|
|
70
|
+
property-based testing.
|
|
71
|
+
"""
|
|
72
|
+
inst = DistributedTestSequencer(
|
|
73
|
+
self._title,
|
|
74
|
+
self._testroot,
|
|
75
|
+
includes=self.includes,
|
|
76
|
+
excludes=self.excludes,
|
|
77
|
+
metafilters=self.metafilters,
|
|
78
|
+
job_seed=self.job_seed,
|
|
79
|
+
output_context=self._output_context,
|
|
80
|
+
)
|
|
81
|
+
return inst
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class DefaultDistributedTestJob(DistributedTestJob):
|
|
85
|
+
"""
|
|
86
|
+
The :class:`DefaultDistributedTestJob` is utilized as a job container when a
|
|
87
|
+
job was not specified.
|
|
88
|
+
"""
|
|
89
|
+
name = "Test Job"
|
|
90
|
+
description = "Unspecified test job."
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""
|
|
2
|
+
.. module:: distributedtestsequencer
|
|
3
|
+
:platform: Darwin, Linux, Unix, Windows
|
|
4
|
+
:synopsis: Module containing the :class:`DistributedTestSequencer` type which is use to control
|
|
5
|
+
the flow of the automation environment startup and test execution sequence for a distributed
|
|
6
|
+
automation environment.
|
|
7
|
+
|
|
8
|
+
.. moduleauthor:: Myron Walker <myron.walker@gmail.com>
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__author__ = "Myron Walker"
|
|
12
|
+
__copyright__ = "Copyright 2025, Code Craft AI LLC"
|
|
13
|
+
__credits__ = []
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
from typing import List, Optional, Sequence
|
|
17
|
+
|
|
18
|
+
import collections
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import sys
|
|
22
|
+
|
|
23
|
+
from ccai.thisapp.paths import get_path_for_output
|
|
24
|
+
|
|
25
|
+
from ccai.testbase.coupling.integrationcoupling import IntegrationCoupling
|
|
26
|
+
from ccai.testbase.markers import MetaFilter
|
|
27
|
+
|
|
28
|
+
from ccai.landscaping.landscape import Landscape
|
|
29
|
+
|
|
30
|
+
from ccai.testbase.output.outputcontext import IOutputContext
|
|
31
|
+
from ccai.testbase.sequencing.testsequencer import TestSequencer
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class DistributedTestSequencer(TestSequencer):
|
|
35
|
+
"""
|
|
36
|
+
The :class:`DistributedTestSequencer` is a state machine that helps to orchestrate the flow fo the test run. It ensures
|
|
37
|
+
that the steps of the test flow are consistent between runs.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(self, jobtitle: str, root: str, includes: Sequence[str], excludes: Sequence[str], metafilters: List[MetaFilter], job_seed: int = None,
|
|
41
|
+
output_context: Optional[IOutputContext] = None):
|
|
42
|
+
"""
|
|
43
|
+
Creates a 'TestSequencer' object which is used to discover the tests and control the flow of a test run.
|
|
44
|
+
|
|
45
|
+
:param jobtitle: The name of the test job.
|
|
46
|
+
:param root: The path to the root folder that is the base of the tests.
|
|
47
|
+
:param includes: List of expressions used to determine which tests to include.
|
|
48
|
+
(scope):(package).(package)@(module)#(testname)
|
|
49
|
+
:param excludes: List of expressions used to determine which tests to exclued from the included tests.
|
|
50
|
+
:param metafilters: List of expressions used to filter tests by metadata
|
|
51
|
+
:param job_seed: Job-level seed forwarded to the parent :class:`TestSequencer` for
|
|
52
|
+
reproducible property-based testing.
|
|
53
|
+
:param output_context: The :class:`IOutputContext` forwarded to the parent
|
|
54
|
+
:class:`TestSequencer` for resolving artefact directories.
|
|
55
|
+
"""
|
|
56
|
+
super().__init__(jobtitle=jobtitle, root=root, includes=includes, excludes=excludes, metafilters=metafilters, job_seed=job_seed,
|
|
57
|
+
output_context=output_context)
|
|
58
|
+
|
|
59
|
+
self._landscape = None
|
|
60
|
+
return
|
|
61
|
+
|
|
62
|
+
def attach_to_framework(self, landscape: Landscape):
|
|
63
|
+
"""
|
|
64
|
+
Goes through all the integrations and provides them with an opportunity to
|
|
65
|
+
attach to the test environment.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
self._landscape = landscape
|
|
69
|
+
|
|
70
|
+
integ_type: IntegrationCoupling
|
|
71
|
+
|
|
72
|
+
for _, integ_type in self._integrations.items():
|
|
73
|
+
integ_type.attach_to_framework(landscape=landscape)
|
|
74
|
+
|
|
75
|
+
return
|
|
76
|
+
|
|
77
|
+
def attach_to_environment(self, landscape):
|
|
78
|
+
"""
|
|
79
|
+
Goes through all the integrations and provides them with an opportunity to
|
|
80
|
+
attach to the test environment.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
results_dir = get_path_for_output()
|
|
84
|
+
|
|
85
|
+
environment_dict = self.create_password_masked_environment()
|
|
86
|
+
|
|
87
|
+
package_dict = collections.OrderedDict()
|
|
88
|
+
|
|
89
|
+
package_names = [k for k in sys.modules.keys()]
|
|
90
|
+
package_names.sort()
|
|
91
|
+
for pname in package_names:
|
|
92
|
+
nxtmod = sys.modules[pname]
|
|
93
|
+
if pname.find(".") == -1 and hasattr(nxtmod, "__file__"):
|
|
94
|
+
package_dict[pname] = nxtmod.__file__
|
|
95
|
+
|
|
96
|
+
startup_dict = collections.OrderedDict([
|
|
97
|
+
("command", " ".join(sys.argv)),
|
|
98
|
+
("environment", environment_dict),
|
|
99
|
+
("packages", package_dict)
|
|
100
|
+
])
|
|
101
|
+
|
|
102
|
+
startup_full = os.path.join(results_dir, "startup-configuration.json")
|
|
103
|
+
with open(startup_full, 'w') as suf:
|
|
104
|
+
json.dump(startup_dict, suf, indent=True)
|
|
105
|
+
|
|
106
|
+
integ_type: IntegrationCoupling
|
|
107
|
+
|
|
108
|
+
for _, integ_type in self._integrations.items():
|
|
109
|
+
integ_type.attach_to_environment(landscape=landscape)
|
|
110
|
+
|
|
111
|
+
return
|