baygon 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
baygon/__init__.py ADDED
@@ -0,0 +1,17 @@
1
+ """Baygon is a tool to run tests on executables."""
2
+
3
+ from .executable import Executable
4
+ from .schema import Schema
5
+ from .suite import TestCase, TestGroup, TestSuite
6
+
7
+ __all__ = ["TestCase", "TestGroup", "TestSuite", "Executable", "Schema"]
8
+
9
+ try:
10
+ from . import version
11
+
12
+ __version__ = version.version
13
+ except ImportError:
14
+ __version__ = None
15
+
16
+ __year__ = 2022
17
+ __copyright__ = f"Copyright {__year__} HEIG-VD"
baygon/__main__.py ADDED
@@ -0,0 +1,385 @@
1
+ """Main CLI command."""
2
+
3
+ import logging
4
+ import os
5
+ import sys
6
+ import time
7
+
8
+ import click
9
+ from rich.box import SQUARE_DOUBLE_HEAD
10
+ from rich.console import Console
11
+ from rich.rule import Rule
12
+ from rich.style import Style
13
+ from rich.table import Table
14
+
15
+ from . import TestCase, TestGroup, TestSuite, __copyright__, __version__
16
+ from .error import InvalidExecutableError
17
+ from .helpers import create_command_line
18
+
19
+ logging.basicConfig(level=logging.INFO)
20
+ logger = logging.getLogger("baygon")
21
+
22
+ console = Console()
23
+
24
+
25
+ def display_pad(pad=0):
26
+ """Display a pad of spaces to align nested output."""
27
+ click.secho("." * pad if pad > 0 else " ", nl=False, dim=pad == 0)
28
+
29
+
30
+ def test_name_length(test):
31
+ """Compute the length of test name."""
32
+ return len(f"{test.id.pad()}Test {test.id}: {test.name}")
33
+
34
+
35
+ def display_test_name(test):
36
+ """Display the name of a test."""
37
+ click.secho(f"{test.id.pad()}Test {test.id}: ", nl=False, bold=True)
38
+ click.secho(f"{test.name}", nl=False, bold=False)
39
+
40
+
41
+ def display_table(data):
42
+ grey_line_style = Style(color="grey37")
43
+ title_style = Style(bold=False, color="white")
44
+ table = Table(
45
+ title="Test Summary",
46
+ title_style=title_style,
47
+ border_style=grey_line_style,
48
+ box=SQUARE_DOUBLE_HEAD,
49
+ )
50
+
51
+ table.add_column("ID", justify="left")
52
+ table.add_column("Test Name", justify="left", style=grey_line_style)
53
+ table.add_column("Points", justify="center", style=grey_line_style)
54
+ table.add_column("Status", justify="center", style=grey_line_style)
55
+
56
+ for test in data:
57
+ status = ""
58
+ points = ""
59
+ if test.get("status") is not None:
60
+ status = "[green]PASS[/green]" if test["status"] else "[red]FAIL[/red]"
61
+
62
+ points = f"{test['earned']}/{test['points']}"
63
+
64
+ table.add_row(
65
+ str(test["id"]), (" " * len(test["id"])) + test["name"], points, status
66
+ )
67
+ console.print(table)
68
+
69
+
70
+ class OneLineExceptionFormatter(logging.Formatter):
71
+ """A formatter that displays the exception on a single line."""
72
+
73
+ def format_exception(self, exc_info):
74
+ """Format an exception."""
75
+ result = super().format_exception(exc_info)
76
+ return repr(result)
77
+
78
+ def format(self, record):
79
+ """Format a log record."""
80
+ result = super().format(record)
81
+ if record.exc_text:
82
+ result = result.replace("\n", "")
83
+ return result
84
+
85
+
86
+ class Runner:
87
+ """Test runner."""
88
+
89
+ def __init__(self, executable=None, config=None, **kwargs):
90
+ self.executable = executable
91
+ self.limit = kwargs.get("limit", -1)
92
+ click.secho(f"Using configuration file: {config}")
93
+ self.test_suite = TestSuite(path=config, executable=self.executable)
94
+
95
+ # Forward cli arguments to the test suite
96
+ if "verbose" in kwargs and kwargs["verbose"] != 0:
97
+ self.test_suite.config["verbose"] = kwargs["verbose"]
98
+ if "report" in kwargs and kwargs["report"] is not None:
99
+ self.test_suite.config["report"] = kwargs["report"]
100
+ if "format" in kwargs and kwargs["format"] is not None:
101
+ self.test_suite.config["format"] = kwargs["format"]
102
+ if "table" in kwargs and kwargs["table"] is not None:
103
+ self.test_suite.config["table"] = kwargs["table"]
104
+
105
+ self.verbose = kwargs.get("verbose", 0)
106
+ if self.verbose > 0:
107
+ click.echo("Verbose level set to " + str(self.verbose))
108
+
109
+ if self.verbose > 3:
110
+ self._init_logger("DEBUG")
111
+
112
+ self.align_column = 0
113
+
114
+ self.failures = 0
115
+ self.successes = 0
116
+ self.skipped = 0
117
+ self.run_time = 0
118
+
119
+ self.points_earned = 0
120
+ self.points_total = 0
121
+
122
+ self.summary = []
123
+
124
+ def _init_logger(self, loglevel):
125
+ handler = logging.StreamHandler()
126
+ formatter = OneLineExceptionFormatter(logging.BASIC_FORMAT)
127
+ handler.setFormatter(formatter)
128
+ root = logging.getLogger()
129
+ root.setLevel(os.environ.get("LOGLEVEL", loglevel))
130
+ root.addHandler(handler)
131
+
132
+ def get_report(self):
133
+ self.test_suite.get_points()
134
+ return {
135
+ "failures": self.failures,
136
+ "successes": self.successes,
137
+ "skipped": self.skipped,
138
+ "total": self.failures + self.successes + self.skipped,
139
+ "time": self.run_time,
140
+ "points": {
141
+ "total": self.points_total,
142
+ "earned": self.points_earned,
143
+ },
144
+ }
145
+
146
+ def run(self):
147
+ """Run the tests."""
148
+ start_time = time.time()
149
+ self.align_column = self._max_length(self.test_suite) + 10
150
+
151
+ self.failures = 0
152
+ self.successes = 0
153
+ self.skipped = 0
154
+ self.points_total = 0
155
+ self.points_earned = 0
156
+
157
+ self._traverse_group(self.test_suite)
158
+
159
+ if self.test_suite.config.get("table"):
160
+ display_table(self.summary)
161
+
162
+ tests = self.failures + self.successes + self.skipped
163
+ seconds = round(time.time() - start_time, 2)
164
+ self.run_time = seconds
165
+ console.print(f"\nRan [bold]{tests}[/bold] tests in [bold]{seconds} s[/bold].")
166
+
167
+ if self.points_total > 0:
168
+ click.secho(
169
+ f"Points: {self.points_earned}/{self.points_total}",
170
+ bold=True,
171
+ )
172
+
173
+ if self.failures > 0:
174
+ ratio = 100 - round(
175
+ self.failures / (self.failures + self.successes) * 100, 2
176
+ )
177
+ click.secho(
178
+ (f"{self.failures} failed, {self.successes} " f"passed ({ratio}% ok)."),
179
+ fg="yellow",
180
+ bold=True,
181
+ )
182
+ click.secho("\nfail.", fg="red", bold=True)
183
+ else:
184
+ click.secho("\nok.", fg="green")
185
+
186
+ if self.skipped > 0:
187
+ click.secho(
188
+ (
189
+ f"{self.skipped} test(s) skipped, "
190
+ "some executables may be missing."
191
+ ),
192
+ fg="yellow",
193
+ )
194
+
195
+ report = self.test_suite.config.get("report")
196
+ report_format = self.test_suite.config.get("format")
197
+ if report:
198
+ if not report_format:
199
+ if report.endswith(".yaml"):
200
+ report_format = "yaml"
201
+ else:
202
+ report_format = "json"
203
+ save_report(self.get_report(), report, report_format)
204
+
205
+ return self.failures
206
+
207
+ def _max_length(self, tests):
208
+ length = 0
209
+ for test in tests:
210
+ length = max(test_name_length(test), length)
211
+ if isinstance(test, TestGroup):
212
+ length = max(self._max_length(test), length)
213
+ return length
214
+
215
+ def hook_ran_command(self, cmd, stdin, stdout, stderr, exit_status, **kwargs):
216
+ """Capture executed command for logging purpose."""
217
+ # If element have a space, wrap it with quotes
218
+
219
+ cmdline = create_command_line(cmd)
220
+ cin = f' <<< "{stdin.decode("utf-8")}"' if stdin else ""
221
+ self.ran_commands.append(
222
+ f"{cmdline}{cin} (exited with: {exit_status})\n{stdout}{stderr}"
223
+ )
224
+
225
+ def pad(self, text, length):
226
+ """Shift text to right
227
+ >>> pad("hello", 5)"
228
+ ' hello'
229
+ >>> pad("hello\nworld", 3)"
230
+ ' hello\n world'
231
+ """
232
+ return "\n".join(" " * length + line for line in text.splitlines())
233
+
234
+ def display_test_verbose(self, test, issues, verbose=0): # noqa: C901
235
+
236
+ p = test
237
+ name = []
238
+ while True:
239
+ if p is None or getattr(p, "name", None) is None:
240
+ break
241
+ if p.name:
242
+ name.append(p.name)
243
+ p = p.parent
244
+ name = " / ".join(reversed(name))
245
+
246
+ if verbose >= 2:
247
+ console.print("\n")
248
+ console.print(Rule(f"Test {test.id}: {name}", align="left", style="bold"))
249
+
250
+ if verbose >= 3:
251
+ click.secho(
252
+ self.pad("".join([f"$ {cmd}\n" for cmd in self.ran_commands]), 2),
253
+ dim=True,
254
+ )
255
+
256
+ if test.status == "skipped":
257
+ if verbose > 0:
258
+ click.secho(" SKIPPED", fg="yellow")
259
+ else:
260
+ click.secho("S", fg="yellow", nl=False)
261
+ self.skipped += 1
262
+ return
263
+
264
+ if test.status == "passed":
265
+ if verbose > 0:
266
+ click.secho(" PASSED", fg="green")
267
+ else:
268
+ click.secho(".", fg="green", nl=False)
269
+ self.successes += 1
270
+ return
271
+
272
+ if test.status == "failed":
273
+ if verbose > 0:
274
+ click.secho(" FAILED", fg="red")
275
+ else:
276
+ click.secho("F", fg="red", nl=False)
277
+ self.failures += 1
278
+
279
+ for k, issue in enumerate(issues):
280
+ click.secho(str(issue))
281
+
282
+ def _run_test(self, test):
283
+ self.ran_commands = []
284
+ hook = self.hook_ran_command
285
+ issues = test.run(hook)
286
+ if issues is None:
287
+ test.status = "skipped"
288
+ elif not issues:
289
+ test.status = "passed"
290
+ else:
291
+ test.status = "failed"
292
+
293
+ self.display_test_verbose(test, issues, self.verbose)
294
+
295
+ earned_points = test.points if test.status == "passed" else 0
296
+ self.points_total += test.points
297
+ self.points_earned += earned_points
298
+ self.summary.append(
299
+ {
300
+ "id": test.id,
301
+ "name": test.name,
302
+ "status": test.status,
303
+ "points": test.points,
304
+ "earned": earned_points,
305
+ }
306
+ )
307
+
308
+ def _traverse_group(self, tests):
309
+ for test in tests:
310
+ if self.limit > 0 and self.failures > self.limit:
311
+ break
312
+
313
+ if isinstance(test, TestGroup):
314
+ self.summary.append(
315
+ {
316
+ "id": test.id,
317
+ "name": test.name,
318
+ }
319
+ )
320
+ self._traverse_group(test)
321
+ elif isinstance(test, TestCase):
322
+ self._run_test(test)
323
+ return self.failures
324
+
325
+
326
+ def save_report(data, filename, format):
327
+ """Save the report to a file."""
328
+ if format == "json":
329
+ import json
330
+
331
+ with open(filename, "w") as fp:
332
+ json.dump(data, fp, indent=2, sort_keys=True)
333
+ elif format == "yaml":
334
+ import yaml
335
+
336
+ with open(filename, "w") as fp:
337
+ yaml.dump(data, fp)
338
+
339
+
340
+ def version():
341
+ """Display the version."""
342
+ print(f"Baygon version {__version__} {__copyright__}")
343
+
344
+
345
+ @click.command()
346
+ @click.argument("executable", required=False, type=click.Path(exists=True))
347
+ @click.option("--version", is_flag=True, help="Shows version")
348
+ @click.option("-v", "--verbose", count=True, help="Shows more details")
349
+ @click.option("-l", "--limit", type=int, default=-1, help="Limit errors to N")
350
+ @click.option("-d", "--debug", is_flag=True, default=False, help="Debug mode")
351
+ @click.option("-r", "--report", type=click.Path(), help="Report file")
352
+ @click.option("-t", "--table", is_flag=True, default=False, help="Summary table")
353
+ @click.option(
354
+ "-f", "--format", type=click.Choice(["json", "yaml"]), help="Report format"
355
+ )
356
+ @click.option(
357
+ "-c",
358
+ "--config",
359
+ type=click.Path(exists=True),
360
+ help="Choose config file (.yml or .json)",
361
+ )
362
+ def cli(debug, **kwargs):
363
+ """Baygon functional test runner."""
364
+ if kwargs.get("version"):
365
+ version()
366
+ sys.exit(0)
367
+
368
+ if debug:
369
+ logger.setLevel(logging.DEBUG)
370
+ logger.debug("Debug mode enabled.")
371
+ runner = Runner(**kwargs)
372
+ runner.run()
373
+ else:
374
+ try:
375
+ runner = Runner(**kwargs)
376
+ runner.run()
377
+ except InvalidExecutableError as error:
378
+ click.secho(f"\nError: {error}", fg="red", bold=True, err=True)
379
+ sys.exit(1)
380
+
381
+ click.echo("")
382
+
383
+
384
+ if __name__ == "__main__":
385
+ cli()
baygon/error.py ADDED
@@ -0,0 +1,17 @@
1
+ """Errors for Baygon"""
2
+
3
+
4
+ class BaygonError(Exception):
5
+ """Base class for Baygon errors"""
6
+
7
+
8
+ class ConfigError(BaygonError):
9
+ """Raised when a config value is not valid"""
10
+
11
+
12
+ class InvalidExecutableError(BaygonError):
13
+ """Raised when an executable is not found"""
14
+
15
+
16
+ class InvalidFilterError(BaygonError):
17
+ """Raised when a filter is not found"""
baygon/eval.py ADDED
@@ -0,0 +1,41 @@
1
+ """Helper functions used in `eval` filter.
2
+ These functions are injected into the kernel and made available from
3
+ the mustaches templates.
4
+
5
+ Examples:
6
+
7
+ >>> reset()
8
+ >>> iter()
9
+ 0
10
+ >>> iter()
11
+ 1
12
+ >>> iter()
13
+ 2
14
+ >>> iter(100, 10)
15
+ 100
16
+ >>> iter(100, 10)
17
+ 110
18
+ >>> iter(ctx='foo')
19
+ 0
20
+ >>> iter(ctx='foo')
21
+ 1
22
+ >>> reset()
23
+ >>> iter()
24
+ 0
25
+ >>> iter(100, 10)
26
+ 100
27
+ """
28
+
29
+ _context = {}
30
+
31
+
32
+ def reset():
33
+ """Reset the context."""
34
+ _context.clear()
35
+
36
+
37
+ def iter(start=0, step=1, ctx=None):
38
+ """Custom iterator for eval input filter."""
39
+ ctx = (start, step, ctx)
40
+ _context[ctx] = _context.get(ctx, start - step) + step
41
+ return _context[ctx]
baygon/executable.py ADDED
@@ -0,0 +1,117 @@
1
+ """Executable class. To be used with the Test class."""
2
+
3
+ import logging
4
+ import os
5
+ import shutil
6
+ import subprocess
7
+ import typing
8
+ from collections import namedtuple
9
+ from pathlib import Path
10
+
11
+ from .error import InvalidExecutableError
12
+
13
+ logger = logging.getLogger("baygon")
14
+
15
+ Outputs = namedtuple("Outputs", ["exit_status", "stdout", "stderr"])
16
+
17
+ forbidden_binaries = ["rm", "mv", "dd", "wget", "mkfs"]
18
+
19
+
20
+ def get_env(env: typing.Optional[str] = None) -> dict:
21
+ """Get the environment variables to be used for the subprocess."""
22
+ return {**os.environ, **(env or {})}
23
+
24
+
25
+ class Executable:
26
+ """An executable program.
27
+
28
+ Convenient execution and access to program outputs such as:
29
+
30
+ - Exit status
31
+ - Standard output
32
+ - Standard error
33
+
34
+ For example:
35
+
36
+ >>> e = Executable('echo')
37
+ >>> e
38
+ Executable<echo>
39
+ >>> e('-n', 'Hello World')
40
+ Outputs(exit_status=0, stdout='Hello World', stderr='')
41
+ >>> e('-n', 'Hello World').stdout
42
+ 'Hello World'
43
+ """
44
+
45
+ def __new__(cls, filename):
46
+ if isinstance(filename, cls):
47
+ return filename
48
+
49
+ return super().__new__(cls) if filename else None
50
+
51
+ def __init__(self, filename, encoding="utf-8"):
52
+ """Create an executable object.
53
+
54
+ :param filename: The path of the executable.
55
+ :param encoding: The encoding to be used for the outputs, default is UTF-8.
56
+ """
57
+ if isinstance(filename, self.__class__):
58
+ self.filename = filename.filename
59
+ self.encoding = filename.encoding
60
+ else:
61
+ self.filename = filename
62
+ self.encoding = encoding
63
+
64
+ if not self._is_executable(self.filename):
65
+ if "/" not in filename and shutil.which(filename) is not None:
66
+ if filename in forbidden_binaries:
67
+ raise InvalidExecutableError(f"Program '{filename}' is forbidden!")
68
+ filename = shutil.which(filename)
69
+ else:
70
+ raise InvalidExecutableError(
71
+ f"Program '{filename}' is not an executable!"
72
+ )
73
+
74
+ def run(self, *args, stdin=None, env=None, hook=None):
75
+ """Run the program and grab all the outputs."""
76
+
77
+ cmd = [self.filename, *[str(a) for a in args]]
78
+
79
+ with subprocess.Popen(
80
+ cmd,
81
+ stdout=subprocess.PIPE,
82
+ stdin=subprocess.PIPE,
83
+ stderr=subprocess.PIPE,
84
+ env=env,
85
+ ) as proc:
86
+
87
+ if stdin is not None:
88
+ stdin = stdin.encode(self.encoding)
89
+
90
+ stdout, stderr = proc.communicate(input=stdin)
91
+
92
+ if stdout is not None:
93
+ stdout = stdout.decode(self.encoding)
94
+ if stderr is not None:
95
+ stderr = stderr.decode(self.encoding)
96
+
97
+ if hook and callable(hook):
98
+ hook(
99
+ cmd=cmd,
100
+ stdin=stdin,
101
+ stdout=stdout,
102
+ stderr=stderr,
103
+ exit_status=proc.returncode,
104
+ )
105
+
106
+ return Outputs(proc.returncode, stdout, stderr)
107
+
108
+ def __call__(self, *args, **kwargs):
109
+ return self.run(*args, **kwargs)
110
+
111
+ def __repr__(self):
112
+ return f"{self.__class__.__name__}<{self.filename}>"
113
+
114
+ @staticmethod
115
+ def _is_executable(filename):
116
+ path = Path(filename)
117
+ return path.is_file() and os.access(path, os.X_OK)