tenzir-test 1.0.2__py3-none-any.whl → 1.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tenzir_test/cli.py +32 -1
- tenzir_test/run.py +215 -20
- tenzir_test/runners/__init__.py +1 -1
- tenzir_test/runners/custom_python_fixture_runner.py +2 -0
- tenzir_test/runners/ext_runner.py +13 -0
- tenzir_test/runners/shell_runner.py +2 -0
- {tenzir_test-1.0.2.dist-info → tenzir_test-1.1.1.dist-info}/METADATA +1 -1
- {tenzir_test-1.0.2.dist-info → tenzir_test-1.1.1.dist-info}/RECORD +11 -11
- {tenzir_test-1.0.2.dist-info → tenzir_test-1.1.1.dist-info}/WHEEL +0 -0
- {tenzir_test-1.0.2.dist-info → tenzir_test-1.1.1.dist-info}/entry_points.txt +0 -0
- {tenzir_test-1.0.2.dist-info → tenzir_test-1.1.1.dist-info}/licenses/LICENSE +0 -0
tenzir_test/cli.py
CHANGED
|
@@ -11,6 +11,15 @@ import click
|
|
|
11
11
|
from . import __version__, run as runtime
|
|
12
12
|
|
|
13
13
|
|
|
14
|
+
class _UnindentedEpilogCommand(click.Command):
|
|
15
|
+
"""Command that renders the epilog without indentation."""
|
|
16
|
+
|
|
17
|
+
def format_epilog(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
|
|
18
|
+
if self.epilog:
|
|
19
|
+
formatter.write_paragraph()
|
|
20
|
+
formatter.write(self.epilog)
|
|
21
|
+
|
|
22
|
+
|
|
14
23
|
def _normalize_exit_code(value: object) -> int:
|
|
15
24
|
"""Cast arbitrary exit codes to integers."""
|
|
16
25
|
|
|
@@ -22,7 +31,18 @@ def _normalize_exit_code(value: object) -> int:
|
|
|
22
31
|
|
|
23
32
|
|
|
24
33
|
@click.command(
|
|
34
|
+
cls=_UnindentedEpilogCommand,
|
|
25
35
|
context_settings={"help_option_names": ["-h", "--help"]},
|
|
36
|
+
epilog="""\
|
|
37
|
+
Examples:
|
|
38
|
+
tenzir-test Run all tests in the project
|
|
39
|
+
tenzir-test tests/alerts/ Run all tests in a directory
|
|
40
|
+
tenzir-test tests/basic.tql Run a specific test file
|
|
41
|
+
tenzir-test -u tests/new.tql Run test and update its baseline
|
|
42
|
+
tenzir-test -p -k tests/debug/ Debug with output streaming and kept temps
|
|
43
|
+
|
|
44
|
+
Documentation: https://docs.tenzir.com/reference/test-framework/
|
|
45
|
+
""",
|
|
26
46
|
)
|
|
27
47
|
@click.version_option(
|
|
28
48
|
__version__,
|
|
@@ -51,6 +71,7 @@ def _normalize_exit_code(value: object) -> int:
|
|
|
51
71
|
@click.argument(
|
|
52
72
|
"tests",
|
|
53
73
|
nargs=-1,
|
|
74
|
+
metavar="[TEST]...",
|
|
54
75
|
type=click.Path(path_type=Path, resolve_path=False),
|
|
55
76
|
)
|
|
56
77
|
@click.option("-u", "--update", is_flag=True, help="Update reference outputs.")
|
|
@@ -156,7 +177,17 @@ def cli(
|
|
|
156
177
|
verbose: bool,
|
|
157
178
|
all_projects: bool,
|
|
158
179
|
) -> int:
|
|
159
|
-
"""Execute
|
|
180
|
+
"""Execute test scenarios and compare output against baselines.
|
|
181
|
+
|
|
182
|
+
Discovers and runs tests under the project root, comparing actual output
|
|
183
|
+
against reference .txt files. Use --update to regenerate baselines.
|
|
184
|
+
|
|
185
|
+
\b
|
|
186
|
+
TEST paths can be:
|
|
187
|
+
- Individual test files (e.g., tests/basic.tql)
|
|
188
|
+
- Directories to run all tests within (e.g., tests/alerts/)
|
|
189
|
+
- Omitted to run all discovered tests in the project
|
|
190
|
+
"""
|
|
160
191
|
|
|
161
192
|
package_paths: list[Path] = []
|
|
162
193
|
for entry in package_dirs:
|
tenzir_test/run.py
CHANGED
|
@@ -587,6 +587,7 @@ def run_subprocess(
|
|
|
587
587
|
check: bool = False,
|
|
588
588
|
text: bool = False,
|
|
589
589
|
force_capture: bool = False,
|
|
590
|
+
stdin_data: bytes | None = None,
|
|
590
591
|
**kwargs: Any,
|
|
591
592
|
) -> subprocess.CompletedProcess[bytes] | subprocess.CompletedProcess[str]:
|
|
592
593
|
"""Execute a subprocess honoring passthrough configuration.
|
|
@@ -597,10 +598,30 @@ def run_subprocess(
|
|
|
597
598
|
|
|
598
599
|
Runner authors should prefer this helper over direct ``subprocess`` calls so
|
|
599
600
|
passthrough semantics remain consistent across implementations.
|
|
601
|
+
|
|
602
|
+
Args:
|
|
603
|
+
args: Command and arguments to execute.
|
|
604
|
+
capture_output: Whether to capture stdout/stderr (ignored in passthrough mode).
|
|
605
|
+
check: If True, raise CalledProcessError on non-zero exit.
|
|
606
|
+
text: If True, decode stdout/stderr as text. Note: stdin_data must still
|
|
607
|
+
be bytes even when text=True, as subprocess.run expects bytes for input.
|
|
608
|
+
force_capture: Capture output even in passthrough mode.
|
|
609
|
+
stdin_data: Bytes to send to the process's stdin. Use :func:`get_stdin_content`
|
|
610
|
+
to read from a .stdin file. When None, stdin is not connected.
|
|
611
|
+
**kwargs: Additional arguments passed to subprocess.run. The keys 'stdout',
|
|
612
|
+
'stderr', 'capture_output', and 'input' are managed by this function
|
|
613
|
+
and will raise TypeError if provided.
|
|
614
|
+
|
|
615
|
+
Returns:
|
|
616
|
+
A CompletedProcess instance with returncode, stdout, and stderr.
|
|
617
|
+
|
|
618
|
+
Raises:
|
|
619
|
+
TypeError: If stdout, stderr, capture_output, or input are passed in kwargs.
|
|
620
|
+
subprocess.CalledProcessError: If check=True and process exits non-zero.
|
|
600
621
|
"""
|
|
601
622
|
|
|
602
|
-
if any(key in kwargs for key in {"stdout", "stderr", "capture_output"}):
|
|
603
|
-
raise TypeError("run_subprocess manages stdout/stderr automatically")
|
|
623
|
+
if any(key in kwargs for key in {"stdout", "stderr", "capture_output", "input"}):
|
|
624
|
+
raise TypeError("run_subprocess manages stdout/stderr/input automatically")
|
|
604
625
|
|
|
605
626
|
passthrough = is_passthrough_enabled()
|
|
606
627
|
stream_output = passthrough and not force_capture
|
|
@@ -622,6 +643,7 @@ def run_subprocess(
|
|
|
622
643
|
stdout=stdout,
|
|
623
644
|
stderr=stderr,
|
|
624
645
|
text=text,
|
|
646
|
+
input=stdin_data,
|
|
625
647
|
**kwargs,
|
|
626
648
|
)
|
|
627
649
|
|
|
@@ -1144,11 +1166,10 @@ def _format_relative_path(path: Path, base: Path) -> str:
|
|
|
1144
1166
|
|
|
1145
1167
|
|
|
1146
1168
|
def _marker_for_selection(selection: ProjectSelection) -> str:
|
|
1169
|
+
is_package = packages.is_package_dir(selection.root)
|
|
1147
1170
|
if selection.kind == "root":
|
|
1148
|
-
return "■"
|
|
1149
|
-
if
|
|
1150
|
-
return "○"
|
|
1151
|
-
return "□"
|
|
1171
|
+
return "●" if is_package else "■"
|
|
1172
|
+
return "○" if is_package else "□"
|
|
1152
1173
|
|
|
1153
1174
|
|
|
1154
1175
|
def _print_execution_plan(plan: ExecutionPlan, *, display_base: Path) -> int:
|
|
@@ -1166,8 +1187,12 @@ def _print_execution_plan(plan: ExecutionPlan, *, display_base: Path) -> int:
|
|
|
1166
1187
|
return 0
|
|
1167
1188
|
|
|
1168
1189
|
print(f"{INFO} found {len(active)} projects")
|
|
1190
|
+
root_base = plan.root.root
|
|
1169
1191
|
for marker, selection in active:
|
|
1170
|
-
|
|
1192
|
+
if selection.kind == "satellite":
|
|
1193
|
+
name = _format_relative_path(selection.root, root_base)
|
|
1194
|
+
else:
|
|
1195
|
+
name = selection.root.name or selection.root.as_posix()
|
|
1171
1196
|
print(f"{INFO} {marker} {name}")
|
|
1172
1197
|
return len(active)
|
|
1173
1198
|
|
|
@@ -1198,6 +1223,7 @@ def _default_test_config() -> TestConfig:
|
|
|
1198
1223
|
"retry": 1,
|
|
1199
1224
|
"suite": None,
|
|
1200
1225
|
"package_dirs": tuple(),
|
|
1226
|
+
"pre_compare": tuple(),
|
|
1201
1227
|
}
|
|
1202
1228
|
|
|
1203
1229
|
|
|
@@ -1206,13 +1232,17 @@ def _canonical_config_key(key: str) -> str:
|
|
|
1206
1232
|
return "fixtures"
|
|
1207
1233
|
if key in {"package_dirs", "package-dirs"}:
|
|
1208
1234
|
return "package_dirs"
|
|
1235
|
+
if key in {"pre_compare", "pre-compare"}:
|
|
1236
|
+
return "pre_compare"
|
|
1209
1237
|
return key
|
|
1210
1238
|
|
|
1211
1239
|
|
|
1212
1240
|
ConfigOrigin = Literal["directory", "frontmatter"]
|
|
1213
1241
|
|
|
1214
1242
|
|
|
1215
|
-
def _raise_config_error(
|
|
1243
|
+
def _raise_config_error(
|
|
1244
|
+
location: Path | str, message: str, line_number: int | None = None
|
|
1245
|
+
) -> typing.NoReturn:
|
|
1216
1246
|
base = str(location)
|
|
1217
1247
|
if line_number is not None:
|
|
1218
1248
|
base = f"{base}:{line_number}"
|
|
@@ -1243,7 +1273,6 @@ def _normalize_fixtures_value(
|
|
|
1243
1273
|
f"Invalid value for 'fixtures', expected string or list, got '{value}'",
|
|
1244
1274
|
line_number,
|
|
1245
1275
|
)
|
|
1246
|
-
return tuple()
|
|
1247
1276
|
|
|
1248
1277
|
fixtures: list[str] = []
|
|
1249
1278
|
for entry in raw:
|
|
@@ -1304,7 +1333,6 @@ def _normalize_inputs_value(
|
|
|
1304
1333
|
f"Invalid value for 'inputs', expected string, got '{value}'",
|
|
1305
1334
|
line_number,
|
|
1306
1335
|
)
|
|
1307
|
-
return None
|
|
1308
1336
|
|
|
1309
1337
|
|
|
1310
1338
|
def _normalize_package_dirs_value(
|
|
@@ -1321,7 +1349,6 @@ def _normalize_package_dirs_value(
|
|
|
1321
1349
|
f"Invalid value for 'package-dirs', expected list of strings, got '{value}'",
|
|
1322
1350
|
line_number,
|
|
1323
1351
|
)
|
|
1324
|
-
return tuple()
|
|
1325
1352
|
base_dir = _extract_location_path(location).parent
|
|
1326
1353
|
normalized: list[str] = []
|
|
1327
1354
|
for entry in value:
|
|
@@ -1331,7 +1358,6 @@ def _normalize_package_dirs_value(
|
|
|
1331
1358
|
f"Invalid package-dirs entry '{entry}', expected string",
|
|
1332
1359
|
line_number,
|
|
1333
1360
|
)
|
|
1334
|
-
continue
|
|
1335
1361
|
raw = os.fspath(entry).strip()
|
|
1336
1362
|
if not raw:
|
|
1337
1363
|
_raise_config_error(
|
|
@@ -1339,7 +1365,6 @@ def _normalize_package_dirs_value(
|
|
|
1339
1365
|
"Invalid package-dirs entry: must be non-empty string",
|
|
1340
1366
|
line_number,
|
|
1341
1367
|
)
|
|
1342
|
-
continue
|
|
1343
1368
|
path = Path(raw)
|
|
1344
1369
|
if not path.is_absolute():
|
|
1345
1370
|
path = base_dir / path
|
|
@@ -1351,6 +1376,58 @@ def _normalize_package_dirs_value(
|
|
|
1351
1376
|
return tuple(normalized)
|
|
1352
1377
|
|
|
1353
1378
|
|
|
1379
|
+
def _normalize_pre_compare_value(
|
|
1380
|
+
value: typing.Any,
|
|
1381
|
+
*,
|
|
1382
|
+
location: Path | str,
|
|
1383
|
+
line_number: int | None = None,
|
|
1384
|
+
) -> tuple[str, ...]:
|
|
1385
|
+
entries: typing.Any
|
|
1386
|
+
if isinstance(value, list):
|
|
1387
|
+
entries = value
|
|
1388
|
+
elif isinstance(value, str):
|
|
1389
|
+
try:
|
|
1390
|
+
parsed = yaml.safe_load(value)
|
|
1391
|
+
except yaml.YAMLError:
|
|
1392
|
+
parsed = None
|
|
1393
|
+
if isinstance(parsed, list):
|
|
1394
|
+
entries = parsed
|
|
1395
|
+
else:
|
|
1396
|
+
entries = [value]
|
|
1397
|
+
else:
|
|
1398
|
+
_raise_config_error(
|
|
1399
|
+
location,
|
|
1400
|
+
f"Invalid value for 'pre-compare', expected string or list, got '{value}'",
|
|
1401
|
+
line_number,
|
|
1402
|
+
)
|
|
1403
|
+
|
|
1404
|
+
transforms: list[str] = []
|
|
1405
|
+
valid_names = set(_TRANSFORMS.keys())
|
|
1406
|
+
for entry in entries:
|
|
1407
|
+
if not isinstance(entry, str):
|
|
1408
|
+
_raise_config_error(
|
|
1409
|
+
location,
|
|
1410
|
+
f"Invalid pre-compare entry '{entry}', expected string",
|
|
1411
|
+
line_number,
|
|
1412
|
+
)
|
|
1413
|
+
name = entry.strip()
|
|
1414
|
+
if not name:
|
|
1415
|
+
_raise_config_error(
|
|
1416
|
+
location,
|
|
1417
|
+
"Pre-compare transform names must be non-empty strings",
|
|
1418
|
+
line_number,
|
|
1419
|
+
)
|
|
1420
|
+
if name not in valid_names:
|
|
1421
|
+
valid_list = ", ".join(sorted(valid_names))
|
|
1422
|
+
_raise_config_error(
|
|
1423
|
+
location,
|
|
1424
|
+
f"Unknown pre-compare transform '{name}', valid transforms: {valid_list}",
|
|
1425
|
+
line_number,
|
|
1426
|
+
)
|
|
1427
|
+
transforms.append(name)
|
|
1428
|
+
return tuple(transforms)
|
|
1429
|
+
|
|
1430
|
+
|
|
1354
1431
|
def _assign_config_option(
|
|
1355
1432
|
config: TestConfig,
|
|
1356
1433
|
key: str,
|
|
@@ -1370,6 +1447,7 @@ def _assign_config_option(
|
|
|
1370
1447
|
"inputs",
|
|
1371
1448
|
"retry",
|
|
1372
1449
|
"package_dirs",
|
|
1450
|
+
"pre_compare",
|
|
1373
1451
|
}
|
|
1374
1452
|
if origin == "directory":
|
|
1375
1453
|
valid_keys.add("suite")
|
|
@@ -1416,7 +1494,6 @@ def _assign_config_option(
|
|
|
1416
1494
|
f"Invalid value for '{canonical}', expected 'true' or 'false', got '{value}'",
|
|
1417
1495
|
line_number,
|
|
1418
1496
|
)
|
|
1419
|
-
return
|
|
1420
1497
|
|
|
1421
1498
|
if canonical == "timeout":
|
|
1422
1499
|
if isinstance(value, int):
|
|
@@ -1429,7 +1506,6 @@ def _assign_config_option(
|
|
|
1429
1506
|
f"Invalid value for 'timeout', expected integer, got '{value}'",
|
|
1430
1507
|
line_number,
|
|
1431
1508
|
)
|
|
1432
|
-
return
|
|
1433
1509
|
if timeout_value <= 0:
|
|
1434
1510
|
_raise_config_error(
|
|
1435
1511
|
location,
|
|
@@ -1485,7 +1561,6 @@ def _assign_config_option(
|
|
|
1485
1561
|
f"Invalid value for 'retry', expected integer, got '{value}'",
|
|
1486
1562
|
line_number,
|
|
1487
1563
|
)
|
|
1488
|
-
return
|
|
1489
1564
|
if retry_value <= 0:
|
|
1490
1565
|
_raise_config_error(
|
|
1491
1566
|
location,
|
|
@@ -1495,6 +1570,11 @@ def _assign_config_option(
|
|
|
1495
1570
|
config[canonical] = retry_value
|
|
1496
1571
|
return
|
|
1497
1572
|
|
|
1573
|
+
if canonical == "pre_compare":
|
|
1574
|
+
transforms = _normalize_pre_compare_value(value, location=location, line_number=line_number)
|
|
1575
|
+
config[canonical] = transforms
|
|
1576
|
+
return
|
|
1577
|
+
|
|
1498
1578
|
if canonical == "runner":
|
|
1499
1579
|
if not isinstance(value, str):
|
|
1500
1580
|
_raise_config_error(
|
|
@@ -1857,10 +1937,18 @@ def get_test_env_and_config_args(
|
|
|
1857
1937
|
candidate = test.parent / candidate
|
|
1858
1938
|
inputs_path = str(candidate.resolve())
|
|
1859
1939
|
env["TENZIR_INPUTS"] = inputs_path
|
|
1860
|
-
# Check for inline input file (.input extension)
|
|
1940
|
+
# Check for inline input file (.input extension).
|
|
1941
|
+
# Note: .input and .stdin extensions are reserved; see RESERVED_EXTENSIONS
|
|
1942
|
+
# in tenzir_test.runners.ext_runner for the authoritative list.
|
|
1861
1943
|
inline_input = test.with_suffix(".input")
|
|
1862
1944
|
if inline_input.is_file():
|
|
1863
|
-
|
|
1945
|
+
validated_input = _validate_path_within_root(inline_input, "input file")
|
|
1946
|
+
env["TENZIR_INPUT"] = str(validated_input)
|
|
1947
|
+
# Check for stdin file (.stdin extension)
|
|
1948
|
+
stdin_file = test.with_suffix(".stdin")
|
|
1949
|
+
if stdin_file.is_file():
|
|
1950
|
+
validated_stdin = _validate_path_within_root(stdin_file, "stdin file")
|
|
1951
|
+
env["TENZIR_STDIN"] = str(validated_stdin)
|
|
1864
1952
|
if config_file.exists():
|
|
1865
1953
|
env.setdefault("TENZIR_CONFIG", str(config_file))
|
|
1866
1954
|
if node_config_file.exists():
|
|
@@ -1875,6 +1963,32 @@ def get_test_env_and_config_args(
|
|
|
1875
1963
|
return env, config_args
|
|
1876
1964
|
|
|
1877
1965
|
|
|
1966
|
+
def _validate_path_within_root(path: Path, description: str) -> Path:
|
|
1967
|
+
"""Validate that a resolved path is within the project ROOT.
|
|
1968
|
+
|
|
1969
|
+
This provides defense-in-depth against symlink traversal attacks where a
|
|
1970
|
+
malicious .stdin or .input file could be a symlink pointing outside the
|
|
1971
|
+
project directory.
|
|
1972
|
+
|
|
1973
|
+
Args:
|
|
1974
|
+
path: The path to validate (will be resolved to follow symlinks).
|
|
1975
|
+
description: Human-readable description for error messages (e.g., "stdin file").
|
|
1976
|
+
|
|
1977
|
+
Returns:
|
|
1978
|
+
The resolved path if validation passes.
|
|
1979
|
+
|
|
1980
|
+
Raises:
|
|
1981
|
+
RuntimeError: If the resolved path is outside the project root.
|
|
1982
|
+
"""
|
|
1983
|
+
resolved = path.resolve()
|
|
1984
|
+
try:
|
|
1985
|
+
resolved.relative_to(ROOT.resolve())
|
|
1986
|
+
except ValueError:
|
|
1987
|
+
rel_path = _relativize_path(path)
|
|
1988
|
+
raise RuntimeError(f"{description} '{rel_path}' resolves outside project root") from None
|
|
1989
|
+
return resolved
|
|
1990
|
+
|
|
1991
|
+
|
|
1878
1992
|
def _apply_fixture_env(env: dict[str, str], fixtures: tuple[str, ...]) -> None:
|
|
1879
1993
|
if fixtures:
|
|
1880
1994
|
env["TENZIR_TEST_FIXTURES"] = ",".join(fixtures)
|
|
@@ -1882,6 +1996,45 @@ def _apply_fixture_env(env: dict[str, str], fixtures: tuple[str, ...]) -> None:
|
|
|
1882
1996
|
env.pop("TENZIR_TEST_FIXTURES", None)
|
|
1883
1997
|
|
|
1884
1998
|
|
|
1999
|
+
def get_stdin_content(env: dict[str, str]) -> bytes | None:
|
|
2000
|
+
"""Read stdin content from a file specified by TENZIR_STDIN environment variable.
|
|
2001
|
+
|
|
2002
|
+
This function is intended for runner authors who need to provide stdin content
|
|
2003
|
+
to test processes. The TENZIR_STDIN variable is automatically set by the test
|
|
2004
|
+
harness when a ``.stdin`` file exists alongside the test file.
|
|
2005
|
+
|
|
2006
|
+
Args:
|
|
2007
|
+
env: Environment dictionary, typically from :func:`get_test_env_and_config_args`.
|
|
2008
|
+
|
|
2009
|
+
Returns:
|
|
2010
|
+
The file contents as bytes if TENZIR_STDIN is set and the file exists,
|
|
2011
|
+
otherwise None. Empty files return empty bytes (b"").
|
|
2012
|
+
|
|
2013
|
+
Raises:
|
|
2014
|
+
RuntimeError: If TENZIR_STDIN is set but the file cannot be read
|
|
2015
|
+
(e.g., file not found, permission denied).
|
|
2016
|
+
|
|
2017
|
+
Example:
|
|
2018
|
+
Runner authors can use this with :func:`run_subprocess`::
|
|
2019
|
+
|
|
2020
|
+
env, config_args = get_test_env_and_config_args(test)
|
|
2021
|
+
stdin_content = get_stdin_content(env)
|
|
2022
|
+
completed = run_subprocess(
|
|
2023
|
+
["my-command"],
|
|
2024
|
+
env=env,
|
|
2025
|
+
stdin_data=stdin_content,
|
|
2026
|
+
...
|
|
2027
|
+
)
|
|
2028
|
+
"""
|
|
2029
|
+
stdin_path = env.get("TENZIR_STDIN")
|
|
2030
|
+
if not stdin_path:
|
|
2031
|
+
return None
|
|
2032
|
+
try:
|
|
2033
|
+
return Path(stdin_path).read_bytes()
|
|
2034
|
+
except OSError as e:
|
|
2035
|
+
raise RuntimeError(f"Failed to read stdin file '{stdin_path}': {e.strerror}") from e
|
|
2036
|
+
|
|
2037
|
+
|
|
1885
2038
|
def set_debug_logging(enabled: bool) -> None:
|
|
1886
2039
|
global _debug_logging
|
|
1887
2040
|
_debug_logging = enabled
|
|
@@ -2795,6 +2948,43 @@ def _format_lines_changed(total: int) -> str:
|
|
|
2795
2948
|
return f"{_BLOCK_INDENT}└ {total} {line} changed"
|
|
2796
2949
|
|
|
2797
2950
|
|
|
2951
|
+
def _transform_sort(output: bytes) -> bytes:
|
|
2952
|
+
"""Sort output lines lexicographically.
|
|
2953
|
+
|
|
2954
|
+
Uses surrogateescape to preserve undecodable bytes as surrogate escapes,
|
|
2955
|
+
allowing the transform to handle binary data gracefully.
|
|
2956
|
+
"""
|
|
2957
|
+
if not output:
|
|
2958
|
+
return output
|
|
2959
|
+
has_trailing_newline = output.endswith(b"\n")
|
|
2960
|
+
text = output.decode("utf-8", errors="surrogateescape")
|
|
2961
|
+
lines = text.splitlines(keepends=False)
|
|
2962
|
+
sorted_lines = sorted(lines)
|
|
2963
|
+
result = "\n".join(sorted_lines)
|
|
2964
|
+
if has_trailing_newline:
|
|
2965
|
+
result += "\n"
|
|
2966
|
+
return result.encode("utf-8", errors="surrogateescape")
|
|
2967
|
+
|
|
2968
|
+
|
|
2969
|
+
# Transforms are intentionally simple and hardcoded rather than using the plugin
|
|
2970
|
+
# architecture (like runners and fixtures). Rationale:
|
|
2971
|
+
# - Transforms are core comparison utilities, not user-extensible features
|
|
2972
|
+
# - Currently only one transform exists; extensibility can be added if needed
|
|
2973
|
+
# - Pre-compare transforms are rarely customized per-project compared to runners
|
|
2974
|
+
# - If custom transforms become necessary, this can be refactored to use a
|
|
2975
|
+
# registry pattern similar to runners/__init__.py
|
|
2976
|
+
_TRANSFORMS: dict[str, typing.Callable[[bytes], bytes]] = {
|
|
2977
|
+
"sort": _transform_sort,
|
|
2978
|
+
}
|
|
2979
|
+
|
|
2980
|
+
|
|
2981
|
+
def apply_pre_compare(output: bytes, transforms: tuple[str, ...]) -> bytes:
|
|
2982
|
+
"""Apply pre-compare transforms in order."""
|
|
2983
|
+
for name in transforms:
|
|
2984
|
+
output = _TRANSFORMS[name](output)
|
|
2985
|
+
return output
|
|
2986
|
+
|
|
2987
|
+
|
|
2798
2988
|
def print_diff(expected: bytes, actual: bytes, path: Path) -> None:
|
|
2799
2989
|
if should_suppress_failure_output():
|
|
2800
2990
|
return
|
|
@@ -2960,12 +3150,14 @@ def run_simple_test(
|
|
|
2960
3150
|
"-f",
|
|
2961
3151
|
str(test),
|
|
2962
3152
|
]
|
|
3153
|
+
stdin_content = get_stdin_content(env)
|
|
2963
3154
|
completed = run_subprocess(
|
|
2964
3155
|
cmd,
|
|
2965
3156
|
timeout=timeout,
|
|
2966
3157
|
env=env,
|
|
2967
3158
|
capture_output=not passthrough_mode,
|
|
2968
3159
|
cwd=str(ROOT),
|
|
3160
|
+
stdin_data=stdin_content,
|
|
2969
3161
|
)
|
|
2970
3162
|
good = completed.returncode == 0
|
|
2971
3163
|
output = b""
|
|
@@ -3038,12 +3230,15 @@ def run_simple_test(
|
|
|
3038
3230
|
return False
|
|
3039
3231
|
log_comparison(test, ref_path, mode="comparing")
|
|
3040
3232
|
expected = ref_path.read_bytes()
|
|
3041
|
-
|
|
3233
|
+
pre_compare = cast(tuple[str, ...], test_config.get("pre_compare", tuple()))
|
|
3234
|
+
expected_transformed = apply_pre_compare(expected, pre_compare)
|
|
3235
|
+
output_transformed = apply_pre_compare(output, pre_compare)
|
|
3236
|
+
if expected_transformed != output_transformed:
|
|
3042
3237
|
if interrupt_requested():
|
|
3043
3238
|
report_interrupted_test(test)
|
|
3044
3239
|
else:
|
|
3045
3240
|
report_failure(test, "")
|
|
3046
|
-
print_diff(
|
|
3241
|
+
print_diff(expected_transformed, output_transformed, ref_path)
|
|
3047
3242
|
return False
|
|
3048
3243
|
success(test)
|
|
3049
3244
|
return True
|
tenzir_test/runners/__init__.py
CHANGED
|
@@ -133,7 +133,7 @@ def get_runner_for_test(test_path: Path) -> Runner:
|
|
|
133
133
|
runner_name = runner_value
|
|
134
134
|
if runner_name in RUNNERS_BY_NAME:
|
|
135
135
|
return RUNNERS_BY_NAME[runner_name]
|
|
136
|
-
raise ValueError(f"
|
|
136
|
+
raise ValueError(f"runner '{runner_name}' not found; define it in <project>/runners/")
|
|
137
137
|
|
|
138
138
|
|
|
139
139
|
register(ShellRunner())
|
|
@@ -106,12 +106,14 @@ class CustomPythonFixture(ExtRunner):
|
|
|
106
106
|
env["TENZIR_PYTHON_FIXTURE_TIMEOUT"] = str(timeout)
|
|
107
107
|
if node_requested and endpoint:
|
|
108
108
|
env["TENZIR_PYTHON_FIXTURE_ENDPOINT"] = endpoint
|
|
109
|
+
stdin_content = run_mod.get_stdin_content(env)
|
|
109
110
|
completed = run_mod.run_subprocess(
|
|
110
111
|
cmd,
|
|
111
112
|
timeout=timeout,
|
|
112
113
|
env=env,
|
|
113
114
|
capture_output=not passthrough,
|
|
114
115
|
check=True,
|
|
116
|
+
stdin_data=stdin_content,
|
|
115
117
|
)
|
|
116
118
|
ref_path = test.with_suffix(".txt")
|
|
117
119
|
if completed.returncode != 0:
|
|
@@ -5,9 +5,22 @@ from pathlib import Path
|
|
|
5
5
|
from ._utils import get_run_module
|
|
6
6
|
from .runner import Runner
|
|
7
7
|
|
|
8
|
+
# Extensions reserved by the framework and not available for custom runners.
|
|
9
|
+
# These extensions have special meaning in the test harness:
|
|
10
|
+
# - .txt: baseline output file for comparison
|
|
11
|
+
# - .input: inline input data exposed via TENZIR_INPUT env var
|
|
12
|
+
# - .stdin: stdin content fed to test process via stdin_data parameter
|
|
13
|
+
#
|
|
14
|
+
# Custom runner authors who need stdin support should use get_stdin_content()
|
|
15
|
+
# and pass the result to run_subprocess(stdin_data=...). See the ShellRunner
|
|
16
|
+
# and CustomPythonFixture implementations for reference.
|
|
17
|
+
RESERVED_EXTENSIONS = frozenset({"input", "stdin", "txt"})
|
|
18
|
+
|
|
8
19
|
|
|
9
20
|
class ExtRunner(Runner):
|
|
10
21
|
def __init__(self, *, name: str, ext: str) -> None:
|
|
22
|
+
if ext in RESERVED_EXTENSIONS:
|
|
23
|
+
raise ValueError(f"runner '{name}' uses reserved extension '.{ext}'")
|
|
11
24
|
super().__init__(name=name)
|
|
12
25
|
self._ext = ext
|
|
13
26
|
|
|
@@ -49,6 +49,7 @@ class ShellRunner(ExtRunner):
|
|
|
49
49
|
env["PATH"] = shell_path_prefix
|
|
50
50
|
|
|
51
51
|
try:
|
|
52
|
+
stdin_content = run_mod.get_stdin_content(env)
|
|
52
53
|
completed = run_mod.run_subprocess(
|
|
53
54
|
["sh", "-eu", str(test)],
|
|
54
55
|
env=env,
|
|
@@ -57,6 +58,7 @@ class ShellRunner(ExtRunner):
|
|
|
57
58
|
check=not expect_error,
|
|
58
59
|
text=False,
|
|
59
60
|
cwd=str(run_mod.ROOT),
|
|
61
|
+
stdin_data=stdin_content,
|
|
60
62
|
)
|
|
61
63
|
except subprocess.CalledProcessError as exc:
|
|
62
64
|
completed = exc # treat like CompletedProcess for diagnostics
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tenzir-test
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.1
|
|
4
4
|
Summary: Reusable test execution framework extracted from the Tenzir repository.
|
|
5
5
|
Project-URL: Homepage, https://github.com/tenzir/test
|
|
6
6
|
Project-URL: Repository, https://github.com/tenzir/test
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
tenzir_test/__init__.py,sha256=k7V6Pbjaa8SAy6t4KnaauHTyfnyVEwc1VGtH823MANU,1181
|
|
2
2
|
tenzir_test/_python_runner.py,sha256=ZODuZ6ll21gxBRj34iq-7lOmZOiqzFyu7wnPnS0K648,3130
|
|
3
|
-
tenzir_test/cli.py,sha256=
|
|
3
|
+
tenzir_test/cli.py,sha256=RfhHKji02CDG1OQU0_ZpCqyEuAOYJwEEZeDD_hqayHg,7438
|
|
4
4
|
tenzir_test/config.py,sha256=z4ayS62SfOLNwrEgNktVeulyQ2QW4KUlN1KX0Za1NDM,2110
|
|
5
5
|
tenzir_test/packages.py,sha256=cTCQdGjCS1XmuKyiwh0ew-z9tHn6J-xZ6nvBP-hU8bc,948
|
|
6
6
|
tenzir_test/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
-
tenzir_test/run.py,sha256=
|
|
7
|
+
tenzir_test/run.py,sha256=EhMzfiJN7MjY6Dt5nQXkCnzq6RvlMktD0HrEEaW1yVc,140759
|
|
8
8
|
tenzir_test/engine/__init__.py,sha256=5APwy90YDm7rmL_qCZfToAcfbQthcZ8yV2_ExXKqaqE,110
|
|
9
9
|
tenzir_test/engine/operations.py,sha256=OCYjuMHyMAaay4s08u2Sl7oE-PmgeXumylp7R8GYIH4,950
|
|
10
10
|
tenzir_test/engine/registry.py,sha256=LXCr6TGlv1sR1m1eboTk7SrbS2IVErc3PqUuHxGA2xk,594
|
|
@@ -12,17 +12,17 @@ tenzir_test/engine/state.py,sha256=zolKmUWpTEajoq4gY8KYv_puCeS_DOpOTExBwLulNjo,8
|
|
|
12
12
|
tenzir_test/engine/worker.py,sha256=WwIkx1m_ANNveQjNisy5-qpbUZl_DfDXxVIfdxABKjc,140
|
|
13
13
|
tenzir_test/fixtures/__init__.py,sha256=H8deAoYg7HMPsZZ2UTO2KE1N82CbCQqN0bXk8q7B9ww,18441
|
|
14
14
|
tenzir_test/fixtures/node.py,sha256=iNT3H0y_9g-CzJO7-EPrAYBOX69O-g3lnuQiAvGqThc,11524
|
|
15
|
-
tenzir_test/runners/__init__.py,sha256=
|
|
15
|
+
tenzir_test/runners/__init__.py,sha256=q68F75vsPwUrnzRZzu2PMuQystpVUFhmsZQPPxfvBtU,4633
|
|
16
16
|
tenzir_test/runners/_utils.py,sha256=BWv7UEPGa01l4tGTCg5i_22NblIyRw8vjk_5NIf1x_c,467
|
|
17
|
-
tenzir_test/runners/custom_python_fixture_runner.py,sha256=
|
|
17
|
+
tenzir_test/runners/custom_python_fixture_runner.py,sha256=xTc0x8_Y5DhkCz85ePmrpoWWAVEnzD9ZiFPNxEKppFY,7438
|
|
18
18
|
tenzir_test/runners/diff_runner.py,sha256=zRLeoqwqLdIpP5weq20K0VQdIXPy7QFmZ462Hlg0L0k,5477
|
|
19
|
-
tenzir_test/runners/ext_runner.py,sha256=
|
|
19
|
+
tenzir_test/runners/ext_runner.py,sha256=jO-eDqzCQY1pM_5W2Uvci8xy-oTel1kVA2hDSAo-05w,1449
|
|
20
20
|
tenzir_test/runners/runner.py,sha256=LtlD8huQOSmD7RyYDnKeCuI4Y6vhxGXMKsHA2qgfWN0,989
|
|
21
|
-
tenzir_test/runners/shell_runner.py,sha256=
|
|
21
|
+
tenzir_test/runners/shell_runner.py,sha256=vWCNTWQGVoatM98jN1UJaVrZJR2oPnqpPUs0-ud7rrc,6185
|
|
22
22
|
tenzir_test/runners/tenzir_runner.py,sha256=464FFYS_mh6l-ehccc-S8cIUO1MxdapwQL5X3PmMkMI,1006
|
|
23
23
|
tenzir_test/runners/tql_runner.py,sha256=2ZLMf3TIKwcOvaOFrVvvhzK-EcWmGOUZxKkbSoByyQA,248
|
|
24
|
-
tenzir_test-1.
|
|
25
|
-
tenzir_test-1.
|
|
26
|
-
tenzir_test-1.
|
|
27
|
-
tenzir_test-1.
|
|
28
|
-
tenzir_test-1.
|
|
24
|
+
tenzir_test-1.1.1.dist-info/METADATA,sha256=0pKON62eJrS_LulcE-yujdlnzg6v6ZR7Nq79FWUZs3I,3065
|
|
25
|
+
tenzir_test-1.1.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
26
|
+
tenzir_test-1.1.1.dist-info/entry_points.txt,sha256=l8DJgiEVrjScdTTo613cZ3PKodOmqrUVIbz-3awfV8w,53
|
|
27
|
+
tenzir_test-1.1.1.dist-info/licenses/LICENSE,sha256=ajMbpcBiSTXI8Rr4t17pvowV-On8DktghfZKxY_A22Q,10750
|
|
28
|
+
tenzir_test-1.1.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|