mcward-cli 1.2.1__tar.gz → 1.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/PKG-INFO +1 -1
  2. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/__init__.py +2 -1
  3. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/commands/__init__.py +2 -0
  4. mcward_cli-1.4.0/src/mcward/cli/commands/bench.py +127 -0
  5. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/commands/daemon.py +14 -24
  6. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/commands/list.py +2 -12
  7. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/commands/test.py +23 -30
  8. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/datapacks.py +41 -17
  9. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/environments.py +12 -6
  10. mcward_cli-1.4.0/src/mcward/cli/reporters/bench.py +60 -0
  11. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/reporters/github.py +3 -1
  12. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/reporters/live.py +2 -1
  13. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/reports/__init__.py +60 -7
  14. mcward_cli-1.4.0/src/mcward/cli/reports/coverage.css +92 -0
  15. mcward_cli-1.4.0/src/mcward/cli/reports/coverage.js +131 -0
  16. mcward_cli-1.4.0/src/mcward/cli/reports/html.py +260 -0
  17. mcward_cli-1.4.0/src/mcward/cli/session.py +62 -0
  18. mcward_cli-1.4.0/tests/conftest.py +10 -0
  19. mcward_cli-1.4.0/tests/test_bench.py +81 -0
  20. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/tests/test_discovery.py +33 -2
  21. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/tests/test_github.py +14 -0
  22. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/tests/test_reports.py +85 -1
  23. mcward_cli-1.2.1/src/mcward/cli/reports/html.py +0 -491
  24. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/.gitignore +0 -0
  25. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/README.md +0 -0
  26. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/pyproject.toml +0 -0
  27. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/commands/install.py +0 -0
  28. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/reporters/__init__.py +0 -0
  29. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/reporters/coverage.py +0 -0
  30. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/reports/junit.py +0 -0
  31. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/reports/lcov.py +0 -0
  32. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/src/mcward/cli/ui.py +0 -0
  33. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/tests/test_cli.py +0 -0
  34. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/tests/test_coverage_report.py +0 -0
  35. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/tests/test_environments.py +0 -0
  36. {mcward_cli-1.2.1 → mcward_cli-1.4.0}/tests/test_live.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: mcward-cli
3
- Version: 1.2.1
3
+ Version: 1.4.0
4
4
  Summary: Command-line interface for Ward, a testing framework for Minecraft datapacks using mcfunction.
5
5
  Author-email: Aksiome <54895777+aksiome@users.noreply.github.com>
6
6
  License: MPL-2.0
@@ -4,7 +4,7 @@ import sys
4
4
 
5
5
  import rich_click as click
6
6
 
7
- from .commands import clean, install, list_versions, start, status, stop, test
7
+ from .commands import bench, clean, install, list_versions, start, status, stop, test
8
8
  from .ui import console
9
9
 
10
10
  click.rich_click.TEXT_MARKUP = "rich"
@@ -36,6 +36,7 @@ def cli() -> None:
36
36
  pass
37
37
 
38
38
 
39
+ cli.add_command(bench)
39
40
  cli.add_command(clean)
40
41
  cli.add_command(install)
41
42
  cli.add_command(list_versions)
@@ -1,11 +1,13 @@
1
1
  """CLI commands."""
2
2
 
3
+ from .bench import bench
3
4
  from .daemon import start, status, stop
4
5
  from .install import clean, install
5
6
  from .list import list_versions
6
7
  from .test import test
7
8
 
8
9
  __all__ = [
10
+ "bench",
9
11
  "clean",
10
12
  "install",
11
13
  "list_versions",
@@ -0,0 +1,127 @@
1
+ """The bench command."""
2
+
3
+ import json
4
+ import sys
5
+ from pathlib import Path
6
+
7
+ import rich_click as click
8
+
9
+ from mcward import BenchReport, WardError, run_bench
10
+
11
+ from ..datapacks import DEFAULT_PATTERNS, discover_datapacks, format_range
12
+ from ..environments import manager, select_compatible, start_environments
13
+ from ..reporters.bench import render_report, render_summary
14
+ from ..ui import console
15
+
16
+
17
+ @click.command()
18
+ @click.option(
19
+ "--pack",
20
+ "-p",
21
+ "packs",
22
+ multiple=True,
23
+ help="Datapack paths or glob patterns",
24
+ )
25
+ @click.option(
26
+ "--version",
27
+ "-v",
28
+ "versions",
29
+ multiple=True,
30
+ help="Minecraft version(s) to bench on, one after the other",
31
+ )
32
+ @click.option(
33
+ "--setup",
34
+ multiple=True,
35
+ metavar="COMMAND",
36
+ help="Run once before each benchmarked command (repeatable)",
37
+ )
38
+ @click.option(
39
+ "--prepare",
40
+ multiple=True,
41
+ metavar="COMMAND",
42
+ help="Run before each timed batch, outside of the timing (repeatable)",
43
+ )
44
+ @click.option(
45
+ "--batch",
46
+ type=click.IntRange(min=1),
47
+ default=None,
48
+ help="Runs per sample. By default the warmup picks it",
49
+ )
50
+ @click.option(
51
+ "--json",
52
+ "json_path",
53
+ type=click.Path(dir_okay=False, writable=True, path_type=Path),
54
+ default=None,
55
+ help="Write the results, with every sample, to a JSON file",
56
+ )
57
+ @click.argument("commands", nargs=-1, required=True)
58
+ def bench(
59
+ versions: tuple[str, ...],
60
+ packs: tuple[str, ...],
61
+ setup: tuple[str, ...],
62
+ prepare: tuple[str, ...],
63
+ batch: int | None,
64
+ json_path: Path | None,
65
+ commands: tuple[str, ...],
66
+ ) -> None:
67
+ """Time commands on a test server and compare them."""
68
+ datapacks = discover_datapacks(packs if packs else DEFAULT_PATTERNS)
69
+ if not datapacks and not versions:
70
+ raise click.ClickException("Datapack not found. Without a pack, give a version with -v")
71
+
72
+ selected = versions or select_compatible(*format_range(datapacks))
73
+ paths = [datapack.path for datapack in datapacks]
74
+ results = []
75
+ failed = False
76
+
77
+ try:
78
+ envs = start_environments([manager.get(v) for v in selected])
79
+ for env in envs:
80
+ console.print()
81
+ if len(envs) > 1:
82
+ console.print(f"[bold]{env.version.name}[/]", highlight=False)
83
+
84
+ reports: list[BenchReport] = []
85
+ with console.status("Running the benchmarks"):
86
+ for report in run_bench(paths, env, commands, setup, prepare, batch or 0):
87
+ reports.append(report)
88
+ for line in render_report(len(reports), report):
89
+ console.print(line, highlight=False)
90
+ console.print()
91
+
92
+ for line in render_summary(reports):
93
+ console.print(line, highlight=False)
94
+
95
+ failed = failed or any(report.failed for report in reports)
96
+ results += [_to_json(env.version.name, report) for report in reports]
97
+ except WardError as e:
98
+ if "Unknown command" in str(e):
99
+ raise click.ClickException(
100
+ "This server runs a Ward mod that cannot bench. "
101
+ "Reinstall it with: mcward install <version> --force"
102
+ ) from e
103
+ raise click.ClickException(str(e)) from e
104
+
105
+ if json_path:
106
+ json_path.write_text(json.dumps({"results": results}, indent=2) + "\n", encoding="utf-8")
107
+ if failed:
108
+ sys.exit(1)
109
+
110
+
111
+ def _to_json(version: str, report: BenchReport) -> dict:
112
+ """One result. Times are nanoseconds per run."""
113
+ data: dict = {"version": version, "command": report.name}
114
+ if report.failed:
115
+ return data | {"failed": report.failed}
116
+ return data | {
117
+ "mean": report.mean,
118
+ "stddev": report.stddev,
119
+ "median": report.median,
120
+ "min": report.fastest,
121
+ "p95": report.p95,
122
+ "batch": report.batch,
123
+ "commands": report.commands,
124
+ "allocated": report.allocated,
125
+ "error": report.error,
126
+ "times": list(report.times),
127
+ }
@@ -1,21 +1,21 @@
1
1
  """The start, stop and status commands."""
2
2
 
3
3
  from concurrent.futures import ThreadPoolExecutor, as_completed
4
- from typing import cast
5
4
 
6
5
  import rich_click as click
7
6
  from rich import box
8
7
  from rich.progress import Progress, SpinnerColumn, TextColumn
9
8
  from rich.table import Table
10
9
 
11
- from mcward import (
12
- InstalledEnvironment,
13
- ProcessError,
14
- RunningEnvironment,
15
- UninstalledEnvironment,
16
- )
10
+ from mcward import ProcessError, RunningEnvironment
17
11
 
18
- from ..environments import get_environment, manager, select_installed, select_running
12
+ from ..environments import (
13
+ get_environment,
14
+ manager,
15
+ select_installed,
16
+ select_running,
17
+ start_environments,
18
+ )
19
19
  from ..ui import console, print_note, print_success, print_warning
20
20
 
21
21
 
@@ -24,16 +24,10 @@ from ..ui import console, print_note, print_success, print_warning
24
24
  def start(version: str | None) -> None:
25
25
  """Start the Ward daemon for a specific version."""
26
26
  env = get_environment(version or select_installed("Select a version to start:"))
27
- if isinstance(env, UninstalledEnvironment):
28
- with console.status(f"Installing version {env.version.name}"):
29
- env = env.install()
30
- print_success(f"Installed version {env.version.name}")
31
- if isinstance(env, InstalledEnvironment):
32
- with console.status(f"Starting version {env.version.name}"):
33
- env = env.start()
34
- print_success(f"Started version {env.version.name} {env.process}")
35
- return
36
- print_warning(f"Version {env.version.name} is already running {env.process}")
27
+ if isinstance(env, RunningEnvironment):
28
+ print_warning(f"Version {env.version.name} is already running {env.process}")
29
+ return
30
+ start_environments([env])
37
31
 
38
32
 
39
33
  @click.command()
@@ -51,7 +45,7 @@ def stop(version: str | None, all: bool = False) -> None:
51
45
  print_warning(f"Version {env.version.name} is not running")
52
46
  return
53
47
 
54
- envs = _running_environments()
48
+ envs = manager.running_environments()
55
49
  if not envs:
56
50
  print_note("No versions running")
57
51
  return
@@ -78,7 +72,7 @@ def stop(version: str | None, all: bool = False) -> None:
78
72
  @click.command()
79
73
  def status() -> None:
80
74
  """Show status of running daemons."""
81
- envs = _running_environments()
75
+ envs = manager.running_environments()
82
76
  if not envs:
83
77
  print_note("No versions running")
84
78
  return
@@ -98,7 +92,3 @@ def status() -> None:
98
92
  table.add_row(env.version.name, str(env.process.pid), str(env.process.port), state)
99
93
 
100
94
  console.print("", table)
101
-
102
-
103
- def _running_environments() -> list[RunningEnvironment]:
104
- return [cast(RunningEnvironment, manager.get(v.name)) for v in manager.list_running()]
@@ -4,7 +4,7 @@ import rich_click as click
4
4
 
5
5
  from mcward import Version
6
6
 
7
- from ..environments import curate_versions, manager
7
+ from ..environments import curate_versions, manager, version_labels
8
8
  from ..ui import console
9
9
 
10
10
 
@@ -44,7 +44,7 @@ def _print_remote(installed: list[Version], available: list[Version]) -> None:
44
44
  if version.name not in available_names:
45
45
  _print_version(version, color="yellow")
46
46
 
47
- labels = _version_labels(curated)
47
+ labels = version_labels(curated)
48
48
  for version in curated[:10]:
49
49
  color, marker = ("green", "✓") if version.name in installed_names else ("blue", "○")
50
50
  _print_version(version, marker, color, labels.get(version, ""))
@@ -54,16 +54,6 @@ def _print_remote(installed: list[Version], available: list[Version]) -> None:
54
54
  )
55
55
 
56
56
 
57
- def _version_labels(curated: list[Version]) -> dict[Version, str]:
58
- """Label the newest release as latest and the newest snapshot as snapshot."""
59
- labels = {}
60
- if latest := max((v for v in curated if not v.is_snapshot), default=None):
61
- labels[latest] = "latest"
62
- if snapshot := max((v for v in curated if v.is_snapshot), default=None):
63
- labels[snapshot] = "snapshot"
64
- return labels
65
-
66
-
67
57
  def _print_version(
68
58
  version: Version,
69
59
  marker: str = "✓",
@@ -5,13 +5,9 @@ from pathlib import Path
5
5
 
6
6
  import rich_click as click
7
7
 
8
- from mcward import CoverageIgnores, WardError
9
-
10
8
  from ..datapacks import DEFAULT_PATTERNS, discover_datapacks
11
- from ..environments import manager, select_compatible, start_environments
12
- from ..reporters import github, live
13
- from ..reports import parse_coverage_report, report_session
14
- from ..ui import console
9
+ from ..reports import parse_coverage_report
10
+ from ..session import run_session
15
11
 
16
12
 
17
13
  @click.command()
@@ -48,6 +44,13 @@ from ..ui import console
48
44
  help="Write coverage as 'lcov' or 'html', with an optional path (repeatable); "
49
45
  "implies --coverage",
50
46
  )
47
+ @click.option(
48
+ "--coverage-min",
49
+ type=click.FloatRange(0, 100),
50
+ default=None,
51
+ metavar="PERCENT",
52
+ help="Fail the run when coverage is below this percentage; implies --coverage",
53
+ )
51
54
  @click.option(
52
55
  "--junit-xml",
53
56
  type=click.Path(dir_okay=False, writable=True, path_type=Path),
@@ -66,6 +69,7 @@ def test(
66
69
  reporter: str,
67
70
  coverage: bool,
68
71
  coverage_reports: tuple[str, ...],
72
+ coverage_min: float | None,
69
73
  junit_xml: Path | None,
70
74
  verbose: bool,
71
75
  selector: str,
@@ -75,27 +79,16 @@ def test(
75
79
  if not datapacks:
76
80
  raise click.ClickException("Datapack not found")
77
81
 
78
- strictest = max(datapacks, key=lambda datapack: datapack.min_format)
79
- loosest = min(datapacks, key=lambda datapack: datapack.max_format)
80
- if strictest.min_format > loosest.max_format:
81
- raise click.ClickException(
82
- f"Datapacks have disjoint pack format ranges: {strictest.path.name} needs "
83
- f">= {strictest.min_format} but {loosest.path.name} caps at {loosest.max_format}"
84
- )
85
-
86
- selected = versions or select_compatible(strictest.min_format, loosest.max_format)
87
- paths = [datapack.path for datapack in datapacks]
88
-
89
- specs = [parse_coverage_report(value) for value in coverage_reports]
90
- enabled = coverage or bool(specs)
91
- run = github.run if reporter == "github" else live.run
92
- try:
93
- ignores = CoverageIgnores.load()
94
- envs = start_environments([manager.get(v) for v in selected])
95
- console.print()
96
- session = run(paths, envs, selector, coverage=enabled, verbose=verbose)
97
- report_session(session, paths, specs, junit_xml, verbose, selector, ignores, enabled)
98
- if session.failed:
99
- sys.exit(1)
100
- except WardError as e:
101
- raise click.ClickException(str(e)) from e
82
+ session = run_session(
83
+ datapacks,
84
+ versions=versions,
85
+ reporter=reporter,
86
+ selector=selector,
87
+ coverage=coverage,
88
+ coverage_specs=[parse_coverage_report(value) for value in coverage_reports],
89
+ coverage_min=coverage_min,
90
+ junit_xml=junit_xml,
91
+ verbose=verbose,
92
+ )
93
+ if session.failed:
94
+ sys.exit(1)
@@ -36,25 +36,30 @@ def discover_datapacks(patterns: Sequence[str]) -> list[DataPack]:
36
36
  matches = glob.glob(pattern, recursive=True, include_hidden=True)
37
37
  paths.update(Path(m).resolve() for m in matches)
38
38
 
39
- return [parse_datapack(p) for p in paths if _is_datapack(p)]
39
+ return [parse_datapack(p) for p in sorted(paths, key=_stacking_key) if _is_datapack(p)]
40
40
 
41
41
 
42
42
  def parse_datapack(path: Path) -> DataPack:
43
43
  """Parse a datapack's pack.mcmeta for format range (directory or zip)."""
44
-
45
- def _parse_format(value: int | list[int]) -> int:
46
- if isinstance(value, int):
47
- return value
48
- if isinstance(value, list) and value:
49
- return value[0]
50
- raise click.ClickException(f"Invalid pack format in {path}")
51
-
52
44
  try:
53
- data = json.loads(_read_mcmeta(path))
54
- pack = data["pack"]
55
- return DataPack(path, _parse_format(pack["min_format"]), _parse_format(pack["max_format"]))
45
+ pack = json.loads(_read_mcmeta(path))["pack"]
46
+ return DataPack(path, _major_format(pack["min_format"]), _major_format(pack["max_format"]))
47
+ except KeyError as e:
48
+ raise click.ClickException(f"Invalid pack.mcmeta in {path}: missing {e}") from e
56
49
  except Exception as e:
57
- raise click.ClickException(str(e)) from e
50
+ raise click.ClickException(f"Invalid pack.mcmeta in {path}: {e}") from e
51
+
52
+
53
+ def format_range(datapacks: Sequence[DataPack]) -> tuple[int, int]:
54
+ """The pack formats that every pack supports."""
55
+ strictest = max(datapacks, key=lambda datapack: datapack.min_format)
56
+ loosest = min(datapacks, key=lambda datapack: datapack.max_format)
57
+ if strictest.min_format > loosest.max_format:
58
+ raise click.ClickException(
59
+ f"Datapacks have disjoint pack format ranges: {strictest.path.name} needs "
60
+ f">= {strictest.min_format} but {loosest.path.name} caps at {loosest.max_format}"
61
+ )
62
+ return strictest.min_format, loosest.max_format
58
63
 
59
64
 
60
65
  def pack_resolver(datapacks: Sequence[Path]) -> FileResolver:
@@ -70,7 +75,8 @@ def pack_resolver(datapacks: Sequence[Path]) -> FileResolver:
70
75
  namespace, path = resource.split(":", 1)
71
76
  extension = ".mcfunction" if folder in MCFUNCTION_FOLDERS else ".json"
72
77
  relative = f"data/{namespace}/{folder}/{path}{extension}"
73
- return next((f for pack in datapacks if (f := workspace_path(pack / relative))), None)
78
+ files = (workspace_path(pack / relative) for pack in reversed(datapacks))
79
+ return next((file for file in files if file), None)
74
80
 
75
81
  return resolve
76
82
 
@@ -85,15 +91,33 @@ def workspace_path(file: Path) -> str | None:
85
91
 
86
92
 
87
93
  def _is_datapack(path: Path) -> bool:
88
- """A directory with a pack.mcmeta, or a zip carrying one at its root."""
94
+ """A directory or a zip with a pack.mcmeta and a data folder at its root.
95
+
96
+ The data folder tells a datapack from a resource pack, which has the same pack.mcmeta.
97
+ """
89
98
  if path.is_dir():
90
- return (path / "pack.mcmeta").exists()
99
+ return (path / "pack.mcmeta").is_file() and (path / "data").is_dir()
91
100
  if path.suffix == ".zip" and zipfile.is_zipfile(path):
92
101
  with zipfile.ZipFile(path) as archive:
93
- return "pack.mcmeta" in archive.namelist()
102
+ names = archive.namelist()
103
+ return "pack.mcmeta" in names and any(name.startswith("data/") for name in names)
94
104
  return False
95
105
 
96
106
 
107
+ def _stacking_key(path: Path) -> tuple[str, Path]:
108
+ """The file name the pack deploys under: a directory is zipped, so it gains the extension."""
109
+ return f"{path.name.removesuffix('.zip')}.zip", path
110
+
111
+
112
+ def _major_format(value: int | list[int]) -> int:
113
+ """A pack format is a number, or a [major, minor] pair."""
114
+ if isinstance(value, int):
115
+ return value
116
+ if isinstance(value, list) and value:
117
+ return value[0]
118
+ raise ValueError(f"unexpected pack format {value!r}")
119
+
120
+
97
121
  def _read_mcmeta(path: Path) -> str:
98
122
  if path.is_file(): # Zipped datapack
99
123
  with zipfile.ZipFile(path) as archive:
@@ -35,6 +35,16 @@ def curate_versions(versions: Iterable[Version]) -> list[Version]:
35
35
  return curated
36
36
 
37
37
 
38
+ def version_labels(versions: Sequence[Version]) -> dict[Version, str]:
39
+ """Label the newest release as latest and the newest snapshot as snapshot."""
40
+ labels = {}
41
+ if latest := max((v for v in versions if not v.is_snapshot), default=None):
42
+ labels[latest] = "latest"
43
+ if snapshot := max((v for v in versions if v.is_snapshot), default=None):
44
+ labels[snapshot] = "snapshot"
45
+ return labels
46
+
47
+
38
48
  def get_environment(version: str) -> Environment:
39
49
  try:
40
50
  return manager.get(version)
@@ -58,12 +68,8 @@ def select_available(message: str) -> str:
58
68
  if not curated:
59
69
  raise click.ClickException("No versions available")
60
70
 
61
- snapshot = next((v for v in curated if v.is_snapshot), None)
62
- latest = next((v for v in curated if not v.is_snapshot), None)
63
- options = []
64
- for version in curated:
65
- hint = "(latest)" if version == latest else "(snapshot)" if version == snapshot else ""
66
- options.append(Option(version.name, hint))
71
+ labels = version_labels(curated)
72
+ options = [Option(v.name, f"({labels[v]})" if v in labels else "") for v in curated]
67
73
  return select(message, options)
68
74
 
69
75
 
@@ -0,0 +1,60 @@
1
+ """Benchmark output: one block per command, then how the commands compare."""
2
+
3
+ from collections.abc import Sequence
4
+
5
+ from mcward import BenchReport, compare
6
+
7
+
8
+ def format_time(nanos: float) -> str:
9
+ """A duration with the unit that keeps it readable."""
10
+ for unit, size in (("s", 1e9), ("ms", 1e6), ("µs", 1e3)):
11
+ if nanos >= size:
12
+ return f"{nanos / size:.1f} {unit}"
13
+ return f"{nanos:.0f} ns"
14
+
15
+
16
+ def format_bytes(count: int) -> str:
17
+ for unit, size in (("MB", 1e6), ("kB", 1e3)):
18
+ if count >= size:
19
+ return f"{count / size:.1f} {unit}"
20
+ return f"{count} B"
21
+
22
+
23
+ def render_report(index: int, report: BenchReport) -> list[str]:
24
+ """The lines of one benchmark, in rich markup."""
25
+ lines = [f"[bold]Benchmark {index}:[/] {report.name}"]
26
+
27
+ if report.failed:
28
+ lines.append(f" [red]✗ not measured: {report.failed}[/]")
29
+ return lines
30
+ if report.error:
31
+ lines.append(f" [yellow]! first run failed: {report.error}[/]")
32
+
33
+ mean, sigma = format_time(report.mean), format_time(report.stddev)
34
+ low, high, median = (format_time(v) for v in (report.fastest, report.p95, report.median))
35
+ share = f"{report.tick_share:.2%}" if report.tick_share >= 0.0001 else "<0.01%"
36
+ lines.append(f" Time (mean ± σ): [green]{mean}[/] ± {sigma} {share} of a tick")
37
+ lines.append(f" Range (min … p95): {low} … {high} median {median}")
38
+
39
+ cost = [f"{report.commands} commands"] if report.commands is not None else []
40
+ if report.allocated is not None:
41
+ cost.append(f"{format_bytes(report.allocated)} allocated")
42
+ if cost:
43
+ lines.append(f" Per run: {', '.join(cost)}")
44
+
45
+ lines.append(f" [dim]{len(report.times)} samples of {report.batch} runs[/]")
46
+ return lines
47
+
48
+
49
+ def render_summary(reports: Sequence[BenchReport]) -> list[str]:
50
+ """How many times faster the fastest command is than each of the others."""
51
+ comparisons = compare(reports)
52
+ if not comparisons:
53
+ return []
54
+
55
+ fastest = min((report for report in reports if report.times), key=lambda r: r.mean)
56
+ lines = ["[bold]Summary[/]", f" [cyan]{fastest.name}[/] ran"]
57
+ for comparison in comparisons:
58
+ ratio = f"{comparison.ratio:.2f} ± {comparison.uncertainty:.2f}"
59
+ lines.append(f" [green]{ratio}[/] times faster than {comparison.report.name}")
60
+ return lines
@@ -62,7 +62,9 @@ def annotations(session: TestSession, resolve: FileResolver) -> list[str]:
62
62
  def _annotate_test(result: TestResult, resolve: FileResolver) -> str:
63
63
  failed = result.status is TestStatus.FAILED
64
64
  versions = [
65
- version for version in result.versions if result.outcomes[version].status is result.status
65
+ version
66
+ for version in result.versions
67
+ if (outcome := result.outcomes.get(version)) and outcome.status is result.status
66
68
  ]
67
69
  outcome = result.outcomes[versions[0]]
68
70
  word = "failed" if failed else "skipped"
@@ -52,7 +52,8 @@ def run(
52
52
  session = TestSession(versions) # rebound by every frame; bound even if none arrives
53
53
  rendered = 0.0
54
54
  with Live(console=console, refresh_per_second=10) as live:
55
- for session in run_tests(datapacks, environments, selector=selector, coverage=coverage):
55
+ frames = run_tests(datapacks, environments, selector=selector, coverage=coverage, tick=0.1)
56
+ for session in frames:
56
57
  # Events can outpace the display many times over; render at frame rate
57
58
  if (now := monotonic()) - rendered < 0.1:
58
59
  continue
@@ -5,7 +5,15 @@ from pathlib import Path
5
5
 
6
6
  import rich_click as click
7
7
 
8
- from mcward import CoverageIgnores, ResolvedCoverage, TestSession, Version, resolve_coverage
8
+ from mcward import (
9
+ CoverageIgnores,
10
+ CoverageMinimum,
11
+ CoverageTotals,
12
+ ResolvedCoverage,
13
+ TestSession,
14
+ Version,
15
+ resolve_coverage,
16
+ )
9
17
 
10
18
  from ..reporters.coverage import render_coverage
11
19
  from ..ui import console, print_warning
@@ -14,6 +22,7 @@ from .junit import write_junit
14
22
  from .lcov import write_lcov
15
23
 
16
24
  __all__ = [
25
+ "coverage_shortfalls",
17
26
  "missing_coverage",
18
27
  "parse_coverage_report",
19
28
  "report_session",
@@ -44,8 +53,12 @@ def report_session(
44
53
  selector: str = "*:*",
45
54
  ignores: CoverageIgnores | None = None,
46
55
  coverage: bool = False,
56
+ minimum: CoverageMinimum | None = None,
47
57
  ) -> None:
48
- """Render the coverage summary and write the requested report files."""
58
+ """Render the coverage summary and write the requested report files.
59
+
60
+ With a minimum, a run that measured coverage and stays under it ends in an error.
61
+ """
49
62
  if junit_xml is not None:
50
63
  write_junit(session, junit_xml)
51
64
  console.print(f"Test results written to [magenta]{junit_xml}[/magenta]")
@@ -55,11 +68,11 @@ def report_session(
55
68
  f"No coverage reported by {version.name}: its ward mod predates coverage, "
56
69
  "reinstall the environment once a newer release supports this Minecraft version"
57
70
  )
58
- if session.coverage:
59
- resolved = {
60
- version: resolve_coverage(coverage, datapacks, selector, ignores)
61
- for version, coverage in session.coverage.items()
62
- }
71
+ resolved = {
72
+ version: resolve_coverage(recorded, datapacks, selector, ignores)
73
+ for version, recorded in session.coverage.items()
74
+ }
75
+ if resolved:
63
76
  console.print(render_coverage(resolved, verbose))
64
77
  for file in write_coverage_reports(session, resolved, specs):
65
78
  console.print(f"Coverage report written to [magenta]{file}[/magenta]")
@@ -68,6 +81,46 @@ def report_session(
68
81
  if session.coverage and not specs:
69
82
  hint += ", or --coverage-report html"
70
83
  console.print(f"[dim]{hint}[/dim]")
84
+ if coverage and minimum is not None:
85
+ if shortfalls := coverage_shortfalls(session, resolved, minimum):
86
+ raise click.ClickException("\n".join(shortfalls))
87
+
88
+
89
+ def coverage_shortfalls(
90
+ session: TestSession,
91
+ resolved: Mapping[Version, ResolvedCoverage],
92
+ minimum: CoverageMinimum,
93
+ ) -> list[str]:
94
+ """One line for each figure under its minimum: the run's, then each namespace's."""
95
+ if minimum.total is None and minimum.namespace is None:
96
+ return []
97
+
98
+ lines = [
99
+ f"No coverage from {version.name} to check against the minimum"
100
+ for version in missing_coverage(session)
101
+ ]
102
+ for version, coverage in resolved.items():
103
+ where = f" on {version.name}" if len(session.versions) > 1 else ""
104
+ total = CoverageTotals.of(coverage.reports)
105
+ if _is_below(total, minimum.total):
106
+ lines.append(
107
+ f"Coverage {total.ratio:.1%}{where} is below the minimum of {minimum.total:g}%"
108
+ )
109
+ for namespace in sorted({report.namespace for report in coverage.reports}):
110
+ own = CoverageTotals.of(r for r in coverage.reports if r.namespace == namespace)
111
+ if _is_below(own, minimum.namespace):
112
+ lines.append(
113
+ f"Coverage {own.ratio:.1%} of {namespace}{where} is below "
114
+ f"the minimum of {minimum.namespace:g}% per namespace"
115
+ )
116
+ return lines
117
+
118
+
119
+ def _is_below(totals: CoverageTotals, minimum: float | None) -> bool:
120
+ """Compared as it is printed, so a figure shown as 80.0% reaches a minimum of 80."""
121
+ if minimum is None or totals.ratio is None:
122
+ return False
123
+ return round(totals.ratio * 100, 1) < minimum
71
124
 
72
125
 
73
126
  def missing_coverage(session: TestSession) -> list[Version]: