stepup 4.1.0rc2__tar.gz → 4.1.0rc4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/PKG-INFO +1 -1
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/path.py +14 -6
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/pytest.py +229 -58
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup.egg-info/PKG-INFO +1 -1
- stepup-4.1.0rc4/stepup.egg-info/scm_version.json +8 -0
- stepup-4.1.0rc2/stepup.egg-info/scm_version.json +0 -8
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/.gitattributes +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/.zenodo.json +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/CITATION.cff +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/CLAUDE.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/CONTRIBUTING.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/LICENSE +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/LICENSES/CC-BY-SA-4.0.txt +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/LICENSES/LGPL-3.0-or-later.txt +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/LICENSES/LicenseRef-StepUp-Logo-v1.txt +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/MANIFEST.in +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/README.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/REUSE.toml +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/pyproject.toml +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/setup.cfg +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/snipwise.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/CLAUDE.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/__init__.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/__main__.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/api.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/call.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cattrs.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/CLAUDE.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/__init__.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/browse.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/build.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/clean.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/config.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/config_loader.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/director_log.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/interact.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/reporter.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/status.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/terminal.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/cli/tool.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/constants.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/CLAUDE.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/__init__.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/__main__.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/builder.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/executor.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/finalize.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/handler.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/hash_queue.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/job.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/reporter.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/scheduler.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/serve.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/startup.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/usage.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/director/watcher.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/enums.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/exceptions.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/extapi.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/graph/CLAUDE.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/graph/__init__.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/graph/file.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/graph/outcome.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/graph/pending.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/graph/sqlite3.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/graph/static_tree.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/graph/step.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/graph/trellis.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/graph/workflow.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/hash.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/launch/CLAUDE.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/launch/__init__.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/launch/capture.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/launch/command.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/launch/copy.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/launch/forkserver.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/launch/subprocess.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/launch/worker.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/logo.svg +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/nglob.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/platform/CLAUDE.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/platform/__init__.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/platform/asyncio.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/platform/cgroups.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/platform/oscompat.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/platform/popen.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/platform/stdio.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/platform/winapi.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/render_jinja.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/rpc/CLAUDE.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/rpc/__init__.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/rpc/client.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/rpc/protocol.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/rpc/server.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/rpc/transport.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/script.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/stepinfo.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/tracebacks.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/utils.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/watch/CLAUDE.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/watch/__init__.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/watch/backend.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/watch/inotify.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/watch/select.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup/core/watch/watchfiles.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup.egg-info/SOURCES.txt +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup.egg-info/dependency_links.txt +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup.egg-info/entry_points.txt +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup.egg-info/requires.txt +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup.egg-info/scm_file_list.json +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/stepup.egg-info/top_level.txt +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/README.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/docs/make_stable_redirects.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/imports/leaves.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/macci/README.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/macci/macci.sh +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/perf/.gitignore +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/perf/analyze_perf.py +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/winvm/README.md +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/winvm/oem/install.bat +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/winvm/oem/setup.ps1 +0 -0
- {stepup-4.1.0rc2 → stepup-4.1.0rc4}/tools/winvm/winvm.sh +0 -0
|
@@ -58,25 +58,33 @@ or one of its variants.
|
|
|
58
58
|
|
|
59
59
|
|
|
60
60
|
def _nt_to_slash(path: str) -> str:
|
|
61
|
-
"""Write a Windows path with `/` separators and an upper-case drive letter.
|
|
61
|
+
r"""Write a Windows path with `/` separators and an upper-case drive letter.
|
|
62
62
|
|
|
63
|
+
The prefix of an extended-length path is removed:
|
|
64
|
+
`\\?\C:\dir` becomes `C:/dir`, and `\\?\UNC\server\share` becomes `//server/share`.
|
|
63
65
|
The drive letter is only changed when it is followed by a separator or by nothing,
|
|
64
66
|
because a colon after a one-letter file name may also start the name of a data stream.
|
|
65
67
|
"""
|
|
66
68
|
# The unbound `str.replace` returns an exact `str`, also for an instance of a subclass.
|
|
67
69
|
path = str.replace(path, "\\", "/")
|
|
70
|
+
if path.startswith("//?/UNC/"):
|
|
71
|
+
path = "//" + path[8:]
|
|
72
|
+
elif path.startswith("//?/"):
|
|
73
|
+
path = path[4:]
|
|
68
74
|
if path[1:2] == ":" and path[2:3] in ("/", "") and "a" <= path[0] <= "z":
|
|
69
75
|
path = path[0].upper() + path[1:]
|
|
70
76
|
return path
|
|
71
77
|
|
|
72
78
|
|
|
73
79
|
def to_slash(path: str) -> str:
|
|
74
|
-
"""Convert a path string received from the operating system to StepUp's spelling.
|
|
80
|
+
r"""Convert a path string received from the operating system to StepUp's spelling.
|
|
75
81
|
|
|
76
82
|
Inside StepUp, `/` is the only path separator on every platform.
|
|
77
|
-
On Windows, backslashes are replaced by
|
|
83
|
+
On Windows, backslashes are replaced by `/`, the drive letter is written in upper case,
|
|
84
|
+
and the prefix `\\?\` of an extended-length path is removed,
|
|
78
85
|
because the operating system does not distinguish these spellings,
|
|
79
86
|
while StepUp compares paths as strings.
|
|
87
|
+
Some programs report paths with this prefix, as observed with Typst 0.15 on Windows.
|
|
80
88
|
On other platforms, a backslash is an ordinary file name character
|
|
81
89
|
and the path is returned unchanged.
|
|
82
90
|
|
|
@@ -219,15 +227,15 @@ class _NtPath(BasePath):
|
|
|
219
227
|
|
|
220
228
|
|
|
221
229
|
class Path(_NtPath if sys.platform == "win32" else BasePath):
|
|
222
|
-
"""A file system path with `/` as its only separator, on every platform.
|
|
230
|
+
r"""A file system path with `/` as its only separator, on every platform.
|
|
223
231
|
|
|
224
232
|
This is a subclass of `path.Path` from the [path](https://pypi.org/project/path/) library,
|
|
225
233
|
and therefore also of `str`.
|
|
226
234
|
It preserves a leading `./` and a trailing `/`, which carry meaning in StepUp.
|
|
227
235
|
|
|
228
236
|
On POSIX systems, it behaves exactly like `path.Path`.
|
|
229
|
-
On Windows, the constructor replaces backslashes by
|
|
230
|
-
|
|
237
|
+
On Windows, the constructor replaces backslashes by `/`,
|
|
238
|
+
writes the drive letter in upper case and removes the prefix `\\?\` of an extended-length path,
|
|
231
239
|
and all methods return paths with `/` separators, e.g. `C:/Users/me/project/plan.py`.
|
|
232
240
|
Windows accepts such paths in all its file system functions.
|
|
233
241
|
Use `os.path.normpath()` to obtain the native spelling when another program requires it.
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import ast
|
|
6
6
|
import asyncio
|
|
7
7
|
import difflib
|
|
8
|
+
import fnmatch
|
|
8
9
|
import functools
|
|
9
10
|
import importlib.util
|
|
10
11
|
import os
|
|
@@ -14,8 +15,10 @@ import shutil
|
|
|
14
15
|
import signal
|
|
15
16
|
import subprocess
|
|
16
17
|
import sys
|
|
18
|
+
from collections.abc import Callable, Sequence
|
|
17
19
|
from typing import TextIO
|
|
18
20
|
|
|
21
|
+
import attrs
|
|
19
22
|
import pytest
|
|
20
23
|
|
|
21
24
|
from .cli.director_log import scan_director_log
|
|
@@ -30,7 +33,19 @@ from .watch.select import select_watcher_backend
|
|
|
30
33
|
if sys.platform == "win32":
|
|
31
34
|
from .platform.winapi import JobObject
|
|
32
35
|
|
|
33
|
-
__all__ = (
|
|
36
|
+
__all__ = (
|
|
37
|
+
"GRAPH_OUTPUT",
|
|
38
|
+
"OUTPUT_RULES",
|
|
39
|
+
"REPORTER_OUTPUT",
|
|
40
|
+
"ConventionTests",
|
|
41
|
+
"OutputRule",
|
|
42
|
+
"mask_reporter_output",
|
|
43
|
+
"normalize_reporter_output",
|
|
44
|
+
"remove_digests",
|
|
45
|
+
"remove_hashes",
|
|
46
|
+
"run_example",
|
|
47
|
+
"run_plan",
|
|
48
|
+
)
|
|
34
49
|
|
|
35
50
|
|
|
36
51
|
def pytest_configure():
|
|
@@ -87,6 +102,33 @@ CHANGE_LINE = re.compile(r"^ ?(?:UPDATED|DELETED) │ (.*)$")
|
|
|
87
102
|
TRACEBACK_FILE = re.compile(r'^( *File ")([^"\n]*)(?=")', re.MULTILINE)
|
|
88
103
|
"""The file name on a line of a Python traceback."""
|
|
89
104
|
|
|
105
|
+
TRACEBACK_SOURCE = re.compile(
|
|
106
|
+
r'^((?P<indent>[ |]*)File "[^"\n]*", line [^\n]*\n(?P=indent) [^\n]*\n)'
|
|
107
|
+
r"(?:(?P=indent) [^\n]*\n)+",
|
|
108
|
+
re.MULTILINE,
|
|
109
|
+
)
|
|
110
|
+
"""A frame of a Python traceback that shows more than the first line of its source.
|
|
111
|
+
|
|
112
|
+
The first group holds the line with the file name and the first line of the source.
|
|
113
|
+
The lines that follow it are carets under the source,
|
|
114
|
+
or further lines of a statement that spans several lines.
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
TRACEBACK_LINE_NUMBER = re.compile(r", line \d+, in ")
|
|
118
|
+
"""The line number on a line of a Python traceback that names a function."""
|
|
119
|
+
|
|
120
|
+
DIRECTOR_SOCKET_LINE = re.compile(r"^ *DIRECTOR │ Listening on .*\n", re.MULTILINE)
|
|
121
|
+
"""The line of the reporter output that names the socket of the director."""
|
|
122
|
+
|
|
123
|
+
DIRECTOR_USAGE_LINE = re.compile(r"^ *DIRECTOR │ Wall .*\n", re.MULTILINE)
|
|
124
|
+
"""The line of the reporter output with the resources used by the director and the steps."""
|
|
125
|
+
|
|
126
|
+
STDERR_PAGE = re.compile(re.escape(STDERR_BEGIN) + r".*?" + re.escape(STDERR_END), re.DOTALL)
|
|
127
|
+
"""The page of the reporter output with the standard error of a step."""
|
|
128
|
+
|
|
129
|
+
GRAPH_DIGEST_LINE = re.compile(r"^ *(?:inp_|out_)?digest = [ 0-9a-f]{71}\n", re.MULTILINE)
|
|
130
|
+
"""A line with the digest of a file or a step in the text format of the workflow graph."""
|
|
131
|
+
|
|
90
132
|
FILE_MODE = re.compile(r"(?<=[ (]mode )([-?a-z]{10}) ➜ ([-?a-z]{10})")
|
|
91
133
|
"""The change of a file mode in the reporter output, e.g. `mode ?--------- ➜ -rw-r--r--`."""
|
|
92
134
|
|
|
@@ -148,7 +190,166 @@ def sort_change_runs(text: str) -> str:
|
|
|
148
190
|
return "\n".join(result)
|
|
149
191
|
|
|
150
192
|
|
|
151
|
-
|
|
193
|
+
def normalize_reporter_output(text: str) -> str:
|
|
194
|
+
"""Rewrite the output of `stepup build` into a form that is the same in every run.
|
|
195
|
+
|
|
196
|
+
This removes or replaces the parts that depend on the platform, the Python version,
|
|
197
|
+
the run itself, or the exact source code of StepUp:
|
|
198
|
+
|
|
199
|
+
- the separators in the file names of tracebacks, see `slash_traceback_files`,
|
|
200
|
+
- all but the first line of the source of each frame in a traceback,
|
|
201
|
+
because Python versions differ in the carets and the lines they add to it,
|
|
202
|
+
- the line numbers in tracebacks,
|
|
203
|
+
- the socket of the director, which is in a new temporary directory in every run,
|
|
204
|
+
- the body of the standard error page, which depends on the OS and the Python version,
|
|
205
|
+
- the resources used by the director and the steps.
|
|
206
|
+
"""
|
|
207
|
+
text = slash_traceback_files(text)
|
|
208
|
+
text = TRACEBACK_SOURCE.sub(r"\1", text)
|
|
209
|
+
text = TRACEBACK_LINE_NUMBER.sub(", line ---, in ", text)
|
|
210
|
+
text = DIRECTOR_SOCKET_LINE.sub("", text)
|
|
211
|
+
text = STDERR_PAGE.sub(STDERR_BEGIN + "\n(stripped)\n" + STDERR_END, text)
|
|
212
|
+
return DIRECTOR_USAGE_LINE.sub("", text)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def mask_reporter_output(text: str) -> str:
|
|
216
|
+
"""Hide the parts of the output of `stepup build` that may differ in this environment.
|
|
217
|
+
|
|
218
|
+
When the file watcher is `watchfiles`,
|
|
219
|
+
the order of consecutive `UPDATED` and `DELETED` lines is hidden,
|
|
220
|
+
because the changes observed together are reported in sorted order,
|
|
221
|
+
see `stepup.core.watch.watchfiles.WatchfilesBackend` and `sort_change_runs`.
|
|
222
|
+
On a platform without permission bits of files,
|
|
223
|
+
the file modes and the digests of step hashes are hidden, see `mask_file_modes`.
|
|
224
|
+
"""
|
|
225
|
+
if select_watcher_backend() == "watchfiles":
|
|
226
|
+
text = sort_change_runs(text)
|
|
227
|
+
if not file_modes_available():
|
|
228
|
+
text = mask_file_modes(text)
|
|
229
|
+
return text
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def remove_digests(text: str) -> str:
|
|
233
|
+
"""Remove the digests of files and steps from the text format of the workflow graph.
|
|
234
|
+
|
|
235
|
+
They change with every change to the contents of a file,
|
|
236
|
+
so an example tests the contents of its results explicitly instead.
|
|
237
|
+
"""
|
|
238
|
+
return GRAPH_DIGEST_LINE.sub("", text)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def finalize_output(text: str, workdir: Path) -> str:
|
|
242
|
+
"""Bring a normalized output into the form of an expected file, whatever its contents.
|
|
243
|
+
|
|
244
|
+
The directory of the example and the current directory are replaced by `${CASE}`
|
|
245
|
+
and `${PWD}`, which keeps the paths of the machine out of the expected files.
|
|
246
|
+
The directory of the example comes first, because it may be inside the current directory.
|
|
247
|
+
Trailing whitespace is removed from every line and from the end of the text,
|
|
248
|
+
so that the pre-commit hooks `trailing-whitespace` and `end-of-file-fixer`
|
|
249
|
+
leave the expected files unchanged.
|
|
250
|
+
|
|
251
|
+
Parameters
|
|
252
|
+
----------
|
|
253
|
+
text
|
|
254
|
+
The output, after the normalization of its `OutputRule`.
|
|
255
|
+
workdir
|
|
256
|
+
The directory in which the example ran.
|
|
257
|
+
|
|
258
|
+
Returns
|
|
259
|
+
-------
|
|
260
|
+
finalized
|
|
261
|
+
The output as it is stored in an expected file.
|
|
262
|
+
"""
|
|
263
|
+
text = text.replace(workdir, "${CASE}").replace(Path.cwd(), "${PWD}")
|
|
264
|
+
return re.sub(r"[ \t]+$", "", text, flags=re.MULTILINE).rstrip()
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _unchanged(text: str) -> str:
|
|
268
|
+
"""Return the text as it is."""
|
|
269
|
+
return text
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
@attrs.define(frozen=True)
|
|
273
|
+
class OutputRule:
|
|
274
|
+
"""How `run_example` treats the outputs of an example whose names match a pattern.
|
|
275
|
+
|
|
276
|
+
A rule has two stages, which differ in what happens to the parts they hide.
|
|
277
|
+
The `normalize` stage loses them,
|
|
278
|
+
because its result is what `overwrite_expected` writes to the expected file.
|
|
279
|
+
It is meant for what differs between runs, machines, platforms or Python versions,
|
|
280
|
+
and for what would make the expected file change for no good reason.
|
|
281
|
+
The `mask` stage only hides them while comparing,
|
|
282
|
+
so that the expected file keeps them.
|
|
283
|
+
It is meant for what may only differ in some environments,
|
|
284
|
+
and is still worth comparing in the others.
|
|
285
|
+
"""
|
|
286
|
+
|
|
287
|
+
pattern: str = attrs.field(validator=attrs.validators.instance_of(str))
|
|
288
|
+
"""The pattern for the part of the file name after `expected` or `current`.
|
|
289
|
+
|
|
290
|
+
It is matched with `fnmatch.fnmatchcase`,
|
|
291
|
+
e.g. `_stdout*.txt` matches `expected_stdout.txt` and `expected_stdout2.txt`.
|
|
292
|
+
"""
|
|
293
|
+
|
|
294
|
+
normalize: Callable[[str], str] = attrs.field(
|
|
295
|
+
default=_unchanged, kw_only=True, validator=attrs.validators.is_callable()
|
|
296
|
+
)
|
|
297
|
+
"""Rewrite the current output into the form that is stored in the expected file.
|
|
298
|
+
|
|
299
|
+
It receives the output as the example wrote it,
|
|
300
|
+
and its result is finalized by `finalize_output`.
|
|
301
|
+
"""
|
|
302
|
+
|
|
303
|
+
mask: Callable[[str], str] = attrs.field(
|
|
304
|
+
default=_unchanged, kw_only=True, validator=attrs.validators.is_callable()
|
|
305
|
+
)
|
|
306
|
+
"""Hide the parts that may legitimately differ in the present environment.
|
|
307
|
+
|
|
308
|
+
It is applied to the finalized current output and to the expected output,
|
|
309
|
+
only for their comparison.
|
|
310
|
+
"""
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
REPORTER_OUTPUT = OutputRule(
|
|
314
|
+
"_stdout*.txt", normalize=normalize_reporter_output, mask=mask_reporter_output
|
|
315
|
+
)
|
|
316
|
+
"""The rule for the output of `stepup build`."""
|
|
317
|
+
|
|
318
|
+
GRAPH_OUTPUT = OutputRule("_graph*.txt", normalize=remove_digests)
|
|
319
|
+
"""The rule for the text format of the workflow graph, as written by `stepup graph`."""
|
|
320
|
+
|
|
321
|
+
OUTPUT_RULES = (REPORTER_OUTPUT, GRAPH_OUTPUT)
|
|
322
|
+
"""The rules that `run_example` applies by default."""
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def select_output_rule(rules: Sequence[OutputRule], suffix: str) -> OutputRule:
|
|
326
|
+
"""Select the first rule whose pattern matches a suffix of an output file name.
|
|
327
|
+
|
|
328
|
+
Parameters
|
|
329
|
+
----------
|
|
330
|
+
rules
|
|
331
|
+
The rules to select from, in order of precedence.
|
|
332
|
+
suffix
|
|
333
|
+
The part of the file name after `expected` or `current`.
|
|
334
|
+
|
|
335
|
+
Returns
|
|
336
|
+
-------
|
|
337
|
+
rule
|
|
338
|
+
The selected rule, or a rule that leaves the output unchanged when none matches.
|
|
339
|
+
"""
|
|
340
|
+
for rule in rules:
|
|
341
|
+
if fnmatch.fnmatchcase(suffix, rule.pattern):
|
|
342
|
+
return rule
|
|
343
|
+
return OutputRule("*")
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
async def run_example(
|
|
347
|
+
srcdir: Path,
|
|
348
|
+
tmpdir: Path,
|
|
349
|
+
overwrite_expected: bool = False,
|
|
350
|
+
*,
|
|
351
|
+
rules: Sequence[OutputRule] = OUTPUT_RULES,
|
|
352
|
+
):
|
|
152
353
|
"""Run an example use case in a temporary directory and check the outputs.
|
|
153
354
|
|
|
154
355
|
The script `main.sh` in the example is the entry point for the test case.
|
|
@@ -170,14 +371,14 @@ async def run_example(srcdir: Path, tmpdir: Path, overwrite_expected=False):
|
|
|
170
371
|
On Windows, it runs with the bash of `stepup.core.platform.popen.find_shell`,
|
|
171
372
|
in a hidden console and a Job Object of its own, which is terminated.
|
|
172
373
|
|
|
173
|
-
All files in the srcdir starting with `
|
|
174
|
-
starting with `
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
374
|
+
All files in the srcdir starting with `expected` will be compared to corresponding files
|
|
375
|
+
starting with `current` in the temporary directory after completion of the example.
|
|
376
|
+
Each pair is treated by the first of the `rules` that matches the rest of the file name.
|
|
377
|
+
Its `normalize` stage and `finalize_output` turn the current output
|
|
378
|
+
into the form of an expected file,
|
|
379
|
+
after which both are passed through its `mask` stage and compared.
|
|
380
|
+
The default rules, `OUTPUT_RULES`, cover the output of `stepup build`
|
|
381
|
+
and the text format of the workflow graph.
|
|
181
382
|
|
|
182
383
|
Parameters
|
|
183
384
|
----------
|
|
@@ -187,6 +388,9 @@ async def run_example(srcdir: Path, tmpdir: Path, overwrite_expected=False):
|
|
|
187
388
|
The temporary directory where the example is executed.
|
|
188
389
|
overwrite_expected
|
|
189
390
|
Update the expected outputs in the source with the outputs from the tmpdir.
|
|
391
|
+
rules
|
|
392
|
+
The rules for the outputs of the example, in order of precedence.
|
|
393
|
+
An output that matches none of them is only finalized.
|
|
190
394
|
"""
|
|
191
395
|
workdir = tmpdir / "example"
|
|
192
396
|
shutil.copytree(srcdir, workdir)
|
|
@@ -213,8 +417,8 @@ async def run_example(srcdir: Path, tmpdir: Path, overwrite_expected=False):
|
|
|
213
417
|
pairs = []
|
|
214
418
|
|
|
215
419
|
for path_exp in sorted(srcdir.glob("expected*.*")):
|
|
216
|
-
|
|
217
|
-
path_cur = workdir /
|
|
420
|
+
suffix = path_exp.basename()[len("expected") :]
|
|
421
|
+
path_cur = workdir / f"current{suffix}"
|
|
218
422
|
if not path_cur.is_file():
|
|
219
423
|
raise AssertionError(
|
|
220
424
|
f"{path_cur.basename()} was not created and main.sh exited with "
|
|
@@ -222,53 +426,26 @@ async def run_example(srcdir: Path, tmpdir: Path, overwrite_expected=False):
|
|
|
222
426
|
f"it died on."
|
|
223
427
|
)
|
|
224
428
|
with open(path_cur, encoding="utf-8") as fh:
|
|
225
|
-
cur = fh.read()
|
|
429
|
+
cur = fh.read()
|
|
226
430
|
|
|
227
431
|
# Print the current output for debugging purposes,
|
|
228
|
-
#
|
|
432
|
+
# because the normalization below loses information.
|
|
229
433
|
print()
|
|
230
|
-
print(f"########## {
|
|
434
|
+
print(f"########## {path_exp.basename()} ##########")
|
|
231
435
|
print()
|
|
232
|
-
print(cur)
|
|
233
|
-
|
|
234
|
-
# Normalize output before comparing:
|
|
235
|
-
# - Python prints the file names of a traceback with the separators of the platform
|
|
236
|
-
cur = slash_traceback_files(cur)
|
|
237
|
-
cur = cur.replace(Path.cwd(), "${PWD}")
|
|
238
|
-
cur = cur.replace(workdir, "${CASE}")
|
|
239
|
-
# - The director's socket path contains a random temporary directory
|
|
240
|
-
cur = re.sub(r"DIRECTOR │ Listening on .*\n", "", cur)
|
|
241
|
-
# - Exact line numbers in exceptions change often and do not matter here
|
|
242
|
-
cur = re.sub(r", line \d+, in ", ", line ---, in ", cur)
|
|
243
|
-
# - Remove new types of traceback output not introduced after Python 3.11,
|
|
244
|
-
# which is the oldest version we support.
|
|
245
|
-
cur = re.sub(r"^ \.{3}<\d+ lines>\.{3}\n", "", cur, flags=re.MULTILINE)
|
|
246
|
-
cur = re.sub(r"^ \)\n", "", cur, flags=re.MULTILINE)
|
|
247
|
-
cur = re.sub(r" \~*\^*\n", "", cur, flags=re.MULTILINE)
|
|
248
|
-
# - Remove trailing whitespace
|
|
249
|
-
cur = re.sub(r"[ \t]+?(\n|\Z)", r"\1", cur)
|
|
250
|
-
# - Remove digests: they change often,
|
|
251
|
-
# so the content of results must be tested explicitly.
|
|
252
|
-
cur = re.sub(r" {10}(.{4})digest = [ 0-9a-f]{71}\n", "", cur)
|
|
253
|
-
# - Strip the body of the standard error page: it is sensitive to OS and Python version
|
|
254
|
-
cur = re.sub(
|
|
255
|
-
STDERR_BEGIN + r".*?" + STDERR_END,
|
|
256
|
-
STDERR_BEGIN + "\n(stripped)\n" + STDERR_END,
|
|
257
|
-
cur,
|
|
258
|
-
flags=re.DOTALL,
|
|
259
|
-
)
|
|
260
|
-
# - Timings are not deterministic
|
|
261
|
-
cur = re.sub(r"DIRECTOR │ Wall .*\n", "", cur)
|
|
436
|
+
print(cur.rstrip())
|
|
262
437
|
|
|
263
|
-
|
|
438
|
+
rule = select_output_rule(rules, suffix)
|
|
439
|
+
cur = finalize_output(rule.normalize(cur), workdir)
|
|
264
440
|
if overwrite_expected:
|
|
265
|
-
path_exp = srcdir / fn_exp
|
|
266
441
|
with open(path_exp, "w", encoding="utf-8", newline="\n") as fh:
|
|
267
|
-
|
|
442
|
+
# An empty output stays empty, as `end-of-file-fixer` would make it.
|
|
443
|
+
if cur != "":
|
|
444
|
+
print(cur, file=fh)
|
|
268
445
|
else:
|
|
269
446
|
with open(path_exp, encoding="utf-8") as fh:
|
|
270
447
|
exp = fh.read().rstrip()
|
|
271
|
-
pairs.append((path_exp, cur, exp))
|
|
448
|
+
pairs.append((path_exp, rule, cur, exp))
|
|
272
449
|
finally:
|
|
273
450
|
for path_log in [workdir / MAIN_LOG, *sorted(workdir.glob(STEPUP_DIR / "*.log"))]:
|
|
274
451
|
print()
|
|
@@ -280,16 +457,10 @@ async def run_example(srcdir: Path, tmpdir: Path, overwrite_expected=False):
|
|
|
280
457
|
print(f"########## main.sh return code: {returncode} ##########")
|
|
281
458
|
|
|
282
459
|
# Check late for errors, to maximize the printed output.
|
|
283
|
-
sort_changes = select_watcher_backend() == "watchfiles"
|
|
284
|
-
mask_modes = not file_modes_available()
|
|
285
460
|
different = []
|
|
286
|
-
for path_exp,
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
exp = sort_change_runs(exp)
|
|
290
|
-
if mask_modes:
|
|
291
|
-
cur = mask_file_modes(cur)
|
|
292
|
-
exp = mask_file_modes(exp)
|
|
461
|
+
for path_exp, rule, cur_finalized, exp_finalized in pairs:
|
|
462
|
+
cur = rule.mask(cur_finalized)
|
|
463
|
+
exp = rule.mask(exp_finalized)
|
|
293
464
|
if cur != exp:
|
|
294
465
|
different.append(path_exp)
|
|
295
466
|
print()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|