stepup 4.0.0rc4__tar.gz → 4.0.0rc5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stepup-4.0.0rc4/stepup.egg-info → stepup-4.0.0rc5}/PKG-INFO +1 -1
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/browse.py +29 -14
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/builder.py +0 -4
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/director.py +38 -26
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/executor.py +2 -5
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/extapi.py +54 -28
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/step.py +42 -45
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/tui.py +12 -11
- {stepup-4.0.0rc4 → stepup-4.0.0rc5/stepup.egg-info}/PKG-INFO +1 -1
- stepup-4.0.0rc5/stepup.egg-info/scm_version.json +8 -0
- stepup-4.0.0rc4/stepup.egg-info/scm_version.json +0 -8
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/CLAUDE.md +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/LICENSE +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/MANIFEST.in +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/README.md +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/pyproject.toml +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/setup.cfg +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/__init__.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/__main__.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/api.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/asyncio.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/call.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/cattrs.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/clean.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/config.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/constants.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/enums.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/exceptions.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/file.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/hash.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/hasher.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/interact.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/job.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/logo.svg +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/nglob.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/path.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/pytest.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/render_jinja.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/reporter.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/rpc.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/scheduler.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/script.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/sqlite3.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/startup.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/static_tree.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/stepinfo.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/trellis.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/utils.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/watcher.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/workflow.py +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/SOURCES.txt +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/dependency_links.txt +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/entry_points.txt +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/requires.txt +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/scm_file_list.json +0 -0
- {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/top_level.txt +0 -0
|
@@ -405,7 +405,7 @@ class GraphServer(BaseHTTPRequestHandler):
|
|
|
405
405
|
if kind == "step":
|
|
406
406
|
sql_props = (
|
|
407
407
|
"SELECT state, need, duration, rescheduled_info, subshell, env_overrides,"
|
|
408
|
-
"_safe, _check_safe, _implied_need, _tail_time, _check_after "
|
|
408
|
+
"stdout, stderr, _safe, _check_safe, _implied_need, _tail_time, _check_after "
|
|
409
409
|
"FROM step WHERE node = ?"
|
|
410
410
|
)
|
|
411
411
|
(
|
|
@@ -415,6 +415,8 @@ class GraphServer(BaseHTTPRequestHandler):
|
|
|
415
415
|
rescheduled_info,
|
|
416
416
|
subshell,
|
|
417
417
|
env_overrides,
|
|
418
|
+
stdout,
|
|
419
|
+
stderr,
|
|
418
420
|
safe,
|
|
419
421
|
check_safe,
|
|
420
422
|
implied_need_id,
|
|
@@ -487,23 +489,30 @@ class GraphServer(BaseHTTPRequestHandler):
|
|
|
487
489
|
ngm = pickle.loads(row[0])
|
|
488
490
|
yield f"<p>{[ngs.pattern for ngs in ngm.nglob_singles]} {ngm.subs}</p>"
|
|
489
491
|
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
yield f"<p>{out_kind}:</p>"
|
|
497
|
-
yield f"<pre>{html.escape(out_content)}</pre>"
|
|
492
|
+
if stdout != "":
|
|
493
|
+
yield "<h3>Standard Output</h3>"
|
|
494
|
+
yield f"<pre>{html.escape(stdout)}</pre>"
|
|
495
|
+
if stderr != "":
|
|
496
|
+
yield "<h3>Standard Error</h3>"
|
|
497
|
+
yield f"<pre>{html.escape(stderr)}</pre>"
|
|
498
498
|
|
|
499
499
|
sql_sub = (
|
|
500
|
-
"SELECT cmd, workdir, env_overrides, returncode, shell, stdin
|
|
501
|
-
"WHERE node = ? ORDER BY seq"
|
|
500
|
+
"SELECT cmd, workdir, env_overrides, returncode, shell, stdin, stdout, stderr "
|
|
501
|
+
"FROM step_subprocess WHERE node = ? ORDER BY seq"
|
|
502
502
|
)
|
|
503
503
|
subs = list(self.con.execute(sql_sub, (node_i,)))
|
|
504
504
|
if len(subs) > 0:
|
|
505
505
|
yield "<h3>Subprocesses</h3>"
|
|
506
|
-
for
|
|
506
|
+
for (
|
|
507
|
+
cmd,
|
|
508
|
+
workdir,
|
|
509
|
+
env_overrides,
|
|
510
|
+
returncode,
|
|
511
|
+
shell_int,
|
|
512
|
+
stdin,
|
|
513
|
+
stdout,
|
|
514
|
+
stderr,
|
|
515
|
+
) in subs:
|
|
507
516
|
line = format_subprocess(
|
|
508
517
|
cmd,
|
|
509
518
|
workdir,
|
|
@@ -512,9 +521,15 @@ class GraphServer(BaseHTTPRequestHandler):
|
|
|
512
521
|
shell=bool(shell_int),
|
|
513
522
|
)
|
|
514
523
|
yield f"<pre>{html.escape(line)}</pre>"
|
|
515
|
-
if stdin
|
|
516
|
-
yield '<div class="indent"><p>
|
|
524
|
+
if stdin != "":
|
|
525
|
+
yield '<div class="indent"><p>stdin:</p>'
|
|
517
526
|
yield f"<pre>{html.escape(stdin)}</pre></div>"
|
|
527
|
+
if stdout != "":
|
|
528
|
+
yield '<div class="indent"><p>stdout:</p>'
|
|
529
|
+
yield f"<pre>{html.escape(stdout)}</pre></div>"
|
|
530
|
+
if stderr != "":
|
|
531
|
+
yield '<div class="indent"><p>stderr:</p>'
|
|
532
|
+
yield f"<pre>{html.escape(stderr)}</pre></div>"
|
|
518
533
|
|
|
519
534
|
elif kind == "file":
|
|
520
535
|
(state_i, digest, mode, mtime, size, inode) = self.con.execute(
|
|
@@ -103,9 +103,6 @@ class Builder:
|
|
|
103
103
|
infra_env: dict = attrs.field(kw_only=True, factory=dict)
|
|
104
104
|
"""Environment variables from the director for step child processes, overriding `os.environ`."""
|
|
105
105
|
|
|
106
|
-
max_output_size: int = attrs.field(kw_only=True, default=0)
|
|
107
|
-
"""Maximum bytes of stdout/stderr stored per step stream in the DB; 0 = unlimited."""
|
|
108
|
-
|
|
109
106
|
executor: Executor = attrs.field(init=False)
|
|
110
107
|
"""The executor that runs the steps as asyncio tasks in this process."""
|
|
111
108
|
|
|
@@ -120,7 +117,6 @@ class Builder:
|
|
|
120
117
|
self.explain_rerun,
|
|
121
118
|
mp_ctx=self.mp_ctx,
|
|
122
119
|
infra_env=self.infra_env,
|
|
123
|
-
max_output_size=self.max_output_size,
|
|
124
120
|
)
|
|
125
121
|
|
|
126
122
|
async def loop(self, stop_event: asyncio.Event):
|
|
@@ -106,18 +106,18 @@ async def async_main(
|
|
|
106
106
|
await reporter("DIRECTOR", f"Listening on {args.director_socket} (StepUp Core {version})")
|
|
107
107
|
try:
|
|
108
108
|
returncode = await serve(
|
|
109
|
-
args.director_socket,
|
|
110
|
-
njob,
|
|
111
|
-
reporter,
|
|
112
|
-
args.clean,
|
|
113
|
-
args.duration,
|
|
114
|
-
args.explain_rerun,
|
|
115
|
-
args.
|
|
116
|
-
args.
|
|
117
|
-
args.
|
|
118
|
-
args.
|
|
119
|
-
args.
|
|
120
|
-
db,
|
|
109
|
+
director_socket_path=args.director_socket,
|
|
110
|
+
njob=args.jobs,
|
|
111
|
+
reporter=reporter,
|
|
112
|
+
do_clean=args.clean,
|
|
113
|
+
use_duration=args.duration,
|
|
114
|
+
explain_rerun=args.explain_rerun,
|
|
115
|
+
fix_epoch=args.fix_epoch,
|
|
116
|
+
show_perf=args.show_perf,
|
|
117
|
+
do_watch=args.watch,
|
|
118
|
+
do_watch_first=args.watch_first,
|
|
119
|
+
available_resources=args.resources,
|
|
120
|
+
db=db,
|
|
121
121
|
mp_ctx=mp_ctx,
|
|
122
122
|
)
|
|
123
123
|
except Exception as exc:
|
|
@@ -178,6 +178,14 @@ def parse_args() -> argparse.Namespace:
|
|
|
178
178
|
"When given as a real number with digits after the comma, "
|
|
179
179
|
"it is multiplied with the number of available cores. [default=%(default)s]",
|
|
180
180
|
)
|
|
181
|
+
parser.add_argument(
|
|
182
|
+
"--fix-epoch",
|
|
183
|
+
default=True,
|
|
184
|
+
action=argparse.BooleanOptionalAction,
|
|
185
|
+
help="Set the SOURCE_DATE_EPOCH environment variable to 315532800. "
|
|
186
|
+
"This corresponds to 1980-01-01 00:00:00 UTC. "
|
|
187
|
+
"(If the variable is already set, it will be used as-is.) ",
|
|
188
|
+
)
|
|
181
189
|
parser.add_argument(
|
|
182
190
|
"--fork-runpy",
|
|
183
191
|
default=False,
|
|
@@ -204,12 +212,6 @@ def parse_args() -> argparse.Namespace:
|
|
|
204
212
|
action=argparse.BooleanOptionalAction,
|
|
205
213
|
help="Add performance details after completed step.",
|
|
206
214
|
)
|
|
207
|
-
parser.add_argument(
|
|
208
|
-
"--max-output-size",
|
|
209
|
-
type=int,
|
|
210
|
-
default=0,
|
|
211
|
-
help="Maximum bytes of stdout/stderr stored per step stream; 0 = unlimited.",
|
|
212
|
-
)
|
|
213
215
|
parser.add_argument(
|
|
214
216
|
"--resources",
|
|
215
217
|
default=None,
|
|
@@ -256,17 +258,18 @@ def interpret_jobs(jobs: Decimal) -> int:
|
|
|
256
258
|
|
|
257
259
|
|
|
258
260
|
async def serve(
|
|
261
|
+
*,
|
|
259
262
|
director_socket_path: Path,
|
|
260
263
|
njob: int,
|
|
261
264
|
reporter: ReporterClient,
|
|
262
265
|
do_clean: bool,
|
|
263
266
|
use_duration: bool,
|
|
264
267
|
explain_rerun: bool,
|
|
268
|
+
fix_epoch: bool,
|
|
265
269
|
show_perf: bool,
|
|
266
270
|
do_watch: bool,
|
|
267
271
|
do_watch_first: bool,
|
|
268
272
|
available_resources: str | None,
|
|
269
|
-
max_output_size: int,
|
|
270
273
|
db: DBSession,
|
|
271
274
|
mp_ctx=None,
|
|
272
275
|
) -> ReturnCode:
|
|
@@ -298,9 +301,6 @@ async def serve(
|
|
|
298
301
|
available_resources
|
|
299
302
|
A dictionary of named resources and their available quantities,
|
|
300
303
|
e.g. `{"cpu": 4, "gpu": 1}`. Defaults to an empty dict.
|
|
301
|
-
max_output_size
|
|
302
|
-
The maximum number of bytes of stdout/stderr stored per step stream in the
|
|
303
|
-
workflow database. `0` (the default) means unlimited (no truncation).
|
|
304
304
|
mp_ctx
|
|
305
305
|
A `multiprocessing` forkserver context for Python step execution and file hashing,
|
|
306
306
|
or `None` to use plain subprocesses.
|
|
@@ -325,6 +325,8 @@ async def serve(
|
|
|
325
325
|
"STEPUP_ROOT": str(Path.cwd()),
|
|
326
326
|
"STEPUP_LOG_LEVEL": logging.getLevelName(logging.root.level),
|
|
327
327
|
}
|
|
328
|
+
if fix_epoch and "SOURCE_DATE_EPOCH" not in os.environ:
|
|
329
|
+
infra_env["SOURCE_DATE_EPOCH"] = "315532800"
|
|
328
330
|
|
|
329
331
|
# Create basic components
|
|
330
332
|
dir_queue = asyncio.Queue() if do_watch else None
|
|
@@ -347,7 +349,6 @@ async def serve(
|
|
|
347
349
|
do_remove_outdated=do_clean,
|
|
348
350
|
mp_ctx=mp_ctx,
|
|
349
351
|
infra_env=infra_env,
|
|
350
|
-
max_output_size=max_output_size,
|
|
351
352
|
)
|
|
352
353
|
stop_event = asyncio.Event()
|
|
353
354
|
director_handler = DirectorHandler(
|
|
@@ -576,8 +577,10 @@ class DirectorHandler:
|
|
|
576
577
|
workdir: str,
|
|
577
578
|
env_overrides: dict[str, str] | None,
|
|
578
579
|
returncode: int,
|
|
579
|
-
shell: bool
|
|
580
|
-
stdin: str
|
|
580
|
+
shell: bool,
|
|
581
|
+
stdin: str,
|
|
582
|
+
stdout: str,
|
|
583
|
+
stderr: str,
|
|
581
584
|
):
|
|
582
585
|
"""Record a subprocess invocation made by a wrapper step.
|
|
583
586
|
|
|
@@ -588,7 +591,16 @@ class DirectorHandler:
|
|
|
588
591
|
"""
|
|
589
592
|
async with self.db:
|
|
590
593
|
step = self.workflow.node(Step, step_i)
|
|
591
|
-
step.record_subprocess(
|
|
594
|
+
step.record_subprocess(
|
|
595
|
+
cmd,
|
|
596
|
+
workdir,
|
|
597
|
+
env_overrides,
|
|
598
|
+
returncode,
|
|
599
|
+
shell,
|
|
600
|
+
stdin,
|
|
601
|
+
stdout,
|
|
602
|
+
stderr,
|
|
603
|
+
)
|
|
592
604
|
|
|
593
605
|
@allow_rpc
|
|
594
606
|
async def getinfo(self, step_i: int) -> StepInfo:
|
|
@@ -611,9 +611,6 @@ class Executor:
|
|
|
611
611
|
infra_env: dict = attrs.field(kw_only=True, factory=dict)
|
|
612
612
|
"""Environment variables from the director for step child processes, overriding `os.environ`."""
|
|
613
613
|
|
|
614
|
-
max_output_size: int = attrs.field(kw_only=True, default=0)
|
|
615
|
-
"""Maximum bytes of stdout/stderr stored per step stream in the DB; 0 = unlimited."""
|
|
616
|
-
|
|
617
614
|
running: set = attrs.field(init=False, factory=set)
|
|
618
615
|
"""The set of `Run` instances whose command is currently running."""
|
|
619
616
|
|
|
@@ -736,8 +733,8 @@ class Executor:
|
|
|
736
733
|
# so a crash cannot leave a completed step without its output (or vice
|
|
737
734
|
# versa). run.stdout/run.stderr stay untruncated; store_output truncates a
|
|
738
735
|
# copy internally, so report() below still forwards the full text to the TUI.
|
|
739
|
-
|
|
740
|
-
step.store_output(
|
|
736
|
+
max_output_size = int(os.getenv("STEPUP_MAX_OUTPUT_SIZE", "0"))
|
|
737
|
+
step.store_output(run.stdout, run.stderr, max_output_size)
|
|
741
738
|
step_counts = self.workflow.get_step_counts()
|
|
742
739
|
await self.reporter.update_step_counts(step_counts)
|
|
743
740
|
|
|
@@ -35,7 +35,7 @@ from collections.abc import Callable, Collection, Iterator
|
|
|
35
35
|
from path import Path
|
|
36
36
|
|
|
37
37
|
from .path import StrPath, coerce_str, translate
|
|
38
|
-
from .
|
|
38
|
+
from .step import truncate_output
|
|
39
39
|
from .utils import CaseSensitiveTemplate
|
|
40
40
|
|
|
41
41
|
__all__ = (
|
|
@@ -47,18 +47,29 @@ __all__ = (
|
|
|
47
47
|
)
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
def
|
|
51
|
-
"""Return a text representation of `
|
|
50
|
+
def _prepare_stream(data: str | bytes | None, max_size: int = 0) -> str:
|
|
51
|
+
"""Return a text representation of `data` suitable for the archival record.
|
|
52
52
|
|
|
53
|
-
`
|
|
53
|
+
`None` is returned unchanged.
|
|
54
|
+
|
|
55
|
+
`str` remains unchanged if `max_size` is `0` (unlimited)
|
|
56
|
+
or the string is shorter than `max_size`.
|
|
57
|
+
Otherwise, the string is truncated to `max_size` characters
|
|
58
|
+
and a note is appended to indicate that it was truncated.
|
|
59
|
+
|
|
60
|
+
`bytes` are summarized into a short,
|
|
54
61
|
human-readable placeholder (byte length and a truncated SHA-256), because the
|
|
55
|
-
archival `stdin`
|
|
56
|
-
nor meaningful to a human inspecting the database.
|
|
62
|
+
archival columns (`stdin`, `stdout`, `stderr`) are `TEXT` and a raw binary blob
|
|
63
|
+
is neither valid UTF-8 nor meaningful to a human inspecting the database.
|
|
57
64
|
"""
|
|
58
|
-
if
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
65
|
+
if data is None:
|
|
66
|
+
return ""
|
|
67
|
+
if isinstance(data, str):
|
|
68
|
+
return truncate_output(data, max_size)
|
|
69
|
+
if isinstance(data, bytes):
|
|
70
|
+
digest = hashlib.sha256(data).hexdigest()[:16]
|
|
71
|
+
return f"<{len(data)} bytes of binary data, sha256={digest}>"
|
|
72
|
+
raise TypeError(f"data must be str, bytes, or None, not {type(data).__name__}")
|
|
62
73
|
|
|
63
74
|
|
|
64
75
|
@contextlib.contextmanager
|
|
@@ -113,6 +124,8 @@ def record_subprocess(
|
|
|
113
124
|
env_overrides: dict[str, str] | None = None,
|
|
114
125
|
shell: bool = False,
|
|
115
126
|
stdin: str | bytes | None = None,
|
|
127
|
+
stdout: str | bytes | None = None,
|
|
128
|
+
stderr: str | bytes | None = None,
|
|
116
129
|
) -> None:
|
|
117
130
|
"""Record a subprocess invocation (already run by the caller) for archival purposes.
|
|
118
131
|
|
|
@@ -146,25 +159,30 @@ def record_subprocess(
|
|
|
146
159
|
shell
|
|
147
160
|
Whether `cmd` was executed via a shell (i.e. `subprocess.run(..., shell=True)`).
|
|
148
161
|
This is stored and used when formatting the invocation for display.
|
|
149
|
-
stdin
|
|
150
|
-
The standard input
|
|
151
|
-
A `str` is stored verbatim
|
|
152
|
-
|
|
153
|
-
|
|
162
|
+
stdin, stdout, stderr
|
|
163
|
+
The standard input/output/error of the subprocess, or `None` when not captured.
|
|
164
|
+
A `str` is stored verbatim (subject to the director's `--max-output-size` cap for
|
|
165
|
+
`stdout`/`stderr`). `bytes` (e.g. a pickle blob) are not stored raw; they are
|
|
166
|
+
recorded as a short summary (byte length and a truncated SHA-256), since the
|
|
167
|
+
archival record is `TEXT` and informative rather than authoritative.
|
|
154
168
|
"""
|
|
155
169
|
from stepup.core.api import RPC_CLIENT, _get_step_i # noqa: PLC0415
|
|
156
170
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
171
|
+
step_i = _get_step_i()
|
|
172
|
+
if step_i < 0:
|
|
173
|
+
return
|
|
174
|
+
max_output_size = int(os.getenv("STEPUP_MAX_OUTPUT_SIZE", "0"))
|
|
175
|
+
RPC_CLIENT.call.record_subprocess(
|
|
176
|
+
step_i,
|
|
177
|
+
cmd,
|
|
178
|
+
translate(workdir),
|
|
179
|
+
env_overrides,
|
|
180
|
+
returncode,
|
|
181
|
+
shell,
|
|
182
|
+
_prepare_stream(stdin, max_output_size),
|
|
183
|
+
_prepare_stream(stdout, max_output_size),
|
|
184
|
+
_prepare_stream(stderr, max_output_size),
|
|
185
|
+
)
|
|
168
186
|
|
|
169
187
|
|
|
170
188
|
def run_subprocess(
|
|
@@ -179,7 +197,8 @@ def run_subprocess(
|
|
|
179
197
|
"""Run a subprocess and record it for archival purposes.
|
|
180
198
|
|
|
181
199
|
This is the convenience wrapper for the case where an extension step wraps an executable.
|
|
182
|
-
The invocation
|
|
200
|
+
The invocation, its return code, and its captured stdout/stderr are recorded via
|
|
201
|
+
`record_subprocess` (subject to the `STEPUP_MAX_OUTPUT_SIZE` cap).
|
|
183
202
|
|
|
184
203
|
Parameters
|
|
185
204
|
----------
|
|
@@ -268,7 +287,14 @@ def run_subprocess(
|
|
|
268
287
|
argv = shlex.split(cmd)
|
|
269
288
|
cp = subprocess.run(argv, **run_kwargs) # noqa: PLW1510
|
|
270
289
|
record_subprocess(
|
|
271
|
-
cmd,
|
|
290
|
+
cmd,
|
|
291
|
+
cp.returncode,
|
|
292
|
+
workdir=workdir,
|
|
293
|
+
env_overrides=env_overrides,
|
|
294
|
+
shell=shell,
|
|
295
|
+
stdin=stdin,
|
|
296
|
+
stdout=cp.stdout,
|
|
297
|
+
stderr=cp.stderr,
|
|
272
298
|
)
|
|
273
299
|
if check and cp.returncode != 0:
|
|
274
300
|
if cp.stdout:
|
|
@@ -70,6 +70,10 @@ CREATE TABLE IF NOT EXISTS step (
|
|
|
70
70
|
env_overrides TEXT,
|
|
71
71
|
-- JSON-encoded dict[str, str] of step-specific environment variable overrides, or NULL.
|
|
72
72
|
-- Applied to the child process environment when the step runs.
|
|
73
|
+
stdout TEXT NOT NULL DEFAULT '',
|
|
74
|
+
-- Captured standard output of the step's command.
|
|
75
|
+
stderr TEXT NOT NULL DEFAULT '',
|
|
76
|
+
-- Captured standard error of the step's command.
|
|
73
77
|
|
|
74
78
|
-- Metadata
|
|
75
79
|
_safe INTEGER NOT NULL CHECK(_safe IN (0, 1)),
|
|
@@ -139,16 +143,6 @@ CREATE TABLE IF NOT EXISTS step_resource (
|
|
|
139
143
|
) WITHOUT ROWID;
|
|
140
144
|
CREATE INDEX IF NOT EXISTS step_resource_name ON step_resource(name);
|
|
141
145
|
|
|
142
|
-
CREATE TABLE IF NOT EXISTS step_output (
|
|
143
|
-
node INTEGER NOT NULL,
|
|
144
|
-
kind TEXT NOT NULL, -- 'stdout' or 'stderr' (extensible)
|
|
145
|
-
content TEXT NOT NULL,
|
|
146
|
-
PRIMARY KEY (node, kind),
|
|
147
|
-
-- ON DELETE CASCADE removes these rows when the node row is deleted, matching the
|
|
148
|
-
-- other satellite tables (step_hash / env_var / step_resource / step_subprocess).
|
|
149
|
-
FOREIGN KEY (node) REFERENCES node(i) ON DELETE CASCADE
|
|
150
|
-
) WITHOUT ROWID;
|
|
151
|
-
|
|
152
146
|
CREATE TABLE IF NOT EXISTS step_subprocess (
|
|
153
147
|
node INTEGER NOT NULL, -- step node
|
|
154
148
|
seq INTEGER NOT NULL, -- order of invocation within the step
|
|
@@ -158,9 +152,11 @@ CREATE TABLE IF NOT EXISTS step_subprocess (
|
|
|
158
152
|
returncode INTEGER NOT NULL,
|
|
159
153
|
shell INTEGER NOT NULL DEFAULT 0, -- 1 if cmd was run via a shell
|
|
160
154
|
stdin TEXT, -- standard input fed to the subprocess, or NULL
|
|
155
|
+
stdout TEXT, -- captured standard output, or NULL if not captured
|
|
156
|
+
stderr TEXT, -- captured standard error, or NULL if not captured
|
|
161
157
|
PRIMARY KEY (node, seq),
|
|
162
158
|
-- ON DELETE CASCADE removes these rows when the node row is deleted, matching the
|
|
163
|
-
-- other satellite tables (step_hash / env_var / step_resource
|
|
159
|
+
-- other satellite tables (step_hash / env_var / step_resource).
|
|
164
160
|
-- Step.clean_before_run() still clears them explicitly between runs of a surviving step.
|
|
165
161
|
FOREIGN KEY (node) REFERENCES node(i) ON DELETE CASCADE
|
|
166
162
|
) WITHOUT ROWID;
|
|
@@ -279,7 +275,7 @@ class Step(Node):
|
|
|
279
275
|
self.db.execute(
|
|
280
276
|
"INSERT OR REPLACE INTO step "
|
|
281
277
|
"VALUES(:node, :state, :need, 1.0, '', :subshell, NULL, "
|
|
282
|
-
":safe, :check_safe, :implied_need, 1.0, :check_need)",
|
|
278
|
+
"NULL, NULL, :safe, :check_safe, :implied_need, 1.0, :check_need)",
|
|
283
279
|
{
|
|
284
280
|
"node": self.i,
|
|
285
281
|
"need": need.value,
|
|
@@ -342,7 +338,7 @@ class Step(Node):
|
|
|
342
338
|
"""Perform a cleanup right before the detached node is removed from the graph.
|
|
343
339
|
|
|
344
340
|
The satellite rows (step, env_var, nglob_multi, step_hash, step_resource,
|
|
345
|
-
|
|
341
|
+
step_subprocess) are removed automatically by `ON DELETE CASCADE`
|
|
346
342
|
when the node row is deleted, so only the dependency edges are handled here.
|
|
347
343
|
"""
|
|
348
344
|
self.del_suppliers()
|
|
@@ -809,41 +805,37 @@ class Step(Node):
|
|
|
809
805
|
def delete_hash(self):
|
|
810
806
|
self.db.execute("DELETE FROM step_hash WHERE node = ?", (self.i,))
|
|
811
807
|
|
|
812
|
-
def store_output(self,
|
|
813
|
-
"""Persist captured
|
|
814
|
-
|
|
815
|
-
Any previously stored row for this `(node, kind)` pair is removed first,
|
|
816
|
-
so a stream that produced output on an earlier run but is empty now does not
|
|
817
|
-
leave a stale row.
|
|
808
|
+
def store_output(self, stdout: str, stderr: str, max_size: int) -> None:
|
|
809
|
+
"""Persist captured stdout/stderr for this step in a single update.
|
|
818
810
|
|
|
819
811
|
Parameters
|
|
820
812
|
----------
|
|
821
|
-
|
|
822
|
-
The
|
|
823
|
-
|
|
824
|
-
The
|
|
813
|
+
stdout
|
|
814
|
+
The captured standard output of the step's command (untruncated).
|
|
815
|
+
stderr
|
|
816
|
+
The captured standard error of the step's command (untruncated).
|
|
825
817
|
max_size
|
|
826
|
-
Maximum number of UTF-8 bytes to store, or `0` for unlimited.
|
|
818
|
+
Maximum number of UTF-8 bytes to store per stream, or `0` for unlimited.
|
|
827
819
|
See `truncate_output`.
|
|
828
820
|
"""
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
821
|
+
self.db.execute(
|
|
822
|
+
"UPDATE step SET stdout = ?, stderr = ? WHERE node = ?",
|
|
823
|
+
(
|
|
824
|
+
truncate_output(stdout, max_size),
|
|
825
|
+
truncate_output(stderr, max_size),
|
|
826
|
+
self.i,
|
|
827
|
+
),
|
|
828
|
+
)
|
|
837
829
|
|
|
838
|
-
def get_output(self
|
|
839
|
-
"""Return the stored
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
830
|
+
def get_output(self) -> tuple[str, str]:
|
|
831
|
+
"""Return the stored (stdout, stderr) for this step, as empty strings if absent."""
|
|
832
|
+
return self.db.execute(
|
|
833
|
+
"SELECT stdout, stderr FROM step WHERE node = ?", (self.i,)
|
|
834
|
+
).fetchone()
|
|
843
835
|
|
|
844
836
|
def delete_outputs(self) -> None:
|
|
845
|
-
"""Remove
|
|
846
|
-
self.db.execute("
|
|
837
|
+
"""Remove the stored stdout/stderr for this step."""
|
|
838
|
+
self.db.execute("UPDATE step SET stdout = '', stderr = '' WHERE node = ?", (self.i,))
|
|
847
839
|
|
|
848
840
|
def record_subprocess(
|
|
849
841
|
self,
|
|
@@ -851,8 +843,10 @@ class Step(Node):
|
|
|
851
843
|
workdir: str,
|
|
852
844
|
env_overrides: dict[str, str] | None,
|
|
853
845
|
returncode: int,
|
|
854
|
-
shell: bool
|
|
855
|
-
stdin: str
|
|
846
|
+
shell: bool,
|
|
847
|
+
stdin: str,
|
|
848
|
+
stdout: str,
|
|
849
|
+
stderr: str,
|
|
856
850
|
) -> None:
|
|
857
851
|
"""Record a subprocess invocation made by this (wrapper) step.
|
|
858
852
|
|
|
@@ -874,8 +868,9 @@ class Step(Node):
|
|
|
874
868
|
shell
|
|
875
869
|
Whether `cmd` was executed via a shell (`subprocess.run(..., shell=True)`).
|
|
876
870
|
stdin
|
|
877
|
-
The standard input fed to the subprocess as a string
|
|
878
|
-
|
|
871
|
+
The standard input fed to the subprocess as a string.
|
|
872
|
+
stdout, stderr
|
|
873
|
+
The captured standard output/error of the subprocess as a string.
|
|
879
874
|
"""
|
|
880
875
|
# The per-step sequence number is assigned here, under the director's DBSession,
|
|
881
876
|
# so concurrent steps cannot collide and a re-run (which clears the rows first)
|
|
@@ -885,8 +880,8 @@ class Step(Node):
|
|
|
885
880
|
).fetchone()
|
|
886
881
|
self.db.execute(
|
|
887
882
|
"INSERT INTO step_subprocess "
|
|
888
|
-
"(node, seq, cmd, workdir, env_overrides, returncode, shell, stdin) "
|
|
889
|
-
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
883
|
+
"(node, seq, cmd, workdir, env_overrides, returncode, shell, stdin, stdout, stderr) "
|
|
884
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
890
885
|
(
|
|
891
886
|
self.i,
|
|
892
887
|
seq,
|
|
@@ -896,6 +891,8 @@ class Step(Node):
|
|
|
896
891
|
returncode,
|
|
897
892
|
int(shell),
|
|
898
893
|
stdin,
|
|
894
|
+
stdout,
|
|
895
|
+
stderr,
|
|
899
896
|
),
|
|
900
897
|
)
|
|
901
898
|
|
|
@@ -124,15 +124,16 @@ async def async_build(args: argparse.Namespace, default_resources: str):
|
|
|
124
124
|
argv.append("--fork-runpy")
|
|
125
125
|
if args.preload_modules:
|
|
126
126
|
argv.append(f"--preload-modules={args.preload_modules}")
|
|
127
|
+
if not args.clean:
|
|
128
|
+
argv.append("--no-clean")
|
|
127
129
|
if not args.duration:
|
|
128
130
|
argv.append("--no-duration")
|
|
129
131
|
if args.explain_rerun:
|
|
130
132
|
argv.append("--explain-rerun")
|
|
131
|
-
if not args.
|
|
132
|
-
argv.append("--no-
|
|
133
|
+
if not args.fix_epoch:
|
|
134
|
+
argv.append("--no-fix-epoch")
|
|
133
135
|
if args.show_perf > 1:
|
|
134
136
|
argv.append("--show-perf")
|
|
135
|
-
argv.append(f"--max-output-size={args.max_output_size}")
|
|
136
137
|
args.resources = merge_resources(default_resources, args.resources)
|
|
137
138
|
if args.resources:
|
|
138
139
|
argv.append(f"--resources={args.resources}")
|
|
@@ -313,6 +314,14 @@ def _add_build_parser(subparsers, loader: ConfigLoader, name: str, help_text: st
|
|
|
313
314
|
"When given as a real number with digits after the comma, "
|
|
314
315
|
"it is multiplied with the number of available cores. [default=%(default)s]",
|
|
315
316
|
)
|
|
317
|
+
parser.add_argument(
|
|
318
|
+
"--fix-epoch",
|
|
319
|
+
default=True,
|
|
320
|
+
action=argparse.BooleanOptionalAction,
|
|
321
|
+
help="Set the SOURCE_DATE_EPOCH environment variable to 315532800. "
|
|
322
|
+
"This corresponds to 1980-01-01 00:00:00 UTC. "
|
|
323
|
+
"(If the variable is already set, it will be used as-is.) ",
|
|
324
|
+
)
|
|
316
325
|
parser.add_argument(
|
|
317
326
|
"--fork-runpy",
|
|
318
327
|
default=(sys.platform == "linux"),
|
|
@@ -348,14 +357,6 @@ def _add_build_parser(subparsers, loader: ConfigLoader, name: str, help_text: st
|
|
|
348
357
|
action="count",
|
|
349
358
|
help="Show the performance info on each line. Repeat for more detailed info.",
|
|
350
359
|
)
|
|
351
|
-
parser.add_argument(
|
|
352
|
-
"--max-output-size",
|
|
353
|
-
type=int,
|
|
354
|
-
default=0,
|
|
355
|
-
help="Maximum number of bytes of stdout/stderr stored per step stream in the "
|
|
356
|
-
"database; 0 means unlimited. The full output is always shown in the TUI. "
|
|
357
|
-
"[default: %(default)s]",
|
|
358
|
-
)
|
|
359
360
|
resources_action = parser.add_argument(
|
|
360
361
|
"--resources",
|
|
361
362
|
"-r",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|