stepup 4.0.0rc4__tar.gz → 4.0.0rc5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {stepup-4.0.0rc4/stepup.egg-info → stepup-4.0.0rc5}/PKG-INFO +1 -1
  2. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/browse.py +29 -14
  3. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/builder.py +0 -4
  4. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/director.py +38 -26
  5. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/executor.py +2 -5
  6. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/extapi.py +54 -28
  7. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/step.py +42 -45
  8. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/tui.py +12 -11
  9. {stepup-4.0.0rc4 → stepup-4.0.0rc5/stepup.egg-info}/PKG-INFO +1 -1
  10. stepup-4.0.0rc5/stepup.egg-info/scm_version.json +8 -0
  11. stepup-4.0.0rc4/stepup.egg-info/scm_version.json +0 -8
  12. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/CLAUDE.md +0 -0
  13. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/LICENSE +0 -0
  14. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/MANIFEST.in +0 -0
  15. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/README.md +0 -0
  16. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/pyproject.toml +0 -0
  17. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/setup.cfg +0 -0
  18. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/__init__.py +0 -0
  19. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/__main__.py +0 -0
  20. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/api.py +0 -0
  21. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/asyncio.py +0 -0
  22. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/call.py +0 -0
  23. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/cattrs.py +0 -0
  24. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/clean.py +0 -0
  25. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/config.py +0 -0
  26. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/constants.py +0 -0
  27. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/enums.py +0 -0
  28. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/exceptions.py +0 -0
  29. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/file.py +0 -0
  30. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/hash.py +0 -0
  31. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/hasher.py +0 -0
  32. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/interact.py +0 -0
  33. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/job.py +0 -0
  34. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/logo.svg +0 -0
  35. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/nglob.py +0 -0
  36. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/path.py +0 -0
  37. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/pytest.py +0 -0
  38. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/render_jinja.py +0 -0
  39. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/reporter.py +0 -0
  40. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/rpc.py +0 -0
  41. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/scheduler.py +0 -0
  42. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/script.py +0 -0
  43. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/sqlite3.py +0 -0
  44. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/startup.py +0 -0
  45. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/static_tree.py +0 -0
  46. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/stepinfo.py +0 -0
  47. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/trellis.py +0 -0
  48. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/utils.py +0 -0
  49. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/watcher.py +0 -0
  50. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup/core/workflow.py +0 -0
  51. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/SOURCES.txt +0 -0
  52. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/dependency_links.txt +0 -0
  53. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/entry_points.txt +0 -0
  54. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/requires.txt +0 -0
  55. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/scm_file_list.json +0 -0
  56. {stepup-4.0.0rc4 → stepup-4.0.0rc5}/stepup.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup
3
- Version: 4.0.0rc4
3
+ Version: 4.0.0rc5
4
4
  Summary: StepUp Core provides the basic framework for the StepUp build tool
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: GPL-3.0-or-later
@@ -405,7 +405,7 @@ class GraphServer(BaseHTTPRequestHandler):
405
405
  if kind == "step":
406
406
  sql_props = (
407
407
  "SELECT state, need, duration, rescheduled_info, subshell, env_overrides,"
408
- "_safe, _check_safe, _implied_need, _tail_time, _check_after "
408
+ "stdout, stderr, _safe, _check_safe, _implied_need, _tail_time, _check_after "
409
409
  "FROM step WHERE node = ?"
410
410
  )
411
411
  (
@@ -415,6 +415,8 @@ class GraphServer(BaseHTTPRequestHandler):
415
415
  rescheduled_info,
416
416
  subshell,
417
417
  env_overrides,
418
+ stdout,
419
+ stderr,
418
420
  safe,
419
421
  check_safe,
420
422
  implied_need_id,
@@ -487,23 +489,30 @@ class GraphServer(BaseHTTPRequestHandler):
487
489
  ngm = pickle.loads(row[0])
488
490
  yield f"<p>{[ngs.pattern for ngs in ngm.nglob_singles]} {ngm.subs}</p>"
489
491
 
490
- sql_out = "SELECT kind, content FROM step_output WHERE node = ? ORDER BY kind"
491
- first = True
492
- for out_kind, out_content in self.con.execute(sql_out, (node_i,)):
493
- if first:
494
- yield "<h3>Output</h3>"
495
- first = False
496
- yield f"<p>{out_kind}:</p>"
497
- yield f"<pre>{html.escape(out_content)}</pre>"
492
+ if stdout != "":
493
+ yield "<h3>Standard Output</h3>"
494
+ yield f"<pre>{html.escape(stdout)}</pre>"
495
+ if stderr != "":
496
+ yield "<h3>Standard Error</h3>"
497
+ yield f"<pre>{html.escape(stderr)}</pre>"
498
498
 
499
499
  sql_sub = (
500
- "SELECT cmd, workdir, env_overrides, returncode, shell, stdin FROM step_subprocess "
501
- "WHERE node = ? ORDER BY seq"
500
+ "SELECT cmd, workdir, env_overrides, returncode, shell, stdin, stdout, stderr "
501
+ "FROM step_subprocess WHERE node = ? ORDER BY seq"
502
502
  )
503
503
  subs = list(self.con.execute(sql_sub, (node_i,)))
504
504
  if len(subs) > 0:
505
505
  yield "<h3>Subprocesses</h3>"
506
- for cmd, workdir, env_overrides, returncode, shell_int, stdin in subs:
506
+ for (
507
+ cmd,
508
+ workdir,
509
+ env_overrides,
510
+ returncode,
511
+ shell_int,
512
+ stdin,
513
+ stdout,
514
+ stderr,
515
+ ) in subs:
507
516
  line = format_subprocess(
508
517
  cmd,
509
518
  workdir,
@@ -512,9 +521,15 @@ class GraphServer(BaseHTTPRequestHandler):
512
521
  shell=bool(shell_int),
513
522
  )
514
523
  yield f"<pre>{html.escape(line)}</pre>"
515
- if stdin is not None:
516
- yield '<div class="indent"><p>with stdin:</p>'
524
+ if stdin != "":
525
+ yield '<div class="indent"><p>stdin:</p>'
517
526
  yield f"<pre>{html.escape(stdin)}</pre></div>"
527
+ if stdout != "":
528
+ yield '<div class="indent"><p>stdout:</p>'
529
+ yield f"<pre>{html.escape(stdout)}</pre></div>"
530
+ if stderr != "":
531
+ yield '<div class="indent"><p>stderr:</p>'
532
+ yield f"<pre>{html.escape(stderr)}</pre></div>"
518
533
 
519
534
  elif kind == "file":
520
535
  (state_i, digest, mode, mtime, size, inode) = self.con.execute(
@@ -103,9 +103,6 @@ class Builder:
103
103
  infra_env: dict = attrs.field(kw_only=True, factory=dict)
104
104
  """Environment variables from the director for step child processes, overriding `os.environ`."""
105
105
 
106
- max_output_size: int = attrs.field(kw_only=True, default=0)
107
- """Maximum bytes of stdout/stderr stored per step stream in the DB; 0 = unlimited."""
108
-
109
106
  executor: Executor = attrs.field(init=False)
110
107
  """The executor that runs the steps as asyncio tasks in this process."""
111
108
 
@@ -120,7 +117,6 @@ class Builder:
120
117
  self.explain_rerun,
121
118
  mp_ctx=self.mp_ctx,
122
119
  infra_env=self.infra_env,
123
- max_output_size=self.max_output_size,
124
120
  )
125
121
 
126
122
  async def loop(self, stop_event: asyncio.Event):
@@ -106,18 +106,18 @@ async def async_main(
106
106
  await reporter("DIRECTOR", f"Listening on {args.director_socket} (StepUp Core {version})")
107
107
  try:
108
108
  returncode = await serve(
109
- args.director_socket,
110
- njob,
111
- reporter,
112
- args.clean,
113
- args.duration,
114
- args.explain_rerun,
115
- args.show_perf,
116
- args.watch,
117
- args.watch_first,
118
- args.resources,
119
- args.max_output_size,
120
- db,
109
+ director_socket_path=args.director_socket,
110
+ njob=args.jobs,
111
+ reporter=reporter,
112
+ do_clean=args.clean,
113
+ use_duration=args.duration,
114
+ explain_rerun=args.explain_rerun,
115
+ fix_epoch=args.fix_epoch,
116
+ show_perf=args.show_perf,
117
+ do_watch=args.watch,
118
+ do_watch_first=args.watch_first,
119
+ available_resources=args.resources,
120
+ db=db,
121
121
  mp_ctx=mp_ctx,
122
122
  )
123
123
  except Exception as exc:
@@ -178,6 +178,14 @@ def parse_args() -> argparse.Namespace:
178
178
  "When given as a real number with digits after the comma, "
179
179
  "it is multiplied with the number of available cores. [default=%(default)s]",
180
180
  )
181
+ parser.add_argument(
182
+ "--fix-epoch",
183
+ default=True,
184
+ action=argparse.BooleanOptionalAction,
185
+ help="Set the SOURCE_DATE_EPOCH environment variable to 315532800. "
186
+ "This corresponds to 1980-01-01 00:00:00 UTC. "
187
+ "(If the variable is already set, it will be used as-is.) ",
188
+ )
181
189
  parser.add_argument(
182
190
  "--fork-runpy",
183
191
  default=False,
@@ -204,12 +212,6 @@ def parse_args() -> argparse.Namespace:
204
212
  action=argparse.BooleanOptionalAction,
205
213
  help="Add performance details after completed step.",
206
214
  )
207
- parser.add_argument(
208
- "--max-output-size",
209
- type=int,
210
- default=0,
211
- help="Maximum bytes of stdout/stderr stored per step stream; 0 = unlimited.",
212
- )
213
215
  parser.add_argument(
214
216
  "--resources",
215
217
  default=None,
@@ -256,17 +258,18 @@ def interpret_jobs(jobs: Decimal) -> int:
256
258
 
257
259
 
258
260
  async def serve(
261
+ *,
259
262
  director_socket_path: Path,
260
263
  njob: int,
261
264
  reporter: ReporterClient,
262
265
  do_clean: bool,
263
266
  use_duration: bool,
264
267
  explain_rerun: bool,
268
+ fix_epoch: bool,
265
269
  show_perf: bool,
266
270
  do_watch: bool,
267
271
  do_watch_first: bool,
268
272
  available_resources: str | None,
269
- max_output_size: int,
270
273
  db: DBSession,
271
274
  mp_ctx=None,
272
275
  ) -> ReturnCode:
@@ -298,9 +301,6 @@ async def serve(
298
301
  available_resources
299
302
  A dictionary of named resources and their available quantities,
300
303
  e.g. `{"cpu": 4, "gpu": 1}`. Defaults to an empty dict.
301
- max_output_size
302
- The maximum number of bytes of stdout/stderr stored per step stream in the
303
- workflow database. `0` (the default) means unlimited (no truncation).
304
304
  mp_ctx
305
305
  A `multiprocessing` forkserver context for Python step execution and file hashing,
306
306
  or `None` to use plain subprocesses.
@@ -325,6 +325,8 @@ async def serve(
325
325
  "STEPUP_ROOT": str(Path.cwd()),
326
326
  "STEPUP_LOG_LEVEL": logging.getLevelName(logging.root.level),
327
327
  }
328
+ if fix_epoch and "SOURCE_DATE_EPOCH" not in os.environ:
329
+ infra_env["SOURCE_DATE_EPOCH"] = "315532800"
328
330
 
329
331
  # Create basic components
330
332
  dir_queue = asyncio.Queue() if do_watch else None
@@ -347,7 +349,6 @@ async def serve(
347
349
  do_remove_outdated=do_clean,
348
350
  mp_ctx=mp_ctx,
349
351
  infra_env=infra_env,
350
- max_output_size=max_output_size,
351
352
  )
352
353
  stop_event = asyncio.Event()
353
354
  director_handler = DirectorHandler(
@@ -576,8 +577,10 @@ class DirectorHandler:
576
577
  workdir: str,
577
578
  env_overrides: dict[str, str] | None,
578
579
  returncode: int,
579
- shell: bool = False,
580
- stdin: str | None = None,
580
+ shell: bool,
581
+ stdin: str,
582
+ stdout: str,
583
+ stderr: str,
581
584
  ):
582
585
  """Record a subprocess invocation made by a wrapper step.
583
586
 
@@ -588,7 +591,16 @@ class DirectorHandler:
588
591
  """
589
592
  async with self.db:
590
593
  step = self.workflow.node(Step, step_i)
591
- step.record_subprocess(cmd, workdir, env_overrides, returncode, shell, stdin)
594
+ step.record_subprocess(
595
+ cmd,
596
+ workdir,
597
+ env_overrides,
598
+ returncode,
599
+ shell,
600
+ stdin,
601
+ stdout,
602
+ stderr,
603
+ )
592
604
 
593
605
  @allow_rpc
594
606
  async def getinfo(self, step_i: int) -> StepInfo:
@@ -611,9 +611,6 @@ class Executor:
611
611
  infra_env: dict = attrs.field(kw_only=True, factory=dict)
612
612
  """Environment variables from the director for step child processes, overriding `os.environ`."""
613
613
 
614
- max_output_size: int = attrs.field(kw_only=True, default=0)
615
- """Maximum bytes of stdout/stderr stored per step stream in the DB; 0 = unlimited."""
616
-
617
614
  running: set = attrs.field(init=False, factory=set)
618
615
  """The set of `Run` instances whose command is currently running."""
619
616
 
@@ -736,8 +733,8 @@ class Executor:
736
733
  # so a crash cannot leave a completed step without its output (or vice
737
734
  # versa). run.stdout/run.stderr stay untruncated; store_output truncates a
738
735
  # copy internally, so report() below still forwards the full text to the TUI.
739
- step.store_output("stdout", run.stdout, self.max_output_size)
740
- step.store_output("stderr", run.stderr, self.max_output_size)
736
+ max_output_size = int(os.getenv("STEPUP_MAX_OUTPUT_SIZE", "0"))
737
+ step.store_output(run.stdout, run.stderr, max_output_size)
741
738
  step_counts = self.workflow.get_step_counts()
742
739
  await self.reporter.update_step_counts(step_counts)
743
740
 
@@ -35,7 +35,7 @@ from collections.abc import Callable, Collection, Iterator
35
35
  from path import Path
36
36
 
37
37
  from .path import StrPath, coerce_str, translate
38
- from .rpc import SocketSyncRPCClient
38
+ from .step import truncate_output
39
39
  from .utils import CaseSensitiveTemplate
40
40
 
41
41
  __all__ = (
@@ -47,18 +47,29 @@ __all__ = (
47
47
  )
48
48
 
49
49
 
50
- def _summarize_binary_stdin(stdin: str | bytes | None) -> str | None:
51
- """Return a text representation of `stdin` suitable for the archival record.
50
+ def _prepare_stream(data: str | bytes | None, max_size: int = 0) -> str:
51
+ """Return a text representation of `data` suitable for the archival record.
52
52
 
53
- `str` and `None` pass through unchanged. `bytes` are summarized into a short,
53
+ `None` is returned unchanged.
54
+
55
+ `str` remains unchanged if `max_size` is `0` (unlimited)
56
+ or the string is shorter than `max_size`.
57
+ Otherwise, the string is truncated to `max_size` characters
58
+ and a note is appended to indicate that it was truncated.
59
+
60
+ `bytes` are summarized into a short,
54
61
  human-readable placeholder (byte length and a truncated SHA-256), because the
55
- archival `stdin` column is `TEXT` and a raw binary blob is neither valid UTF-8
56
- nor meaningful to a human inspecting the database.
62
+ archival columns (`stdin`, `stdout`, `stderr`) are `TEXT` and a raw binary blob
63
+ is neither valid UTF-8 nor meaningful to a human inspecting the database.
57
64
  """
58
- if isinstance(stdin, bytes):
59
- digest = hashlib.sha256(stdin).hexdigest()[:16]
60
- return f"<{len(stdin)} bytes of binary stdin, sha256={digest}>"
61
- return stdin
65
+ if data is None:
66
+ return ""
67
+ if isinstance(data, str):
68
+ return truncate_output(data, max_size)
69
+ if isinstance(data, bytes):
70
+ digest = hashlib.sha256(data).hexdigest()[:16]
71
+ return f"<{len(data)} bytes of binary data, sha256={digest}>"
72
+ raise TypeError(f"data must be str, bytes, or None, not {type(data).__name__}")
62
73
 
63
74
 
64
75
  @contextlib.contextmanager
@@ -113,6 +124,8 @@ def record_subprocess(
113
124
  env_overrides: dict[str, str] | None = None,
114
125
  shell: bool = False,
115
126
  stdin: str | bytes | None = None,
127
+ stdout: str | bytes | None = None,
128
+ stderr: str | bytes | None = None,
116
129
  ) -> None:
117
130
  """Record a subprocess invocation (already run by the caller) for archival purposes.
118
131
 
@@ -146,25 +159,30 @@ def record_subprocess(
146
159
  shell
147
160
  Whether `cmd` was executed via a shell (i.e. `subprocess.run(..., shell=True)`).
148
161
  This is stored and used when formatting the invocation for display.
149
- stdin
150
- The standard input that was fed to the subprocess, or `None`.
151
- A `str` is stored verbatim. `bytes` (e.g. a pickle blob) are not stored raw;
152
- they are recorded as a short summary (byte length and a truncated SHA-256),
153
- since the archival record is `TEXT` and informative rather than authoritative.
162
+ stdin, stdout, stderr
163
+ The standard input/output/error of the subprocess, or `None` when not captured.
164
+ A `str` is stored verbatim (subject to the director's `--max-output-size` cap for
165
+ `stdout`/`stderr`). `bytes` (e.g. a pickle blob) are not stored raw; they are
166
+ recorded as a short summary (byte length and a truncated SHA-256), since the
167
+ archival record is `TEXT` and informative rather than authoritative.
154
168
  """
155
169
  from stepup.core.api import RPC_CLIENT, _get_step_i # noqa: PLC0415
156
170
 
157
- if isinstance(RPC_CLIENT, SocketSyncRPCClient):
158
- step_i = _get_step_i()
159
- RPC_CLIENT.call.record_subprocess(
160
- step_i,
161
- cmd,
162
- translate(workdir),
163
- env_overrides,
164
- returncode,
165
- shell,
166
- _summarize_binary_stdin(stdin),
167
- )
171
+ step_i = _get_step_i()
172
+ if step_i < 0:
173
+ return
174
+ max_output_size = int(os.getenv("STEPUP_MAX_OUTPUT_SIZE", "0"))
175
+ RPC_CLIENT.call.record_subprocess(
176
+ step_i,
177
+ cmd,
178
+ translate(workdir),
179
+ env_overrides,
180
+ returncode,
181
+ shell,
182
+ _prepare_stream(stdin, max_output_size),
183
+ _prepare_stream(stdout, max_output_size),
184
+ _prepare_stream(stderr, max_output_size),
185
+ )
168
186
 
169
187
 
170
188
  def run_subprocess(
@@ -179,7 +197,8 @@ def run_subprocess(
179
197
  """Run a subprocess and record it for archival purposes.
180
198
 
181
199
  This is the convenience wrapper for the case where an extension step wraps an executable.
182
- The invocation and its return code are recorded via `record_subprocess`.
200
+ The invocation, its return code, and its captured stdout/stderr are recorded via
201
+ `record_subprocess` (subject to the `STEPUP_MAX_OUTPUT_SIZE` cap).
183
202
 
184
203
  Parameters
185
204
  ----------
@@ -268,7 +287,14 @@ def run_subprocess(
268
287
  argv = shlex.split(cmd)
269
288
  cp = subprocess.run(argv, **run_kwargs) # noqa: PLW1510
270
289
  record_subprocess(
271
- cmd, cp.returncode, workdir=workdir, env_overrides=env_overrides, shell=shell, stdin=stdin
290
+ cmd,
291
+ cp.returncode,
292
+ workdir=workdir,
293
+ env_overrides=env_overrides,
294
+ shell=shell,
295
+ stdin=stdin,
296
+ stdout=cp.stdout,
297
+ stderr=cp.stderr,
272
298
  )
273
299
  if check and cp.returncode != 0:
274
300
  if cp.stdout:
@@ -70,6 +70,10 @@ CREATE TABLE IF NOT EXISTS step (
70
70
  env_overrides TEXT,
71
71
  -- JSON-encoded dict[str, str] of step-specific environment variable overrides, or NULL.
72
72
  -- Applied to the child process environment when the step runs.
73
+ stdout TEXT NOT NULL DEFAULT '',
74
+ -- Captured standard output of the step's command.
75
+ stderr TEXT NOT NULL DEFAULT '',
76
+ -- Captured standard error of the step's command.
73
77
 
74
78
  -- Metadata
75
79
  _safe INTEGER NOT NULL CHECK(_safe IN (0, 1)),
@@ -139,16 +143,6 @@ CREATE TABLE IF NOT EXISTS step_resource (
139
143
  ) WITHOUT ROWID;
140
144
  CREATE INDEX IF NOT EXISTS step_resource_name ON step_resource(name);
141
145
 
142
- CREATE TABLE IF NOT EXISTS step_output (
143
- node INTEGER NOT NULL,
144
- kind TEXT NOT NULL, -- 'stdout' or 'stderr' (extensible)
145
- content TEXT NOT NULL,
146
- PRIMARY KEY (node, kind),
147
- -- ON DELETE CASCADE removes these rows when the node row is deleted, matching the
148
- -- other satellite tables (step_hash / env_var / step_resource / step_subprocess).
149
- FOREIGN KEY (node) REFERENCES node(i) ON DELETE CASCADE
150
- ) WITHOUT ROWID;
151
-
152
146
  CREATE TABLE IF NOT EXISTS step_subprocess (
153
147
  node INTEGER NOT NULL, -- step node
154
148
  seq INTEGER NOT NULL, -- order of invocation within the step
@@ -158,9 +152,11 @@ CREATE TABLE IF NOT EXISTS step_subprocess (
158
152
  returncode INTEGER NOT NULL,
159
153
  shell INTEGER NOT NULL DEFAULT 0, -- 1 if cmd was run via a shell
160
154
  stdin TEXT, -- standard input fed to the subprocess, or NULL
155
+ stdout TEXT, -- captured standard output, or NULL if not captured
156
+ stderr TEXT, -- captured standard error, or NULL if not captured
161
157
  PRIMARY KEY (node, seq),
162
158
  -- ON DELETE CASCADE removes these rows when the node row is deleted, matching the
163
- -- other satellite tables (step_hash / env_var / step_resource / step_output).
159
+ -- other satellite tables (step_hash / env_var / step_resource).
164
160
  -- Step.clean_before_run() still clears them explicitly between runs of a surviving step.
165
161
  FOREIGN KEY (node) REFERENCES node(i) ON DELETE CASCADE
166
162
  ) WITHOUT ROWID;
@@ -279,7 +275,7 @@ class Step(Node):
279
275
  self.db.execute(
280
276
  "INSERT OR REPLACE INTO step "
281
277
  "VALUES(:node, :state, :need, 1.0, '', :subshell, NULL, "
282
- ":safe, :check_safe, :implied_need, 1.0, :check_need)",
278
+ "NULL, NULL, :safe, :check_safe, :implied_need, 1.0, :check_need)",
283
279
  {
284
280
  "node": self.i,
285
281
  "need": need.value,
@@ -342,7 +338,7 @@ class Step(Node):
342
338
  """Perform a cleanup right before the detached node is removed from the graph.
343
339
 
344
340
  The satellite rows (step, env_var, nglob_multi, step_hash, step_resource,
345
- step_output, step_subprocess) are removed automatically by `ON DELETE CASCADE`
341
+ step_subprocess) are removed automatically by `ON DELETE CASCADE`
346
342
  when the node row is deleted, so only the dependency edges are handled here.
347
343
  """
348
344
  self.del_suppliers()
@@ -809,41 +805,37 @@ class Step(Node):
809
805
  def delete_hash(self):
810
806
  self.db.execute("DELETE FROM step_hash WHERE node = ?", (self.i,))
811
807
 
812
- def store_output(self, kind: str, content: str, max_size: int) -> None:
813
- """Persist captured output of one stream for this step.
814
-
815
- Any previously stored row for this `(node, kind)` pair is removed first,
816
- so a stream that produced output on an earlier run but is empty now does not
817
- leave a stale row.
808
+ def store_output(self, stdout: str, stderr: str, max_size: int) -> None:
809
+ """Persist captured stdout/stderr for this step in a single update.
818
810
 
819
811
  Parameters
820
812
  ----------
821
- kind
822
- The stream identifier, e.g. `'stdout'` or `'stderr'`.
823
- content
824
- The full captured text (untruncated).
813
+ stdout
814
+ The captured standard output of the step's command (untruncated).
815
+ stderr
816
+ The captured standard error of the step's command (untruncated).
825
817
  max_size
826
- Maximum number of UTF-8 bytes to store, or `0` for unlimited.
818
+ Maximum number of UTF-8 bytes to store per stream, or `0` for unlimited.
827
819
  See `truncate_output`.
828
820
  """
829
- # Delete the existing row for this kind first: INSERT OR REPLACE cannot
830
- # remove a row whose content is now empty, so an explicit DELETE is needed.
831
- self.db.execute("DELETE FROM step_output WHERE node = ? AND kind = ?", (self.i, kind))
832
- if content:
833
- self.db.execute(
834
- "INSERT INTO step_output VALUES (?, ?, ?)",
835
- (self.i, kind, truncate_output(content, max_size)),
836
- )
821
+ self.db.execute(
822
+ "UPDATE step SET stdout = ?, stderr = ? WHERE node = ?",
823
+ (
824
+ truncate_output(stdout, max_size),
825
+ truncate_output(stderr, max_size),
826
+ self.i,
827
+ ),
828
+ )
837
829
 
838
- def get_output(self, kind: str) -> str:
839
- """Return the stored output for one stream, or an empty string if absent."""
840
- sql = "SELECT content FROM step_output WHERE node = ? AND kind = ?"
841
- row = self.db.execute(sql, (self.i, kind)).fetchone()
842
- return row[0] if row else ""
830
+ def get_output(self) -> tuple[str, str]:
831
+ """Return the stored (stdout, stderr) for this step, as empty strings if absent."""
832
+ return self.db.execute(
833
+ "SELECT stdout, stderr FROM step WHERE node = ?", (self.i,)
834
+ ).fetchone()
843
835
 
844
836
  def delete_outputs(self) -> None:
845
- """Remove all stored output rows for this step."""
846
- self.db.execute("DELETE FROM step_output WHERE node = ?", (self.i,))
837
+ """Remove the stored stdout/stderr for this step."""
838
+ self.db.execute("UPDATE step SET stdout = '', stderr = '' WHERE node = ?", (self.i,))
847
839
 
848
840
  def record_subprocess(
849
841
  self,
@@ -851,8 +843,10 @@ class Step(Node):
851
843
  workdir: str,
852
844
  env_overrides: dict[str, str] | None,
853
845
  returncode: int,
854
- shell: bool = False,
855
- stdin: str | None = None,
846
+ shell: bool,
847
+ stdin: str,
848
+ stdout: str,
849
+ stderr: str,
856
850
  ) -> None:
857
851
  """Record a subprocess invocation made by this (wrapper) step.
858
852
 
@@ -874,8 +868,9 @@ class Step(Node):
874
868
  shell
875
869
  Whether `cmd` was executed via a shell (`subprocess.run(..., shell=True)`).
876
870
  stdin
877
- The standard input fed to the subprocess as a string, or `None` when no input
878
- was provided. Stored verbatim.
871
+ The standard input fed to the subprocess as a string.
872
+ stdout, stderr
873
+ The captured standard output/error of the subprocess as a string.
879
874
  """
880
875
  # The per-step sequence number is assigned here, under the director's DBSession,
881
876
  # so concurrent steps cannot collide and a re-run (which clears the rows first)
@@ -885,8 +880,8 @@ class Step(Node):
885
880
  ).fetchone()
886
881
  self.db.execute(
887
882
  "INSERT INTO step_subprocess "
888
- "(node, seq, cmd, workdir, env_overrides, returncode, shell, stdin) "
889
- "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
883
+ "(node, seq, cmd, workdir, env_overrides, returncode, shell, stdin, stdout, stderr) "
884
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
890
885
  (
891
886
  self.i,
892
887
  seq,
@@ -896,6 +891,8 @@ class Step(Node):
896
891
  returncode,
897
892
  int(shell),
898
893
  stdin,
894
+ stdout,
895
+ stderr,
899
896
  ),
900
897
  )
901
898
 
@@ -124,15 +124,16 @@ async def async_build(args: argparse.Namespace, default_resources: str):
124
124
  argv.append("--fork-runpy")
125
125
  if args.preload_modules:
126
126
  argv.append(f"--preload-modules={args.preload_modules}")
127
+ if not args.clean:
128
+ argv.append("--no-clean")
127
129
  if not args.duration:
128
130
  argv.append("--no-duration")
129
131
  if args.explain_rerun:
130
132
  argv.append("--explain-rerun")
131
- if not args.clean:
132
- argv.append("--no-clean")
133
+ if not args.fix_epoch:
134
+ argv.append("--no-fix-epoch")
133
135
  if args.show_perf > 1:
134
136
  argv.append("--show-perf")
135
- argv.append(f"--max-output-size={args.max_output_size}")
136
137
  args.resources = merge_resources(default_resources, args.resources)
137
138
  if args.resources:
138
139
  argv.append(f"--resources={args.resources}")
@@ -313,6 +314,14 @@ def _add_build_parser(subparsers, loader: ConfigLoader, name: str, help_text: st
313
314
  "When given as a real number with digits after the comma, "
314
315
  "it is multiplied with the number of available cores. [default=%(default)s]",
315
316
  )
317
+ parser.add_argument(
318
+ "--fix-epoch",
319
+ default=True,
320
+ action=argparse.BooleanOptionalAction,
321
+ help="Set the SOURCE_DATE_EPOCH environment variable to 315532800. "
322
+ "This corresponds to 1980-01-01 00:00:00 UTC. "
323
+ "(If the variable is already set, it will be used as-is.) ",
324
+ )
316
325
  parser.add_argument(
317
326
  "--fork-runpy",
318
327
  default=(sys.platform == "linux"),
@@ -348,14 +357,6 @@ def _add_build_parser(subparsers, loader: ConfigLoader, name: str, help_text: st
348
357
  action="count",
349
358
  help="Show the performance info on each line. Repeat for more detailed info.",
350
359
  )
351
- parser.add_argument(
352
- "--max-output-size",
353
- type=int,
354
- default=0,
355
- help="Maximum number of bytes of stdout/stderr stored per step stream in the "
356
- "database; 0 means unlimited. The full output is always shown in the TUI. "
357
- "[default: %(default)s]",
358
- )
359
360
  resources_action = parser.add_argument(
360
361
  "--resources",
361
362
  "-r",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup
3
- Version: 4.0.0rc4
3
+ Version: 4.0.0rc5
4
4
  Summary: StepUp Core provides the basic framework for the StepUp build tool
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: GPL-3.0-or-later
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "4.0.0rc5",
3
+ "distance": 0,
4
+ "node": "g3e71666082cc117f5178ca3f224a945f326e0aaf",
5
+ "dirty": false,
6
+ "branch": "HEAD",
7
+ "node_date": "2026-06-30"
8
+ }
@@ -1,8 +0,0 @@
1
- {
2
- "tag": "4.0.0rc4",
3
- "distance": 0,
4
- "node": "gee637b237769106288f71c6327c1b8ce1b48388f",
5
- "dirty": false,
6
- "branch": "HEAD",
7
- "node_date": "2026-06-28"
8
- }
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes