stepup 4.0.1__tar.gz → 4.0.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {stepup-4.0.1 → stepup-4.0.3}/PKG-INFO +5 -4
  2. {stepup-4.0.1 → stepup-4.0.3}/pyproject.toml +6 -3
  3. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/api.py +83 -39
  4. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/director.py +7 -1
  5. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/executor.py +7 -3
  6. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/extapi.py +42 -4
  7. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/path.py +65 -8
  8. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/reporter.py +2 -2
  9. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/watcher.py +1 -1
  10. {stepup-4.0.1 → stepup-4.0.3}/stepup.egg-info/PKG-INFO +5 -4
  11. {stepup-4.0.1 → stepup-4.0.3}/stepup.egg-info/requires.txt +4 -2
  12. {stepup-4.0.1 → stepup-4.0.3}/stepup.egg-info/scm_file_list.json +16 -0
  13. stepup-4.0.3/stepup.egg-info/scm_version.json +8 -0
  14. stepup-4.0.1/stepup.egg-info/scm_version.json +0 -8
  15. {stepup-4.0.1 → stepup-4.0.3}/.zenodo.json +0 -0
  16. {stepup-4.0.1 → stepup-4.0.3}/CITATION.cff +0 -0
  17. {stepup-4.0.1 → stepup-4.0.3}/CLAUDE.md +0 -0
  18. {stepup-4.0.1 → stepup-4.0.3}/CONTRIBUTING.md +0 -0
  19. {stepup-4.0.1 → stepup-4.0.3}/LICENSE +0 -0
  20. {stepup-4.0.1 → stepup-4.0.3}/LICENSES/CC-BY-SA-4.0.txt +0 -0
  21. {stepup-4.0.1 → stepup-4.0.3}/LICENSES/LGPL-3.0-or-later.txt +0 -0
  22. {stepup-4.0.1 → stepup-4.0.3}/LICENSES/LicenseRef-StepUp-Logo-v1.txt +0 -0
  23. {stepup-4.0.1 → stepup-4.0.3}/MANIFEST.in +0 -0
  24. {stepup-4.0.1 → stepup-4.0.3}/README.md +0 -0
  25. {stepup-4.0.1 → stepup-4.0.3}/REUSE.toml +0 -0
  26. {stepup-4.0.1 → stepup-4.0.3}/setup.cfg +0 -0
  27. {stepup-4.0.1 → stepup-4.0.3}/snipwise.md +0 -0
  28. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/CLAUDE.md +0 -0
  29. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/__init__.py +0 -0
  30. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/__main__.py +0 -0
  31. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/asyncio.py +0 -0
  32. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/browse.py +0 -0
  33. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/builder.py +0 -0
  34. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/call.py +0 -0
  35. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/cattrs.py +0 -0
  36. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/cgroups.py +0 -0
  37. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/clean.py +0 -0
  38. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/config_loader.py +0 -0
  39. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/config_tool.py +0 -0
  40. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/constants.py +0 -0
  41. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/enums.py +0 -0
  42. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/exceptions.py +0 -0
  43. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/file.py +0 -0
  44. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/finalize.py +0 -0
  45. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/hash.py +0 -0
  46. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/hash_queue.py +0 -0
  47. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/interact.py +0 -0
  48. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/job.py +0 -0
  49. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/logo.svg +0 -0
  50. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/nglob.py +0 -0
  51. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/outcome.py +0 -0
  52. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/pending.py +0 -0
  53. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/pytest.py +0 -0
  54. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/render_jinja.py +0 -0
  55. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/rpc.py +0 -0
  56. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/run.py +0 -0
  57. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/scheduler.py +0 -0
  58. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/script.py +0 -0
  59. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/sqlite3.py +0 -0
  60. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/startup.py +0 -0
  61. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/static_tree.py +0 -0
  62. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/status.py +0 -0
  63. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/step.py +0 -0
  64. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/stepinfo.py +0 -0
  65. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/tool.py +0 -0
  66. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/tracebacks.py +0 -0
  67. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/trellis.py +0 -0
  68. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/tui.py +0 -0
  69. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/usage.py +0 -0
  70. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/utils.py +0 -0
  71. {stepup-4.0.1 → stepup-4.0.3}/stepup/core/workflow.py +0 -0
  72. {stepup-4.0.1 → stepup-4.0.3}/stepup.egg-info/SOURCES.txt +0 -0
  73. {stepup-4.0.1 → stepup-4.0.3}/stepup.egg-info/dependency_links.txt +0 -0
  74. {stepup-4.0.1 → stepup-4.0.3}/stepup.egg-info/entry_points.txt +0 -0
  75. {stepup-4.0.1 → stepup-4.0.3}/stepup.egg-info/top_level.txt +0 -0
  76. {stepup-4.0.1 → stepup-4.0.3}/tools/.gitignore +0 -0
  77. {stepup-4.0.1 → stepup-4.0.3}/tools/analyze_perf.py +0 -0
  78. {stepup-4.0.1 → stepup-4.0.3}/tools/leaves.py +0 -0
  79. {stepup-4.0.1 → stepup-4.0.3}/tools/make_stable_redirects.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup
3
- Version: 4.0.1
3
+ Version: 4.0.3
4
4
  Summary: A dynamic Python build tool for reproducible workflows
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: LGPL-3.0-or-later
@@ -34,14 +34,15 @@ Requires-Dist: path>=16.14.0
34
34
  Requires-Dist: pyyaml>=6.0
35
35
  Requires-Dist: rich>=13.0.0
36
36
  Provides-Extra: tests
37
- Requires-Dist: matplotlib>=3.8.0; extra == "tests"
38
- Requires-Dist: numpy>=1.26.0; extra == "tests"
39
37
  Requires-Dist: pytest>=8.2.0; extra == "tests"
40
38
  Requires-Dist: pytest-asyncio>=0.24; extra == "tests"
41
39
  Requires-Dist: pytest-timeout>=2.3.1; extra == "tests"
42
40
  Requires-Dist: pytest-xdist>=3.6.1; extra == "tests"
41
+ Provides-Extra: examples
42
+ Requires-Dist: matplotlib>=3.8.0; extra == "examples"
43
+ Requires-Dist: numpy>=1.26.0; extra == "examples"
43
44
  Provides-Extra: dev
44
- Requires-Dist: stepup[tests]; extra == "dev"
45
+ Requires-Dist: stepup[examples,tests]; extra == "dev"
45
46
  Requires-Dist: mkdocs; extra == "dev"
46
47
  Requires-Dist: mkdocstrings[python]; extra == "dev"
47
48
  Requires-Dist: mkdocs-macros-plugin; extra == "dev"
@@ -64,15 +64,18 @@ dynamic = ["version"]
64
64
  # because it resolves the oldest supported versions of the direct dependencies,
65
65
  # which is only meaningful for packages that the tests actually exercise.
66
66
  tests = [
67
- "matplotlib>=3.8.0",
68
- "numpy>=1.26.0",
69
67
  "pytest>=8.2.0",
70
68
  "pytest-asyncio>=0.24",
71
69
  "pytest-timeout>=2.3.1",
72
70
  "pytest-xdist>=3.6.1",
73
71
  ]
72
+ # What the documentation examples need, which the smoke workflow runs.
73
+ examples = [
74
+ "matplotlib>=3.8.0",
75
+ "numpy>=1.26.0",
76
+ ]
74
77
  dev = [
75
- "stepup[tests]",
78
+ "stepup[tests,examples]",
76
79
  "mkdocs",
77
80
  "mkdocstrings[python]",
78
81
  "mkdocs-macros-plugin",
@@ -10,10 +10,21 @@ For arguments where these affixes are significant
10
10
  (the `dst` of `copy`, local executables, and the path variants of `getenv`),
11
11
  pass a `str` or a `path.Path` to preserve them.
12
12
 
13
+ Relative paths passed to this API are relative to the current working directory
14
+ of the calling process, just like paths passed to `open()`.
15
+ The environment variables `HERE` and `ROOT` are never read by this API,
16
+ because they are out of date in a process that changed its working directory.
17
+ Where a path argument or `getenv()` refers to them,
18
+ their values are computed with [`get_here()`][stepup.core.path.get_here]
19
+ and [`get_root()`][stepup.core.path.get_root] instead,
20
+ for the directory to which the path is relative,
21
+ and they are not recorded as environment dependencies of the step.
22
+ See [`HERE` and `ROOT`](../advanced_topics/here_and_root.md) for more details.
23
+
13
24
  Local variables holding a path use a prefix to state which form the path is in:
14
25
 
15
26
  - `su_`: after environment variable substitution and normalization,
16
- still relative to the working directory of the caller.
27
+ still relative to the current working directory.
17
28
  - `tr_`: after translation to the working directory of the director,
18
29
  which is the only form sent over RPC.
19
30
 
@@ -62,6 +73,8 @@ from .path import (
62
73
  coerce_str,
63
74
  format_local_executable,
64
75
  get_affixes,
76
+ get_here,
77
+ get_root,
65
78
  make_path_out,
66
79
  translate,
67
80
  translate_back,
@@ -98,6 +111,11 @@ __all__ = (
98
111
 
99
112
  logger = logging.getLogger(__name__)
100
113
 
114
+
115
+ # Variables whose values are computed for the current working directory,
116
+ # instead of being read from the environment, where they may be out of date.
117
+ _COMPUTED_ENV_VARS = {"HERE": get_here, "ROOT": get_root}
118
+
101
119
  # The `command` argument of `step()`, `run()` and `plan()`:
102
120
  # either a path-like object holding the command text,
103
121
  # or a callable that builds the command text from the step's own paths.
@@ -161,7 +179,7 @@ def static(*paths: StrPath | Iterable[StrPath] | NamedGlob) -> list[Path]:
161
179
  paths
162
180
  A sorted list of the files declared and the static tree roots registered by this call,
163
181
  tree roots with a trailing slash.
164
- Relative to the caller's working directory.
182
+ Relative to the current working directory.
165
183
 
166
184
  Raises
167
185
  ------
@@ -261,7 +279,7 @@ def static(*paths: StrPath | Iterable[StrPath] | NamedGlob) -> list[Path]:
261
279
  if len(tr_trees) + len(tr_files) + len(tr_patterns) > 0:
262
280
  get_rpc_client().call.declare_static(get_job_i(), tr_trees, tr_files, tr_patterns)
263
281
 
264
- # Report what this call covers, relative to the caller's working directory.
282
+ # Report what this call covers, relative to the current working directory.
265
283
  return sorted(
266
284
  {Path(su_path) / "" for su_path in su_lit_dirs + su_match_dirs}
267
285
  | {Path(su_path) for su_path in su_lit_files + su_match_files}
@@ -403,7 +421,7 @@ def step(
403
421
  workdir
404
422
  The directory where the command must be executed.
405
423
  The path is normalized before further processing.
406
- If this is a relative path, it is relative to the work directory of the caller.
424
+ If this is a relative path, it is relative to the current working directory.
407
425
  (The default is the current directory.)
408
426
  It does not need to exist yet: StepUp creates it right before running the command,
409
427
  and removes it again when the step leaves the workflow and the directory is empty.
@@ -467,6 +485,9 @@ def step(
467
485
  at the time this function is called, not when the step is executed.
468
486
 
469
487
  Relative paths in `inp`, `out`, and `vol` are relative to the working directory of the new step.
488
+ The same holds for `${HERE}` and `${ROOT}` in these paths,
489
+ so they have the same meaning as in the command of the step.
490
+ In `workdir`, they refer to the current working directory.
470
491
  """
471
492
  # Pre-process the arguments for the Director process.
472
493
  # The command is resolved after the environment variable substitution below,
@@ -505,10 +526,10 @@ def step(
505
526
  )
506
527
 
507
528
  with subs_env_vars() as subs_env:
508
- su_inp_paths = [subs_env(inp_path) for inp_path in inp_paths]
509
- su_out_paths = [subs_env(out_path) for out_path in out_paths]
510
- su_vol_paths = [subs_env(vol_path) for vol_path in vol_paths]
511
529
  su_workdir = subs_env(workdir)
530
+ su_inp_paths = [subs_env(inp_path, su_workdir) for inp_path in inp_paths]
531
+ su_out_paths = [subs_env(out_path, su_workdir) for out_path in out_paths]
532
+ su_vol_paths = [subs_env(vol_path, su_workdir) for vol_path in vol_paths]
512
533
 
513
534
  # Build the command text from the substituted paths when a callable is given.
514
535
  command = _resolve_command(command, su_inp_paths, su_out_paths, su_vol_paths)
@@ -628,6 +649,7 @@ def call(
628
649
  Resource constraints for this step.
629
650
  args_file
630
651
  Full filename for the serialized arguments.
652
+ Relative paths are assumed to be relative to `workdir`.
631
653
  When given, arguments are written to this file (format inferred from extension)
632
654
  and passed via `--inp=<args_file>`;
633
655
  when absent, a JSON string is embedded directly in the command.
@@ -662,10 +684,10 @@ def call(
662
684
  # Perform environment variable substitutions before building the command.
663
685
  # This is somewhat redundant with the substitutions performed in `step()`.
664
686
  with subs_env_vars() as subs_env:
665
- su_inp_paths = [subs_env(inp_path) for inp_path in coerce_paths(inp)]
666
- su_out_paths = [subs_env(out_path) for out_path in coerce_paths(out)]
667
687
  su_workdir = subs_env(workdir)
668
- su_args_file = subs_env(args_file)
688
+ su_inp_paths = [subs_env(inp_path, su_workdir) for inp_path in coerce_paths(inp)]
689
+ su_out_paths = [subs_env(out_path, su_workdir) for out_path in coerce_paths(out)]
690
+ su_args_file = subs_env(args_file, su_workdir)
669
691
 
670
692
  # Validate executable path format.
671
693
  if os.sep not in executable:
@@ -702,8 +724,9 @@ def call(
702
724
  command = f"{shlex.quote(executable)} {function} {shlex.quote(json_str)}"
703
725
  step_inp = [executable, *su_inp_paths]
704
726
  else:
705
- # dumpns(do_amend=True) calls amend(out=args_file) before writing.
706
- dumpns(su_args_file, forwarded)
727
+ # The step reads args_file relative to its workdir,
728
+ # whereas the planner writes it relative to its own working directory.
729
+ _dumpns((su_workdir / su_args_file).normpath(), forwarded)
707
730
  command = f"{shlex.quote(executable)} {function} --inp={shlex.quote(su_args_file)}"
708
731
  step_inp = [executable, *su_inp_paths, su_args_file]
709
732
 
@@ -753,7 +776,7 @@ class EnvSubstitutor:
753
776
  used_env: set[str] = attrs.field(factory=set)
754
777
  """The names of the environment variables substituted so far."""
755
778
 
756
- def _substitute(self, path: StrPath) -> Path:
779
+ def _substitute(self, path: StrPath, workdir: StrPath) -> Path:
757
780
  """Substitute the environment variables in a path, without normalizing it."""
758
781
  template = _CaseSensitiveTemplate(coerce_str(path))
759
782
  if not template.is_valid():
@@ -762,6 +785,8 @@ class EnvSubstitutor:
762
785
  for name in template.get_identifiers():
763
786
  if name.startswith("*"):
764
787
  mapping[name] = f"${{{name}}}"
788
+ elif name in _COMPUTED_ENV_VARS:
789
+ mapping[name] = str(_COMPUTED_ENV_VARS[name](workdir))
765
790
  else:
766
791
  value = os.getenv(name)
767
792
  if value is None:
@@ -770,7 +795,7 @@ class EnvSubstitutor:
770
795
  self.used_env.add(name)
771
796
  return Path(path if len(mapping) == 0 else template.substitute(mapping))
772
797
 
773
- def __call__(self, path: StrPath | None) -> Path | None:
798
+ def __call__(self, path: StrPath | None, workdir: StrPath = ".") -> Path | None:
774
799
  """Substitute the environment variables in a path and normalize the result.
775
800
 
776
801
  The leading `./` and trailing `/` of the substituted path are restored
@@ -783,6 +808,12 @@ class EnvSubstitutor:
783
808
  ----------
784
809
  path
785
810
  The path to substitute, or `None`.
811
+ workdir
812
+ The directory to which `path` is relative,
813
+ itself relative to the current working directory.
814
+ `${HERE}` and `${ROOT}` are computed for this directory,
815
+ with [`get_here()`][stepup.core.path.get_here]
816
+ and [`get_root()`][stepup.core.path.get_root].
786
817
 
787
818
  Returns
788
819
  -------
@@ -796,7 +827,9 @@ class EnvSubstitutor:
796
827
  When the path contains invalid shell variable identifiers,
797
828
  or references an environment variable that is not defined.
798
829
  """
799
- return None if path is None else _keep_affixes(self._substitute(path), Path.normpath)
830
+ if path is None:
831
+ return None
832
+ return _keep_affixes(self._substitute(path, workdir), Path.normpath)
800
833
 
801
834
 
802
835
  @contextlib.contextmanager
@@ -865,7 +898,7 @@ def amend(
865
898
  ----------
866
899
  inp
867
900
  Files required by the step.
868
- Relative paths are relative to the step's working directory.
901
+ Relative paths are relative to the current working directory.
869
902
  Directory inputs are not supported.
870
903
  env
871
904
  Environment variables to which the step is sensitive.
@@ -873,11 +906,11 @@ def amend(
873
906
  such that the step cannot be skipped.
874
907
  out
875
908
  Files created by the step.
876
- Relative paths are relative to the step's working directory.
909
+ Relative paths are relative to the current working directory.
877
910
  Directory outputs are not supported.
878
911
  vol
879
912
  Volatile files created by the step.
880
- Relative paths are relative to the step's working directory.
913
+ Relative paths are relative to the current working directory.
881
914
  Directory outputs are not supported.
882
915
 
883
916
  Raises
@@ -956,10 +989,8 @@ def amend(
956
989
  su_inp_paths = {subs_env(inp_path) for inp_path in inp_paths}
957
990
  su_out_paths = {subs_env(out_path) for out_path in out_paths}
958
991
  su_vol_paths = {subs_env(vol_path) for vol_path in vol_paths}
959
- # The checks use the substituted paths, not the translated ones below:
960
- # they look at the file system,
961
- # which this process sees relative to its own working directory,
962
- # not relative to the director's.
992
+ # The checks use the substituted paths, not the translated ones below,
993
+ # because they look at the file system relative to the current working directory.
963
994
  _check_no_directories(su_inp_paths)
964
995
  _check_no_directories(su_out_paths)
965
996
  _check_no_directories(su_vol_paths)
@@ -1076,10 +1107,9 @@ def get_info() -> StepInfo:
1076
1107
  step_info
1077
1108
  Holds relevant information of the current step, useful for defining follow-up steps.
1078
1109
  For consistency with other functions in this module, the `inp`, `out` and `vol`
1079
- paths are relative to the working directory of the step.
1110
+ paths are relative to the current working directory.
1080
1111
  """
1081
1112
  step_info = get_rpc_client().call.get_step_info(get_job_i())
1082
- # Update paths to make them relative to the working directory of the step.
1083
1113
  step_info.inp = sorted(translate_back(inp) for inp in step_info.inp)
1084
1114
  step_info.out = sorted(translate_back(out) for out in step_info.out)
1085
1115
  step_info.vol = sorted(translate_back(vol) for vol in step_info.vol)
@@ -1214,7 +1244,7 @@ def run(
1214
1244
  Holds relevant information of the step, useful for defining follow-up steps.
1215
1245
  """
1216
1246
  command, inp_paths, out_paths, vol_paths = _resolve_run_command(
1217
- command, inp=inp, out=out, vol=vol
1247
+ command, inp=inp, out=out, vol=vol, workdir=workdir
1218
1248
  )
1219
1249
  command, exe, env_overrides = _prepare_run_command(
1220
1250
  command, shell=shell, need_relative_exe=False
@@ -1307,7 +1337,7 @@ def plan(
1307
1337
  """
1308
1338
  # Note that we do not use `run()` here because we need to set `need=Need.PLAN`.
1309
1339
  command, inp_paths, out_paths, vol_paths = _resolve_run_command(
1310
- command, inp=inp, out=out, vol=vol
1340
+ command, inp=inp, out=out, vol=vol, workdir=workdir
1311
1341
  )
1312
1342
  command, exe, env_overrides = _prepare_run_command(command, shell=False, need_relative_exe=True)
1313
1343
  inp_paths = [coerce_path(exe), *inp_paths]
@@ -1390,6 +1420,9 @@ def getenv(
1390
1420
  ----------
1391
1421
  name
1392
1422
  The name of the environment variable, which is retrieved with `os.getenv`.
1423
+ The values of `HERE` and `ROOT` are computed
1424
+ with [`get_here()`][stepup.core.path.get_here]
1425
+ and [`get_root()`][stepup.core.path.get_root] instead.
1393
1426
  default
1394
1427
  The value to return when the environment variable is unset.
1395
1428
  path
@@ -1397,9 +1430,9 @@ def getenv(
1397
1430
  A `Path` instance will be returned.
1398
1431
  Shell variables are substituted (once) in such paths.
1399
1432
  back
1400
- Set to `True` to translate the path back to the working directory of the caller.
1433
+ Set to `True` to translate the path back to the current working directory.
1401
1434
  If the path is relative, it is assumed to be relative to StepUp's working directory.
1402
- It will be translated to become relative to the working directory of the caller.
1435
+ It will be translated to become relative to the current working directory.
1403
1436
  This implies `path=True`.
1404
1437
  multi
1405
1438
  Set to `True` if the variable is a list of paths.
@@ -1426,7 +1459,8 @@ def getenv(
1426
1459
  path = path or back or multi
1427
1460
  if default is not None:
1428
1461
  default = coerce_str(default)
1429
- value = os.getenv(name, default)
1462
+ compute = _COMPUTED_ENV_VARS.get(name)
1463
+ value = os.getenv(name, default) if compute is None else str(compute())
1430
1464
  # Do not amend environment variables set for the step by the executor.
1431
1465
  # See `stepup.core.executor.Executor._run_command`.
1432
1466
  if name not in RESERVED_ENV_VARS:
@@ -1544,10 +1578,8 @@ def loadns(
1544
1578
  The directory that `Path` variables are made relative to.
1545
1579
  If not given, the current working directory is used.
1546
1580
  This is only relevant for variables loaded from Python files.
1547
- A relative `Path` in such a file is interpreted
1548
- relative to the current working directory,
1549
- not relative to the parent of the variable file,
1550
- even though the file itself is executed with that parent as working directory.
1581
+ Such a file is executed with its parent directory as working directory,
1582
+ so a relative `Path` in it is interpreted relative to that parent directory.
1551
1583
  do_amend
1552
1584
  If `True`, the current step is amended with the loaded files as input dependencies.
1553
1585
 
@@ -1592,7 +1624,7 @@ def loadns(
1592
1624
  if name.startswith("_"):
1593
1625
  continue
1594
1626
  if isinstance(value, Path):
1595
- value = Path(value).relpath(dir_out)
1627
+ value = (dir_py / value).relpath(dir_out)
1596
1628
  variables[name] = value
1597
1629
  else:
1598
1630
  raise StepUpError(f"unsupported variable file format: {path_var}")
@@ -1625,6 +1657,16 @@ def dumpns(path: StrPath, data: dict[str, Any] | SimpleNamespace, *, do_amend: b
1625
1657
  """
1626
1658
  with subs_env_vars() as subs_env:
1627
1659
  su_path = subs_env(path)
1660
+ _dumpns(su_path, data, do_amend=do_amend)
1661
+
1662
+
1663
+ def _dumpns(
1664
+ su_path: Path, data: dict[str, Any] | SimpleNamespace, *, do_amend: bool = True
1665
+ ) -> None:
1666
+ """Write variables to a JSON or YAML file whose path is already substituted.
1667
+
1668
+ See [`dumpns()`][stepup.core.api.dumpns] for the parameters.
1669
+ """
1628
1670
  if do_amend:
1629
1671
  amend(out=su_path)
1630
1672
  if isinstance(data, SimpleNamespace):
@@ -1927,6 +1969,7 @@ def _resolve_run_command(
1927
1969
  inp: Iterable[StrPath] | StrPath,
1928
1970
  out: Iterable[StrPath] | StrPath,
1929
1971
  vol: Iterable[StrPath] | StrPath,
1972
+ workdir: StrPath,
1930
1973
  ) -> tuple[str, list[Path], list[Path], list[Path]]:
1931
1974
  """Resolve a `run()`/`plan()` command before the local executable is detected.
1932
1975
 
@@ -1940,8 +1983,8 @@ def _resolve_run_command(
1940
1983
  ----------
1941
1984
  command
1942
1985
  The `command` argument as given by the user.
1943
- inp, out, vol
1944
- The `inp`, `out` and `vol` arguments as given by the user.
1986
+ inp, out, vol, workdir
1987
+ The `inp`, `out`, `vol` and `workdir` arguments as given by the user.
1945
1988
 
1946
1989
  Returns
1947
1990
  -------
@@ -1960,9 +2003,10 @@ def _resolve_run_command(
1960
2003
  # which is idempotent.
1961
2004
  # The repeated `amend(env=...)` is filtered out by `_AMEND_HISTORY`.
1962
2005
  with subs_env_vars() as subs_env:
1963
- su_inp_paths = [subs_env(inp_path) for inp_path in inp_paths]
1964
- su_out_paths = [subs_env(out_path) for out_path in out_paths]
1965
- su_vol_paths = [subs_env(vol_path) for vol_path in vol_paths]
2006
+ su_workdir = subs_env(workdir)
2007
+ su_inp_paths = [subs_env(inp_path, su_workdir) for inp_path in inp_paths]
2008
+ su_out_paths = [subs_env(out_path, su_workdir) for out_path in out_paths]
2009
+ su_vol_paths = [subs_env(vol_path, su_workdir) for vol_path in vol_paths]
1966
2010
  command = _resolve_command(command, su_inp_paths, su_out_paths, su_vol_paths)
1967
2011
  else:
1968
2012
  command = coerce_str(command)
@@ -728,7 +728,13 @@ async def watch_first_loop(watcher: Watcher, handler: "DirectorHandler", stop_ev
728
728
  """Run pending steps 0.5 seconds after the watcher observes a file change."""
729
729
  with watcher.subscribe_changes() as changed_event:
730
730
  while True:
731
- await watcher.busy_watching.wait()
731
+ # Every wait below must also watch `stop_event`,
732
+ # because the director shuts down only after this loop has ended.
733
+ # Once the shutdown has begun, the watch phase never starts again,
734
+ # so waiting for it alone would block the shutdown forever.
735
+ await wait_for_any_event(watcher.busy_watching, stop_event)
736
+ if stop_event.is_set():
737
+ break
732
738
  await wait_for_any_event(changed_event, stop_event)
733
739
  if stop_event.is_set():
734
740
  break
@@ -827,8 +827,12 @@ class Executor:
827
827
  # Client code may use the following:
828
828
  env["STEPUP_STEP_INP_DIGEST"] = run.inp_digest.hex()
829
829
  env["STEPUP_STEP_NEED"] = need.name
830
- env["ROOT"] = str(Path.cwd().relpath(workdir))
831
- env["HERE"] = str(Path(workdir).relpath())
830
+ # The root is the working directory of the director, not `${STEPUP_ROOT}`,
831
+ # which may be set to another directory in the environment of the director.
832
+ # Physical paths are used, for consistency with `get_here()` and `get_root()`.
833
+ real_workdir = Path(workdir).realpath()
834
+ env["ROOT"] = str(Path.cwd().relpath(real_workdir))
835
+ env["HERE"] = str(real_workdir.relpath(Path.cwd()))
832
836
  # Note: the variables defined here must be listed in `RESERVED_ENV_VARS`.
833
837
 
834
838
  suspended_before = self.suspended_total
@@ -954,4 +958,4 @@ class Executor:
954
958
  pages = [("Outdated dynamic dependencies", page_change)]
955
959
  if len(page_same) > 0:
956
960
  pages.append(("Remained the same (or missing)", page_same))
957
- await self.reporter("DROPAMEND", run.description, pages)
961
+ await self.reporter("UNAMEND", run.description, pages)
@@ -12,17 +12,18 @@ import os
12
12
  import shlex
13
13
  import subprocess
14
14
  import sys
15
- from collections.abc import Iterable, Iterator
15
+ from collections.abc import Iterable, Iterator, Mapping
16
16
 
17
17
  from path import Path
18
18
 
19
19
  from .api import get_job_i, get_rpc_client, getenv
20
20
  from .exceptions import ConsistencyError, StepUpError
21
- from .path import StrPath, get_stepup_root, translate
21
+ from .path import StrPath, get_here, get_root, get_stepup_root, translate
22
22
  from .step import truncate_output
23
23
  from .utils import extract_env_overrides
24
24
 
25
25
  __all__ = (
26
+ "child_env",
26
27
  "filter_dependencies",
27
28
  "get_local_import_paths",
28
29
  "record_subprocess",
@@ -30,6 +31,41 @@ __all__ = (
30
31
  )
31
32
 
32
33
 
34
+ #
35
+ # Subprocess Environment
36
+ #
37
+
38
+
39
+ def child_env(workdir: StrPath, env: Mapping[str, str] | None = None) -> dict[str, str]:
40
+ """Return an environment for a child process with `HERE` and `ROOT` updated for its workdir.
41
+
42
+ StepUp itself never reads `HERE` and `ROOT` from the environment,
43
+ because it computes them with [`get_here()`][stepup.core.path.get_here]
44
+ and [`get_root()`][stepup.core.path.get_root].
45
+ Updating them is still useful when the child process runs user-written code,
46
+ such as a notebook, which may combine them with its own relative paths.
47
+ See [HERE and ROOT](../advanced_topics/here_and_root.md) for more details.
48
+
49
+ Parameters
50
+ ----------
51
+ workdir
52
+ The working directory of the child process, relative to the current working directory.
53
+ env
54
+ The environment to start from, `os.environ` by default.
55
+ It is not modified.
56
+
57
+ Returns
58
+ -------
59
+ env_child
60
+ A copy of `env` in which `HERE` and `ROOT` are set for the child process,
61
+ as computed by `get_here(workdir)` and `get_root(workdir)`.
62
+ """
63
+ env_child = dict(os.environ if env is None else env)
64
+ env_child["HERE"] = str(get_here(workdir))
65
+ env_child["ROOT"] = str(get_root(workdir))
66
+ return env_child
67
+
68
+
33
69
  #
34
70
  # Subprocess Recording
35
71
  #
@@ -192,8 +228,10 @@ def run_subprocess(
192
228
  In either case, the caller is responsible for proper quoting.
193
229
  workdir
194
230
  The working directory of the subprocess as a path or string,
195
- relative to the step's own working directory.
231
+ relative to the current working directory.
196
232
  It is passed to `subprocess.run` as `cwd`.
233
+ The environment variables `HERE` and `ROOT` of the subprocess are updated accordingly,
234
+ see `child_env()`.
197
235
  stdin
198
236
  Standard input fed to the subprocess, or `None`.
199
237
  A `str` is passed to `subprocess.run` as-is and implies `text=True`.
@@ -236,7 +274,7 @@ def run_subprocess(
236
274
  else:
237
275
  env_overrides, cmd = extract_env_overrides(cmd)
238
276
  text = _resolve_text_mode(stdin, text)
239
- run_env = dict(os.environ)
277
+ run_env = child_env(workdir)
240
278
  if env_overrides is not None:
241
279
  run_env.update(env_overrides)
242
280
  cp = subprocess.run(
@@ -2,6 +2,7 @@
2
2
  # SPDX-License-Identifier: LGPL-3.0-or-later
3
3
  """Specialized path operations."""
4
4
 
5
+ import functools
5
6
  import os
6
7
  import shlex
7
8
  from collections.abc import Iterable
@@ -20,6 +21,8 @@ __all__ = (
20
21
  "dir_range_upper",
21
22
  "format_local_executable",
22
23
  "get_affixes",
24
+ "get_here",
25
+ "get_root",
23
26
  "get_stepup_root",
24
27
  "make_path_out",
25
28
  "parent_dir",
@@ -233,6 +236,64 @@ def get_stepup_root() -> Path:
233
236
  return Path(os.getenv("STEPUP_ROOT", os.getcwd())).absolute()
234
237
 
235
238
 
239
+ def get_here(workdir: StrPath = ".") -> Path:
240
+ """Get the relative path from the StepUp root to a working directory.
241
+
242
+ This is the Python counterpart of the environment variable `HERE`,
243
+ which is not consulted, so the result is also correct in a process
244
+ that changed its working directory without updating `HERE`.
245
+ See [HERE and ROOT](../advanced_topics/here_and_root.md) for more details.
246
+
247
+ Parameters
248
+ ----------
249
+ workdir
250
+ The working directory, relative to the current working directory.
251
+ The default is the current working directory itself.
252
+
253
+ Returns
254
+ -------
255
+ here
256
+ The relative path from the StepUp root to `workdir`.
257
+ Both directories are physical paths, with all symbolic links resolved.
258
+ The path starts with `../` when `workdir` lies outside the StepUp root.
259
+ """
260
+ return coerce_path(workdir).realpath().relpath(_get_real_root())
261
+
262
+
263
+ def get_root(workdir: StrPath = ".") -> Path:
264
+ """Get the relative path from a working directory to the StepUp root.
265
+
266
+ This is the Python counterpart of the environment variable `ROOT`,
267
+ which is not consulted, so the result is also correct in a process
268
+ that changed its working directory without updating `ROOT`.
269
+ See [HERE and ROOT](../advanced_topics/here_and_root.md) for more details.
270
+
271
+ Parameters
272
+ ----------
273
+ workdir
274
+ The working directory, relative to the current working directory.
275
+ The default is the current working directory itself.
276
+
277
+ Returns
278
+ -------
279
+ root
280
+ The relative path from `workdir` to the StepUp root.
281
+ Both directories are physical paths, with all symbolic links resolved.
282
+ """
283
+ return _get_real_root().relpath(coerce_path(workdir).realpath())
284
+
285
+
286
+ def _get_real_root() -> Path:
287
+ """Get the physical StepUp root, with all symbolic links resolved."""
288
+ return _realpath_cached(get_stepup_root())
289
+
290
+
291
+ @functools.lru_cache(maxsize=16)
292
+ def _realpath_cached(path: Path) -> Path:
293
+ """Resolve symbolic links, with caching because API functions may translate many paths."""
294
+ return path.realpath()
295
+
296
+
236
297
  def translate(path: StrPath, workdir: StrPath = ".") -> Path:
237
298
  """Normalize the path and, if relative, make it relative to `STEPUP_ROOT`.
238
299
 
@@ -243,7 +304,7 @@ def translate(path: StrPath, workdir: StrPath = ".") -> Path:
243
304
  If relative, it is assumed to be relative to `workdir`.
244
305
  workdir
245
306
  The working directory.
246
- If relative, it is assumed to be relative to `HERE`.
307
+ If relative, it is assumed to be relative to the current working directory.
247
308
 
248
309
  Returns
249
310
  -------
@@ -255,9 +316,7 @@ def translate(path: StrPath, workdir: StrPath = ".") -> Path:
255
316
  workdir = coerce_path(workdir).normpath()
256
317
  path = workdir / path
257
318
  if not workdir.isabs():
258
- root = get_stepup_root()
259
- here = Path(os.getenv("HERE", Path(".").relpath(root)))
260
- path = (root / here / path).normpath().relpath(root)
319
+ path = path.relpath(_get_real_root())
261
320
  return path
262
321
 
263
322
 
@@ -271,7 +330,7 @@ def translate_back(path: StrPath, workdir: StrPath = ".") -> Path:
271
330
  If relative, it is assumed to be relative to `STEPUP_ROOT`.
272
331
  workdir
273
332
  The working directory.
274
- If relative, it is assumed to be relative to `HERE`.
333
+ If relative, it is assumed to be relative to the current working directory.
275
334
 
276
335
  Returns
277
336
  -------
@@ -284,9 +343,7 @@ def translate_back(path: StrPath, workdir: StrPath = ".") -> Path:
284
343
  if workdir.isabs() and path.startswith(workdir):
285
344
  path = Path(path).relpath(workdir)
286
345
  else:
287
- root = get_stepup_root()
288
- here = Path(os.getenv("HERE", Path(".").relpath(root)))
289
- path = Path(root / path).relpath(root / here / workdir)
346
+ path = (_get_real_root() / path).relpath(workdir)
290
347
  return path
291
348
 
292
349
 
@@ -41,12 +41,12 @@ TAG_COLORS = {
41
41
  "SUCCESS": "green",
42
42
  # Workflow-related details
43
43
  "DELETED": "cyan",
44
- "DROPAMEND": "cyan",
44
+ "UNAMEND": "cyan",
45
45
  "NOSKIP": "cyan",
46
46
  "DEFERRED": "cyan",
47
47
  "REMOVE": "cyan",
48
48
  "SKIP": "cyan",
49
- "UNCHANGED": "cyan",
49
+ "SAMEHASH": "cyan",
50
50
  "UPDATED": "cyan",
51
51
  # Events outside the workflow
52
52
  "DIRECTOR": "magenta",
@@ -221,7 +221,7 @@ class Watcher:
221
221
  # so its round trips are made after the transaction closes,
222
222
  # instead of holding the database write lock for the duration of the network traffic.
223
223
  for path in dropped:
224
- await self.reporter("UNCHANGED", path)
224
+ await self.reporter("SAMEHASH", path)
225
225
 
226
226
  # Reset the watcher state.
227
227
  # The subscriber events are cleared together with the sets they refer to,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup
3
- Version: 4.0.1
3
+ Version: 4.0.3
4
4
  Summary: A dynamic Python build tool for reproducible workflows
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: LGPL-3.0-or-later
@@ -34,14 +34,15 @@ Requires-Dist: path>=16.14.0
34
34
  Requires-Dist: pyyaml>=6.0
35
35
  Requires-Dist: rich>=13.0.0
36
36
  Provides-Extra: tests
37
- Requires-Dist: matplotlib>=3.8.0; extra == "tests"
38
- Requires-Dist: numpy>=1.26.0; extra == "tests"
39
37
  Requires-Dist: pytest>=8.2.0; extra == "tests"
40
38
  Requires-Dist: pytest-asyncio>=0.24; extra == "tests"
41
39
  Requires-Dist: pytest-timeout>=2.3.1; extra == "tests"
42
40
  Requires-Dist: pytest-xdist>=3.6.1; extra == "tests"
41
+ Provides-Extra: examples
42
+ Requires-Dist: matplotlib>=3.8.0; extra == "examples"
43
+ Requires-Dist: numpy>=1.26.0; extra == "examples"
43
44
  Provides-Extra: dev
44
- Requires-Dist: stepup[tests]; extra == "dev"
45
+ Requires-Dist: stepup[examples,tests]; extra == "dev"
45
46
  Requires-Dist: mkdocs; extra == "dev"
46
47
  Requires-Dist: mkdocstrings[python]; extra == "dev"
47
48
  Requires-Dist: mkdocs-macros-plugin; extra == "dev"
@@ -10,7 +10,7 @@ rich>=13.0.0
10
10
  asyncinotify>=4.0.9
11
11
 
12
12
  [dev]
13
- stepup[tests]
13
+ stepup[examples,tests]
14
14
  mkdocs
15
15
  mkdocstrings[python]
16
16
  mkdocs-macros-plugin
@@ -19,9 +19,11 @@ reuse
19
19
  snipwise<2.0a0
20
20
  yappi
21
21
 
22
- [tests]
22
+ [examples]
23
23
  matplotlib>=3.8.0
24
24
  numpy>=1.26.0
25
+
26
+ [tests]
25
27
  pytest>=8.2.0
26
28
  pytest-asyncio>=0.24
27
29
  pytest-timeout>=2.3.1
@@ -365,6 +365,15 @@
365
365
  "tests/examples/amend_already_declared/main.sh",
366
366
  "tests/examples/amend_already_declared/plan.py",
367
367
  "tests/examples/amend_already_declared/work.py",
368
+ "tests/examples/amend_chdir/.gitignore",
369
+ "tests/examples/amend_chdir/README.txt",
370
+ "tests/examples/amend_chdir/expected_graph.txt",
371
+ "tests/examples/amend_chdir/expected_stdout.txt",
372
+ "tests/examples/amend_chdir/main.sh",
373
+ "tests/examples/amend_chdir/plan.py",
374
+ "tests/examples/amend_chdir/sub/child.py",
375
+ "tests/examples/amend_chdir/sub/data.txt",
376
+ "tests/examples/amend_chdir/work.py",
368
377
  "tests/examples/amend_delete_both/.gitignore",
369
378
  "tests/examples/amend_delete_both/README.txt",
370
379
  "tests/examples/amend_delete_both/expected_graph1.txt",
@@ -589,6 +598,13 @@
589
598
  "tests/examples/call_args_file_no_ext/main.sh",
590
599
  "tests/examples/call_args_file_no_ext/plan.py",
591
600
  "tests/examples/call_args_file_no_ext/work.py",
601
+ "tests/examples/call_args_file_workdir/.gitignore",
602
+ "tests/examples/call_args_file_workdir/expected_graph.txt",
603
+ "tests/examples/call_args_file_workdir/expected_stdout.txt",
604
+ "tests/examples/call_args_file_workdir/main.sh",
605
+ "tests/examples/call_args_file_workdir/plan.py",
606
+ "tests/examples/call_args_file_workdir/sub/data.txt",
607
+ "tests/examples/call_args_file_workdir/sub/work.py",
592
608
  "tests/examples/call_args_file_yaml/.gitignore",
593
609
  "tests/examples/call_args_file_yaml/data.txt",
594
610
  "tests/examples/call_args_file_yaml/expected_graph.txt",
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "4.0.3",
3
+ "distance": 0,
4
+ "node": "g20ae41cab865ce9d4d903a366a9e2fb884258b80",
5
+ "dirty": false,
6
+ "branch": "HEAD",
7
+ "node_date": "2026-09-19"
8
+ }
@@ -1,8 +0,0 @@
1
- {
2
- "tag": "4.0.1",
3
- "distance": 0,
4
- "node": "g6a42284bb4b55375a994cb0286af00d1c327dbab",
5
- "dirty": false,
6
- "branch": "HEAD",
7
- "node_date": "2026-09-02"
8
- }
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes