stepup 3.2.0__tar.gz → 3.2.2__tar.gz

{stepup-3.2.0/stepup.egg-info → stepup-3.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup
-Version: 3.2.0
+Version: 3.2.2
 Summary: StepUp Core provides the basic framework for the StepUp build tool
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/__init__.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/__main__.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/actions.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/api.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/asyncio.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/browse.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #
@@ -23,6 +23,7 @@ import argparse
 import contextlib
 import importlib.resources
 import os
+import pickle
 import sqlite3
 import stat
 import traceback
@@ -366,8 +367,9 @@ class GraphServer(BaseHTTPRequestHandler):
 
         # Format the state (if a file or a step)
         if kind == "step":
-            (state_i, mandatory_i, dirty) = self.con.execute(
-                "SELECT state, mandatory, dirty FROM step WHERE node = ?", (node_i,)
+            sql_props = "SELECT state, pool, block, mandatory, dirty FROM step WHERE node = ?"
+            state_i, pool, block, mandatory_i, dirty = self.con.execute(
+                sql_props, (node_i,)
             ).fetchone()
             state = StepState(state_i)
             mandatory = Mandatory(mandatory_i)
@@ -379,6 +381,38 @@ class GraphServer(BaseHTTPRequestHandler):
                 yield '<p>Dirty: <span class="dirty">YES</span></p>'
             else:
                 yield '<p>Dirty: <span class="clean">NO</span></p>'
+            if pool is not None:
+                yield f"<p><b>Pool:</b> {pool}</p>"
+            if block:
+                yield "<p><b>This step is blocked.</b></p>"
+
+            sql_env = "SELECT name, amended FROM env_var WHERE node = ?"
+            env_vars = list(self.con.execute(sql_env, (node_i,)))
+            if len(env_vars) > 0:
+                yield "<h3>Uses Environment Variables</h3><ul>"
+                for env_var, amended in env_vars:
+                    yield (
+                        f"<li>{env_var} <i>[amended]</i></li>" if amended else f"<li>{env_var}</li>"
+                    )
+                yield "</ul>"
+
+            sql_ngm = "SELECT data FROM nglob_multi WHERE node = ?"
+            ngms = list(self.con.execute(sql_ngm, (node_i,)))
+            if len(ngms) > 0:
+                yield "<h3>Defines NGlob Multis</h3><ul>"
+                for row in ngms:
+                    ngm = pickle.loads(row[0])
+                    yield f"<li>{[ngs.pattern for ngs in ngm.nglob_singles]} {ngm.subs}</li>"
+                yield "</ul>"
+
+            sql_pooldefs = "SELECT name, size FROM pool_definition WHERE node = ?"
+            pooldefs = list(self.con.execute(sql_pooldefs, (node_i,)))
+            if len(pooldefs) > 0:
+                yield "<h3>Defines Pools</h3><ul>"
+                for pool, size in pooldefs:
+                    yield f"<li>{pool} = {size}</li>"
+                yield "</ul>"
+
         elif kind == "file":
             (state_i, digest, mode, mtime, size, inode) = self.con.execute(
                 "SELECT state, digest, mode, mtime, size, inode FROM file WHERE node = ?", (node_i,)
@@ -427,10 +461,7 @@ class GraphServer(BaseHTTPRequestHandler):
             "WHERE dependency.supplier = ? ORDER BY node.kind, node.label",
             (node_i,),
         ):
-            node_str = self._format_node(cons_i, cons_kind, cons_label, False, state)
-            if amended:
-                node_str += " <b>[amended]</b>"
-            yield node_str
+            yield self._format_node(cons_i, cons_kind, cons_label, False, state, amended)
         yield "</table>"
 
         # Format the suppliers
@@ -442,10 +473,7 @@ class GraphServer(BaseHTTPRequestHandler):
             "WHERE dependency.consumer = ? ORDER BY node.kind, node.label",
             (node_i,),
         ):
-            node_str = self._format_node(sup_i, sup_kind, sup_label, False, state)
-            if amended:
-                node_str += " <b>[amended]</b>"
-            yield node_str
+            yield self._format_node(sup_i, sup_kind, sup_label, False, state, amended)
         yield "</table>"
 
     def _search(self, env, args):
@@ -494,14 +522,24 @@ class GraphServer(BaseHTTPRequestHandler):
     # --- helpers ---
 
     def _format_node(
-        self, i: int, kind: str, label: str, orphan: bool, state: int | None = None
+        self,
+        i: int,
+        kind: str,
+        label: str,
+        orphan: bool,
+        state: int | None = None,
+        amended: bool = False,
     ) -> str:
         sym = KIND_SYMBOLS.get(kind, f"?{kind}?")
         node_str = f"{label}"
         if i is not None:
             node_str = f'<a href="/node/?i={i}">{node_str}</a>'
+        if len(label) == 0:
+            node_str = f"[{kind}]"
         if orphan:
             node_str = f"({node_str})"
+        if amended:
+            node_str += " <i>[amended]</i>"
         if state is None:
             state_str = ""
         elif kind == "file":

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/call.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/cascade.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #
@@ -398,7 +398,7 @@ class Node:
         Parameters
         ----------
         supplier
-            Other node that supplies to thise node.
+            Other node that supplies to this node.
 
         Returns
         -------
@@ -832,7 +832,7 @@ class Cascade:
         while cleaned_some:
             cleaned_some = False
             # Look for orphans without consumers or products.
-            # As long is nodes have consumers or products, they cannot be removed.
+            # As long nodes have consumers or products, they cannot be removed.
             query = (
                 "SELECT i, kind, label FROM node WHERE orphan = TRUE AND "
                 "NOT EXISTS (SELECT 1 FROM node AS cnode WHERE node.i = cnode.creator) AND "

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/clean.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/deferred_glob.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/director.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #
@@ -34,7 +34,10 @@ from importlib.metadata import version as get_version
 import attrs
 from path import Path
 
-from stepup.core.step import Step
+try:
+    import yappi
+except ImportError:
+    yappi = None
 
 from .asyncio import wait_for_events
 from .enums import ReturnCode, StepState
@@ -46,6 +49,7 @@ from .rpc import allow_rpc, serve_socket_rpc
 from .runner import Runner
 from .scheduler import Scheduler
 from .startup import startup_from_db
+from .step import Step
 from .stepinfo import StepInfo
 from .utils import DBLock, check_plan, mynormpath
 from .watcher import WATCHER_AVAILABLE, Watcher
@@ -70,6 +74,16 @@ async def async_main():
     )
     print(f"SOCKET {args.director_socket}", file=sys.stderr)
     print(f"PID {os.getpid()}", file=sys.stderr)
+    print(f"LOG_LEVEL {args.log_level}", file=sys.stderr)
+    if args.yappi:
+        if yappi is None:
+            print(
+                "Yappi profiling requested, but the yappi module is not installed.",
+                file=sys.stderr,
+            )
+        else:
+            yappi.set_clock_type("cpu")
+            yappi.start(builtins=True, profile_threads=True)
     async with ReporterClient.socket(args.reporter_socket) as reporter:
         num_workers = interpret_num_workers(args.num_workers)
         await reporter.set_num_workers(num_workers)
@@ -94,6 +108,10 @@ async def async_main():
         finally:
             await reporter("DIRECTOR", "See you!")
             await reporter.shutdown()
+            if args.yappi and yappi is not None:
+                yappi.stop()
+                stats = yappi.get_func_stats()
+                stats.save(".stepup/director.prof", type="pstat")
         sys.exit(returncode.value)
 
 
@@ -169,6 +187,12 @@ def parse_args() -> argparse.Namespace:
         action=argparse.BooleanOptionalAction,
         help="Do not remove outdated output files.",
     )
+    parser.add_argument(
+        "--yappi",
+        default=False,
+        action=argparse.BooleanOptionalAction,
+        help="Profile the director with Yappi (must be installed).",
+    )
     args = parser.parse_args()
     if WATCHER_AVAILABLE:
         if args.watch_first:

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/enums.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/exceptions.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/file.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #
@@ -299,7 +299,7 @@ class File(Node):
             # Local import to avoid cyclic imports.
             from .step import Step  # noqa: PLC0415
 
-            for step in self.consumers(Step):
+            for step in self.consumers(Step, include_orphans=True):
                 step.mark_pending()
         elif state != FileState.OUTDATED:
             raise ValueError(f"Cannot make file oudated when its state is {state}")

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/hash.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/interact.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/job.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

stepup-3.2.2/stepup/core/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 26 26"><path d="M9.5 0C7.5 0 6 1.6 6 3.5v3.4c0 .6-.3 1.1-.8 1.3A3.7 3.7 0 0 0 3 11.5v3.4c0 .6-.3 1.1-.7 1.3A3.6 3.6 0 0 0 0 19.5v3c0 2 1.6 3.5 3.5 3.5h19c2 0 3.5-1.6 3.5-3.5v-3c0-2-1.6-3.5-3.5-3.5-.8 0-1.5-.7-1.5-1.5v-3c0-2-1.6-3.5-3.5-3.5-.8 0-1.5-.7-1.5-1.5v-3c0-2-1.6-3.5-3.5-3.5Z"/><path id="a" fill="#fff" d="M12.5 8h-3C8.7 8 8 7.3 8 6.5v-3C8 2.7 8.7 2 9.5 2h3c.8 0 1.5.7 1.5 1.5v3c0 .8-.7 1.5-1.5 1.5Z"/><use xlink:href="#a" transform="translate(10 16)"/><use xlink:href="#a" transform="translate(2 16)"/><use xlink:href="#a" transform="translate(-6 16)"/><use xlink:href="#a" transform="translate(-3 8)"/><use xlink:href="#a" transform="translate(5 8)"/></svg>

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/nglob.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/pytest.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/render_jinja.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/reporter.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/rpc.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/runner.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/scheduler.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/script.py

@@ -1,5 +1,5 @@
 # StepUp Script extends StepUp with scripts that combine planning and running.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Script.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/startup.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/step.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #
@@ -219,14 +219,16 @@ class Step(Node):
 
     def format_properties(self) -> Iterator[tuple[str, str]]:
         """Iterate over key-value pairs that represent the properties of the node."""
-        state, pool, block, mandatory, _, _ = self.properties()
+        state, pool, block, mandatory, dirty, _ = self.properties()
         yield "state", state.name
         if mandatory != Mandatory.YES:
             yield "mandatory", mandatory.name
         if pool is not None:
             yield "pool", pool
+        if dirty:
+            yield "dirty", "yes"
         if block:
-            yield "block", block
+            yield "blocked", "yes"
 
         sql = "SELECT name, amended FROM env_var WHERE node = ?"
         label = "env_var"
@@ -985,6 +987,6 @@ class Step(Node):
             logger.info("Mark %s step PENDING: %s", state.name, self.label)
             self.set_state(StepState.PENDING)
             # First make all consumers (output files) pending
-            for file in self.consumers(File):
+            for file in self.consumers(File, include_orphans=True):
                 if file.get_state() == FileState.BUILT:
                     file.mark_outdated()

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/stepinfo.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/tui.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #
@@ -123,6 +123,8 @@ async def async_boot(args: argparse.Namespace):
                 argv.append("--watch-first")
         if not args.clean:
             argv.append("--no-clean")
+        if args.yappi:
+            argv.append("--yappi")
         returncode = 1  # Internal error unless it is overriden later by the director subprocess
         try:
             with open(".stepup/director.log", "w") as log_file:
@@ -299,7 +301,15 @@ def boot_subcommand(subparsers) -> callable:
         default=os.getenv("STEPUP_PERF", None),
         nargs="?",
         const="500",
-        help="Run the director under perf record, by default at a frequency of 500 Hz. "
+        help="Profile the director with perf, by default at a frequency of %(const)s Hz. "
         "(Only supported on Linux with perf installed.)",
     )
+    parser.add_argument(
+        "--yappi",
+        default=string_to_bool(os.getenv("STEPUP_YAPPI", "0")),
+        action=argparse.BooleanOptionalAction,
+        help="Profile the director with Yappi (must be installed). "
+        "This produces a .stepup/director.prof file that can be analyzed with "
+        "tools like SnakeViz.",
+    )
     return boot_tool

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/utils.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #
@@ -21,6 +21,7 @@
 
 import asyncio
 import contextlib
+import logging
 import os
 import re
 import shlex
@@ -37,6 +38,7 @@ __all__ = (
     "DBLock",
     "check_inp_path",
     "check_plan",
+    "filter_dependencies",
     "format_command",
     "format_digest",
     "get_local_import_paths",
@@ -44,6 +46,7 @@ __all__ = (
     "myabsolute",
     "mynormpath",
     "myparent",
+    "myrealpath",
     "myrelpath",
     "remove_path",
     "string_to_bool",
@@ -53,6 +56,9 @@ __all__ = (
 )
 
 
+logger = logging.getLogger(__name__)
+
+
 #
 # Custom path operations
 #
@@ -66,6 +72,14 @@ def mynormpath(path: str) -> Path:
     return result
 
 
+def myrealpath(path: str) -> Path:
+    """Like Path.realpath path but keep the trailing separator"""
+    result = Path(path).realpath()
+    if path.endswith(os.sep) and not result.endswith("/"):
+        result = result / ""
+    return result
+
+
 def myrelpath(path: str, start: str = ".") -> Path:
     """Like Path.relpath path but keep the trailing separator"""
     result = Path(path).relpath(start)
@@ -257,20 +271,21 @@ def filter_dependencies(paths: Collection[str]) -> set[Path]:
             raise ValueError(f"Invalid filter item: {filter_item}")
         prefix = filter_item[1:]
         if not prefix.startswith("/"):
-            prefix = mynormpath(stepup_root / prefix)
+            prefix = myrealpath(stepup_root / prefix)
         rules.append((prefix, keep))
 
     # Filter paths according to the rules.
     result = set()
+    realpwd = myrealpath(os.getcwd())
     for path in paths:
-        abspath = myabsolute(path)
+        abspath = myrealpath(path)
         for prefix, keep in rules:
             if abspath.startswith(prefix):
                 if keep:
-                    result.add(myrelpath(path))
+                    result.add(myrelpath(abspath, realpwd))
                 break
         else:
-            raise AssertionError("No matching rule found for path: {path}")
+            raise AssertionError(f"No matching rule found for path: {path}")
     return result
 
 

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/watcher.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/worker.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #
@@ -25,6 +25,8 @@ import contextlib
 import ctypes
 import inspect
 import io
+import json
+import logging
 import os
 import queue
 import resource
@@ -48,7 +50,7 @@ from .reporter import ReporterClient
 from .rpc import AsyncRPCClient, allow_rpc, serve_socket_rpc
 from .scheduler import Scheduler
 from .step import Step, split_step_label
-from .utils import DBLock
+from .utils import DBLock, string_to_bool
 from .workflow import Workflow
 
 __all__ = ("WorkThread", "WorkerClient", "WorkerHandler", "WorkerStep")
@@ -59,6 +61,9 @@ __all__ = ("WorkThread", "WorkerClient", "WorkerHandler", "WorkerStep")
 #
 
 
+logger = logging.getLogger(__name__)
+
+
 @attrs.define
 class WorkerClient:
     """Client interface to a worker, used by the director process."""
@@ -109,6 +114,7 @@ class WorkerClient:
             self.director_socket_path,
             worker_socket_path,
             str(self.idx),
+            f"--log-level={logging.getLevelName(logging.root.level)}",
         ]
         if self.show_perf:
             argv.append("--show-perf")
@@ -223,8 +229,6 @@ class WorkerClient:
                 return True
 
             # Delegate the calculation of the output part of the step hash to the worker.
-            # With skipping=True, the worker knows the outputs should not have changed
-            # and will report it on screen.
             new_step_hash, new_out_hashes = await self.compute_out_step_hash(step, new_step_hash)
 
             if step_hash.out_digest != new_step_hash.out_digest:
@@ -516,10 +520,18 @@ class WorkerStep:
 
 
 PYCODE_WRAPPER = """\
+import logging
+import os
 import sys
 import runpy
+from path import Path
 from stepup.core.api import amend
 from stepup.core.utils import get_local_import_paths
+logging.basicConfig(
+    format="%(asctime)s  %(levelname)8s  %(name)24s  ::  %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+    level=logging.{log_level},
+)
 sys.argv = {argv}
 try:
     runpy.run_path({script}, run_name="__main__")
@@ -645,7 +657,11 @@ class WorkThread(threading.Thread):
         # Run the script
         return self.runsh_verbose(
             f"{sys.executable} -",
-            PYCODE_WRAPPER.format(argv=repr([script, *args]), script=repr(script)),
+            PYCODE_WRAPPER.format(
+                argv=repr([script, *args]),
+                script=repr(script),
+                log_level=logging.getLevelName(logging.root.level),
+            ),
         )
 
 
@@ -690,6 +706,38 @@ def check_executable(executable: Path, shebang: str | None = None) -> bool:
     return True
 
 
+@attrs.define
+class Timer:
+    """Context managers to construct a JSON string with times spent in typical worker tasks."""
+
+    start: float = attrs.field(init=False, factory=perf_counter)
+    sections: dict[str, float] = attrs.field(init=False, factory=dict)
+
+    @contextlib.contextmanager
+    def section(self, section: str):
+        """Context manager for a timed section.
+
+        Parameters
+        ----------
+        section
+            The name of the section.
+        """
+        time = perf_counter()
+        try:
+            yield
+        finally:
+            time = perf_counter() - time
+            self.sections[section] = self.sections.get(section, 0.0) + time
+
+    def report(self):
+        """Write the timings line to standard error."""
+        time = perf_counter() - self.start
+        sections = self.sections.copy()
+        sections["idle/other"] = time - sum(sections.values())
+        sections["total"] = time
+        print(f"TIMINGS: {json.dumps(sections)}", file=sys.stderr)
+
+
 @attrs.define
 class WorkerHandler:
     """RPC Handler in the worker process to respond to requests from the WorkerClient."""
@@ -699,6 +747,7 @@ class WorkerHandler:
     explain_rerun: bool = attrs.field()
     stop_event: asyncio.Event = attrs.field(factory=asyncio.Event)
     step: WorkerStep | None = attrs.field(init=False, default=None)
+    timer: Timer = attrs.field(init=False, factory=Timer)
 
     @allow_rpc
     def shutdown(self):
@@ -760,18 +809,19 @@ class WorkerHandler:
             The new hash of the step, with the input part already computed, if available.
             `None` is yielded if, unexpectedly, some inputs are missing or have changed.
         """
-        if self.step is not None:
-            raise RPCError(
-                "Worker cannot initiate two steps at the same time. "
-                f"Still working on {self.step.action}"
-            )
+        with self.timer.section("new_step"):
+            if self.step is not None:
+                raise RPCError(
+                    "Worker cannot initiate two steps at the same time. "
+                    f"Still working on {self.step.action}"
+                )
 
-        # Create the step
-        action, workdir = split_step_label(label)
-        self.step = WorkerStep(i, action, workdir)
+            # Create the step
+            action, workdir = split_step_label(label)
+            self.step = WorkerStep(i, action, workdir)
 
-        # Create initial StepHash
-        return self.compute_inp_step_hash(inp_hashes, env_vars, check_hash)[0]
+            # Create initial StepHash
+            return self.compute_inp_step_hash(inp_hashes, env_vars, check_hash)[0]
 
     def compute_inp_step_hash(
         self,
@@ -802,41 +852,44 @@ class WorkerHandler:
             so they can be updated in StepUp's workflow database if desired.
             Unchanged ones are not included.
         """
-        # Check the input hashes
-        messages = []
-        new_inp_hashes = []
-        all_inp_hashes = []
-        for path, old_file_hash in old_inp_hashes:
-            new_file_hash = old_file_hash.regen(path)
-            all_inp_hashes.append((path, new_file_hash))
-            if check_hash and new_file_hash != old_file_hash:
-                if new_file_hash.is_unknown:
-                    messages.append(f"Input vanished unexpectedly: {path} ")
-                else:
-                    messages.append(
-                        f"Input changed unexpectedly: {path} "
-                        + fmt_file_hash_diff(old_file_hash, new_file_hash)
-                    )
-                new_inp_hashes.append((path, new_file_hash))
-
-        # If there are unexpected issues with inputs, bail out.
-        if len(messages) > 0:
-            self.step.inp_messages = messages
-            self.step.success = False
-            return None, new_inp_hashes
+        with self.timer.section("compute_inp_step_hash"):
+            # Check the input hashes
+            messages = []
+            new_inp_hashes = []
+            all_inp_hashes = []
+            for path, old_file_hash in old_inp_hashes:
+                new_file_hash = old_file_hash.regen(path)
+                all_inp_hashes.append((path, new_file_hash))
+                if check_hash and new_file_hash != old_file_hash:
+                    if new_file_hash.is_unknown:
+                        messages.append(f"Input vanished unexpectedly: {path} ")
+                    else:
+                        messages.append(
+                            f"Input changed unexpectedly: {path} "
+                            + fmt_file_hash_diff(old_file_hash, new_file_hash)
+                        )
+                    new_inp_hashes.append((path, new_file_hash))
+
+            # If there are unexpected issues with inputs, bail out.
+            if len(messages) > 0:
+                self.step.inp_messages = messages
+                self.step.success = False
+                return None, new_inp_hashes
 
-        # Add the environment variables to the hash
-        env_var_values = [(env_var, os.environ.get(env_var)) for env_var in env_vars]
+            # Add the environment variables to the hash
+            env_var_values = [(env_var, os.environ.get(env_var)) for env_var in env_vars]
 
-        # Create the StepHash
-        label = self.step.action
-        if self.step.workdir != "./":
-            label += f"  # wd={self.step.workdir}"
-        result = StepHash.from_inp(f"{label}", self.explain_rerun, all_inp_hashes, env_var_values)
+            # Create the StepHash
+            label = self.step.action
+            if self.step.workdir != "./":
+                label += f"  # wd={self.step.workdir}"
+            result = StepHash.from_inp(
+                f"{label}", self.explain_rerun, all_inp_hashes, env_var_values
+            )
 
-        # Copy the inp_digest, because it can be useful for some actions.
-        self.step.inp_digest = result.inp_digest
-        return result, []
+            # Copy the inp_digest, because it can be useful for some actions.
+            self.step.inp_digest = result.inp_digest
+            return result, []
 
     @allow_rpc
     def compute_out_step_hash(
@@ -864,22 +917,23 @@ class WorkerHandler:
             so they can be updated in StepUp's workflow database if desired.
             Unchanged ones are not included.
         """
-        # Check the output hashes
-        new_out_hashes = []
-        all_out_hashes = []
-        for path, old_file_hash in sorted(old_out_hashes):
-            new_file_hash = old_file_hash.regen(path)
-            all_out_hashes.append((path, new_file_hash))
-            if new_file_hash != old_file_hash:
-                new_out_hashes.append((path, new_file_hash))
-            if new_file_hash.is_unknown:
-                self.step.out_missing.append(path)
-                self.step.success = False
+        with self.timer.section("compute_out_step_hash"):
+            # Check the output hashes
+            new_out_hashes = []
+            all_out_hashes = []
+            for path, old_file_hash in sorted(old_out_hashes):
+                new_file_hash = old_file_hash.regen(path)
+                all_out_hashes.append((path, new_file_hash))
+                if new_file_hash != old_file_hash:
+                    new_out_hashes.append((path, new_file_hash))
+                if new_file_hash.is_unknown:
+                    self.step.out_missing.append(path)
+                    self.step.success = False
 
-        # Update the step hash
-        if step_hash is not None:
-            step_hash = step_hash.evolve_out(all_out_hashes)
-        return step_hash, new_out_hashes
+            # Update the step hash
+            if step_hash is not None:
+                step_hash = step_hash.evolve_out(all_out_hashes)
+            return step_hash, new_out_hashes
 
     @allow_rpc
     def compute_full_step_hash(
@@ -939,57 +993,58 @@ class WorkerHandler:
 
     @allow_rpc
     async def run(self):
-        await self.reporter("START", self.step.description)
-        await self.reporter.start_step(self.step.description, self.step.i)
-
-        # For internal use in actions:
-        os.environ["STEPUP_STEP_I"] = str(self.step.i)
-        # Client code may use the following:
-        os.environ["STEPUP_STEP_INP_DIGEST"] = self.step.inp_digest.hex()
-        os.environ["ROOT"] = str(Path.cwd().relpath(self.step.workdir))
-        os.environ["HERE"] = str(self.step.workdir.relpath())
-        # Note: the variables defined here should be listed in stepup.core.api.getenv
-
-        # Create IO redirection for stdout and stderr
-        step_err = io.StringIO()
-        step_out = io.StringIO()
-        with (
-            contextlib.chdir(self.step.workdir),
-            contextlib.redirect_stderr(step_err),
-            contextlib.redirect_stdout(step_out),
-        ):
+        with self.timer.section("run"):
+            await self.reporter("START", self.step.description)
+            await self.reporter.start_step(self.step.description, self.step.i)
+
+            # For internal use in actions:
+            os.environ["STEPUP_STEP_I"] = str(self.step.i)
+            # Client code may use the following:
+            os.environ["STEPUP_STEP_INP_DIGEST"] = self.step.inp_digest.hex()
+            os.environ["ROOT"] = str(Path.cwd().relpath(self.step.workdir))
+            os.environ["HERE"] = str(self.step.workdir.relpath())
+            # Note: the variables defined here should be listed in stepup.core.api.getenv
+
+            # Create IO redirection for stdout and stderr
+            step_err = io.StringIO()
+            step_out = io.StringIO()
+            with (
+                contextlib.chdir(self.step.workdir),
+                contextlib.redirect_stderr(step_err),
+                contextlib.redirect_stdout(step_out),
+            ):
+                if self.show_perf:
+                    ru_initial = resource.getrusage(resource.RUSAGE_CHILDREN)
+                    pt_initial = perf_counter()
+                self.step.thread = WorkThread(self.step.action)
+                self.step.thread.start()
+                await self.step.thread.done.wait()
+                self.step.thread.join()
+                self.step.returncode = self.step.thread.returncode
+                self.step.thread = None
+            self.step.stdout = step_out.getvalue()
+            self.step.stderr = step_err.getvalue()
+
+            # Clean up environment variables (to avoid potential confusion)
+            del os.environ["STEPUP_STEP_I"]
+            del os.environ["STEPUP_STEP_INP_DIGEST"]
+            del os.environ["ROOT"]
+            del os.environ["HERE"]
+
+            # Process results of the step.
             if self.show_perf:
-                ru_initial = resource.getrusage(resource.RUSAGE_CHILDREN)
-                pt_initial = perf_counter()
-            self.step.thread = WorkThread(self.step.action)
-            self.step.thread.start()
-            await self.step.thread.done.wait()
-            self.step.thread.join()
-            self.step.returncode = self.step.thread.returncode
-            self.step.thread = None
-        self.step.stdout = step_out.getvalue()
-        self.step.stderr = step_err.getvalue()
-
-        # Clean up environment variables (to avoid potential confusion)
-        del os.environ["STEPUP_STEP_I"]
-        del os.environ["STEPUP_STEP_INP_DIGEST"]
-        del os.environ["ROOT"]
-        del os.environ["HERE"]
-
-        # Process results of the step.
-        if self.show_perf:
-            ru_final = resource.getrusage(resource.RUSAGE_CHILDREN)
-            utime = ru_final.ru_utime - ru_initial.ru_utime
-            stime = ru_final.ru_stime - ru_initial.ru_stime
-            wtime = perf_counter() - pt_initial
-            ru_lines = [
-                f"User CPU time [s]:   {utime:9.4f}",
-                f"System CPU time [s]: {stime:9.4f}",
-                f"Total CPU time [s]:  {utime + stime:9.4f}",
-                f"Wall time [s]:       {wtime:9.4f}",
-            ]
-            self.step.perf_info = "\n".join(ru_lines)
-        self.step.success = self.step.returncode == 0
+                ru_final = resource.getrusage(resource.RUSAGE_CHILDREN)
+                utime = ru_final.ru_utime - ru_initial.ru_utime
+                stime = ru_final.ru_stime - ru_initial.ru_stime
+                wtime = perf_counter() - pt_initial
+                ru_lines = [
+                    f"User CPU time [s]:   {utime:9.4f}",
+                    f"System CPU time [s]: {stime:9.4f}",
+                    f"Total CPU time [s]:  {utime + stime:9.4f}",
+                    f"Wall time [s]:       {wtime:9.4f}",
+                ]
+                self.step.perf_info = "\n".join(ru_lines)
+            self.step.success = self.step.returncode == 0
 
     @allow_rpc
     def get_success(self) -> bool:
@@ -1004,51 +1059,52 @@ class WorkerHandler:
 
     @allow_rpc
     async def report(self):
-        pages = []
-        if not self.step.success:
-            # Format command such that it can be copied and pasted into a shell.
-            command = "stepup act "
-            if any(word.startswith("-") for word in shlex.split(self.step.action)):
-                command += "-- "
-            command += self.step.action
-            if self.step.workdir != "./":
-                command = f"(cd {self.step.workdir} && {command})"
-            lines = [f"Command               {command}"]
-            # Other info on the execution of the step
-            if self.step.returncode is not None:
-                lines.append(f"Return code           {self.step.returncode}")
-            pages.append(("Step info", "\n".join(lines)))
-        if len(self.step.perf_info) > 0:
-            pages.append(("Performance details", self.step.perf_info))
-        if self.step.rescheduled_info != "":
-            pages.append(
-                (
-                    "Rescheduling due to unavailable amended inputs",
-                    self.step.rescheduled_info,
+        with self.timer.section("report"):
+            pages = []
+            if not self.step.success:
+                # Format command such that it can be copied and pasted into a shell.
+                command = "stepup act "
+                if any(word.startswith("-") for word in shlex.split(self.step.action)):
+                    command += "-- "
+                command += self.step.action
+                if self.step.workdir != "./":
+                    command = f"(cd {self.step.workdir} && {command})"
+                lines = [f"Command               {command}"]
+                # Other info on the execution of the step
+                if self.step.returncode is not None:
+                    lines.append(f"Return code           {self.step.returncode}")
+                pages.append(("Step info", "\n".join(lines)))
+            if len(self.step.perf_info) > 0:
+                pages.append(("Performance details", self.step.perf_info))
+            if self.step.rescheduled_info != "":
+                pages.append(
+                    (
+                        "Rescheduling due to unavailable amended inputs",
+                        self.step.rescheduled_info,
+                    )
                 )
-            )
-        else:
-            if len(self.step.inp_messages) > 0:
-                self.step.inp_messages.sort()
-                pages.append(("Invalid inputs", "\n".join(self.step.inp_messages)))
-            if len(self.step.out_missing) > 0:
-                self.step.out_missing.sort()
-                pages.append(("Expected outputs not created", "\n".join(self.step.out_missing)))
-        stdout = self.step.stdout.rstrip()
-        if len(stdout) > 0:
-            pages.append(("Standard output", stdout))
-        stderr = self.step.stderr.rstrip()
-        if len(stderr) > 0:
-            pages.append(("Standard error", stderr))
-        if self.step.rescheduled_info != "":
-            action = "RESCHEDULE"
-        elif self.step.success:
-            action = "SUCCESS"
-        else:
-            action = "FAIL"
-        await self.reporter.stop_step(self.step.i)
-        await self.reporter(action, self.step.description, pages)
-        self.step = None
+            else:
+                if len(self.step.inp_messages) > 0:
+                    self.step.inp_messages.sort()
+                    pages.append(("Invalid inputs", "\n".join(self.step.inp_messages)))
+                if len(self.step.out_missing) > 0:
+                    self.step.out_missing.sort()
+                    pages.append(("Expected outputs not created", "\n".join(self.step.out_missing)))
+            stdout = self.step.stdout.rstrip()
+            if len(stdout) > 0:
+                pages.append(("Standard output", stdout))
+            stderr = self.step.stderr.rstrip()
+            if len(stderr) > 0:
+                pages.append(("Standard error", stderr))
+            if self.step.rescheduled_info != "":
+                action = "RESCHEDULE"
+            elif self.step.success:
+                action = "SUCCESS"
+            else:
+                action = "FAIL"
+            await self.reporter.stop_step(self.step.i)
+            await self.reporter(action, self.step.description, pages)
+            self.step = None
 
     @allow_rpc
     async def skip(self, step_hash: StepHash):
@@ -1090,6 +1146,14 @@ def parse_args():
     parser.add_argument("director_socket", type=Path, help="Socket of the director")
     parser.add_argument("worker_socket", type=Path, help="Socket of the worker (to be created)")
     parser.add_argument("worker_idx", type=int, help="Worker index")
+    debug = string_to_bool(os.getenv("STEPUP_DEBUG", "0"))
+    parser.add_argument(
+        "--log-level",
+        "-l",
+        default=os.getenv("STEPUP_LOG_LEVEL", "DEBUG" if debug else "WARNING").upper(),
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+        help="Set the logging level. [default=%(default)s]",
+    )
     parser.add_argument(
         "--reporter",
         "-r",
@@ -1117,13 +1181,20 @@ def parse_args():
 
 async def async_main():
     args = parse_args()
+    logging.basicConfig(
+        format="%(asctime)s  %(levelname)8s  %(name)24s  ::  %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        level=args.log_level,
+    )
     os.environ["STEPUP_DIRECTOR_SOCKET"] = args.director_socket
     os.environ["STEPUP_ROOT"] = str(Path.cwd())
     print(f"PID {os.getpid()}", file=sys.stderr)
+    print(f"LOG_LEVEL {args.log_level}", file=sys.stderr)
     async with ReporterClient.socket(args.reporter_socket) as reporter:
         # Create the worker handler for the RPC server.
         handler = WorkerHandler(reporter, args.show_perf, args.explain_rerun)
         await serve_socket_rpc(handler, args.worker_socket, handler.stop_event)
+        handler.timer.report()
 
 
 def main():

{stepup-3.2.0 → stepup-3.2.2}/stepup/core/workflow.py

@@ -1,5 +1,5 @@
 # StepUp Core provides the basic framework for the StepUp build tool.
-# © 2024–2025 Toon Verstraelen
+# Copyright 2024-2026 Toon Verstraelen
 #
 # This file is part of StepUp Core.
 #
@@ -1118,6 +1118,21 @@ class Workflow(Cascade):
         return self.declare_missing(dg, matching_paths)
 
     def clean(self):
+        # Search for parent directories of STEPUP_ROOT that are no longer used.
+        # The are not cleaned up like other unused directories,
+        # because they are treated as "always static" with the root node as creator.
+        parent_stack = []
+        iparent = 1
+        while True:
+            node = self.find(File, "../" * iparent)
+            if node is None:
+                break
+            parent_stack.insert(0, node)
+            iparent += 1
+        for node in parent_stack:
+            if any(node.consumers()):
+                break
+            node.orphan()
         # Get rid of deferred glob files that are no longer used.
         for dg in self.nodes(DeferredGlob):
             files = sorted(dg.products(), reverse=True, key=(lambda node: node.path))

{stepup-3.2.0 → stepup-3.2.2/stepup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup
-Version: 3.2.0
+Version: 3.2.2
 Summary: StepUp Core provides the basic framework for the StepUp build tool
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later

stepup-3.2.0/stepup/core/logo.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="26" height="26"><g transform="translate(-5)"><path d="M14.5 0c-2 0-3.5 1.6-3.5 3.5v3.4c0 .6-.3 1.1-.8 1.3A3.7 3.7 0 0 0 8 11.5v3.4c0 .6-.3 1.1-.7 1.3A3.6 3.6 0 0 0 5 19.5v3c0 2 1.6 3.5 3.5 3.5h19c2 0 3.5-1.6 3.5-3.5v-3c0-2-1.6-3.5-3.5-3.5-.8 0-1.5-.7-1.5-1.5v-3c0-2-1.6-3.5-3.5-3.5-.8 0-1.5-.7-1.5-1.5v-3c0-2-1.6-3.5-3.5-3.5Z" overflow="visible" paint-order="stroke markers fill"/><path id="a" fill="#fff" d="M17.5 8h-3c-.8 0-1.5-.7-1.5-1.5v-3c0-.8.7-1.5 1.5-1.5h3c.8 0 1.5.7 1.5 1.5v3c0 .8-.7 1.5-1.5 1.5z" baseline-shift="baseline" display="inline" overflow="visible" paint-order="stroke markers fill" stop-color="#000" vector-effect="none"/><use xlink:href="#a" transform="translate(10 16)"/><use xlink:href="#a" transform="translate(2 16)"/><use xlink:href="#a" transform="translate(-6 16)"/><use xlink:href="#a" transform="translate(-3 8)"/><use xlink:href="#a" transform="translate(5 8)"/></g></svg>