stepup 3.2.1__tar.gz → 3.2.3__tar.gz

{stepup-3.2.1/stepup.egg-info → stepup-3.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup
-Version: 3.2.1
+Version: 3.2.3
 Summary: StepUp Core provides the basic framework for the StepUp build tool
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/browse.py

@@ -24,7 +24,6 @@ import contextlib
 import importlib.resources
 import os
 import pickle
-import sqlite3
 import stat
 import traceback
 from collections.abc import Iterator
@@ -37,6 +36,7 @@ from path import Path
 
 from .enums import FileState, Mandatory, StepState
 from .hash import fmt_digest
+from .sqlite3 import connect
 
 
 def browse_subcommand(subparser: argparse.ArgumentParser) -> callable:
@@ -275,8 +275,8 @@ class GraphServer(BaseHTTPRequestHandler):
             print("Loading database...")
             if self.con is not None:
                 self.con.close()
-            self.con = sqlite3.Connection(":memory:")
-            src = sqlite3.Connection(self.path_db)
+            self.con = connect(":memory:")
+            src = connect(self.path_db)
             try:
                 src.backup(self.con)
             finally:
@@ -415,7 +415,9 @@ class GraphServer(BaseHTTPRequestHandler):
 
         elif kind == "file":
             (state_i, digest, mode, mtime, size, inode) = self.con.execute(
-                "SELECT state, digest, mode, mtime, size, inode FROM file WHERE node = ?", (node_i,)
+                "SELECT state, digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM file "
+                "WHERE node = ?",
+                (node_i,),
             ).fetchone()
             state = FileState(state_i)
             yield f'<p><b>State:</b> <span class="{state.name.lower()}">{state.name}</span></p>'

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/cascade.py

@@ -562,6 +562,9 @@ class Cascade:
         # While making this change, the enums were also made more intuitive.
         # Schema 2 became outdated due to the worker actions.
         # Schema 3 became outdated due to a change in step table (dirty field).
+
+        # Delayed Schema updates, for version 5:
+        # - Use UINT64 with PARSE_DECLTYPES instead of PARSE_COLNAMES.
         return 4
 
     @classmethod

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/clean.py

@@ -29,7 +29,8 @@ from rich.console import Console
 from .cascade import DROP_CONSUMERS, INITIAL_CONSUMERS, RECURSE_CONSUMERS
 from .enums import FileState
 from .hash import FileHash
-from .utils import mynormpath, sqlite3_copy_in_memory, translate, translate_back
+from .sqlite3 import copy_db_in_memory
+from .utils import mynormpath, translate, translate_back
 
 
 def clean_subcommand(subparser: argparse.ArgumentParser) -> callable:
@@ -91,7 +92,7 @@ def clean_tool(args: argparse.Namespace):
     # Copy the database in memory and work on the copy.
     root = Path(os.getenv("STEPUP_ROOT", "."))
     path_db = root / ".stepup/graph.db"
-    with sqlite3_copy_in_memory(path_db) as con:
+    with copy_db_in_memory(path_db) as con:
         clean(con, tr_paths, args)
 
 
@@ -175,7 +176,7 @@ def fmtnum(i: int):
 CREATE_INITIAL_PATHS = "CREATE TABLE temp.initial_path(path TEXT PRIMARY KEY) WITHOUT ROWID"
 
 SELECT_OUTPUTS = f"""
-SELECT label, file.state, orphan, digest, mode, mtime, size, inode FROM node
+SELECT label, file.state, orphan, digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM node
 JOIN all_consumer ON node.i = all_consumer.current
 JOIN file ON file.node = all_consumer.current
 WHERE file.state in

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/director.py

@@ -24,7 +24,6 @@ import asyncio
 import logging
 import os
 import signal
-import sqlite3
 import sys
 import time
 import traceback
@@ -34,7 +33,10 @@ from importlib.metadata import version as get_version
 import attrs
 from path import Path
 
-from stepup.core.step import Step
+try:
+    import yappi
+except ImportError:
+    yappi = None
 
 from .asyncio import wait_for_events
 from .enums import ReturnCode, StepState
@@ -45,7 +47,9 @@ from .reporter import ReporterClient
 from .rpc import allow_rpc, serve_socket_rpc
 from .runner import Runner
 from .scheduler import Scheduler
+from .sqlite3 import connect
 from .startup import startup_from_db
+from .step import Step
 from .stepinfo import StepInfo
 from .utils import DBLock, check_plan, mynormpath
 from .watcher import WATCHER_AVAILABLE, Watcher
@@ -71,6 +75,15 @@ async def async_main():
     print(f"SOCKET {args.director_socket}", file=sys.stderr)
     print(f"PID {os.getpid()}", file=sys.stderr)
     print(f"LOG_LEVEL {args.log_level}", file=sys.stderr)
+    if args.yappi:
+        if yappi is None:
+            print(
+                "Yappi profiling requested, but the yappi module is not installed.",
+                file=sys.stderr,
+            )
+        else:
+            yappi.set_clock_type("cpu")
+            yappi.start(builtins=True, profile_threads=True)
     async with ReporterClient.socket(args.reporter_socket) as reporter:
         num_workers = interpret_num_workers(args.num_workers)
         await reporter.set_num_workers(num_workers)
@@ -95,6 +108,10 @@ async def async_main():
         finally:
             await reporter("DIRECTOR", "See you!")
             await reporter.shutdown()
+            if args.yappi and yappi is not None:
+                yappi.stop()
+                stats = yappi.get_func_stats()
+                stats.save(".stepup/director.prof", type="pstat")
         sys.exit(returncode.value)
 
 
@@ -170,6 +187,12 @@ def parse_args() -> argparse.Namespace:
         action=argparse.BooleanOptionalAction,
         help="Do not remove outdated output files.",
     )
+    parser.add_argument(
+        "--yappi",
+        default=False,
+        action=argparse.BooleanOptionalAction,
+        help="Profile the director with Yappi (must be installed).",
+    )
     args = parser.parse_args()
     if WATCHER_AVAILABLE:
         if args.watch_first:
@@ -235,7 +258,7 @@ async def serve(
     check_plan("plan.py")
 
     # Create basic components
-    con = sqlite3.connect(".stepup/graph.db", cached_statements=1024)
+    con = connect(".stepup/graph.db")
     dblock = DBLock(con)
     workflow = Workflow(con)
     scheduler = Scheduler(workflow.job_queue, workflow.config_queue, workflow.job_queue_changed)

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/file.py

@@ -29,6 +29,7 @@ from path import Path
 from .cascade import Node
 from .enums import DirWatch, FileState
 from .hash import FileHash
+from .sqlite3 import UInt64
 from .utils import format_digest
 
 if TYPE_CHECKING:
@@ -102,7 +103,10 @@ class File(Node):
         # If the file was previously BUILT or OUTDATED, and created again as AWAITED,
         # it should copy that state
         if state == FileState.AWAITED:
-            sql = "SELECT state, digest, mode, mtime, size, inode FROM file WHERE node = ?"
+            sql = (
+                "SELECT state, digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM file "
+                "WHERE node = ?"
+            )
             row = self.con.execute(sql, (self.i,)).fetchone()
             if row is not None and row[0] in (FileState.BUILT.value, FileState.OUTDATED.value):
                 state = FileState(row[0])
@@ -121,7 +125,7 @@ class File(Node):
                 "mode": mode,
                 "mtime": mtime,
                 "size": size,
-                "inode": inode,
+                "inode": UInt64(inode),
             },
         )
         # If the state is BUILT, mark it as OUTDATED to force a rebuild.
@@ -215,7 +219,7 @@ class File(Node):
         self.con.execute(sql, (state.value, self.i))
 
     def get_hash(self) -> FileHash:
-        sql = "SELECT digest, mode, mtime, size, inode FROM file WHERE node = ?"
+        sql = "SELECT digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM file WHERE node = ?"
         row = self.con.execute(sql, (self.i,)).fetchone()
         return FileHash(*row)
 
@@ -299,7 +303,7 @@ class File(Node):
             # Local import to avoid cyclic imports.
             from .step import Step  # noqa: PLC0415
 
-            for step in self.consumers(Step):
+            for step in self.consumers(Step, include_orphans=True):
                 step.mark_pending()
         elif state != FileState.OUTDATED:
             raise ValueError(f"Cannot make file oudated when its state is {state}")

stepup-3.2.3/stepup/core/sqlite3.py

@@ -0,0 +1,83 @@
+# StepUp Core provides the basic framework for the StepUp build tool.
+# Copyright 2024-2026 Toon Verstraelen
+#
+# This file is part of StepUp Core.
+#
+# StepUp Core is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# StepUp Core is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+#
+# --
+"""Wrapper for SQLite3 functionality."""
+
+import contextlib
+import os
+import sqlite3
+from collections.abc import Iterator
+from typing import Self
+
+__all__ = ("UInt64", "connect", "copy_db_in_memory")
+
+
+class UInt64(int):
+    """A wrapper to tell SQLite this int should be treated as an unsigned 64-bit value."""
+
+    MAX_SIGNED_64 = 2**63 - 1
+    MAX_WRAPAROUND_64 = 2**64
+    MAX_UNSIGNED_64 = MAX_WRAPAROUND_64 - 1
+
+    @staticmethod
+    def adapt(val: Self) -> int:
+        if not (0 <= val <= UInt64.MAX_UNSIGNED_64):
+            raise ValueError(f"Value {val} out of UINT64 range")
+        return val - UInt64.MAX_WRAPAROUND_64 if val > UInt64.MAX_SIGNED_64 else val
+
+    @staticmethod
+    def convert(val: bytes) -> Self:
+        val = int(val)
+        if val < 0:
+            val += UInt64.MAX_WRAPAROUND_64
+        return UInt64(val)
+
+
+sqlite3.register_adapter(UInt64, UInt64.adapt)
+sqlite3.register_converter("UINT64", UInt64.convert)
+
+
+def connect(path: str | os.PathLike[str], **kwargs) -> sqlite3.Connection:
+    """Connect to a SQLite database, with the appropriate settings for StepUp.
+
+    The following deviations from the default settings are used:
+
+    - Types can be detected from column names,
+      which allows us to use the custom UINT64 type for file inodes.
+    - The `cached_statements` parameter is set to a large value to improve
+      performance when executing many similar statements.
+    """
+    my_kwargs = {"cached_statements": 1024, "detect_types": sqlite3.PARSE_COLNAMES}
+    my_kwargs.update(kwargs)
+    return sqlite3.connect(path, **my_kwargs)
+
+
+@contextlib.contextmanager
+def copy_db_in_memory(path_db) -> Iterator[sqlite3.Connection]:
+    """Copy an SQLite database into memory and yield the connection."""
+    dst = connect(":memory:")
+    try:
+        src = connect(path_db)
+        try:
+            src.backup(dst)
+        finally:
+            src.close()
+        yield dst
+    finally:
+        dst.close()

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/startup.py

@@ -124,7 +124,7 @@ async def scan_file_changes(
 ) -> tuple[set[str], set[str]]:
     """Check all files in the workflow for changes."""
     sql = (
-        "SELECT label, state, digest, mode, mtime, size, inode "
+        "SELECT label, state, digest, mode, mtime, size, inode AS 'inode [UINT64]' "
         "FROM node JOIN file ON node.i = file.node AND state NOT IN (?, ?) AND NOT orphan"
     )
     data = (FileState.AWAITED.value, FileState.VOLATILE.value)

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/step.py

@@ -558,7 +558,7 @@ class Step(Node):
             fields.append("state")
             join_file = True
         if yield_hash:
-            fields.extend(["digest", "mode", "mtime", "size", "inode"])
+            fields.extend(["digest", "mode", "mtime", "size", "inode AS 'inode [UINT64]'"])
             join_file = True
         if yield_orphan:
             fields.append("orphan")
@@ -731,7 +731,7 @@ class Step(Node):
         sql = (
             "SELECT node.label, node.orphan, file.state, "
             "EXISTS (SELECT 1 FROM amended_dep WHERE amended_dep.i = dep.i), "
-            "file.digest, file.mode, file.mtime, file.size, file.inode "
+            "file.digest, file.mode, file.mtime, file.size, file.inode AS 'inode [UINT64]' "
             "FROM node JOIN dependency AS dep ON node.i = dep.supplier "
             "JOIN file ON file.node = node.i "
             "WHERE dep.consumer = ?"
@@ -987,6 +987,6 @@ class Step(Node):
             logger.info("Mark %s step PENDING: %s", state.name, self.label)
             self.set_state(StepState.PENDING)
             # First make all consumers (output files) pending
-            for file in self.consumers(File):
+            for file in self.consumers(File, include_orphans=True):
                 if file.get_state() == FileState.BUILT:
                     file.mark_outdated()

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/tui.py

@@ -123,6 +123,8 @@ async def async_boot(args: argparse.Namespace):
                 argv.append("--watch-first")
         if not args.clean:
             argv.append("--no-clean")
+        if args.yappi:
+            argv.append("--yappi")
         returncode = 1  # Internal error unless it is overriden later by the director subprocess
         try:
             with open(".stepup/director.log", "w") as log_file:
@@ -299,7 +301,15 @@ def boot_subcommand(subparsers) -> callable:
         default=os.getenv("STEPUP_PERF", None),
         nargs="?",
         const="500",
-        help="Run the director under perf record, by default at a frequency of 500 Hz. "
+        help="Profile the director with perf, by default at a frequency of %(const)s Hz. "
         "(Only supported on Linux with perf installed.)",
     )
+    parser.add_argument(
+        "--yappi",
+        default=string_to_bool(os.getenv("STEPUP_YAPPI", "0")),
+        action=argparse.BooleanOptionalAction,
+        help="Profile the director with Yappi (must be installed). "
+        "This produces a .stepup/director.prof file that can be analyzed with "
+        "tools like SnakeViz.",
+    )
     return boot_tool

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/utils.py

@@ -20,7 +20,6 @@
 """Small utilities used throughout."""
 
 import asyncio
-import contextlib
 import logging
 import os
 import re
@@ -28,7 +27,7 @@ import shlex
 import sqlite3
 import string
 import sys
-from collections.abc import Collection, Iterator
+from collections.abc import Collection
 
 import attrs
 from path import Path
@@ -429,18 +428,3 @@ def string_to_bool(v: str | bool) -> bool:
             return False
         raise ValueError(f"Cannot interpret '{v}' as a boolean value.")
     raise TypeError(f"Expected a boolean value or string. Got {type(v).__name__}")
-
-
-@contextlib.contextmanager
-def sqlite3_copy_in_memory(path_db) -> Iterator[sqlite3.Connection]:
-    """Copy an SQLite database into memory and yield the connection."""
-    dst = sqlite3.Connection(":memory:")
-    try:
-        src = sqlite3.Connection(path_db)
-        try:
-            src.backup(dst)
-        finally:
-            src.close()
-        yield dst
-    finally:
-        dst.close()

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/worker.py

@@ -25,6 +25,7 @@ import contextlib
 import ctypes
 import inspect
 import io
+import json
 import logging
 import os
 import queue
@@ -228,8 +229,6 @@ class WorkerClient:
                 return True
 
             # Delegate the calculation of the output part of the step hash to the worker.
-            # With skipping=True, the worker knows the outputs should not have changed
-            # and will report it on screen.
             new_step_hash, new_out_hashes = await self.compute_out_step_hash(step, new_step_hash)
 
             if step_hash.out_digest != new_step_hash.out_digest:
@@ -707,6 +706,38 @@ def check_executable(executable: Path, shebang: str | None = None) -> bool:
     return True
 
 
+@attrs.define
+class Timer:
+    """Context managers to construct a JSON string with times spent in typical worker tasks."""
+
+    start: float = attrs.field(init=False, factory=perf_counter)
+    sections: dict[str, float] = attrs.field(init=False, factory=dict)
+
+    @contextlib.contextmanager
+    def section(self, section: str):
+        """Context manager for a timed section.
+
+        Parameters
+        ----------
+        section
+            The name of the section.
+        """
+        time = perf_counter()
+        try:
+            yield
+        finally:
+            time = perf_counter() - time
+            self.sections[section] = self.sections.get(section, 0.0) + time
+
+    def report(self):
+        """Write the timings line to standard error."""
+        time = perf_counter() - self.start
+        sections = self.sections.copy()
+        sections["idle/other"] = time - sum(sections.values())
+        sections["total"] = time
+        print(f"TIMINGS: {json.dumps(sections)}", file=sys.stderr)
+
+
 @attrs.define
 class WorkerHandler:
     """RPC Handler in the worker process to respond to requests from the WorkerClient."""
@@ -716,6 +747,7 @@ class WorkerHandler:
     explain_rerun: bool = attrs.field()
     stop_event: asyncio.Event = attrs.field(factory=asyncio.Event)
     step: WorkerStep | None = attrs.field(init=False, default=None)
+    timer: Timer = attrs.field(init=False, factory=Timer)
 
     @allow_rpc
     def shutdown(self):
@@ -777,18 +809,19 @@ class WorkerHandler:
             The new hash of the step, with the input part already computed, if available.
             `None` is yielded if, unexpectedly, some inputs are missing or have changed.
         """
-        if self.step is not None:
-            raise RPCError(
-                "Worker cannot initiate two steps at the same time. "
-                f"Still working on {self.step.action}"
-            )
+        with self.timer.section("new_step"):
+            if self.step is not None:
+                raise RPCError(
+                    "Worker cannot initiate two steps at the same time. "
+                    f"Still working on {self.step.action}"
+                )
 
-        # Create the step
-        action, workdir = split_step_label(label)
-        self.step = WorkerStep(i, action, workdir)
+            # Create the step
+            action, workdir = split_step_label(label)
+            self.step = WorkerStep(i, action, workdir)
 
-        # Create initial StepHash
-        return self.compute_inp_step_hash(inp_hashes, env_vars, check_hash)[0]
+            # Create initial StepHash
+            return self.compute_inp_step_hash(inp_hashes, env_vars, check_hash)[0]
 
     def compute_inp_step_hash(
         self,
@@ -819,41 +852,44 @@ class WorkerHandler:
             so they can be updated in StepUp's workflow database if desired.
             Unchanged ones are not included.
         """
-        # Check the input hashes
-        messages = []
-        new_inp_hashes = []
-        all_inp_hashes = []
-        for path, old_file_hash in old_inp_hashes:
-            new_file_hash = old_file_hash.regen(path)
-            all_inp_hashes.append((path, new_file_hash))
-            if check_hash and new_file_hash != old_file_hash:
-                if new_file_hash.is_unknown:
-                    messages.append(f"Input vanished unexpectedly: {path} ")
-                else:
-                    messages.append(
-                        f"Input changed unexpectedly: {path} "
-                        + fmt_file_hash_diff(old_file_hash, new_file_hash)
-                    )
-                new_inp_hashes.append((path, new_file_hash))
-
-        # If there are unexpected issues with inputs, bail out.
-        if len(messages) > 0:
-            self.step.inp_messages = messages
-            self.step.success = False
-            return None, new_inp_hashes
+        with self.timer.section("compute_inp_step_hash"):
+            # Check the input hashes
+            messages = []
+            new_inp_hashes = []
+            all_inp_hashes = []
+            for path, old_file_hash in old_inp_hashes:
+                new_file_hash = old_file_hash.regen(path)
+                all_inp_hashes.append((path, new_file_hash))
+                if check_hash and new_file_hash != old_file_hash:
+                    if new_file_hash.is_unknown:
+                        messages.append(f"Input vanished unexpectedly: {path} ")
+                    else:
+                        messages.append(
+                            f"Input changed unexpectedly: {path} "
+                            + fmt_file_hash_diff(old_file_hash, new_file_hash)
+                        )
+                    new_inp_hashes.append((path, new_file_hash))
+
+            # If there are unexpected issues with inputs, bail out.
+            if len(messages) > 0:
+                self.step.inp_messages = messages
+                self.step.success = False
+                return None, new_inp_hashes
 
-        # Add the environment variables to the hash
-        env_var_values = [(env_var, os.environ.get(env_var)) for env_var in env_vars]
+            # Add the environment variables to the hash
+            env_var_values = [(env_var, os.environ.get(env_var)) for env_var in env_vars]
 
-        # Create the StepHash
-        label = self.step.action
-        if self.step.workdir != "./":
-            label += f"  # wd={self.step.workdir}"
-        result = StepHash.from_inp(f"{label}", self.explain_rerun, all_inp_hashes, env_var_values)
+            # Create the StepHash
+            label = self.step.action
+            if self.step.workdir != "./":
+                label += f"  # wd={self.step.workdir}"
+            result = StepHash.from_inp(
+                f"{label}", self.explain_rerun, all_inp_hashes, env_var_values
+            )
 
-        # Copy the inp_digest, because it can be useful for some actions.
-        self.step.inp_digest = result.inp_digest
-        return result, []
+            # Copy the inp_digest, because it can be useful for some actions.
+            self.step.inp_digest = result.inp_digest
+            return result, []
 
     @allow_rpc
     def compute_out_step_hash(
@@ -881,22 +917,23 @@ class WorkerHandler:
             so they can be updated in StepUp's workflow database if desired.
             Unchanged ones are not included.
         """
-        # Check the output hashes
-        new_out_hashes = []
-        all_out_hashes = []
-        for path, old_file_hash in sorted(old_out_hashes):
-            new_file_hash = old_file_hash.regen(path)
-            all_out_hashes.append((path, new_file_hash))
-            if new_file_hash != old_file_hash:
-                new_out_hashes.append((path, new_file_hash))
-            if new_file_hash.is_unknown:
-                self.step.out_missing.append(path)
-                self.step.success = False
+        with self.timer.section("compute_out_step_hash"):
+            # Check the output hashes
+            new_out_hashes = []
+            all_out_hashes = []
+            for path, old_file_hash in sorted(old_out_hashes):
+                new_file_hash = old_file_hash.regen(path)
+                all_out_hashes.append((path, new_file_hash))
+                if new_file_hash != old_file_hash:
+                    new_out_hashes.append((path, new_file_hash))
+                if new_file_hash.is_unknown:
+                    self.step.out_missing.append(path)
+                    self.step.success = False
 
-        # Update the step hash
-        if step_hash is not None:
-            step_hash = step_hash.evolve_out(all_out_hashes)
-        return step_hash, new_out_hashes
+            # Update the step hash
+            if step_hash is not None:
+                step_hash = step_hash.evolve_out(all_out_hashes)
+            return step_hash, new_out_hashes
 
     @allow_rpc
     def compute_full_step_hash(
@@ -956,57 +993,58 @@ class WorkerHandler:
 
     @allow_rpc
     async def run(self):
-        await self.reporter("START", self.step.description)
-        await self.reporter.start_step(self.step.description, self.step.i)
-
-        # For internal use in actions:
-        os.environ["STEPUP_STEP_I"] = str(self.step.i)
-        # Client code may use the following:
-        os.environ["STEPUP_STEP_INP_DIGEST"] = self.step.inp_digest.hex()
-        os.environ["ROOT"] = str(Path.cwd().relpath(self.step.workdir))
-        os.environ["HERE"] = str(self.step.workdir.relpath())
-        # Note: the variables defined here should be listed in stepup.core.api.getenv
-
-        # Create IO redirection for stdout and stderr
-        step_err = io.StringIO()
-        step_out = io.StringIO()
-        with (
-            contextlib.chdir(self.step.workdir),
-            contextlib.redirect_stderr(step_err),
-            contextlib.redirect_stdout(step_out),
-        ):
+        with self.timer.section("run"):
+            await self.reporter("START", self.step.description)
+            await self.reporter.start_step(self.step.description, self.step.i)
+
+            # For internal use in actions:
+            os.environ["STEPUP_STEP_I"] = str(self.step.i)
+            # Client code may use the following:
+            os.environ["STEPUP_STEP_INP_DIGEST"] = self.step.inp_digest.hex()
+            os.environ["ROOT"] = str(Path.cwd().relpath(self.step.workdir))
+            os.environ["HERE"] = str(self.step.workdir.relpath())
+            # Note: the variables defined here should be listed in stepup.core.api.getenv
+
+            # Create IO redirection for stdout and stderr
+            step_err = io.StringIO()
+            step_out = io.StringIO()
+            with (
+                contextlib.chdir(self.step.workdir),
+                contextlib.redirect_stderr(step_err),
+                contextlib.redirect_stdout(step_out),
+            ):
+                if self.show_perf:
+                    ru_initial = resource.getrusage(resource.RUSAGE_CHILDREN)
+                    pt_initial = perf_counter()
+                self.step.thread = WorkThread(self.step.action)
+                self.step.thread.start()
+                await self.step.thread.done.wait()
+                self.step.thread.join()
+                self.step.returncode = self.step.thread.returncode
+                self.step.thread = None
+            self.step.stdout = step_out.getvalue()
+            self.step.stderr = step_err.getvalue()
+
+            # Clean up environment variables (to avoid potential confusion)
+            del os.environ["STEPUP_STEP_I"]
+            del os.environ["STEPUP_STEP_INP_DIGEST"]
+            del os.environ["ROOT"]
+            del os.environ["HERE"]
+
+            # Process results of the step.
             if self.show_perf:
-                ru_initial = resource.getrusage(resource.RUSAGE_CHILDREN)
-                pt_initial = perf_counter()
-            self.step.thread = WorkThread(self.step.action)
-            self.step.thread.start()
-            await self.step.thread.done.wait()
-            self.step.thread.join()
-            self.step.returncode = self.step.thread.returncode
-            self.step.thread = None
-        self.step.stdout = step_out.getvalue()
-        self.step.stderr = step_err.getvalue()
-
-        # Clean up environment variables (to avoid potential confusion)
-        del os.environ["STEPUP_STEP_I"]
-        del os.environ["STEPUP_STEP_INP_DIGEST"]
-        del os.environ["ROOT"]
-        del os.environ["HERE"]
-
-        # Process results of the step.
-        if self.show_perf:
-            ru_final = resource.getrusage(resource.RUSAGE_CHILDREN)
-            utime = ru_final.ru_utime - ru_initial.ru_utime
-            stime = ru_final.ru_stime - ru_initial.ru_stime
-            wtime = perf_counter() - pt_initial
-            ru_lines = [
-                f"User CPU time [s]:   {utime:9.4f}",
-                f"System CPU time [s]: {stime:9.4f}",
-                f"Total CPU time [s]:  {utime + stime:9.4f}",
-                f"Wall time [s]:       {wtime:9.4f}",
-            ]
-            self.step.perf_info = "\n".join(ru_lines)
-        self.step.success = self.step.returncode == 0
+                ru_final = resource.getrusage(resource.RUSAGE_CHILDREN)
+                utime = ru_final.ru_utime - ru_initial.ru_utime
+                stime = ru_final.ru_stime - ru_initial.ru_stime
+                wtime = perf_counter() - pt_initial
+                ru_lines = [
+                    f"User CPU time [s]:   {utime:9.4f}",
+                    f"System CPU time [s]: {stime:9.4f}",
+                    f"Total CPU time [s]:  {utime + stime:9.4f}",
+                    f"Wall time [s]:       {wtime:9.4f}",
+                ]
+                self.step.perf_info = "\n".join(ru_lines)
+            self.step.success = self.step.returncode == 0
 
     @allow_rpc
     def get_success(self) -> bool:
@@ -1021,51 +1059,52 @@ class WorkerHandler:
 
     @allow_rpc
     async def report(self):
-        pages = []
-        if not self.step.success:
-            # Format command such that it can be copied and pasted into a shell.
-            command = "stepup act "
-            if any(word.startswith("-") for word in shlex.split(self.step.action)):
-                command += "-- "
-            command += self.step.action
-            if self.step.workdir != "./":
-                command = f"(cd {self.step.workdir} && {command})"
-            lines = [f"Command               {command}"]
-            # Other info on the execution of the step
-            if self.step.returncode is not None:
-                lines.append(f"Return code           {self.step.returncode}")
-            pages.append(("Step info", "\n".join(lines)))
-        if len(self.step.perf_info) > 0:
-            pages.append(("Performance details", self.step.perf_info))
-        if self.step.rescheduled_info != "":
-            pages.append(
-                (
-                    "Rescheduling due to unavailable amended inputs",
-                    self.step.rescheduled_info,
+        with self.timer.section("report"):
+            pages = []
+            if not self.step.success:
+                # Format command such that it can be copied and pasted into a shell.
+                command = "stepup act "
+                if any(word.startswith("-") for word in shlex.split(self.step.action)):
+                    command += "-- "
+                command += self.step.action
+                if self.step.workdir != "./":
+                    command = f"(cd {self.step.workdir} && {command})"
+                lines = [f"Command               {command}"]
+                # Other info on the execution of the step
+                if self.step.returncode is not None:
+                    lines.append(f"Return code           {self.step.returncode}")
+                pages.append(("Step info", "\n".join(lines)))
+            if len(self.step.perf_info) > 0:
+                pages.append(("Performance details", self.step.perf_info))
+            if self.step.rescheduled_info != "":
+                pages.append(
+                    (
+                        "Rescheduling due to unavailable amended inputs",
+                        self.step.rescheduled_info,
+                    )
                 )
-            )
-        else:
-            if len(self.step.inp_messages) > 0:
-                self.step.inp_messages.sort()
-                pages.append(("Invalid inputs", "\n".join(self.step.inp_messages)))
-            if len(self.step.out_missing) > 0:
-                self.step.out_missing.sort()
-                pages.append(("Expected outputs not created", "\n".join(self.step.out_missing)))
-        stdout = self.step.stdout.rstrip()
-        if len(stdout) > 0:
-            pages.append(("Standard output", stdout))
-        stderr = self.step.stderr.rstrip()
-        if len(stderr) > 0:
-            pages.append(("Standard error", stderr))
-        if self.step.rescheduled_info != "":
-            action = "RESCHEDULE"
-        elif self.step.success:
-            action = "SUCCESS"
-        else:
-            action = "FAIL"
-        await self.reporter.stop_step(self.step.i)
-        await self.reporter(action, self.step.description, pages)
-        self.step = None
+            else:
+                if len(self.step.inp_messages) > 0:
+                    self.step.inp_messages.sort()
+                    pages.append(("Invalid inputs", "\n".join(self.step.inp_messages)))
+                if len(self.step.out_missing) > 0:
+                    self.step.out_missing.sort()
+                    pages.append(("Expected outputs not created", "\n".join(self.step.out_missing)))
+            stdout = self.step.stdout.rstrip()
+            if len(stdout) > 0:
+                pages.append(("Standard output", stdout))
+            stderr = self.step.stderr.rstrip()
+            if len(stderr) > 0:
+                pages.append(("Standard error", stderr))
+            if self.step.rescheduled_info != "":
+                action = "RESCHEDULE"
+            elif self.step.success:
+                action = "SUCCESS"
+            else:
+                action = "FAIL"
+            await self.reporter.stop_step(self.step.i)
+            await self.reporter(action, self.step.description, pages)
+            self.step = None
 
     @allow_rpc
     async def skip(self, step_hash: StepHash):
@@ -1155,6 +1194,7 @@ async def async_main():
         # Create the worker handler for the RPC server.
         handler = WorkerHandler(reporter, args.show_perf, args.explain_rerun)
         await serve_socket_rpc(handler, args.worker_socket, handler.stop_event)
+        handler.timer.report()
 
 
 def main():

{stepup-3.2.1 → stepup-3.2.3}/stepup/core/workflow.py

@@ -39,6 +39,7 @@ from .exceptions import GraphError
 from .file import File
 from .hash import FileHash, fmt_digest
 from .nglob import NGlobMulti, convert_nglob_to_regex, iter_wildcard_names
+from .sqlite3 import UInt64
 from .step import Step
 from .utils import myparent, string_to_bool
 
@@ -568,13 +569,14 @@ class Workflow(Cascade):
             sql = "INSERT INTO temp.missing VALUES (?)"
             self.con.executemany(sql, ((file.i,) for file in deferred))
             sql = (
-                "SELECT label, digest, mtime, mode, size, inode FROM temp.missing "
+                "SELECT label, digest, mode, mtime, size, inode AS 'inode [UINT64]' "
+                "FROM temp.missing "
                 "JOIN node ON node.i = temp.missing.node "
                 "JOIN file ON file.node = temp.missing.node"
             )
             return [
-                (path, FileHash(digest, mtime, mode, size, inode))
-                for path, digest, mtime, mode, size, inode in self.con.execute(sql)
+                (path, FileHash(digest, mode, mtime, size, inode))
+                for path, digest, mode, mtime, size, inode in self.con.execute(sql)
             ]
         finally:
             self.con.execute("DROP TABLE IF EXISTS temp.missing")
@@ -597,7 +599,7 @@ class Workflow(Cascade):
             self.con.execute("CREATE TABLE temp.paths(path TEXT PRIMARY KEY)")
             self.con.executemany("INSERT INTO temp.paths VALUES (?)", ((path,) for path in paths))
             sql = (
-                "SELECT label, digest, mode, mtime, size, inode FROM node "
+                "SELECT label, digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM node "
                 "JOIN file ON file.node = node.i JOIN temp.paths ON label = temp.paths.path"
             )
             return [
@@ -735,7 +737,7 @@ class Workflow(Cascade):
             "UPDATE file SET state = ?, digest = ?, mode = ?, mtime = ?, size = ?, inode = ? "
             "WHERE node = ?",
             (
-                (state.value, fh.digest, fh.mode, fh.mtime, fh.size, fh.inode, i)
+                (state.value, fh.digest, fh.mode, fh.mtime, fh.size, UInt64(fh.inode), i)
                 for i, state, fh in new_states_hashes
             ),
         )
@@ -1141,7 +1143,7 @@ class Workflow(Cascade):
                     file.orphan()
         # Delete outputs of steps that are no longer mandatory.
         cur = self.con.execute(
-            "SELECT label, digest, mode, mtime, size, inode FROM file "
+            "SELECT label, digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM file "
             "JOIN node ON node.i = file.node "
             "JOIN dependency ON node.i = consumer "
             "JOIN step ON step.node = supplier "

{stepup-3.2.1 → stepup-3.2.3/stepup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup
-Version: 3.2.1
+Version: 3.2.3
 Summary: StepUp Core provides the basic framework for the StepUp build tool
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later

{stepup-3.2.1 → stepup-3.2.3}/stepup.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ stepup/core/rpc.py
 stepup/core/runner.py
 stepup/core/scheduler.py
 stepup/core/script.py
+stepup/core/sqlite3.py
 stepup/core/startup.py
 stepup/core/step.py
 stepup/core/stepinfo.py