stepup 3.2.2__tar.gz → 3.2.3__tar.gz

{stepup-3.2.2/stepup.egg-info → stepup-3.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup
-Version: 3.2.2
+Version: 3.2.3
 Summary: StepUp Core provides the basic framework for the StepUp build tool
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later

{stepup-3.2.2 → stepup-3.2.3}/stepup/core/browse.py

@@ -24,7 +24,6 @@ import contextlib
 import importlib.resources
 import os
 import pickle
-import sqlite3
 import stat
 import traceback
 from collections.abc import Iterator
@@ -37,6 +36,7 @@ from path import Path
 
 from .enums import FileState, Mandatory, StepState
 from .hash import fmt_digest
+from .sqlite3 import connect
 
 
 def browse_subcommand(subparser: argparse.ArgumentParser) -> callable:
@@ -275,8 +275,8 @@ class GraphServer(BaseHTTPRequestHandler):
             print("Loading database...")
             if self.con is not None:
                 self.con.close()
-            self.con = sqlite3.Connection(":memory:")
-            src = sqlite3.Connection(self.path_db)
+            self.con = connect(":memory:")
+            src = connect(self.path_db)
             try:
                 src.backup(self.con)
             finally:
@@ -415,7 +415,9 @@ class GraphServer(BaseHTTPRequestHandler):
 
         elif kind == "file":
             (state_i, digest, mode, mtime, size, inode) = self.con.execute(
-                "SELECT state, digest, mode, mtime, size, inode FROM file WHERE node = ?", (node_i,)
+                "SELECT state, digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM file "
+                "WHERE node = ?",
+                (node_i,),
             ).fetchone()
             state = FileState(state_i)
             yield f'<p><b>State:</b> <span class="{state.name.lower()}">{state.name}</span></p>'

{stepup-3.2.2 → stepup-3.2.3}/stepup/core/cascade.py

@@ -562,6 +562,9 @@ class Cascade:
         # While making this change, the enums were also made more intuitive.
         # Schema 2 became outdated due to the worker actions.
         # Schema 3 became outdated due to a change in step table (dirty field).
+
+        # Delayed Schema updates, for version 5:
+        # - Use UINT64 with PARSE_DECLTYPES instead of PARSE_COLNAMES.
         return 4
 
     @classmethod

{stepup-3.2.2 → stepup-3.2.3}/stepup/core/clean.py

@@ -29,7 +29,8 @@ from rich.console import Console
 from .cascade import DROP_CONSUMERS, INITIAL_CONSUMERS, RECURSE_CONSUMERS
 from .enums import FileState
 from .hash import FileHash
-from .utils import mynormpath, sqlite3_copy_in_memory, translate, translate_back
+from .sqlite3 import copy_db_in_memory
+from .utils import mynormpath, translate, translate_back
 
 
 def clean_subcommand(subparser: argparse.ArgumentParser) -> callable:
@@ -91,7 +92,7 @@ def clean_tool(args: argparse.Namespace):
     # Copy the database in memory and work on the copy.
     root = Path(os.getenv("STEPUP_ROOT", "."))
     path_db = root / ".stepup/graph.db"
-    with sqlite3_copy_in_memory(path_db) as con:
+    with copy_db_in_memory(path_db) as con:
         clean(con, tr_paths, args)
 
 
@@ -175,7 +176,7 @@ def fmtnum(i: int):
 CREATE_INITIAL_PATHS = "CREATE TABLE temp.initial_path(path TEXT PRIMARY KEY) WITHOUT ROWID"
 
 SELECT_OUTPUTS = f"""
-SELECT label, file.state, orphan, digest, mode, mtime, size, inode FROM node
+SELECT label, file.state, orphan, digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM node
 JOIN all_consumer ON node.i = all_consumer.current
 JOIN file ON file.node = all_consumer.current
 WHERE file.state in

{stepup-3.2.2 → stepup-3.2.3}/stepup/core/director.py

@@ -24,7 +24,6 @@ import asyncio
 import logging
 import os
 import signal
-import sqlite3
 import sys
 import time
 import traceback
@@ -48,6 +47,7 @@ from .reporter import ReporterClient
 from .rpc import allow_rpc, serve_socket_rpc
 from .runner import Runner
 from .scheduler import Scheduler
+from .sqlite3 import connect
 from .startup import startup_from_db
 from .step import Step
 from .stepinfo import StepInfo
@@ -258,7 +258,7 @@ async def serve(
     check_plan("plan.py")
 
     # Create basic components
-    con = sqlite3.connect(".stepup/graph.db", cached_statements=1024)
+    con = connect(".stepup/graph.db")
     dblock = DBLock(con)
     workflow = Workflow(con)
     scheduler = Scheduler(workflow.job_queue, workflow.config_queue, workflow.job_queue_changed)

{stepup-3.2.2 → stepup-3.2.3}/stepup/core/file.py

@@ -29,6 +29,7 @@ from path import Path
 from .cascade import Node
 from .enums import DirWatch, FileState
 from .hash import FileHash
+from .sqlite3 import UInt64
 from .utils import format_digest
 
 if TYPE_CHECKING:
@@ -102,7 +103,10 @@ class File(Node):
         # If the file was previously BUILT or OUTDATED, and created again as AWAITED,
         # it should copy that state
         if state == FileState.AWAITED:
-            sql = "SELECT state, digest, mode, mtime, size, inode FROM file WHERE node = ?"
+            sql = (
+                "SELECT state, digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM file "
+                "WHERE node = ?"
+            )
             row = self.con.execute(sql, (self.i,)).fetchone()
             if row is not None and row[0] in (FileState.BUILT.value, FileState.OUTDATED.value):
                 state = FileState(row[0])
@@ -121,7 +125,7 @@ class File(Node):
                 "mode": mode,
                 "mtime": mtime,
                 "size": size,
-                "inode": inode,
+                "inode": UInt64(inode),
             },
         )
         # If the state is BUILT, mark it as OUTDATED to force a rebuild.
@@ -215,7 +219,7 @@ class File(Node):
         self.con.execute(sql, (state.value, self.i))
 
     def get_hash(self) -> FileHash:
-        sql = "SELECT digest, mode, mtime, size, inode FROM file WHERE node = ?"
+        sql = "SELECT digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM file WHERE node = ?"
         row = self.con.execute(sql, (self.i,)).fetchone()
         return FileHash(*row)
 

stepup-3.2.3/stepup/core/sqlite3.py

@@ -0,0 +1,83 @@
+# StepUp Core provides the basic framework for the StepUp build tool.
+# Copyright 2024-2026 Toon Verstraelen
+#
+# This file is part of StepUp Core.
+#
+# StepUp Core is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# StepUp Core is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+#
+# --
+"""Wrapper for SQLite3 functionality."""
+
+import contextlib
+import os
+import sqlite3
+from collections.abc import Iterator
+from typing import Self
+
+__all__ = ("UInt64", "connect", "copy_db_in_memory")
+
+
+class UInt64(int):
+    """A wrapper to tell SQLite this int should be treated as an unsigned 64-bit value."""
+
+    MAX_SIGNED_64 = 2**63 - 1
+    MAX_WRAPAROUND_64 = 2**64
+    MAX_UNSIGNED_64 = MAX_WRAPAROUND_64 - 1
+
+    @staticmethod
+    def adapt(val: Self) -> int:
+        if not (0 <= val <= UInt64.MAX_UNSIGNED_64):
+            raise ValueError(f"Value {val} out of UINT64 range")
+        return val - UInt64.MAX_WRAPAROUND_64 if val > UInt64.MAX_SIGNED_64 else val
+
+    @staticmethod
+    def convert(val: bytes) -> Self:
+        val = int(val)
+        if val < 0:
+            val += UInt64.MAX_WRAPAROUND_64
+        return UInt64(val)
+
+
+sqlite3.register_adapter(UInt64, UInt64.adapt)
+sqlite3.register_converter("UINT64", UInt64.convert)
+
+
+def connect(path: str | os.PathLike[str], **kwargs) -> sqlite3.Connection:
+    """Connect to a SQLite database, with the appropriate settings for StepUp.
+
+    The following deviations from the default settings are used:
+
+    - Types can be detected from column names,
+      which allows us to use the custom UINT64 type for file inodes.
+    - The `cached_statements` parameter is set to a large value to improve
+      performance when executing many similar statements.
+    """
+    my_kwargs = {"cached_statements": 1024, "detect_types": sqlite3.PARSE_COLNAMES}
+    my_kwargs.update(kwargs)
+    return sqlite3.connect(path, **my_kwargs)
+
+
+@contextlib.contextmanager
+def copy_db_in_memory(path_db) -> Iterator[sqlite3.Connection]:
+    """Copy an SQLite database into memory and yield the connection."""
+    dst = connect(":memory:")
+    try:
+        src = connect(path_db)
+        try:
+            src.backup(dst)
+        finally:
+            src.close()
+        yield dst
+    finally:
+        dst.close()

{stepup-3.2.2 → stepup-3.2.3}/stepup/core/startup.py

@@ -124,7 +124,7 @@ async def scan_file_changes(
 ) -> tuple[set[str], set[str]]:
     """Check all files in the workflow for changes."""
     sql = (
-        "SELECT label, state, digest, mode, mtime, size, inode "
+        "SELECT label, state, digest, mode, mtime, size, inode AS 'inode [UINT64]' "
         "FROM node JOIN file ON node.i = file.node AND state NOT IN (?, ?) AND NOT orphan"
     )
     data = (FileState.AWAITED.value, FileState.VOLATILE.value)

{stepup-3.2.2 → stepup-3.2.3}/stepup/core/step.py

@@ -558,7 +558,7 @@ class Step(Node):
             fields.append("state")
             join_file = True
         if yield_hash:
-            fields.extend(["digest", "mode", "mtime", "size", "inode"])
+            fields.extend(["digest", "mode", "mtime", "size", "inode AS 'inode [UINT64]'"])
             join_file = True
         if yield_orphan:
             fields.append("orphan")
@@ -731,7 +731,7 @@ class Step(Node):
         sql = (
             "SELECT node.label, node.orphan, file.state, "
             "EXISTS (SELECT 1 FROM amended_dep WHERE amended_dep.i = dep.i), "
-            "file.digest, file.mode, file.mtime, file.size, file.inode "
+            "file.digest, file.mode, file.mtime, file.size, file.inode AS 'inode [UINT64]' "
             "FROM node JOIN dependency AS dep ON node.i = dep.supplier "
             "JOIN file ON file.node = node.i "
             "WHERE dep.consumer = ?"

{stepup-3.2.2 → stepup-3.2.3}/stepup/core/utils.py

@@ -20,7 +20,6 @@
 """Small utilities used throughout."""
 
 import asyncio
-import contextlib
 import logging
 import os
 import re
@@ -28,7 +27,7 @@ import shlex
 import sqlite3
 import string
 import sys
-from collections.abc import Collection, Iterator
+from collections.abc import Collection
 
 import attrs
 from path import Path
@@ -429,18 +428,3 @@ def string_to_bool(v: str | bool) -> bool:
             return False
         raise ValueError(f"Cannot interpret '{v}' as a boolean value.")
     raise TypeError(f"Expected a boolean value or string. Got {type(v).__name__}")
-
-
-@contextlib.contextmanager
-def sqlite3_copy_in_memory(path_db) -> Iterator[sqlite3.Connection]:
-    """Copy an SQLite database into memory and yield the connection."""
-    dst = sqlite3.Connection(":memory:")
-    try:
-        src = sqlite3.Connection(path_db)
-        try:
-            src.backup(dst)
-        finally:
-            src.close()
-        yield dst
-    finally:
-        dst.close()

{stepup-3.2.2 → stepup-3.2.3}/stepup/core/workflow.py

@@ -39,6 +39,7 @@ from .exceptions import GraphError
 from .file import File
 from .hash import FileHash, fmt_digest
 from .nglob import NGlobMulti, convert_nglob_to_regex, iter_wildcard_names
+from .sqlite3 import UInt64
 from .step import Step
 from .utils import myparent, string_to_bool
 
@@ -568,13 +569,14 @@ class Workflow(Cascade):
             sql = "INSERT INTO temp.missing VALUES (?)"
             self.con.executemany(sql, ((file.i,) for file in deferred))
             sql = (
-                "SELECT label, digest, mtime, mode, size, inode FROM temp.missing "
+                "SELECT label, digest, mode, mtime, size, inode AS 'inode [UINT64]' "
+                "FROM temp.missing "
                 "JOIN node ON node.i = temp.missing.node "
                 "JOIN file ON file.node = temp.missing.node"
             )
             return [
-                (path, FileHash(digest, mtime, mode, size, inode))
-                for path, digest, mtime, mode, size, inode in self.con.execute(sql)
+                (path, FileHash(digest, mode, mtime, size, inode))
+                for path, digest, mode, mtime, size, inode in self.con.execute(sql)
             ]
         finally:
             self.con.execute("DROP TABLE IF EXISTS temp.missing")
@@ -597,7 +599,7 @@ class Workflow(Cascade):
             self.con.execute("CREATE TABLE temp.paths(path TEXT PRIMARY KEY)")
             self.con.executemany("INSERT INTO temp.paths VALUES (?)", ((path,) for path in paths))
             sql = (
-                "SELECT label, digest, mode, mtime, size, inode FROM node "
+                "SELECT label, digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM node "
                 "JOIN file ON file.node = node.i JOIN temp.paths ON label = temp.paths.path"
             )
             return [
@@ -735,7 +737,7 @@ class Workflow(Cascade):
             "UPDATE file SET state = ?, digest = ?, mode = ?, mtime = ?, size = ?, inode = ? "
             "WHERE node = ?",
             (
-                (state.value, fh.digest, fh.mode, fh.mtime, fh.size, fh.inode, i)
+                (state.value, fh.digest, fh.mode, fh.mtime, fh.size, UInt64(fh.inode), i)
                 for i, state, fh in new_states_hashes
             ),
         )
@@ -1141,7 +1143,7 @@ class Workflow(Cascade):
                     file.orphan()
         # Delete outputs of steps that are no longer mandatory.
         cur = self.con.execute(
-            "SELECT label, digest, mode, mtime, size, inode FROM file "
+            "SELECT label, digest, mode, mtime, size, inode AS 'inode [UINT64]' FROM file "
             "JOIN node ON node.i = file.node "
             "JOIN dependency ON node.i = consumer "
             "JOIN step ON step.node = supplier "

{stepup-3.2.2 → stepup-3.2.3/stepup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup
-Version: 3.2.2
+Version: 3.2.3
 Summary: StepUp Core provides the basic framework for the StepUp build tool
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later

{stepup-3.2.2 → stepup-3.2.3}/stepup.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ stepup/core/rpc.py
 stepup/core/runner.py
 stepup/core/scheduler.py
 stepup/core/script.py
+stepup/core/sqlite3.py
 stepup/core/startup.py
 stepup/core/step.py
 stepup/core/stepinfo.py