Nuitka-winsvc 2.7.7__cp313-cp313-win_amd64.whl

nuitka/utils/MemoryUsage.py

@@ -0,0 +1,165 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" Tools for tracing memory usage at compiled time.
+
+"""
+
+from nuitka.containers.OrderedDicts import OrderedDict
+from nuitka.Tracing import memory_logger, printLine
+
+from .Utils import isMacOS, isWin32Windows
+
+
+def getOwnProcessMemoryUsage():
+    """Memory usage of own process in bytes."""
+
+    if isWin32Windows():
+        # adapted from http://code.activestate.com/recipes/578513
+        import ctypes.wintypes
+
+        # Lets allow this to match Windows API it reflects,
+        # pylint: disable=invalid-name
+        class PROCESS_MEMORY_COUNTERS_EX(ctypes.Structure):
+            _fields_ = [
+                ("cb", ctypes.wintypes.DWORD),
+                ("PageFaultCount", ctypes.wintypes.DWORD),
+                ("PeakWorkingSetSize", ctypes.c_size_t),
+                ("WorkingSetSize", ctypes.c_size_t),
+                ("QuotaPeakPagedPoolUsage", ctypes.c_size_t),
+                ("QuotaPagedPoolUsage", ctypes.c_size_t),
+                ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t),
+                ("QuotaNonPagedPoolUsage", ctypes.c_size_t),
+                ("PagefileUsage", ctypes.c_size_t),
+                ("PeakPagefileUsage", ctypes.c_size_t),
+                ("PrivateUsage", ctypes.c_size_t),
+            ]
+
+        GetProcessMemoryInfo = ctypes.windll.psapi.GetProcessMemoryInfo
+        GetProcessMemoryInfo.argtypes = (
+            ctypes.wintypes.HANDLE,
+            ctypes.POINTER(PROCESS_MEMORY_COUNTERS_EX),
+            ctypes.wintypes.DWORD,
+        )
+        GetProcessMemoryInfo.restype = ctypes.wintypes.BOOL
+
+        counters = PROCESS_MEMORY_COUNTERS_EX()
+        rv = GetProcessMemoryInfo(
+            ctypes.windll.kernel32.GetCurrentProcess(),
+            ctypes.byref(counters),
+            ctypes.sizeof(counters),
+        )
+
+        if not rv:
+            raise ctypes.WinError()
+
+        return counters.PrivateUsage
+    else:
+        import resource  # Posix only code, pylint: disable=I0021,import-error
+
+        # The value is from "getrusage", which has OS dependent scaling, at least
+        # macOS and Linux are different. Others maybe too.
+        if isMacOS():
+            factor = 1
+        else:
+            factor = 1024
+
+        return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * factor
+
+
+def getHumanReadableProcessMemoryUsage():
+    return formatMemoryUsageValue(getOwnProcessMemoryUsage())
+
+
+class MemoryWatch(object):
+    def __init__(self):
+        self.start = getOwnProcessMemoryUsage()
+        self.stop = None
+
+    def finish(self, message):
+        self.stop = getOwnProcessMemoryUsage()
+
+        _logMemoryInfo(message, self.value())
+
+    def asStr(self):
+        return formatMemoryUsageValue(self.value())
+
+    def value(self):
+        return self.stop - self.start
+
+
+_memory_infos = OrderedDict()
+
+
+def getMemoryInfos():
+    return _memory_infos
+
+
+def collectMemoryUsageValue(memory_usage_name):
+    assert memory_usage_name not in _memory_infos, memory_usage_name
+    _memory_infos[memory_usage_name] = getOwnProcessMemoryUsage()
+
+    return _memory_infos[memory_usage_name]
+
+
+def formatMemoryUsageValue(value):
+    if abs(value) < 1024 * 1014:
+        return "%.2f KB (%d bytes)" % (value / 1024.0, value)
+    elif abs(value) < 1024 * 1014 * 1024:
+        return "%.2f MB (%d bytes)" % (value / (1024 * 1024.0), value)
+    elif abs(value) < 1024 * 1014 * 1024 * 1024:
+        return "%.2f GB (%d bytes)" % (value / (1024 * 1024 * 1024.0), value)
+    else:
+        return "%d bytes" % value
+
+
+def _logMemoryInfo(message, memory_usage):
+    if message:
+        memory_logger.info("%s: %s" % (message, formatMemoryUsageValue(memory_usage)))
+
+
+def reportMemoryUsage(identifier, message):
+    memory_usage = collectMemoryUsageValue(identifier)
+
+    _logMemoryInfo(message, memory_usage)
+
+
+def startMemoryTracing():
+    try:
+        import tracemalloc
+    except ImportError:
+        pass
+    else:
+        tracemalloc.start()
+
+
+def showMemoryTrace():
+    try:
+        import tracemalloc
+    except ImportError:
+        pass
+    else:
+        snapshot = tracemalloc.take_snapshot()
+        stats = snapshot.statistics("lineno")
+
+        printLine("Top 50 memory allocations:")
+        for count, stat in enumerate(stats):
+            if count == 50:
+                break
+            printLine(stat)
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/utils/ModuleNames.py

@@ -0,0 +1,317 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" Module names are common string type, which deserves special operations.
+
+These are used in Nuitka for module and package names in most places, and
+allow to easily make checks on them.
+
+"""
+
+import fnmatch
+import os
+
+
+def checkModuleName(value):
+    return ".." not in str(value) and not (
+        str(value).endswith(".") or str(value) == "."
+    )
+
+
+# Trigger names for shared use.
+post_module_load_trigger_name = "-postLoad"
+pre_module_load_trigger_name = "-preLoad"
+
+trigger_names = (pre_module_load_trigger_name, post_module_load_trigger_name)
+
+
+def makeTriggerModuleName(module_name, trigger_name):
+    assert trigger_name in trigger_names
+
+    return ModuleName(module_name + trigger_name)
+
+
+# Multidist prefix
+_multi_dist_prefix = "multidist-"
+
+
+def makeMultidistModuleName(count, suffix):
+    return ModuleName("%s%d-%s" % (_multi_dist_prefix, count, suffix))
+
+
+class ModuleName(str):
+    def __init__(self, value):
+        assert checkModuleName(value), value
+
+        # TODO: Disallow some conversion, e.g. from module, function, etc.
+        # objects, and white list what types we accept.
+
+        str.__init__(value)
+
+    @staticmethod
+    def makeModuleNameInPackage(module_name, package_name):
+        """Create a module name in a package.
+
+        Args:
+            - module_name (str or ModuleName) module name to put below the package
+            - package_name (str or ModuleName or None) package to put below
+
+        Returns:
+            Module name "package_name.module_name" or if "package_name" is None
+            then simply "module_name".
+
+        Notes:
+            Prefer this factory function over manually duplicating the pattern
+            behind it.
+
+        """
+
+        if package_name is not None:
+            return ModuleName(package_name + "." + module_name)
+        else:
+            return ModuleName(module_name)
+
+    def __repr__(self):
+        return "<ModuleName '%s'>" % str(self)
+
+    def asString(self):
+        """Get a simply str value.
+
+        Notes:
+            This should only be used to create constant values for code
+            generation, there is no other reason to lower the type of
+            these values otherwise.
+        """
+
+        return str(self)
+
+    def asPath(self):
+        return str(self).replace(".", os.path.sep)
+
+    def getPackageName(self):
+        """Get the package name if any.
+
+        Returns:
+            ModuleName of the containing package or None if already
+            top level.
+        """
+
+        return self.splitModuleBasename()[0]
+
+    def getParentPackageNames(self):
+        """Yield parent packages in descending order."""
+        parent_packages = []
+
+        parent_package = self.getPackageName()
+        while parent_package is not None:
+            parent_packages.append(parent_package)
+
+            parent_package = parent_package.getPackageName()
+
+        for parent_package in reversed(parent_packages):
+            yield parent_package
+
+    def getRelativePackageName(self, level):
+        assert level >= 0
+
+        parts = self.asString().split(".")
+
+        while level > 0:
+            if not parts:
+                return None
+
+            del parts[-1]
+            level -= 1
+
+        return ModuleName(".".join(parts))
+
+    def getTopLevelPackageName(self):
+        """Get the top level package name.
+
+        Returns:
+            ModuleName of the top level name.
+        """
+        package_name = self.getPackageName()
+
+        if package_name is None:
+            return self
+        else:
+            return package_name.getTopLevelPackageName()
+
+    def getBasename(self):
+        """Get leaf name of the module without package part.
+
+        Returns:
+            ModuleName without package.
+        """
+        return self.splitModuleBasename()[1]
+
+    def splitModuleBasename(self):
+        """Split a module into package name and module name."""
+
+        if "." in self:
+            package_part = ModuleName(self[: self.rfind(".")])
+            module_name = ModuleName(self[self.rfind(".") + 1 :])
+        else:
+            package_part = None
+            module_name = self
+
+        return package_part, module_name
+
+    def splitPackageName(self):
+        """Split a module into the top level package name and remaining module name."""
+
+        if "." in self:
+            package_part = ModuleName(self[: self.find(".")])
+            module_name = ModuleName(self[self.find(".") + 1 :])
+        else:
+            package_part = None
+            module_name = self
+
+        return package_part, module_name
+
+    def hasNamespace(self, package_name):
+        return self == package_name or self.isBelowNamespace(package_name)
+
+    def hasOneOfNamespaces(self, *package_names):
+        """Check if a module name is below one of many namespaces.
+
+        Args:
+            - package_names: Star argument that allows also lists and tuples
+
+        Returns:
+            bool - module name is below one of the packages.
+        """
+
+        for package_name in package_names:
+            if type(package_name) in (tuple, list, set):
+                if self.hasOneOfNamespaces(*package_name):
+                    return True
+            elif self.hasNamespace(package_name):
+                return True
+
+        return False
+
+    def isBelowNamespace(self, package_name):
+        assert type(package_name) in (str, ModuleName), package_name
+
+        # Avoid startswith on these.
+        return str(self).startswith(package_name + ".")
+
+    def getChildNamed(self, *args):
+        """Get a child package with these names added."""
+        return ModuleName(".".join([self] + list(args)))
+
+    def getSiblingNamed(self, *args):
+        """Get a sub-package relative to this child package."""
+        return self.getPackageName().getChildNamed(*args)
+
+    def relocateModuleNamespace(self, parent_old, parent_new):
+        """Get a module name, where the top level part is translated from old to new."""
+        assert self.hasNamespace(parent_old)
+
+        submodule_name_str = str(self)[len(str(parent_old)) + 1 :]
+
+        if submodule_name_str:
+            return ModuleName(parent_new).getChildNamed(submodule_name_str)
+        else:
+            return ModuleName(parent_new)
+
+    def getChildNameFromPackage(self, package_name):
+        """Get child a module name part for a name in the package."""
+        assert self.hasNamespace(package_name)
+
+        submodule_name_str = str(self)[len(str(package_name)) + 1 :]
+        return ModuleName(submodule_name_str)
+
+    def matchesToShellPattern(self, pattern):
+        """Match a module name to a patterns
+
+        Args:
+            pattern:
+                Complies with fnmatch.fnmatch description
+                or also is below the package. So "*.tests" will matches to also
+                "something.tests.MyTest", thereby allowing to match whole
+                packages with one pattern only.
+        Returns:
+            Tuple of two values, where the first value is the result, second value
+            explains why the pattern matched and how.
+        """
+
+        if self == pattern:
+            return True, "is exact match of '%s'" % pattern
+        elif self.isBelowNamespace(pattern):
+            return True, "is package content of '%s'" % pattern
+        elif fnmatch.fnmatch(self.asString(), pattern):
+            return True, "matches pattern '%s'" % pattern
+        elif fnmatch.fnmatch(self.asString(), pattern + ".*"):
+            return True, "is package content of match to pattern '%s'" % pattern
+        else:
+            return False, None
+
+    def matchesToShellPatterns(self, patterns):
+        """Match a module name to a list of patterns
+
+        Args:
+            patterns:
+                List of patterns that comply with fnmatch.fnmatch description
+                or also is below the package. So "*.tests" will matches to also
+                "something.tests.MyTest", thereby allowing to match whole
+                packages with one pattern only.
+        Returns:
+            Tuple of two values, where the first value is the result, second value
+            explains which pattern matched and how.
+        """
+
+        for pattern in patterns:
+            match, reason = self.matchesToShellPattern(pattern)
+
+            if match:
+                return match, reason
+
+        # No match result
+        return False, None
+
+    def isFakeModuleName(self):
+        return str(self).endswith(trigger_names)
+
+    def isMultidistModuleName(self):
+        return str(self).startswith(_multi_dist_prefix)
+
+    # Reject APIs being used. TODO: Maybe make this a decorator for reuse.
+    # TODO: Add rsplit and subscript operations too.
+    for _func_name in ("split", "startswith", "endswith", "__mod__"):
+        code = """\
+def %(func_name)s(*args, **kwargs):
+    from nuitka.Errors import NuitkaCodeDeficit
+    raise NuitkaCodeDeficit('''
+Do not use %(func_name)s on ModuleName objects, use e.g.
+.hasNamespace(),
+.getBasename(),
+.getTopLevelPackageName()
+.hasOneOfNamespaces()
+
+Check API documentation of nuitka.utils.ModuleNames.ModuleName for more
+variations.
+''')
+""" % {
+            "func_name": _func_name
+        }
+
+        exec(code)  # Avoid code duplication, pylint: disable=exec-used
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/utils/PackageResources.py

@@ -0,0 +1,44 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" Access package data. """
+
+import os
+import pkgutil
+
+from nuitka.importing.Importing import locateModule
+
+from .FileOperations import getFileContents
+
+
+def getPackageData(package_name, resource):
+    """Get the package data, but without loading the code, i.e. we try and avoids its loader.
+
+    If it's absolutely necessary, we fallback to "pkgutil.get_data" but
+    only if we absolutely cannot find it easily.
+    """
+    package_directory = locateModule(package_name, None, 0)[1]
+
+    if package_directory is not None:
+        resource_filename = os.path.join(package_directory, resource)
+
+        if os.path.exists(resource_filename):
+            return getFileContents(resource_filename, mode="rb")
+
+    return pkgutil.get_data(package_name.asString(), resource)
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/utils/ReExecute.py

@@ -0,0 +1,152 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+"""Ability to restart Nuitka, needed for removing site module effects and using Python PGO after compile.
+
+Note: This avoids imports at all costs, such that initial startup doesn't do more
+than necessary.
+
+spell-checker: ignore execl, Popen
+"""
+
+import os
+import sys
+
+
+def callExecProcess(args, uac):
+    """Do exec in a portable way preserving exit code.
+
+    On Windows, unfortunately there is no real exec, so we have to spawn
+    a new process instead.
+    """
+
+    # We better flush these, "os.execl" won't do it anymore.
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    # On Windows "os.execl" does not work properly
+    if os.name == "nt":
+        import subprocess
+
+        args = list(args)
+        del args[1]
+
+        try:
+            # Context manager is not available on all Python versions, pylint: disable=consider-using-with
+            process = subprocess.Popen(args=args, shell=uac)
+            process.communicate()
+            # No point in cleaning up, just exit the hard way.
+            try:
+                os._exit(process.returncode)
+            except OverflowError:
+                # Seems negative values go wrong otherwise,
+                # see https://bugs.python.org/issue28474
+                os._exit(process.returncode - 2**32)
+        except KeyboardInterrupt:
+            # There was a more relevant stack trace already, so abort this
+            # right here.
+            os._exit(2)
+        except OSError as e:
+            print("Error, executing: %s" % e)
+            os._exit(2)
+
+    else:
+        # The star arguments is the API of execl
+        os.execl(*args)
+
+
+def setLaunchingNuitkaProcessEnvironmentValue(environment_variable_name, value):
+    os.environ[environment_variable_name] = str(os.getpid()) + ":" + value
+
+
+def reExecuteNuitka(pgo_filename):
+    # Execute with full path as the process name, so it can find itself and its
+    # libraries.
+    args = [sys.executable, sys.executable]
+
+    if sys.version_info >= (3, 7) and sys.flags.utf8_mode:
+        args += ["-X", "utf8"]
+
+    if sys.version_info >= (3, 11):
+        args += ["-X", "frozen_modules=off"]
+
+    if "nuitka.__main__" in sys.modules:
+        our_filename = sys.modules["nuitka.__main__"].__file__
+    else:
+        our_filename = sys.modules["__main__"].__file__
+
+    args += ["-S", our_filename]
+
+    setLaunchingNuitkaProcessEnvironmentValue(
+        "NUITKA_BINARY_NAME", sys.modules["__main__"].__file__
+    )
+    setLaunchingNuitkaProcessEnvironmentValue(
+        "NUITKA_PACKAGE_HOME",
+        os.path.dirname(os.path.abspath(sys.modules["nuitka"].__path__[0])),
+    )
+
+    if pgo_filename is not None:
+        args.append("--pgo-python-input=%s" % pgo_filename)
+    else:
+        setLaunchingNuitkaProcessEnvironmentValue("NUITKA_SYS_PREFIX", sys.prefix)
+
+    # Same arguments as before.
+    args += sys.argv[1:]
+
+    from nuitka.importing.PreloadedPackages import (
+        detectPreLoadedPackagePaths,
+        detectPthImportedPackages,
+    )
+
+    os.environ["NUITKA_NAMESPACES"] = repr(detectPreLoadedPackagePaths())
+
+    if "site" in sys.modules:
+        site_filename = sys.modules["site"].__file__
+        if site_filename.endswith(".pyc"):
+            site_filename = site_filename[:-4] + ".py"
+
+        setLaunchingNuitkaProcessEnvironmentValue("NUITKA_SITE_FILENAME", site_filename)
+
+        # Note: As side effect, this might modify the "sys.path" too.
+        os.environ["NUITKA_PTH_IMPORTED"] = repr(detectPthImportedPackages())
+
+        user_site = getattr(sys.modules["site"], "USER_SITE", None)
+        if user_site is not None:
+            os.environ["NUITKA_USER_SITE"] = repr(user_site)
+
+    setLaunchingNuitkaProcessEnvironmentValue("NUITKA_PYTHONPATH", repr(sys.path))
+
+    # In some environments, initial "sys.path" does not contain enough to load
+    # "ast" module, which however we use to decode "NUITKA_PYTHONPATH", this
+    # helps solve the chicken and egg problem.
+    import ast
+
+    setLaunchingNuitkaProcessEnvironmentValue(
+        "NUITKA_PYTHONPATH_AST", os.path.dirname(ast.__file__)
+    )
+
+    if sys.flags.no_site:
+        os.environ["NUITKA_NOSITE_FLAG"] = "1"
+
+    os.environ["PYTHONHASHSEED"] = "0"
+
+    setLaunchingNuitkaProcessEnvironmentValue("NUITKA_RE_EXECUTION", "1")
+
+    # Does not return:
+    callExecProcess(args, uac=False)
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.