Nuitka-winsvc 1.8.6__cp312-cp312-win_amd64.whl → 1.9.3__cp312-cp312-win_amd64.whl

nuitka/utils/FileOperations.py

@@ -30,15 +30,16 @@ import glob
 import os
 import shutil
 import stat
+import sys
 import tempfile
 import time
 from contextlib import contextmanager
 
 from nuitka.__past__ import (  # pylint: disable=redefined-builtin
     PermissionError,
-    WindowsError,
     basestring,
     raw_input,
+    unicode,
 )
 from nuitka.PythonVersions import python_version
 from nuitka.Tracing import (
@@ -51,7 +52,13 @@ from nuitka.Tracing import (
 
 from .Importing import importFromInlineCopy
 from .ThreadedExecutor import RLock, getThreadIdent
-from .Utils import isLinux, isMacOS, isWin32OrPosixWindows, isWin32Windows
+from .Utils import (
+    isLinux,
+    isMacOS,
+    isWin32OrPosixWindows,
+    isWin32Windows,
+    raiseWindowsError,
+)
 
 # Locking seems to be only required for Windows mostly, but we can keep
 # it for all.
@@ -259,7 +266,9 @@ def getDirectoryRealPath(path):
         path with symlinks resolved
 
     Notes:
-        Workaround for Windows symlink is applied.
+        Workaround for Windows symlink is applied. This function is not recursive
+        at all with older Python, i.e. only the last part, the directory itself
+        is being resolved there.
 
     """
     path = os.path.realpath(path)
@@ -272,6 +281,45 @@ def getDirectoryRealPath(path):
     return path
 
 
+def getFilenameRealPath(path):
+    """Get os.path.realpath with Python2 and Windows symlink workaround applied.
+
+    Args:
+        path: path to get realpath of
+
+    Returns:
+        path with symlinks resolved
+
+    Notes:
+        Workaround for Windows symlinks are applied, this works recursive and
+        assumes that the path given itself is a file and not a directory, and
+        doesn't handle file symlinks at the end on older Python currently, but
+        we shouldn't deal with those.
+    """
+    path = os.path.realpath(path)
+
+    # Attempt to resolve Windows symlinks older Python
+    if os.name == "nt":
+        path = path.strip(os.path.sep)
+
+        if os.path.sep in path:
+            dirname = os.path.dirname(path)
+            filename = os.path.basename(path)
+
+            if dirname:
+                dirname = getDirectoryRealPath(dirname)
+
+                # Drive letters do not get slashes from "os.path.join", so
+                # we inject this here and normalize the path afterwards to
+                # remove any duplication added.
+                if os.path.sep not in dirname:
+                    dirname = dirname + os.path.sep
+
+                return os.path.normpath(os.path.join(dirname, filename))
+
+    return path
+
+
 def listDir(path):
     """Give a sorted listing of a path.
 
@@ -293,8 +341,28 @@ def listDir(path):
     """
     real_path = getDirectoryRealPath(path)
 
+    # The "os.listdir" output needs to be unicode paths, or else it can be unusable
+    # for Python2 on Windows at least. We try to go back on the result.
+    if str is bytes and type(real_path) is str:
+        real_path = unicode(real_path)
+
+    def _tryDecodeToStr(value):
+        if str is bytes:
+            if type(value) is unicode:
+                # File system paths, that should be usable for names of modules,
+                # as Python2 code objects will e.g. hate unicode values.
+
+                # spell-checker: ignore getfilesystemencoding
+                try:
+                    return value.decode(sys.getfilesystemencoding())
+                except UnicodeDecodeError:
+                    return value
+        else:
+            return value
+
     return sorted(
-        [(os.path.join(path, filename), filename) for filename in os.listdir(real_path)]
+        (_tryDecodeToStr(os.path.join(path, filename)), _tryDecodeToStr(filename))
+        for filename in os.listdir(real_path)
     )
 
 
@@ -577,11 +645,19 @@ def deleteFile(path, must_exist):
             raise OSError("Does not exist", path)
 
 
-def splitPath(path):
-    """Split path, skipping empty elements."""
-    return tuple(
-        element for element in os.path.split(path.rstrip(os.path.sep)) if element
-    )
+def searchPrefixPath(path, element):
+    """Search element and return prefix in path, if any."""
+
+    while path:
+        if os.path.normcase(os.path.basename(path)) == os.path.normcase(element):
+            return path
+
+        new_path = os.path.dirname(path)
+        if new_path == path:
+            break
+        path = new_path
+
+    return None
 
 
 def getFilenameExtension(path):
@@ -977,6 +1053,10 @@ def isFilenameBelowPath(path, filename, consider_short=True):
             filename = getExternalUsePath(filename)
             path = getExternalUsePath(path)
 
+            if isWin32Windows():
+                if getWindowsDrive(path) != getWindowsDrive(filename):
+                    return False
+
             result = os.path.relpath(filename, path).split(os.path.sep)[0] != ".."
 
     return result
@@ -1021,10 +1101,7 @@ def getWindowsShortPathName(filename):
             if ctypes.GetLastError() == 5:
                 return filename
 
-            raise WindowsError(
-                ctypes.GetLastError(), ctypes.FormatError(ctypes.GetLastError())
-            )
-
+            raiseWindowsError("getWindowsShortPathName for %s" % filename)
         if output_buf_size >= needed:
             # Short paths should be ASCII. Don't return unicode without a need,
             # as e.g. Scons hates that in environment variables.
@@ -1068,10 +1145,7 @@ def getWindowsLongPathName(filename):
             if ctypes.GetLastError() == 5:
                 return filename
 
-            raise WindowsError(
-                ctypes.GetLastError(), ctypes.FormatError(ctypes.GetLastError())
-            )
-
+            raiseWindowsError("getWindowsLongPathName for %s" % filename)
         if output_buf_size >= needed:
             return output_buf.value
         else:
@@ -1097,6 +1171,8 @@ def getExternalUsePath(filename, only_dirname=False):
 
     if os.name == "nt":
         if filename not in _external_use_path_cache:
+            filename = getFilenameRealPath(filename)
+
             asked_filename = filename
 
             if only_dirname:
@@ -1202,6 +1278,10 @@ def getLinkTarget(filename):
     return is_link, filename
 
 
+# Late import and optional to be there.
+atomicwrites = None
+
+
 def replaceFileAtomic(source_path, dest_path):
     """
     Move ``src`` to ``dst``. If ``dst`` exists, it will be silently
@@ -1216,9 +1296,12 @@ def replaceFileAtomic(source_path, dest_path):
     if python_version >= 0x300:
         os.replace(source_path, dest_path)
     else:
-        importFromInlineCopy("atomicwrites", must_exist=True).replace_atomic(
-            source_path, dest_path
-        )
+        global atomicwrites  # singleton, pylint: disable=global-statement
+
+        if atomicwrites is None:
+            atomicwrites = importFromInlineCopy("atomicwrites", must_exist=True)
+
+        atomicwrites.replace_atomic(source_path, dest_path)
 
 
 def resolveShellPatternToFilenames(pattern):

nuitka/utils/Hashing.py

@@ -21,10 +21,9 @@ Offers support for hashing incrementally and files esp. without having
 to read their contents.
 """
 
-import hashlib
 from binascii import crc32
 
-from nuitka.__past__ import unicode
+from nuitka.__past__ import md5, unicode
 
 from .FileOperations import openTextFile
 
@@ -64,7 +63,7 @@ class HashBase(object):
 
 class Hash(HashBase):
     def __init__(self):
-        self.hash = hashlib.md5()
+        self.hash = md5()
 
     def updateFromBytes(self, value):
         self.hash.update(value)

nuitka/utils/Importing.py

@@ -26,6 +26,7 @@ import sys
 from nuitka.PythonVersions import python_version
 from nuitka.Tracing import general
 
+from .ModuleNames import ModuleName
 from .Utils import withNoDeprecationWarning
 
 
@@ -163,7 +164,7 @@ def _importFromFolder(logger, module_name, path, must_exist, message):
         del sys.path[0]
 
 
-def importFromInlineCopy(module_name, must_exist):
+def importFromInlineCopy(module_name, must_exist, delete_module=False):
     """Import a module from the inline copy stage."""
 
     folder_name = os.path.normpath(
@@ -181,7 +182,7 @@ def importFromInlineCopy(module_name, must_exist):
     elif python_version < 0x360 and os.path.exists(candidate_35):
         folder_name = candidate_35
 
-    return _importFromFolder(
+    module = _importFromFolder(
         module_name=module_name,
         path=folder_name,
         must_exist=must_exist,
@@ -189,6 +190,11 @@ def importFromInlineCopy(module_name, must_exist):
         logger=general,
     )
 
+    if delete_module and module_name in sys.modules:
+        del sys.modules[module_name]
+
+    return module
+
 
 _compile_time_modules = {}
 
@@ -224,7 +230,30 @@ def isBuiltinModuleName(module_name):
     else:
         import _imp
 
-    return _imp.is_builtin(module_name)
+    result = _imp.is_builtin(module_name) or _imp.is_frozen(module_name)
+
+    # Some frozen modules are not actually in that list, e.g.
+    # "importlib._bootstrap_external" on Python3.10 doesn't report to
+    # "_imp.is_frozen()" above, so we check if it's already loaded and from the
+    # "FrozenImporter" by name.
+    if result is False and module_name in sys.modules:
+        module = sys.modules[module_name]
+
+        if hasattr(module, "__loader__"):
+            loader = module.__loader__
+
+            try:
+                result = loader.__name__ == "FrozenImporter"
+            except AttributeError:
+                pass
+
+    return result
+
+
+# Have a set for quicker lookups, and we cannot have "__main__" in there.
+builtin_module_names = set(
+    module_name for module_name in sys.builtin_module_names if module_name != "__main__"
+)
 
 
 def getModuleFilenameSuffixes():
@@ -251,3 +280,31 @@ def getModuleFilenameSuffixes():
             yield suffix, "PY_SOURCE"
         for suffix in importlib.machinery.BYTECODE_SUFFIXES:
             yield suffix, "PY_COMPILED"
+
+
+def getModuleNameAndKindFromFilenameSuffix(module_filename):
+    """Given a filename, decide the module name and kind.
+
+    Args:
+        module_name - file path of the module
+    Returns:
+        Tuple with the name of the module basename, and the kind of the
+        module derived from the file suffix. Can be None, None if is is not a
+        known file suffix.
+    Notes:
+        This doesn't handle packages at all.
+    """
+    if module_filename.endswith(".py"):
+        return ModuleName(os.path.basename(module_filename)[:-3]), "py"
+
+    if module_filename.endswith(".pyc"):
+        return ModuleName(os.path.basename(module_filename)[:-4]), "pyc"
+
+    for suffix in getSharedLibrarySuffixes():
+        if module_filename.endswith(suffix):
+            return (
+                ModuleName(os.path.basename(module_filename)[: -len(suffix)]),
+                "extension",
+            )
+
+    return None, None

nuitka/utils/InstalledPythons.py

@@ -20,9 +20,12 @@
 import os
 import sys
 
-from nuitka.__past__ import WindowsError  # pylint: disable=I0021,redefined-builtin
 from nuitka.containers.OrderedSets import OrderedSet
-from nuitka.PythonVersions import python_version_str
+from nuitka.PythonFlavors import isAnacondaPython, isMSYS2MingwPython
+from nuitka.PythonVersions import (
+    getInstalledPythonRegistryPaths,
+    python_version_str,
+)
 
 from .Execution import (
     NuitkaCalledProcessError,
@@ -55,11 +58,34 @@ class InstalledPython(object):
         return int(major) * 256 + int(minor) * 16
 
     def isAnacondaPython(self):
-        # TODO: This may not yet going to work really
+        if self.python_exe == sys.executable:
+            return isAnacondaPython()
+
+        # TODO: May not yet work really.
         return os.path.exists(
             os.path.join(os.path.dirname(self.python_exe), "..", "conda-meta")
         )
 
+    def isMSYS2MingwPython(self):
+        if self.python_exe == sys.executable:
+            return isMSYS2MingwPython()
+
+        # TODO: May not yet work really.
+        return (
+            os.path.exists(
+                os.path.join(os.path.dirname(self.python_exe), "..", "..", "msys2.ini")
+            )
+            and os.path.basename(os.path.dirname(self.python_exe)) == "mingw64"
+        )
+
+    def getPreferredPackageType(self):
+        if self.isAnacondaPython():
+            return "conda"
+        elif self.isMSYS2MingwPython():
+            return "pacman"
+        else:
+            return "pip"
+
     # Necessary for Python 2.7, otherwise SyntaxError is given on exec.
     @staticmethod
     def _exec(code, context):
@@ -124,38 +150,6 @@ class InstalledPython(object):
 _installed_pythons = {}
 
 
-def getInstalledPythonRegistryPaths(python_version):
-    """Yield all Pythons as found in the Windows registry."""
-    # Windows only code, pylint: disable=I0021,import-error,undefined-variable
-    if str is bytes:
-        import _winreg as winreg  # pylint: disable=I0021,import-error,no-name-in-module
-    else:
-        import winreg  # pylint: disable=I0021,import-error,no-name-in-module
-
-    for hkey_branch in (winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER):
-        for arch_key in (0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY):
-            for suffix in "", "-32", "-arm64":
-                try:
-                    key = winreg.OpenKey(
-                        hkey_branch,
-                        r"SOFTWARE\Python\PythonCore\%s%s\InstallPath"
-                        % (python_version, suffix),
-                        0,
-                        winreg.KEY_READ | arch_key,
-                    )
-
-                    install_dir = os.path.normpath(winreg.QueryValue(key, ""))
-                except WindowsError:
-                    pass
-                else:
-                    candidate = os.path.normpath(
-                        os.path.join(install_dir, "python.exe")
-                    )
-
-                    if os.path.exists(candidate):
-                        yield candidate
-
-
 def _getPythonInstallPathsWindows(python_version):
     """Find Python installation on Windows.
 
@@ -181,7 +175,7 @@ def _getPythonInstallPathsWindows(python_version):
 
         seen.add(candidate)
 
-    for candidate in getInstalledPythonRegistryPaths(python_version=python_version):
+    for candidate in getInstalledPythonRegistryPaths(python_version):
         if candidate not in seen:
             seen.add(candidate)
             yield candidate
@@ -211,6 +205,7 @@ def findPythons(python_version):
         result.add(InstalledPython(python_exe=candidate, python_version=python_version))
 
     _installed_pythons[python_version] = result
+
     return result
 
 

nuitka/utils/Jinja2.py

@@ -43,6 +43,7 @@ def unlikely_or_likely_from(value):
 
 
 _jinja2 = None
+_markupsafe = None
 
 # For pkg resources, we need to keep a reference, after we delete it from
 # "sys.modules" again.
@@ -51,20 +52,25 @@ _loaded_pkg_resources = None
 
 
 def getJinja2Package():
-    global _jinja2, _loaded_pkg_resources  # singleton package using a cache, pylint: disable=global-statement
+    global _jinja2, _markupsafe, _loaded_pkg_resources  # singleton package using a cache, pylint: disable=global-statement
 
     # Import dependencies, sadly we get to manage this ourselves.
-    importFromInlineCopy("markupsafe", must_exist=True)
+    if _markupsafe is None:
+        _markupsafe = importFromInlineCopy("markupsafe", must_exist=True)
 
     # Newer Jinja2 may not use it, but we load it and remove it, so it
     # does not interfere with anything else.
     if "pkg_resources" not in sys.modules:
-        _loaded_pkg_resources = importFromInlineCopy("pkg_resources", must_exist=False)
+        if _loaded_pkg_resources is None:
+            _loaded_pkg_resources = importFromInlineCopy(
+                "pkg_resources", must_exist=False
+            )
 
-    _jinja2 = importFromInlineCopy("jinja2", must_exist=True)
+    if _jinja2 is None:
+        _jinja2 = importFromInlineCopy("jinja2", must_exist=True)
 
     # Unload if it was us loading it, as the inline copy is incomplete.
-    if _loaded_pkg_resources is not None:
+    if _loaded_pkg_resources is not None and "pkg_resources" in sys.modules:
         del sys.modules["pkg_resources"]
 
     return _jinja2

nuitka/utils/ModuleNames.py

@@ -86,7 +86,7 @@ class ModuleName(str):
             return ModuleName(module_name)
 
     def __repr__(self):
-        return "<ModuleName %s>" % str(self)
+        return "<ModuleName '%s'>" % str(self)
 
     def asString(self):
         """Get a simply str value.
@@ -225,6 +225,13 @@ class ModuleName(str):
         else:
             return ModuleName(parent_new)
 
+    def getChildNameFromPackage(self, package_name):
+        """Get child a module name part for a name in the package."""
+        assert self.hasNamespace(package_name)
+
+        submodule_name_str = str(self)[len(str(package_name)) + 1 :]
+        return ModuleName(submodule_name_str)
+
     def matchesToShellPattern(self, pattern):
         """Match a module name to a list of patterns
 
@@ -290,9 +297,10 @@ Do not use %(func_name)s on ModuleName objects, use e.g.
 .hasNamespace(),
 .getBasename(),
 .getTopLevelPackageName()
-.hasOneOfNamespaces
+.hasOneOfNamespaces()
 
-Check API documentation of nuitka.utils.ModuleNames.ModuleName
+Check API documentation of nuitka.utils.ModuleNames.ModuleName for more
+variations.
 ''')
 """ % {
             "func_name": _func_name

nuitka/utils/ReExecute.py

@@ -73,6 +73,9 @@ def reExecuteNuitka(pgo_filename):
     if sys.version_info >= (3, 7) and sys.flags.utf8_mode:
         args += ["-X", "utf8"]
 
+    if sys.version_info >= (3, 11):
+        args += ["-X", "frozen_modules=off"]
+
     if "nuitka.__main__" in sys.modules:
         our_filename = sys.modules["nuitka.__main__"].__file__
     else:
@@ -110,6 +113,10 @@ def reExecuteNuitka(pgo_filename):
         # Note: As side effect, this might modify the "sys.path" too.
         os.environ["NUITKA_PTH_IMPORTED"] = repr(detectPthImportedPackages())
 
+        user_site = getattr(sys.modules["site"], "USER_SITE")
+        if user_site is not None:
+            os.environ["NUITKA_USER_SITE"] = repr(user_site)
+
     os.environ["NUITKA_PYTHONPATH"] = repr(sys.path)
 
     # In some environments, initial "sys.path" does not contain enough to load

nuitka/utils/SharedLibraries.py

@@ -44,7 +44,13 @@ from .FileOperations import (
     makeContainingPath,
     withMadeWritableFileMode,
 )
-from .Utils import isAlpineLinux, isLinux, isMacOS, isWin32Windows
+from .Utils import (
+    isAlpineLinux,
+    isLinux,
+    isMacOS,
+    isWin32Windows,
+    raiseWindowsError,
+)
 from .WindowsResources import (
     RT_MANIFEST,
     VsFixedFileInfoStructure,
@@ -115,6 +121,8 @@ def locateDLL(dll_name):
         left = left[: left.rfind(b" (")]
 
         if python_version >= 0x300:
+            # spell-checker: ignore getfilesystemencoding
+
             left = left.decode(sys.getfilesystemencoding())
             right = right.decode(sys.getfilesystemencoding())
 
@@ -147,7 +155,7 @@ def _removeSxsFromDLL(filename):
         filename: Filename to remove SxS manifests from
     """
     # There may be more files that need this treatment, these are from scans
-    # with the "find_sxs_modules" tool.
+    # with the "find_sxs_modules" tool. spell-checker: ignore winxpgui
     if os.path.normcase(os.path.basename(filename)) not in (
         "sip.pyd",
         "win32ui.pyd",
@@ -207,7 +215,7 @@ def _getDLLVersionWindows(filename):
     # This cannot really fail anymore.
     assert success
 
-    # Look for codepages
+    # Look for code pages
     VerQueryValueA = ctypes.windll.version.VerQueryValueA
     VerQueryValueA.argtypes = (
         ctypes.wintypes.LPCVOID,
@@ -233,6 +241,7 @@ def _getDLLVersionWindows(filename):
     return (ms >> 16) & 0xFFFF, ms & 0xFFFF, (ls >> 16) & 0xFFFF, ls & 0xFFFF
 
 
+# spell-checker: ignore readelf
 _readelf_usage = "The 'readelf' is used to analyse dependencies on ELF using systems and required to be found."
 
 
@@ -244,6 +253,7 @@ def _getSharedLibraryRPATHElf(filename):
     )
 
     for line in output.split(b"\n"):
+        # spell-checker: ignore RUNPATH
         if b"RPATH" in line or b"RUNPATH" in line:
             result = line[line.find(b"[") + 1 : line.rfind(b"]")]
 
@@ -258,10 +268,19 @@ def _getSharedLibraryRPATHElf(filename):
 _otool_output_cache = {}
 
 
+def _getMacOSArchOption():
+    macos_target_arch = getMacOSTargetArch()
+
+    if macos_target_arch != "universal":
+        return ("-arch", macos_target_arch)
+    else:
+        return ()
+
+
 def _getOToolCommandOutput(otool_option, filename):
     filename = os.path.abspath(filename)
 
-    command = ("otool", otool_option, filename)
+    command = ("otool",) + _getMacOSArchOption() + (otool_option, filename)
 
     if otool_option == "-L":
         cache_key = command, os.environ.get("DYLD_LIBRARY_PATH")
@@ -474,6 +493,7 @@ def getPyWin32Dir():
     Notes:
         This is needed for standalone mode only.
     """
+    # spell-checker: ignore pywin32
 
     for path_element in sys.path:
         if not path_element:
@@ -653,6 +673,8 @@ def copyDllFile(source_path, dist_dir, dest_path, executable):
             )
 
     if isMacOS():
+        # spell-checker: ignore xattr
+
         executeToolChecked(
             logger=postprocessing_logger,
             command=("xattr", "-c", target_filename),
@@ -689,10 +711,7 @@ def getWindowsRunningProcessModuleFilename(handle):
 
     res = GetModuleFileName(handle, buf, MAX_PATH)
     if res == 0:
-        # Windows only code, pylint: disable=I0021,undefined-variable
-        raise WindowsError(
-            ctypes.GetLastError(), ctypes.FormatError(ctypes.GetLastError())
-        )
+        raiseWindowsError("getWindowsRunningProcessModuleFilename")
 
     return os.path.normcase(buf.value)
 
@@ -726,10 +745,7 @@ def _getWindowsRunningProcessModuleHandles():
     )
 
     if not res:
-        # Windows only code, pylint: disable=I0021,undefined-variable
-        raise WindowsError(
-            ctypes.GetLastError(), ctypes.FormatError(ctypes.GetLastError())
-        )
+        raiseWindowsError("getWindowsRunningProcessModuleHandles")
 
     return tuple(handle for handle in handles if handle is not None)
 
@@ -782,10 +798,18 @@ Error, needs 'nm' on your system, to detect exported DLL symbols."""
 
 
 def getDllExportedSymbols(logger, filename):
-    if isLinux():
+    if isLinux or isMacOS():
+        if isLinux():
+            command = ("nm", "-D", filename)
+        elif isMacOS():
+            command = ("nm", "-gU", filename) + _getMacOSArchOption()
+        else:
+            # Need to add e.g. FreeBSD here.
+            assert False
+
         output = executeToolChecked(
             logger=logger,
-            command=("nm", "-D", filename),
+            command=command,
             absence_message=_nm_usage,
         )