Nuitka-winsvc 2.7.7__cp313-cp313-win_amd64.whl

nuitka/freezer/DllDependenciesWin32.py

@@ -0,0 +1,315 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+"""DLL dependency scan methods for Win32 Windows.
+
+Note: MSYS2, aka POSIX Windows is dealt with in the "DllDependenciesPosix" module.
+"""
+
+import os
+import sys
+
+from nuitka.__past__ import iterItems
+from nuitka.build.SconsUtils import readSconsReport
+from nuitka.containers.OrderedSets import OrderedSet
+from nuitka.Options import isExperimental, isShowProgress
+from nuitka.plugins.Plugins import Plugins
+from nuitka.PythonFlavors import isAnacondaPython
+from nuitka.PythonVersions import getSystemPrefixPath
+from nuitka.Tracing import inclusion_logger
+from nuitka.utils.AppDirs import getCacheDir
+from nuitka.utils.FileOperations import (
+    areSamePaths,
+    getDirectoryRealPath,
+    getFileContentByLine,
+    getSubDirectoriesWithDlls,
+    isFilenameSameAsOrBelowPath,
+    listDllFilesFromDirectory,
+    makePath,
+    putTextFileContents,
+    withFileLock,
+)
+from nuitka.utils.Hashing import Hash
+from nuitka.utils.SharedLibraries import getPEFileUsedDllNames, getPyWin32Dir
+from nuitka.utils.Utils import getArchitecture
+from nuitka.Version import version_string
+
+from .DependsExe import detectDLLsWithDependencyWalker
+from .DllDependenciesCommon import getPackageSpecificDLLDirectories
+
+_scan_dir_cache = {}
+
+
+def detectDLLsWithPEFile(binary_filename, scan_dirs):
+    pe_dll_names = getPEFileUsedDllNames(binary_filename)
+
+    result = OrderedSet()
+
+    # Get DLL imports from PE file
+    for dll_name in pe_dll_names:
+        dll_name = dll_name.lower()
+
+        # Search DLL path from scan dirs
+        for scan_dir in scan_dirs:
+            dll_filename = os.path.normcase(
+                os.path.abspath(os.path.join(scan_dir, dll_name))
+            )
+
+            if os.path.isfile(dll_filename):
+                result.add(dll_filename)
+
+                break
+        else:
+            if dll_name.startswith("API-MS-WIN-") or dll_name.startswith("EXT-MS-WIN-"):
+                continue
+
+            # Ignore this runtime DLL of Python2, will be coming via manifest.
+            # spell-checker: ignore msvcr90
+            if dll_name == "msvcr90.dll":
+                continue
+
+            # Ignore API DLLs, they can come in from PATH, but we do not want to
+            # include them.
+            if dll_name.startswith("api-ms-win-"):
+                continue
+
+            # Ignore UCRT runtime, this must come from OS, spell-checker: ignore ucrtbase
+            if dll_name == "ucrtbase.dll":
+                continue
+
+    return result
+
+
+def detectBinaryPathDLLsWin32(
+    is_main_executable,
+    source_dir,
+    original_dir,
+    binary_filename,
+    package_name,
+    use_path,
+    use_cache,
+    update_cache,
+):
+    # For ARM64 and on user request, we can use "pefile" for dependency detection.
+    dependency_tool = (
+        "pefile"
+        if (getArchitecture() == "arm64" or isExperimental("force-dependencies-pefile"))
+        else "depends.exe"
+    )
+
+    # This is the caching mechanism and plugin handling for DLL imports.
+    if use_cache or update_cache:
+
+        cache_filename = _getCacheFilename(
+            dependency_tool=dependency_tool,
+            is_main_executable=is_main_executable,
+            source_dir=source_dir,
+            original_dir=original_dir,
+            binary_filename=binary_filename,
+            package_name=package_name,
+            use_path=use_path,
+        )
+
+        if use_cache:
+            with withFileLock():
+                if not os.path.exists(cache_filename):
+                    use_cache = False
+
+        if use_cache:
+            result = OrderedSet()
+
+            for line in getFileContentByLine(cache_filename):
+                line = line.strip()
+
+                # Detect files that have become missing by ignoring the cache.
+                if not os.path.exists(line):
+                    break
+
+                result.add(line)
+            else:
+                return result
+
+    if isShowProgress():
+        inclusion_logger.info("Analyzing dependencies of '%s'." % binary_filename)
+
+    scan_dirs = _getScanDirectories(
+        package_name=package_name, original_dir=original_dir, use_path=use_path
+    )
+
+    if dependency_tool == "depends.exe":
+        result = detectDLLsWithDependencyWalker(
+            binary_filename=binary_filename,
+            source_dir=source_dir,
+            scan_dirs=scan_dirs,
+        )
+    else:
+        result = detectDLLsWithPEFile(
+            binary_filename=binary_filename,
+            scan_dirs=scan_dirs,
+        )
+
+    if update_cache:
+        putTextFileContents(filename=cache_filename, contents=result)
+
+    return result
+
+
+_path_contributions = {}
+
+
+def _getPathContribution(use_path):
+    """Contributions from PATH environment variable.
+
+    These are ignored if use_path is False, but for
+    Anaconda we need to keep those elements pointing
+    to inside of it.
+    """
+    if use_path not in _path_contributions:
+        _path_contributions[use_path] = OrderedSet()
+
+        for path_dir in os.environ["PATH"].split(";"):
+            if not os.path.isdir(path_dir):
+                continue
+
+            # spell-checker: ignore SYSTEMROOT
+            if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
+                continue
+            if areSamePaths(
+                path_dir, os.path.join(os.environ["SYSTEMROOT"], "System32")
+            ):
+                continue
+            if areSamePaths(
+                path_dir, os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")
+            ):
+                continue
+
+            # For Anaconda, we cannot ignore PATH on its inside.
+            if not use_path:
+                if not isAnacondaPython() or not isFilenameSameAsOrBelowPath(
+                    filename=path_dir,
+                    path=(
+                        getSystemPrefixPath(),
+                        sys.prefix,
+                    ),
+                ):
+                    continue
+
+            _path_contributions[use_path].add(path_dir)
+
+    return _path_contributions[use_path]
+
+
+def _getScanDirectories(package_name, original_dir, use_path):
+    # TODO: Move PyWin32 specific stuff to yaml dll section
+
+    cache_key = package_name, original_dir
+
+    if cache_key in _scan_dir_cache:
+        return _scan_dir_cache[cache_key]
+
+    scan_dirs = [os.path.dirname(sys.executable), getSystemPrefixPath()]
+
+    if package_name is not None:
+        scan_dirs.extend(
+            getPackageSpecificDLLDirectories(
+                package_name=package_name,
+                consider_plugins=False,
+            )
+        )
+
+    if original_dir is not None:
+        scan_dirs.append(original_dir)
+        scan_dirs.extend(getSubDirectoriesWithDlls(original_dir))
+
+    if package_name is not None and package_name.isBelowNamespace("win32com"):
+        py_win32_dir = getPyWin32Dir()
+
+        if py_win32_dir is not None:
+            scan_dirs.append(py_win32_dir)
+
+    scan_dirs.extend(_getPathContribution(use_path=use_path))
+
+    result = []
+
+    # Remove directories that hold no DLLs.
+    for scan_dir in scan_dirs:
+        scan_dir = getDirectoryRealPath(scan_dir)
+
+        # Not a directory, or no DLLs, or not accessible, no use.
+        try:
+            if not os.path.isdir(scan_dir) or not any(
+                listDllFilesFromDirectory(scan_dir)
+            ):
+                continue
+        except OSError:
+            continue
+
+        result.append(os.path.realpath(scan_dir))
+
+    _scan_dir_cache[cache_key] = result
+    return result
+
+
+def _getCacheFilename(
+    dependency_tool,
+    is_main_executable,
+    source_dir,
+    original_dir,
+    binary_filename,
+    package_name,
+    use_path,
+):
+    original_filename = os.path.join(original_dir, os.path.basename(binary_filename))
+    original_filename = os.path.normcase(original_filename)
+
+    hash_value = Hash()
+
+    if is_main_executable:
+        # Normalize main program name for caching as well, but need to use the
+        # scons information to distinguish different compilers, so we use
+        # different libs there.
+        hash_value.updateFromValues(
+            "".join(
+                key + value
+                for key, value in iterItems(readSconsReport(source_dir=source_dir))
+                # Ignore values, that are variable per compilation.
+                if key not in ("CLCACHE_STATS", "CCACHE_LOGFILE", "CCACHE_DIR")
+            )
+        )
+    else:
+        hash_value.updateFromValues(original_filename)
+        hash_value.updateFromFile(filename=original_filename)
+
+    # Have different values for different Python major versions.
+    hash_value.updateFromValues(sys.version, sys.executable)
+
+    # Plugins may change their influence.
+    hash_value.updateFromValues(*Plugins.getCacheContributionValues(package_name))
+
+    # Take Nuitka version into account as well, ought to catch code changes.
+    hash_value.updateFromValues(version_string)
+
+    # Using PATH or not, should also be considered different.
+    if use_path:
+        hash_value.updateFromValues(os.getenv("PATH"))
+
+    cache_dir = os.path.join(getCacheDir("library_dependencies"), dependency_tool)
+    makePath(cache_dir)
+
+    return os.path.join(cache_dir, hash_value.asHexDigest())
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/freezer/ImportDetection.py

@@ -0,0 +1,359 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" Detect imports made for code by Python.
+
+In the freezer, this is a step done to detect the technically needed modules to
+initialize the CPython interpreter.
+
+"""
+
+import os
+import pkgutil
+import sys
+
+from nuitka.importing.StandardLibrary import (
+    getStandardLibraryPaths,
+    isStandardLibraryNoAutoInclusionModule,
+    isStandardLibraryPath,
+    scanStandardLibraryPath,
+)
+from nuitka.Options import isStandaloneMode
+from nuitka.PythonVersions import python_version
+from nuitka.Tracing import general, printError
+from nuitka.utils.Execution import executeProcess
+from nuitka.utils.FileOperations import areSamePaths
+from nuitka.utils.ModuleNames import ModuleName
+
+
+def _detectImports(command):
+    # This is pretty complicated stuff, with variants to deal with.
+    # pylint: disable=too-many-branches,too-many-statements
+
+    # Print statements for stuff to show, the modules loaded.
+    if python_version >= 0x300:
+        command += """
+print("\\n".join(sorted(
+    "import %s # sourcefile %s" % (module.__name__, module.__file__)
+    for module in sys.modules.values()
+    if getattr(module, "__file__", None) not in (None, "<frozen>"
+))), file = sys.stderr)"""
+
+    reduced_path = [
+        path_element
+        for path_element in sys.path
+        if not areSamePaths(path_element, ".")
+        if not areSamePaths(
+            path_element, os.path.dirname(sys.modules["__main__"].__file__)
+        )
+    ]
+
+    # Make sure the right import path (the one Nuitka binary is running with)
+    # is used.
+    command = (
+        "import sys; sys.path = %s; sys.real_prefix = sys.prefix;" % repr(reduced_path)
+    ) + command
+
+    if str is not bytes:
+        command = command.encode("utf8")
+
+    _stdout, stderr, exit_code = executeProcess(
+        command=(
+            sys.executable,
+            "-s",
+            "-S",
+            "-v",
+            "-c",
+            "import sys;exec(sys.stdin.read())",
+        ),
+        stdin=command,
+        env=dict(os.environ, PYTHONIOENCODING="utf-8"),
+    )
+
+    assert type(stderr) is bytes
+
+    # Don't let errors here go unnoticed.
+    if exit_code != 0:
+        # An error by the user pressing CTRL-C should not lead to the below output.
+        if b"KeyboardInterrupt" in stderr:
+            general.sysexit("Pressed CTRL-C while detecting early imports.")
+
+        general.warning("There is a problem with detecting imports, CPython said:")
+        for line in stderr.split(b"\n"):
+            printError(line)
+        general.sysexit("Error, please report the issue with above output.")
+
+    detections = []
+
+    for line in stderr.replace(b"\r", b"").split(b"\n"):
+        if line.startswith(b"import "):
+            parts = line.split(b" # ", 2)
+
+            module_name = parts[0].split(b" ", 2)[1].strip(b"'")
+            origin = parts[1].split()[0]
+
+            if python_version >= 0x300:
+                module_name = module_name.decode("utf8")
+
+            module_name = ModuleName(module_name)
+
+            if origin == b"precompiled":
+                # This is a ".pyc" file that was imported, even before we have a
+                # chance to do anything, we need to preserve it.
+                filename = parts[1][len(b"precompiled from ") :]
+                if python_version >= 0x300:
+                    filename = filename.decode("utf8")
+
+                # Do not leave standard library when freezing.
+                if not isStandardLibraryPath(filename):
+                    continue
+
+                detections.append((module_name, 3, "precompiled", filename))
+            elif origin == b"from" and python_version < 0x300:
+                filename = parts[1][len(b"from ") :]
+                if str is not bytes:  # For consistency, and maybe later reuse
+                    filename = filename.decode("utf8")
+
+                # Do not leave standard library when freezing.
+                if not isStandardLibraryPath(filename):
+                    continue
+
+                if filename.endswith(".py"):
+                    detections.append((module_name, 2, "sourcefile", filename))
+                else:
+                    assert False
+            elif origin == b"sourcefile":
+                filename = parts[1][len(b"sourcefile ") :]
+                if python_version >= 0x300:
+                    filename = filename.decode("utf8")
+
+                # Do not leave standard library when freezing.
+                if not isStandardLibraryPath(filename):
+                    continue
+
+                # Python3 started lying in "__name__" for the "_collections_abc" module
+                # as well, which then is interpreted wrongly, this affects at least 3.9
+                if os.path.basename(filename) in (
+                    "_collections_abc.py",
+                    "_collections_abc.pyc",
+                ):
+                    # Note: This was only useful for 3.13.0, and 3.13.1 already
+                    # didn't need it anymore. So precise version check for a major
+                    # version is correct here.
+                    if python_version == 0x3D0:
+                        detections.append((module_name, 2, "sourcefile", filename))
+
+                    module_name = ModuleName("_collections_abc")
+
+                if filename.endswith(".py"):
+                    detections.append((module_name, 2, "sourcefile", filename))
+                elif filename.endswith(".pyc"):
+                    detections.append((module_name, 3, "precompiled", filename))
+                elif not filename.endswith("<frozen>"):
+                    # Python3 started lying in "__name__" for the "_decimal"
+                    # calls itself "decimal", which then is wrong and also
+                    # clashes with "decimal" proper
+                    if python_version >= 0x300 and module_name == "decimal":
+                        module_name = ModuleName("_decimal")
+
+                    detections.append((module_name, 2, "extension", filename))
+            elif origin == b"dynamically":
+                # Shared library in early load, happens on RPM based systems and
+                # or self compiled Python installations.
+                filename = parts[1][len(b"dynamically loaded from ") :]
+                if python_version >= 0x300:
+                    filename = filename.decode("utf8")
+
+                # Do not leave standard library when freezing.
+                if not isStandardLibraryPath(filename):
+                    continue
+
+                detections.append((module_name, 1, "extension", filename))
+
+    module_names = set()
+
+    for module_name, _priority, kind, filename in sorted(detections):
+        if isStandardLibraryNoAutoInclusionModule(module_name):
+            continue
+
+        if kind == "extension":
+            # Extension modules are not tracked outside of standalone
+            # mode.
+            if not isStandaloneMode():
+                continue
+
+            # That is not a shared library, but looks like one.
+            if module_name == "__main__":
+                continue
+
+            module_names.add(module_name)
+        elif kind == "precompiled":
+            module_names.add(module_name)
+        elif kind == "sourcefile":
+            module_names.add(module_name)
+        else:
+            assert False, kind
+
+    return module_names
+
+
+def _detectEarlyImports():
+    encoding_names = [
+        m[1] for m in pkgutil.iter_modules(sys.modules["encodings"].__path__)
+    ]
+
+    if os.name != "nt":
+        # On posix systems, and posix Python variants on Windows, these won't
+        # work and fail to import.
+        for encoding_name in ("mbcs", "cp65001", "oem"):
+            if encoding_name in encoding_names:
+                encoding_names.remove(encoding_name)
+
+    # Not for startup.
+    for non_locale_encoding in (
+        "bz2_codec",
+        "idna",
+        "base64_codec",
+        "hex_codec",
+        "rot_13",
+        "punycode",
+    ):
+        if non_locale_encoding in encoding_names:
+            encoding_names.remove(non_locale_encoding)
+
+    import_code = ";".join(
+        "import encodings.%s" % encoding_name
+        for encoding_name in sorted(encoding_names)
+    )
+
+    import_code += ";import locale;"
+
+    # For Python3 we patch inspect without knowing if it is used.
+    if python_version >= 0x300:
+        import_code += "import inspect;import importlib._bootstrap"
+
+    return _detectImports(command=import_code)
+
+
+_early_modules_names = None
+
+
+def detectEarlyImports():
+    if not isStandaloneMode():
+        return ()
+
+    global _early_modules_names  # singleton, pylint: disable=global-statement
+
+    if _early_modules_names is None:
+        _early_modules_names = tuple(sorted(_detectEarlyImports()))
+
+    return _early_modules_names
+
+
+def _detectStdlibAutoInclusionModules():
+    if not isStandaloneMode():
+        return ()
+
+    stdlib_modules = set()
+
+    # Scan the standard library paths (multiple in case of virtualenv).
+    for stdlib_dir in getStandardLibraryPaths():
+        for module_name in scanStandardLibraryPath(stdlib_dir):
+            if not isStandardLibraryNoAutoInclusionModule(module_name):
+                stdlib_modules.add(module_name)
+
+    # Put here ones that should be imported first.
+    first_ones = ("Tkinter",)
+
+    # We have to fight zombie modules in this, some things, e.g. Tkinter
+    # on newer Python 2.7, comes back after failure without a second error
+    # being raised, leading to other issues. So we fight it after each
+    # module that was tried, and prevent re-try by adding a meta path
+    # based loader that will never load it again, and remove it from the
+    # "sys.modules" over and over, once it sneaks back. The root cause is
+    # that extension modules sometimes only raise an error when first
+    # imported, not the second time around.
+    # Otherwise this just makes imports of everything so we can see where
+    # it comes from and what it requires.
+
+    import_code = """
+imports = %r
+
+failed = set()
+
+class ImportBlocker(object):
+    def find_module(self, fullname, path = None):
+        if fullname in failed:
+            return self
+
+        return None
+
+    def load_module(self, name):
+        raise ImportError("%%s has failed before" %% name)
+
+sys.meta_path.insert(0, ImportBlocker())
+
+for imp in imports:
+    try:
+        __import__(imp)
+    except (ImportError, SyntaxError):
+        failed.add(imp)
+    except ValueError as e:
+        if "cannot contain null bytes" in e.args[0]:
+            failed.add(imp)
+        else:
+            sys.stderr.write("PROBLEM with '%%s'\\n" %% imp)
+            raise
+    except Exception:
+        sys.stderr.write("PROBLEM with '%%s'\\n" %% imp)
+        raise
+
+    for fail in failed:
+        if fail in sys.modules:
+            del sys.modules[fail]
+""" % (
+        tuple(
+            module_name.asString()
+            for module_name in sorted(
+                stdlib_modules, key=lambda name: (name not in first_ones, name)
+            )
+        ),
+    )
+
+    return _detectImports(command=import_code)
+
+
+_stdlib_modules_names = None
+
+
+def detectStdlibAutoInclusionModules():
+    if not isStandaloneMode():
+        return ()
+
+    global _stdlib_modules_names  # singleton, pylint: disable=global-statement
+
+    if _stdlib_modules_names is None:
+        _stdlib_modules_names = _detectStdlibAutoInclusionModules()
+
+        for module_name in detectEarlyImports():
+            _stdlib_modules_names.discard(module_name)
+
+        _stdlib_modules_names = tuple(sorted(_stdlib_modules_names))
+
+    return _stdlib_modules_names
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.