Nuitka-winsvc 2.4.8__cp311-cp311-win_amd64.whl → 2.5.1__cp311-cp311-win_amd64.whl

nuitka/freezer/Standalone.py

@@ -32,6 +32,7 @@ from nuitka.PythonFlavors import (
     getHomebrewInstallPath,
     isAnacondaPython,
     isHomebrewPython,
+    isMSYS2MingwPython,
 )
 from nuitka.PythonVersions import getSystemPrefixPath
 from nuitka.Tracing import general, inclusion_logger
@@ -115,6 +116,7 @@ def _detectBinaryDLLs(
     original_filename,
     binary_filename,
     package_name,
+    use_path,
     use_cache,
     update_cache,
 ):
@@ -141,6 +143,7 @@ def _detectBinaryDLLs(
                 original_dir=os.path.dirname(original_filename),
                 binary_filename=binary_filename,
                 package_name=package_name,
+                use_path=use_path,
                 use_cache=use_cache,
                 update_cache=update_cache,
             )
@@ -227,7 +230,11 @@ def copyDllsUsed(dist_dir, standalone_entry_points):
     )
 
 
-def _reduceToPythonPath(used_dlls):
+_excluded_system_dlls = set()
+
+
+def _reduceToPythonPath(used_dll_paths):
+    """Remove DLLs outside of python path."""
     inside_paths = getPythonUnpackedSearchPath()
 
     if isAnacondaPython():
@@ -236,28 +243,67 @@ def _reduceToPythonPath(used_dlls):
     if isHomebrewPython():
         inside_paths.insert(0, getHomebrewInstallPath())
 
+    if isMSYS2MingwPython():
+        inside_paths.insert(0, os.path.join(getSystemPrefixPath(), "bin"))
+
     def decideInside(dll_filename):
         return any(
             isFilenameBelowPath(path=inside_path, filename=dll_filename)
             for inside_path in inside_paths
         )
 
-    used_dlls = set(
-        dll_filename for dll_filename in used_dlls if decideInside(dll_filename)
-    )
+    kept_used_dll_paths = OrderedSet()
+    removed_dll_paths = OrderedSet()
+
+    for dll_filename in used_dll_paths:
+        if decideInside(dll_filename):
+            kept_used_dll_paths.add(dll_filename)
+        else:
+            if dll_filename not in _excluded_system_dlls:
+                _excluded_system_dlls.add(dll_filename)
+                inclusion_logger.info("Not including system DLL '%s'" % dll_filename)
+
+            removed_dll_paths.add(dll_filename)
 
-    return used_dlls
+    return kept_used_dll_paths, removed_dll_paths
+
+
+_removed_dll_usages = {}
+
+
+def getRemovedUsedDllsInfo():
+    return _removed_dll_usages.items()
 
 
 def _detectUsedDLLs(standalone_entry_point, source_dir):
+    # TODO: We are handling a bunch of cases here, but also some special cases,
+    # that should live elsewhere.
+    # pylint: disable=too-many-branches,too-many-locals
+
+    if standalone_entry_point.module_name is not None:
+        module_name, module_filename, _kind, finding = locateModule(
+            standalone_entry_point.module_name, parent_package=None, level=0
+        )
+
+        # TODO: How can this be None at all.
+        if module_filename is not None and isStandardLibraryPath(module_filename):
+            allow_outside_dependencies = True
+        else:
+            allow_outside_dependencies = Plugins.decideAllowOutsideDependencies(
+                standalone_entry_point.module_name
+            )
+    else:
+        allow_outside_dependencies = False
+
     binary_filename = standalone_entry_point.source_path
     try:
         used_dll_paths = _detectBinaryDLLs(
             is_main_executable=standalone_entry_point.kind == "executable",
             source_dir=source_dir,
             original_filename=standalone_entry_point.source_path,
-            binary_filename=standalone_entry_point.source_path,
+            binary_filename=binary_filename,
             package_name=standalone_entry_point.package_name,
+            use_path=allow_outside_dependencies,
             use_cache=not shallNotUseDependsExeCachedResults(),
             update_cache=not shallNotStoreDependsExeCachedResults(),
         )
@@ -265,6 +311,14 @@ def _detectUsedDLLs(standalone_entry_point, source_dir):
         inclusion_logger.info(
             "Not including due to forbidden DLL '%s'." % binary_filename
         )
+    except (RuntimeError, Exception) as e:
+        inclusion_logger.warning(
+            """\
+Error, cannot detect used DLLs for DLL '%s' in package '%s' due to: %s"""
+            % (binary_filename, standalone_entry_point.package_name, str(e))
+        )
+
+        raise
     else:
         # Plugins generally decide if they allow dependencies from the outside
         # based on the package name.
@@ -280,21 +334,25 @@ def _detectUsedDLLs(standalone_entry_point, source_dir):
             ), standalone_entry_point.module_name
             assert finding == "absolute", standalone_entry_point.module_name
 
-            if isStandardLibraryPath(module_filename):
-                allow_outside_dependencies = True
-            else:
-                allow_outside_dependencies = Plugins.decideAllowOutsideDependencies(
-                    standalone_entry_point.module_name
-                )
+            if not allow_outside_dependencies:
+                used_dll_paths, removed_dll_paths = _reduceToPythonPath(used_dll_paths)
+
+                if removed_dll_paths:
+                    _removed_dll_usages[standalone_entry_point] = (
+                        "no outside dependencies allowed for '%s'"
+                        % standalone_entry_point.module_name,
+                        removed_dll_paths,
+                    )
 
-            if allow_outside_dependencies is False:
-                used_dll_paths = _reduceToPythonPath(used_dll_paths)
+        # Lets be sure of this
+        assert type(used_dll_paths) is OrderedSet, type(used_dll_paths)
 
-        # Allow plugins can prevent inclusion, this may discard things from used_dlls.
+        # Allow plugins can prevent inclusion, this may discard things from
+        # used_dlls through its return value.
         removed_dlls = Plugins.removeDllDependencies(
-            dll_filename=binary_filename, dll_filenames=used_dll_paths
+            dll_filename=binary_filename, dll_filenames=OrderedSet(used_dll_paths)
         )
-        used_dll_paths = tuple(OrderedSet(used_dll_paths) - OrderedSet(removed_dlls))
+        used_dll_paths = used_dll_paths - removed_dlls
 
         for used_dll_path in used_dll_paths:
             extension_standalone_entry_point = getIncludedExtensionModule(used_dll_path)
@@ -309,7 +367,7 @@ def _detectUsedDLLs(standalone_entry_point, source_dir):
                     "openvino",
                     "av",
                 )
-                and areInSamePaths(standalone_entry_point.source_path, used_dll_path)
+                and areInSamePaths(binary_filename, used_dll_path)
             ):
                 # TODO: If used by a DLL from the same folder, put it there,
                 # otherwise top level, but for now this is limited to a few cases

nuitka/importing/Importing.py

@@ -36,7 +36,11 @@ from nuitka.PythonVersions import python_version
 from nuitka.Tracing import my_print, recursion_logger
 from nuitka.tree.ReformulationMultidist import locateMultidistModule
 from nuitka.utils.AppDirs import getCacheDir
-from nuitka.utils.FileOperations import listDir, removeDirectory
+from nuitka.utils.FileOperations import (
+    getNormalizedPath,
+    listDir,
+    removeDirectory,
+)
 from nuitka.utils.Hashing import getFileContentsHash
 from nuitka.utils.Importing import (
     builtin_module_names,
@@ -826,10 +830,10 @@ def locateModule(module_name, parent_package, level):
 
     assert module_package is None or (
         type(module_package) is ModuleName and module_package != ""
-    ), repr(module_package)
+    ), ("Must not attempt to locate %r" % module_name)
 
     if module_filename is not None:
-        module_filename = os.path.normpath(module_filename)
+        module_filename = getNormalizedPath(module_filename)
 
         module_name, module_kind = getModuleNameAndKindFromFilename(module_filename)
         module_name = ModuleName.makeModuleNameInPackage(module_name, module_package)
@@ -874,7 +878,7 @@ def decideModuleSourceRef(filename, module_name, is_main, is_fake, logger):
     is_package = False
 
     if is_main and os.path.isdir(filename):
-        source_filename = os.path.join(filename, "__main__.py")
+        source_filename = getNormalizedPath(os.path.join(filename, "__main__.py"))
 
         if not os.path.isfile(source_filename):
             sys.stderr.write(
@@ -904,7 +908,7 @@ def decideModuleSourceRef(filename, module_name, is_main, is_fake, logger):
     elif isPackageDir(filename):
         is_package = True
 
-        source_filename = os.path.join(filename, "__init__.py")
+        source_filename = getNormalizedPath(os.path.join(filename, "__init__.py"))
 
         if not os.path.isfile(source_filename):
             source_ref = SourceCodeReferences.fromFilename(
@@ -913,7 +917,7 @@ def decideModuleSourceRef(filename, module_name, is_main, is_fake, logger):
             is_namespace = True
         else:
             source_ref = SourceCodeReferences.fromFilename(
-                filename=os.path.abspath(source_filename)
+                filename=getNormalizedPath(os.path.abspath(source_filename))
             )
 
     else:
@@ -1042,6 +1046,7 @@ _stdlib_module_raises = {
     "_interpreters": False,
     "_interpqueues": False,
     "_sysconfig": False,
+    "_suggestions": False,
 }
 
 
@@ -1052,6 +1057,28 @@ def isNonRaisingBuiltinModule(module_name):
     return _stdlib_module_raises.get(module_name)
 
 
+def _getChildPackageNames(module_name):
+    module_name = ModuleName(module_name)
+
+    _module_name, module_filename, _module_kind, _finding = locateModule(
+        parent_package=None, module_name=module_name, level=0
+    )
+
+    package_dir = module_filename
+
+    for module_info in iter_modules([package_dir]):
+        child_name = module_name.getChildNamed(module_info.name)
+        yield child_name
+
+        if module_info.ispkg:
+            for sub_module_name in getChildPackageNames(child_name):
+                yield sub_module_name
+
+
+def getChildPackageNames(module_name):
+    return tuple(_getChildPackageNames(module_name))
+
+
 #     Part of "Nuitka", an optimizing Python compiler that is compatible and
 #     integrates with CPython, but also works on its own.
 #

nuitka/importing/Recursion.py

@@ -339,12 +339,23 @@ def _addIncludedModule(module, package_only):
                         module_package=module.getFullName(),
                         package_only=False,
                     )
-                elif sub_path.endswith(".py"):
+                elif sub_filename.endswith(".py"):
                     checkPluginSinglePath(
                         sub_path,
                         module_package=module.getFullName(),
                         package_only=False,
                     )
+                else:
+                    for suffix in getSharedLibrarySuffixes():
+                        if (
+                            sub_filename.endswith(suffix)
+                            and "." not in sub_filename[: -len(suffix)]
+                        ):
+                            checkPluginSinglePath(
+                                sub_path,
+                                module_package=module.getFullName(),
+                                package_only=False,
+                            )
 
     elif module.isCompiledPythonModule() or module.isUncompiledPythonModule():
         ModuleRegistry.addRootModule(module)

nuitka/importing/StandardLibrary.py

@@ -141,53 +141,55 @@ def isStandardLibraryPath(filename):
     return _is_standard_library_path_cache[filename]
 
 
-# Some modules we want to exclude entirely.
-_excluded_stdlib_modules = ["__main__.py", "__init__.py", "antigravity.py"]
-
-if not isWin32Windows():
-    # On posix systems, and posix Python variants on Windows, this won't
-    # work.
-    _excluded_stdlib_modules.append("wintypes.py")
-    _excluded_stdlib_modules.append("cp65001.py")
-
-
 def scanStandardLibraryPath(stdlib_dir):
     # There is a lot of filtering here, done in branches, so there is many of
-    # them, but that's acceptable, pylint: disable=too-many-branches,too-many-statements
+    # them, but that's acceptable, pylint: disable=too-many-branches
 
     for root, dirs, filenames in os.walk(stdlib_dir):
         import_path = root[len(stdlib_dir) :].strip("/\\")
         import_path = import_path.replace("\\", ".").replace("/", ".")
 
+        def _removeFilenamesIfPresent(*remove_filenames):
+            # pylint: intended for loop usage, pylint: disable=cell-var-from-loop
+            for remove_filename in remove_filenames:
+                if remove_filename in filenames:
+                    filenames.remove(remove_filename)
+
+        def _removeDirsIfPresent(*remove_dirs):
+            # pylint: intended for loop usage, pylint: disable=cell-var-from-loop
+            for remove_dir in remove_dirs:
+                if remove_dir in dirs:
+                    dirs.remove(remove_dir)
+
+        _removeDirsIfPresent("__pycache__")
+
+        # Ignore ".idea", ".git" and similar folders, they are not modules
+        dirs[:] = [dirname for dirname in dirs if not dirname.startswith(".")]
+
         if import_path == "":
-            if "site-packages" in dirs:
-                dirs.remove("site-packages")
-            if "dist-packages" in dirs:
-                dirs.remove("dist-packages")
-            if "vendor-packages" in dirs:
-                dirs.remove("vendor-packages")
-            if "test" in dirs:
-                dirs.remove("test")
-            if "turtledemo" in dirs:
-                dirs.remove("turtledemo")
-
-            if "ensurepip" in filenames:
-                filenames.remove("ensurepip")
-            if "ensurepip" in dirs:
-                dirs.remove("ensurepip")
-
-            if "_ios_support.py" in filenames and not isMacOS():
-                filenames.remove("_ios_support.py")
-
-            # Ignore "lib-dynload" and "lib-tk" and alike.
+            # Ignore "lib-dynload" and "lib-tk" and alike, spell-checker: ignore dynload
             dirs[:] = [
                 dirname
                 for dirname in dirs
                 if not dirname.startswith("lib-")
-                if dirname != "Tools"
                 if not dirname.startswith("plat-")
             ]
 
+            _removeDirsIfPresent(
+                "site-packages",
+                "dist-packages",
+                "vendor-packages",
+                "test",
+                "ensurepip",
+                "turtledemo",
+                "Tools",
+            )
+            _removeFilenamesIfPresent("ensurepip")
+
+            if not isMacOS():
+                _removeFilenamesIfPresent("_ios_support.py")
+
+        # Ignore tests from selected packages, spell-checker: ignore bsddb
         if import_path in (
             "tkinter",
             "Tkinter",
@@ -199,38 +201,40 @@ def scanStandardLibraryPath(stdlib_dir):
             "email",
             "bsddb",
         ):
-            if "test" in dirs:
-                dirs.remove("test")
+            _removeDirsIfPresent("test")
+
+        if import_path in ("lib2to3", "json", "distutils"):
+            _removeDirsIfPresent("tests")
 
         if import_path == "distutils.command":
             # Misbehaving and crashing while importing the world.
-            if "bdist_conda.py" in filenames:
-                filenames.remove("bdist_conda.py")
-
-        if import_path in ("lib2to3", "json", "distutils"):
-            if "tests" in dirs:
-                dirs.remove("tests")
+            _removeFilenamesIfPresent("bdist_conda.py")
 
         if import_path == "asyncio":
-            if "test_utils.py" in filenames:
-                filenames.remove("test_utils.py")
+            _removeFilenamesIfPresent("test_utils.py")
 
-        if python_version >= 0x340 and isWin32Windows():
+        if python_version >= 0x300 and isPosixWindows():
             if import_path == "multiprocessing":
-                filenames.remove("popen_fork.py")
-                filenames.remove("popen_forkserver.py")
-                filenames.remove("popen_spawn_posix.py")
+                _removeFilenamesIfPresent(
+                    "popen_fork.py", "popen_forkserver.py", "popen_spawn_posix.py"
+                )
 
-        if python_version >= 0x300 and isPosixWindows():
             if import_path == "curses":
-                filenames.remove("has_key.py")
+                _removeFilenamesIfPresent("has_key.py")
 
         if isNetBSD():
             if import_path == "xml.sax":
-                filenames.remove("expatreader.py")
+                _removeFilenamesIfPresent("expatreader.py")
+
+        _removeFilenamesIfPresent("__main__.py", "__init__.py", "antigravity.py")
+
+        if not isWin32Windows():
+            # On POSIX systems, and on POSIX Python variants on Windows, this
+            # won't work.
+            _removeFilenamesIfPresent("wintypes.py", "cp65001.py")
 
         for filename in filenames:
-            if filename.endswith(".py") and filename not in _excluded_stdlib_modules:
+            if filename.endswith(".py"):
                 module_name = filename[:-3]
 
                 if import_path == "":
@@ -238,13 +242,6 @@ def scanStandardLibraryPath(stdlib_dir):
                 else:
                     yield ModuleName(import_path + "." + module_name)
 
-        if python_version >= 0x300:
-            if "__pycache__" in dirs:
-                dirs.remove("__pycache__")
-
-        # Ignore ".idea", ".git" and similar folders, they are not modules
-        dirs[:] = [dirname for dirname in dirs if not dirname.startswith(".")]
-
         for dirname in dirs:
             if import_path == "":
                 yield ModuleName(dirname)
@@ -255,6 +252,7 @@ def scanStandardLibraryPath(stdlib_dir):
 _stdlib_no_auto_inclusion_list = (
     # Avoid this to be included, implicit usages will be rare, but it triggers
     # the Nuitka plugin "multiprocessing" that is always enabled.
+    # spell-checker: ignore _pydecimal,_posixsubprocess,pyexpat,sitecustomize
     "multiprocessing",
     "_multiprocessing",
     # Implicit usages of these will be rare, but it can have that costly extension module
@@ -369,13 +367,13 @@ _stdlib_no_auto_inclusion_list = (
     "py_compile",
     "msilib",
     "_opcode",
-    # tzdata is not always needed
+    # time zone data is not always needed
     "zoneinfo",
     # tkinter under all its names
     "Tkinter",
     "tkinter",
     "_tkinter",
-    # lib-tk from Python2
+    # lib-tk from Python2, spell-checker: ignore Tkdnd,Tkconstants
     "Tix",
     "FixTk",
     "ScrolledText",

nuitka/nodes/AsyncgenNodes.py

@@ -76,14 +76,11 @@ class ExpressionAsyncgenObjectBody(ExpressionFunctionEntryPointBase):
     def getFunctionName(self):
         return self.name
 
-    def markAsNeedsGeneratorReturnHandling(self, value):
-        self.needs_generator_return_exit = max(self.needs_generator_return_exit, value)
-
-    def needsGeneratorReturnHandling(self):
-        return self.needs_generator_return_exit == 2
+    def markAsNeedsGeneratorReturnHandling(self):
+        self.needs_generator_return_exit = True
 
     def needsGeneratorReturnExit(self):
-        return bool(self.needs_generator_return_exit)
+        return self.needs_generator_return_exit
 
     @staticmethod
     def needsCreation():

nuitka/nodes/BuiltinIteratorNodes.py

@@ -155,7 +155,7 @@ class ExpressionBuiltinIterForUnpack(ExpressionBuiltinIter1):
         # Rewrite exceptions to correct message.
         if (
             result_node.isExpressionRaiseException()
-            and result_node.subnode_exception_type.isExpressionBuiltinExceptionRef()
+            and result_node.subnode_exception_type.isExpressionBuiltinMakeException()
             and result_node.subnode_exception_type.getExceptionName() == "TypeError"
         ):
             return makeRaiseTypeErrorExceptionReplacementFromTemplateAndValue(

nuitka/nodes/BuiltinRefNodes.py

@@ -18,15 +18,10 @@ from nuitka.Builtins import (
     builtin_type_names,
 )
 from nuitka.Options import hasPythonFlagNoAsserts
-from nuitka.PythonVersions import python_version
 from nuitka.specs import BuiltinParameterSpecs
 
 from .ConstantRefNodes import makeConstantRefNode
-from .ExceptionNodes import (
-    ExpressionBuiltinMakeException,
-    ExpressionBuiltinMakeExceptionImportError,
-    ExpressionBuiltinMakeExceptionModuleNotFoundError,
-)
+from .ExceptionNodes import makeBuiltinMakeExceptionNode
 from .ExpressionBases import (
     CompileTimeConstantExpressionBase,
     ExpressionBase,
@@ -267,33 +262,22 @@ class ExpressionBuiltinExceptionRef(ExpressionBuiltinRefBase):
     def computeExpressionCall(self, call_node, call_args, call_kw, trace_collection):
         exception_name = self.getExceptionName()
 
-        def createBuiltinMakeException(args, name=None, path=None, source_ref=None):
-            if exception_name == "ImportError" and python_version >= 0x300:
-                return ExpressionBuiltinMakeExceptionImportError(
-                    args=args,
-                    name=name,
-                    path=path,
-                    source_ref=source_ref,
-                )
-            elif exception_name == "ModuleNotFoundError" and python_version >= 0x360:
-                return ExpressionBuiltinMakeExceptionModuleNotFoundError(
-                    args=args,
-                    name=name,
-                    path=path,
-                    source_ref=source_ref,
-                )
-            else:
-                # We expect to only get the star arguments for these.
-                assert name is None
-                assert path is None
-
-                return ExpressionBuiltinMakeException(
-                    exception_name=exception_name, args=args, source_ref=source_ref
-                )
+        def createBuiltinMakeExceptionNode(
+            args, name=None, path=None, obj=None, source_ref=None
+        ):
+            return makeBuiltinMakeExceptionNode(
+                exception_name=exception_name,
+                args=args,
+                name=name,
+                path=path,
+                obj=obj,
+                for_raise=False,
+                source_ref=source_ref,
+            )
 
         new_node = BuiltinParameterSpecs.extractBuiltinArgs(
             node=call_node,
-            builtin_class=createBuiltinMakeException,
+            builtin_class=createBuiltinMakeExceptionNode,
             builtin_spec=BuiltinParameterSpecs.makeBuiltinExceptionParameterSpec(
                 exception_name=exception_name
             ),