Nuitka-winsvc 2.4.8__cp311-cp311-win_amd64.whl → 2.5.1__cp311-cp311-win_amd64.whl

nuitka/containers/OrderedSets.py

@@ -1,15 +1,115 @@
 """ This module is only an abstraction of OrderedSet which is not present in
 Python at all.
 
+spell-checker: ignore orderedset,orderedsets
+
 """
 
-try:
-    # spell-checker: ignore orderedset
-    from orderedset import OrderedSet
-except ImportError:
+import os
+import subprocess
+import sys
+
+from nuitka.PythonVersions import python_version
+from nuitka.utils.InlineCopies import getDownloadCopyFolder
+
+recommended_orderedset_package_name, recommended_orderedset_module_name = (
+    ("ordered-set", "ordered_set")
+    if python_version >= 0x370
+    else ("orderedset", "orderedsets")
+)
+
+
+def _tryImportOrderedSet():
     try:
-        from ordered_set import OrderedSet
+        from orderedset import OrderedSet as result
     except ImportError:
+        try:
+            from ordered_set import OrderedSet as result
+        except ImportError:
+            return None
+
+    return result
+
+
+OrderedSet = _tryImportOrderedSet()
+
+
+def _findDownloadSitePackagesDir(download_folder):
+    for root, dirnames, _filenames in os.walk(download_folder):
+        found_candidate = None
+        for candidate in ("site-packages", "dist-packages", "vendor-packages"):
+            if candidate in dirnames:
+                # Unclear which one to use.
+                if found_candidate is not None:
+                    return
+
+                found_candidate = candidate
+
+        if found_candidate:
+            return os.path.join(root, found_candidate)
+
+
+def tryDownloadPackageName(package_name, module_name, package_version):
+    download_folder = getDownloadCopyFolder()
+
+    site_packages_folder = _findDownloadSitePackagesDir(download_folder)
+
+    if site_packages_folder is not None:
+        candidate = os.path.join(site_packages_folder, module_name)
+
+        if os.path.exists(candidate):
+            return site_packages_folder
+
+    if os.getenv("NUITKA_ASSUME_YES_FOR_DOWNLOADS") in ("1", "true", "yes"):
+        if package_version is not None:
+            package_spec = "%s==%s" % (package_name, package_version)
+        else:
+            package_spec = package_name
+
+        exit_code = subprocess.call(
+            [
+                sys.executable,
+                "-m",
+                "pip",
+                "install",
+                "--root",
+                download_folder,
+                package_spec,
+            ],
+            shell=False,
+        )
+
+        if exit_code != 0:
+            return None
+
+        if site_packages_folder is None:
+            site_packages_folder = _findDownloadSitePackagesDir(download_folder)
+
+    if site_packages_folder is not None:
+        candidate = os.path.join(site_packages_folder, module_name)
+
+        if os.path.exists(candidate):
+            return site_packages_folder
+
+
+def _tryDownloadOrderedSet():
+    return tryDownloadPackageName(
+        recommended_orderedset_package_name,
+        recommended_orderedset_module_name,
+        package_version=None,
+    )
+
+
+if OrderedSet is None:
+    downloaded_pip = _tryDownloadOrderedSet()
+
+    try:
+        sys.path.insert(0, downloaded_pip)
+        OrderedSet = _tryImportOrderedSet()
+    finally:
+        del sys.path[0]
+
+    if OrderedSet is None:
         from .OrderedSetsFallback import OrderedSet
 
 

nuitka/distutils/Build.py

@@ -23,6 +23,20 @@ class NuitkaBuildMetaBackend(setuptools.build_meta._BuildMetaBackend):
     def build_wheel(
         self, wheel_directory, config_settings=None, metadata_directory=None
     ):
+        # Allow falling back to setuptools when the `build_with_nuitka` configuration setting is set to true.
+        if config_settings:
+            build_with_nuitka = config_settings.pop("build_with_nuitka", "true").lower()
+
+            if build_with_nuitka not in ("true", "false"):
+                raise ValueError(
+                    "When passing the 'build_with_nuitka' setting, it must either be 'true' or 'false'."
+                )
+
+            if build_with_nuitka == "false":
+                return super().build_wheel(
+                    wheel_directory, config_settings, metadata_directory
+                )
+
         os.environ["NUITKA_TOML_FILE"] = os.path.join(os.getcwd(), "pyproject.toml")
 
         with suppress_known_deprecation():

nuitka/distutils/DistutilCommands.py

@@ -21,6 +21,10 @@ from nuitka.importing.Importing import (
     locateModule,
 )
 from nuitka.PythonVersions import python_version
+from nuitka.reports.CompilationReportReader import (
+    getEmbeddedDataFilenames,
+    parseCompilationReport,
+)
 from nuitka.Tracing import wheel_logger
 from nuitka.utils.Execution import check_call
 from nuitka.utils.FileOperations import deleteFile, getFileList, renameFile
@@ -31,6 +35,7 @@ def setupNuitkaDistutilsCommands(dist, keyword, value):
     # If the user project setup.py includes the key "build_with_nuitka=True" all
     # build operations (build, bdist_wheel, install etc) will run via Nuitka.
     # pylint: disable=unused-argument
+    # spell-checker: ignore cmdclass
 
     if not value:
         return
@@ -213,6 +218,9 @@ class build(distutils.command.build.build):
 
     @staticmethod
     def _parseOptionsEntry(option, value):
+        if option == "build_with_nuitka":
+            return
+
         option = "--" + option.lstrip("-")
 
         if type(value) is tuple and len(value) == 2 and value[0] == "setup.py":
@@ -231,7 +239,8 @@ class build(distutils.command.build.build):
             yield "%s=%s" % (option, value)
 
     def _build(self, build_lib):
-        # High complexity, pylint: disable=too-many-branches,too-many-locals
+        # High complexity,
+        # pylint: disable=too-many-branches,too-many-locals,too-many-statements
 
         old_dir = os.getcwd()
 
@@ -249,6 +258,8 @@ class build(distutils.command.build.build):
         # Search in the build directory preferably.
         addMainScriptDirectory(main_package_dir)
 
+        embedded_data_files = set()
+
         for is_package, module_name in self._findBuildTasks():
             # Nuitka wants the main package by filename, probably we should stop
             # needing that.
@@ -276,7 +287,12 @@ class build(distutils.command.build.build):
                 "--enable-plugin=pylint-warnings",
                 "--output-dir=%s" % output_dir,
                 "--nofollow-import-to=*.tests",
+                "--nowarn-mnemonic=compiled-package-hidden-by-package",
                 "--remove-output",
+                # Note: For when we are debugging module mode of Nuitka, not for general use.
+                # "--debug",
+                # "--trace",
+                # "--python-flag=-v"
             ]
 
             if package_name is not None:
@@ -319,12 +335,25 @@ class build(distutils.command.build.build):
                 ):
                     command.extend(self._parseOptionsEntry(option, value))
 
+            report_filename = None
+
             # Process any extra options from setuptools
             if "nuitka" in self.distribution.command_options:
                 for option, value in self.distribution.command_options[
                     "nuitka"
                 ].items():
-                    command.extend(self._parseOptionsEntry(option, value))
+                    for option in self._parseOptionsEntry(option, value):
+                        command.append(option)
+
+                        if option.startswith("--report="):
+                            report_filename = option.split("=", 1)[1]
+
+            if report_filename is None:
+                command.append("--report=compilation-report.xml")
+                report_filename = "compilation-report.xml"
+                delete_report = True
+            else:
+                delete_report = False
 
             command.append(main_filename)
 
@@ -337,17 +366,26 @@ class build(distutils.command.build.build):
                 "Finished compilation of '%s'." % module_name.asString(), style="green"
             )
 
-        self.build_lib = build_lib
+            report = parseCompilationReport(report_filename)
 
-        os.chdir(old_dir)
+            embedded_data_files.update(getEmbeddedDataFilenames(report))
+
+            if delete_report:
+                os.unlink(report_filename)
+
+        self.build_lib = build_lib
 
         # Remove Python code from build folder, that's our job now.
-        for root, _, filenames in os.walk(build_lib):
-            for filename in filenames:
-                fullpath = os.path.join(root, filename)
+        for filename in getFileList(
+            build_lib, only_suffixes=(".py", ".pyw", ".pyc", ".pyo")
+        ):
+            os.unlink(filename)
 
-                if fullpath.lower().endswith((".py", ".pyw", ".pyc", ".pyo")):
-                    os.unlink(fullpath)
+        # Remove data files from build folder, that's our job now too
+        for filename in embedded_data_files:
+            os.unlink(filename)
+
+        os.chdir(old_dir)
 
 
 # Required by distutils, used as command name, pylint: disable=invalid-name
@@ -355,7 +393,8 @@ class install(distutils.command.install.install):
     # pylint: disable=attribute-defined-outside-init
     def finalize_options(self):
         distutils.command.install.install.finalize_options(self)
-        # Ensure the purelib folder is not used
+        # Ensure the "purelib" folder is not used
+        # spell-checker: ignore purelib,platlib
         self.install_lib = self.install_platlib
 
 
@@ -376,6 +415,7 @@ class bdist_nuitka(wheel.bdist_wheel.bdist_wheel):
         # Force module to use correct platform in name
         self.root_is_pure = False
 
+    # virtual method name, spell-checker: ignore wheelfile
     def write_wheelfile(self, wheelfile_base, generator=None):
         if generator is None:
             from nuitka.Version import getNuitkaVersion

nuitka/finalizations/Finalization.py

@@ -14,9 +14,9 @@ from nuitka.tree import Operations
 from .FinalizeMarkups import FinalizeMarkups
 
 
-def prepareCodeGeneration(tree):
-    visitor = FinalizeMarkups()
-    Operations.visitTree(tree, visitor)
+def prepareCodeGeneration(module):
+    visitor = FinalizeMarkups(module)
+    Operations.visitTree(module, visitor)
 
 
 #     Part of "Nuitka", an optimizing Python compiler that is compatible and

nuitka/finalizations/FinalizeMarkups.py

@@ -23,6 +23,9 @@ from nuitka.tree.Operations import VisitorNoopMixin
 
 
 class FinalizeMarkups(VisitorNoopMixin):
+    def __init__(self, module):
+        self.module = module
+
     def onEnterNode(self, node):
         try:
             self._onEnterNode(node)
@@ -37,26 +40,17 @@ class FinalizeMarkups(VisitorNoopMixin):
         # This has many different things it deals with, so there need to be a
         # lot of branches and statements, pylint: disable=too-many-branches
 
-        # Also all self specific things have been done on the outside,
-        # pylint: disable=no-self-use
         if node.isStatementReturn() or node.isStatementGeneratorReturn():
-            search = node
-
-            in_tried_block = False
-
             # Search up to the containing function, and check for a try/finally
             # containing the "return" statement.
-            search = search.getParentReturnConsumer()
+            search = node.getParentReturnConsumer()
 
             if (
                 search.isExpressionGeneratorObjectBody()
                 or search.isExpressionCoroutineObjectBody()
                 or search.isExpressionAsyncgenObjectBody()
             ):
-                if in_tried_block:
-                    search.markAsNeedsGeneratorReturnHandling(2)
-                else:
-                    search.markAsNeedsGeneratorReturnHandling(1)
+                search.markAsNeedsGeneratorReturnHandling()
 
         if node.isExpressionFunctionCreation():
             if (
@@ -72,11 +66,10 @@ class FinalizeMarkups(VisitorNoopMixin):
             function_body = node.getFunctionBody()
             parent_module = function_body.getParentModule()
 
-            node_module = node.getParentModule()
-            if node_module is not parent_module:
+            if self.module is not parent_module:
                 function_body.markAsCrossModuleUsed()
 
-                node_module.addCrossUsedFunction(function_body)
+                self.module.addCrossUsedFunction(function_body)
 
         if node.isStatementAssignmentVariable():
             target_var = node.getVariable()
@@ -93,6 +86,9 @@ class FinalizeMarkups(VisitorNoopMixin):
                     # TODO: This might be bad.
                     assign_source.removeMarkAsInplaceSuspect()
 
+            if target_var.isModuleVariable():
+                pass
+
         if python_version < 0x300 and node.isStatementPublishException():
             node.getParentStatementsFrame().markAsFrameExceptionPreserving()
 

nuitka/freezer/DependsExe.py

@@ -3,12 +3,14 @@
 
 """ Interface to depends.exe on Windows.
 
-We use depends.exe to investigate needed DLLs of Python DLLs.
+We use "depends.exe" to investigate needed DLLs of Python DLLs.
 
 """
 
 import os
 
+# pylint: disable=I0021,import-error,redefined-builtin
+from nuitka.__past__ import WindowsError
 from nuitka.containers.OrderedSets import OrderedSet
 from nuitka.Options import assumeYesForDownloads
 from nuitka.Tracing import inclusion_logger
@@ -18,6 +20,7 @@ from nuitka.utils.FileOperations import (
     deleteFile,
     getExternalUsePath,
     getFileContentByLine,
+    getNormalizedPath,
     getWindowsLongPathName,
     isFilenameBelowPath,
     isFilesystemEncodable,
@@ -65,6 +68,7 @@ def _attemptToFindNotFoundDLL(dll_filename):
     if dll_filename in currently_loaded_dlls:
         return currently_loaded_dlls[dll_filename]
 
+    # Lets try the Windows system, spell-checker: ignore systemroot
     dll_filename = os.path.join(
         os.environ["SYSTEMROOT"],
         "SysWOW64" if getArchitecture() == "x86_64" else "System32",
@@ -132,7 +136,13 @@ def _parseDependsExeOutput2(lines):
             continue
 
         dll_filename = os.path.abspath(dll_filename)
-        dll_filename = getWindowsLongPathName(dll_filename)
+
+        # Ignore errors trying to resolve the filename. Sometimes Chinese
+        # directory paths do not resolve to long filenames.
+        try:
+            dll_filename = getWindowsLongPathName(dll_filename)
+        except WindowsError:
+            pass
 
         dll_name = os.path.basename(dll_filename)
 
@@ -152,7 +162,7 @@ def _parseDependsExeOutput2(lines):
 
         assert os.path.isfile(dll_filename), (dll_filename, line)
 
-        result.add(os.path.normcase(os.path.abspath(dll_filename)))
+        result.add(getNormalizedPath(os.path.normcase(dll_filename)))
 
     return result
 

nuitka/freezer/DllDependenciesMacOS.py

@@ -57,7 +57,12 @@ def _detectPythonRpaths():
 
 
 def detectBinaryPathDLLsMacOS(
-    original_dir, binary_filename, package_name, keep_unresolved, recursive
+    original_dir,
+    binary_filename,
+    package_name,
+    keep_unresolved,
+    recursive,
+    recursive_dlls=None,
 ):
     assert os.path.exists(binary_filename), binary_filename
 
@@ -91,13 +96,25 @@ def detectBinaryPathDLLsMacOS(
     if recursive:
         merged_result = OrderedDict(resolved_result)
 
+        # For recursive DLL detection, cycle may exist, so we keep track of what
+        # was seen so far.
+        if recursive_dlls is None:
+            recursive_dlls = set([binary_filename])
+        else:
+            recursive_dlls = set(recursive_dlls)
+            recursive_dlls.add(binary_filename)
+
         for sub_dll_filename in resolved_result:
+            if sub_dll_filename in recursive_dlls:
+                continue
+
             _, sub_result = detectBinaryPathDLLsMacOS(
                 original_dir=os.path.dirname(sub_dll_filename),
                 binary_filename=sub_dll_filename,
                 package_name=package_name,
                 keep_unresolved=True,
                 recursive=True,
+                recursive_dlls=recursive_dlls,
             )
 
             merged_result.update(sub_result)

nuitka/freezer/DllDependenciesWin32.py

@@ -43,6 +43,7 @@ def detectBinaryPathDLLsWin32(
     original_dir,
     binary_filename,
     package_name,
+    use_path,
     use_cache,
     update_cache,
 ):
@@ -55,6 +56,7 @@ def detectBinaryPathDLLsWin32(
             original_dir=original_dir,
             binary_filename=binary_filename,
             package_name=package_name,
+            use_path=use_path,
         )
 
         if use_cache:
@@ -79,7 +81,9 @@ def detectBinaryPathDLLsWin32(
     if isShowProgress():
         inclusion_logger.info("Analyzing dependencies of '%s'." % binary_filename)
 
-    scan_dirs = _getScanDirectories(package_name, original_dir)
+    scan_dirs = _getScanDirectories(
+        package_name=package_name, original_dir=original_dir, use_path=use_path
+    )
 
     result = detectDLLsWithDependencyWalker(
         binary_filename=binary_filename, source_dir=source_dir, scan_dirs=scan_dirs
@@ -91,7 +95,7 @@ def detectBinaryPathDLLsWin32(
     return result
 
 
-def _getScanDirectories(package_name, original_dir):
+def _getScanDirectories(package_name, original_dir, use_path):
     # TODO: Move PyWin32 specific stuff to yaml dll section
     # pylint: disable=too-many-branches
 
@@ -120,18 +124,23 @@ def _getScanDirectories(package_name, original_dir):
         if py_win32_dir is not None:
             scan_dirs.append(py_win32_dir)
 
-    for path_dir in os.environ["PATH"].split(";"):
-        if not os.path.isdir(path_dir):
-            continue
+    if use_path:
+        for path_dir in os.environ["PATH"].split(";"):
+            if not os.path.isdir(path_dir):
+                continue
 
-        if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
-            continue
-        if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"], "System32")):
-            continue
-        if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")):
-            continue
+            if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
+                continue
+            if areSamePaths(
+                path_dir, os.path.join(os.environ["SYSTEMROOT"], "System32")
+            ):
+                continue
+            if areSamePaths(
+                path_dir, os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")
+            ):
+                continue
 
-        scan_dirs.append(path_dir)
+            scan_dirs.append(path_dir)
 
     result = []
 
@@ -161,6 +170,7 @@ def _getCacheFilename(
     original_dir,
     binary_filename,
     package_name,
+    use_path,
 ):
     original_filename = os.path.join(original_dir, os.path.basename(binary_filename))
     original_filename = os.path.normcase(original_filename)
@@ -192,6 +202,10 @@ def _getCacheFilename(
     # Take Nuitka version into account as well, ought to catch code changes.
     hash_value.updateFromValues(version_string)
 
+    # Using PATH or not, should also be considered different.
+    if use_path:
+        hash_value.updateFromValues(os.getenv("PATH"))
+
     cache_dir = os.path.join(getCacheDir("library_dependencies"), dependency_tool)
     makePath(cache_dir)
 

nuitka/freezer/ImportDetection.py

@@ -137,6 +137,9 @@ print("\\n".join(sorted(
                     "_collections_abc.py",
                     "_collections_abc.pyc",
                 ):
+                    if python_version >= 0x3D0:
+                        detections.append((module_name, 2, "sourcefile", filename))
+
                     module_name = ModuleName("_collections_abc")
 
                 if filename.endswith(".py"):

nuitka/freezer/IncludedDataFiles.py

@@ -19,6 +19,7 @@ from nuitka.Options import (
     getShallIncludeDataFiles,
     getShallIncludeExternallyDataFilePatterns,
     getShallIncludePackageData,
+    getShallIncludeRawDirs,
     getShallNotIncludeDataFilePatterns,
     isAcceleratedMode,
     isOnefileMode,
@@ -35,6 +36,7 @@ from nuitka.utils.FileOperations import (
     getFileList,
     getFilenameExtension,
     getFileSize,
+    getNormalizedPath,
     isFilenameBelowPath,
     isLegalPath,
     isRelativePath,
@@ -98,9 +100,9 @@ class IncludedDataFile(object):
 
         self.kind = kind
         self.source_path = source_path
-        self.dest_path = os.path.normpath(dest_path)
+        self.dest_path = getNormalizedPath(dest_path)
 
-        is_legal, illegal_reason = isLegalPath(dest_path)
+        is_legal, illegal_reason = isLegalPath(self.dest_path)
         if not is_legal:
             general.sysexit(
                 "Error, cannot add data file with '%s' path, as '%s'"
@@ -166,11 +168,12 @@ def makeIncludedDataFile(source_path, dest_path, reason, tracer, tags):
     # In accelerated mode, data files can be everywhere, but they cannot
     # change place.
     if isAcceleratedMode():
-        if "package_data" not in tags and not areSamePaths(source_path, dest_path):
-            tracer.sysexit(
-                "Error, cannot change paths for data files in accelerated mode from '%s' to '%s'."
-                % (source_path, dest_path)
-            )
+        if "copy" in tags:
+            if "package_data" not in tags and not areSamePaths(source_path, dest_path):
+                tracer.sysexit(
+                    "Error, cannot change paths for data files in accelerated mode from '%s' to '%s'."
+                    % (source_path, dest_path)
+                )
     else:
         inside = True
         if not isRelativePath(dest_path):
@@ -341,6 +344,8 @@ def getIncludedDataFiles():
 
 
 def _addIncludedDataFilesFromFileOptions():
+    # Many different option variants, pylint: disable=too-many-branches
+
     for pattern, source_path, dest_path, arg in getShallIncludeDataFiles():
         filenames = resolveShellPatternToFilenames(pattern)
 
@@ -392,6 +397,24 @@ def _addIncludedDataFilesFromFileOptions():
         if count == 0:
             options_logger.warning("No data files in directory '%s.'" % source_path)
 
+    for source_path, dest_path in getShallIncludeRawDirs():
+        count = 0
+
+        for included_datafile in makeIncludedDataDirectory(
+            source_path=source_path,
+            dest_path=os.path.normpath(dest_path),
+            reason="specified raw dir '%s' on command line" % source_path,
+            tracer=options_logger,
+            tags="user",
+            raw=True,
+        ):
+            yield included_datafile
+
+            count += 1
+
+        if count == 0:
+            options_logger.warning("No files in raw directory '%s.'" % source_path)
+
 
 def addIncludedDataFilesFromFileOptions():
     """Early data files, from user options that work with file system."""
@@ -538,7 +561,7 @@ def _reportDataFiles():
 
 
 def _checkPathConflict(dest_path, standalone_entry_points):
-    assert os.path.normpath(dest_path) == dest_path
+    assert getNormalizedPath(dest_path) == dest_path
 
     while dest_path:
         for standalone_entry_point in standalone_entry_points:

nuitka/freezer/IncludedEntryPoints.py

@@ -17,6 +17,7 @@ from nuitka.Options import getShallNotIncludeDllFilePatterns, isShowInclusion
 from nuitka.Tracing import general, inclusion_logger
 from nuitka.utils.FileOperations import (
     areSamePaths,
+    getNormalizedPath,
     getReportPath,
     hasFilenameExtension,
     haveSameFileContents,
@@ -51,7 +52,7 @@ def _makeIncludedEntryPoint(
     assert type(executable) is bool, executable
 
     # Make sure outside code uses sane paths only.
-    assert source_path == os.path.normpath(source_path), source_path
+    assert source_path == getNormalizedPath(source_path), source_path
 
     # Avoid obvious mistakes, these files won't be binaries or DLL ever, right?
     assert not hasFilenameExtension(path=source_path, extensions=(".qml", ".json"))
@@ -60,7 +61,7 @@ def _makeIncludedEntryPoint(
         logger=logger,
         kind=kind,
         source_path=source_path,
-        dest_path=os.path.normpath(dest_path),
+        dest_path=getNormalizedPath(dest_path),
         module_name=module_name,
         package_name=package_name,
         executable=executable,