Nuitka-winsvc 1.8__cp311-cp311-win_amd64.whl → 2.4.8__cp311-cp311-win_amd64.whl

nuitka/MainControl.py

@@ -1,20 +1,6 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ This is the main actions of Nuitka.
 
 This can do all the steps to translate one module to a target language using
@@ -28,8 +14,15 @@ import os
 import sys
 
 from nuitka.build.DataComposerInterface import runDataComposer
-from nuitka.build.SconsUtils import getSconsReportValue, readSconsReport
-from nuitka.code_generation.ConstantCodes import addDistributionMetadataValue
+from nuitka.build.SconsUtils import (
+    getSconsReportValue,
+    readSconsErrorReport,
+    readSconsReport,
+)
+from nuitka.code_generation.ConstantCodes import (
+    addDistributionMetadataValue,
+    getDistributionMetadataValues,
+)
 from nuitka.freezer.IncludedDataFiles import (
     addIncludedDataFilesFromFileOptions,
     addIncludedDataFilesFromPackageOptions,
@@ -51,6 +44,7 @@ from nuitka.Options import (
     hasPythonFlagNoDocStrings,
     hasPythonFlagNoWarnings,
     hasPythonFlagUnbuffered,
+    isExperimental,
 )
 from nuitka.plugins.Plugins import Plugins
 from nuitka.PostProcessing import executePostProcessing
@@ -61,19 +55,21 @@ from nuitka.Progress import (
 )
 from nuitka.PythonFlavors import (
     isApplePython,
+    isArchPackagePython,
     isDebianPackagePython,
     isFedoraPackagePython,
     isNuitkaPython,
     isPyenvPython,
 )
 from nuitka.PythonVersions import (
+    getModuleLinkerLibs,
     getPythonABI,
     getSupportedPythonVersions,
     python_version,
     python_version_str,
 )
 from nuitka.Serialization import ConstantAccessor
-from nuitka.Tracing import general, inclusion_logger
+from nuitka.Tracing import general, inclusion_logger, pgo_logger
 from nuitka.tree import SyntaxErrors
 from nuitka.tree.ReformulationMultidist import createMultidistMainSourceCode
 from nuitka.utils import InstanceCounters
@@ -91,19 +87,21 @@ from nuitka.utils.FileOperations import (
     removeDirectory,
     resetDirectory,
 )
-from nuitka.utils.Importing import getSharedLibrarySuffix
+from nuitka.utils.Importing import (
+    getPackageDirFilename,
+    getSharedLibrarySuffix,
+)
 from nuitka.utils.MemoryUsage import reportMemoryUsage, showMemoryTrace
 from nuitka.utils.ModuleNames import ModuleName
 from nuitka.utils.ReExecute import callExecProcess, reExecuteNuitka
 from nuitka.utils.StaticLibraries import getSystemStaticLibPythonPath
-from nuitka.utils.Utils import isMacOS, isWin32Windows
+from nuitka.utils.Utils import getArchitecture, isMacOS, isWin32Windows
 from nuitka.Version import getCommercialVersion, getNuitkaVersion
 
 from . import ModuleRegistry, Options, OutputDirectories
 from .build.SconsInterface import (
     asBoolStr,
     cleanSconsDirectory,
-    getSconsDataPath,
     runScons,
     setCommonSconsOptions,
 )
@@ -123,16 +121,6 @@ from .tree.SourceHandling import writeSourceCode
 from .TreeXML import dumpTreeXMLToFile
 
 
-def _setupFromMainFilenames():
-    main_filenames = Options.getMainEntryPointFilenames()
-    for filename in main_filenames:
-        # Inform the importing layer about the main script directory, so it can use
-        # it when attempting to follow imports.
-        Importing.addMainScriptDirectory(
-            main_dir=os.path.dirname(os.path.abspath(filename))
-        )
-
-
 def _createMainModule():
     """Create a node tree.
 
@@ -142,24 +130,19 @@ def _createMainModule():
     directory paths.
 
     """
-    # Many cases to deal with, pylint: disable=too-many-branches
+    # Many cases and details to deal with, pylint: disable=too-many-branches
 
     Plugins.onBeforeCodeParsing()
 
-    main_filenames = Options.getMainEntryPointFilenames()
-
     # First, build the raw node tree from the source code.
-    if len(main_filenames) > 1:
+    if Options.isMultidistMode():
         assert not Options.shallMakeModule()
 
         main_module = buildMainModuleTree(
-            # TODO: Should not be given.
-            filename=main_filenames[0],
-            source_code=createMultidistMainSourceCode(main_filenames),
+            source_code=createMultidistMainSourceCode(),
         )
     else:
         main_module = buildMainModuleTree(
-            filename=main_filenames[0],
             source_code=None,
         )
 
@@ -174,7 +157,11 @@ def _createMainModule():
                 % distribution_name
             )
 
-        addDistributionMetadataValue(distribution_name, distribution)
+        addDistributionMetadataValue(
+            distribution_name=distribution_name,
+            distribution=distribution,
+            reason="user requested",
+        )
 
     # First remove old object files and old generated files, old binary or
     # module, and standalone mode program directory if any, they can only do
@@ -187,11 +174,19 @@ def _createMainModule():
     # Prepare the ".dist" directory, throwing away what was there before.
     if Options.isStandaloneMode():
         standalone_dir = OutputDirectories.getStandaloneDirectoryPath(bundle=False)
-        resetDirectory(path=standalone_dir, ignore_errors=True)
+        resetDirectory(
+            path=standalone_dir,
+            logger=general,
+            ignore_errors=True,
+            extra_recommendation="Stop previous binary.",
+        )
 
         if Options.shallCreateAppBundle():
             resetDirectory(
-                path=changeFilenameExtension(standalone_dir, ".app"), ignore_errors=True
+                path=changeFilenameExtension(standalone_dir, ".app"),
+                logger=general,
+                ignore_errors=True,
+                extra_recommendation=None,
             )
 
     # Delete result file, to avoid confusion with previous build and to
@@ -246,7 +241,9 @@ def _createMainModule():
             )
 
         Recursion.checkPluginSinglePath(
-            plugin_filename=module_filename, module_package=module_name.getPackageName()
+            plugin_filename=module_filename,
+            module_package=module_name.getPackageName(),
+            package_only=True,
         )
 
     # Allow plugins to add more modules based on the initial set being complete.
@@ -262,6 +259,14 @@ def _createMainModule():
     if Options.isStandaloneMode():
         checkFreezingModuleSet()
 
+    # Check if distribution meta data is included, that cannot be used.
+    for distribution_name, meta_data_value in getDistributionMetadataValues():
+        if not ModuleRegistry.hasDoneModule(meta_data_value.module_name):
+            inclusion_logger.sysexit(
+                "Error, including metadata for distribution '%s' without including related package '%s'."
+                % (distribution_name, meta_data_value.module_name)
+            )
+
     # Allow plugins to comment on final module set.
     Plugins.onModuleCompleteSet()
 
@@ -280,7 +285,7 @@ def dumpTreeXML():
     filename = Options.getXMLDumpOutputFilename()
 
     if filename is not None:
-        with openTextFile(filename, "w") as output_file:
+        with openTextFile(filename, "wb") as output_file:
             # XML output only.
             for module in ModuleRegistry.getDoneModules():
                 dumpTreeXMLToFile(tree=module.asXml(), output_file=output_file)
@@ -524,11 +529,11 @@ def _runCPgoBinary():
                 source_dir=OutputDirectories.getSourceDirectoryPath(), key="PATH"
             ),
         ):
-            _exit_code = _runPgoBinary()
+            exit_code_pgo = _runPgoBinary()
 
         pgo_data_collected = os.path.exists(msvc_pgc_filename)
     else:
-        _exit_code = _runPgoBinary()
+        exit_code_pgo = _runPgoBinary()
 
         # gcc file suffix, spell-checker: ignore gcda
         gcc_constants_pgo_filename = os.path.join(
@@ -537,12 +542,21 @@ def _runCPgoBinary():
 
         pgo_data_collected = os.path.exists(gcc_constants_pgo_filename)
 
+    if exit_code_pgo != 0:
+        pgo_logger.warning(
+            """\
+Error, the C PGO compiled program error exited. Make sure it works \
+fully before using '--pgo-c' option."""
+        )
+
     if not pgo_data_collected:
-        general.sysexit(
-            "Error, no PGO information produced, did the created binary run at all?"
+        pgo_logger.sysexit(
+            """\
+Error, no C PGO compiled program did not produce expected information, \
+did the created binary run at all?"""
         )
 
-    general.info("Successfully collected C level PGO information.", style="blue")
+    pgo_logger.info("Successfully collected C level PGO information.", style="blue")
 
 
 def _runPythonPgoBinary():
@@ -553,11 +567,14 @@ def _runPythonPgoBinary():
     pgo_filename = OutputDirectories.getPgoRunInputFilename()
 
     with withEnvironmentVarOverridden("NUITKA_PGO_OUTPUT", pgo_filename):
-        _exit_code = _runPgoBinary()
+        exit_code = _runPgoBinary()
 
     if not os.path.exists(pgo_filename):
         general.sysexit(
-            "Error, no Python PGO information produced, did the created binary run at all?"
+            """\
+Error, no Python PGO information produced, did the created binary
+run (exit code %d) as expected?"""
+            % exit_code
         )
 
     return pgo_filename
@@ -569,17 +586,15 @@ def runSconsBackend():
     # pylint: disable=too-many-branches,too-many-statements
 
     options = {
-        "result_name": OutputDirectories.getResultBasePath(onefile=False),
         "source_dir": OutputDirectories.getSourceDirectoryPath(),
         "nuitka_python": asBoolStr(isNuitkaPython()),
         "debug_mode": asBoolStr(Options.is_debug),
+        "debugger_mode": asBoolStr(Options.shallRunInDebugger()),
         "python_debug": asBoolStr(Options.shallUsePythonDebug()),
         "module_mode": asBoolStr(Options.shallMakeModule()),
         "full_compat": asBoolStr(Options.is_full_compat),
         "experimental": ",".join(Options.getExperimentalIndications()),
         "trace_mode": asBoolStr(Options.shallTraceExecution()),
-        "python_version": python_version_str,
-        "nuitka_src": getSconsDataPath(),
         "file_reference_mode": Options.getFileReferenceMode(),
         "module_count": "%d" % len(ModuleRegistry.getDoneModules()),
     }
@@ -602,13 +617,12 @@ def runSconsBackend():
 
     if isDebianPackagePython():
         options["debian_python"] = asBoolStr(True)
-
     if isFedoraPackagePython():
         options["fedora_python"] = asBoolStr(True)
-
+    if isArchPackagePython():
+        options["arch_python"] = asBoolStr(True)
     if isApplePython():
         options["apple_python"] = asBoolStr(True)
-
     if isPyenvPython():
         options["pyenv_python"] = asBoolStr(True)
 
@@ -621,12 +635,20 @@ def runSconsBackend():
         if Options.isOnefileTempDirMode():
             options["onefile_temp_mode"] = asBoolStr(True)
 
+    # TODO: Some things are going to hate that, we might need to bundle
+    # for accelerated mode still.
+    if Options.shallCreateAppBundle():
+        options["macos_bundle_mode"] = asBoolStr(True)
+
     if Options.getForcedStdoutPath():
         options["forced_stdout_path"] = Options.getForcedStdoutPath()
 
     if Options.getForcedStderrPath():
         options["forced_stderr_path"] = Options.getForcedStderrPath()
 
+    if Options.isProfile():
+        options["profile_mode"] = asBoolStr(True)
+
     if Options.shallTreatUninstalledPython():
         options["uninstalled_python"] = asBoolStr(True)
 
@@ -635,9 +657,6 @@ def runSconsBackend():
             len(ModuleRegistry.getUncompiledTechnicalModules())
         )
 
-    if Options.isProfile():
-        options["profile_mode"] = asBoolStr(True)
-
     if hasPythonFlagNoWarnings():
         options["no_python_warnings"] = asBoolStr(True)
 
@@ -665,7 +684,7 @@ def runSconsBackend():
     if sys.flags.bytes_warning:
         options["python_sysflag_bytes_warning"] = asBoolStr(True)
 
-    if int(os.environ.get("NUITKA_NOSITE_FLAG", Options.hasPythonFlagNoSite())):
+    if int(os.getenv("NUITKA_NOSITE_FLAG", Options.hasPythonFlagNoSite())):
         options["python_sysflag_no_site"] = asBoolStr(True)
 
     if Options.hasPythonFlagTraceImports():
@@ -691,14 +710,20 @@ def runSconsBackend():
         options["abiflags"] = abiflags
 
     if Options.shallMakeModule():
-        options["module_suffix"] = getSharedLibrarySuffix(preferred=True)
+        options["result_exe"] = OutputDirectories.getResultBasePath(
+            onefile=False
+        ) + getSharedLibrarySuffix(preferred=True)
+
+    link_module_libs = getModuleLinkerLibs()
+    if link_module_libs:
+        options["link_module_libs"] = ",".join(link_module_libs)
 
     env_values = setCommonSconsOptions(options)
 
     # Allow plugins to build definitions.
     env_values.update(Plugins.getBuildDefinitions())
 
-    if Options.shallCreatePgoInput():
+    if Options.shallCreatePythonPgoInput():
         options["pgo_mode"] = "python"
 
         result = runScons(
@@ -717,11 +742,11 @@ def runSconsBackend():
         return True, options
 
         # Need to restart compilation from scratch here.
-    if Options.isPgoMode():
+    if Options.isCPgoMode():
         # For C level PGO, we have a 2 pass system. TODO: Make it more global for onefile
         # and standalone mode proper support, which might need data files to be
         # there, which currently are not yet there, so it won't run.
-        if Options.isPgoMode():
+        if Options.isCPgoMode():
             options["pgo_mode"] = "generate"
 
             result = runScons(
@@ -754,12 +779,7 @@ def runSconsBackend():
     return result
 
 
-def callExecPython(args, clean_path, add_path):
-    old_python_path = os.environ.get("PYTHONPATH")
-
-    if clean_path and old_python_path is not None:
-        os.environ["PYTHONPATH"] = ""
-
+def callExecPython(args, add_path, uac):
     if add_path:
         if "PYTHONPATH" in os.environ:
             os.environ["PYTHONPATH"] += ":" + Options.getOutputDir()
@@ -769,20 +789,24 @@ def callExecPython(args, clean_path, add_path):
     # Add the main arguments, previous separated.
     args += Options.getPositionalArgs()[1:] + Options.getMainArgs()
 
-    callExecProcess(args)
+    callExecProcess(args, uac=uac)
 
 
-def executeMain(binary_filename, clean_path):
+def _executeMain(binary_filename):
     # Wrap in debugger, unless the CMD file contains that call already.
     if Options.shallRunInDebugger() and not Options.shallCreateCmdFileForExecution():
         args = wrapCommandForDebuggerForExec(binary_filename)
     else:
         args = (binary_filename, binary_filename)
 
-    callExecPython(clean_path=clean_path, add_path=False, args=args)
+    callExecPython(
+        args=args,
+        add_path=False,
+        uac=isWin32Windows() and Options.shallAskForWindowsAdminRights(),
+    )
 
 
-def executeModule(tree, clean_path):
+def _executeModule(tree):
     """Execute the extension module just created."""
 
     if python_version < 0x340:
@@ -798,6 +822,15 @@ assert os.path.normcase(os.path.abspath(os.path.normpath(\
 importlib.util.find_spec('%(module_name)s').origin))) == %(expected_filename)r,\
 'Error, cannot launch extension module %(module_name)s, original package is in the way.'"""
 
+    output_dir = os.path.normpath(Options.getOutputDir())
+    if output_dir != ".":
+        python_command_template = (
+            """\
+import sys; sys.path.insert(0, %(output_dir)r)
+"""
+            + python_command_template
+        )
+
     python_command_template += ";__import__('%(module_name)s')"
 
     python_command = python_command_template % {
@@ -807,6 +840,7 @@ importlib.util.find_spec('%(module_name)s').origin))) == %(expected_filename)r,\
                 os.path.normpath(OutputDirectories.getResultFullpath(onefile=False))
             )
         ),
+        "output_dir": output_dir,
     }
 
     if Options.shallRunInDebugger():
@@ -814,7 +848,7 @@ importlib.util.find_spec('%(module_name)s').origin))) == %(expected_filename)r,\
     else:
         args = (sys.executable, "python", "-c", python_command)
 
-    callExecPython(clean_path=clean_path, add_path=True, args=args)
+    callExecPython(args=args, add_path=True, uac=False)
 
 
 def compileTree():
@@ -827,9 +861,11 @@ def compileTree():
 
         reportMemoryUsage(
             "before_c_code_generation",
-            "Total memory usage before generating C code:"
-            if Options.isShowProgress() or Options.isShowMemory()
-            else None,
+            (
+                "Total memory usage before generating C code:"
+                if Options.isShowProgress() or Options.isShowMemory()
+                else None
+            ),
         )
 
         # Now build the target language code for the whole tree.
@@ -855,9 +891,11 @@ def compileTree():
 
     reportMemoryUsage(
         "before_running_scons",
-        "Total memory usage before running scons"
-        if Options.isShowProgress() or Options.isShowMemory()
-        else None,
+        (
+            "Total memory usage before running scons"
+            if Options.isShowProgress() or Options.isShowMemory()
+            else None
+        ),
     )
 
     if Options.isShowMemory():
@@ -878,9 +916,9 @@ def compileTree():
     general.info("Running C compilation via Scons.")
 
     # Run the Scons to build things.
-    result, options = runSconsBackend()
+    result, scons_options = runSconsBackend()
 
-    return result, options
+    return result, scons_options
 
 
 def handleSyntaxError(e):
@@ -938,12 +976,16 @@ def _main():
 
     reportMemoryUsage(
         "after_launch",
-        "Total memory usage before processing:"
-        if Options.isShowProgress() or Options.isShowMemory()
-        else None,
+        (
+            "Total memory usage before processing:"
+            if Options.isShowProgress() or Options.isShowMemory()
+            else None
+        ),
     )
 
-    _setupFromMainFilenames()
+    # Initialize the importing layer from options, main filenames, debugging
+    # options, etc.
+    Importing.setupImportingFromOptions()
 
     addIncludedDataFilesFromFileOptions()
     addIncludedDataFilesFromPackageOptions()
@@ -959,14 +1001,18 @@ def _main():
     dumpTreeXML()
 
     # Make the actual compilation.
-    result, options = compileTree()
+    result, scons_options = compileTree()
 
     # Exit if compilation failed.
     if not result:
-        general.sysexit(exit_code=1, reporting=True)
+        general.sysexit(
+            message="Failed unexpectedly in Scons C backend compilation.",
+            mnemonic="scons-backend-failure",
+            reporting=True,
+        )
 
     # Relaunch in case of Python PGO input to be produced.
-    if Options.shallCreatePgoInput():
+    if Options.shallCreatePythonPgoInput():
         # Will not return.
         pgo_filename = OutputDirectories.getPgoRunInputFilename()
         general.info(
@@ -983,10 +1029,8 @@ def _main():
 
     executePostProcessing()
 
-    copyDataFiles()
-
     if Options.isStandaloneMode():
-        binary_filename = options["result_exe"]
+        binary_filename = scons_options["result_exe"]
 
         setMainEntryPoint(binary_filename)
 
@@ -1005,6 +1049,9 @@ def _main():
             standalone_entry_points=getStandaloneEntryPoints(),
         )
 
+    copyDataFiles(standalone_entry_points=getStandaloneEntryPoints())
+
+    if Options.isStandaloneMode():
         Plugins.onStandaloneDistributionFinished(dist_dir)
 
         if Options.isOnefileMode():
@@ -1013,7 +1060,12 @@ def _main():
             if Options.isRemoveBuildDir():
                 general.info("Removing dist folder '%s'." % dist_dir)
 
-                removeDirectory(path=dist_dir, ignore_errors=False)
+                removeDirectory(
+                    path=dist_dir,
+                    logger=general,
+                    ignore_errors=False,
+                    extra_recommendation=None,
+                )
             else:
                 general.info(
                     "Keeping dist folder '%s' for inspection, no need to use it."
@@ -1028,7 +1080,14 @@ def _main():
 
         # Make sure the scons report is cached before deleting it.
         readSconsReport(source_dir)
-        removeDirectory(path=source_dir, ignore_errors=False)
+        readSconsErrorReport(source_dir)
+
+        removeDirectory(
+            path=source_dir,
+            logger=general,
+            ignore_errors=False,
+            extra_recommendation=None,
+        )
         assert not os.path.exists(source_dir)
     else:
         general.info("Keeping build directory '%s'." % source_dir)
@@ -1040,17 +1099,26 @@ def _main():
     if Options.isStandaloneMode() and isMacOS():
         general.info(
             "Created binary that runs on macOS %s (%s) or higher."
-            % (options["macos_min_version"], options["macos_target_arch"])
+            % (scons_options["macos_min_version"], scons_options["macos_target_arch"])
         )
 
+        if scons_options["macos_target_arch"] != getArchitecture():
+            general.warning(
+                "It will only work as well as 'arch -%s %s %s' does."
+                % (
+                    scons_options["macos_target_arch"],
+                    sys.executable,
+                    Options.getMainEntryPointFilenames()[0],
+                ),
+                mnemonic="macos-cross-compile",
+            )
+
     Plugins.onFinalResult(final_filename)
 
     if Options.shallMakeModule():
         base_path = OutputDirectories.getResultBasePath(onefile=False)
 
-        if os.path.isdir(base_path) and os.path.isfile(
-            os.path.join(base_path, "__init__.py")
-        ):
+        if os.path.isdir(base_path) and getPackageDirFilename(base_path):
             general.warning(
                 """\
 The compilation result is hidden by package directory '%s'. Importing will \
@@ -1068,18 +1136,12 @@ not use compiled code while it exists."""
 
     # Execute the module immediately if option was given.
     if Options.shallExecuteImmediately():
-        general.info("Launching '%s'" % run_filename)
+        general.info("Launching '%s'." % run_filename)
 
         if Options.shallMakeModule():
-            executeModule(
-                tree=main_module,
-                clean_path=Options.shallClearPythonPathEnvironment(),
-            )
+            _executeModule(tree=main_module)
         else:
-            executeMain(
-                binary_filename=run_filename,
-                clean_path=Options.shallClearPythonPathEnvironment(),
-            )
+            _executeMain(run_filename)
     else:
         if run_filename != final_filename:
             general.info(
@@ -1094,7 +1156,33 @@ def main():
     except BaseException:
         try:
             writeCompilationReports(aborted=True)
+        except KeyboardInterrupt:
+            general.warning("""Report writing was prevented by user interrupt.""")
         except BaseException as e:  # Catch all the things, pylint: disable=broad-except
-            general.warning("Report writing was prevented by exception %s" % e)
+            general.warning(
+                """\
+Report writing was prevented by exception %r, use option \
+'--experimental=debug-report-traceback' for full traceback."""
+                % e
+            )
+
+            if isExperimental("debug-report-traceback"):
+                raise
 
         raise
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.