Nuitka-winsvc 2.7.7__cp313-cp313-win_amd64.whl

nuitka/nodes/shapes/__init__.py

@@ -0,0 +1,19 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" Dummy file to make this directory a package. """
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/optimizations/BytecodeDemotion.py

@@ -0,0 +1,105 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" Demotion of compiled modules to bytecode modules.
+
+"""
+
+import marshal
+
+from nuitka.BytecodeCaching import writeImportedModulesNamesToCache
+from nuitka.Bytecodes import compileSourceToBytecode
+from nuitka.freezer.ImportDetection import detectEarlyImports
+from nuitka.importing.ImportCache import (
+    isImportedModuleByName,
+    replaceImportedModule,
+)
+from nuitka.ModuleRegistry import replaceRootModule
+from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
+from nuitka.Options import isShowProgress, isStandaloneMode
+from nuitka.plugins.Plugins import (
+    Plugins,
+    isTriggerModule,
+    replaceTriggerModule,
+)
+from nuitka.Tracing import inclusion_logger
+from nuitka.utils.FileOperations import getNormalizedPath
+
+
+def demoteSourceCodeToBytecode(module_name, source_code, filename):
+    if isStandaloneMode():
+        filename = module_name.asPath() + ".py"
+
+    bytecode = compileSourceToBytecode(source_code, filename)
+
+    bytecode = Plugins.onFrozenModuleBytecode(
+        module_name=module_name, is_package=False, bytecode=bytecode
+    )
+
+    return marshal.dumps(bytecode)
+
+
+def demoteCompiledModuleToBytecode(module):
+    """Demote a compiled module to uncompiled (bytecode)."""
+
+    full_name = module.getFullName()
+    filename = module.getCompileTimeFilename()
+
+    if isShowProgress():
+        inclusion_logger.info(
+            "Demoting module '%s' to bytecode from '%s'."
+            % (full_name.asString(), filename)
+        )
+
+    source_code = module.getSourceCode()
+
+    bytecode = demoteSourceCodeToBytecode(
+        module_name=full_name, source_code=source_code, filename=filename
+    )
+
+    uncompiled_module = makeUncompiledPythonModule(
+        module_name=full_name,
+        reason=module.reason,
+        filename=getNormalizedPath(filename),
+        bytecode=bytecode,
+        is_package=module.isCompiledPythonPackage(),
+        technical=full_name in detectEarlyImports(),
+    )
+
+    used_modules = module.getUsedModules()
+    uncompiled_module.setUsedModules(used_modules)
+
+    distribution_names = module.getUsedDistributions()
+    uncompiled_module.setUsedDistributions(distribution_names)
+
+    module.finalize()
+
+    if isImportedModuleByName(full_name):
+        replaceImportedModule(old=module, new=uncompiled_module)
+    replaceRootModule(old=module, new=uncompiled_module)
+
+    if isTriggerModule(module):
+        replaceTriggerModule(old=module, new=uncompiled_module)
+
+    writeImportedModulesNamesToCache(
+        module_name=full_name,
+        source_code=source_code,
+        used_modules=used_modules,
+        distribution_names=distribution_names,
+    )
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/optimizations/FunctionInlining.py

@@ -0,0 +1,110 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" In-lining of functions.
+
+Done by assigning the argument values to variables, and producing an outline
+from the in-lined function.
+"""
+
+from nuitka.nodes.OutlineNodes import ExpressionOutlineBody
+from nuitka.nodes.VariableAssignNodes import makeStatementAssignmentVariable
+from nuitka.tree.Operations import VisitorNoopMixin, visitTree
+from nuitka.tree.ReformulationTryFinallyStatements import (
+    makeTryFinallyReleaseStatement,
+)
+from nuitka.tree.TreeHelpers import makeStatementsSequence
+
+
+class VariableScopeUpdater(VisitorNoopMixin):
+    def __init__(self, locals_scope, variable_translation):
+        self.locals_scope = locals_scope
+        self.variable_translation = variable_translation
+
+    def onEnterNode(self, node):
+        if hasattr(node, "variable"):
+            if node.variable in self.variable_translation:
+                node.variable = self.variable_translation[node.variable]
+
+        if hasattr(node, "locals_scope"):
+            node.locals_scope = self.locals_scope
+
+
+def updateLocalsScope(provider, locals_scope, variable_translation):
+    visitor = VariableScopeUpdater(
+        locals_scope=locals_scope, variable_translation=variable_translation
+    )
+
+    visitTree(provider, visitor)
+
+
+def convertFunctionCallToOutline(provider, function_body, values, call_source_ref):
+    # This has got to have pretty man details
+    function_source_ref = function_body.getSourceReference()
+
+    outline_body = ExpressionOutlineBody(
+        provider=provider, name="inline", source_ref=function_source_ref
+    )
+
+    # Make a clone first, so we do not harm other references.
+    clone = function_body.subnode_body.makeClone()
+
+    locals_scope_clone, variable_translation = function_body.locals_scope.makeClone(
+        clone
+    )
+
+    # TODO: Lets update all at once maybe, it would take less visits.
+    updateLocalsScope(
+        clone,
+        locals_scope=locals_scope_clone,
+        variable_translation=variable_translation,
+    )
+
+    argument_names = function_body.getParameters().getParameterNames()
+    assert len(argument_names) == len(values), (argument_names, values)
+
+    statements = []
+
+    for argument_name, value in zip(argument_names, values):
+        statements.append(
+            makeStatementAssignmentVariable(
+                variable=variable_translation[argument_name],
+                source=value,
+                source_ref=call_source_ref,
+            )
+        )
+
+    body = makeStatementsSequence(
+        statements=(statements, clone), allow_none=False, source_ref=function_source_ref
+    )
+
+    auto_releases = function_body.getFunctionVariablesWithAutoReleases()
+
+    # TODO: Not possible to auto release with outline bodies too?
+    if auto_releases:
+        body = makeTryFinallyReleaseStatement(
+            provider=outline_body,
+            tried=body,
+            variables=auto_releases,
+            source_ref=function_source_ref,
+        )
+
+    outline_body.setChildBody(body)
+
+    return outline_body
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/optimizations/Graphs.py

@@ -0,0 +1,70 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" Graph optimization states.
+
+These are not the graphs you might be thinking of. This is for rending the
+progress of optimization into images.
+"""
+
+from nuitka import Options
+from nuitka.ModuleRegistry import getDoneModules
+from nuitka.Tracing import general
+
+graph = None
+computation_counters = {}
+
+progressive = False
+
+
+def _addModuleGraph(module, desc):
+    module_graph = module.asGraph(graph, desc)
+
+    return module_graph
+
+
+def onModuleOptimizationStep(module):
+    # Update the graph if active.
+    if graph is not None:
+        computation_counters[module] = computation_counters.get(module, 0) + 1
+
+        if progressive:
+            _addModuleGraph(module, computation_counters[module])
+
+
+def startGraph():
+    # We maintain this globally to make it accessible, pylint: disable=global-statement
+    global graph
+
+    if Options.shallCreateGraph():
+        try:
+            from pygraphviz import AGraph  # pylint: disable=I0021,import-error
+
+            graph = AGraph(name="Optimization", directed=True)
+            graph.layout()
+        except ImportError:
+            general.sysexit("Cannot import pygraphviz module, no graphing capability.")
+
+
+def endGraph(output_filename):
+    if graph is not None:
+        for module in getDoneModules():
+            _addModuleGraph(module, "final")
+
+        graph.draw(output_filename + ".dot", prog="dot")
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/optimizations/Optimization.py

@@ -0,0 +1,363 @@
+#     Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" Control the flow of optimizations applied to node tree.
+
+Applies abstract execution on all so far known modules until no more
+optimization is possible. Every successful optimization to anything might
+make others possible.
+"""
+
+import inspect
+
+from nuitka import ModuleRegistry, Options, Variables
+from nuitka.importing.Importing import addExtraSysPaths
+from nuitka.importing.Recursion import considerUsedModules
+from nuitka.plugins.Plugins import Plugins
+from nuitka.Progress import (
+    closeProgressBar,
+    reportProgressBar,
+    setupProgressBar,
+)
+from nuitka.Tracing import general, optimization_logger, progress_logger
+from nuitka.utils.MemoryUsage import MemoryWatch, reportMemoryUsage
+from nuitka.utils.Timing import TimerReport
+
+from . import Graphs
+from .BytecodeDemotion import demoteCompiledModuleToBytecode
+from .Tags import TagSet
+from .TraceCollections import fetchMergeCounts, withChangeIndicationsTo
+
+tag_set = None
+
+
+def signalChange(tags, source_ref, message):
+    """Indicate a change to the optimization framework."""
+    if message is not None:
+        # Try hard to not call a delayed evaluation of node descriptions.
+
+        if Options.is_verbose:
+            optimization_logger.info(
+                "{source_ref} : {tags} : {message}".format(
+                    source_ref=source_ref.getAsString(),
+                    tags=tags,
+                    message=message() if inspect.isfunction(message) else message,
+                )
+            )
+
+        # assert pass_count < 2
+
+    tag_set.onSignal(tags)
+
+
+def optimizeCompiledPythonModule(module):
+    optimization_logger.info_if_file(
+        "Doing module local optimizations for '{module_name}'.".format(
+            module_name=module.getFullName()
+        ),
+        other_logger=progress_logger,
+    )
+
+    touched = False
+
+    if Options.isShowProgress() and Options.isShowMemory():
+        memory_watch = MemoryWatch()
+
+    # Temporary workaround, since we do some optimization based on the last pass results
+    # that are then not yet fully seen in the traces yet until another time around, we
+    # allow to continue the loop even without changes one more time.
+    unchanged_count = 0
+
+    # Count the micro passes, so we can see how often we looped
+    micro_pass = 0
+
+    while True:
+        micro_pass += 1
+
+        tag_set.clear()
+
+        try:
+            # print("Compute module")
+            with withChangeIndicationsTo(signalChange):
+                scopes_were_incomplete = module.computeModule()
+        except SystemExit:
+            raise
+        except BaseException:
+            general.info("Interrupted while working on '%s'." % module)
+            raise
+
+        if scopes_were_incomplete:
+            tag_set.add("var_usage")
+
+        Graphs.onModuleOptimizationStep(module)
+
+        # Search for local change tags.
+        if not tag_set:
+            unchanged_count += 1
+
+            if unchanged_count == 1 and pass_count == 1:
+                optimization_logger.info_if_file(
+                    "Not changed, but retrying one more time.",
+                    other_logger=progress_logger,
+                )
+                continue
+
+            optimization_logger.info_if_file(
+                "Finished with the module.", other_logger=progress_logger
+            )
+            break
+
+        unchanged_count = 0
+
+        optimization_logger.info_if_file(
+            "Not finished with the module due to following change kinds: %s"
+            % ",".join(sorted(tag_set)),
+            other_logger=progress_logger,
+        )
+
+        # Otherwise we did stuff, so note that for return value.
+        touched = True
+
+    if Options.isShowProgress() and Options.isShowMemory():
+        memory_watch.finish(
+            "Memory usage changed during optimization of '%s'" % (module.getFullName())
+        )
+
+    considerUsedModules(module=module, pass_count=pass_count)
+
+    return touched, micro_pass
+
+
+def optimizeUncompiledPythonModule(module):
+    full_name = module.getFullName()
+    progress_logger.info(
+        "Doing module dependency considerations for '{module_name}':".format(
+            module_name=full_name
+        )
+    )
+
+    # Pick up parent package if any.
+    module.attemptRecursion()
+
+    considerUsedModules(module=module, pass_count=pass_count)
+
+    Plugins.considerImplicitImports(module=module)
+
+
+def optimizeExtensionModule(module):
+    # Pick up parent package if any.
+    module.attemptRecursion()
+
+    Plugins.considerImplicitImports(module=module)
+
+
+def optimizeModule(module):
+    # The tag set is global, so it can track changes without context.
+    # pylint: disable=global-statement
+    global tag_set
+    tag_set = TagSet()
+
+    addExtraSysPaths(Plugins.getModuleSysPathAdditions(module.getFullName()))
+
+    if module.isPythonExtensionModule():
+        optimizeExtensionModule(module)
+        return False, 0
+    elif module.isCompiledPythonModule():
+        return optimizeCompiledPythonModule(module)
+    else:
+        optimizeUncompiledPythonModule(module)
+        return False, 0
+
+
+pass_count = 0
+last_total = 0
+
+
+def _restartProgress():
+    global pass_count  # Singleton, pylint: disable=global-statement
+
+    closeProgressBar()
+    pass_count += 1
+
+    optimization_logger.info_if_file(
+        "PASS %d:" % pass_count, other_logger=progress_logger
+    )
+
+    if not Options.is_verbose or optimization_logger.isFileOutput():
+        setupProgressBar(
+            stage="PASS %d" % pass_count,
+            unit="module",
+            total=ModuleRegistry.getRemainingModulesCount()
+            + ModuleRegistry.getDoneModulesCount(),
+            min_total=last_total,
+        )
+
+
+def _traceProgressModuleStart(current_module):
+    optimization_logger.info_if_file(
+        """\
+Optimizing module '{module_name}', {remaining:d} more modules to go \
+after that.""".format(
+            module_name=current_module.getFullName(),
+            remaining=ModuleRegistry.getRemainingModulesCount(),
+        ),
+        other_logger=progress_logger,
+    )
+
+    reportProgressBar(
+        item=current_module.getFullName(),
+        total=ModuleRegistry.getRemainingModulesCount()
+        + ModuleRegistry.getDoneModulesCount(),
+        update=False,
+    )
+
+    if Options.isShowProgress() and Options.isShowMemory():
+        reportMemoryUsage(
+            "optimization/%d/%s" % (pass_count, current_module.getFullName()),
+            (
+                (
+                    "Total memory usage before optimizing module '%s'"
+                    % current_module.getFullName()
+                )
+                if Options.isShowProgress() or Options.isShowMemory()
+                else None
+            ),
+        )
+
+
+def _traceProgressModuleEnd(current_module):
+    reportProgressBar(
+        item=current_module.getFullName(),
+        total=ModuleRegistry.getRemainingModulesCount()
+        + ModuleRegistry.getDoneModulesCount(),
+        update=True,
+    )
+
+
+def _endProgress():
+    global last_total  # Singleton, pylint: disable=global-statement
+    last_total = closeProgressBar()
+
+
+def restoreFromXML(text):
+    from nuitka.nodes.NodeBases import fromXML
+    from nuitka.TreeXML import fromString
+
+    xml = fromString(text)
+
+    module = fromXML(provider=None, xml=xml)
+
+    return module
+
+
+def makeOptimizationPass():
+    """Make a single pass for optimization, indication potential completion."""
+
+    finished = True
+
+    ModuleRegistry.startTraversal()
+
+    _restartProgress()
+
+    main_module = None
+    stdlib_phase_done = False
+
+    while True:
+        current_module = ModuleRegistry.nextModule()
+
+        if current_module is None:
+            if main_module is not None and pass_count == 1:
+                considerUsedModules(module=main_module, pass_count=-1)
+
+                stdlib_phase_done = True
+                main_module = None
+                continue
+
+            break
+
+        if current_module.isMainModule() and not stdlib_phase_done:
+            main_module = current_module
+
+        _traceProgressModuleStart(current_module)
+
+        module_name = current_module.getFullName()
+
+        with TimerReport(
+            message="Optimizing %s" % module_name, decider=False
+        ) as module_timer:
+            changed, micro_passes = optimizeModule(current_module)
+
+        ModuleRegistry.addModuleOptimizationTimeInformation(
+            module_name=module_name,
+            pass_number=pass_count,
+            time_used=module_timer.getDelta(),
+            micro_passes=micro_passes,
+            merge_counts=fetchMergeCounts(),
+        )
+
+        _traceProgressModuleEnd(current_module)
+
+        if changed:
+            finished = False
+
+    # Unregister collection traces from now unused code, dropping the trace
+    # collections of functions no longer used. This must be done after global
+    # optimization due to cross module usages.
+    for current_module in ModuleRegistry.getDoneModules():
+        if current_module.isCompiledPythonModule():
+            for unused_function in current_module.getUnusedFunctions():
+                Variables.updateVariablesFromCollection(
+                    old_collection=unused_function.trace_collection,
+                    new_collection=None,
+                    source_ref=unused_function.getSourceReference(),
+                )
+
+                unused_function.trace_collection = None
+                unused_function.finalize()
+
+            current_module.subnode_functions = tuple(
+                function
+                for function in current_module.subnode_functions
+                if function in current_module.getUsedFunctions()
+            )
+
+    _endProgress()
+
+    return finished
+
+
+def optimizeModules(output_filename):
+    Graphs.startGraph()
+
+    finished = makeOptimizationPass()
+
+    # Demote compiled modules to bytecode, now that imports had a chance to be resolved, and
+    # dependencies were handled.
+    for module in ModuleRegistry.getDoneModules():
+        if (
+            module.isCompiledPythonModule()
+            and module.getCompilationMode() == "bytecode"
+        ):
+            demoteCompiledModuleToBytecode(module)
+
+    # Second, "endless" pass.
+    while not finished:
+        finished = makeOptimizationPass()
+
+    Graphs.endGraph(output_filename)
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.