Nuitka-winsvc 1.8.6__cp312-cp312-win_amd64.whl → 1.9.3__cp312-cp312-win_amd64.whl

nuitka/plugins/standard/stdlib3.nuitka-package.config.yml

@@ -111,6 +111,13 @@
       change_function:
         '_main': "'(lambda: None)'"
 
+- module-name: 'enum'
+  anti-bloat:
+    - replacements_plain:
+        'def __call__(cls, value,': 'def __call__(cls, value=None,'
+        'return cls.__new__(cls, value)': 'return cls.__new__(cls, value if value is not None or cls.__module__ != "PySide6.QtCore" else 0)'
+      when: 'plugin("pyside6")'
+
 - module-name: 'formatter'
   anti-bloat:
     - description: 'remove module ability to run as a binary'
@@ -327,7 +334,7 @@
               raise\n\
             sys.exit("Nuitka: Need to use '--enable-plugin=tk-inter' option during compilation for tk-inter to work!")
           """, "        ")
-      when: 'not deployment'
+      when: 'not deployment and not plugin("tk-inter")'
   options:
     checks:
       - console: 'recommend'

nuitka/reports/CompilationReportReader.py

@@ -0,0 +1,53 @@
+#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
+#
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+#
+""" This is for use in testing, but also for user tools too."""
+
+
+from nuitka.containers.OrderedDicts import OrderedDict
+from nuitka.TreeXML import fromFile
+from nuitka.utils.ModuleNames import ModuleName
+
+
+def parseCompilationReport(filename):
+    return fromFile(filename)
+
+
+def extractModulesUsedByModule(compilation_report, module_name):
+    # Note: Avoiding usage of "xpath", to lower requirements to not need lxml.
+    for module_node in compilation_report.findall("module"):
+        if module_node.attrib["name"] != module_name:
+            continue
+
+        result = OrderedDict()
+
+        for module_usage_node in module_node.find("module_usages").findall(
+            "module_usage"
+        ):
+            entry = OrderedDict(module_usage_node.attrib)
+
+            used_module_name = ModuleName(entry["name"])
+            del entry["name"]
+            entry["line"] = int(entry["line"])
+            entry["excluded"] = bool(entry["finding"] == "excluded")
+
+            result[used_module_name] = entry
+
+        return result
+
+    # Not found, no usages, user needs to handle that.
+    return None

nuitka/reports/LicenseReport.rst.j2

@@ -22,13 +22,13 @@ This software was compiled with Nuitka and contains parts of the following softw
 
 {% for distribution in all_distributions %}
 
-{{ distribution.metadata["Name"] }}
-===================================
+{{ get_distribution_name(distribution) }}
+=========================================
 
-Version {{ distribution.metadata["Version"] }} of {{ distribution.metadata["Name"] }} was included and is
+Version {{ get_distribution_version(distribution) }} of {{ get_distribution_name(distribution) }} was included and is
 under license "{{ get_distribution_license(distribution) }}".
 
-{% if distribution_modules[distribution] != [distribution.metadata["Name"]] %}
+{% if distribution_modules[distribution] != [get_distribution_name(distribution)] %}
 The contained {{"packages are" if len(distribution_modules[distribution]) > 1 else "package is"}} {{" ,".join(quoted(distribution_modules[distribution]))}}.
 {% endif %}
 

nuitka/reports/Reports.py

@@ -33,6 +33,7 @@ from nuitka.containers.OrderedSets import OrderedSet
 from nuitka.freezer.IncludedDataFiles import getIncludedDataFiles
 from nuitka.freezer.IncludedEntryPoints import getStandaloneEntryPoints
 from nuitka.importing.Importing import getPackageSearchPath
+from nuitka.importing.Recursion import getRecursionDecisions
 from nuitka.ModuleRegistry import (
     getDoneModules,
     getModuleInclusionInfoByName,
@@ -49,7 +50,13 @@ from nuitka.plugins.Plugins import getActivePlugins
 from nuitka.PythonFlavors import getPythonFlavorName
 from nuitka.PythonVersions import getSystemPrefixPath, python_version_full_str
 from nuitka.Tracing import ReportingSystemExit, reports_logger
-from nuitka.utils.Distributions import getDistributionsFromModuleName
+from nuitka.utils.Distributions import (
+    getDistributionInstallerName,
+    getDistributionLicense,
+    getDistributionName,
+    getDistributionsFromModuleName,
+    getDistributionVersion,
+)
 from nuitka.utils.FileOperations import getReportPath, putTextFileContents
 from nuitka.utils.Jinja2 import getTemplate
 from nuitka.utils.MemoryUsage import getMemoryInfos
@@ -88,8 +95,14 @@ def _getReportInputData(aborted):
         for module in getDoneModules()
     )
 
+    module_usages = dict(
+        (module.getFullName(), tuple(module.getUsedModules()))
+        for module in getDoneModules()
+    )
+
     module_distributions = {}
     distribution_modules = {}
+
     for module in getDoneModules():
         module_distributions[module.getFullName()] = getDistributionsFromModuleName(
             module.getFullName()
@@ -100,9 +113,19 @@ def _getReportInputData(aborted):
 
             distribution_modules[_distribution].add(module.getFullName())
 
-    module_usages = dict(
-        (module.getFullName(), module.getUsedModules()) for module in getDoneModules()
-    )
+    module_distribution_usages = {}
+    for module in getDoneModules():
+        module_distribution_usages[module.getFullName()] = OrderedSet()
+
+        for _module_usage in module_usages[module.getFullName()]:
+            if _module_usage.module_name not in module_usages:
+                continue
+
+            module_distribution_usages[module.getFullName()].update(
+                dist
+                for dist in module_distributions[_module_usage.module_name]
+                if dist not in module_distributions[module.getFullName()]
+            )
 
     module_distribution_names = dict(
         (module.getFullName(), module.getUsedDistributions())
@@ -112,10 +135,45 @@ def _getReportInputData(aborted):
     all_distributions = tuple(
         sorted(
             set(sum(module_distributions.values(), ())),
-            key=lambda dist: dist.metadata["Name"],
+            key=getDistributionName,
         )
     )
 
+    module_distribution_installers = dict(
+        (
+            getDistributionName(dist),
+            getDistributionInstallerName(getDistributionName(dist)),
+        )
+        for dist in all_distributions
+    )
+
+    module_exclusions = dict((module.getFullName(), {}) for module in getDoneModules())
+
+    # TODO: The module filename, and other things can be None. Once we change to
+    # namedtuples, we need to adapt the type check.
+    def _replaceNoneWithString(value):
+        if type(value) is tuple:
+            return tuple(_replaceNoneWithString(element) for element in value)
+
+        return value if value is not None else ""
+
+    for (
+        _using_module_name,
+        _module_filename,
+        _module_name,
+        _module_kind,
+        _extra_recursion,
+    ), (_decision, _reason) in sorted(
+        getRecursionDecisions().items(), key=_replaceNoneWithString
+    ):
+        if _decision is not False:
+            continue
+
+        if _using_module_name is None:
+            continue
+
+        module_exclusions[_using_module_name][_module_name] = _reason
+
     memory_infos = getMemoryInfos()
 
     python_exe = sys.executable
@@ -188,6 +246,13 @@ def _addModulesToReport(root, report_input_data, diffable):
             ),
         )
 
+        distributions = report_input_data["module_distributions"][module_name]
+
+        if distributions:
+            module_xml_node.attrib["distribution"] = ",".join(
+                getDistributionName(dist) for dist in distributions
+            )
+
         for plugin_name, influence, detail in report_input_data[
             "module_plugin_influences"
         ][module_name]:
@@ -219,19 +284,19 @@ def _addModulesToReport(root, report_input_data, diffable):
 
             module_xml_node.append(timing_xml_node)
 
-        distributions = report_input_data["module_distributions"][module_name]
+        distributions = report_input_data["module_distribution_usages"][module_name]
 
         if distributions:
             distributions_xml_node = TreeXML.appendTreeElement(
                 module_xml_node,
-                "distributions",
+                "distribution-usages",
             )
 
             for distribution in distributions:
                 TreeXML.appendTreeElement(
                     distributions_xml_node,
-                    "distribution-used",
-                    name=distribution.metadata["Name"],
+                    "distribution-usage",
+                    name=getDistributionName(distribution),
                 )
 
         module_distribution_names = report_input_data["module_distribution_names"][
@@ -247,7 +312,7 @@ def _addModulesToReport(root, report_input_data, diffable):
             for distribution_name, found in module_distribution_names.items():
                 TreeXML.appendTreeElement(
                     module_distribution_names_xml_node,
-                    "distribution",
+                    "distribution-lookup",
                     name=distribution_name,
                     found="yes" if found else "no",
                 )
@@ -258,7 +323,7 @@ def _addModulesToReport(root, report_input_data, diffable):
         )
 
         for used_module in report_input_data["module_usages"][module_name]:
-            TreeXML.appendTreeElement(
+            module_usage_node = TreeXML.appendTreeElement(
                 used_modules_xml_node,
                 "module_usage",
                 name=used_module.module_name.asString(),
@@ -266,6 +331,16 @@ def _addModulesToReport(root, report_input_data, diffable):
                 line=str(used_module.source_ref.getLineNumber()),
             )
 
+            exclusion_reason = report_input_data["module_exclusions"][module_name].get(
+                used_module.module_name
+            )
+
+            # Include reason why a module was excluded unless it is obvious like
+            # with built-in modules.
+            if exclusion_reason is not None and used_module.module_kind != "built-in":
+                module_usage_node.attrib["finding"] = "excluded"
+                module_usage_node.attrib["exclusion_reason"] = exclusion_reason
+
 
 def _addMemoryInfosToReport(performance_xml_node, memory_infos, diffable):
     for key, value in memory_infos.items():
@@ -460,8 +535,11 @@ def writeCompilationReport(report_filename, report_input_data, diffable):
         TreeXML.appendTreeElement(
             distributions_xml_node,
             "distribution",
-            name=distribution.metadata["Name"],
-            version=distribution.metadata["Version"],
+            name=getDistributionName(distribution),
+            version=getDistributionVersion(distribution),
+            installer=report_input_data["module_distribution_installers"][
+                getDistributionName(distribution)
+            ],
         )
 
     python_xml_node = TreeXML.appendTreeElement(
@@ -474,22 +552,27 @@ def writeCompilationReport(report_filename, report_input_data, diffable):
         arch_name=report_input_data["arch_name"],
     )
 
-    search_path_xml_node = TreeXML.appendTreeElement(
-        python_xml_node,
-        "search_path",
-    )
+    search_path = getPackageSearchPath(None)
 
-    for search_path in getPackageSearchPath(None):
-        TreeXML.appendTreeElement(
-            search_path_xml_node,
-            "path",
-            value=_getCompilationReportPath(search_path),
+    if search_path is not None:
+        search_path_xml_node = TreeXML.appendTreeElement(
+            python_xml_node,
+            "search_path",
         )
 
+        for search_path in getPackageSearchPath(None):
+            TreeXML.appendTreeElement(
+                search_path_xml_node,
+                "path",
+                value=_getCompilationReportPath(search_path),
+            )
+
     _addUserDataToReport(root=root, user_data=report_input_data["user_data"])
 
     try:
-        putTextFileContents(filename=report_filename, contents=TreeXML.toString(root))
+        putTextFileContents(
+            filename=report_filename, contents=TreeXML.toString(root), encoding="utf8"
+        )
     except OSError as e:
         reports_logger.warning(
             "Compilation report write to file '%s' failed due to: %s."
@@ -514,25 +597,6 @@ def writeCompilationReportFromTemplate(
         extensions=("jinja2.ext.do",),
     )
 
-    def get_distribution_license(distribution):
-        license_name = distribution.metadata["License"]
-
-        if not license_name or license_name == "UNKNOWN":
-            for classifier in (
-                value
-                for (key, value) in distribution.metadata.items()
-                if "Classifier" in key
-            ):
-                parts = [part.strip() for part in classifier.split("::")]
-                if not parts:
-                    continue
-
-                if parts[0] == "License":
-                    license_name = parts[-1]
-                    break
-
-        return license_name
-
     def quoted(value):
         if isinstance(value, str):
             return "'%s'" % value
@@ -541,7 +605,11 @@ def writeCompilationReportFromTemplate(
 
     report_text = template.render(
         # Get the license text.
-        get_distribution_license=get_distribution_license,
+        get_distribution_license=getDistributionLicense,
+        # get the distribution_name
+        get_distribution_name=getDistributionName,
+        # get the distribution version
+        get_distribution_version=getDistributionVersion,
         # Quote a list of strings.
         quoted=quoted,
         # For checking length of lists.
@@ -563,15 +631,20 @@ def writeCompilationReportFromTemplate(
 
 
 _crash_report_filename = "nuitka-crash-report.xml"
+_crash_report_bug_message = True
 
 
 def _informAboutCrashReport():
     if _crash_report_filename is not None:
+        message = (
+            "Compilation crash report written to file '%s'." % _crash_report_filename
+        )
+
+        if _crash_report_bug_message:
+            message += " Please include it in your bug report."
+
         reports_logger.info(
-            """\
-Compilation crash report written to file '%s'. Please include it in \
-your bug report."""
-            % _crash_report_filename,
+            message,
             style="red",
         )
 
@@ -588,6 +661,15 @@ def writeCompilationReports(aborted):
     ):
         report_filename = _crash_report_filename
 
+        # Inform user about bug reporting of a bug only, if this is not some sort
+        # of reporting exit, these do not constitute definitive bugs of Nuitka but
+        # are often usage errors only.
+
+        # Using global here, as this is really a singleton
+        # pylint: disable=global-statement
+        global _crash_report_bug_message
+        _crash_report_bug_message = sys.exc_info()[0] is not ReportingSystemExit
+
         atexit.register(_informAboutCrashReport)
 
     if report_filename or template_specs:

nuitka/specs/HardImportSpecs.py

@@ -174,3 +174,9 @@ ctypes_cdll_before_38_spec = BuiltinParameterSpec(
     ),
     default_count=4,
 )
+
+builtins_open_since_3_spec = BuiltinParameterSpec(
+    "builtins.open",
+    ("file", "mode", "buffering", "encoding", "errors", "newline", "closefd", "opener"),
+    default_count=7,
+)

nuitka/tools/data_composer/DataComposer.py

@@ -46,6 +46,24 @@ from nuitka.Tracing import data_composer_logger
 from nuitka.utils.FileOperations import getFileSize, listDir, syncFileOutput
 from nuitka.utils.Json import writeJsonToFilename
 
+_max_uint64_t_value = 2**64 - 1
+
+
+def _encodeVariableLength(value):
+    """Get the variable length size encoding of a uint64_t value."""
+
+    assert 0 <= value <= _max_uint64_t_value
+
+    result = b""
+
+    while value >= 128:
+        # Need to take the last seven bits as a byte value
+        result += to_byte((value & 255) | 128)
+        value >>= 7
+
+    # Last byte or whole value small enough.
+    return result + to_byte(value)
+
 
 def scanConstFiles(build_dir):
     result = []
@@ -90,27 +108,24 @@ def _writeConstantValue(output, constant_value):
     elif constant_value is False:
         output.write(b"F")
     elif constant_type is tuple:
-        # TODO: Optimize for size of tuple to be < 256 with dedicated value
-        output.write(b"T" + struct.pack("i", len(constant_value)))
+        output.write(b"T" + _encodeVariableLength(len(constant_value)))
 
         _last_written = None
 
         for element in constant_value:
             _writeConstantValue(output, element)
     elif constant_type is list:
-        # TODO: Optimize for size of list to be < 256 with dedicated value
-        output.write(b"L" + struct.pack("i", len(constant_value)))
+        output.write(b"L" + _encodeVariableLength(len(constant_value)))
 
         _last_written = None
 
         for element in constant_value:
             _writeConstantValue(output, element)
     elif constant_type is dict:
-        # TODO: Optimize for size of dict to be < 256 with dedicated value
-        output.write(b"D" + struct.pack("i", len(constant_value)))
+        output.write(b"D" + _encodeVariableLength(len(constant_value)))
 
         # Write keys first, and values second, such that we allow for the
-        # last_writte to have an impact.
+        # last_written to have an impact.
         items = constant_value.items()
 
         _last_written = None
@@ -121,15 +136,13 @@ def _writeConstantValue(output, constant_value):
         for key, value in items:
             _writeConstantValue(output, value)
     elif constant_type is set:
-        # TODO: Optimize for size of set to be < 256 with dedicated value
-        output.write(b"S" + struct.pack("i", len(constant_value)))
+        output.write(b"S" + _encodeVariableLength(len(constant_value)))
 
         _last_written = None
         for element in constant_value:
             _writeConstantValue(output, element)
     elif constant_type is frozenset:
-        # TODO: Optimize for size of set to be < 256 with dedicated value
-        output.write(b"P" + struct.pack("i", len(constant_value)))
+        output.write(b"P" + _encodeVariableLength(len(constant_value)))
 
         _last_written = None
         for element in constant_value:
@@ -191,7 +204,7 @@ def _writeConstantValue(output, constant_value):
             output.write(b"w" + encoded)
         # Zero termination if possible.
         elif b"\0" in encoded:
-            output.write(b"v" + struct.pack("i", len(encoded)))
+            output.write(b"v" + _encodeVariableLength(len(encoded)))
             output.write(encoded)
         else:
             if str is not bytes and _isAttributeName(constant_value):
@@ -205,7 +218,7 @@ def _writeConstantValue(output, constant_value):
             output.write(b"d" + constant_value)
         # Zero termination if possible.
         elif b"\0" in constant_value:
-            output.write(b"b" + struct.pack("i", len(constant_value)))
+            output.write(b"b" + _encodeVariableLength(len(constant_value)))
             output.write(constant_value)
         else:
             if str is bytes and _isAttributeName(constant_value):
@@ -262,7 +275,7 @@ def _writeConstantValue(output, constant_value):
             output.write(struct.pack("dd", constant_value.real, constant_value.imag))
 
     elif constant_type is bytearray:
-        output.write(b"B" + struct.pack("i", len(constant_value)))
+        output.write(b"B" + _encodeVariableLength(len(constant_value)))
 
         if python_version < 0x270:
             constant_value = constant_value.decode("latin1")
@@ -296,7 +309,7 @@ def _writeConstantValue(output, constant_value):
         output.write(constant_value.__name__.encode("utf8"))
         output.write(b"\0")
     else:
-        assert False, constant_value
+        assert False, (type(constant_value), constant_value)
 
     _last_written = constant_value
 
@@ -422,8 +435,7 @@ def main():
             )
 
             if str is not bytes:
-                # Encoding needs to match generated source code output.
-                encoded_name = name.encode("latin1")
+                encoded_name = name.encode("utf8")
             else:
                 encoded_name = name