Nuitka-winsvc 1.8.6__cp312-cp312-win_amd64.whl → 1.9.3__cp312-cp312-win_amd64.whl

nuitka/tools/onefile_compressor/OnefileCompressor.py

@@ -23,7 +23,6 @@ import os
 import shutil
 import struct
 import sys
-import time
 from contextlib import contextmanager
 
 from nuitka.__past__ import to_byte
@@ -34,14 +33,19 @@ from nuitka.Progress import (
     setupProgressBar,
 )
 from nuitka.Tracing import onefile_logger
-from nuitka.utils.FileOperations import getFileList
-from nuitka.utils.Hashing import HashCRC32
+from nuitka.utils.AppDirs import getCacheDir
+from nuitka.utils.FileOperations import getFileList, getFileSize, makePath
+from nuitka.utils.Hashing import Hash, HashCRC32
 from nuitka.utils.Utils import (
     decoratorRetries,
-    isPosixWindows,
     isWin32OrPosixWindows,
     isWin32Windows,
 )
+from nuitka.Version import version_string
+
+
+def getCompressorLevel(low_memory):
+    return 3 if low_memory else 22
 
 
 def getCompressorFunction(expect_compression, low_memory):
@@ -50,7 +54,7 @@ def getCompressorFunction(expect_compression, low_memory):
     if expect_compression:
         from zstandard import ZstdCompressor  # pylint: disable=I0021,import-error
 
-        compressor_context = ZstdCompressor(level=3 if low_memory else 22)
+        compressor_context = ZstdCompressor(level=getCompressorLevel(low_memory))
 
         @contextmanager
         def useCompressedFile(output_file):
@@ -71,138 +75,166 @@ def getCompressorFunction(expect_compression, low_memory):
         return b"X", useSameFile
 
 
-@contextmanager
-def _openBinaryFileForAppending(onefile_output_filename):
-    max_attempts = 5
+def _attachOnefilePayloadFile(
+    output_file,
+    is_archive,
+    is_compressing,
+    use_compression_cache,
+    low_memory,
+    file_compressor,
+    filename_full,
+    count,
+    dist_dir,
+    filename_encoding,
+    file_checksums,
+    win_path_sep,
+):
+    # Somewhat detail rich, at least unless we make more things mandatory, and
+    # we also need to pass all modes, since this can be run in a separate process
+    # that has no access to options.
+    # pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-statements
 
-    # TODO: This is code duplication with resource handling, should be unified
-    # and with as a context manager.
+    payload_item_size = 0
 
-    for attempt in range(1, max_attempts + 1):
-        try:
-            with open(onefile_output_filename, "ab") as output_file:
-                if attempt != 1:
-                    onefile_logger.warning(
-                        "Succeeded to open binary for payload attachment in attempt %d."
-                        % attempt
-                    )
+    filename_relative = os.path.relpath(filename_full, dist_dir)
 
-                yield output_file
+    reportProgressBar(
+        item=filename_relative,
+        update=False,
+    )
 
-                return
-        except OSError as e:
-            # Only for Windows at this time, other platforms don't have the issue.
-            if not isWin32Windows() and not isPosixWindows():
-                raise
+    # Might be changing from POSIX to Win32 Python on Windows.
+    if win_path_sep:
+        filename_relative = filename_relative.replace("/", "\\")
+    else:
+        filename_relative = filename_relative.replace("\\", "/")
 
-            if e.errno in (110, 13):
-                onefile_logger.warning(
-                    """\
-Failed to open binary for payload attachment in attempt %d.
-Disable Anti-Virus, e.g. Windows Defender for build folders. Retrying after a second of delay."""
-                    % attempt
-                )
-            else:
-                onefile_logger.warning(
-                    """\
-Failed to open binary for payload attachment in attempt %d with error code %d.
-Disable Anti-Virus, e.g. Windows Defender for build folders. Retrying after a second of delay."""
-                    % (attempt, e.errno)
-                )
+    filename_encoded = (filename_relative + "\0").encode(filename_encoding)
 
-            time.sleep(1)
-            continue
+    output_file.write(filename_encoded)
+    payload_item_size += len(filename_encoded)
 
+    file_flags = 0
+    if not isWin32OrPosixWindows() and os.path.islink(filename_full):
+        link_target = os.readlink(filename_full)
 
-def attachOnefilePayload(
-    dist_dir,
-    onefile_output_filename,
-    start_binary,
-    expect_compression,
-    file_checksums,
-    win_path_sep,
-    low_memory,
-):
-    # Somewhat detail rich, pylint: disable=too-many-statements
-    compression_indicator, compressor = getCompressorFunction(
-        expect_compression=expect_compression,
-        low_memory=low_memory,
-    )
+        file_flags |= 2
+        file_header = to_byte(file_flags)
+
+        output_file.write(file_header)
+        payload_item_size += len(file_header)
+
+        link_target_encoded = (link_target + "\0").encode(filename_encoding)
+
+        output_file.write(link_target_encoded)
+        payload_item_size += len(link_target_encoded)
+    else:
+        # This flag is only relevant for non-links.
+        if not isWin32OrPosixWindows() and os.access(filename_full, os.X_OK):
+            file_flags |= 1
+
+        with open(filename_full, "rb") as input_file:
+            input_file.seek(0, 2)
+            input_size = input_file.tell()
+            input_file.seek(0, 0)
 
-    def _attachOnefilePayloadFile(
-        compressed_file, filename_full, dist_dir, filename_encoding
-    ):
-        payload_item_size = 0
+            file_header = b""
 
-        filename_relative = os.path.relpath(filename_full, dist_dir)
+            if not isWin32OrPosixWindows():
+                file_header += to_byte(file_flags)
 
-        reportProgressBar(
-            item=filename_relative,
-            update=False,
-        )
+            file_header += struct.pack("Q", input_size)
 
-        # Might be changing from POSIX to Win32 Python on Windows.
-        if win_path_sep:
-            filename_relative = filename_relative.replace("/", "\\")
-        else:
-            filename_relative = filename_relative.replace("\\", "/")
+            if file_checksums:
+                hash_crc32 = HashCRC32()
+                hash_crc32.updateFromFileHandle(input_file)
+                input_file.seek(0, 0)
+
+                # CRC32 value 0 is avoided, used as error indicator in C code.
+                file_header += struct.pack("I", hash_crc32.asDigest() or 1)
 
-        filename_encoded = (filename_relative + "\0").encode(filename_encoding)
+            if is_archive and is_compressing:
+                compression_cache_filename = _getCacheFilename(
+                    binary_filename=filename_full, low_memory=low_memory
+                )
 
-        compressed_file.write(filename_encoded)
-        payload_item_size += len(filename_encoded)
+                if not os.path.exists(compression_cache_filename):
+                    with open(compression_cache_filename, "wb") as archive_entry_file:
+                        with file_compressor(
+                            archive_entry_file
+                        ) as compressed_file_tmp2:
+                            shutil.copyfileobj(input_file, compressed_file_tmp2)
 
-        file_flags = 0
-        if not isWin32OrPosixWindows() and os.path.islink(filename_full):
-            link_target = os.readlink(filename_full)
+                        compressed_size = archive_entry_file.tell()
+                else:
+                    compressed_size = getFileSize(compression_cache_filename)
 
-            file_flags |= 2
-            file_header = to_byte(file_flags)
+                file_header += struct.pack("I", compressed_size)
 
-            compressed_file.write(file_header)
+            output_file.write(file_header)
             payload_item_size += len(file_header)
 
-            link_target_encoded = (link_target + "\0").encode(filename_encoding)
+            if is_archive and is_compressing:
+                with open(compression_cache_filename, "rb") as archive_entry_file:
+                    pos1 = output_file.tell()
+                    shutil.copyfileobj(archive_entry_file, output_file)
+                    pos2 = output_file.tell()
+                    assert pos2 - pos1 == compressed_size
 
-            compressed_file.write(link_target_encoded)
-            payload_item_size += len(link_target_encoded)
-        else:
-            # This flag is only relevant for non-links.
-            if not isWin32OrPosixWindows() and os.access(filename_full, os.X_OK):
-                file_flags |= 1
+                if count == 0 or not use_compression_cache:
+                    os.unlink(compression_cache_filename)
 
-            with open(filename_full, "rb") as input_file:
-                input_file.seek(0, 2)
-                input_size = input_file.tell()
-                input_file.seek(0, 0)
+                payload_item_size += compressed_size
 
-                file_header = b""
+            else:
+                shutil.copyfileobj(input_file, output_file)
+                payload_item_size += input_size
 
-                if not isWin32OrPosixWindows():
-                    file_header += to_byte(file_flags)
+    reportProgressBar(
+        item=filename_relative,
+        update=True,
+    )
 
-                file_header += struct.pack("Q", input_size)
+    return payload_item_size
 
-                if file_checksums:
-                    hash_crc32 = HashCRC32()
-                    hash_crc32.updateFromFileHandle(input_file)
-                    input_file.seek(0, 0)
 
-                    # CRC32 value 0 is avoided, used as error indicator in C code.
-                    file_header += struct.pack("I", hash_crc32.asDigest() or 1)
+def _getCacheFilename(binary_filename, low_memory):
+    hash_value = Hash()
 
-                compressed_file.write(file_header)
-                payload_item_size += len(file_header)
+    hash_value.updateFromFile(filename=binary_filename)
 
-                shutil.copyfileobj(input_file, compressed_file)
-                payload_item_size += input_size
+    # Have different values for different Python major versions.
+    hash_value.updateFromValues(sys.version, sys.executable)
 
-        reportProgressBar(
-            item=filename_relative,
-            update=True,
-        )
+    # Take Nuitka version into account as well, ought to catch code changes.
+    hash_value.updateFromValues(version_string)
 
-        return payload_item_size
+    # Take zstandard version and compression level into account.
+    from zstandard import __version__
+
+    hash_value.updateFromValues(__version__, getCompressorLevel(low_memory))
+
+    cache_dir = os.path.join(getCacheDir(), "onefile-compression")
+    makePath(cache_dir)
+
+    return os.path.join(cache_dir, hash_value.asHexDigest())
+
+
+def attachOnefilePayload(
+    dist_dir,
+    onefile_output_filename,
+    start_binary,
+    expect_compression,
+    as_archive,
+    use_compression_cache,
+    file_checksums,
+    win_path_sep,
+    low_memory,
+):
+    compression_indicator, compressor = getCompressorFunction(
+        expect_compression=expect_compression,
+        low_memory=low_memory,
+    )
 
     @decoratorRetries(
         logger=onefile_logger,
@@ -235,13 +267,39 @@ def attachOnefilePayload(
                 total=len(file_list),
             )
 
-            with compressor(output_file) as compressed_file:
-                for filename_full in file_list:
+            # Abstract the differences here for the time being.
+            if as_archive:
+                # Fake compressor then
+                @contextmanager
+                def overall_compressor(f):
+                    yield f
+
+                file_compressor = compressor
+                is_archive = True
+            else:
+                overall_compressor = compressor
+
+                @contextmanager
+                def file_compressor(f):
+                    yield f
+
+                is_archive = False
+
+            with overall_compressor(output_file) as compressed_file:
+                for count, filename_full in enumerate(file_list, start=1):
                     payload_size += _attachOnefilePayloadFile(
-                        compressed_file=compressed_file,
+                        output_file=compressed_file,
+                        is_archive=is_archive,
+                        file_compressor=file_compressor,
+                        is_compressing=compression_indicator == b"Y",
+                        use_compression_cache=use_compression_cache,
+                        low_memory=low_memory,
                         filename_full=filename_full,
+                        count=count,
                         dist_dir=dist_dir,
                         filename_encoding=filename_encoding,
+                        file_checksums=file_checksums,
+                        win_path_sep=win_path_sep,
                     )
 
                 # Using empty filename as a terminator.
@@ -261,6 +319,7 @@ def attachOnefilePayload(
                     )
                 )
 
+            # TODO: If put into a resource, this is not really needed anymore.
             if isWin32Windows():
                 # add padding to have the start position at a double world boundary
                 # this is needed on windows so that a possible certificate immediately
@@ -290,6 +349,8 @@ def main():
     file_checksums = sys.argv[4] == "True"
     win_path_sep = sys.argv[5] == "True"
     low_memory = sys.argv[6] == "True"
+    as_archive = sys.argv[7] == "True"
+    use_compression_cache = sys.argv[8] == "True"
 
     if os.environ.get("NUITKA_PROGRESS_BAR") == "1":
         enableProgressBar()
@@ -300,6 +361,8 @@ def main():
         start_binary=start_binary,
         # We wouldn't be here, if that was not the case.
         expect_compression=True,
+        as_archive=as_archive,
+        use_compression_cache=use_compression_cache,
         file_checksums=file_checksums,
         win_path_sep=win_path_sep,
         low_memory=low_memory,

nuitka/tools/podman/__main__.py

@@ -90,6 +90,15 @@ Podman binary in case you do not have it in your path.
 """,
     )
 
+    parser.add_option(
+        "--podman-verbose",
+        action="store_false",
+        dest="quiet",
+        default=True,
+        help="""
+Dot not use podman quietly, giving more messages during build.""",
+    )
+
     parser.add_option(
         "--shared-path",
         action="append",
@@ -159,7 +168,7 @@ def isPodman(podman_path):
     return "podman" in os.path.normcase(os.path.basename(podman_path))
 
 
-def updateContainer(podman_path, container_tag_name, container_file_path):
+def updateContainer(podman_path, container_tag_name, container_file_path, quiet):
     requirements_file = os.path.join(
         os.path.dirname(__file__), "..", "..", "..", "requirements-devel.txt"
     )
@@ -182,13 +191,15 @@ def updateContainer(podman_path, container_tag_name, container_file_path):
             podman_path,
             "build",
             # Tolerate errors checking for image download, and use old one
-            "--quiet",
             "--tag",
             container_tag_name,
             "-f",
             container_file_path,
         ]
 
+        if quiet:
+            command.append("--quiet")
+
         if isPodman(podman_path):
             # Podman only.
             command.append("--pull=newer")
@@ -295,6 +306,9 @@ def _checkContainerArgument(options, default_container_directory):
 def main():
     options = parseOptions()
 
+    if options.command is None:
+        options.command = "python3 -m nuitka --version"
+
     containers_logger.info(
         "Running in container '%s' this command: %s"
         % (options.container_id, options.command)
@@ -342,6 +356,7 @@ def main():
             podman_path=options.podman_path,
             container_tag_name=container_tag_name,
             container_file_path=container_file_path,
+            quiet=options.quiet,
         )
 
     command = [options.podman_path, "run"]

nuitka/tools/scanning/DisplayPackageDLLs.py

@@ -32,6 +32,16 @@ from nuitka.utils.FileOperations import listDllFilesFromDirectory, relpath
 from nuitka.utils.Importing import getSharedLibrarySuffixes
 from nuitka.utils.ModuleNames import ModuleName
 from nuitka.utils.SharedLibraries import getDllExportedSymbols
+from nuitka.utils.Utils import isMacOS
+
+
+def getPythonEntryPointExportedSymbolName(module_name):
+    result = "%s%s" % ("init" if str is bytes else "PyInit_", module_name.asString())
+
+    if isMacOS():
+        result = "_" + result
+
+    return result
 
 
 def isExtensionModule(module_filename):
@@ -47,8 +57,7 @@ def isExtensionModule(module_filename):
                 return None
 
             return (
-                "%s%s" % ("init" if str is bytes else "PyInit_", module_name.asString())
-                in exported_symbols
+                getPythonEntryPointExportedSymbolName(module_name) in exported_symbols
             )
 
     return False

nuitka/tools/scanning/DisplayPackageData.py

@@ -47,7 +47,7 @@ def displayPackageData(module_name):
 
     if not os.path.isdir(package_directory):
         tools_logger.sysexit(
-            "Error, doesn't seem that '%s' is a package on disk."
+            "Error, doesn't seem that '%s' is a package on disk but rather a plain module."
             % module_name.asString()
         )
 

nuitka/tools/specialize/CTypeDescriptions.py

@@ -22,11 +22,12 @@
 import math
 from abc import abstractmethod
 
-from nuitka.__past__ import getMetaClassBase, long
+from nuitka.__past__ import long
 from nuitka.code_generation.Namify import namifyConstant
+from nuitka.utils.SlotMetaClasses import getMetaClassBase
 
 
-class TypeDescBase(getMetaClassBase("Type")):
+class TypeDescBase(getMetaClassBase("Type", require_slots=False)):
     # To be overloaded
     type_name = None
     type_desc = None
@@ -350,7 +351,7 @@ class TypeDescBase(getMetaClassBase("Type")):
             )
         elif slot == "tp_richcompare":
             # Try to detect fallbacks, this needs version specific management
-            # for at least "LONG", maybe others.
+            # for at least "LONG", maybe others. spell-checker: ignore RICHCOMPARE
 
             assert self is object_desc, self
             return "RICHCOMPARE(%s)" % operand
@@ -461,22 +462,26 @@ return %(return_value)s;""" % {
     def hasSameTypeOperationSpecializationCode(self, other, nb_slot, sq_slot):
         # Many cases, pylint: disable=too-many-branches,too-many-return-statements
 
-        cand = self if self is not object_desc else other
+        candidate = self if self is not object_desc else other
 
         # Both are objects, nothing to be done.
-        if cand is object_desc:
+        if candidate is object_desc:
             assert self is object_desc
             assert other is object_desc
             return False
 
-        # Special case for sequence concats/repeats.
-        if sq_slot is not None and not cand.hasSlot(nb_slot) and cand.hasSlot(sq_slot):
+        # Special case for sequence concat/repeats.
+        if (
+            sq_slot is not None
+            and not candidate.hasSlot(nb_slot)
+            and candidate.hasSlot(sq_slot)
+        ):
             slot = sq_slot
         else:
             slot = nb_slot
 
         if slot == "sq_repeat":
-            if cand in (
+            if candidate in (
                 list_desc,
                 tuple_desc,
                 set_desc,
@@ -489,12 +494,12 @@ return %(return_value)s;""" % {
                 return False
 
         if slot == "nb_remainder":
-            if cand in (list_desc, tuple_desc, set_desc, dict_desc):
+            if candidate in (list_desc, tuple_desc, set_desc, dict_desc):
                 # No remainder with themselves.
                 return False
 
         if slot == "nb_multiply":
-            if cand in (
+            if candidate in (
                 str_desc,
                 bytes_desc,
                 list_desc,
@@ -508,7 +513,7 @@ return %(return_value)s;""" % {
         if slot == "nb_add":
             # Tuple and list, etc. use sq_concat.
             # TODO: What about unicode_desc
-            if cand in (
+            if candidate in (
                 str_desc,
                 bytes_desc,
                 tuple_desc,
@@ -520,7 +525,7 @@ return %(return_value)s;""" % {
                 return False
 
         if slot in ("nb_and", "nb_or", "nb_xor"):
-            if cand in (
+            if candidate in (
                 str_desc,
                 bytes_desc,
                 unicode_desc,
@@ -532,7 +537,7 @@ return %(return_value)s;""" % {
                 return False
 
         if slot in ("nb_lshift", "nb_rshift"):
-            if cand in (
+            if candidate in (
                 str_desc,
                 bytes_desc,
                 unicode_desc,
@@ -588,26 +593,29 @@ return %(return_value)s;""" % {
     def getTypeComparisonSpecializationHelper(
         self, other, op_code, target, operand1, operand2
     ):
-        cand1 = self if self is not object_desc else other
-        cand2 = other if other is not object_desc else self
+        candidate1 = self if self is not object_desc else other
+        candidate2 = other if other is not object_desc else self
 
-        if cand1 is object_desc:
+        if candidate1 is object_desc:
             return "", None, None, None, None, None
 
-        if long_desc in (cand1, cand2) and int_desc in (cand1, cand2):
-            if cand1 == int_desc:
+        if long_desc in (candidate1, candidate2) and int_desc in (
+            candidate1,
+            candidate2,
+        ):
+            if candidate1 == int_desc:
                 operand1 = int_desc.getAsLongValueExpression(operand1)
-                cand1 = c_long_desc
-            elif cand2 == int_desc:
+                candidate1 = c_long_desc
+            elif candidate2 == int_desc:
                 operand2 = int_desc.getAsLongValueExpression(operand1)
-                cand2 = c_long_desc
+                candidate2 = c_long_desc
             else:
                 assert False
 
         if (
             target is n_bool_desc
-            and cand1 is cand2
-            and cand1 not in (tuple_desc, list_desc)
+            and candidate1 is candidate2
+            and candidate1 not in (tuple_desc, list_desc)
         ):
             target = c_bool_desc
 
@@ -616,12 +624,12 @@ return %(return_value)s;""" % {
             % (
                 op_code,
                 target.getHelperCodeName(),
-                cand1.getHelperCodeName(),
-                cand2.getHelperCodeName(),
+                candidate1.getHelperCodeName(),
+                candidate2.getHelperCodeName(),
             ),
             target,
-            cand1,
-            cand2,
+            candidate1,
+            candidate2,
             operand1,
             operand2,
         )
@@ -1518,6 +1526,7 @@ class CLongDesc(ConcreteCTypeBase):
 
     @staticmethod
     def getLongValueMediumValueExpression(operand):
+        # spell-checker: disable=sdigit
         return "(sdigit)%s" % (operand)
 
 

nuitka/tools/specialize/SpecializeC.py

@@ -52,7 +52,7 @@ from nuitka.code_generation.ComparisonHelperDefinitions import (
     getSpecializedComparisonOperations,
 )
 from nuitka.code_generation.ImportCodes import getImportModuleHardCodeName
-from nuitka.nodes.ImportNodes import (
+from nuitka.HardImportRegistry import (
     hard_modules,
     hard_modules_non_stdlib,
     hard_modules_version,

nuitka/tools/specialize/SpecializePython.py

@@ -475,6 +475,9 @@ def getCallModuleName(module_name, function_name):
         return "OsSysNodes"
     if module_name == "ctypes":
         return "CtypesNodes"
+    if module_name == "builtins":
+        if function_name == "open":
+            return "BuiltinOpenNodes"
 
     assert False, (module_name, function_name)
 
@@ -729,6 +732,9 @@ def addFromNodes():
     node_factory_translations[
         "ExpressionImportlibMetadataBackportMetadataCall"
     ] = "makeExpressionImportlibMetadataBackportMetadataCall"
+    node_factory_translations[
+        "ExpressionBuiltinsOpenCall"
+    ] = "makeExpressionBuiltinsOpenCall"
 
 
 addFromNodes()
@@ -1068,6 +1074,16 @@ hard_import_node_classes = {}
                     function_name_title=function_name_title,
                     function_name_code=makeCodeCased(function_name),
                     module_name=module_name,
+                    is_stdlib_module=module_name
+                    in (
+                        "builtins",
+                        "os",
+                        "os.path",
+                        "pkgutil",
+                        "ctypes",
+                        "importlib.metadata",
+                        "importlib.resources",
+                    ),
                     module_name_code=makeCodeCased(adaptModuleName(module_name)),
                     module_name_title=module_name_title,
                     call_node_module_name=getCallModuleName(module_name, function_name),