Nuitka-winsvc 1.8__cp311-cp311-win_amd64.whl → 2.5.1__cp311-cp311-win_amd64.whl

nuitka/tools/onefile_compressor/OnefileCompressor.py

@@ -1,20 +1,6 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ Internal tool, compression standalone distribution files and attach to onefile bootstrap binary.
 
 """
@@ -23,7 +9,6 @@ import os
 import shutil
 import struct
 import sys
-import time
 from contextlib import contextmanager
 
 from nuitka.__past__ import to_byte
@@ -34,14 +19,24 @@ from nuitka.Progress import (
     setupProgressBar,
 )
 from nuitka.Tracing import onefile_logger
-from nuitka.utils.FileOperations import getFileList
-from nuitka.utils.Hashing import HashCRC32
+from nuitka.utils.AppDirs import getCacheDir
+from nuitka.utils.FileOperations import (
+    areSamePaths,
+    getFileList,
+    getFileSize,
+    makePath,
+)
+from nuitka.utils.Hashing import Hash, HashCRC32
 from nuitka.utils.Utils import (
     decoratorRetries,
-    isPosixWindows,
     isWin32OrPosixWindows,
     isWin32Windows,
 )
+from nuitka.Version import version_string
+
+
+def getCompressorLevel(low_memory):
+    return 3 if low_memory else 22
 
 
 def getCompressorFunction(expect_compression, low_memory):
@@ -50,7 +45,7 @@ def getCompressorFunction(expect_compression, low_memory):
     if expect_compression:
         from zstandard import ZstdCompressor  # pylint: disable=I0021,import-error
 
-        compressor_context = ZstdCompressor(level=3 if low_memory else 22)
+        compressor_context = ZstdCompressor(level=getCompressorLevel(low_memory))
 
         @contextmanager
         def useCompressedFile(output_file):
@@ -71,138 +66,180 @@ def getCompressorFunction(expect_compression, low_memory):
         return b"X", useSameFile
 
 
-@contextmanager
-def _openBinaryFileForAppending(onefile_output_filename):
-    max_attempts = 5
+def _attachOnefilePayloadFile(
+    output_file,
+    is_archive,
+    is_compressing,
+    use_compression_cache,
+    low_memory,
+    file_compressor,
+    filename_full,
+    count,
+    dist_dir,
+    filename_encoding,
+    file_checksums,
+    win_path_sep,
+):
+    # Somewhat detail rich, at least unless we make more things mandatory, and
+    # we also need to pass all modes, since this can be run in a separate process
+    # that has no access to options.
+    # pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-statements
 
-    # TODO: This is code duplication with resource handling, should be unified
-    # and with as a context manager.
+    payload_item_size = 0
 
-    for attempt in range(1, max_attempts + 1):
-        try:
-            with open(onefile_output_filename, "ab") as output_file:
-                if attempt != 1:
-                    onefile_logger.warning(
-                        "Succeeded to open binary for payload attachment in attempt %d."
-                        % attempt
-                    )
+    filename_relative = os.path.relpath(filename_full, dist_dir)
 
-                yield output_file
+    reportProgressBar(
+        item=filename_relative,
+        update=False,
+    )
 
-                return
-        except OSError as e:
-            # Only for Windows at this time, other platforms don't have the issue.
-            if not isWin32Windows() and not isPosixWindows():
-                raise
+    # Might be changing from POSIX to Win32 Python on Windows.
+    if win_path_sep:
+        filename_relative = filename_relative.replace("/", "\\")
+    else:
+        filename_relative = filename_relative.replace("\\", "/")
 
-            if e.errno in (110, 13):
-                onefile_logger.warning(
-                    """\
-Failed to open binary for payload attachment in attempt %d.
-Disable Anti-Virus, e.g. Windows Defender for build folders. Retrying after a second of delay."""
-                    % attempt
-                )
-            else:
-                onefile_logger.warning(
-                    """\
-Failed to open binary for payload attachment in attempt %d with error code %d.
-Disable Anti-Virus, e.g. Windows Defender for build folders. Retrying after a second of delay."""
-                    % (attempt, e.errno)
-                )
+    filename_encoded = (filename_relative + "\0").encode(filename_encoding)
 
-            time.sleep(1)
-            continue
+    output_file.write(filename_encoded)
+    payload_item_size += len(filename_encoded)
 
+    file_flags = 0
+    if not isWin32OrPosixWindows() and os.path.islink(filename_full):
+        link_target = os.readlink(filename_full)
 
-def attachOnefilePayload(
-    dist_dir,
-    onefile_output_filename,
-    start_binary,
-    expect_compression,
-    file_checksums,
-    win_path_sep,
-    low_memory,
-):
-    # Somewhat detail rich, pylint: disable=too-many-statements
-    compression_indicator, compressor = getCompressorFunction(
-        expect_compression=expect_compression,
-        low_memory=low_memory,
-    )
+        file_flags |= 2
+        file_header = to_byte(file_flags)
 
-    def _attachOnefilePayloadFile(
-        compressed_file, filename_full, dist_dir, filename_encoding
-    ):
-        payload_item_size = 0
+        output_file.write(file_header)
+        payload_item_size += len(file_header)
 
-        filename_relative = os.path.relpath(filename_full, dist_dir)
+        link_target_encoded = (link_target + "\0").encode(filename_encoding)
 
-        reportProgressBar(
-            item=filename_relative,
-            update=False,
-        )
+        output_file.write(link_target_encoded)
+        payload_item_size += len(link_target_encoded)
+    else:
+        # This flag is only relevant for non-links.
+        if not isWin32OrPosixWindows() and os.access(filename_full, os.X_OK):
+            file_flags |= 1
+
+        with open(filename_full, "rb") as input_file:
+            input_file.seek(0, 2)
+            input_size = input_file.tell()
+            input_file.seek(0, 0)
+
+            file_header = b""
 
-        # Might be changing from POSIX to Win32 Python on Windows.
-        if win_path_sep:
-            filename_relative = filename_relative.replace("/", "\\")
-        else:
-            filename_relative = filename_relative.replace("\\", "/")
+            if not isWin32OrPosixWindows():
+                file_header += to_byte(file_flags)
 
-        filename_encoded = (filename_relative + "\0").encode(filename_encoding)
+            file_header += struct.pack("Q", input_size)
 
-        compressed_file.write(filename_encoded)
-        payload_item_size += len(filename_encoded)
+            if file_checksums:
+                hash_crc32 = HashCRC32()
+                hash_crc32.updateFromFileHandle(input_file)
+                input_file.seek(0, 0)
+
+                # CRC32 value 0 is avoided, used as error indicator in C code.
+                file_header += struct.pack("I", hash_crc32.asDigest() or 1)
 
-        file_flags = 0
-        if not isWin32OrPosixWindows() and os.path.islink(filename_full):
-            link_target = os.readlink(filename_full)
+            if is_archive and is_compressing:
+                compression_cache_filename = _getCacheFilename(
+                    binary_filename=filename_full, low_memory=low_memory
+                )
 
-            file_flags |= 2
-            file_header = to_byte(file_flags)
+                if not os.path.exists(compression_cache_filename):
+                    with open(compression_cache_filename, "wb") as archive_entry_file:
+                        with file_compressor(
+                            archive_entry_file
+                        ) as compressed_file_tmp2:
+                            shutil.copyfileobj(input_file, compressed_file_tmp2)
 
-            compressed_file.write(file_header)
+                        compressed_size = archive_entry_file.tell()
+                else:
+                    compressed_size = getFileSize(compression_cache_filename)
+
+                file_header += struct.pack("I", compressed_size)
+
+            output_file.write(file_header)
             payload_item_size += len(file_header)
 
-            link_target_encoded = (link_target + "\0").encode(filename_encoding)
+            if is_archive and is_compressing:
+                with open(compression_cache_filename, "rb") as archive_entry_file:
+                    pos1 = output_file.tell()
+                    shutil.copyfileobj(archive_entry_file, output_file)
+                    pos2 = output_file.tell()
+                    assert pos2 - pos1 == compressed_size
 
-            compressed_file.write(link_target_encoded)
-            payload_item_size += len(link_target_encoded)
-        else:
-            # This flag is only relevant for non-links.
-            if not isWin32OrPosixWindows() and os.access(filename_full, os.X_OK):
-                file_flags |= 1
+                if count == 0 or not use_compression_cache:
+                    os.unlink(compression_cache_filename)
 
-            with open(filename_full, "rb") as input_file:
-                input_file.seek(0, 2)
-                input_size = input_file.tell()
-                input_file.seek(0, 0)
+                payload_item_size += compressed_size
 
-                file_header = b""
+            else:
+                shutil.copyfileobj(input_file, output_file)
+                payload_item_size += input_size
 
-                if not isWin32OrPosixWindows():
-                    file_header += to_byte(file_flags)
+    reportProgressBar(
+        item=filename_relative,
+        update=True,
+    )
 
-                file_header += struct.pack("Q", input_size)
+    return payload_item_size
 
-                if file_checksums:
-                    hash_crc32 = HashCRC32()
-                    hash_crc32.updateFromFileHandle(input_file)
-                    input_file.seek(0, 0)
 
-                    # CRC32 value 0 is avoided, used as error indicator in C code.
-                    file_header += struct.pack("I", hash_crc32.asDigest() or 1)
+def _getCacheFilename(binary_filename, low_memory):
+    hash_value = Hash()
 
-                compressed_file.write(file_header)
-                payload_item_size += len(file_header)
+    hash_value.updateFromFile(filename=binary_filename)
+
+    # Have different values for different Python major versions.
+    hash_value.updateFromValues(sys.version, sys.executable)
+
+    # Take Nuitka version into account as well, ought to catch code changes.
+    hash_value.updateFromValues(version_string)
+
+    # Take zstandard version and compression level into account.
+    from zstandard import __version__  # pylint: disable=I0021,import-error
+
+    hash_value.updateFromValues(__version__, getCompressorLevel(low_memory))
+
+    cache_dir = getCacheDir("onefile-compression")
+    makePath(cache_dir)
+
+    return os.path.join(cache_dir, hash_value.asHexDigest())
 
-                shutil.copyfileobj(input_file, compressed_file)
-                payload_item_size += input_size
 
-        reportProgressBar(
-            item=filename_relative,
-            update=True,
-        )
+def _getInputFileList(dist_dir, start_binary):
+    file_list = getFileList(dist_dir, normalize=False)
+    file_list_size = len(file_list)
+    file_list = [
+        filename for filename in file_list if not areSamePaths(filename, start_binary)
+    ]
+    file_list.insert(0, start_binary)
 
-        return payload_item_size
+    # If this fails, above code failed to find the start binary.
+    assert file_list_size == len(file_list)
+
+    return tuple(file_list)
+
+
+def attachOnefilePayload(
+    dist_dir,
+    onefile_output_filename,
+    start_binary,
+    expect_compression,
+    as_archive,
+    use_compression_cache,
+    file_checksums,
+    win_path_sep,
+    low_memory,
+):
+    compression_indicator, compressor = getCompressorFunction(
+        expect_compression=expect_compression,
+        low_memory=low_memory,
+    )
 
     @decoratorRetries(
         logger=onefile_logger,
@@ -218,9 +255,7 @@ def attachOnefilePayload(
             output_file.write(b"KA" + compression_indicator)
 
             # Move the binary to start immediately to the start position
-            file_list = getFileList(dist_dir, normalize=False)
-            file_list.remove(start_binary)
-            file_list.insert(0, start_binary)
+            file_list = _getInputFileList(dist_dir=dist_dir, start_binary=start_binary)
 
             if isWin32Windows():
                 filename_encoding = "utf-16le"
@@ -235,13 +270,39 @@ def attachOnefilePayload(
                 total=len(file_list),
             )
 
-            with compressor(output_file) as compressed_file:
-                for filename_full in file_list:
+            # Abstract the differences here for the time being.
+            if as_archive:
+                # Fake compressor then
+                @contextmanager
+                def overall_compressor(f):
+                    yield f
+
+                file_compressor = compressor
+                is_archive = True
+            else:
+                overall_compressor = compressor
+
+                @contextmanager
+                def file_compressor(f):
+                    yield f
+
+                is_archive = False
+
+            with overall_compressor(output_file) as compressed_file:
+                for count, filename_full in enumerate(file_list, start=1):
                     payload_size += _attachOnefilePayloadFile(
-                        compressed_file=compressed_file,
+                        output_file=compressed_file,
+                        is_archive=is_archive,
+                        file_compressor=file_compressor,
+                        is_compressing=compression_indicator == b"Y",
+                        use_compression_cache=use_compression_cache,
+                        low_memory=low_memory,
                         filename_full=filename_full,
+                        count=count,
                         dist_dir=dist_dir,
                         filename_encoding=filename_encoding,
+                        file_checksums=file_checksums,
+                        win_path_sep=win_path_sep,
                     )
 
                 # Using empty filename as a terminator.
@@ -261,6 +322,7 @@ def attachOnefilePayload(
                     )
                 )
 
+            # TODO: If put into a resource, this is not really needed anymore.
             if isWin32Windows():
                 # add padding to have the start position at a double world boundary
                 # this is needed on windows so that a possible certificate immediately
@@ -290,8 +352,10 @@ def main():
     file_checksums = sys.argv[4] == "True"
     win_path_sep = sys.argv[5] == "True"
     low_memory = sys.argv[6] == "True"
+    as_archive = sys.argv[7] == "True"
+    use_compression_cache = sys.argv[8] == "True"
 
-    if os.environ.get("NUITKA_PROGRESS_BAR") == "1":
+    if os.getenv("NUITKA_PROGRESS_BAR") == "1":
         enableProgressBar()
 
     attachOnefilePayload(
@@ -300,9 +364,27 @@ def main():
         start_binary=start_binary,
         # We wouldn't be here, if that was not the case.
         expect_compression=True,
+        as_archive=as_archive,
+        use_compression_cache=use_compression_cache,
         file_checksums=file_checksums,
         win_path_sep=win_path_sep,
         low_memory=low_memory,
     )
 
     sys.exit(0)
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/tools/onefile_compressor/__init__.py

@@ -1,5 +1,8 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" Dummy file to make this directory a package. """
+
 #     Part of "Nuitka", an optimizing Python compiler that is compatible and
 #     integrates with CPython, but also works on its own.
 #
@@ -14,5 +17,3 @@
 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #     See the License for the specific language governing permissions and
 #     limitations under the License.
-#
-""" Dummy file to make this directory a package. """

nuitka/tools/onefile_compressor/__main__.py

@@ -1,20 +1,6 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ Internal tool, attach the standalone distribution in compressed form.
 
 """
@@ -38,3 +24,18 @@ if __name__ == "__main__":
     from nuitka.tools.onefile_compressor.OnefileCompressor import main
 
     main()
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/tools/podman/Podman.py

@@ -1,20 +1,6 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ Podman container usage tools. """
 
 from nuitka.utils.Execution import getExecutablePath
@@ -51,3 +37,19 @@ Cannot find 'podman'. Install it from \
                 )
 
     return result
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/tools/podman/__init__.py

@@ -1,5 +1,8 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+""" Dummy file to make this directory a package. """
+
 #     Part of "Nuitka", an optimizing Python compiler that is compatible and
 #     integrates with CPython, but also works on its own.
 #
@@ -14,5 +17,3 @@
 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #     See the License for the specific language governing permissions and
 #     limitations under the License.
-#
-""" Dummy file to make this directory a package. """

nuitka/tools/podman/__main__.py

@@ -1,20 +1,6 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ Launcher for running a script inside a container.
 
 Podman and Docker should both work, but the first one is recommended.
@@ -90,6 +76,15 @@ Podman binary in case you do not have it in your path.
 """,
     )
 
+    parser.add_option(
+        "--podman-verbose",
+        action="store_false",
+        dest="quiet",
+        default=True,
+        help="""
+Dot not use podman quietly, giving more messages during build.""",
+    )
+
     parser.add_option(
         "--shared-path",
         action="append",
@@ -159,7 +154,7 @@ def isPodman(podman_path):
     return "podman" in os.path.normcase(os.path.basename(podman_path))
 
 
-def updateContainer(podman_path, container_tag_name, container_file_path):
+def updateContainer(podman_path, container_tag_name, container_file_path, quiet):
     requirements_file = os.path.join(
         os.path.dirname(__file__), "..", "..", "..", "requirements-devel.txt"
     )
@@ -182,13 +177,15 @@ def updateContainer(podman_path, container_tag_name, container_file_path):
             podman_path,
             "build",
             # Tolerate errors checking for image download, and use old one
-            "--quiet",
             "--tag",
             container_tag_name,
             "-f",
             container_file_path,
         ]
 
+        if quiet:
+            command.append("--quiet")
+
         if isPodman(podman_path):
             # Podman only.
             command.append("--pull=newer")
@@ -295,6 +292,9 @@ def _checkContainerArgument(options, default_container_directory):
 def main():
     options = parseOptions()
 
+    if options.command is None:
+        options.command = "python3 -m nuitka --version"
+
     containers_logger.info(
         "Running in container '%s' this command: %s"
         % (options.container_id, options.command)
@@ -342,6 +342,7 @@ def main():
             podman_path=options.podman_path,
             container_tag_name=container_tag_name,
             container_file_path=container_file_path,
+            quiet=options.quiet,
         )
 
     command = [options.podman_path, "run"]
@@ -402,3 +403,18 @@ def main():
 
 if __name__ == "__main__":
     main()
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.