Nuitka-winsvc 1.8__cp311-cp311-win_amd64.whl → 2.5.1__cp311-cp311-win_amd64.whl

nuitka/utils/FileOperations.py

@@ -1,20 +1,6 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ Utils for file and directory operations.
 
 This provides enhanced and more error resilient forms of standard
@@ -24,34 +10,39 @@ stuff. It will also frequently add sorting for determinism.
 
 from __future__ import print_function
 
+import codecs
 import errno
 import fnmatch
 import glob
 import os
+import pickle
 import shutil
 import stat
+import sys
 import tempfile
 import time
+import unicodedata
 from contextlib import contextmanager
 
 from nuitka.__past__ import (  # pylint: disable=redefined-builtin
+    FileNotFoundError,
     PermissionError,
-    WindowsError,
     basestring,
-    raw_input,
+    unicode,
 )
 from nuitka.PythonVersions import python_version
-from nuitka.Tracing import (
-    flushStandardOutputs,
-    general,
-    my_print,
-    options_logger,
-    printLine,
-)
+from nuitka.Tracing import general, my_print, options_logger, queryUser
 
 from .Importing import importFromInlineCopy
 from .ThreadedExecutor import RLock, getThreadIdent
-from .Utils import isLinux, isMacOS, isWin32OrPosixWindows, isWin32Windows
+from .Utils import (
+    decoratorRetries,
+    isLinux,
+    isMacOS,
+    isWin32OrPosixWindows,
+    isWin32Windows,
+    raiseWindowsError,
+)
 
 # Locking seems to be only required for Windows mostly, but we can keep
 # it for all.
@@ -101,8 +92,11 @@ def areSamePaths(path1, path2):
         even short paths, it then becomes a mere string compare after that.
     """
 
-    path1 = os.path.abspath(os.path.normpath(path1))
-    path2 = os.path.abspath(os.path.normpath(path2))
+    if path1 == path2:
+        return True
+
+    path1 = os.path.abspath(getNormalizedPath(path1))
+    path2 = os.path.abspath(getNormalizedPath(path2))
 
     if os.path.exists(path1) and os.path.exists(path2):
         path1 = getExternalUsePath(path1)
@@ -190,7 +184,11 @@ def makePath(path):
 
     with withFileLock("creating directory %s" % path):
         if not os.path.isdir(path):
-            os.makedirs(path)
+            try:
+                os.makedirs(path)
+            except OSError:
+                if not os.path.exists(path):
+                    raise
 
 
 def makeContainingPath(filename):
@@ -240,9 +238,14 @@ def _getRealPathWindows(path):
             if str is not bytes:
                 result = result.decode("utf8")
 
-            _real_path_windows_cache[path] = os.path.join(
-                os.path.dirname(path), result.rstrip("\r\n")
-            )
+            if result.startswith("UNC\\"):
+                # Avoid network mounts being converted to UNC shared paths by newer
+                # Python versions, many tools won't work with those.
+                _real_path_windows_cache[path] = path
+            else:
+                _real_path_windows_cache[path] = os.path.join(
+                    os.path.dirname(path), result.rstrip("\r\n")
+                )
         else:
             _real_path_windows_cache[path] = path
 
@@ -259,7 +262,9 @@ def getDirectoryRealPath(path):
         path with symlinks resolved
 
     Notes:
-        Workaround for Windows symlink is applied.
+        Workaround for Windows symlink is applied. This function is not recursive
+        at all with older Python, i.e. only the last part, the directory itself
+        is being resolved there.
 
     """
     path = os.path.realpath(path)
@@ -272,6 +277,62 @@ def getDirectoryRealPath(path):
     return path
 
 
+def _restoreWindowsPath(orig_path, path):
+    if path.startswith("\\\\"):
+        drive, _remaining_path = os.path.splitdrive(orig_path)
+
+        if drive and not drive.startswith("\\\\"):
+            drive_real_path = os.path.realpath(drive + "\\")
+            assert path.startswith(drive_real_path)
+
+            path = drive + "\\" + path[len(drive_real_path) :]
+    else:
+        path = path.strip(os.path.sep)
+
+        if os.path.sep in path:
+            dirname = os.path.dirname(path)
+            filename = os.path.basename(path)
+
+            if dirname:
+                dirname = getDirectoryRealPath(dirname)
+
+                # Drive letters do not get slashes from "os.path.join", so
+                # we inject this here and normalize the path afterwards to
+                # remove any duplication added.
+                if os.path.sep not in dirname:
+                    dirname = dirname + os.path.sep
+
+                path = os.path.normpath(os.path.join(dirname, filename))
+
+    return path
+
+
+def getFilenameRealPath(path):
+    """Get os.path.realpath with Python2 and Windows symlink workaround applied.
+
+    Args:
+        path: path to get realpath of
+
+    Returns:
+        path with symlinks resolved
+
+    Notes:
+        Workaround for Windows symlinks are applied, this works recursive and
+        assumes that the path given itself is a file and not a directory, and
+        doesn't handle file symlinks at the end on older Python currently, but
+        we shouldn't deal with those.
+    """
+    orig_path = path
+    path = os.path.realpath(path)
+
+    # Avoid network mounts being converted to UNC shared paths by newer
+    # Python versions, many tools won't work with those.
+    if os.name == "nt":
+        path = _restoreWindowsPath(orig_path=orig_path, path=path)
+
+    return path
+
+
 def listDir(path):
     """Give a sorted listing of a path.
 
@@ -293,8 +354,28 @@ def listDir(path):
     """
     real_path = getDirectoryRealPath(path)
 
+    # The "os.listdir" output needs to be unicode paths, or else it can be unusable
+    # for Python2 on Windows at least. We try to go back on the result.
+    if str is bytes and type(real_path) is str:
+        real_path = unicode(real_path)
+
+    def _tryDecodeToStr(value):
+        if str is bytes:
+            if type(value) is unicode:
+                # File system paths, that should be usable for names of modules,
+                # as Python2 code objects will e.g. hate unicode values.
+
+                # spell-checker: ignore getfilesystemencoding
+                try:
+                    return value.decode(sys.getfilesystemencoding())
+                except UnicodeDecodeError:
+                    return value
+        else:
+            return value
+
     return sorted(
-        [(os.path.join(path, filename), filename) for filename in os.listdir(real_path)]
+        (_tryDecodeToStr(os.path.join(path, filename)), _tryDecodeToStr(filename))
+        for filename in os.listdir(real_path)
     )
 
 
@@ -359,7 +440,7 @@ def getFileList(
             fullname = os.path.join(root, filename)
 
             if normalize:
-                fullname = os.path.normpath(fullname)
+                fullname = getNormalizedPath(fullname)
 
             result.append(fullname)
 
@@ -409,9 +490,9 @@ def getDllBasename(path):
         if compare_path.endswith(suffix):
             return path[: -len(suffix)]
 
-    # Linux us not case sensitive, but lets still do it properly,
-    # sometimes, it is done on non-Linux too. So we split on the
-    # normcase, but only to find out what is going on there.
+    # Linux is not case sensitive, but lets still do it properly, sometimes, it
+    # is done macOS too. So we split on the normcase, but only to find out what
+    # is going on there.
     if ".so." in compare_path:
         return path[: len(compare_path.split(".so.")[0])]
 
@@ -473,7 +554,7 @@ def listExeFilesFromDirectory(path, prefix=None, suffixes=None):
 
     # On Windows, we check exe suffixes, on other platforms we shell all filenames,
     # matching the prefix, but they have to the executable bit set.
-    if suffixes is None and isWin32OrPosixWindows():
+    if not suffixes and isWin32OrPosixWindows():
         suffixes = "exe", "bin"
 
     if suffixes:
@@ -577,11 +658,19 @@ def deleteFile(path, must_exist):
             raise OSError("Does not exist", path)
 
 
-def splitPath(path):
-    """Split path, skipping empty elements."""
-    return tuple(
-        element for element in os.path.split(path.rstrip(os.path.sep)) if element
-    )
+def searchPrefixPath(path, element):
+    """Search element and return prefix in path, if any."""
+
+    while path:
+        if os.path.normcase(os.path.basename(path)) == os.path.normcase(element):
+            return path
+
+        new_path = os.path.dirname(path)
+        if new_path == path:
+            break
+        path = new_path
+
+    return None
 
 
 def getFilenameExtension(path):
@@ -605,7 +694,8 @@ def hasFilenameExtension(path, extensions):
     """Has a filename one of the given extensions.
 
     Note: The extensions should be normalized, i.e. lower case and will match other
-    cases where the file system does that on a platform.
+    cases where the file system does that on a platform. Also they include a dot,
+    e.g. ".qml" is a good value.
     """
 
     extension = getFilenameExtension(path)
@@ -616,7 +706,14 @@ def hasFilenameExtension(path, extensions):
         return extension in extensions
 
 
-def removeDirectory(path, ignore_errors):
+def addFilenameExtension(path, extension):
+    if not hasFilenameExtension(path, extension):
+        path += extension
+
+    return path
+
+
+def removeDirectory(path, logger, ignore_errors, extra_recommendation):
     """Remove a directory recursively.
 
     On Windows, it happens that operations fail, and succeed when retried,
@@ -627,28 +724,52 @@ def removeDirectory(path, ignore_errors):
     it hopefully only briefly.
     """
 
-    def onError(func, path, exc_info):
-        # Try again immediately, ignore what happened, pylint: disable=unused-argument
-        try:
-            func(path)
-        except OSError:
-            time.sleep(0.1)
-
-        func(path)
-
     with withFileLock("removing directory %s" % path):
         if os.path.exists(path):
-            try:
-                shutil.rmtree(path, ignore_errors=False, onerror=onError)
-            except OSError:
-                if ignore_errors:
-                    shutil.rmtree(path, ignore_errors=ignore_errors)
-                else:
-                    raise
 
-
-def resetDirectory(path, ignore_errors):
-    removeDirectory(path=path, ignore_errors=ignore_errors)
+            @decoratorRetries(
+                logger=logger,
+                purpose="delete '%s'" % path,
+                consequence="the path is not fully removed",
+                extra_recommendation=extra_recommendation,
+            )
+            def _removeDirectory():
+                def onError(func, path, exc_info):
+                    # Record what happened what happened, pylint: disable=unused-argument
+                    last_error.append((func, path))
+
+                previous_error = []
+
+                # Try deleting while ignoring errors first.
+                while True:
+                    last_error = []
+                    shutil.rmtree(path, ignore_errors=False, onerror=onError)
+
+                    # onError as a side effect, modifies last_error
+                    if previous_error == last_error:
+                        break
+
+                    previous_error = list(last_error)
+                    time.sleep(0.2)
+
+                # If it still exists, try one more time, this time not ignoring errors.
+                if os.path.exists(path):
+                    try:
+                        shutil.rmtree(path, ignore_errors=ignore_errors)
+                    except OSError:
+                        if not ignore_errors:
+                            raise
+
+            _removeDirectory()
+
+
+def resetDirectory(path, logger, ignore_errors, extra_recommendation):
+    removeDirectory(
+        path=path,
+        logger=logger,
+        ignore_errors=ignore_errors,
+        extra_recommendation=extra_recommendation,
+    )
     makePath(path)
 
 
@@ -660,19 +781,22 @@ def withTemporaryFile(suffix="", mode="w", delete=True, temp_path=None):
         yield temp_file
 
 
-def getFileContentByLine(filename, mode="r", encoding=None):
+def getFileContentByLine(filename, mode="r", encoding=None, errors=None):
     # We read the whole, to keep lock times minimal. We only deal with small
     # files like this normally.
-    return getFileContents(filename, mode, encoding=encoding).splitlines()
+    return getFileContents(
+        filename, mode, encoding=encoding, errors=errors
+    ).splitlines()
 
 
-def getFileContents(filename, mode="r", encoding=None):
+def getFileContents(filename, mode="r", encoding=None, errors=None):
     """Get the contents of a file.
 
     Args:
         filename: str with the file to be read
         mode: "r" for str, "rb" for bytes result
         encoding: optional encoding to used when reading the file, e.g. "utf8"
+        errors: optional error handler decoding the content, as defined in `codecs`
 
     Returns:
         str or bytes - depending on mode.
@@ -680,7 +804,7 @@ def getFileContents(filename, mode="r", encoding=None):
     """
 
     with withFileLock("reading file %s" % filename):
-        with openTextFile(filename, mode, encoding=encoding) as f:
+        with openTextFile(filename, mode, encoding=encoding, errors=errors) as f:
             return f.read()
 
 
@@ -702,18 +826,11 @@ def getFileFirstLine(filename, mode="r", encoding=None):
             return f.readline()
 
 
-def openTextFile(filename, mode, encoding=None):
-    if encoding is not None:
-        import codecs
-
-        return codecs.open(filename, mode, encoding=encoding)
-    else:
-        # Avoid deprecation warning, is now the default.
-        if python_version >= 0x370:
-            mode = mode.replace("U", "")
+def openTextFile(filename, mode, encoding=None, errors=None):
+    if python_version >= 0x370:
+        mode = mode.replace("U", "")
 
-        # Encoding was checked to be not needed.
-        return open(filename, mode)  # pylint: disable=unspecified-encoding
+    return codecs.open(filename, mode, encoding=encoding, errors=errors)
 
 
 def putTextFileContents(filename, contents, encoding=None):
@@ -740,6 +857,22 @@ def putTextFileContents(filename, contents, encoding=None):
             _writeContents(output_file)
 
 
+def putBinaryFileContents(filename, contents):
+    """Write a binary file from given contents.
+
+    Args:
+        filename: str with the file to be created
+        contents: bytes that should be written into the file
+
+    Returns:
+        None
+    """
+
+    with withFileLock("writing file %s" % filename):
+        with openTextFile(filename, "wb") as output_file:
+            output_file.write(contents)
+
+
 def changeTextFileContents(filename, contents, encoding=None, compare_only=False):
     """Write a text file from given contents.
 
@@ -766,7 +899,7 @@ def changeTextFileContents(filename, contents, encoding=None, compare_only=False
 
 @contextmanager
 def withPreserveFileMode(filenames):
-    if type(filenames) is str:
+    if isinstance(filenames, basestring):
         filenames = [filenames]
 
     old_modes = {}
@@ -781,7 +914,7 @@ def withPreserveFileMode(filenames):
 
 @contextmanager
 def withMadeWritableFileMode(filenames):
-    if type(filenames) is str:
+    if isinstance(filenames, basestring):
         filenames = [filenames]
 
     with withPreserveFileMode(filenames):
@@ -839,10 +972,13 @@ def copyTree(source_path, dest_path):
     if python_version >= 0x380:
         # Python 3.8+ has dirs_exist_ok
         return shutil.copytree(source_path, dest_path, dirs_exist_ok=True)
+    else:
 
-    from distutils.dir_util import copy_tree
+        from distutils.dir_util import (  # Older Python only, pylint: disable=I0021,import-error
+            copy_tree,
+        )
 
-    return copy_tree(source_path, dest_path)
+        return copy_tree(source_path, dest_path)
 
 
 def copyFileWithPermissions(source_path, dest_path, dist_dir):
@@ -881,33 +1017,6 @@ def copyFileWithPermissions(source_path, dest_path, dist_dir):
         os.chmod(dest_path, source_mode)
 
 
-def queryUser(question, choices, default, default_non_interactive):
-    assert default in choices, (default, choices)
-    assert default_non_interactive in choices, (default, choices)
-
-    prompt = "%s? %s : " % (
-        question,
-        "/".join(
-            "[%s]" % choice.title() if choice == default else choice.title()
-            for choice in choices
-        ),
-    )
-
-    # Integrates with progress bar by closing it.
-    printLine(prompt, end="")
-    flushStandardOutputs()
-
-    try:
-        reply = raw_input() or default
-    except EOFError:
-        reply = default_non_interactive
-
-    if reply == "y":
-        reply = "yes"
-
-    return reply.lower()
-
-
 def copyFile(source_path, dest_path):
     """Improved version of shutil.copy
 
@@ -977,6 +1086,10 @@ def isFilenameBelowPath(path, filename, consider_short=True):
             filename = getExternalUsePath(filename)
             path = getExternalUsePath(path)
 
+            if isWin32Windows():
+                if getWindowsDrive(path) != getWindowsDrive(filename):
+                    return False
+
             result = os.path.relpath(filename, path).split(os.path.sep)[0] != ".."
 
     return result
@@ -1021,10 +1134,7 @@ def getWindowsShortPathName(filename):
             if ctypes.GetLastError() == 5:
                 return filename
 
-            raise WindowsError(
-                ctypes.GetLastError(), ctypes.FormatError(ctypes.GetLastError())
-            )
-
+            raiseWindowsError("getWindowsShortPathName for %s" % filename)
         if output_buf_size >= needed:
             # Short paths should be ASCII. Don't return unicode without a need,
             # as e.g. Scons hates that in environment variables.
@@ -1058,7 +1168,7 @@ def getWindowsLongPathName(filename):
     while True:
         output_buf = ctypes.create_unicode_buffer(output_buf_size)
         needed = GetLongPathNameW(
-            os.path.abspath(filename), output_buf, output_buf_size
+            getNormalizedPath(os.path.abspath(filename)), output_buf, output_buf_size
         )
 
         if needed == 0:
@@ -1068,10 +1178,7 @@ def getWindowsLongPathName(filename):
             if ctypes.GetLastError() == 5:
                 return filename
 
-            raise WindowsError(
-                ctypes.GetLastError(), ctypes.FormatError(ctypes.GetLastError())
-            )
-
+            raiseWindowsError("getWindowsLongPathName for %s" % filename)
         if output_buf_size >= needed:
             return output_buf.value
         else:
@@ -1096,8 +1203,10 @@ def getExternalUsePath(filename, only_dirname=False):
     filename = os.path.abspath(filename)
 
     if os.name == "nt":
+        key = filename, only_dirname
+
         if filename not in _external_use_path_cache:
-            asked_filename = filename
+            filename = getFilenameRealPath(filename)
 
             if only_dirname:
                 dirname = getWindowsShortPathName(os.path.dirname(filename))
@@ -1106,10 +1215,13 @@ def getExternalUsePath(filename, only_dirname=False):
             else:
                 filename = getWindowsShortPathName(filename)
 
-            _external_use_path_cache[asked_filename] = filename
-            _external_use_path_cache[filename] = filename
+            _external_use_path_cache[key] = filename
+
+            # Looking up again should give same result immediately.
+            key = filename, only_dirname
+            _external_use_path_cache[key] = filename
 
-        return _external_use_path_cache[filename]
+        return _external_use_path_cache[key]
     else:
         return filename
 
@@ -1202,6 +1314,10 @@ def getLinkTarget(filename):
     return is_link, filename
 
 
+# Late import and optional to be there.
+atomicwrites = None
+
+
 def replaceFileAtomic(source_path, dest_path):
     """
     Move ``src`` to ``dst``. If ``dst`` exists, it will be silently
@@ -1216,9 +1332,12 @@ def replaceFileAtomic(source_path, dest_path):
     if python_version >= 0x300:
         os.replace(source_path, dest_path)
     else:
-        importFromInlineCopy("atomicwrites", must_exist=True).replace_atomic(
-            source_path, dest_path
-        )
+        global atomicwrites  # singleton, pylint: disable=global-statement
+
+        if atomicwrites is None:
+            atomicwrites = importFromInlineCopy("atomicwrites", must_exist=True)
+
+        atomicwrites.replace_atomic(source_path, dest_path)
 
 
 def resolveShellPatternToFilenames(pattern):
@@ -1303,3 +1422,113 @@ def syncFileOutput(file_handle):
         except AttributeError:
             # Too old to have "syncfs" available.
             return
+
+
+def isFilesystemEncodable(filename):
+    """Decide if a filename is safe for use as a file system path with tools."""
+    if os.name == "nt" and type(filename) is unicode:
+        value = (
+            unicodedata.normalize("NFKD", filename)
+            .encode("ascii", "ignore")
+            .decode("ascii")
+        )
+
+        return value == filename
+    else:
+        return True
+
+
+def makeFilesystemEncodable(filename):
+    if not os.path.isabs(filename):
+        filename = os.path.abspath(filename)
+
+    if not isFilesystemEncodable(filename):
+        filename = getExternalUsePath(filename)
+
+    return filename
+
+
+def openPickleFile(filename, mode, protocol=-1):
+    file_handle = openTextFile(filename, mode)
+
+    if python_version < 0x300:
+        return file_handle, pickle.Pickler(file_handle, protocol)
+    else:
+        return file_handle, pickle._Pickler(  # pylint: disable=protected-access
+            file_handle, protocol
+        )
+
+
+def isLegalPath(path):
+    illegal_suffixes = "/\\"
+    illegal_chars = "\0"
+
+    if isWin32Windows():
+        illegal_chars += r'*"/<>:|?'
+
+        illegal_chars += "".join(chr(x) for x in range(1, 32))
+        illegal_suffixes += " ."
+
+    if isMacOS():
+        illegal_chars += ":"
+
+    for c in path:
+        if c in illegal_chars:
+            return False, "contains illegal character %r" % c
+
+    for illegal_suffix in illegal_suffixes:
+        if path.endswith(illegal_suffix):
+            return False, "contains illegal suffix %r" % illegal_suffix
+
+        for part in path.split(os.path.sep):
+            if part == ".":
+                continue
+
+            if part.endswith(illegal_suffix):
+                return False, "contains illegal suffix %r in path part %r" % (
+                    part,
+                    illegal_suffix,
+                )
+
+    return True, None
+
+
+def getParentDirectories(path):
+    """Get all parent directories of a path in descending order."""
+
+    while 1:
+        old_path = path
+        path = os.path.dirname(path)
+
+        if not path or path == old_path:
+            return
+
+        yield path
+
+
+def getNormalizedPath(path):
+    """Return normalized path that is also a native path, i.e. only legal characters.
+
+    Needed, because MSYS2 likes to keep "/" in normalized paths.
+    """
+    path = os.path.normpath(path)
+    if isWin32Windows():
+        path = path.replace("/", "\\")
+
+    return path
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.