Nuitka-winsvc 1.8__cp311-cp311-win_amd64.whl → 2.5.1__cp311-cp311-win_amd64.whl

nuitka/tools/testing/Common.py

@@ -1,28 +1,12 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ Common test infrastructure functions. To be used by test runners. """
 
 import ast
 import atexit
 import gc
-import hashlib
 import os
-import re
 import shutil
 import signal
 import sys
@@ -32,7 +16,8 @@ import time
 from contextlib import contextmanager
 from optparse import OptionParser
 
-from nuitka.__past__ import subprocess
+from nuitka.__past__ import md5, subprocess
+from nuitka.Options import getCommercialVersion
 from nuitka.PythonVersions import getTestExecutionPythonVersions, isDebugPython
 from nuitka.Tracing import OurLogger, my_print
 from nuitka.tree.SourceHandling import readSourceCodeFromFilename
@@ -40,6 +25,7 @@ from nuitka.utils.AppDirs import getCacheDir
 from nuitka.utils.Execution import (
     check_output,
     createProcess,
+    executeProcess,
     getNullInput,
     getNullOutput,
 )
@@ -49,6 +35,7 @@ from nuitka.utils.FileOperations import (
     getFileContentByLine,
     getFileContents,
     getFileList,
+    getParentDirectories,
     isFilenameSameAsOrBelowPath,
     makePath,
     openTextFile,
@@ -56,7 +43,7 @@ from nuitka.utils.FileOperations import (
 )
 from nuitka.utils.InstalledPythons import findInstalledPython
 from nuitka.utils.Jinja2 import getTemplate
-from nuitka.utils.Utils import getOS, isMacOS, isWin32Windows
+from nuitka.utils.Utils import isFreeBSD, isLinux, isMacOS, isWin32Windows
 
 from .SearchModes import (
     SearchModeByPattern,
@@ -220,11 +207,16 @@ def getTempDir():
                 os.path.dirname(os.path.abspath(sys.modules["__main__"].__file__))
             )
             + "-",
-            dir=tempfile.gettempdir() if not os.path.exists("/var/tmp") else "/var/tmp",
+            dir=None if not (isLinux() and os.path.exists("/var/tmp")) else "/var/tmp",
         )
 
         def removeTempDir():
-            removeDirectory(path=tmp_dir, ignore_errors=True)
+            removeDirectory(
+                path=tmp_dir,
+                logger=test_logger,
+                ignore_errors=True,
+                extra_recommendation=None,
+            )
 
         atexit.register(removeTempDir)
 
@@ -236,8 +228,9 @@ def convertUsing2to3(path, force=False):
 
     if not force:
         if "xrange" not in getFileContents(path):
-            if check_result(command, stderr=getNullOutput()):
-                return path, False
+            with getNullOutput() as null_output:
+                if check_result(command, stderr=null_output):
+                    return path, False
 
     filename = os.path.basename(path)
 
@@ -266,16 +259,17 @@ def convertUsing2to3(path, force=False):
 
     command += ("-w", "-n", "--no-diffs", new_path)
 
-    try:
-        check_output(command, stderr=getNullOutput())
+    with getNullOutput() as null_output:
+        try:
+            check_output(command, stderr=null_output)
 
-    except subprocess.CalledProcessError:
-        if isWin32Windows():
-            raise
+        except subprocess.CalledProcessError:
+            if isWin32Windows():
+                raise
 
-        command[0:3] = ["2to3"]
+            command[0:3] = ["2to3"]
 
-        check_output(command, stderr=getNullOutput())
+            check_output(command, stderr=null_output)
 
     data = getFileContents(new_path)
 
@@ -305,6 +299,10 @@ def decideFilenameVersionSkip(filename):
 
     assert type(filename) is str, repr(filename)
 
+    # There are commercial only test cases
+    if "commercial" in filename.lower() and getCommercialVersion() is None:
+        return False
+
     # Skip runner scripts by default.
     if filename.startswith("run_"):
         return False
@@ -324,6 +322,10 @@ def decideFilenameVersionSkip(filename):
     if filename.endswith("_37.py") and _python_version >= (3, 8):
         return False
 
+    # Skip tests that require Python 3.11 at maximum.
+    if filename.endswith("_311.py") and _python_version >= (3, 8):
+        return False
+
     # Skip tests that require Python 3.2 at least.
     if filename.endswith("32.py") and _python_version < (3, 2):
         return False
@@ -364,18 +366,27 @@ def decideFilenameVersionSkip(filename):
     if filename.endswith("311.py") and _python_version < (3, 11):
         return False
 
-    return True
+    # Skip tests that require Python 3.12 at least.
+    if filename.endswith("312.py") and _python_version < (3, 12):
+        return False
 
+    # Skip tests that require Python 3.13 at least.
+    if filename.endswith("313.py") and _python_version < (3, 13):
+        return False
 
-def decideNeeds2to3(filename):
-    return _python_version >= (3,) and not re.match(r".*3\d+\.py", filename)
+    return True
 
 
 def _removeCPythonTestSuiteDir():
     # Cleanup, some tests apparently forget that.
     try:
         if os.path.isdir("@test"):
-            removeDirectory("@test", ignore_errors=False)
+            removeDirectory(
+                "@test",
+                logger=test_logger,
+                ignore_errors=False,
+                extra_recommendation=None,
+            )
         elif os.path.isfile("@test"):
             os.unlink("@test")
     except OSError:
@@ -390,9 +401,7 @@ def _removeCPythonTestSuiteDir():
             raise
 
 
-def compareWithCPython(
-    dirname, filename, extra_flags, search_mode, needs_2to3, on_error=None
-):
+def compareWithCPython(dirname, filename, extra_flags, search_mode, on_error=None):
     """Call the comparison tool. For a given directory filename.
 
     The search mode decides if the test case aborts on error or gets extra
@@ -400,19 +409,11 @@ def compareWithCPython(
 
     """
 
-    # Many cases to consider here, pylint: disable=too-many-branches
-
     if dirname is None:
         path = filename
     else:
         path = os.path.join(dirname, filename)
 
-    # Apply 2to3 conversion if necessary.
-    if needs_2to3:
-        path, converted = convertUsing2to3(path)
-    else:
-        converted = False
-
     if os.getenv("NUITKA_TEST_INSTALLED", "") == "1":
         command = [
             sys.executable,
@@ -450,9 +451,6 @@ def compareWithCPython(
 
         search_mode.onErrorDetected("Error exit! %s" % result)
 
-    if converted:
-        os.unlink(path)
-
     if result == 2:
         test_logger.sysexit("Interrupted, with CTRL-C\n", exit_code=2)
 
@@ -477,9 +475,13 @@ def checkCompilesNotWithCPython(dirname, filename, search_mode):
 def checkSucceedsWithCPython(filename):
     command = [_python_executable, filename]
 
-    result = subprocess.call(command, stdout=getNullOutput(), stderr=subprocess.STDOUT)
+    stdout, stderr, exit_code = executeProcess(command)
 
-    return result == 0
+    if exit_code != 0:
+        my_print("stdout", stdout)
+        my_print("stderr", stderr)
+
+    return exit_code == 0
 
 
 def getDebugPython():
@@ -527,7 +529,7 @@ def displayRuntimeTraces(logger, path):
 
     if os.name == "posix":
         # Run with traces to help debugging, specifically in CI environment.
-        if getOS() in ("Darwin", "FreeBSD"):
+        if isMacOS() or isFreeBSD():
             test_logger.info("dtruss:")
             os.system("sudo dtruss %s" % path)
         else:
@@ -536,11 +538,12 @@ def displayRuntimeTraces(logger, path):
 
 
 def hasModule(module_name):
-    result = subprocess.call(
-        (os.environ["PYTHON"], "-c", "import %s" % module_name),
-        stdout=getNullOutput(),
-        stderr=subprocess.STDOUT,
-    )
+    with getNullOutput() as null_output:
+        result = subprocess.call(
+            (os.environ["PYTHON"], "-c", "import %s" % module_name),
+            stdout=null_output,
+            stderr=subprocess.STDOUT,
+        )
 
     return result == 0
 
@@ -584,7 +587,7 @@ def snapObjRefCntMap(before):
             k = "str_overload_" + x.__class__.__name__ + str(x)
         elif type(x) is dict:
             if "__builtins__" in x:
-                k = "<module dict %s>" % x["__name__"]
+                k = "<module dict %s>" % x.get("__name__", "<maybe not a module>")
             elif "__spec__" in x and "__name__" in x:
                 k = "<module dict %s>" % x["__name__"]
             else:
@@ -717,7 +720,7 @@ def checkReferenceCount(checked_function, max_rounds=20, explain=False):
                     my_print("extra:", m1[key], key)
                 elif m1[key] != m2[key]:
                     my_print("*" * 80)
-                    my_print(m1[key], "->", m2[key], key)
+                    my_print(m1[key], "->", m2[key], repr(key))
                 else:
                     pass
 
@@ -738,6 +741,7 @@ def checkReferenceCount(checked_function, max_rounds=20, explain=False):
 
 def createSearchMode():
     # Dealing with many options, pylint: disable=too-many-branches
+    # Return driven, pylint: disable=too-many-return-statements
 
     parser = OptionParser()
 
@@ -759,6 +763,14 @@ Execute only tests matching the pattern. Defaults to all tests.""",
         help="""\
 Execute all tests, continue execution even after failure of one.""",
     )
+    select_group.add_option(
+        "--jobs",
+        action="store",
+        dest="jobs",
+        default=None,
+        help="""\
+Pass this as the --jobs value to Nuitka.""",
+    )
 
     del select_group
 
@@ -790,6 +802,9 @@ Defaults to off.""",
     if options.debug:
         addExtendedExtraOptions("--debug")
 
+    if options.jobs is not None:
+        addExtendedExtraOptions("--jobs=%s" % options.jobs)
+
     if options.show_commands:
         os.environ["NUITKA_TRACE_COMMANDS"] = "1"
 
@@ -811,7 +826,9 @@ Defaults to off.""",
         else:
             return SearchModeImmediate()
     elif mode == "resume":
-        return SearchModeResume(sys.modules["__main__"].__file__)
+        return SearchModeResume(sys.modules["__main__"].__file__, skip=False)
+    elif mode == "skip":
+        return SearchModeResume(sys.modules["__main__"].__file__, skip=True)
     elif mode == "only":
         if options.pattern:
             pattern = options.pattern.replace("/", os.path.sep)
@@ -820,9 +837,9 @@ Defaults to off.""",
             assert False
     elif mode == "coverage":
         return SearchModeCoverage(
-            start_at=options.pattern.replace("/", os.path.sep)
-            if options.pattern
-            else None
+            start_at=(
+                options.pattern.replace("/", os.path.sep) if options.pattern else None
+            )
         )
     else:
         test_logger.sysexit("Error, using unknown search mode %r" % mode)
@@ -840,7 +857,7 @@ def executeReferenceChecked(
 ):
     gc.disable()
 
-    extract_number = lambda name: int(name.replace(prefix, ""))
+    extract_number = lambda name: int(name.replace(prefix, "") or "0")
 
     # Find the function names.
     matching_names = tuple(
@@ -861,20 +878,19 @@ def executeReferenceChecked(
             continue
 
         # Avoid non-raisable output.
-        try:
-            if number in tests_stderr:
-                sys.stderr = getNullOutput()
-        except OSError:  # Windows
-            if not checkReferenceCount(names[name], explain=explain):
-                result = False
-        else:
-            if not checkReferenceCount(names[name], explain=explain):
-                result = False
-
-            if number in tests_stderr:
-                new_stderr = sys.stderr
-                sys.stderr = old_stderr
-                new_stderr.close()
+        with getNullOutput() as null_output:
+            try:
+                if number in tests_stderr:
+                    sys.stderr = null_output
+            except OSError:  # Windows
+                if not checkReferenceCount(names[name], explain=explain):
+                    result = False
+            else:
+                if not checkReferenceCount(names[name], explain=explain):
+                    result = False
+            finally:
+                if number in tests_stderr:
+                    sys.stderr = old_stderr
 
     gc.enable()
     return result
@@ -925,7 +941,7 @@ def withPythonPathChange(python_path):
 
 
 def addExtendedExtraOptions(*args):
-    old_value = os.environ.get("NUITKA_EXTRA_OPTIONS")
+    old_value = os.getenv("NUITKA_EXTRA_OPTIONS")
 
     value = old_value
 
@@ -956,7 +972,8 @@ def withExtendedExtraOptions(*args):
 
 def indentedCode(codes, count):
     """Indent code, used for generating test codes."""
-    return "\n".join(" " * count + line if line else "" for line in codes)
+    indent = " " * count
+    return "\n".join(indent + line if line else "" for line in codes)
 
 
 def convertToPython(doctests, line_filter=None):
@@ -1179,19 +1196,15 @@ def async_iterate(g):
 
 
 def getTestingCacheDir():
-    cache_dir = getCacheDir()
-
-    result = os.path.join(cache_dir, "tests_state")
-    makePath(result)
-    return result
+    cache_dir = getCacheDir("tests_state")
+    makePath(cache_dir)
+    return cache_dir
 
 
 def getTestingCPythonOutputsCacheDir():
-    cache_dir = getCacheDir()
+    cache_dir = getCacheDir("cpython_outputs")
 
-    result = os.path.join(
-        cache_dir, "cpython_outputs", os.environ.get("NUITKA_TEST_SUITE", "")
-    )
+    result = os.path.join(cache_dir, os.getenv("NUITKA_TEST_SUITE", ""))
 
     makePath(result)
     return result
@@ -1203,7 +1216,7 @@ def scanDirectoryForTestCases(dirname, template_context=None):
     filenames = [
         filename
         for filename in filenames
-        if (filename.endswith(".py") and not filename + ".j2" in filenames)
+        if (filename.endswith(".py") and filename + ".j2" not in filenames)
         or filename.endswith(".j2")
     ]
 
@@ -1263,19 +1276,20 @@ def setupCacheHashSalt(test_code_path):
     if os.path.exists(os.path.join(test_code_path, ".git")):
         git_cmd = ["git", "ls-tree", "-r", "HEAD", test_code_path]
 
-        process = subprocess.Popen(
-            args=git_cmd,
-            stdin=getNullInput(),
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        )
+        with getNullInput() as null_input:
+            process = subprocess.Popen(
+                args=git_cmd,
+                stdin=null_input,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
 
         stdout_git, stderr_git = process.communicate()
         assert process.returncode == 0, stderr_git
 
-        salt_value = hashlib.md5(stdout_git)
+        salt_value = md5(stdout_git)
     else:
-        salt_value = hashlib.md5()
+        salt_value = md5()
 
         for filename in getFileList(test_code_path):
             if filename.endswith(".py"):
@@ -1330,20 +1344,24 @@ def checkTestRequirements(filename):
 
         if os.path.isfile(candidate):
             filename = candidate
+        else:
+            filename = os.path.join(filename, getMainProgramFilename(filename))
 
     for line in readSourceCodeFromFilename(None, filename).splitlines():
         if line.startswith("# nuitka-skip-unless-"):
             if line[21:33] == "expression: ":
                 expression = line[33:]
-                result = subprocess.call(
-                    (
-                        os.environ["PYTHON"],
-                        "-c",
-                        "import sys, os; sys.exit(not bool(%s))" % expression,
-                    ),
-                    stdout=getNullOutput(),
-                    stderr=subprocess.STDOUT,
-                )
+
+                with getNullOutput() as null_output:
+                    result = subprocess.call(
+                        (
+                            os.environ["PYTHON"],
+                            "-c",
+                            "import sys, os; sys.exit(not bool(%s))" % expression,
+                        ),
+                        stdout=null_output,
+                        stderr=subprocess.STDOUT,
+                    )
                 if result != 0:
                     return (False, "Expression '%s' evaluated to false" % expression)
 
@@ -1399,6 +1417,7 @@ def checkLoadedFileAccesses(loaded_filenames, current_dir):
     current_dir = os.path.normpath(current_dir)
     current_dir = os.path.normcase(current_dir)
     current_dir_ext = os.path.normcase(getExternalUsePath(current_dir))
+    current_dir_real = os.path.realpath(current_dir)
 
     illegal_accesses = []
 
@@ -1443,6 +1462,17 @@ def checkLoadedFileAccesses(loaded_filenames, current_dir):
         if loaded_filename.startswith(current_dir_ext):
             continue
 
+        if loaded_filename.startswith(current_dir_real):
+            continue
+
+        # Parent directories are OK too
+        if (
+            loaded_filename in getParentDirectories(current_dir)
+            or loaded_filename in getParentDirectories(current_dir_ext)
+            or loaded_filename in getParentDirectories(current_dir_real)
+        ):
+            continue
+
         ignore = True
         for ignored_dir in (
             # System configuration is OK
@@ -1463,6 +1493,7 @@ def checkLoadedFileAccesses(loaded_filenames, current_dir):
             # Themes may of course be loaded.
             "/usr/share/themes",
             # Terminal info files are OK too.
+            "/usr/share/terminfo",
             "/lib/terminfo",
         ):
             if isFilenameSameAsOrBelowPath(ignored_dir, loaded_filename):
@@ -1620,7 +1651,14 @@ def checkLoadedFileAccesses(loaded_filenames, current_dir):
         # Loading from home directories is OK too.
         if any(
             isFilenameSameAsOrBelowPath(path, loaded_filename)
-            for path in ("/home", "/data", "/root", "/Users", "/Library/Preferences")
+            for path in (
+                "/home",
+                "/data",
+                "/root",
+                "/Users",
+                "/Library/Preferences",
+                "/agent",  # Azure
+            )
         ):
             continue
 
@@ -1762,10 +1800,12 @@ def checkLoadedFileAccesses(loaded_filenames, current_dir):
 
         # macOS uses these:
         if loaded_basename in (
+            "libc.dylib",
             "libcrypto.1.0.0.dylib",
             "libssl.1.0.0.dylib",
             "libcrypto.1.1.dylib",
             "libffi.dylib",
+            "libfribidi.dylib",
         ):
             continue
 
@@ -1780,13 +1820,11 @@ def checkLoadedFileAccesses(loaded_filenames, current_dir):
         if isMacOS():
             ignore = True
             for ignored_dir in (
-                "/System/Library/PrivateFrameworks",
-                "/System/Library/CoreServices",
-                "/System/Library/Frameworks/",
-                "/System/Library/dyld",
+                "/System/Library",
                 "/AppleInternal",
                 "/System/Volumes/Preboot",
                 "/usr/lib/system/",
+                "/usr/libexec/swift",
             ):
                 if isFilenameSameAsOrBelowPath(ignored_dir, loaded_filename):
                     ignore = False
@@ -1794,6 +1832,9 @@ def checkLoadedFileAccesses(loaded_filenames, current_dir):
             if not ignore:
                 continue
 
+            if loaded_filename == "/System/Library":
+                continue
+
             if loaded_filename == "/usr/libexec/rosetta/runtime":
                 continue
 
@@ -1884,7 +1925,11 @@ def getLocalWebServerDir(base_dir):
     web_dir = os.path.join(getTempDir(), "local-web-server", base_dir)
 
     if _web_server_process is None:
-        web_server_directory_supporting_pythons = ("3.11", "3.10", "3.9", "3.8", "3.7")
+        web_server_directory_supporting_pythons = tuple(
+            python_version
+            for python_version in getTestExecutionPythonVersions()
+            if python_version not in ("3.6", "3.5", "3.4", "2.7", "2.6")
+        )
 
         web_server_python = findInstalledPython(
             python_versions=web_server_directory_supporting_pythons,
@@ -1935,3 +1980,19 @@ def extractNuitkaVersionFromFilePath(version_filename):
     (version_line,) = [line for line in option_lines if line.startswith("Nuitka V")]
 
     return version_line[8:].rstrip()
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/tools/testing/Constructs.py

@@ -1,20 +1,6 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ Tools for construct tests.
 
 """
@@ -49,3 +35,19 @@ def generateConstructCases(construct_source_code):
             case = 1
 
     return "\n".join(case_1), "\n".join(case_2)
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.