Nuitka-winsvc 1.8.6__cp312-cp312-win_amd64.whl → 1.9.3__cp312-cp312-win_amd64.whl

nuitka/tools/testing/Common.py

@@ -20,7 +20,6 @@
 import ast
 import atexit
 import gc
-import hashlib
 import os
 import re
 import shutil
@@ -32,7 +31,7 @@ import time
 from contextlib import contextmanager
 from optparse import OptionParser
 
-from nuitka.__past__ import subprocess
+from nuitka.__past__ import md5, subprocess
 from nuitka.PythonVersions import getTestExecutionPythonVersions, isDebugPython
 from nuitka.Tracing import OurLogger, my_print
 from nuitka.tree.SourceHandling import readSourceCodeFromFilename
@@ -1273,9 +1272,9 @@ def setupCacheHashSalt(test_code_path):
         stdout_git, stderr_git = process.communicate()
         assert process.returncode == 0, stderr_git
 
-        salt_value = hashlib.md5(stdout_git)
+        salt_value = md5(stdout_git)
     else:
-        salt_value = hashlib.md5()
+        salt_value = md5()
 
         for filename in getFileList(test_code_path):
             if filename.endswith(".py"):

nuitka/tools/testing/OutputComparison.py

@@ -144,7 +144,30 @@ def makeDiffable(output, ignore_warnings, syntax_errors):
         line = thread_re.sub(r"Thread 0xXXXXXXXX", line)
         line = compiled_types_re.sub(r"\1", line)
         line = global_name_error_re.sub(r"\1\2\3", line)
+
         line = module_repr_re.sub(r"\1xxxxx\2", line)
+
+        # Frozen modules of 3.11, _imp._frozen_module_names
+        for module_name in (
+            "zipimport",
+            "abc",
+            "codecs",
+            "io",
+            "_collections_abc",
+            "_sitebuiltins",
+            "genericpath",
+            "ntpath",
+            "posixpath",
+            "os.path",
+            "os",
+            "site",
+            "stat",
+        ):
+            line = line.replace(
+                "<module '%s' (frozen)>" % module_name,
+                "<module '%s' from 'xxxxx'>" % module_name,
+            )
+
         line = non_ascii_error_rt.sub(r"\1 xxxx", line)
         line = timing_re.sub(r"in x.xx seconds", line)
 

nuitka/tools/testing/SearchModes.py

@@ -21,10 +21,10 @@ The test runner can handle found errors, skip tests, etc. with search
 modes, which are implemented here.
 """
 
-import hashlib
 import os
 import sys
 
+from nuitka.__past__ import md5
 from nuitka.utils.FileOperations import (
     areSamePaths,
     getFileContents,
@@ -133,7 +133,7 @@ class SearchModeResume(SearchModeBase):
             tests_path = tests_path.encode("utf8")
             version = version.encode("utf8")
 
-        case_hash = hashlib.md5(tests_path)
+        case_hash = md5(tests_path)
         case_hash.update(version)
 
         from .Common import getTestingCacheDir

nuitka/tools/testing/compare_with_cpython/__main__.py

@@ -21,15 +21,14 @@
 
 """
 
-import hashlib
 import os
 import pickle
 import re
 import sys
 import time
 
+from nuitka.__past__ import md5
 from nuitka.OptionParsing import getNuitkaProjectOptions
-from nuitka.PythonVersions import python_version
 from nuitka.tools.testing.Common import (
     addToPythonPath,
     executeAfterTimePassed,
@@ -137,7 +136,7 @@ def getCPythonResults(cpython_cmd, cpython_cached, force_update, send_kill):
         if str is not bytes:
             hash_input = hash_input.encode("utf8")
 
-        command_hash = hashlib.md5(hash_input)
+        command_hash = md5(hash_input)
 
         for element in cpython_cmd:
             if os.path.exists(element):
@@ -150,7 +149,7 @@ def getCPythonResults(cpython_cmd, cpython_cached, force_update, send_kill):
             hash_salt = hash_salt.encode("utf8")
         command_hash.update(hash_salt)
 
-        if os.name == "nt" and python_version < 0x300:
+        if os.name == "nt" and str is bytes:
             curdir = os.getcwdu()  # spell-checker: ignore getcwdu
         else:
             curdir = os.getcwd()
@@ -264,6 +263,10 @@ def main():
     include_packages = hasArgValues("--include-package")
     include_modules = hasArgValues("--include-module")
     python_flag_m = hasArg("--python-flag=-m")
+    python_version = hasArgValue("--python-version")
+
+    if python_version:
+        python_version = tuple(int(d) for d in python_version.split("."))
 
     plugins_enabled = []
     for count, arg in reversed(tuple(enumerate(args))):
@@ -386,6 +389,9 @@ Taking coverage of '{filename}' using '{python}' with flags {args} ...""".format
             % (repr(os.path.dirname(filename)), mini_script),
         ]
 
+        if python_version and python_version >= (3, 11):
+            cpython_cmd += ["-X", "frozen_modules=off"]
+
         if no_warnings:
             cpython_cmd[1:1] = [
                 "-W",
@@ -400,6 +406,9 @@ Taking coverage of '{filename}' using '{python}' with flags {args} ...""".format
                 "ignore",
             ]
 
+        if python_version and python_version >= (3, 11):
+            cpython_cmd += ["-X", "frozen_modules=off"]
+
         if python_flag_m:
             cpython_cmd += ["-m", os.path.basename(filename)]
             os.chdir(os.path.dirname(filename))

nuitka/tools/testing/measure_construct_performance/__main__.py

@@ -25,11 +25,11 @@ in comparisons.
 
 """
 
-import hashlib
 import os
 import sys
 from optparse import OptionParser
 
+from nuitka.__past__ import md5
 from nuitka.tools.testing.Common import (
     check_output,
     convertUsing2to3,
@@ -111,10 +111,7 @@ def main():
 
     my_print("PYTHON='%s'" % getPythonVersionString())
     my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
-    my_print(
-        "TEST_CASE_HASH='%s'"
-        % hashlib.md5(getFileContents(test_case, "rb")).hexdigest()
-    )
+    my_print("TEST_CASE_HASH='%s'" % md5(getFileContents(test_case, "rb")).hexdigest())
 
     needs_2to3 = decideNeeds2to3(test_case)
 

nuitka/tools/watch/__main__.py

@@ -26,11 +26,17 @@ import sys
 from optparse import OptionParser
 
 from nuitka.containers.OrderedDicts import OrderedDict
+from nuitka.PythonFlavors import isAnacondaPython, isMSYS2MingwPython
 from nuitka.PythonVersions import getTestExecutionPythonVersions
 from nuitka.tools.testing.Common import extractNuitkaVersionFromFilePath
 from nuitka.Tracing import OurLogger
 from nuitka.TreeXML import fromFile
-from nuitka.utils.Execution import check_call, executeProcess
+from nuitka.utils.Execution import (
+    check_call,
+    executeProcess,
+    executeToolChecked,
+    withEnvironmentVarOverridden,
+)
 from nuitka.utils.FileOperations import (
     changeTextFileContents,
     getFileContents,
@@ -44,18 +50,17 @@ from nuitka.utils.Hashing import getFileContentsHash
 from nuitka.utils.InstalledPythons import findPythons
 from nuitka.utils.Utils import isLinux, isMacOS, isWin32Windows
 from nuitka.utils.Yaml import parseYaml
-
-# TODO: Command line interface
-nuitka_update_mode = "newer"
+from nuitka.Version import parseNuitkaVersionToTuple
 
 watch_logger = OurLogger("", base_style="blue")
 
 
-def _compareNuitkaVersions(version_a, version_b):
-    def _numberize(version):
-        return tuple(int(d) for d in version.split("rc")[0].split("."))
+def _compareNuitkaVersions(version_a, version_b, consider_rc):
+    if not consider_rc:
+        version_a = version_a.split("rc")[0]
+        version_b = version_b.split("rc")[0]
 
-    return _numberize(version_a) < _numberize(version_b)
+    return parseNuitkaVersionToTuple(version_a) < parseNuitkaVersionToTuple(version_b)
 
 
 def scanCases(path):
@@ -70,10 +75,13 @@ def scanCases(path):
                 yield case
 
 
-def selectPythons(python_version_req, anaconda):
+def selectPythons(python_version_req, anaconda, msys2_mingw64):
     for _python_version_str, installed_python_for_version in installed_pythons.items():
         for installed_python in installed_python_for_version:
-            if not anaconda and installed_python.isAnacondaPython():
+            if anaconda and not installed_python.isAnacondaPython():
+                continue
+
+            if msys2_mingw64 and not installed_python.isMSYS2MingwPython():
                 continue
 
             if python_version_req is not None:
@@ -90,14 +98,27 @@ def selectPythons(python_version_req, anaconda):
 
 
 def selectOS(os_values):
+    # return driven, pylint: disable=too-many-return-statements
+
     for value in os_values:
-        if value not in ("Linux", "Win32", "macOS"):
+        if value not in ("Linux", "Win32", "macOS", "Win32-MSYS2", "Win32-Anaconda"):
             watch_logger.sysexit("Illegal value for OS: %s" % value)
 
     if isLinux() and "Linux" in os_values:
         return "Linux"
-    if isWin32Windows() and "Win32" in os_values:
-        return "Win32"
+    if isWin32Windows():
+        if isMSYS2MingwPython():
+            if "Win32-MSYS2" in os_values:
+                return "Win32-MSYS2"
+
+            return None
+        if isAnacondaPython():
+            if "Win32-Anaconda" in os_values:
+                return "Win32-Anaconda"
+
+            return None
+        if "Win32" in os_values:
+            return "Win32"
     if isMacOS() and "macOS" in os_values:
         return "macOS"
 
@@ -181,6 +202,35 @@ python_version = "%(python_version)s"
     return changed_pipenv_file, pipenv_filename
 
 
+def _updatePacmanFile(installed_python, case_data, dry_run, result_path):
+    pipenv_filename = os.path.join(result_path, "Pacman.txt")
+    pipenv_package_requirements = []
+
+    for requirement in getPlatformRequirements(
+        installed_python=installed_python, case_data=case_data
+    ):
+        # Ignore spaces in requirements.
+        requirement = requirement.replace(" ", "")
+
+    # TODO: Other indexes, e.g. nvidia might be needed too
+    changed_pipenv_file = changeTextFileContents(
+        pipenv_filename,
+        """\
+[python]
+%(python_version)s
+[packages]
+%(pipenv_package_requirements)s
+"""
+        % {
+            "pipenv_package_requirements": "\n".join(pipenv_package_requirements),
+            "python_version": installed_python.getPythonVersion(),
+        },
+        compare_only=dry_run,
+    )
+
+    return changed_pipenv_file, pipenv_filename
+
+
 def _updatePipenvLockFile(
     installed_python, dry_run, pipenv_filename_full, no_pipenv_update
 ):
@@ -191,6 +241,17 @@ def _updatePipenvLockFile(
                 % pipenv_filename_full
             )
 
+            check_call(
+                [
+                    installed_python.getPythonExe(),
+                    "-m",
+                    "pipenv",
+                    "install",
+                    "--python",
+                    installed_python.getPythonExe(),
+                ]
+            )
+
         elif not dry_run:
             watch_logger.info(
                 "Working with pipenv file '%s' to update virtualenv, may take a while."
@@ -224,10 +285,31 @@ def _updatePipenvLockFile(
             ]
         )
 
+    return "Pipfile.lock"
+
+
+def _updatePacmanLockFile():
+    pacman_lock_filename = "Pacman.lock"
+
+    pacman_output = executeToolChecked(
+        logger=watch_logger,
+        command=["pacman", "-Q"],
+        absence_message="needs pacman to query package status on MSYS2",
+    )
+
+    if str is not bytes:
+        pacman_output = pacman_output.decode("utf8")
+
+    changeTextFileContents(filename=pacman_lock_filename, contents=pacman_output)
+
+    return pacman_lock_filename
 
-def _compileCase(case_data, case_dir, installed_python):
-    check_call(
-        [
+
+def _compileCase(case_data, case_dir, installed_python, lock_filename):
+    preferred_package_type = installed_python.getPreferredPackageType()
+
+    if preferred_package_type == "pip":
+        run_command = [
             installed_python.getPythonExe(),
             "-m",
             "pipenv",
@@ -235,14 +317,29 @@ def _compileCase(case_data, case_dir, installed_python):
             "--python",
             installed_python.getPythonExe(),
             "python",
-            nuitka_binary,
-            os.path.join(case_dir, case_data["filename"]),
-            "--report=compilation-report.xml",
-            "--report-diffable",
-            "--report-user-provided=pipenv_hash=%s"
-            % getFileContentsHash("Pipfile.lock"),
         ]
-    )
+        extra_options = []
+    elif preferred_package_type == "pacman":
+        run_command = ["python"]
+
+        # TODO: Bug in MSYS2 ccache, avoid using that.
+        extra_options = ["--disable-ccache"]
+    else:
+        assert False
+
+    with withEnvironmentVarOverridden("NUITKA_LAUNCH_TOKEN", "1"):
+        check_call(
+            run_command
+            + [
+                nuitka_binary,
+                os.path.join(case_dir, case_data["filename"]),
+                "--report=compilation-report.xml",
+                "--report-diffable",
+                "--report-user-provided=pipenv_hash=%s"
+                % getFileContentsHash(lock_filename),
+            ]
+            + extra_options
+        )
 
     if case_data["interactive"] == "no":
         binaries = getFileList(
@@ -254,7 +351,7 @@ def _compileCase(case_data, case_dir, installed_python):
         if len(binaries) != 1:
             sys.exit("Error, failed to identify created binary.")
 
-        stdout, stderr, exit_nuitka = executeProcess([binaries[0]])
+        stdout, stderr, exit_nuitka = executeProcess([binaries[0]], timeout=5 * 60)
 
         if exit_nuitka != 0:
             sys.exit(
@@ -268,40 +365,63 @@ def _compileCase(case_data, case_dir, installed_python):
 
 
 def _updateCase(
-    case_dir, case_data, dry_run, no_pipenv_update, installed_python, result_path
+    case_dir,
+    case_data,
+    dry_run,
+    no_pipenv_update,
+    nuitka_update_mode,
+    installed_python,
+    result_path,
 ):
+    # Many details and cases due to package method being handled here.
+    # pylint: disable=too-many-branches,too-many-locals
+
     # Not good for dry run, but tough life.
     makePath(result_path)
 
     # Update the pipenv file in any case, ought to be stable but we follow
     # global changes this way.
-    changed_pipenv_file, pipenv_filename = _updatePipenvFile(
-        installed_python=installed_python,
-        case_data=case_data,
-        dry_run=dry_run,
-        result_path=result_path,
-    )
+    preferred_package_type = installed_python.getPreferredPackageType()
+    if preferred_package_type == "pip":
+        changed_pipenv_file, pipenv_filename = _updatePipenvFile(
+            installed_python=installed_python,
+            case_data=case_data,
+            dry_run=dry_run,
+            result_path=result_path,
+        )
 
-    pipenv_filename_full = os.path.join(case_dir, pipenv_filename)
+        pipenv_filename_full = os.path.join(case_dir, pipenv_filename)
 
-    if dry_run and changed_pipenv_file:
-        watch_logger.info("Would create pipenv file '%s'." % pipenv_filename_full)
-        return
+        if dry_run and changed_pipenv_file:
+            watch_logger.info("Would create pipenv file '%s'." % pipenv_filename_full)
+            return
 
-    with withDirectoryChange(result_path):
-        # Update or create lockfile of pipenv.
-        _updatePipenvLockFile(
+        with withDirectoryChange(result_path):
+            # Update or create lockfile of pipenv.
+            lock_filename = _updatePipenvLockFile(
+                installed_python=installed_python,
+                dry_run=dry_run,
+                pipenv_filename_full=pipenv_filename_full,
+                no_pipenv_update=no_pipenv_update,
+            )
+    elif preferred_package_type == "pacman":
+        changed_pipenv_file, pipenv_filename = _updatePacmanFile(
             installed_python=installed_python,
+            case_data=case_data,
             dry_run=dry_run,
-            pipenv_filename_full=pipenv_filename_full,
-            no_pipenv_update=no_pipenv_update,
+            result_path=result_path,
         )
 
-        # Check if compilation is required.
+        with withDirectoryChange(result_path):
+            # Update or create lockfile of pipenv.
+            lock_filename = _updatePacmanLockFile()
+
+    # Check if compilation is required.
+    with withDirectoryChange(result_path):
         if os.path.exists("compilation-report.xml"):
             old_report_root = fromFile("compilation-report.xml")
 
-            existing_hash = getFileContentsHash("Pipfile.lock")
+            existing_hash = getFileContentsHash(lock_filename)
             old_report_root_hash = (
                 old_report_root.find("user-data").find("pipenv_hash").text
             )
@@ -311,7 +431,9 @@ def _updateCase(
             if nuitka_update_mode == "force":
                 need_compile = True
             elif nuitka_update_mode == "newer":
-                if _compareNuitkaVersions(old_nuitka_version, nuitka_version):
+                if _compareNuitkaVersions(
+                    old_nuitka_version, nuitka_version, consider_rc=True
+                ):
                     need_compile = True
                 else:
                     if existing_hash != old_report_root_hash:
@@ -349,10 +471,11 @@ def _updateCase(
                 case_data=case_data,
                 case_dir=case_dir,
                 installed_python=installed_python,
+                lock_filename=lock_filename,
             )
 
 
-def updateCase(case_dir, case_data, dry_run, no_pipenv_update):
+def updateCase(case_dir, case_data, dry_run, no_pipenv_update, nuitka_update_mode):
     case_name = case_data["case"]
 
     # Wrong OS maybe.
@@ -364,13 +487,15 @@ def updateCase(case_dir, case_data, dry_run, no_pipenv_update):
 
     # Too old Nuitka version maybe.
     if nuitka_min_version is not None and _compareNuitkaVersions(
-        nuitka_version, nuitka_min_version
+        nuitka_version, nuitka_min_version, consider_rc=False
     ):
         return
 
     # For all relevant Pythons applicable to this case.
     for installed_python in selectPythons(
-        anaconda=case_data["anaconda"] == "yes",
+        # TODO: Enable Anaconda support through options/detection.
+        anaconda="Anaconda" in os_name,
+        msys2_mingw64="MSYS2" in os_name,
         python_version_req=case_data.get("python_version_req"),
     ):
         watch_logger.info("Consider with Python %s." % installed_python)
@@ -386,18 +511,20 @@ def updateCase(case_dir, case_data, dry_run, no_pipenv_update):
             case_data=case_data,
             dry_run=dry_run,
             no_pipenv_update=no_pipenv_update,
+            nuitka_update_mode=nuitka_update_mode,
             installed_python=installed_python,
             result_path=result_path,
         )
 
 
-def updateCases(case_dir, dry_run, no_pipenv_update):
+def updateCases(case_dir, dry_run, no_pipenv_update, nuitka_update_mode):
     for case_data in parseYaml(getFileContents("case.yml", mode="rb")):
         updateCase(
             case_dir=case_dir,
             case_data=case_data,
             dry_run=dry_run,
             no_pipenv_update=no_pipenv_update,
+            nuitka_update_mode=nuitka_update_mode,
         )
 
 
@@ -415,15 +542,6 @@ def main():
 
     parser = OptionParser()
 
-    parser.add_option(
-        "--dry-run",
-        action="store_false",
-        dest="dry_run",
-        default=False,
-        help="""\
-Do not change anything, just report what would be done. Default %default.""",
-    )
-
     parser.add_option(
         "--python-version",
         action="append",
@@ -451,6 +569,25 @@ Nuitka binary to compile with. Defaults to one near the nuitka-watch usage.""",
 Do not update the pipenv environment. Best to see only effect of Nuitka update. Default %default.""",
     )
 
+    parser.add_option(
+        "--dry-run",
+        action="store_false",
+        dest="dry_run",
+        default=False,
+        help="""\
+Do not change anything, just report what would be done. Not yet perfectly true. Default %default.""",
+    )
+
+    parser.add_option(
+        "--nuitka-update-mode",
+        action="store",
+        choices=("newer", "force", "never"),
+        dest="nuitka_update_mode",
+        default="newer",
+        help="""\
+Recompile even if the versions seems not changed. Default %default.""",
+    )
+
     options, positional_args = parser.parse_args()
 
     assert len(positional_args) <= 1, positional_args
@@ -490,6 +627,7 @@ Do not update the pipenv environment. Best to see only effect of Nuitka update.
                     os.path.dirname(case_filename),
                     dry_run=options.dry_run,
                     no_pipenv_update=options.no_pipenv_update,
+                    nuitka_update_mode=options.nuitka_update_mode,
                 )
 
 

nuitka/tree/Building.py

@@ -824,7 +824,10 @@ def buildParseTree(provider, ast_tree, source_ref, is_module, is_main):
             statements.append(
                 StatementExpressionOnly(
                     expression=makeExpressionImportModuleFixed(
-                        module_name="site", source_ref=source_ref
+                        using_module_name=provider.getParentModule().getFullName(),
+                        module_name="site",
+                        value_name="site",
+                        source_ref=source_ref,
                     ),
                     source_ref=source_ref,
                 )
@@ -837,7 +840,10 @@ def buildParseTree(provider, ast_tree, source_ref, is_module, is_main):
                 statements.append(
                     StatementExpressionOnly(
                         expression=makeExpressionImportModuleFixed(
-                            module_name=path_imported_name, source_ref=source_ref
+                            using_module_name=provider.getParentModule().getFullName(),
+                            module_name=path_imported_name,
+                            value_name=path_imported_name.getTopLevelPackageName(),
+                            source_ref=source_ref,
                         ),
                         source_ref=source_ref,
                     )