Nuitka-winsvc 1.8__cp311-cp311-win_amd64.whl → 2.5.1__cp311-cp311-win_amd64.whl

nuitka/importing/Importing.py

@@ -1,20 +1,6 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ Locating modules and package source on disk.
 
 The actual import of a module would already execute code that changes things.
@@ -35,27 +21,31 @@ it's from the standard library, one can abuse the attribute ``__file__`` of the
 
 """
 
-
 import collections
-import hashlib
 import os
 import sys
 import zipfile
 
-from nuitka import Options, SourceCodeReferences
+from nuitka import SourceCodeReferences
 from nuitka.__past__ import iter_modules
 from nuitka.containers.Namedtuples import makeNamedtupleClass
 from nuitka.containers.OrderedSets import OrderedSet
-from nuitka.importing import StandardLibrary
 from nuitka.plugins.Plugins import Plugins
 from nuitka.PythonFlavors import isNuitkaPython
 from nuitka.PythonVersions import python_version
 from nuitka.Tracing import my_print, recursion_logger
 from nuitka.tree.ReformulationMultidist import locateMultidistModule
 from nuitka.utils.AppDirs import getCacheDir
-from nuitka.utils.FileOperations import listDir, removeDirectory
+from nuitka.utils.FileOperations import (
+    getNormalizedPath,
+    listDir,
+    removeDirectory,
+)
+from nuitka.utils.Hashing import getFileContentsHash
 from nuitka.utils.Importing import (
+    builtin_module_names,
     getModuleFilenameSuffixes,
+    getPackageDirFilename,
     getSharedLibrarySuffixes,
     isBuiltinModuleName,
 )
@@ -63,12 +53,53 @@ from nuitka.utils.ModuleNames import ModuleName
 from nuitka.utils.SharedLibraries import (
     hasUniversalOrMatchingMacOSArchitecture,
 )
-from nuitka.utils.Utils import isMacOS, isWin32OrPosixWindows
+from nuitka.utils.Utils import (
+    getLaunchingNuitkaProcessEnvironmentValue,
+    isMacOS,
+    isWin32OrPosixWindows,
+)
 
 from .IgnoreListing import isIgnoreListedNotExistingModule
 from .PreloadedPackages import getPreloadedPackagePath, isPreloadedPackagePath
+from .StandardLibrary import isStandardLibraryPath
+
+# Debug traces, enabled via --explain-imports
+_debug_module_finding = None
+
+# Preference as expressed via --prefer-source-code
+_prefer_source_code_over_extension_modules = None
+
+
+def setupImportingFromOptions():
+    """Set up the importing layer from giving options."""
+
+    # Should only be used inside of here.
+    from nuitka import Options
+
+    # singleton, pylint: disable=global-statement
+    global _debug_module_finding
+    _debug_module_finding = Options.shallExplainImports()
+
+    global _prefer_source_code_over_extension_modules
+    _prefer_source_code_over_extension_modules = (
+        Options.shallPreferSourceCodeOverExtensionModules()
+    )
+
+    # Lets try and have this complete, please report failures.
+    if Options.is_debug and not isNuitkaPython():
+        _checkRaisingBuiltinComplete()
+
+    main_filenames = Options.getMainEntryPointFilenames()
+    for filename in main_filenames:
+        # Inform the importing layer about the main script directory, so it can use
+        # it when attempting to follow imports.
+        addMainScriptDirectory(main_dir=os.path.dirname(os.path.abspath(filename)))
+
+
+def _checkRaisingBuiltinComplete():
+    for module_name in builtin_module_names:
+        assert module_name in _stdlib_module_raises, module_name
 
-_debug_module_finding = Options.shallExplainImports()
 
 warned_about = set()
 
@@ -108,6 +139,38 @@ def makeModuleUsageAttempt(
     )
 
 
+def makeParentModuleUsagesAttempts(module_usage_attempt):
+    result = []
+
+    for parent_package_name in module_usage_attempt.module_name.getParentPackageNames():
+        (
+            _parent_package_name,
+            parent_module_filename,
+            parent_module_kind,
+            parent_module_finding,
+        ) = locateModule(
+            module_name=parent_package_name,
+            parent_package=None,
+            level=0,
+        )
+
+        result.append(
+            makeModuleUsageAttempt(
+                module_name=parent_package_name,
+                filename=parent_module_filename,
+                finding=parent_module_finding,
+                module_kind=parent_module_kind,
+                level=0,
+                source_ref=module_usage_attempt.source_ref,
+                reason="import path parent",
+            )
+        )
+
+    result.append(module_usage_attempt)
+
+    return tuple(result)
+
+
 def addMainScriptDirectory(main_dir):
     """Initialize the main script directory.
 
@@ -130,7 +193,7 @@ def hasMainScriptDirectory():
 def isPackageDir(dirname):
     """Decide if a directory is a package.
 
-    Before Python3.3 it's required to have a "__init__.py" file, but then
+    Before Python3 it's required to have a "__init__.py" file, but then
     it became impossible to decide, and for extra fun, there is also the
     extra packages provided via "*.pth" file tricks by "site.py" loading.
     """
@@ -174,6 +237,16 @@ def getModuleNameAndKindFromFilename(module_filename):
             )
 
     if os.path.isdir(module_filename):
+        package_filename = getPackageDirFilename(module_filename)
+
+        if package_filename is not None:
+            for suffix in getSharedLibrarySuffixes():
+                if package_filename.endswith(suffix):
+                    return (
+                        ModuleName(os.path.basename(module_filename)),
+                        "extension",
+                    )
+
         return ModuleName(os.path.basename(module_filename)), "py"
 
     return None, None
@@ -183,10 +256,10 @@ def isIgnoreListedImportMaker(source_ref):
     if isNuitkaPython():
         return True
 
-    return StandardLibrary.isStandardLibraryPath(source_ref.getFilename())
+    return isStandardLibraryPath(source_ref.getFilename())
 
 
-def warnAbout(importing, module_name, level, source_ref):
+def warnAboutNotFoundImport(importing, module_name, level, source_ref):
     # This probably should not be dealt with here
     if module_name == "":
         return
@@ -275,10 +348,19 @@ def findModule(module_name, parent_package, level):
         # TODO: Should give a warning and return not found if the levels
         # exceed the package name.
         if parent_package is not None:
-            parent_package = parent_package.getRelativePackageName(level)
+            parent_package = parent_package.getRelativePackageName(level - 1)
         else:
             return None, None, None, "not-found"
 
+    if level == 1 and not module_name:
+        # Not actually allowed, but we only catch that at run-time.
+        if parent_package is None:
+            return None, None, None, "not-found"
+
+        module_name = parent_package
+        parent_package = None
+        level = 0
+
     # Try relative imports first if we have a parent package.
     if level != 0 and parent_package is not None:
         if module_name:
@@ -313,37 +395,11 @@ def findModule(module_name, parent_package, level):
 
             return full_name.getPackageName(), module_filename, module_kind, "relative"
 
-    if level < 1 and module_name != "":
+    if level < 1 and module_name:
         module_name = normalizePackageName(module_name)
 
         package_name = module_name.getPackageName()
 
-        # Built-in module names must not be searched any further.
-        if module_name in sys.builtin_module_names:
-            if _debug_module_finding:
-                my_print(
-                    "findModule: Absolute imported module '%s' in as built-in':"
-                    % (module_name,)
-                )
-            return package_name, None, None, "built-in"
-
-        # Frozen module names are similar to built-in, but there is no list of
-        # them, therefore check loader name. Not useful at this time
-        # to make a difference with built-in.
-        if python_version >= 0x300 and module_name in sys.modules:
-            loader = getattr(sys.modules[module_name], "__loader__", None)
-
-            if (
-                loader is not None
-                and getattr(loader, "__name__", "") == "FrozenImporter"
-            ):
-                if _debug_module_finding:
-                    my_print(
-                        "findModule: Absolute imported module '%s' in as frozen':"
-                        % (module_name,)
-                    )
-                return package_name, None, None, "built-in"
-
         preloaded_path = getPreloadedPackagePath(module_name)
 
         if preloaded_path is not None:
@@ -393,10 +449,7 @@ def _reportCandidates(package_name, module_name, candidate, candidates):
         else module_name
     )
 
-    if (
-        candidate.priority == 1
-        and Options.shallPreferSourceCodeOverExtensionModules() is None
-    ):
+    if candidate.priority == 1 and _prefer_source_code_over_extension_modules is None:
         for c in candidates:
             # Don't compare to itself and don't consider unused bytecode a problem.
             if c is candidate or c.priority == 3:
@@ -454,7 +507,7 @@ def _findModuleInPath2(package_name, module_name, search_path):
     # Higher values are lower priority.
     priority_map = {
         "PY_COMPILED": 3,
-        "PY_SOURCE": 0 if Options.shallPreferSourceCodeOverExtensionModules() else 2,
+        "PY_SOURCE": 0 if _prefer_source_code_over_extension_modules else 2,
         "C_EXTENSION": 1,
     }
 
@@ -473,9 +526,6 @@ def _findModuleInPath2(package_name, module_name, search_path):
             found = False
 
             for suffix, module_type in getModuleFilenameSuffixes():
-                if module_type == "C_EXTENSION":
-                    continue
-
                 package_file_name = "__init__" + suffix
 
                 file_path = os.path.join(package_directory, package_file_name)
@@ -557,7 +607,11 @@ def _findModuleInPath2(package_name, module_name, search_path):
     if (
         found_candidate.module_type == "C_EXTENSION"
         and isMacOS()
-        and not hasUniversalOrMatchingMacOSArchitecture(found_candidate.full_path)
+        and not hasUniversalOrMatchingMacOSArchitecture(
+            getPackageDirFilename(found_candidate.full_path)
+            if os.path.isdir(found_candidate.full_path)
+            else found_candidate.full_path
+        )
     ):
         # Not usable for target architecture.
         raise ImportError
@@ -582,26 +636,31 @@ def _unpackPathElement(path_entry):
     if not path_entry:
         return "."  # empty means current directory
 
-    if os.path.isfile(path_entry) and path_entry.lower().endswith(".egg"):
-        if path_entry not in _egg_files:
-            with open(path_entry, "rb") as f:
-                checksum = hashlib.md5(f.read()).hexdigest()
-
-            target_dir = os.path.join(getCacheDir(), "egg-content", checksum)
-
-            if not os.path.exists(target_dir):
-                try:
-                    # Not all Python versions allow using with here, pylint: disable=consider-using-with
-                    zip_ref = zipfile.ZipFile(path_entry, "r")
-                    zip_ref.extractall(target_dir)
-                    zip_ref.close()
-                except BaseException:
-                    removeDirectory(target_dir, ignore_errors=True)
-                    raise
+    if os.path.isfile(path_entry):
+        if path_entry.lower().endswith((".egg", ".zip")):
+            if path_entry not in _egg_files:
+                checksum = getFileContentsHash(path_entry)
+
+                target_dir = os.path.join(getCacheDir("egg-content"), checksum)
+
+                if not os.path.exists(target_dir):
+                    try:
+                        # Not all Python versions allow using with here, pylint: disable=consider-using-with
+                        zip_ref = zipfile.ZipFile(path_entry, "r")
+                        zip_ref.extractall(target_dir)
+                        zip_ref.close()
+                    except BaseException:
+                        removeDirectory(
+                            target_dir,
+                            logger=recursion_logger,
+                            ignore_errors=True,
+                            extra_recommendation=None,
+                        )
+                        raise
 
-            _egg_files[path_entry] = target_dir
+                _egg_files[path_entry] = target_dir
 
-        return _egg_files[path_entry]
+            return _egg_files[path_entry]
 
     return path_entry
 
@@ -614,7 +673,8 @@ def getPythonUnpackedSearchPath():
 
 
 def getPackageSearchPath(package_name):
-    assert _main_paths
+    if not _main_paths:
+        return None
 
     if package_name is None:
         result = (
@@ -632,8 +692,9 @@ def getPackageSearchPath(package_name):
 
             if isPackageDir(package_dir):
                 result.append(package_dir)
-                # Hack for "uniconverter". TODO: Move this to plug-in decision. This
+                # Hack for "uniconvertor". TODO: Move this to plug-in decision. This
                 # fails the above test, but at run time should be a package.
+                # spell-checker: ignore uniconvertor
             elif package_name == "uniconvertor.app.modules":
                 result.append(package_dir)
 
@@ -671,14 +732,14 @@ def _findModuleInPath(module_name):
     # executed, while we normally search in PYTHONPATH after it was executed,
     # and on some systems, that fails.
     if package_name is None and module_name == "site":
-        candidate = os.environ.get("NUITKA_SITE_FILENAME", "")
+        candidate = getLaunchingNuitkaProcessEnvironmentValue("NUITKA_SITE_FILENAME")
 
         if candidate:
             return candidate, "py"
 
     # Free pass for built-in modules, they need not exist.
     if package_name is None and isBuiltinModuleName(module_name):
-        return None
+        return None, "built-in"
 
     search_path = getPackageSearchPath(package_name)
 
@@ -698,7 +759,7 @@ def _findModuleInPath(module_name):
             module_name if package_name is None else package_name + "." + module_name,
         )
 
-        return None
+        return None, None
 
     if _debug_module_finding:
         my_print("_findModuleInPath: _findModuleInPath2 gave", module_filename)
@@ -737,6 +798,8 @@ def _findModule(module_name):
         module_search_cache[key] = ImportError
         raise
 
+    # assert len(module_search_cache[key]) == 2, (module_name, module_search_cache[key])
+
     return module_search_cache[key]
 
 
@@ -747,9 +810,10 @@ def locateModule(module_name, parent_package, level):
     as with "__import__" built-in.
 
     Returns:
-        Returns a triple of module name the module has considering
+        Returns a tuple of module name the module has considering
         package containing it, and filename of it which can be a
-        directory for packages, and the location method used.
+        directory for packages, the module kind, and the finding
+        kind.
     """
 
     if module_name.isMultidistModuleName():
@@ -762,17 +826,14 @@ def locateModule(module_name, parent_package, level):
     )
 
     # Allowing ourselves to be lazy.
-    if module_filename is None:
-        module_kind = None
-    else:
-        assert module_kind is not None, module_filename
+    assert module_kind is not None or module_filename is None, module_name
 
     assert module_package is None or (
         type(module_package) is ModuleName and module_package != ""
-    ), repr(module_package)
+    ), ("Must not attempt to locate %r" % module_name)
 
     if module_filename is not None:
-        module_filename = os.path.normpath(module_filename)
+        module_filename = getNormalizedPath(module_filename)
 
         module_name, module_kind = getModuleNameAndKindFromFilename(module_filename)
         module_name = ModuleName.makeModuleNameInPackage(module_name, module_package)
@@ -817,7 +878,7 @@ def decideModuleSourceRef(filename, module_name, is_main, is_fake, logger):
     is_package = False
 
     if is_main and os.path.isdir(filename):
-        source_filename = os.path.join(filename, "__main__.py")
+        source_filename = getNormalizedPath(os.path.join(filename, "__main__.py"))
 
         if not os.path.isfile(source_filename):
             sys.stderr.write(
@@ -847,7 +908,7 @@ def decideModuleSourceRef(filename, module_name, is_main, is_fake, logger):
     elif isPackageDir(filename):
         is_package = True
 
-        source_filename = os.path.join(filename, "__init__.py")
+        source_filename = getNormalizedPath(os.path.join(filename, "__init__.py"))
 
         if not os.path.isfile(source_filename):
             source_ref = SourceCodeReferences.fromFilename(
@@ -856,7 +917,7 @@ def decideModuleSourceRef(filename, module_name, is_main, is_fake, logger):
             is_namespace = True
         else:
             source_ref = SourceCodeReferences.fromFilename(
-                filename=os.path.abspath(source_filename)
+                filename=getNormalizedPath(os.path.abspath(source_filename))
             )
 
     else:
@@ -872,3 +933,163 @@ def decideModuleSourceRef(filename, module_name, is_main, is_fake, logger):
         source_ref,
         source_filename,
     )
+
+
+# spell-checker: ignore _posixsubprocess,pyexpat,xxsubtype
+
+_stdlib_module_raises = {
+    "__builtin__": False,
+    "_abc": False,
+    "_ast": False,
+    "_bisect": False,
+    "_blake2": False,
+    "_bytesio": False,
+    "_codecs": False,
+    "_codecs_cn": False,
+    "_codecs_hk": False,
+    "_codecs_iso2022": False,
+    "_codecs_jp": False,
+    "_codecs_kr": False,
+    "_codecs_tw": False,
+    "_collections": False,
+    "_contextvars": False,
+    "_csv": False,
+    "_datetime": False,
+    "_elementtree": False,
+    "_fileio": False,
+    "_functools": False,
+    "_heapq": False,
+    "_hotshot": False,
+    "_imp": False,
+    "_io": False,
+    "_json": False,
+    "_locale": False,
+    "_lsprof": False,
+    "_md5": False,
+    "_multibytecodec": False,
+    "_opcode": False,
+    "_operator": False,
+    "_peg_parser": False,
+    "_pickle": False,
+    "_posixsubprocess": False,
+    "_random": False,
+    "_subprocess": False,
+    "_sha": False,  # TODO: Not entirely clear if that's true
+    "_sha1": False,
+    "_sha2": False,
+    "_sha256": False,
+    "_sha3": False,
+    "_sha512": False,
+    "_signal": False,
+    "_socket": False,
+    "_sre": False,
+    "_stat": False,
+    "_statistics": False,
+    "_string": False,
+    "_struct": False,
+    "_symtable": False,
+    "_thread": False,
+    "_tracemalloc": False,
+    "_tokenize": False,
+    "_typing": False,
+    "_warnings": False,
+    "_weakref": False,
+    "_winapi": False,
+    "_winreg": False,
+    "_xxsubinterpreters": False,
+    "array": False,
+    "atexit": False,
+    "audioop": False,
+    "binascii": False,
+    "builtins": False,
+    "cmath": False,
+    "cStringIO": False,
+    "cPickle": False,
+    "datetime": False,
+    "errno": False,
+    "exceptions": False,
+    "faulthandler": False,
+    "fcntl": False,
+    "future_builtins": False,
+    "gc": False,
+    "grp": False,
+    "itertools": False,
+    "imageop": False,
+    "imp": False,
+    "operator": False,
+    "marshal": False,
+    "math": False,
+    "mmap": False,
+    "msvcrt": False,
+    "nt": False,
+    "parser": False,
+    "posix": False,
+    "pwd": False,
+    "pyexpat": False,
+    "select": False,
+    "signal": False,
+    "strop": False,
+    "spwd": False,
+    "sys": False,
+    "syslog": False,
+    "time": False,
+    "thread": False,
+    "unicodedata": False,
+    "winreg": False,
+    "xxsubtype": False,
+    "zipimport": False,
+    "zlib": False,
+    "_ssl": True,
+    "_xxinterpchannels": False,
+    # TODO: 3.13 clarify if this is actually true
+    "_interpchannels": False,
+    "_interpreters": False,
+    "_interpqueues": False,
+    "_sysconfig": False,
+    "_suggestions": False,
+}
+
+
+def isNonRaisingBuiltinModule(module_name):
+    assert isBuiltinModuleName(module_name), module_name
+
+    # Return None, if we don't know.
+    return _stdlib_module_raises.get(module_name)
+
+
+def _getChildPackageNames(module_name):
+    module_name = ModuleName(module_name)
+
+    _module_name, module_filename, _module_kind, _finding = locateModule(
+        parent_package=None, module_name=module_name, level=0
+    )
+
+    package_dir = module_filename
+
+    for module_info in iter_modules([package_dir]):
+        child_name = module_name.getChildNamed(module_info.name)
+        yield child_name
+
+        if module_info.ispkg:
+            for sub_module_name in getChildPackageNames(child_name):
+                yield sub_module_name
+
+
+def getChildPackageNames(module_name):
+    return tuple(_getChildPackageNames(module_name))
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

nuitka/importing/PreloadedPackages.py

@@ -1,20 +1,6 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ This module abstracts what site.py is normally doing in .pth files.
 
 This tries to extract "namespaces" packages that were manually created and
@@ -27,6 +13,7 @@ import sys
 
 from nuitka.Tracing import recursion_logger
 from nuitka.utils.FileOperations import getFileContentByLine, listDir
+from nuitka.utils.ModuleNames import ModuleName
 
 
 def getLoadedPackages():
@@ -154,8 +141,24 @@ def setPthImportedPackages(value):
     # We need to set this from the outside, pylint: disable=global-statement
     global pth_imported_packages
 
-    pth_imported_packages = value
+    pth_imported_packages = tuple(ModuleName(module_name) for module_name in value)
 
 
 def getPthImportedPackages():
     return pth_imported_packages
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.