Nuitka-winsvc 1.8.6__cp312-cp312-win_amd64.whl → 1.9.3__cp312-cp312-win_amd64.whl

nuitka/plugins/standard/DllFilesPlugin.py

@@ -29,6 +29,11 @@ import sys
 from nuitka.Options import isStandaloneMode
 from nuitka.plugins.PluginBase import NuitkaPluginBase
 from nuitka.PythonVersions import python_version
+from nuitka.utils.Distributions import (
+    getDistributionFromModuleName,
+    getDistributionName,
+    isDistributionSystemPackage,
+)
 from nuitka.utils.FileOperations import (
     listDllFilesFromDirectory,
     listExeFilesFromDirectory,
@@ -82,6 +87,8 @@ class NuitkaPluginDllFiles(NuitkaPluginBase):
             exe = dll_config.get("executable", "no") == "yes"
 
             suffixes = dll_config.get("suffixes")
+            if suffixes is not None:
+                suffixes = tuple(suffix.lstrip(".") for suffix in suffixes)
 
             for prefix in dll_config.get("prefixes"):
                 if exe:
@@ -96,6 +103,7 @@ class NuitkaPluginDllFiles(NuitkaPluginBase):
                                     filename,
                                 )
                             ),
+                            module_name=full_name,
                             package_name=full_name,
                             reason="Yaml config of '%s'" % full_name.asString(),
                         )
@@ -111,6 +119,7 @@ class NuitkaPluginDllFiles(NuitkaPluginBase):
                                     filename,
                                 )
                             ),
+                            module_name=full_name,
                             package_name=full_name,
                             reason="Yaml config of '%s'" % full_name.asString(),
                         )
@@ -143,6 +152,7 @@ class NuitkaPluginDllFiles(NuitkaPluginBase):
             yield self.makeExeEntryPoint(
                 source_path=filename,
                 dest_path=dest_path,
+                module_name=full_name,
                 package_name=full_name,
                 reason="Yaml config of '%s'" % full_name.asString(),
             )
@@ -150,6 +160,7 @@ class NuitkaPluginDllFiles(NuitkaPluginBase):
             yield self.makeDllEntryPoint(
                 source_path=filename,
                 dest_path=dest_path,
+                module_name=full_name,
                 package_name=full_name,
                 reason="Yaml config of '%s'" % full_name.asString(),
             )
@@ -259,6 +270,7 @@ conditions are missing, or this version of the module needs treatment added."""
                 yield self.makeDllEntryPoint(
                     source_path=uuid_dll_path,
                     dest_path=os.path.basename(uuid_dll_path),
+                    module_name=full_name,
                     package_name=None,
                     reason="needed by uuid package",
                 )
@@ -271,6 +283,7 @@ conditions are missing, or this version of the module needs treatment added."""
             yield self.makeDllEntryPoint(
                 source_path=xtwrapper_dll_path,
                 dest_path=os.path.basename(xtwrapper_dll_path),
+                module_name=full_name,
                 package_name=None,
                 reason="needed by 'iptc'",
             )
@@ -280,6 +293,7 @@ conditions are missing, or this version of the module needs treatment added."""
                     module.getCompileTimeDirectory(), "libsecp256k1.dll"
                 ),
                 dest_path=os.path.join(full_name.getPackageName(), "libsecp256k1.dll"),
+                module_name=full_name,
                 package_name=full_name.getPackageName(),
                 reason="needed by 'coincurve._libsecp256k1'",
             )
@@ -333,6 +347,7 @@ conditions are missing, or this version of the module needs treatment added."""
                         yield self.makeDllEntryPoint(
                             source_path=pythoncom_dll_path,
                             dest_path=pythoncom_filename,
+                            module_name=full_name,
                             package_name=None,
                             reason="needed by '%s'" % full_name.asString(),
                         )
@@ -376,3 +391,33 @@ conditions are missing, or this version of the module needs treatment added."""
                 return True
 
         return None
+
+    def decideAllowOutsideDependencies(self, module_name):
+        distribution = None
+
+        assert module_name != "_ctypes", self.config.get(
+            module_name, section="import-hacks"
+        )
+
+        while 1:
+            for entry in self.config.get(module_name, section="import-hacks"):
+                if self.evaluateCondition(
+                    full_name=module_name, condition=entry.get("when", "True")
+                ):
+                    if "package-system-dlls" in entry:
+                        return entry["package-system-dlls"] == "yes"
+
+            if distribution is None:
+                distribution = getDistributionFromModuleName(module_name)
+
+            module_name = module_name.getPackageName()
+            if not module_name:
+                break
+
+        if distribution is None:
+            return None
+
+        if not isDistributionSystemPackage(getDistributionName(distribution)):
+            return False
+        else:
+            return None

nuitka/plugins/standard/GiPlugin.py

@@ -15,7 +15,7 @@
 #     See the License for the specific language governing permissions and
 #     limitations under the License.
 #
-""" Support for gi typelib files
+""" Support for gi typelib files and DLLs
 """
 import os
 
@@ -86,16 +86,29 @@ if not os.environ.get("GI_TYPELIB_PATH"):
 
     @standalone_only
     def getExtraDlls(self, module):
-        if module.getFullName() == "gi._gi":
-            gtk_dll_path = self.locateDLL("gtk-3")
-
-            if gtk_dll_path is None:
-                gtk_dll_path = self.locateDLL("gtk-3-0")
+        def tryLocateAndLoad(dll_name):
+            # Support various name forms in MSYS2 over time.
+            dll_path = self.locateDLL(dll_name)
+            if dll_path is None:
+                dll_path = self.locateDLL("%s" % dll_name)
+            if dll_path is None:
+                dll_path = self.locateDLL("lib%s" % dll_name)
 
-            if gtk_dll_path is not None:
+            if dll_path is not None:
                 yield self.makeDllEntryPoint(
-                    source_path=gtk_dll_path,
-                    dest_path=os.path.basename(gtk_dll_path),
-                    package_name=None,
+                    source_path=dll_path,
+                    dest_path=os.path.basename(dll_path),
+                    module_name="gi._gi",
+                    package_name="gi",
                     reason="needed by 'gi._gi'",
                 )
+
+        if module.getFullName() == "gi._gi":
+            # TODO: Get local relevant DLL names from GI
+            for dll_name in (
+                "gtk-3-0",
+                "soup-2.4-1",
+                "soup-gnome-2.4-1",
+                "libsecret-1-0",
+            ):
+                yield tryLocateAndLoad(dll_name)

nuitka/plugins/standard/GlfwPlugin.py

@@ -114,6 +114,7 @@ class NuitkaPluginGlfw(NuitkaPluginBase):
             yield self.makeDllEntryPoint(
                 source_path=dll_filename,
                 dest_path=os.path.join("glfw", os.path.basename(dll_filename)),
+                module_name="glfw",
                 package_name="glfw.library",
                 reason="needed by 'glfw'",
             )

nuitka/plugins/standard/ImplicitImports.py

@@ -22,10 +22,13 @@ be told that. This encodes the knowledge we have for various modules. Feel free
 to add to this and submit patches to make it more complete.
 """
 
+import ast
 import fnmatch
 import os
 
 from nuitka.__past__ import iter_modules
+from nuitka.importing.Importing import locateModule
+from nuitka.importing.Recursion import decideRecursion
 from nuitka.plugins.PluginBase import NuitkaPluginBase
 from nuitka.utils.ModuleNames import ModuleName
 from nuitka.utils.Utils import isMacOS, isWin32Windows
@@ -42,6 +45,8 @@ class NuitkaPluginImplicitImports(NuitkaPluginBase):
     def __init__(self):
         self.config = getYamlPackageConfiguration()
 
+        self.lazy_loader_usages = {}
+
     @staticmethod
     def isAlwaysEnabled():
         return True
@@ -68,19 +73,22 @@ class NuitkaPluginImplicitImports(NuitkaPluginBase):
                     module_name=ModuleName(current),
                 )
 
-                for sub_module in iter_modules([module_filename]):
-                    if not fnmatch.fnmatch(sub_module.name, part):
-                        continue
+                if module_filename is not None:
+                    for sub_module in iter_modules([module_filename]):
+                        if not fnmatch.fnmatch(sub_module.name, part):
+                            continue
 
-                    if count == len(parts) - 1:
-                        yield current.getChildNamed(sub_module.name)
-                    else:
-                        child_name = current.getChildNamed(sub_module.name).asString()
+                        if count == len(parts) - 1:
+                            yield current.getChildNamed(sub_module.name)
+                        else:
+                            child_name = current.getChildNamed(
+                                sub_module.name
+                            ).asString()
 
-                        for value in self._resolveModulePattern(
-                            child_name + "." + ".".join(parts[count + 1 :])
-                        ):
-                            yield value
+                            for value in self._resolveModulePattern(
+                                child_name + "." + ".".join(parts[count + 1 :])
+                            ):
+                                yield value
 
                 return
             else:
@@ -339,14 +347,18 @@ class NuitkaPluginImplicitImports(NuitkaPluginBase):
                 ):
                     yield item
 
-    def onModuleSourceCode(self, module_name, source_code):
+    def onModuleSourceCode(self, module_name, source_filename, source_code):
+        # Too much code here, pylint: disable=too-many-branches
+        # TODO: Move the ones that would be possible to yaml config,
+        # e.g. the numexpr hack.
+
         if module_name == "numexpr.cpuinfo":
             # We cannot intercept "is" tests, but need it to be "isinstance",
             # so we patch it on the file. TODO: This is only temporary, in
             # the future, we may use optimization that understands the right
             # hand size of the "is" argument well enough to allow for our
             # type too.
-            return source_code.replace(
+            source_code = source_code.replace(
                 "type(attr) is types.MethodType", "isinstance(attr, types.MethodType)"
             )
 
@@ -365,11 +377,159 @@ __file__ = (__nuitka_binary_dir + '%ssite.py') if '__nuitka_binary_dir' in dict(
                 "PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []"
             )
 
-            return source_code
+        # Source code should use lazy_loader, this may not be good enough
+        # for all things yet.
+        attach_call_replacements = (
+            (
+                "lazy.attach_stub(__name__, __file__)",
+                "lazy.attach('%(module_name)s', %(submodules)s, %(attrs)s)",
+            ),
+        )
+
+        for attach_call, attach_call_replacement in attach_call_replacements:
+            if attach_call in source_code:
+                result = self._handleLazyLoad(
+                    module_name=module_name,
+                    source_filename=source_filename,
+                )
+
+                # Inline the values, to avoid the data files.
+                if result is not None:
+                    replacement = attach_call_replacement % {
+                        "module_name": module_name.asString(),
+                        "submodules": tuple(
+                            sub_module_name.asString() for sub_module_name in result[0]
+                        ),
+                        "attrs": dict(
+                            (
+                                sub_module_name.getChildNameFromPackage(
+                                    module_name
+                                ).asString(),
+                                module_attributes,
+                            )
+                            for (sub_module_name, module_attributes) in sorted(
+                                result[1].items()
+                            )
+                        ),
+                    }
+
+                    source_code = source_code.replace(attach_call, replacement)
+
+        if module_name == "huggingface_hub":
+            # Special handling for huggingface that uses the source code variant
+            # of lazy module. spell-checker: ignore huggingface
+            if (
+                "__getattr__, __dir__, __all__ = _attach(__name__, submodules=[], submod_attrs=_SUBMOD_ATTRS)"
+                in source_code
+            ):
+                huggingface_hub_lazy_loader_info = self.queryRuntimeInformationSingle(
+                    setup_codes="import huggingface_hub",
+                    value="huggingface_hub._SUBMOD_ATTRS",
+                    info_name="huggingface_hub_lazy_loader",
+                )
+
+                self._addLazyLoader(
+                    module_name,
+                    submodules=(),
+                    submodule_attrs=dict(
+                        ("." + submodule_name, attributes)
+                        for (
+                            submodule_name,
+                            attributes,
+                        ) in huggingface_hub_lazy_loader_info.items()
+                    ),
+                )
+
+        if module_name == "pydantic":
+            # Pydantic has its own lazy loading, spell-checker: ignore pydantic
+            if "def __getattr__(" in source_code:
+                pydantic_info = self.queryRuntimeInformationSingle(
+                    setup_codes="import pydantic",
+                    value="pydantic._dynamic_imports",
+                    info_name="pydantic_lazy_loader",
+                )
+
+                pydantic_lazy_loader_info = {}
+                for key, value in pydantic_info.items():
+                    # Older pydantic had only a string for the attribute.
+                    if type(value) is tuple:
+                        value = "".join(value).rstrip(".")
+
+                    if value not in pydantic_lazy_loader_info:
+                        pydantic_lazy_loader_info[value] = []
+                    pydantic_lazy_loader_info[value].append(key)
+
+                self._addLazyLoader(
+                    module_name=module_name,
+                    submodules=(),
+                    submodule_attrs=pydantic_lazy_loader_info,
+                )
 
-        # Do nothing by default.
         return source_code
 
+    def _addLazyLoader(self, module_name, submodules, submodule_attrs):
+        """Add lazy loader information for a module.
+
+        Args:
+            module_name: name of the module to work on
+            submodules: list of attributes that are known submodules
+            submodule_attrs: dict of module name to list of attributes
+
+        Notes:
+            It converts to modules names on the fly. If in submodule_attr
+            the module name starts with a "." then it's relative to the
+            module_name value.
+
+        """
+
+        submodules = tuple(ModuleName(submodule) for submodule in submodules)
+
+        submodule_attrs = dict(
+            (
+                module_name.getChildNamed(submodule[1:])
+                if submodule.startswith(".")
+                else ModuleName(submodule),
+                tuple(attribute_names),
+            )
+            for (submodule, attribute_names) in sorted(submodule_attrs.items())
+        )
+
+        self.lazy_loader_usages[module_name] = (
+            submodules,
+            submodule_attrs,
+        )
+
+    def _handleLazyLoad(self, module_name, source_filename):
+        pyi_filename = source_filename + "i"
+
+        if os.path.exists(pyi_filename):
+            try:
+                import lazy_loader
+            except ImportError:
+                pass
+            else:
+                with open(pyi_filename, "rb") as f:
+                    stub_node = ast.parse(f.read())
+
+                # We are using private code here, to avoid use duplicating,
+                # pylint: disable=protected-access
+                visitor = lazy_loader._StubVisitor()
+                visitor.visit(stub_node)
+
+                self._addLazyLoader(
+                    module_name=module_name,
+                    submodules=visitor._submodules,
+                    submodule_attrs=dict(
+                        ("." + submodule_name, attributes)
+                        for (
+                            submodule_name,
+                            attributes,
+                        ) in visitor._submod_attrs.items()
+                    ),
+                )
+
+                return self.lazy_loader_usages[module_name]
+
     def createPreModuleLoadCode(self, module):
         full_name = module.getFullName()
 
@@ -442,3 +602,95 @@ __file__ = (__nuitka_binary_dir + '%ssite.py') if '__nuitka_binary_dir' in dict(
     def decideCompilation(self, module_name):
         if module_name.hasOneOfNamespaces(self.unworthy_namespaces):
             return "bytecode"
+
+    def onModuleUsageLookAhead(
+        self, module_name, module_filename, module_kind, get_module_source
+    ):
+        # Getting the source code will also trigger our modification
+        # and potentially tell us if any lazy loading applies.
+        if get_module_source() is None:
+            return
+
+        if module_name in self.lazy_loader_usages:
+            from nuitka.HardImportRegistry import (
+                addModuleAttributeFactory,
+                addModuleDynamicHard,
+                addModuleTrust,
+                trust_module,
+                trust_node,
+            )
+
+            addModuleDynamicHard(module_name)
+
+            sub_module_names, sub_module_attr = self.lazy_loader_usages[module_name]
+
+            for sub_module_name in sub_module_names:
+                addModuleTrust(module_name, sub_module_name, trust_module)
+
+                sub_module_name = module_name.getChildNamed(sub_module_name)
+                addModuleDynamicHard(sub_module_name)
+
+                _lookAhead(using_module_name=module_name, module_name=sub_module_name)
+
+            for (
+                sub_module_name,
+                attribute_names,
+            ) in sub_module_attr.items():
+                addModuleDynamicHard(sub_module_name)
+
+                _lookAhead(using_module_name=module_name, module_name=sub_module_name)
+
+                for attribute_name in attribute_names:
+                    addModuleTrust(module_name, attribute_name, trust_node)
+                    addModuleAttributeFactory(
+                        module_name,
+                        attribute_name,
+                        makeExpressionImportModuleNameHardExistsAfterImportFactory(
+                            sub_module_name=sub_module_name,
+                            attribute_name=attribute_name,
+                        ),
+                    )
+
+
+def makeExpressionImportModuleNameHardExistsAfterImportFactory(
+    sub_module_name,
+    attribute_name,
+):
+    from nuitka.HardImportRegistry import trust_node_factory
+    from nuitka.nodes.ImportHardNodes import (
+        ExpressionImportModuleNameHardExists,
+    )
+
+    key = (sub_module_name, attribute_name)
+    if key in trust_node_factory:
+        return lambda source_ref: trust_node_factory[key](source_ref=source_ref)
+
+    return lambda source_ref: ExpressionImportModuleNameHardExists(
+        module_name=sub_module_name,
+        import_name=attribute_name,
+        module_guaranteed=False,
+        source_ref=source_ref,
+    )
+
+
+def _lookAhead(using_module_name, module_name):
+    (
+        _module_name,
+        package_filename,
+        package_module_kind,
+        finding,
+    ) = locateModule(
+        module_name=module_name,
+        parent_package=None,
+        level=0,
+    )
+
+    assert module_name == _module_name
+
+    if finding != "not-found":
+        decideRecursion(
+            using_module_name=using_module_name,
+            module_filename=package_filename,
+            module_name=module_name,
+            module_kind=package_module_kind,
+        )

nuitka/plugins/standard/KivyPlugin.py

@@ -129,6 +129,7 @@ except ImportError:
                 yield self.makeDllEntryPoint(
                     source_path=full_path,
                     dest_path=target_filename,
+                    module_name=full_name,
                     package_name=full_name,
                     reason="needed by 'kivy'",
                 )

nuitka/plugins/standard/MatplotlibPlugin.py

@@ -24,6 +24,7 @@ from nuitka.plugins.Plugins import (
     getActiveQtPluginBindingName,
     hasActivePlugin,
 )
+from nuitka.utils.Execution import NuitkaCalledProcessError
 from nuitka.utils.FileOperations import getFileContentByLine
 from nuitka.utils.Jinja2 import renderTemplateFromString
 
@@ -68,9 +69,10 @@ class NuitkaPluginMatplotlib(NuitkaPluginBase):
             There might exist a local version outside 'matplotlib/mpl-data' which
             we then must use instead. Determine its name by asking matplotlib.
         """
-        info = self.queryRuntimeInformationMultiple(
-            info_name="matplotlib_info",
-            setup_codes="""
+        try:
+            info = self.queryRuntimeInformationMultiple(
+                info_name="matplotlib_info",
+                setup_codes="""
 from matplotlib import matplotlib_fname, get_backend, __version__
 try:
     from matplotlib import get_data_path
@@ -78,17 +80,24 @@ except ImportError:
     from matplotlib import _get_data_path as get_data_path
 from inspect import getsource
 """,
-            values=(
-                ("matplotlibrc_filename", "matplotlib_fname()"),
-                ("backend", "get_backend()"),
-                ("data_path", "get_data_path()"),
-                ("matplotlib_version", "__version__"),
-                (
-                    "needs_matplotlibdata_env",
-                    "'MATPLOTLIBDATA' in getsource(get_data_path) or 'MATPLOTLIBRC' in getsource(get_data_path)",
+                values=(
+                    ("matplotlibrc_filename", "matplotlib_fname()"),
+                    ("backend", "get_backend()"),
+                    ("data_path", "get_data_path()"),
+                    ("matplotlib_version", "__version__"),
                 ),
-            ),
-        )
+            )
+        except NuitkaCalledProcessError:
+            if "MPLBACKEND" not in os.environ:
+                self.sysexit(
+                    """\
+Error, failed to detect matplotlib backend. Please set 'MPLBACKEND' \
+environment variable during compilation.""",
+                    mnemonic="""\
+https://matplotlib.org/stable/users/installing/environment_variables_faq.html#envvar-MPLBACKEND""",
+                )
+
+            raise
 
         if info is None:
             self.sysexit("Error, it seems 'matplotlib' is not installed or broken.")
@@ -106,6 +115,18 @@ from inspect import getsource
                 "mpl-data missing, matplotlib installation appears to be broken"
             )
 
+        self.info(
+            "Using %s backend '%s'."
+            % (
+                (
+                    "configuration file or default"
+                    if "MPLBACKEND" not in os.environ
+                    else "as per 'MPLBACKEND' environment variable"
+                ),
+                matplotlib_info.backend,
+            )
+        )
+
         # Include the "mpl-data" files.
         yield self.makeIncludedDataDirectory(
             source_path=matplotlib_info.data_path,
@@ -133,7 +154,7 @@ from inspect import getsource
                 # old config file has a backend definition
                 found = True
 
-        if not found and matplotlib_info.matplotlib_version < "3":
+        if not found and matplotlib_info.matplotlib_version < "4":
             # Set the backend, so even if it was run time determined, we now enforce it.
             new_lines.append("backend: %s" % matplotlib_info.backend)
 
@@ -152,12 +173,12 @@ from inspect import getsource
         # some special handling for matplotlib:
         # depending on whether 'tk-inter' resp. 'qt-plugins' are enabled,
         # their matplotlib backends are included.
-        if hasActivePlugin("tk-inter"):
-            if module_name in (
-                "matplotlib.backends.backend_tk",
-                "matplotlib.backends.backend_tkagg",
-                "matplotlib.backend.tkagg",
-            ):
+        if module_name in (
+            "matplotlib.backends.backend_tk",
+            "matplotlib.backends.backend_tkagg",
+            "matplotlib.backend.tkagg",
+        ):
+            if hasActivePlugin("tk-inter"):
                 return True, "Needed for tkinter matplotlib backend"
 
     def createPreModuleLoadCode(self, module):
@@ -175,16 +196,13 @@ from inspect import getsource
         """
 
         # The version may not need the environment variable.
-        if (
-            module.getFullName() == "matplotlib"
-            and self._getMatplotlibInfo().needs_matplotlibdata_env
-        ):
+        if module.getFullName() == "matplotlib":
             code = renderTemplateFromString(
                 r"""
 import os
 os.environ["MATPLOTLIBDATA"] = os.path.join(__nuitka_binary_dir, "matplotlib", "mpl-data")
 os.environ["MATPLOTLIBRC"] = os.path.join(__nuitka_binary_dir, "matplotlib", "mpl-data", "matplotlibrc")
-os.environ["MPLBACKEND"] = {{matplotlib_info.backend}}
+os.environ["MPLBACKEND"] = "{{matplotlib_info.backend}}"
 {% if qt_binding_name %}
 os.environ["QT_API"] = "{{qt_binding_name}}"
 {% endif %}

nuitka/plugins/standard/OptionsNannyPlugin.py

@@ -60,9 +60,7 @@ class NuitkaPluginOptionsNanny(NuitkaPluginBase):
             return
 
         if condition != "True":
-            problem_desc = (
-                "incomplete support due to not passing condition %s" % condition
-            )
+            problem_desc = "incomplete support due untrue condition '%s'" % condition
         else:
             problem_desc = "incomplete support"
 
@@ -105,10 +103,11 @@ Otherwise a terminal window will open"""
 
                 self.info(
                     """\
-Note, when using '%s', consider using '--disable-console' option. %s. However for \
-debugging, terminal output is the easiest way to see informative traceback \
+Note, when using '%s', consider using '--disable-console' option. %s. However\
+for debugging, terminal output is the easiest way to see informative traceback \
 and error information, so delay this until your program working and remove \
-once you find it non-working."""
+once you find it non-working, and use '--enable-console' to make it explicit \
+and not see this message."""
                     % (full_name, downside_message)
                 )
 

nuitka/plugins/standard/PkgResourcesPlugin.py

@@ -90,7 +90,7 @@ sys.exit(%(module_name)s.%(main_name)s)
             "main_name": main_name,
         }
 
-    def onModuleSourceCode(self, module_name, source_code):
+    def onModuleSourceCode(self, module_name, source_filename, source_code):
         if module_name == "__main__":
             match = re.search(
                 "\n# EASY-INSTALL-ENTRY-SCRIPT: '(.*?)','(.*?)','(.*?)'", source_code