Nuitka-winsvc 1.8__cp311-cp311-win_amd64.whl → 2.4.8__cp311-cp311-win_amd64.whl

nuitka/plugins/standard/AntiBloatPlugin.py

@@ -1,20 +1,6 @@
-#     Copyright 2023, Kay Hayen, mailto:kay.hayen@gmail.com
-#
-#     Part of "Nuitka", an optimizing Python compiler that is compatible and
-#     integrates with CPython, but also works on its own.
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
+#     Copyright 2024, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
 """ Standard plug-in to avoid bloat at compile time.
 
 Nuitka hard codes stupid monkey patching normally not needed here and avoids
@@ -23,23 +9,24 @@ that to be done and causing massive degradations.
 """
 
 import ast
-import os
 import re
-import sys
 
 from nuitka.containers.OrderedDicts import OrderedDict
 from nuitka.Errors import NuitkaForbiddenImportEncounter
 from nuitka.ModuleRegistry import getModuleByName
-from nuitka.plugins.PluginBase import NuitkaPluginBase
+from nuitka.Options import isExperimental
+from nuitka.plugins.YamlPluginBase import NuitkaYamlPluginBase
 from nuitka.utils.ModuleNames import ModuleName
-from nuitka.utils.Yaml import getYamlPackageConfiguration
 
 # spell-checker: ignore dask,numba,statsmodels,matplotlib,sqlalchemy,ipykernel
 
 _mode_choices = ("error", "warning", "nofollow", "allow")
 
 
-class NuitkaPluginAntiBloat(NuitkaPluginBase):
+class NuitkaPluginAntiBloat(NuitkaYamlPluginBase):
+    # Lots of details, a bunch of state is cached and tracked across functions
+    # pylint: disable=too-many-instance-attributes
+
     plugin_name = "anti-bloat"
     plugin_desc = (
         "Patch stupid imports out of widely used library modules source codes."
@@ -54,6 +41,7 @@ class NuitkaPluginAntiBloat(NuitkaPluginBase):
         noinclude_setuptools_mode,
         noinclude_pytest_mode,
         noinclude_unittest_mode,
+        noinclude_pydoc_mode,
         noinclude_ipython_mode,
         noinclude_dask_mode,
         noinclude_numba_mode,
@@ -61,7 +49,10 @@ class NuitkaPluginAntiBloat(NuitkaPluginBase):
         custom_choices,
         show_changes,
     ):
-        # Many details, due to many repetitive arguments, pylint: disable=too-many-branches,too-many-statements
+        # Many details, due to many repetitive arguments,
+        # pylint: disable=too-many-branches,too-many-locals,too-many-statements
+
+        NuitkaYamlPluginBase.__init__(self)
 
         self.show_changes = show_changes
 
@@ -72,6 +63,8 @@ class NuitkaPluginAntiBloat(NuitkaPluginBase):
             noinclude_pytest_mode = noinclude_default_mode
         if noinclude_unittest_mode is None:
             noinclude_unittest_mode = noinclude_default_mode
+        if noinclude_pydoc_mode is None:
+            noinclude_pydoc_mode = noinclude_default_mode
         if noinclude_ipython_mode is None:
             noinclude_ipython_mode = noinclude_default_mode
         if noinclude_dask_mode is None:
@@ -79,8 +72,6 @@ class NuitkaPluginAntiBloat(NuitkaPluginBase):
         if noinclude_numba_mode is None:
             noinclude_numba_mode = noinclude_default_mode
 
-        self.config = getYamlPackageConfiguration()
-
         self.handled_modules = OrderedDict()
 
         # These should be checked, to allow disabling anti-bloat contents.
@@ -92,6 +83,34 @@ class NuitkaPluginAntiBloat(NuitkaPluginBase):
                 noinclude_setuptools_mode,
                 "setuptools",
             )
+            self.handled_modules["triton"] = (
+                noinclude_setuptools_mode,
+                "setuptools",
+            )
+            self.handled_modules["Cython"] = (
+                noinclude_setuptools_mode,
+                "setuptools",
+            )
+            self.handled_modules["cython"] = (
+                noinclude_setuptools_mode,
+                "setuptools",
+            )
+            self.handled_modules["pyximport"] = (
+                noinclude_setuptools_mode,
+                "setuptools",
+            )
+            self.handled_modules["paddle.utils.cpp_extension"] = (
+                noinclude_setuptools_mode,
+                "setuptools",
+            )
+            self.handled_modules["torch.utils.cpp_extension"] = (
+                noinclude_setuptools_mode,
+                "setuptools",
+            )
+            self.handled_modules["numpy.distutils"] = (
+                noinclude_setuptools_mode,
+                "setuptools",
+            )
         else:
             self.control_tags["use_setuptools"] = True
 
@@ -109,6 +128,10 @@ class NuitkaPluginAntiBloat(NuitkaPluginBase):
                 noinclude_pytest_mode,
                 "pytest",
             )
+            self.handled_modules["distributed.utils_test"] = (
+                noinclude_pytest_mode,
+                "pytest",
+            )
         else:
             self.control_tags["use_pytest"] = True
 
@@ -124,6 +147,14 @@ class NuitkaPluginAntiBloat(NuitkaPluginBase):
                 noinclude_unittest_mode,
                 "unittest",
             )
+            self.handled_modules["keras.src.testing_infra"] = (
+                noinclude_unittest_mode,
+                "unittest",
+            )
+            self.handled_modules["tf_keras.src.testing_infra"] = (
+                noinclude_unittest_mode,
+                "unittest",
+            )
         else:
             self.control_tags["use_unittest"] = True
 
@@ -138,6 +169,10 @@ class NuitkaPluginAntiBloat(NuitkaPluginBase):
                 noinclude_ipython_mode,
                 "IPython",
             )
+            self.handled_modules["altair._magics"] = (
+                noinclude_ipython_mode,
+                "IPython",
+            )
         else:
             self.control_tags["use_ipython"] = True
 
@@ -147,6 +182,11 @@ class NuitkaPluginAntiBloat(NuitkaPluginBase):
         else:
             self.control_tags["use_dask"] = True
 
+        if noinclude_pydoc_mode != "allow":
+            self.handled_modules["pydoc"] = noinclude_pydoc_mode, "pydoc"
+        else:
+            self.control_tags["use_pydoc"] = True
+
         if noinclude_numba_mode != "allow":
             self.handled_modules["numba"] = noinclude_numba_mode, "numba"
             self.handled_modules["sparse"] = noinclude_numba_mode, "numba"
@@ -199,6 +239,12 @@ form 'module_name:[%s]'."""
         # information given for that.
         self.no_auto_follows = {}
 
+        # Keep track of modules prevented from being followed at all.
+        self.no_follows = OrderedDict()
+
+        # Cache execution context for anti-bloat configs.
+        self.context_codes = {}
+
     def getEvaluationConditionControlTags(self):
         return self.control_tags
 
@@ -257,6 +303,17 @@ What to do if a unittest import is encountered. This package can be big with
 dependencies, and should definitely be avoided.""",
         )
 
+        group.add_option(
+            "--noinclude-pydoc-mode",
+            action="store",
+            dest="noinclude_pydoc_mode",
+            choices=_mode_choices,
+            default=None,
+            help="""\
+What to do if a pydoc import is encountered. This package use is mark of useless
+code for deployments and should be avoided.""",
+        )
+
         group.add_option(
             "--noinclude-IPython-mode",
             action="store",
@@ -313,73 +370,64 @@ which can and should be a top level package and then one choice, "error",
 "warning", "nofollow", e.g. PyQt5:error.""",
         )
 
+    def _getContextCode(self, module_name, anti_bloat_config):
+        context_code = anti_bloat_config.get("context", "")
+        if type(context_code) in (tuple, list):
+            context_code = "\n".join(context_code)
+
+        if context_code not in self.context_codes:
+            context = {}
+
+            try:
+                # We trust the yaml files, pylint: disable=exec-used
+                exec(context_code, context)
+            except Exception as e:  # pylint: disable=broad-except
+                self.sysexit(
+                    """\
+Error, cannot exec module '%s', context code '%s' due to: %s"""
+                    % (module_name, context_code, e)
+                )
+
+            self.context_codes[context_code] = context
+
+        return dict(self.context_codes[context_code])
+
     def _onModuleSourceCode(self, module_name, anti_bloat_config, source_code):
-        # Complex dealing with many cases, pylint: disable=too-many-branches,too-many-locals,too-many-statements
+        # Complex dealing with many cases, pylint: disable=too-many-branches
 
         description = anti_bloat_config.get("description", "description not given")
 
         # To allow detection if it did anything.
         change_count = 0
 
-        context = {"sys": sys, "os": os}
-
-        context_code = anti_bloat_config.get("context", "")
-        if type(context_code) in (tuple, list):
-            context_code = "\n".join(context_code)
-        context["version"] = self.getPackageVersion
-
-        # We trust the yaml files, pylint: disable=eval-used,exec-used
-        context_ready = not bool(context_code)
-
-        for replace_src, replace_code in anti_bloat_config.get(
-            "replacements", {}
+        for replace_src, replace_code in (
+            anti_bloat_config.get("replacements") or {}
         ).items():
             # Avoid the eval, if the replace doesn't hit.
             if replace_src not in source_code:
                 continue
 
             if replace_code:
-                if not context_ready:
-                    try:
-                        exec(context_code, context)
-                    except Exception as e:  # pylint: disable=broad-except
-                        self.sysexit(
-                            """\
-Error, cannot exec module '%s', execute context code '%s' due to: %s"""
-                            % (module_name, context_code, e)
-                        )
-
-                    context_ready = True
-
-                if "__dirname__" in replace_code:
-                    context["__dirname__"] = self.locateModule(module_name)
-
-                try:
-                    replace_dst = eval(replace_code, context)
-                except Exception as e:  # pylint: disable=broad-except
-                    self.sysexit(
-                        """\
-Error, cannot eval module '%s' replacement expression code '%s' in '%s' due to: %s"""
-                        % (module_name, replace_code, context_code, e)
-                    )
+                replace_dst = self.evaluateExpression(
+                    full_name=module_name,
+                    expression=replace_code,
+                    config_name="module '%s' config 'replacements' " % module_name,
+                    extra_context=self._getContextCode(
+                        module_name=module_name, anti_bloat_config=anti_bloat_config
+                    ),
+                    single_value=True,
+                )
             else:
                 replace_dst = ""
 
-            if type(replace_dst) is not str:
-                self.sysexit(
-                    """\
-Error, module '%s' replacement expression code for '%s' needs to generate string, not %s"""
-                    % (module_name, replace_code, type(replace_dst))
-                )
-
             old = source_code
             source_code = source_code.replace(replace_src, replace_dst)
 
             if old != source_code:
                 change_count += 1
 
-        for replace_src, replace_dst in anti_bloat_config.get(
-            "replacements_plain", {}
+        for replace_src, replace_dst in (
+            anti_bloat_config.get("replacements_plain") or {}
         ).items():
             old = source_code
             source_code = source_code.replace(replace_src, replace_dst)
@@ -387,32 +435,31 @@ Error, module '%s' replacement expression code for '%s' needs to generate string
             if old != source_code:
                 change_count += 1
 
-        for replace_src, replace_dst in anti_bloat_config.get(
-            "replacements_re", {}
+        for replace_src, replace_dst in (
+            anti_bloat_config.get("replacements_re") or {}
         ).items():
             old = source_code
-            source_code = re.sub(replace_src, replace_dst, source_code)
+            source_code = re.sub(replace_src, replace_dst, source_code, re.S)
 
             if old != source_code:
                 change_count += 1
+            elif isExperimental("display-anti-bloat-mismatches"):
+                self.info("No match in %s no match %r" % (module_name, replace_src))
 
         append_code = anti_bloat_config.get("append_result", "")
         if type(append_code) in (tuple, list):
             append_code = "\n".join(append_code)
 
         if append_code:
-            if not context_ready:
-                exec(context_code, context)
-                context_ready = True
-
-            try:
-                append_result = eval(append_code, context)
-            except Exception as e:  # pylint: disable=broad-except
-                self.sysexit(
-                    """\
-Error, cannot evaluate module '%s' append code '%s' in '%s' due to: %s"""
-                    % (module_name, append_code, context_code, e)
-                )
+            append_result = self.evaluateExpression(
+                full_name=module_name,
+                expression=append_code,
+                config_name="module '%s' config 'append_code'" % module_name,
+                extra_context=self._getContextCode(
+                    module_name=module_name, anti_bloat_config=anti_bloat_config
+                ),
+                single_value=True,
+            )
 
             source_code += "\n" + append_result
             change_count += 1
@@ -446,7 +493,7 @@ Error, cannot evaluate module '%s' append code '%s' in '%s' due to: %s"""
 
         return source_code
 
-    def onModuleSourceCode(self, module_name, source_code):
+    def onModuleSourceCode(self, module_name, source_filename, source_code):
         for anti_bloat_config in self.config.get(module_name, section="anti-bloat"):
             if self.evaluateCondition(
                 full_name=module_name, condition=anti_bloat_config.get("when", "True")
@@ -462,14 +509,6 @@ Error, cannot evaluate module '%s' append code '%s' in '%s' due to: %s"""
     def _onFunctionBodyParsing(
         self, module_name, anti_bloat_config, function_name, body
     ):
-        context = {}
-        context_code = anti_bloat_config.get("context", "")
-        if type(context_code) in (tuple, list):
-            context_code = "\n".join(context_code)
-
-        # We trust the yaml files, pylint: disable=eval-used,exec-used
-        context_ready = not bool(context_code)
-
         replace_code = anti_bloat_config.get("change_function", {}).get(function_name)
 
         if replace_code == "un-callable":
@@ -481,18 +520,16 @@ Error, cannot evaluate module '%s' append code '%s' in '%s' due to: %s"""
         if replace_code is None:
             return False
 
-        if not context_ready:
-            exec(context_code, context)
-            context_ready = True
-
-        try:
-            replacement = eval(replace_code, context)
-        except Exception as e:  # pylint: disable=broad-except
-            self.sysexit(
-                """\
-Error, cannot eval module '%s' function '%s' replacement code '%s' in '%s' due to: %s"""
-                % (module_name, function_name, replace_code, context_code, e)
-            )
+        replacement = self.evaluateExpression(
+            full_name=module_name,
+            expression=replace_code,
+            config_name="module '%s' config 'change_function' of '%s'"
+            % (module_name, function_name),
+            extra_context=self._getContextCode(
+                module_name=module_name, anti_bloat_config=anti_bloat_config
+            ),
+            single_value=True,
+        )
 
         # Single node is required, extract the generated module body with
         # single expression only statement value or a function body.
@@ -533,6 +570,140 @@ Error, cannot eval module '%s' function '%s' replacement code '%s' in '%s' due t
 
         return result
 
+    def _onClassBodyParsing(self, module_name, anti_bloat_config, class_name, node):
+        replace_code = anti_bloat_config.get("change_class", {}).get(class_name)
+
+        if replace_code == "un-usable":
+            replace_code = """
+'''
+class %(class_name)s:
+    def __init__(self, *args, **kwargs):
+        raise RuntimeError("Must not call %(module_name)s.%(class_name)s")
+'''
+""" % {
+                "module_name": module_name,
+                "class_name": class_name,
+            }
+
+        if replace_code is None:
+            return False
+
+        replacement = self.evaluateExpression(
+            full_name=module_name,
+            expression=replace_code,
+            config_name="module '%s' config 'change_class' of '%s'"
+            % (module_name, class_name),
+            extra_context=self._getContextCode(
+                module_name=module_name, anti_bloat_config=anti_bloat_config
+            ),
+            single_value=True,
+        )
+
+        # Single node is required, extract the generated module body with
+        # single expression only statement value or a function body.
+        replacement = ast.parse(replacement).body[0]
+
+        node.body[:] = replacement.body
+        node.bases = replacement.bases
+
+        if self.show_changes:
+            self.info(
+                "Updated module '%s' class '%s'." % (module_name.asString(), class_name)
+            )
+
+        return True
+
+    def onClassBodyParsing(self, module_name, class_name, node):
+        result = False
+
+        config = self.config.get(module_name, section="anti-bloat")
+
+        if config:
+            for anti_bloat_config in config:
+                if self._onClassBodyParsing(
+                    module_name=module_name,
+                    anti_bloat_config=anti_bloat_config,
+                    class_name=class_name,
+                    node=node,
+                ):
+                    result = True
+
+        return result
+
+    def _getModuleBloatModeOverrides(self, using_module_name, intended_module_name):
+        # Finding a matching configuration aborts the search, not finding one
+        # means default behavior should apply.
+        for _config_module_name, bloat_mode_overrides in self.getYamlConfigItem(
+            module_name=using_module_name,
+            section="anti-bloat",
+            item_name="bloat-mode-overrides",
+            default={},
+            decide_relevant=(lambda config_item: intended_module_name in config_item),
+            recursive=True,
+        ):
+            return bloat_mode_overrides[intended_module_name]
+
+        return None
+
+    def decideAnnotations(self, module_name):
+        # Finding a matching configuration aborts the search, not finding one
+        # means default behavior should apply.
+        for _config_module_name, annotations_config_value in self.getYamlConfigItem(
+            module_name=module_name,
+            section="anti-bloat",
+            item_name="annotations",
+            default=None,
+            decide_relevant=(lambda config_item: config_item in ("yes", "no")),
+            recursive=True,
+        ):
+            return annotations_config_value == "yes"
+
+        return None
+
+    def decideDocStrings(self, module_name):
+        # Finding a matching configuration aborts the search, not finding one
+        # means default behavior should apply.
+        for _config_module_name, doc_strings_config_value in self.getYamlConfigItem(
+            module_name=module_name,
+            section="anti-bloat",
+            item_name="doc_strings",
+            default=None,
+            decide_relevant=(lambda config_item: config_item in ("yes", "no")),
+            recursive=True,
+        ):
+            return doc_strings_config_value == "yes"
+
+        return None
+
+    def decideAsserts(self, module_name):
+        # Finding a matching configuration aborts the search, not finding one
+        # means default behavior should apply.
+        for _config_module_name, asserts_config_value in self.getYamlConfigItem(
+            module_name=module_name,
+            section="anti-bloat",
+            item_name="asserts",
+            default=None,
+            decide_relevant=(lambda config_item: config_item in ("yes", "no")),
+            recursive=True,
+        ):
+            return asserts_config_value == "yes"
+
+        return None
+
+    def _applyNoFollowConfiguration(self, module_name):
+        for (
+            config_of_module_name,
+            no_follow_pattern,
+            description,
+        ) in self.getYamlConfigItemItems(
+            module_name=module_name,
+            section="anti-bloat",
+            item_name="no-follow",
+            decide_relevant=lambda key, value: True,
+            recursive=True,
+        ):
+            self.no_follows[no_follow_pattern] = (config_of_module_name, description)
+
     def onModuleRecursion(
         self,
         module_name,
@@ -542,6 +713,11 @@ Error, cannot eval module '%s' function '%s' replacement code '%s' in '%s' due t
         source_ref,
         reason,
     ):
+        # pylint: disable=too-many-branches
+
+        # First off, activate "no-follow" configurations of this module.
+        self._applyNoFollowConfiguration(module_name=module_name)
+
         # Do not even look at these. It's either included by a module that is in standard
         # library, or included for a module in standard library.
         if reason == "stdlib" or (
@@ -552,6 +728,8 @@ Error, cannot eval module '%s' function '%s' replacement code '%s' in '%s' due t
 
         # This will allow "unittest.mock" to pass "unittest". It's kind of a hack and
         # hopefully unusual.
+        if module_name == "unittest" and reason == "import path parent":
+            return
         if module_name == "unittest.mock" and module_name not in self.handled_modules:
             return
 
@@ -567,6 +745,15 @@ Error, cannot eval module '%s' function '%s' replacement code '%s' in '%s' due t
                 return
 
             if module_name.hasNamespace(handled_module_name):
+                if using_module_name is not None:
+                    override_mode = self._getModuleBloatModeOverrides(
+                        using_module_name=using_module_name,
+                        intended_module_name=intended_module_name,
+                    )
+
+                    if override_mode is not None:
+                        mode = override_mode
+
                 # Make sure the compilation aborts or warns if asked to
                 if mode == "error":
                     raise NuitkaForbiddenImportEncounter(
@@ -585,13 +772,20 @@ Error, cannot eval module '%s' function '%s' replacement code '%s' in '%s' due t
                     )
 
                     if key not in self.warnings_given:
+                        if handled_module_name == intended_module_name:
+                            handled_module_name_desc = "'%s'" % handled_module_name
+                        else:
+                            handled_module_name_desc = (
+                                "'%s' (intending to avoid '%s')"
+                                % (handled_module_name, intended_module_name)
+                            )
+
                         self.warning(
                             """\
-Undesirable import of '%s' (intending to avoid '%s') in \
-'%s' (at '%s') encountered. It may slow down compilation."""
+Undesirable import of %s in '%s' (at '%s') encountered. It may \
+slow down compilation."""
                             % (
-                                handled_module_name,
-                                intended_module_name,
+                                handled_module_name_desc,
                                 using_module_name,
                                 source_ref.getAsString(),
                             ),
@@ -623,25 +817,43 @@ Undesirable import of '%s' (intending to avoid '%s') in \
                     )
 
         if using_module_name is not None:
-            config = self.config.get(using_module_name, section="anti-bloat")
-
-            if config:
-                for anti_bloat_config in config:
-                    no_auto_follows = anti_bloat_config.get("no-auto-follow", {})
-
-                    for no_auto_follow, description in no_auto_follows.items():
-                        if module_name.hasNamespace(no_auto_follow):
-                            if self.evaluateCondition(
-                                full_name=module_name,
-                                condition=anti_bloat_config.get("when", "True"),
-                            ):
-                                self.no_auto_follows[no_auto_follow] = description
-
-                                return (
-                                    False,
-                                    "according to yaml 'no-auto-follow' configuration of '%s'"
-                                    % using_module_name,
-                                )
+
+            def decideRelevant(key, value):
+                # Only checking keys of configs, pylint: disable=unused-argument
+
+                return module_name.hasNamespace(key)
+
+            for (
+                config_of_module_name,
+                no_auto_follow,
+                description,
+            ) in self.getYamlConfigItemItems(
+                module_name=using_module_name,
+                section="anti-bloat",
+                item_name="no-auto-follow",
+                decide_relevant=decideRelevant,
+                recursive=True,
+            ):
+                assert module_name.hasNamespace(no_auto_follow), no_auto_follow
+
+                self.no_auto_follows[no_auto_follow] = description
+
+                return (
+                    False,
+                    "according to yaml 'no-auto-follow' configuration of '%s' for '%s'"
+                    % (config_of_module_name, no_auto_follow),
+                )
+
+        for no_follow_pattern, (
+            config_of_module_name,
+            description,
+        ) in self.no_follows.items():
+            if module_name.matchesToShellPattern(no_follow_pattern)[0]:
+                return (
+                    False,
+                    "according to yaml 'no-follow' configuration of '%s' for '%s'"
+                    % (config_of_module_name, no_follow_pattern),
+                )
 
         # Do not provide an opinion about it.
         return None
@@ -675,3 +887,19 @@ Undesirable import of '%s' (intending to avoid '%s') in \
 Not including '%s' automatically in order to avoid bloat, but this may cause: %s."""
                     % (module_name, description)
                 )
+
+
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.