pforge 1.0.0__py3-none-any.whl

pforge/Builtins.py

@@ -0,0 +1,288 @@
+#     Copyright 2026, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+"""Built-ins module. Information about built-ins of the running Python."""
+
+import sys
+from types import BuiltinFunctionType, FunctionType, GeneratorType, ModuleType
+
+from pforge.__past__ import GenericAlias, UnionType, builtins
+from pforge.containers.OrderedDicts import OrderedDict
+from pforge.PythonVersions import python_version
+
+
+def _getBuiltinExceptionNames():
+    def isExceptionName(builtin_name):
+        if builtin_name.endswith("Error") or builtin_name.endswith("Exception"):
+            return True
+        elif builtin_name in (
+            "StopIteration",
+            "GeneratorExit",
+            "SystemExit",
+            "NotImplemented",
+            "KeyboardInterrupt",
+            "StopAsyncIteration",
+        ):
+            return True
+        else:
+            return False
+
+    exceptions = OrderedDict()
+
+    # Hide Python3 changes for built-in exception names
+    if python_version < 0x300:
+        import exceptions as builtin_exceptions  # python2 code, pylint: disable=import-error
+
+        for key in sorted(dir(builtin_exceptions)):
+            name = str(key)
+
+            if isExceptionName(name):
+                exceptions[name] = getattr(builtin_exceptions, key)
+
+        for key in sorted(dir(builtins)):
+            name = str(key)
+
+            if isExceptionName(name):
+                exceptions[name] = getattr(builtins, key)
+    else:
+        for key in sorted(dir(builtins)):
+            if isExceptionName(key):
+                exceptions[key] = getattr(builtins, key)
+
+    return list(exceptions.keys()), exceptions
+
+
+builtin_exception_names, builtin_exception_values = _getBuiltinExceptionNames()
+builtin_exception_values_list = tuple(builtin_exception_values.values())
+
+
+# Just to make sure it's covering these cases correctly.
+assert "TypeError" in builtin_exception_names
+assert "ValueError" in builtin_exception_names
+assert "StopIteration" in builtin_exception_names
+assert "GeneratorExit" in builtin_exception_names
+assert "AssertionError" in builtin_exception_names
+assert "BaseException" in builtin_exception_names
+assert "Exception" in builtin_exception_names
+assert "NotImplemented" in builtin_exception_names
+
+assert "StopAsyncIteration" in builtin_exception_names or python_version < 0x350
+
+
+def getBuiltinExceptionIdentifier(exception_name):
+    """Get the C identifier of a built-in exception name.
+
+    Notes:
+        Usually that is the name prefixed with "PyExc_", but some built-in
+        exceptions have a C name that differs from their name.
+
+    Args:
+        exception_name: Name of the built-in exception.
+
+    Returns:
+        C identifier of the built-in exception.
+    """
+    assert "PyExc" not in exception_name, exception_name
+
+    if exception_name == "NotImplemented":
+        return "Py_NotImplemented"
+
+    # The builtin name has a leading underscore, but the C name does not.
+    if exception_name == "_IncompleteInputError":
+        return "PyExc_IncompleteInputError"
+
+    return "PyExc_%s" % exception_name
+
+
+def _getBuiltinNames():
+    names = [str(x) for x in dir(builtins)]
+    names.sort()
+
+    for builtin_exception_name in builtin_exception_names:
+        if builtin_exception_name in names:
+            names.remove(builtin_exception_name)
+
+    names.remove("__doc__")
+    names.remove("__name__")
+    names.remove("__package__")
+
+    if "__loader__" in names:
+        names.remove("__loader__")
+
+    if "__spec__" in names:
+        names.remove("__spec__")
+
+    warnings = []
+
+    for builtin_name in names:
+        if builtin_name.endswith("Warning"):
+            warnings.append(builtin_name)
+
+    for builtin_name in warnings:
+        names.remove(builtin_name)
+
+    return names, warnings
+
+
+builtin_names, builtin_warnings = _getBuiltinNames()
+builtin_named_values = dict((getattr(builtins, x), x) for x in builtin_names)
+builtin_named_values_list = tuple(builtin_named_values)
+
+assert type in builtin_named_values
+
+assert "__import__" in builtin_names
+assert "int" in builtin_names
+assert "type" in builtin_names
+
+assert "__doc__" not in builtin_names
+assert "sys" not in builtin_names
+
+builtin_all_names = builtin_names + builtin_exception_names + builtin_warnings
+
+
+def getBuiltinTypeNames():
+    result = []
+
+    for builtin_name in builtin_names:
+        if isinstance(getattr(builtins, builtin_name), type):
+            result.append(builtin_name)
+
+    return tuple(sorted(result))
+
+
+builtin_type_names = getBuiltinTypeNames()
+
+
+def _getAnonBuiltins():
+    # We use the order when encoding in the constants blob. Therefore it is imported
+    # to not reorder these values, the C side uses the absolute indexes.
+    anon_names = OrderedDict()
+    anon_codes = OrderedDict()
+
+    # Strangely these are not in Python3 types module
+    anon_names["NoneType"] = type(None)
+    anon_codes["NoneType"] = "Py_TYPE(Py_None)"
+
+    anon_names["ellipsis"] = type(Ellipsis)  # see above
+    anon_codes["ellipsis"] = "&PyEllipsis_Type"
+
+    anon_names["NotImplementedType"] = type(NotImplemented)
+    anon_codes["NotImplementedType"] = "Py_TYPE(Py_NotImplemented)"
+
+    anon_names["function"] = FunctionType
+    anon_codes["function"] = "&PyFunction_Type"
+
+    anon_names["generator"] = GeneratorType
+    anon_codes["generator"] = "&PyGenerator_Type"
+
+    anon_names["builtin_function_or_method"] = BuiltinFunctionType
+    anon_codes["builtin_function_or_method"] = "&PyCFunction_Type"
+
+    # Can't really have it until we have __pforge__
+    # "compiled_function"          : BuiltinFunctionType,
+    # "compiled_generator"         : GeneratorType, # see above
+    # anon_codes["compiled_function"] =  "&Pforge_Function_Type"
+    # anon_codes["compiled_generator"] =  "&Pforge_Generator_Type"
+
+    anon_names["code"] = type(_getAnonBuiltins.__code__)
+    anon_codes["code"] = "&PyCode_Type"
+
+    anon_names["module"] = ModuleType
+    anon_codes["module"] = "&PyModule_Type"
+
+    if python_version < 0x300:
+        # There are only there for Python2,
+        # pylint: disable=I0021,no-name-in-module
+        from types import ClassType, InstanceType, MethodType
+
+        # We can to be sure of the type here, so use open without encoding, pylint: disable=unspecified-encoding
+        with open(sys.executable) as any_file:
+            anon_names["file"] = type(any_file)
+        anon_codes["file"] = "&PyFile_Type"
+
+        anon_names["classobj"] = ClassType
+        anon_codes["classobj"] = "&PyClass_Type"
+
+        anon_names["instance"] = InstanceType
+        anon_codes["instance"] = "&PyInstance_Type"
+
+        anon_names["instancemethod"] = MethodType
+        anon_codes["instancemethod"] = "&PyMethod_Type"
+
+    if python_version >= 0x270:
+        anon_names["version_info"] = type(sys.version_info)
+        anon_codes["version_info"] = 'Py_TYPE(Pforge_SysGetObject("version_info"))'
+
+    if python_version >= 0x390:
+        assert GenericAlias is not None
+
+        anon_names["GenericAlias"] = GenericAlias
+        anon_codes["GenericAlias"] = "&Py_GenericAliasType"
+
+    if UnionType is not None:
+        # 3.10 only code, pylint: disable=I0021,unsupported-binary-operation
+
+        # Note: The name changed in Python3.14 to "Union", before it was UnionType.
+        union_name = UnionType.__name__
+
+        anon_names[union_name] = type(int | str)
+        anon_codes[union_name] = "Pforge_PyUnion_Type"
+
+    return anon_names, anon_codes
+
+
+builtin_anon_names, builtin_anon_codes = _getAnonBuiltins()
+builtin_anon_values = OrderedDict((b, a) for a, b in builtin_anon_names.items())
+
+# For being able to check if it's not hashable, we need something not using
+# a hash.
+builtin_anon_value_list = tuple(builtin_anon_values)
+
+_exception_size_cache = {}
+
+
+def _getExceptionObjectSize(exception_type):
+    if exception_type not in _exception_size_cache:
+        # spell-checker: ignore getsizeof
+        _exception_size_cache[exception_type] = sys.getsizeof(exception_type())
+
+    return _exception_size_cache[exception_type]
+
+
+_base_exception_size = _getExceptionObjectSize(BaseException)
+
+
+def isBaseExceptionSimpleExtension(exception_type):
+    def checkBases(exc_type):
+        for base_exception_type in exc_type.__bases__:
+            if base_exception_type is BaseException:
+                return True
+
+            if checkBases(base_exception_type):
+                return True
+
+        return False
+
+    return (
+        checkBases(exception_type)
+        and _getExceptionObjectSize(exception_type) == _base_exception_size
+    )
+
+
+#     Part of "Pforge", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the GNU Affero General Public License, Version 3 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        https://www.gnu.org/licenses/agpl-3.0.txt
+#
+#     See also: "Pforge Runtime Library Exception, Version 1.0" in file
+#     "LICENSE-RUNTIME.txt" for additional permissions granted under Section 7.
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

pforge/BytecodeCaching.py

@@ -0,0 +1,209 @@
+#     Copyright 2026, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+"""Caching of compiled code.
+
+Initially this deals with preserving compiled module state after bytecode demotion
+such that it allows to restore it directly.
+"""
+
+import os
+import sys
+
+from pforge.containers.OrderedSets import OrderedSet
+from pforge.importing.Importing import (
+    isSyntheticModuleUsage,
+    locateModule,
+    makeModuleUsageAttempt,
+)
+from pforge.ModuleRegistry import getModuleOptimizationTimingInfos
+from pforge.plugins.Hooks import getPluginsCacheContributionValues
+from pforge.utils.AppDirs import getCacheDir
+from pforge.utils.FileOperations import getNormalizedPathJoin
+from pforge.utils.Hashing import Hash, getStringHash
+from pforge.utils.Json import loadJsonFromFilename, writeJsonToFilename
+from pforge.utils.ModuleNames import ModuleName
+from pforge.Version import version_string
+
+
+def getBytecodeCacheDir():
+    return getCacheDir("module-cache", create=True)
+
+
+def _getCacheFilename(module_name, extension):
+    return getNormalizedPathJoin(
+        getBytecodeCacheDir(), "%s.%s" % (module_name, extension)
+    )
+
+
+def makeCacheName(module_name, source_code):
+    module_config_hash = _getModuleConfigHash(module_name)
+
+    return (
+        module_name.asLegalFilename()
+        + "@"
+        + module_config_hash
+        + "@"
+        + getStringHash(source_code)
+    )
+
+
+def hasCachedImportedModuleUsageAttempts(module_name, source_code, source_ref):
+    result = getCachedImportedModuleUsageAttempts(
+        module_name=module_name, source_code=source_code, source_ref=source_ref
+    )
+
+    return result is not None
+
+
+# Bump this is format is changed or enhanced implementation might different ones.
+_cache_format_version = 9
+
+
+def getCachedImportedModuleUsageAttempts(module_name, source_code, source_ref):
+    cache_name = makeCacheName(module_name, source_code)
+    cache_filename = _getCacheFilename(cache_name, "json")
+
+    if not os.path.exists(cache_filename):
+        return None
+
+    data = loadJsonFromFilename(cache_filename)
+
+    if data is None:
+        return None
+
+    if data.get("file_format_version") != _cache_format_version:
+        return None
+
+    if data["module_name"] != module_name:
+        return None
+
+    result = OrderedSet()
+
+    for module_used in data["modules_used"]:
+        used_module_name = ModuleName(module_used["module_name"])
+
+        # Retry the module scan to see if it still gives same result
+        if module_used["finding"] == "relative":
+            _used_module_name, filename, module_kind, finding = locateModule(
+                module_name=used_module_name.getBasename(),
+                parent_package=used_module_name.getPackageName(),
+                level=1,
+            )
+        else:
+            _used_module_name, filename, module_kind, finding = locateModule(
+                module_name=used_module_name, parent_package=None, level=0
+            )
+
+        if (
+            finding != module_used["finding"]
+            or module_kind != module_used["module_kind"]
+        ):
+            assert module_name != "email._header_value_parser", (
+                finding,
+                module_used["finding"],
+            )
+
+            return None
+
+        # We store only the line number and class, so the source
+        # reference is rebuilt from those.
+        used_source_ref = source_ref.atLineNumber(module_used["source_ref_line"])
+        if module_used["source_ref_internal"]:
+            used_source_ref = used_source_ref.atInternal()
+
+        result.add(
+            makeModuleUsageAttempt(
+                module_name=used_module_name,
+                filename=filename,
+                finding=module_used["finding"],
+                module_kind=module_used["module_kind"],
+                # TODO: Level might have to be dropped.
+                level=0,
+                source_ref=used_source_ref,
+                reason=module_used["reason"],
+            )
+        )
+
+    for module_used in data["distribution_names"]:
+        # TODO: Consider distributions found and not found and return None if
+        # something changed there.
+        pass
+
+    # The Json doesn't store integer keys.
+    for pass_timing_info in data["timing_infos"]:
+        pass_timing_info[5] = dict(
+            (int(key), value) for (key, value) in pass_timing_info[5].items()
+        )
+
+    return result, data["timing_infos"]
+
+
+def writeImportedModulesNamesToCache(
+    module_name,
+    source_code,
+    used_modules,
+    distribution_names,
+):
+    cache_name = makeCacheName(module_name, source_code)
+    cache_filename = _getCacheFilename(cache_name, "json")
+
+    used_modules = [
+        module_usage.asDict()
+        for module_usage in used_modules
+        # Synthetic usages, e.g. plugin provided implicit imports, are not
+        # stored, as they depend on the plugin state, and are added again on
+        # every compilation run.
+        if not isSyntheticModuleUsage(module_usage)
+    ]
+
+    for module_usage in used_modules:
+        source_ref = module_usage.pop("source_ref")
+        module_usage["source_ref_line"] = source_ref.getLineNumber()
+        module_usage["source_ref_internal"] = source_ref.isInternal()
+
+    data = {
+        "file_format_version": _cache_format_version,
+        "module_name": module_name.asString(),
+        # We use a tuple, so preserve the order.
+        "modules_used": used_modules,
+        "distribution_names": distribution_names,
+        "timing_infos": tuple(
+            timing_info.asTuple()
+            for timing_info in getModuleOptimizationTimingInfos(module_name)
+        ),
+    }
+
+    writeJsonToFilename(filename=cache_filename, contents=data)
+
+
+def _getModuleConfigHash(full_name):
+    """Calculate hash value for package packages importable for a module of this name."""
+    hash_value = Hash()
+
+    # Plugins may change their influence.
+    hash_value.updateFromValues(*getPluginsCacheContributionValues(full_name))
+
+    # Take Pforge and Python version into account as well, ought to catch code changes.
+    hash_value.updateFromValues(version_string, sys.version)
+
+    return hash_value.asHexDigest()
+
+
+#     Part of "Pforge", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the GNU Affero General Public License, Version 3 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        https://www.gnu.org/licenses/agpl-3.0.txt
+#
+#     See also: "Pforge Runtime Library Exception, Version 1.0" in file
+#     "LICENSE-RUNTIME.txt" for additional permissions granted under Section 7.
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

pforge/Bytecodes.py

@@ -0,0 +1,124 @@
+#     Copyright 2026, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+"""Handle bytecode and compile source code to bytecode."""
+
+import ast
+
+from pforge.options.Options import (
+    hasPythonFlagNoAsserts,
+    hasPythonFlagNoDocStrings,
+)
+from pforge.PythonVersions import python_version
+from pforge.tree.TreeHelpers import getKind
+
+doc_having = tuple(
+    getattr(ast, candidate)
+    for candidate in ("FunctionDef", "ClassDef", "AsyncFunctionDef")
+    if hasattr(ast, candidate)
+)
+
+
+def _removeDocFromBody(node):
+    if node.body and getKind(node.body[0]) == "Expr":
+        if getKind(node.body[0].value) == "Str":  # python3.7 or earlier
+            node.body[0].value.s = ""
+        elif getKind(node.body[0].value) == "Constant":  # python3.8
+            node.body[0].value.value = ""
+
+
+def compileSourceToBytecode(source_code, filename):
+    """Compile given source code into bytecode."""
+
+    # Prepare compile call with AST tree.
+    tree = ast.parse(source_code, filename)
+
+    # Do we need to remove doc strings.
+    remove_doc_strings_from_tree = hasPythonFlagNoDocStrings()
+
+    # For Python2, we need to do this manually.
+    remove_asserts_from_tree = hasPythonFlagNoAsserts() and str is bytes
+
+    if remove_doc_strings_from_tree or remove_asserts_from_tree:
+        # Module level docstring.
+        if remove_doc_strings_from_tree:
+            _removeDocFromBody(tree)
+
+        for node in ast.walk(tree):
+            if remove_asserts_from_tree:
+                node_type = type(node)
+
+                if node_type is ast.Name:
+                    if node.id == "__debug__":
+                        node.id = "False"
+
+                elif node_type is ast.Assert:
+                    # Cannot really remove the assertion node easily, lets just replace it with
+                    # "assert 1" and remove the assert msg. Probably not worth more effort for
+                    # Python2 at this time.
+                    node.test = ast.Num()
+                    node.test.n = 1
+                    node.test.lineno = node.lineno
+                    node.test.col_offset = node.col_offset
+                    node.msg = None
+
+            # Check if it's a docstring having node type.
+            if remove_doc_strings_from_tree and isinstance(node, doc_having):
+                _removeDocFromBody(node)
+
+    if str is bytes:
+        bytecode = compile(
+            tree,
+            filename=filename,
+            mode="exec",
+            dont_inherit=True,
+        )
+    else:
+        # Let the handling of __debug__ happen within compile built-in.
+        optimize = 0
+        if hasPythonFlagNoAsserts():
+            optimize = 1
+
+        bytecode = compile(
+            tree, filename=filename, mode="exec", dont_inherit=True, optimize=optimize
+        )
+
+    return bytecode
+
+
+if python_version < 0x300:
+    bytecode_header_size = 8
+elif python_version < 0x370:
+    # Python 3 added the source size to the pyc header.
+    bytecode_header_size = 12
+else:
+    # Python 3.7 switched to the PEP 552 header layout.
+    bytecode_header_size = 16
+
+
+def loadCodeObjectData(bytecode_filename):
+    """Load bytecode from a file."""
+
+    # Ignoring magic numbers, etc. which we don't have to care for much as
+    # CPython already checked them (would have rejected it otherwise).
+    with open(bytecode_filename, "rb") as f:
+        return f.read()[bytecode_header_size:]
+
+
+#     Part of "Pforge", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the GNU Affero General Public License, Version 3 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        https://www.gnu.org/licenses/agpl-3.0.txt
+#
+#     See also: "Pforge Runtime Library Exception, Version 1.0" in file
+#     "LICENSE-RUNTIME.txt" for additional permissions granted under Section 7.
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.

pforge/CacheCleanup.py

@@ -0,0 +1,59 @@
+#     Copyright 2026, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
+
+
+"""Cleanup of caches for Pforge.
+
+This is triggered by "--clean-cache=" usage, and can cleanup all kinds of
+caches and is supposed to run before or instead of Pforge compilation.
+"""
+
+import os
+
+from pforge.Tracing import cache_logger
+from pforge.utils.AppDirs import getCacheDir, removeCacheDir
+
+
+def _cleanCacheDirectory(cache_name, cache_basename):
+    from pforge.options.Options import shallCleanCache
+
+    if shallCleanCache(cache_name):
+        cache_dir = getCacheDir(cache_basename, create=False)
+
+        if os.path.exists(cache_dir):
+            cache_logger.info(
+                "Cleaning cache '%s' directory '%s'." % (cache_name, cache_dir)
+            )
+            removeCacheDir(
+                cache_basename=cache_basename,
+                logger=cache_logger,
+                ignore_errors=False,
+                extra_recommendation=None,
+            )
+            cache_logger.info("Done.")
+
+
+def cleanCaches():
+    _cleanCacheDirectory("ccache", "ccache")
+    _cleanCacheDirectory("clcache", "clcache")
+    _cleanCacheDirectory("zig", "zig")
+    _cleanCacheDirectory("bytecode", "module-cache")
+    _cleanCacheDirectory("dll-dependencies", "library_dependencies")
+
+
+#     Part of "Pforge", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the GNU Affero General Public License, Version 3 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        https://www.gnu.org/licenses/agpl-3.0.txt
+#
+#     See also: "Pforge Runtime Library Exception, Version 1.0" in file
+#     "LICENSE-RUNTIME.txt" for additional permissions granted under Section 7.
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.