fastpython-native 0.1.0__py3-none-any.whl

fastpython/compiler/toolchain.py

@@ -0,0 +1,147 @@
+"""
+Compiler toolchain for FastPython.
+Handles discovery, configuration, and execution of host C/C++ compilers (MSVC, GCC, Clang)
+to compile C-extensions targeting the host CPU architecture.
+"""
+
+import os
+import sys
+import shutil
+import subprocess
+from pathlib import Path
+from typing import List, Optional
+from ..hardware.cpu import get_cpu_info, CPUFeatures
+
+
+class CompilationError(Exception):
+    pass
+
+
+class CompilerToolchain:
+    def __init__(self):
+        self.cpu_info: CPUFeatures = get_cpu_info()
+        self.toolchain_type = self._detect_toolchain()
+
+    def _detect_toolchain(self) -> str:
+        if sys.platform == "win32":
+            # On Windows, try MSVC first
+            try:
+                from setuptools._distutils._msvccompiler import MSVCCompiler
+                comp = MSVCCompiler()
+                comp.initialize()
+                if comp.cc and os.path.exists(comp.cc):
+                    return "msvc"
+            except Exception:
+                pass
+            if shutil.which("gcc"):
+                return "gcc"
+            if shutil.which("clang"):
+                return "clang"
+        else:
+            if shutil.which("gcc"):
+                return "gcc"
+            if shutil.which("clang"):
+                return "clang"
+
+        raise RuntimeError("No suitable native C/C++ compiler found on the system. Please ensure MSVC, GCC, or Clang is installed.")
+
+    def compile_extension(
+        self,
+        c_source_path: str,
+        output_pyd_path: str,
+        fastmath: bool = True,
+        parallel: bool = True,
+        extra_flags: Optional[List[str]] = None
+    ) -> str:
+        """
+        Compile a C source file into a native Python extension (.pyd or .so).
+        """
+        out_dir = str(Path(output_pyd_path).parent.resolve())
+        os.makedirs(out_dir, exist_ok=True)
+
+        if self.toolchain_type == "msvc":
+            return self._compile_msvc(c_source_path, output_pyd_path, fastmath, parallel, extra_flags)
+        else:
+            return self._compile_gcc_or_clang(c_source_path, output_pyd_path, fastmath, parallel, extra_flags)
+
+    def _compile_msvc(
+        self,
+        c_source_path: str,
+        output_pyd_path: str,
+        fastmath: bool,
+        parallel: bool,
+        extra_flags: Optional[List[str]]
+    ) -> str:
+        from setuptools._distutils._msvccompiler import MSVCCompiler
+
+        try:
+            compiler = MSVCCompiler()
+            compiler.initialize()
+
+            flags = self.cpu_info.get_msvc_flags(fastmath=fastmath, parallel=parallel)
+            if extra_flags:
+                flags.extend(extra_flags)
+
+            py_include = os.path.join(sys.base_prefix, "include")
+            py_lib = os.path.join(sys.base_prefix, "libs")
+
+            temp_dir = str(Path(output_pyd_path).parent)
+
+            # Compile source to .obj
+            obj_files = compiler.compile(
+                [c_source_path],
+                output_dir=temp_dir,
+                include_dirs=[py_include],
+                extra_postargs=flags
+            )
+
+            # Link into .pyd with linker optimizations
+            py_lib_name = f"python{sys.version_info.major}{sys.version_info.minor}"
+            compiler.link_shared_object(
+                obj_files,
+                output_pyd_path,
+                libraries=[py_lib_name],
+                library_dirs=[py_lib],
+                extra_postargs=["/OPT:REF", "/OPT:ICF"]
+            )
+
+            if not os.path.exists(output_pyd_path):
+                raise CompilationError(f"Linking failed: {output_pyd_path} was not created.")
+
+            return output_pyd_path
+
+        except Exception as e:
+            raise CompilationError(f"MSVC compilation failed: {e}") from e
+
+    def _compile_gcc_or_clang(
+        self,
+        c_source_path: str,
+        output_pyd_path: str,
+        fastmath: bool,
+        parallel: bool,
+        extra_flags: Optional[List[str]]
+    ) -> str:
+        compiler_cmd = "gcc" if self.toolchain_type == "gcc" else "clang"
+        flags = self.cpu_info.get_gcc_flags(fastmath=fastmath, parallel=parallel)
+        if extra_flags:
+            flags.extend(extra_flags)
+
+        py_include = os.path.join(sys.base_prefix, "include")
+        py_lib = os.path.join(sys.base_prefix, "libs")
+        py_lib_name = f"python{sys.version_info.major}{sys.version_info.minor}"
+
+        cmd = [
+            compiler_cmd,
+            "-shared",
+            c_source_path,
+            "-o", output_pyd_path,
+            f"-I{py_include}",
+            f"-L{py_lib}",
+            f"-l{py_lib_name}",
+        ] + flags
+
+        proc = subprocess.run(cmd, capture_output=True, text=True)
+        if proc.returncode != 0:
+            raise CompilationError(f"Compilation command failed with code {proc.returncode}:\n{proc.stderr}")
+
+        return output_pyd_path

fastpython/hardware/__init__.py

@@ -0,0 +1,3 @@
+from .cpu import CPUFeatures, get_cpu_info
+
+__all__ = ["CPUFeatures", "get_cpu_info"]

fastpython/hardware/cpu.py

@@ -0,0 +1,209 @@
+"""
+Hardware detection module for FastPython.
+Identifies host CPU architecture, instruction set extensions (AVX, AVX2, AVX-512, FMA, NEON),
+core count, and generates optimal compiler optimization flags.
+"""
+
+import os
+import sys
+import platform
+import ctypes
+from ctypes import wintypes
+from dataclasses import dataclass
+from typing import List, Dict, Any
+
+
+@dataclass
+class CPUFeatures:
+    vendor: str
+    brand: str
+    architecture: str
+    physical_cores: int
+    logical_cores: int
+    has_avx: bool
+    has_avx2: bool
+    has_avx512f: bool
+    has_fma: bool
+    has_sse41: bool
+    has_sse42: bool
+    has_bmi1: bool
+    has_bmi2: bool
+    has_neon: bool
+
+    def summary(self) -> str:
+        features = []
+        if self.has_avx512f:
+            features.append("AVX-512")
+        elif self.has_avx2:
+            features.append("AVX2")
+        elif self.has_avx:
+            features.append("AVX")
+
+        if self.has_fma:
+            features.append("FMA3")
+        if self.has_sse42:
+            features.append("SSE4.2")
+        if self.has_bmi2:
+            features.append("BMI2")
+        if self.has_neon:
+            features.append("NEON")
+
+        feat_str = ", ".join(features) if features else "Baseline x86"
+        return f"{self.brand} ({self.logical_cores} threads, Features: {feat_str})"
+
+    def get_msvc_flags(self, fastmath: bool = True, parallel: bool = True) -> List[str]:
+        # Ultra-native bare-metal optimization flags for MSVC
+        flags = [
+            "/O2",        # Maximize speed
+            "/Oi",        # Enable intrinsic functions
+            "/Ot",        # Favor fast code over small code
+            "/Gy",        # Separate functions for linker
+            "/Gw",        # Optimize global data
+            "/GS-",       # Disable stack buffer security checks in compute kernels
+            "/Ob2",       # Inline any suitable function
+        ]
+        if self.has_avx512f:
+            flags.append("/arch:AVX512")
+        elif self.has_avx2:
+            flags.append("/arch:AVX2")
+        elif self.has_avx:
+            flags.append("/arch:AVX")
+
+        if fastmath:
+            flags.append("/fp:fast")
+        if parallel and self.logical_cores > 1:
+            flags.append("/openmp")
+        return flags
+
+    def get_gcc_flags(self, fastmath: bool = True, parallel: bool = True) -> List[str]:
+        flags = [
+            "-O3",
+            "-march=native",
+            "-fPIC",
+            "-fomit-frame-pointer",
+            "-funroll-loops",
+        ]
+        if fastmath:
+            flags.append("-ffast-math")
+        if parallel and self.logical_cores > 1:
+            flags.append("-fopenmp")
+        return flags
+
+
+def _detect_x86_cpuid() -> Dict[str, Any]:
+    """Execute CPUID on Windows AMD64 or Linux/macOS to query instruction sets."""
+    features = {
+        "vendor": "Unknown",
+        "brand": platform.processor() or "x86_64 Processor",
+        "has_avx": False,
+        "has_avx2": False,
+        "has_avx512f": False,
+        "has_fma": False,
+        "has_sse41": False,
+        "has_sse42": False,
+        "has_bmi1": False,
+        "has_bmi2": False,
+    }
+
+    if sys.platform == "win32" and platform.machine() in ("AMD64", "x86_64"):
+        try:
+            kernel32 = ctypes.windll.kernel32
+            kernel32.VirtualAlloc.restype = ctypes.c_void_p
+            kernel32.VirtualAlloc.argtypes = [wintypes.LPVOID, ctypes.c_size_t, wintypes.DWORD, wintypes.DWORD]
+            kernel32.VirtualFree.restype = wintypes.BOOL
+            kernel32.VirtualFree.argtypes = [wintypes.LPVOID, ctypes.c_size_t, wintypes.DWORD]
+
+            code = bytes([
+                0x48, 0x89, 0x5C, 0x24, 0x08,  # mov [rsp+8], rbx
+                0x89, 0xC8,                    # mov eax, ecx
+                0x89, 0xD1,                    # mov ecx, edx
+                0x0F, 0xA2,                    # cpuid
+                0x41, 0x89, 0x00,              # mov [r8], eax
+                0x41, 0x89, 0x58, 0x04,        # mov [r8+4], ebx
+                0x41, 0x89, 0x48, 0x08,        # mov [r8+8], ecx
+                0x41, 0x89, 0x50, 0x0C,        # mov [r8+12], edx
+                0x48, 0x8B, 0x5C, 0x24, 0x08,  # mov rbx, [rsp+8]
+                0xC3                           # ret
+            ])
+            addr = kernel32.VirtualAlloc(None, len(code), 0x3000, 0x40)
+            if addr:
+                ctypes.memmove(addr, code, len(code))
+                func_type = ctypes.CFUNCTYPE(None, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint32 * 4))
+                cpuid_call = func_type(addr)
+
+                regs = (ctypes.c_uint32 * 4)()
+
+                # Vendor string (Leaf 0)
+                cpuid_call(0, 0, ctypes.byref(regs))
+                vendor_bytes = bytearray()
+                for reg_idx in [1, 3, 2]:  # ebx, edx, ecx
+                    vendor_bytes.extend(int(regs[reg_idx]).to_bytes(4, 'little'))
+                features["vendor"] = vendor_bytes.decode('ascii', errors='ignore').strip()
+
+                # Features (Leaf 1)
+                cpuid_call(1, 0, ctypes.byref(regs))
+                eax, ebx, ecx, edx = regs
+                features["has_fma"] = bool(ecx & (1 << 12))
+                features["has_sse41"] = bool(ecx & (1 << 19))
+                features["has_sse42"] = bool(ecx & (1 << 20))
+                features["has_avx"] = bool(ecx & (1 << 28))
+
+                # Extended features (Leaf 7)
+                cpuid_call(7, 0, ctypes.byref(regs))
+                eax, ebx, ecx, edx = regs
+                features["has_bmi1"] = bool(ebx & (1 << 3))
+                features["has_avx2"] = bool(ebx & (1 << 5))
+                features["has_bmi2"] = bool(ebx & (1 << 8))
+                features["has_avx512f"] = bool(ebx & (1 << 16))
+
+                # Brand String (Leaf 0x80000002 .. 0x80000004)
+                brand_bytes = bytearray()
+                for leaf in [0x80000002, 0x80000003, 0x80000004]:
+                    cpuid_call(leaf, 0, ctypes.byref(regs))
+                    for r in regs:
+                        brand_bytes.extend(int(r).to_bytes(4, 'little'))
+                brand = brand_bytes.decode('ascii', errors='ignore').strip()
+                if brand:
+                    features["brand"] = brand
+
+                kernel32.VirtualFree(addr, 0, 0x8000)
+        except Exception:
+            pass
+
+    return features
+
+
+_CACHED_CPU_INFO = None
+
+
+def get_cpu_info() -> CPUFeatures:
+    """Retrieve and cache host CPU capabilities."""
+    global _CACHED_CPU_INFO
+    if _CACHED_CPU_INFO is not None:
+        return _CACHED_CPU_INFO
+
+    arch = platform.machine()
+    logical_cores = os.cpu_count() or 4
+    physical_cores = max(1, logical_cores // 2)
+
+    has_neon = arch.lower() in ("arm64", "aarch64")
+
+    detected = _detect_x86_cpuid()
+
+    _CACHED_CPU_INFO = CPUFeatures(
+        vendor=detected["vendor"],
+        brand=detected["brand"],
+        architecture=arch,
+        physical_cores=physical_cores,
+        logical_cores=logical_cores,
+        has_avx=detected["has_avx"],
+        has_avx2=detected["has_avx2"],
+        has_avx512f=detected["has_avx512f"],
+        has_fma=detected["has_fma"],
+        has_sse41=detected["has_sse41"],
+        has_sse42=detected["has_sse42"],
+        has_bmi1=detected["has_bmi1"],
+        has_bmi2=detected["has_bmi2"],
+        has_neon=has_neon,
+    )
+    return _CACHED_CPU_INFO

fastpython/importer.py

@@ -0,0 +1,181 @@
+"""
+Automatic Project Importer & AST Hook for FastPython.
+Intercepts imports of project modules, detects computational functions,
+and compiles them Ahead-of-Time (AOT) to hardware-native machine code.
+"""
+
+import sys
+import os
+import ast
+from pathlib import Path
+from importlib.abc import MetaPathFinder, Loader
+from importlib.machinery import ModuleSpec
+from typing import Optional, List, Set
+
+from .hardware.cpu import get_cpu_info
+
+
+def is_acceleration_candidate(node: ast.FunctionDef) -> bool:
+    """
+    Check if a function is an ideal candidate for hardware machine code acceleration:
+    contains loops (for/while) and arithmetic, and avoids non-kernel constructs.
+    """
+    has_loop = False
+    has_arithmetic = False
+
+    for child in ast.walk(node):
+        # Disqualify generators or complex dynamic features
+        if isinstance(child, (ast.Yield, ast.YieldFrom, ast.AsyncFunctionDef, ast.Try)):
+            return False
+        if isinstance(child, (ast.For, ast.While)):
+            has_loop = True
+        if isinstance(child, (ast.BinOp, ast.AugAssign, ast.Compare)):
+            has_arithmetic = True
+
+    return has_loop or (has_arithmetic and len(node.body) > 1)
+
+
+class FastPythonLoader(Loader):
+    def __init__(self, fullname: str, filepath: str, verbose: bool = True):
+        self.fullname = fullname
+        self.filepath = filepath
+        self.verbose = verbose
+
+    def create_module(self, spec: ModuleSpec):
+        return None  # Use default module creation
+
+    def exec_module(self, module):
+        module.__file__ = self.filepath
+
+        with open(self.filepath, "r", encoding="utf-8") as f:
+            source = f.read()
+
+        try:
+            tree = ast.parse(source, filename=self.filepath)
+        except Exception:
+            # Fall back to standard execution if parsing fails
+            exec(compile(source, self.filepath, "exec"), module.__dict__)
+            return
+
+        candidates: List[str] = []
+        for node in tree.body:
+            if isinstance(node, ast.FunctionDef) and is_acceleration_candidate(node):
+                candidates.append(node.name)
+                # Inject decorator: @_fastpython_compile(fastmath=True)
+                call_node = ast.Call(
+                    func=ast.Name(id="_fastpython_compile", ctx=ast.Load()),
+                    args=[],
+                    keywords=[
+                        ast.keyword(arg="fastmath", value=ast.Constant(value=True)),
+                        ast.keyword(arg="verbose", value=ast.Constant(value=self.verbose)),
+                    ]
+                )
+                node.decorator_list.insert(0, call_node)
+
+        if candidates:
+            if self.verbose:
+                cpu = get_cpu_info()
+                print(f"[FastPython.Auto] Accelerating module '{self.fullname}' ({len(candidates)} functions: {', '.join(candidates)}) for {cpu.brand.strip()}...")
+
+            # Inject import of compile_native at top of module
+            import_stmt = ast.ImportFrom(
+                module="fastpython",
+                names=[ast.alias(name="compile_native", asname="_fastpython_compile")],
+                level=0
+            )
+            tree.body.insert(0, import_stmt)
+            ast.fix_missing_locations(tree)
+
+        code_obj = compile(tree, filename=self.filepath, mode="exec")
+        exec(code_obj, module.__dict__)
+
+
+class FastPythonFinder(MetaPathFinder):
+    def __init__(self, project_root: str, verbose: bool = True):
+        self.project_root = Path(project_root).resolve()
+        self.verbose = verbose
+        self._cache_ignore_prefixes = {
+            sys.base_prefix.lower(),
+            sys.prefix.lower(),
+        }
+
+    def find_spec(self, fullname: str, path: Optional[List[str]] = None, target=None) -> Optional[ModuleSpec]:
+        # Ignore fastpython internal modules
+        if fullname.startswith("fastpython"):
+            return None
+
+        # Determine file path
+        mod_parts = fullname.split(".")
+        search_dirs = [self.project_root]
+        if path:
+            search_dirs = [Path(p).resolve() for p in path]
+        else:
+            # Check all sys.path directories belonging to the project
+            for p in sys.path:
+                try:
+                    p_path = Path(p).resolve()
+                    if p_path not in search_dirs and self._is_in_project(p_path):
+                        search_dirs.append(p_path)
+                except Exception:
+                    pass
+
+        for base_dir in search_dirs:
+            # Check for module.py
+            candidate_file = base_dir / f"{mod_parts[-1]}.py"
+            if candidate_file.is_file():
+                resolved = candidate_file.resolve()
+                if self._is_in_project(resolved):
+                    return ModuleSpec(
+                        fullname,
+                        FastPythonLoader(fullname, str(resolved), verbose=self.verbose),
+                        origin=str(resolved)
+                    )
+
+            # Check for package/__init__.py
+            candidate_pkg = base_dir / mod_parts[-1] / "__init__.py"
+            if candidate_pkg.is_file():
+                resolved = candidate_pkg.resolve()
+                if self._is_in_project(resolved):
+                    spec = ModuleSpec(
+                        fullname,
+                        FastPythonLoader(fullname, str(resolved), verbose=self.verbose),
+                        origin=str(resolved),
+                        is_package=True
+                    )
+                    spec.submodule_search_locations = [str(candidate_pkg.parent)]
+                    return spec
+
+        return None
+
+    def _is_in_project(self, file_path: Path) -> bool:
+        path_str = str(file_path).lower()
+        for ignore in self._cache_ignore_prefixes:
+            if path_str.startswith(ignore) and "site-packages" in path_str:
+                return False
+        try:
+            file_path.relative_to(self.project_root)
+            return True
+        except ValueError:
+            return False
+
+
+_HOOK_INSTALLED = False
+
+
+def auto_accelerate(project_dir: Optional[str] = None, verbose: bool = True):
+    """
+    Installs the FastPython automatic compilation hook.
+    Every module imported in the project will be analyzed and compiled AOT into
+    hardware-native machine code before execution.
+    """
+    global _HOOK_INSTALLED
+    if _HOOK_INSTALLED:
+        return
+
+    root = project_dir or os.getcwd()
+    finder = FastPythonFinder(project_root=root, verbose=verbose)
+    sys.meta_path.insert(0, finder)
+    _HOOK_INSTALLED = True
+    if verbose:
+        cpu = get_cpu_info()
+        print(f"[FastPython.Auto] Hook active for project '{root}'. Hardware: {cpu.summary()}")

fastpython/memory/__init__.py

@@ -0,0 +1,3 @@
+from .aligned_mem import allocate, AlignedBuffer
+
+__all__ = ["allocate", "AlignedBuffer"]

fastpython/server/__init__.py

@@ -0,0 +1,38 @@
+import time
+from typing import Optional, Dict, Any
+from . import native_server
+
+
+class FastWebServer:
+    """
+    High-performance, hardware-native web server.
+    Bypasses Python GIL, uses AVX2 SIMD request line parsing,
+    and distributes TCP connections across all hardware CPU threads.
+    """
+    def __init__(self, port: int = 8080, threads: int = 16):
+        self.port = port
+        self.threads = threads
+        self.is_running = False
+
+    def start(self):
+        if not self.is_running:
+            native_server.start(self.port, self.threads)
+            self.is_running = True
+
+    def stop(self):
+        if self.is_running:
+            native_server.stop()
+            self.is_running = False
+
+    def stats(self) -> Dict[str, Any]:
+        return native_server.get_stats()
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.stop()
+
+
+__all__ = ["FastWebServer", "native_server"]

fastpython/simd/__init__.py

@@ -0,0 +1,10 @@
+try:
+    from .fast_simd import find_char, find_crlf, parse_http_request, dumps
+except ImportError:
+    # If not yet compiled in package directory
+    find_char = None
+    find_crlf = None
+    parse_http_request = None
+    dumps = None
+
+__all__ = ["find_char", "find_crlf", "parse_http_request", "dumps"]