pyodide-build 0.26.0.dev0__tar.gz → 0.26.2__tar.gz

pyodide_build-0.26.2/.gitignore

@@ -0,0 +1,43 @@
+*.a
+*.o
+*.pyc
+*.gen.*
+src/js/generated
+*.out.*
+
+*.egg-info/
+
+__pycache__
+
+.docker_home
+.hypothesis
+.idea
+.mozilla
+.mypy_cache/
+.pytest_cache/
+.vscode
+.venv*
+tsconfig.tsbuildinfo
+
+build
+ccache
+cpython/downloads
+cpython/installs
+cpython/build.log
+dist*/
+docs/_build/
+emsdk/emsdk
+geckodriver.log
+node_modules
+packages/.artifacts
+packages/.build
+packages/.libs
+packages/*/build.log*
+packages/build-logs
+dist/
+pyodide-build/**/build.log
+xbuildenv/
+pytest-pyodide
+tools/symlinks
+xbuildenv/
+.pyodide-xbuildenv*

{pyodide-build-0.26.0.dev0 → pyodide_build-0.26.2}/PKG-INFO

@@ -1,43 +1,41 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: pyodide-build
-Version: 0.26.0.dev0
+Version: 0.26.2
 Summary: "Tools for building Pyodide"
-Author: Pyodide developers
-License: MPL-2.0
 Project-URL: Homepage, https://github.com/pyodide/pyodide
 Project-URL: Bug Tracker, https://github.com/pyodide/pyodide/issues
 Project-URL: Documentation, https://pyodide.org/en/stable/
-Classifier: Programming Language :: Python :: 3
+Author: Pyodide developers
+License: MPL-2.0
+License-File: LICENSE
 Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
 Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.12
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: pyyaml
-Requires-Dist: ruamel.yaml
+Requires-Dist: auditwheel-emscripten~=0.0.9
+Requires-Dist: build~=1.2.0
+Requires-Dist: cmake>=3.24
+Requires-Dist: loky
 Requires-Dist: packaging
-Requires-Dist: wheel
-Requires-Dist: build>=1.0.0
-Requires-Dist: virtualenv
-Requires-Dist: pydantic<2,>=1.10.2
+Requires-Dist: pydantic<3,>=2
 Requires-Dist: pyodide-cli~=0.2.1
-Requires-Dist: cmake>=3.24
-Requires-Dist: unearth~=0.6
+Requires-Dist: pyodide-lock==0.1.0a6
+Requires-Dist: pyyaml
 Requires-Dist: requests
-Requires-Dist: types-requests
-Requires-Dist: typer
-Requires-Dist: auditwheel-emscripten~=0.0.9
-Requires-Dist: pyodide-lock==0.1.0a4
 Requires-Dist: resolvelib
 Requires-Dist: rich
-Requires-Dist: loky
+Requires-Dist: ruamel-yaml
+Requires-Dist: typer
+Requires-Dist: types-requests
+Requires-Dist: unearth~=0.6
+Requires-Dist: virtualenv
+Requires-Dist: wheel
 Provides-Extra: test
-Requires-Dist: pytest<8.0.0; extra == "test"
-Requires-Dist: pytest-pyodide==0.56.2; extra == "test"
-Requires-Dist: packaging; extra == "test"
-Provides-Extra: deploy
-Requires-Dist: boto3; extra == "deploy"
-Requires-Dist: moto; extra == "deploy"
+Requires-Dist: packaging; extra == 'test'
+Requires-Dist: pytest-httpserver; extra == 'test'
+Requires-Dist: pytest-pyodide==0.57.0; extra == 'test'
+Requires-Dist: pytest<8.0.0; extra == 'test'
+Description-Content-Type: text/markdown
 
 # pyodide-build
 

pyodide_build-0.26.2/pyodide_build/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.26.2"

pyodide_build-0.26.2/pyodide_build/_f2c_fixes.py

@@ -0,0 +1,198 @@
+import os
+import re
+import subprocess
+from pathlib import Path
+from textwrap import dedent
+
+
+def fix_f2c_input(f2c_input: Path) -> None:
+    if f2c_input.name.endswith("_flapack-f2pywrappers.f"):
+        content = f2c_input.read_text()
+        content = content.replace("character cmach", "integer cmach")
+        content = content.replace("character norm", "integer norm")
+        f2c_input.write_text(content)
+        return
+
+    if f2c_input.name in [
+        "_lapack_subroutine_wrappers.f",
+        "_blas_subroutine_wrappers.f",
+    ]:
+        content = f2c_input.read_text()
+        content = content.replace("character", "integer")
+        content = content.replace(
+            "ret = chla_transtype(", "call chla_transtype(ret, 1,"
+        )
+        f2c_input.write_text(content)
+
+
+def fix_f2c_output(f2c_output: Path) -> None:
+    """
+    This function is called on the name of each C output file. It fixes up the C
+    output in various ways to compensate for the lack of f2c support for Fortran
+    90 and Fortran 95.
+    """
+    if f2c_output.name == "_lapack_subroutine_wrappers.c":
+        content = f2c_output.read_text()
+        content = content.replace("integer chla_transtype__", "void chla_transtype__")
+        f2c_output.write_text(content)
+        return
+
+    if f2c_output.name.endswith("eupd.c"):
+        content = f2c_output.read_text()
+        content = re.sub(
+            r"ftnlen\s*(howmny_len|bmat_len),?", "", content, flags=re.MULTILINE
+        )
+        f2c_output.write_text(content)
+        return
+
+    if f2c_output.name.endswith("lansvd.c"):
+        content = f2c_output.read_text()
+        content += dedent(
+            """
+            #include <time.h>
+
+            int second_(real *t) {
+                *t = clock()/1000;
+                return 0;
+            }
+            """
+        )
+        f2c_output.write_text(content)
+        return
+
+
+def scipy_fix_cfile(path: Path) -> None:
+    """
+    Replace void return types with int return types in various generated .c and
+    .h files. We can't achieve this with a simple patch because these files are
+    not in the sdist, they are generated as part of the build.
+    """
+    text = path.read_text()
+    text = text.replace("extern void F_WRAPPEDFUNC", "extern int F_WRAPPEDFUNC")
+    text = text.replace("extern void F_FUNC", "extern int F_FUNC")
+    text = text.replace("void (*f2py_func)", "int (*f2py_func)")
+    text = text.replace("static void cb_", "static int cb_")
+    text = text.replace("typedef void(*cb_", "typedef int(*cb_")
+    text = text.replace("void(*)", "int(*)")
+    text = text.replace("static void f2py_setup_", "static int f2py_setup_")
+
+    if path.name.endswith("_flapackmodule.c"):
+        text = text.replace(",size_t", "")
+        text = re.sub(r",slen\([a-z]*\)\)", ")", text)
+
+    path.write_text(text)
+
+    for lib in ["lapack", "blas"]:
+        if path.name.endswith(f"cython_{lib}.c"):
+            header_name = f"_{lib}_subroutines.h"
+            header_dir = path.parent
+            header_path = find_header(header_dir, header_name)
+
+            header_text = header_path.read_text()
+            header_text = header_text.replace("void F_FUNC", "int F_FUNC")
+            header_path.write_text(header_text)
+
+
+def find_header(source_dir: Path, header_name: str) -> Path:
+    """
+    Find the header file that corresponds to a source file.
+    """
+    while not (header_path := source_dir / header_name).exists():
+        # meson copies the source files into a subdirectory of the build
+        source_dir = source_dir.parent
+        if source_dir == Path("/"):
+            raise RuntimeError(f"Could not find header file {header_name}")
+
+    return header_path
+
+
+def scipy_fixes(args: list[str]) -> None:
+    for arg in args:
+        if arg.endswith(".c"):
+            scipy_fix_cfile(Path(arg))
+
+
+def replay_f2c(args: list[str], dryrun: bool = False) -> list[str] | None:
+    """Apply f2c to compilation arguments
+
+    Parameters
+    ----------
+    args
+       input compiler arguments
+    dryrun
+       if False run f2c on detected fortran files
+
+    Returns
+    -------
+    new_args
+       output compiler arguments
+
+
+    Examples
+    --------
+
+    >>> replay_f2c(['gfortran', 'test.f'], dryrun=True)
+    ['gcc', 'test.c']
+    """
+    f2c_path = os.environ.get("F2C_PATH", "f2c")
+
+    new_args = ["gcc"]
+    found_source = False
+    for arg in args[1:]:
+        if not arg.endswith((".f", ".F")):
+            new_args.append(arg)
+            continue
+        found_source = True
+        filepath = Path(arg).resolve()
+        new_args.append(arg[:-2] + ".c")
+        if dryrun:
+            continue
+        fix_f2c_input(Path(arg))
+        if arg.endswith(".F"):
+            # .F files apparently expect to be run through the C
+            # preprocessor (they have #ifdef's in them)
+            # Use gfortran frontend, as gcc frontend might not be
+            # present on osx
+            # The file-system might be not case-sensitive,
+            # so take care to handle this by renaming.
+            # For preprocessing and further operation the
+            # expected file-name and extension needs to be preserved.
+            subprocess.check_call(
+                [
+                    "gfortran",
+                    "-E",
+                    "-C",
+                    "-P",
+                    filepath,
+                    "-o",
+                    filepath.with_suffix(".f77"),
+                ]
+            )
+            filepath = filepath.with_suffix(".f77")
+        # -R flag is important, it means that Fortran functions that
+        # return real e.g. sdot will be transformed into C functions
+        # that return float. For historic reasons, by default f2c
+        # transform them into functions that return a double. Using -R
+        # allows to match what OpenBLAS has done when they f2ced their
+        # Fortran files, see
+        # https://github.com/xianyi/OpenBLAS/pull/3539#issuecomment-1493897254
+        # for more details
+        with (
+            open(filepath) as input_pipe,
+            open(filepath.with_suffix(".c"), "w") as output_pipe,
+        ):
+            subprocess.check_call(
+                [f2c_path, "-R"],
+                stdin=input_pipe,
+                stdout=output_pipe,
+                cwd=filepath.parent,
+            )
+        fix_f2c_output(Path(arg[:-2] + ".c"))
+
+    new_args_str = " ".join(args)
+    if ".so" in new_args_str and "libgfortran.so" not in new_args_str:
+        found_source = True
+
+    if not found_source:
+        return None
+    return new_args

{pyodide-build-0.26.0.dev0 → pyodide_build-0.26.2}/pyodide_build/_py_compile.py

@@ -122,9 +122,10 @@ def _compile(
         else:
             compression = zipfile.ZIP_STORED
 
-        with zipfile.ZipFile(
-            input_path
-        ) as fh_zip_in, TemporaryDirectory() as temp_dir_str:
+        with (
+            zipfile.ZipFile(input_path) as fh_zip_in,
+            TemporaryDirectory() as temp_dir_str,
+        ):
             temp_dir = Path(temp_dir_str)
             output_path_tmp = temp_dir / output_name
             with zipfile.ZipFile(

{pyodide-build-0.26.0.dev0 → pyodide_build-0.26.2}/pyodide_build/bash_runner.py

@@ -106,7 +106,7 @@ def get_bash_runner(
     extra_envs: dict[str, str],
 ) -> Iterator[BashRunnerWithSharedEnvironment]:
     pyodide_root = get_pyodide_root()
-    env = get_build_environment_vars()
+    env = get_build_environment_vars(pyodide_root)
     env.update(extra_envs)
 
     with BashRunnerWithSharedEnvironment(env=env) as b:

{pyodide-build-0.26.0.dev0 → pyodide_build-0.26.2}/pyodide_build/build_env.py

@@ -5,7 +5,6 @@ import functools
 import os
 import re
 import subprocess
-import tomllib
 from collections.abc import Iterator
 from contextlib import nullcontext, redirect_stdout
 from io import StringIO
@@ -13,8 +12,8 @@ from pathlib import Path
 
 from packaging.tags import Tag, compatible_tags, cpython_tags
 
-from .common import exit_with_stdio, xbuildenv_dirname
-from .logger import logger
+from .common import search_pyproject_toml, xbuildenv_dirname
+from .config import ConfigManager
 from .recipe import load_all_recipes
 
 RUST_BUILD_PRELUDE = """
@@ -23,47 +22,6 @@ rustup target add wasm32-unknown-emscripten --toolchain ${RUST_TOOLCHAIN}
 """
 
 
-BUILD_VARS: set[str] = {
-    "CARGO_BUILD_TARGET",
-    "CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER",
-    "HOME",
-    "HOSTINSTALLDIR",
-    "HOSTSITEPACKAGES",
-    "NUMPY_LIB",
-    "PATH",
-    "PLATFORM_TRIPLET",
-    "PIP_CONSTRAINT",
-    "PYMAJOR",
-    "PYMICRO",
-    "PYMINOR",
-    "PYO3_CROSS_INCLUDE_DIR",
-    "PYO3_CROSS_LIB_DIR",
-    "PYODIDE_EMSCRIPTEN_VERSION",
-    "PYODIDE_JOBS",
-    "PYODIDE_PACKAGE_ABI",
-    "PYODIDE_ROOT",
-    "PYTHON_ARCHIVE_SHA256",
-    "PYTHON_ARCHIVE_URL",
-    "PYTHONINCLUDE",
-    "PYTHONPATH",
-    "PYVERSION",
-    "RUSTFLAGS",
-    "RUST_TOOLCHAIN",
-    "SIDE_MODULE_CFLAGS",
-    "SIDE_MODULE_CXXFLAGS",
-    "SIDE_MODULE_LDFLAGS",
-    "STDLIB_MODULE_CFLAGS",
-    "SYSCONFIGDATA_DIR",
-    "SYSCONFIG_NAME",
-    "TARGETINSTALLDIR",
-    "WASM_LIBRARY_DIR",
-    "CMAKE_TOOLCHAIN_FILE",
-    "PYO3_CONFIG_FILE",
-    "MESON_CROSS_FILE",
-    "PKG_CONFIG_LIBDIR",
-}
-
-
 @dataclasses.dataclass(eq=False, order=False, kw_only=True)
 class BuildArgs:
     """
@@ -114,14 +72,16 @@ def _init_xbuild_env(*, quiet: bool = False) -> Path:
     -------
         The path to the Pyodide root directory inside the xbuild environment
     """
-    from . import install_xbuildenv  # avoid circular import
+    from .xbuildenv import CrossBuildEnvManager  # avoid circular import
 
-    # TODO: Do not hardcode the path
     xbuildenv_path = Path(xbuildenv_dirname()).resolve()
-
     context = redirect_stdout(StringIO()) if quiet else nullcontext()
     with context:
-        return install_xbuildenv.install(xbuildenv_path, download=True)
+        manager = CrossBuildEnvManager(xbuildenv_path)
+        if manager.current_version is None:
+            manager.install()
+
+        return manager.pyodide_root
 
 
 @functools.cache
@@ -136,24 +96,13 @@ def search_pyodide_root(curdir: str | Path, *, max_depth: int = 10) -> Path | No
     by looking for the pyproject.toml file in the parent directories
     which contains [tool.pyodide] section.
     """
+    pyproject_path, pyproject_file = search_pyproject_toml(curdir, max_depth)
 
-    # We want to include "curdir" in parent_dirs, so add a garbage suffix
-    parent_dirs = (Path(curdir) / "garbage").parents[:max_depth]
+    if pyproject_path is None or pyproject_file is None:
+        return None
 
-    for base in parent_dirs:
-        pyproject_file = base / "pyproject.toml"
-
-        if not pyproject_file.is_file():
-            continue
-
-        try:
-            with pyproject_file.open("rb") as f:
-                configs = tomllib.load(f)
-        except tomllib.TOMLDecodeError as e:
-            raise ValueError(f"Could not parse {pyproject_file}.") from e
-
-        if "tool" in configs and "pyodide" in configs["tool"]:
-            return base
+    if "tool" in pyproject_file and "_pyodide" in pyproject_file["tool"]:
+        return pyproject_path.parent
 
     return None
 
@@ -164,84 +113,33 @@ def in_xbuildenv() -> bool:
 
 
 @functools.cache
-def get_build_environment_vars() -> dict[str, str]:
+def get_build_environment_vars(pyodide_root: Path) -> dict[str, str]:
     """
     Get common environment variables for the in-tree and out-of-tree build.
     """
-    env = _get_make_environment_vars().copy()
-
-    # Allow users to overwrite the build environment variables by setting
-    # host environment variables.
-    # TODO: Add modifiable configuration file instead.
-    # (https://github.com/pyodide/pyodide/pull/3737/files#r1161247201)
-    env.update({key: os.environ[key] for key in BUILD_VARS if key in os.environ})
-    env["PYODIDE"] = "1"
-
-    tools_dir = Path(__file__).parent / "tools"
-
-    if "CMAKE_TOOLCHAIN_FILE" not in env:
-        env["CMAKE_TOOLCHAIN_FILE"] = str(
-            tools_dir / "cmake/Modules/Platform/Emscripten.cmake"
-        )
-
-    if "PYO3_CONFIG_FILE" not in env:
-        env["PYO3_CONFIG_FILE"] = str(tools_dir / "pyo3_config.ini")
-
-    if "MESON_CROSS_FILE" not in env:
-        env["MESON_CROSS_FILE"] = str(tools_dir / "emscripten.meson.cross")
-
-    hostsitepackages = env["HOSTSITEPACKAGES"]
-    pythonpath = [
-        hostsitepackages,
-    ]
-    env["PYTHONPATH"] = ":".join(pythonpath)
-
-    return env
-
-
-def _get_make_environment_vars(*, pyodide_root: Path | None = None) -> dict[str, str]:
-    """Load environment variables from Makefile.envs
-
-    This allows us to set all build vars in one place
-
-    Parameters
-    ----------
-    pyodide_root
-        The root directory of the Pyodide repository. If None, this will be inferred.
-    """
-
-    PYODIDE_ROOT = get_pyodide_root() if pyodide_root is None else pyodide_root
-    environment = {}
-    result = subprocess.run(
-        ["make", "-f", str(PYODIDE_ROOT / "Makefile.envs"), ".output_vars"],
-        capture_output=True,
-        text=True,
-        env={"PYODIDE_ROOT": str(PYODIDE_ROOT)},
+    config_manager = ConfigManager(pyodide_root)
+    env = config_manager.to_env()
+
+    env.update(
+        {
+            # This environment variable is used for packages to detect if they are built
+            # for pyodide during build time
+            "PYODIDE": "1",
+            # This is the legacy environment variable used for the aforementioned purpose
+            "PYODIDE_PACKAGE_ABI": "1",
+            "PYTHONPATH": env["HOSTSITEPACKAGES"],
+        }
     )
 
-    if result.returncode != 0:
-        logger.error("ERROR: Failed to load environment variables from Makefile.envs")
-        exit_with_stdio(result)
-
-    for line in result.stdout.splitlines():
-        equalPos = line.find("=")
-        if equalPos != -1:
-            varname = line[0:equalPos]
-
-            if varname not in BUILD_VARS:
-                continue
-
-            value = line[equalPos + 1 :]
-            value = value.strip("'").strip()
-            environment[varname] = value
-    return environment
+    return env
 
 
 def get_build_flag(name: str) -> str:
     """
     Get a value of a build flag.
     """
-    build_vars = get_build_environment_vars()
+    pyodide_root = get_pyodide_root()
+    build_vars = get_build_environment_vars(pyodide_root)
     if name not in build_vars:
         raise ValueError(f"Unknown build flag: {name}")
 
@@ -293,6 +191,11 @@ def platform() -> str:
     return f"emscripten_{version}_wasm32"
 
 
+def wheel_platform() -> str:
+    abi_version = get_build_flag("PYODIDE_ABI_VERSION")
+    return f"pyodide_{abi_version}_wasm32"
+
+
 def pyodide_tags() -> Iterator[Tag]:
     """
     Returns the sequence of tag triples for the Pyodide interpreter.
@@ -301,10 +204,10 @@ def pyodide_tags() -> Iterator[Tag]:
     """
     PYMAJOR = get_pyversion_major()
     PYMINOR = get_pyversion_minor()
-    PLATFORM = platform()
+    PLATFORMS = [platform(), wheel_platform()]
     python_version = (int(PYMAJOR), int(PYMINOR))
-    yield from cpython_tags(platforms=[PLATFORM], python_version=python_version)
-    yield from compatible_tags(platforms=[PLATFORM], python_version=python_version)
+    yield from cpython_tags(platforms=PLATFORMS, python_version=python_version)
+    yield from compatible_tags(platforms=PLATFORMS, python_version=python_version)
     # Following line can be removed once packaging 22.0 is released and we update to it.
     yield Tag(interpreter=f"cp{PYMAJOR}{PYMINOR}", abi="none", platform="any")
 

{pyodide-build-0.26.0.dev0 → pyodide_build-0.26.2}/pyodide_build/buildall.py

@@ -100,7 +100,7 @@ class BasePackage:
 class Package(BasePackage):
     def __init__(self, pkgdir: Path, config: MetaConfig):
         self.pkgdir = pkgdir
-        self.meta = config.copy(deep=True)
+        self.meta = config.model_copy(deep=True)
 
         self.name = self.meta.package.name
         self.version = self.meta.package.version
@@ -772,16 +772,14 @@ def generate_lockfile(
 ) -> PyodideLockSpec:
     """Generate the package.json file"""
 
-    from . import __version__
-
     # Build package.json data.
     [platform, _, arch] = build_env.platform().rpartition("_")
     info = {
         "arch": arch,
         "platform": platform,
-        # This assumes that pyodide-build version == pyodide version.
-        "version": __version__,
+        "version": build_env.get_build_flag("PYODIDE_VERSION"),
         "python": sys.version.partition(" ")[0],
+        "abi_version": build_env.get_build_flag("PYODIDE_ABI_VERSION"),
     }
     packages = generate_packagedata(output_dir, pkg_map)
     lock_spec = PyodideLockSpec(info=info, packages=packages)
@@ -875,7 +873,7 @@ def install_packages(
     """
     Install packages into the output directory.
     - copies build artifacts (wheel, zip, ...) to the output directory
-    - create pyodide_lock.json
+    - create pyodide-lock.json
 
 
     pkg_map
@@ -905,15 +903,15 @@ def install_packages(
 def set_default_build_args(build_args: BuildArgs) -> BuildArgs:
     args = dataclasses.replace(build_args)
 
-    if args.cflags is None:
-        args.cflags = build_env.get_build_flag("SIDE_MODULE_CFLAGS")  # type: ignore[unreachable]
-    if args.cxxflags is None:
-        args.cxxflags = build_env.get_build_flag("SIDE_MODULE_CXXFLAGS")  # type: ignore[unreachable]
-    if args.ldflags is None:
-        args.ldflags = build_env.get_build_flag("SIDE_MODULE_LDFLAGS")  # type: ignore[unreachable]
-    if args.target_install_dir is None:
-        args.target_install_dir = build_env.get_build_flag("TARGETINSTALLDIR")  # type: ignore[unreachable]
-    if args.host_install_dir is None:
-        args.host_install_dir = build_env.get_build_flag("HOSTINSTALLDIR")  # type: ignore[unreachable]
+    if not args.cflags:
+        args.cflags = build_env.get_build_flag("SIDE_MODULE_CFLAGS")
+    if not args.cxxflags:
+        args.cxxflags = build_env.get_build_flag("SIDE_MODULE_CXXFLAGS")
+    if not args.ldflags:
+        args.ldflags = build_env.get_build_flag("SIDE_MODULE_LDFLAGS")
+    if not args.target_install_dir:
+        args.target_install_dir = build_env.get_build_flag("TARGETINSTALLDIR")
+    if not args.host_install_dir:
+        args.host_install_dir = build_env.get_build_flag("HOSTINSTALLDIR")
 
     return args