nimporter-plus 0.0.2__py3-none-any.whl

nimporter_plus/__init__.py

@@ -0,0 +1,21 @@
+import sys
+import os
+from icecream import ic, colorama
+
+ic.configureOutput(
+    includeContext=True,
+    prefix=f'{colorama.Fore.CYAN}ic|{colorama.Fore.RESET} ',
+
+    # https://github.com/gruns/icecream/issues/35#issuecomment-908730426
+    outputFunction=lambda *args: print(*args)
+)
+
+ic.enabled = 'NIMPORTER_INSTRUMENT' in os.environ
+
+from nimporter_plus.lib import (
+    WINDOWS, MACOS, LINUX, EXT_DIR, PLATFORM_TABLE, ARCH_TABLE, compiler_args, get_host_info
+)
+
+import nimporter_plus.nimporter  # Register importers
+
+from nimporter_plus.nexporter import get_nim_extensions, build_nim_extensions

nimporter_plus/cli.py

@@ -0,0 +1,206 @@
+from __future__ import annotations
+
+import os
+import sys
+import time
+import shutil
+import pathlib
+import argparse
+import subprocess
+from typing import Any
+from pathlib import Path
+from cookiecutter.main import cookiecutter
+from nimporter_plus.lib import *
+from nimporter_plus.nimporter import *
+
+# TODO(pbz): Need to move this to a doc/tutorial
+SETUPPY_TEMPLATE: str = f'''
+# Setup.py tutorial:
+# https://github.com/navdeep-G/setup.py
+# Edit `packages=` to fit your requirements
+
+import setuptools, pathlib, sysconfig
+from setuptools.command.build_ext import build_ext
+import nimporter_plus
+
+
+class NoSuffixBuilder(build_ext):
+    """
+    Optional.
+
+    Removes the target platform, architecture, and Python version from the
+    final artifact.
+
+    Example:
+        The artifact: `module.linux-x86_64.cpython.3.8.5.so`
+        Becomes: `module.so`
+    """
+    def get_ext_filename(self, ext_name: str) -> str:
+        filename = super().get_ext_filename(ext_name)
+        ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
+        return filename.replace(ext_suffix, '') + pathlib.Path(filename).suffix
+
+
+setuptools.setup(
+    name="{ pathlib.Path().absolute().name }",
+    packages=[..],  # Please read the above tutorial
+    ext_modules=nimporter_plus.build_nim_extensions()
+    cmdclass={{"build_ext": NoSuffixBuilder}},
+)
+'''
+
+
+def nimporter_list() -> None:
+    for extension in find_extensions(Path()):
+        print(extension)
+        return
+
+
+def nimporter_clean(path: Path) -> None:
+    for item in path.iterdir():
+        item_full_path = item.resolve().absolute()
+
+        if item.is_dir():
+            DELETE_THESE_DIRS = {EXT_DIR, '__pycache__', '.pytest_cache'}
+
+            if item.stem in DELETE_THESE_DIRS:
+                print('Deleting', item_full_path)
+                shutil.rmtree(item)
+
+            elif item.name.endswith('.egg-info'):
+                print('Deleting', item_full_path)
+                shutil.rmtree(item)
+
+            elif item.stem == 'dist' and (item.parent / 'setup.py').exists():
+                print('Deleting', item_full_path)
+                shutil.rmtree(item)
+
+            elif item.stem == 'build' and (item.parent / 'setup.py').exists():
+                print('Deleting', item_full_path)
+                shutil.rmtree(item)
+
+            else:
+                nimporter_clean(item)
+    return
+
+
+def nimporter_init(extension_type: str, extension_name: str) -> None:
+    print(f'Initializing new extension {extension_type} "{extension_name}"')
+
+    if extension_type == 'mod':
+        Path(f'{extension_name}.nim').write_text(
+            'import nimpy\n\nproc add(a: int, b: int): int {.exportpy.} =\n'
+            '    return a + b\n'
+        )
+
+    elif extension_type == 'lib':
+        cookiecutter(
+            template='https://github.com/Pebaz/template-nimporter-ext-lib',
+            extra_context=dict(ext_name=extension_name),
+            no_input=True
+        )
+
+    else:
+        raise ValueError(
+            f'Extension is not one of [`mod`, `lib`], got: {extension_type}'
+        )
+    return
+
+
+def nimporter_compile() -> None:
+    def current_time_ms() -> float:
+        return round(time.time() * 1000)
+
+    overall_start = current_time_ms()
+
+    nimporter_clean(Path())
+
+    for ext in find_extensions(Path()):
+        print(
+            f'Building Extension {"Lib" if ext.is_dir() else "Mod"}: '
+            f'{ext.name}'
+        )
+
+        start = current_time_ms()
+        module_path = ext if ext.is_file() else ext / f'{ext.name}.nim'
+        compile_extension_to_lib(ExtLib(module_path, Path(), ext.is_dir()))
+        print('  Completed in', current_time_ms() - start, 'ms')
+
+    print(
+        'Completed all in',
+        (current_time_ms() - overall_start) / 1000.0,
+        'seconds'
+    )
+    return
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description='Nimporter CLI')
+    subs = parser.add_subparsers(dest='cmd', required=True)
+
+    # List command
+    subs.add_parser('list', help='List Nim extensions starting in current dir')
+
+    # Clean command
+    subs.add_parser(
+        'clean',
+        help=(
+            'Run in project root to recursively remove __pycache__, '
+            '.egg-info, build, and dist folders used by Nimporter'
+        )
+    )
+
+    # Init command
+    init = subs.add_parser(
+        'init',
+        help='Initializes the folder structure of a new extension'
+    )
+    init.add_argument(
+        'extension_type',
+        type=str,
+        help=(
+            'Either `mod` or `lib`. Extension modules are single files, '
+            'extension libraries are are fully configurable mini Nim projects'
+        )
+    )
+    init.add_argument(
+        'extension_name',
+        type=str,
+        help='The importable name of the extension module or library'
+    )
+
+    # Compile command
+    subs.add_parser(
+        'compile',
+        help='Precompile all extensions exactly as if they were imported'
+    )
+
+    return parser
+
+
+def main(cli_args: list[str] | None = None) -> int:
+    args = build_parser().parse_args(cli_args or sys.argv[1:])
+
+    if args.cmd == 'list':
+        nimporter_list()
+
+    elif args.cmd == 'clean':
+        # cwd = pathlib.Path()
+        # print('Cleaning Directory:', cwd.resolve())
+        # clean(cwd)
+        nimporter_clean(Path().resolve().absolute())
+
+    # elif args.cmd == 'bundle': # nimporter_bundle has nit been defined yet.
+        # nimporter_bundle(args.exp)
+
+    elif args.cmd == 'compile':
+        nimporter_compile()
+
+    elif args.cmd == 'init':
+        nimporter_init(args.extension_type, args.extension_name)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

nimporter_plus/lib.py

@@ -0,0 +1,345 @@
+from __future__ import annotations
+
+import os
+import sys
+import shlex
+import shutil
+import hashlib
+import tempfile
+import platform
+import sysconfig
+import subprocess
+import cpuinfo
+from typing import Any, Text
+from collections.abc import Iterator
+from pathlib import Path
+from contextlib import contextmanager
+from icecream import ic
+
+PathParts = tuple[str, str, str] | tuple[str] | tuple[str, str]
+
+PYTHON: str = 'python' if sys.platform == 'win32' else 'python3'
+PIP: str = 'pip' if shutil.which('pip') else 'pip3'
+PYTHON_LIB_EXT: str = sysconfig.get_config_var('EXT_SUFFIX')
+WINDOWS: str = 'windows'
+MACOS: str = 'darwin'
+LINUX: str = 'linux'
+EXT_DIR: str = 'nim-extensions'
+
+PLATFORM_TABLE: dict[str, str] = {  # Keys are known to Python and values are Nim-understood
+    'windows': 'Windows',
+    'darwin': 'MacOSX',
+    'linux': 'Linux',
+}
+
+ARCH_TABLE: dict[str, str] = {  # Keys are known to Python and values are Nim-understood
+    'x86_32': 'i386',
+    'x86_64': 'amd64',
+    'arm_8': 'arm64',
+    'aarch64': 'arm64',
+    'arm64': 'arm64',
+}
+
+compiler_args: list[str] = [
+]
+
+ALWAYS_ARGS: list[str] = [
+    'nimble',  # Installs dependencies :)
+    'c',
+    '--accept',  # Allow installing dependencies
+    '--skipUserCfg',
+    '--app:lib',
+    '--backend:c',
+    '--threads:on',
+    '--warning[ProveInit]:off',  # https://github.com/Pebaz/nimporter/issues/41
+]
+
+
+def find_extensions(path: Path) -> list[Path]:
+    nim_exts = []
+
+    for item in path.iterdir():
+        if item.is_dir() and list(item.glob('*.nimble')):
+            # Treat directory as one single Extension
+            (nimble_file,) = item.glob('*.nimble')
+
+            # NOTE(pbz): Folder must contain .nimble file of exact same name
+            nim_file = nimble_file.parent / (nimble_file.stem + '.nim')
+
+            # NOTE(pbz): Folder must contain Nim file of exact same name.
+            if nim_file.exists():
+                nim_exts.append(item)
+
+        elif item.is_dir():
+            # Treat item as directory
+            nim_exts.extend(find_extensions(item))
+
+        elif item.suffix == '.nim':
+            # Treat item as a Nim Extension.
+            nim_exts.append(item)
+
+    return nim_exts
+
+
+def get_import_path(path: Path, root: Path) -> str:
+    """
+    Coerce proper import path using root path
+
+    Args:
+        library(bool): hint as to how to treat the `module_path` parameter.
+        module_name(str): name of nim module.
+        module_path(Path): path to nim module.
+        root(Path): path to project root.
+
+    Returns:
+        str: Returns the path of the nim module.
+    """
+
+    def get_import_prefix(module_path: Path, root: Path) -> PathParts:
+        root_path = root.resolve()
+        full_path = module_path.resolve()
+
+        assert full_path >= root_path, 'Extension path is not within root dir.'
+
+        return ic(full_path.parts[len(root_path.parts):])
+
+    library = path.is_dir()
+    module_name = path.stem
+    module_path = path if path.is_file() else path / path.stem
+    import_prefix = get_import_prefix(module_path.parent, root)
+    module_part = tuple() if library else (module_name,)
+    import_path = '.'.join(import_prefix + module_part)
+
+    return ic(import_path)
+
+
+@contextmanager
+def convert_to_lib_if_needed(path: Path) -> Iterator[Path]:
+    if path.is_file():
+        package_root = None
+        for parent in path.parents:
+            nimbles = list(parent.glob('*.nimble'))
+            matching = [n for n in nimbles if n.stem in (path.stem, parent.name)]
+            if matching:
+                package_root = parent
+                break
+
+        if package_root:
+            yield package_root
+        else:
+            with tempfile.TemporaryDirectory() as compilation_dir:
+                shutil.copy(path, compilation_dir)
+                fake_nimble = Path(compilation_dir) / path.with_suffix('.nimble').name
+                fake_nimble.write_text('requires "nimpy"\n')
+                yield Path(compilation_dir)
+    else:
+        yield path
+
+
+def run_process(
+    process_args: list[str],
+    show_output: bool = False,
+) -> tuple[int, bytes | Text, bytes | Text]:
+    """
+    Invokes the compiler (or any executable) and returns the output.
+
+    While this can (and has been) used to call executables other than Nim
+    and Nimble, it should be noted that the warnings and hints are artifacts
+    of being mainly targeted as a Nim compiler invoker.
+
+    Args:
+        process_args(list): the arg being the executable and the rest are args.
+
+    Returns:
+        A tuple containing any errors, warnings, or hints from the
+        compilation process.
+    """
+    try:
+        process = subprocess.run(
+            process_args,
+            stdout=None if show_output else subprocess.PIPE,
+            stderr=None if show_output else subprocess.PIPE,
+        )
+    except FileNotFoundError:
+        executable = process_args[0] if process_args else "Command"
+        raise FileNotFoundError(
+            f"Cannot find '{executable}'. Please ensure it is installed and in your system PATH."
+        )
+
+    code, out, err = process.returncode, process.stdout, process.stderr
+    out = out.decode(errors='ignore') if out else '' # type: ignore[assignment]
+    err = err.decode(errors='ignore') if err else '' # type: ignore[assignment]
+
+    return code, out, err
+
+
+@contextmanager
+def cd(path: Path) -> Iterator[Path]:
+    "Convenience function to step in and out of a directory temporarily."
+    cwd = os.getcwd()
+    os.chdir(path)
+    try:
+        yield path
+    finally:
+        os.chdir(cwd)
+
+
+def get_c_compiler_used_to_build_python() -> str:
+    "This func is included just to be a bit clearer as to its significance."
+    return 'vcc' if 'MSC' in sys.version else 'gcc'
+
+
+def get_host_info() -> tuple[str, str, str]:
+    """
+    Returns the host platform, architecture, and C compiler used to build the
+    running Python process.
+    """
+    # Calling get_cpu_info() is expensive
+    if not getattr(get_host_info, 'host_arch', None):
+        setattr(get_host_info, 'host_arch', cpuinfo.get_cpu_info()['arch'].lower())
+        # get_host_info.host_arch = cpuinfo.get_cpu_info()['arch'].lower()
+
+    return ic((
+        platform.system().lower(),
+        get_host_info.host_arch, # type: ignore[attr-defined]
+        get_c_compiler_used_to_build_python()
+    ))
+
+
+def ensure_nimpy() -> None:
+    """
+    Makes sure that the Nimpy Nim library is installed.
+
+    Verifies that the [Nimpy Library](https://github.com/yglukhov/nimpy) is
+    installed and installs it otherwise.
+
+    NOTE: Nimporter would not be possible without Nimpy. Thank you
+    Yuriy Glukhov for making this project possible!
+    """
+    ic()
+
+    show_output = 'NIMPORTER_INSTRUMENT' in os.environ
+    code, *_ = run_process(shlex.split('nimble path nimpy'), show_output)
+
+    if code != 0:
+        ic()
+        nimble_args = shlex.split('nimble install nimpy --accept')
+        code, _, stderr = run_process(nimble_args, show_output)
+
+        if code:
+            raise CompilationFailedException(stderr)
+    return
+
+
+class NimporterException(Exception):
+    "Base exception for Nimporter's exception hierarchy."
+    pass
+
+
+class CompilationFailedException(NimporterException):
+    def __init__(self, stderr: bytes | str) -> None:
+        if isinstance(stderr, bytes):
+            stderr_str = stderr.decode(errors='ignore')
+        else:
+            stderr_str = str(stderr)
+        super().__init__(
+            f'\n{"="*60}\nNim Compilation Failed:\n{stderr_str.strip()}\n{"="*60}'
+        )
+        return
+
+
+class ImportFailedException(NimporterException):
+    "Custom exception for when compilation succeeds but importing fails."
+    pass
+
+
+def hash_extension(module_path: Path) -> bytes:
+    """
+    Convenience function to hash an extension module or extension library recursively,
+    including any associated project configuration files (.nimble, config.nims, .cfg).
+    """
+    digest = hashlib.sha256()
+    root_folder = module_path.parent if module_path.is_file() else module_path
+
+    def walk_folder(path: Path) -> Iterator[Path]:
+        for item in sorted(path.iterdir()):
+            if item.is_dir():
+                if item.name in ('__pycache__', 'nimcache', '.git', '.venv', 'build', 'dist', '.pytest_cache', 'node_modules'):
+                    continue
+                yield from walk_folder(item)
+            elif item.suffix in ('.nim', '.nimble', '.nims', '.cfg', '.json', '.c', '.h'):
+                yield item
+
+    for item in walk_folder(root_folder):
+        digest.update(str(item.relative_to(root_folder)).encode())
+        with item.open('rb') as f:
+            while buf := f.read(65536):
+                digest.update(buf)
+
+    # Also check parent directories up to project root for .nimble or config.nims
+    for parent in root_folder.parents:
+        if parent.name in ('dl', 'Users', 'home', ''):
+            break
+        configs = sorted([
+            c for c in parent.iterdir()
+            if c.is_file() and c.suffix in ('.nimble', '.nims', '.cfg')
+        ])
+        for cfg in configs:
+            digest.update(str(cfg).encode())
+            with cfg.open('rb') as f:
+                while buf := f.read(65536):
+                    digest.update(buf)
+
+    ic(digest.hexdigest())
+    return digest.digest()
+
+
+class ExtLib:
+    """
+    All extensions are assumed to be libraries only. Modules are convert to
+    libraries as needed.
+    """
+    def __init__(self, path: Path, root: Path, library_hint: bool) -> None:
+        """
+        Args:
+            path(str): the relative path to the Nim file (for both lib & mod).
+        """
+        self.library = all((
+            any(path.parent.glob(f'{path.stem}.nim')),
+            any(path.parent.glob(f'{path.stem}.nimble')),
+            any(path.parent.glob(f'{path.stem}.nim.cfg'))
+        ))
+
+        assert library_hint and self.library if library_hint else True, (
+            f'ExtLib must define: {path.stem}.nim, {path.stem}.nimble, and '
+            f'{path.stem}.nim.cfg'
+        )
+
+        self.symbol = path.stem
+        self.module_path = path
+
+        if self.library:
+            self.relative_path = path.parent
+            self.full_path = path.parent.resolve().absolute()
+
+        else:
+            self.relative_path = path
+            self.full_path = path.resolve().absolute()
+
+        self.pycache = self.full_path.parent / '__pycache__'
+
+        self.import_namespace = get_import_path(self.relative_path, root)
+        self.hash_filename = self.pycache / f'{self.symbol}.hash'
+        self.build_artifact = (
+            self.pycache / f'{self.symbol}{PYTHON_LIB_EXT}'
+        )
+        return
+
+    def __str__(self) -> str:
+        return f'<ExtLib {self.import_namespace}>'
+
+    def __repr__(self) -> str:
+        return str(self)
+
+    def __format__(self, *args, **kwargs) -> str: # type: ignore[no-untyped-def]
+        return str(self)