hyperray 0.0.1__py3-none-any.whl

hyperdrone/__init__.py

@@ -0,0 +1,53 @@
+"""hyperdrone — drone simulation stack for RLtools.
+
+Subpackages:
+  hyperdrone.render   — raytracing renderer (self-contained, no drone dynamics)
+  hyperdrone.dynamics — L2F multirotor simulator (vectorized, stateful Sim; pure dynamics)
+  hyperdrone.env      — the RL environment: the C++ MultiEnvironment behind the batch verbs
+  hyperdrone.gym      — optional Gymnasium VectorEnv adapter (pip install "hyperdrone[gym]")
+
+Render and dynamics are independent peers coupled by DLPack camera bases; env integrates them through the C++ environment. Native components are JIT-compiled per set of compile-time constants and
+cached under ~/.cache/hyperdrone (override with HYPERDRONE_CACHE_DIR). Importing
+hyperdrone (or any subpackage) is cheap; the first build happens on first use.
+set_verbosity(level) selects the native log level (error|warning|info|debug, default info).
+"""
+import importlib
+
+__all__ = ["set_verbosity", "verbosity"]
+
+__version__ = "0.0.1"
+
+_SUBPACKAGES = ("render", "dynamics", "env", "gym")
+_VERBOSITY_LEVELS = ("error", "warning", "info", "debug")
+_verbosity = "info"
+_logging_extensions = []
+
+
+def set_verbosity(level):
+    """Set the level for subsequent native diagnostic checks in loaded and future extensions."""
+    global _verbosity
+    if level not in _VERBOSITY_LEVELS:
+        raise ValueError(f"hyperdrone: invalid verbosity {level!r} ({'|'.join(_VERBOSITY_LEVELS)})")
+    _verbosity = level
+    for extension in _logging_extensions:
+        extension._set_verbosity(_VERBOSITY_LEVELS.index(level))
+
+
+def verbosity():
+    return _verbosity
+
+
+def _register_extension(extension):
+    if hasattr(extension, "_set_verbosity"):
+        extension._set_verbosity(_VERBOSITY_LEVELS.index(_verbosity))
+        _logging_extensions.append(extension)
+
+
+def __getattr__(name):
+    if name in _SUBPACKAGES:
+        return importlib.import_module(f".{name}", __name__)
+    raise AttributeError(f"module 'hyperdrone' has no attribute {name}")
+
+
+def __dir__():
+    return sorted(list(globals()) + list(_SUBPACKAGES))

hyperdrone/_cuda/__init__.py

@@ -0,0 +1,76 @@
+"""hyperdrone._cuda — CUDA staging helpers shared by render and dynamics.
+
+The native CudaBuffer class is compiled into every CUDA-capable component core
+(render on OptiX, dynamics on CUDA). This package defines the Python-side views and
+resolves a provider registered by render or a CUDA Sim, keeping the dependency direction render/dynamics -> _cuda.
+"""
+import numpy as np
+
+_providers = []
+
+
+def register_provider(load_core):
+    """Register a zero-argument core loader that may expose CudaBuffer."""
+    if load_core not in _providers:
+        _providers.append(load_core)
+
+
+def _core():
+    errors = []
+    for load_core in _providers:
+        try:
+            core = load_core()
+        except Exception as error:  # provider may fail to build on this machine
+            errors.append(str(error))
+            continue
+        if getattr(core, "HAS_CUDA", False):
+            return core
+    detail = ("; ".join(errors)) if errors else "no CUDA-capable component core is available"
+    raise RuntimeError(
+        "hyperdrone: CUDA staging needs a CUDA-capable component (render on the OptiX "
+        f"backend, or a CUDA Sim); initialize one first ({detail})"
+    )
+
+
+class CudaView:
+    """DLPack producer over one slice of an uploaded CUDA buffer. Fresh capsule per
+    __dlpack__ call; holds the parent buffer alive."""
+
+    def __init__(self, buffer, index):
+        self._buffer = buffer
+        self._index = index
+
+    def __dlpack__(self, stream=None, **_kwargs):
+        if self._index is None:
+            return self._buffer.dlpack()
+        return self._buffer.slice_dlpack(self._index)
+
+    def __dlpack_device__(self):
+        return (2, self._buffer.device_id)
+
+
+class TensorSet:
+    """A host float32 array uploaded to CUDA device memory once; view(i) exposes slice i
+    along axis 0 as a zero-copy DLPack CUDA tensor (e.g. one camera set per slice)."""
+
+    def __init__(self, array):
+        self._buffer = _core().CudaBuffer(np.ascontiguousarray(array, dtype=np.float32))
+
+    @property
+    def shape(self):
+        return tuple(self._buffer.shape)
+
+    @property
+    def num_sets(self):
+        return self.shape[0]
+
+    @property
+    def raw(self):
+        return self._buffer
+
+    def view(self, index=None):
+        return CudaView(self._buffer, index)
+
+
+def upload(array):
+    return TensorSet(array)

hyperdrone/_dlpack.py

@@ -0,0 +1,30 @@
+"""DLPack plumbing shared by the render, dynamics, and env packages."""
+
+KDL_CPU = 1
+KDL_CUDA = 2
+
+
+class DLPackBuffer:
+    """DLPack producer over a live native buffer. Capsules are single-use, so each
+    __dlpack__ call mints a fresh one; the owner reference keeps the memory alive."""
+
+    def __init__(self, owner, make_capsule, device_type, device_id=0, order_consumer=None):
+        self._owner = owner
+        self._make_capsule = make_capsule
+        self._device_type = device_type
+        self._device_id = device_id
+        self._order_consumer = order_consumer
+
+    def __dlpack__(self, stream=None, **_kwargs):
+        if self._order_consumer is not None and stream not in (None, -1):
+            self._order_consumer(int(stream))
+        return self._make_capsule()
+
+    def __dlpack_device__(self):
+        return (self._device_type, self._device_id)
+
+
+def is_cuda(value):
+    """True for DLPack producers that report CUDA device memory."""
+    device = getattr(value, "__dlpack_device__", None)
+    return device is not None and device()[0] == KDL_CUDA

hyperdrone/_jit/__init__.py

@@ -0,0 +1,131 @@
+"""hyperdrone._jit — shared compile-and-cache infrastructure.
+
+A *component* is a native code unit under hyperdrone/_native/<name>/: an optional nanobind
+core module built once per build tree, plus JIT artifacts compiled per configuration:
+Python extension modules or shared libraries with a C ABI.
+
+Configs implement the JitConfig protocol: defines() -> {NAME: value} compile definitions,
+canonical() -> stable human-readable string of the configuration, key() -> its short hash,
+which names the artifact. Config files land as configs/<key>.txt inside the
+build tree, one NAME=VALUE line each; the shared hyperdrone_jit.cmake turns each file into
+a MODULE target named <component>_<key>.
+
+Build trees live at <cache>/<component>-<variant>-<tag> (one per component and variant so
+switching variants never invalidates another's cache); all configure/build steps run under
+a per-tree file lock so concurrent processes cannot race one CMake tree.
+"""
+import hashlib
+import importlib.util
+import sys
+from dataclasses import dataclass, field
+
+from .. import _register_extension
+from . import _build
+from ._workspace import BuildError, build_dir, cache_root, native_root, source_root, workspace_tag
+
+__all__ = [
+    "BuildError",
+    "Component",
+    "build_dir",
+    "cache_root",
+    "canonical_key",
+    "core",
+    "ensure",
+    "native_root",
+    "module",
+    "source_root",
+    "workspace_tag",
+]
+
+_core_modules = {}
+
+
+@dataclass(frozen=True)
+class Component:
+    name: str
+    variant: str
+    core_target: str = None
+    needs_cuda: bool = False
+    cmake_defines: tuple = field(default_factory=tuple)
+    source_dir: str = None  # default: hyperdrone/_native/<name>
+
+
+def canonical_key(canonical):
+    return hashlib.sha1(canonical.encode()).hexdigest()[:16]
+
+
+def _write_config_file(directory, config):
+    import os
+    config_dir = directory / "configs"
+    config_dir.mkdir(exist_ok=True)
+    config_file = config_dir / f"{config.key()}.txt"
+    if config_file.exists():
+        return False
+    lines = "".join(f"{name}={value}\n" for name, value in sorted(config.defines().items()))
+    # per-process staging name: concurrent writers of the same config must not race on one
+    # staging path; the final rename is atomic and idempotent (identical content)
+    staging = config_dir / f"{config.key()}.{os.getpid()}.tmp"
+    staging.write_text(lines)
+    staging.rename(config_file)
+    return True
+
+
+def ensure(component, config):
+    """Build (if necessary) and return the path of the JIT artifact for this config."""
+    directory = _build.ensure_configured(component)
+    fresh = _write_config_file(directory, config)
+    artifact = directory / "jit" / f"{component.name}_{config.key()}.so"
+    if fresh or not artifact.exists():
+        # the target only exists after the configs/ glob is re-run
+        _build.reconfigure(component)
+    _build.build_target(component, f"{component.name}_{config.key()}")
+    return artifact
+
+
+def core(component):
+    """Build (if necessary), load, and cache the component's nanobind core module.
+
+    Extension modules are identified by name process-wide, so one component core can only
+    be loaded for one variant per process (e.g. dynamics on cpu OR cuda, not both)."""
+    if component.core_target is None:
+        raise BuildError(f"hyperdrone: component {component.name} declares no core module")
+    cached = _core_modules.get(component.core_target)
+    if cached is not None:
+        variant, module = cached
+        if variant != component.variant:
+            raise BuildError(
+                f"hyperdrone: {component.core_target} is already loaded for variant "
+                f"'{variant}'; one process can use only one {component.name} variant"
+            )
+        return module
+    directory = build_dir(component)
+    if (directory / "CMakeCache.txt").exists() and not sorted(directory.glob(f"{component.core_target}*.so")):
+        # a core target added since this workspace was configured exists only after a reconfigure
+        _build.reconfigure(component)
+    directory = _build.build_target(component, component.core_target)
+    matches = sorted(directory.glob(f"{component.core_target}*.so"))
+    if not matches:
+        raise BuildError(f"hyperdrone: {component.core_target} module not found in {directory}")
+    extension = _load_extension(component.core_target, matches[0])
+    _core_modules[component.core_target] = (component.variant, extension)
+    return extension
+
+
+def _load_extension(name, path):
+    cached = sys.modules.get(name)
+    if cached is not None:
+        return cached
+    specification = importlib.util.spec_from_file_location(name, path)
+    extension = importlib.util.module_from_spec(specification)
+    specification.loader.exec_module(extension)
+    sys.modules[name] = extension
+    _register_extension(extension)
+    return extension
+
+
+def module(component, config):
+    """Build and import a configuration's Python extension, after its shared core."""
+    if component.core_target is not None:
+        core(component)
+    artifact = ensure(component, config)
+    return _load_extension(f"hyperdrone_{component.name}_{component.variant}_{config.key()}", artifact)

hyperdrone/_jit/_build.py

@@ -0,0 +1,113 @@
+import os
+import subprocess
+from pathlib import Path
+
+from ._lock import FileLock
+from ._patches import seed_dependencies
+from ._toolchain import environment
+from ._workspace import BuildError, build_dir, dependencies_root, native_root, source_root
+
+BUILD_JOBS = os.environ.get("HYPERDRONE_BUILD_JOBS", "5")
+
+
+def lock(component):
+    directory = build_dir(component)
+    return FileLock(directory.parent / f"{directory.name}.lock")
+
+
+def run(arguments, env):
+    process = subprocess.run(
+        [str(argument) for argument in arguments],
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env,
+    )
+    if process.returncode != 0:
+        tail = "\n".join(process.stdout.splitlines()[-60:])
+        raise BuildError(
+            f"hyperdrone: command failed: {' '.join(str(argument) for argument in arguments)}\n{tail}"
+        )
+    return process.stdout
+
+
+def source_dir(component):
+    if component.source_dir is not None:
+        from pathlib import Path
+        return Path(component.source_dir)
+    return native_root() / component.name
+
+
+def fetchcontent_base(directory):
+    return (dependencies_root() / directory.name).resolve()
+
+
+def configured_fetchcontent_base(directory):
+    cache_file = directory / "CMakeCache.txt"
+    if not cache_file.exists():
+        return None
+    prefix = "FETCHCONTENT_BASE_DIR:"
+    for line in cache_file.read_text().splitlines():
+        if line.startswith(prefix):
+            return Path(line.split("=", 1)[1]).resolve()
+    return None
+
+
+def configure(component, directory):
+    import sys
+    seeded = seed_dependencies(directory.name)
+    arguments = [
+        "cmake",
+        "-S", source_dir(component),
+        "-B", directory,
+        "-DCMAKE_BUILD_TYPE=Release",
+        f"-DRLTOOLS_ROOT={source_root()}",
+        f"-DHYPERDRONE_VARIANT={component.variant.upper()}",
+        f"-DHYPERDRONE_NATIVE_ROOT={native_root()}",
+        f"-DPython_EXECUTABLE={sys.executable}",
+        # fully qualified: the rl_tools root CMakeLists cannot append the build-dir
+        # basename to an externally-passed value (non-FORCE cache set)
+        f"-DFETCHCONTENT_BASE_DIR={fetchcontent_base(directory)}",
+    ]
+    for fetch_name, path in sorted(seeded.items()):
+        arguments.append(f"-DFETCHCONTENT_SOURCE_DIR_{fetch_name}={path}")
+    for name, value in component.cmake_defines:
+        arguments.append(f"-D{name}={value}")
+    if os.environ.get("HYPERDRONE_OFFLINE"):
+        arguments.append("-DRL_TOOLS_OFFLINE_BUILD=ON")
+    run(arguments, environment(component))
+
+
+def ensure_configured(component):
+    directory = build_dir(component)
+    expected_fetchcontent_base = fetchcontent_base(directory)
+    if configured_fetchcontent_base(directory) == expected_fetchcontent_base:
+        return directory
+    with lock(component):
+        if configured_fetchcontent_base(directory) != expected_fetchcontent_base:
+            fresh = not (directory / "CMakeCache.txt").exists()
+            directory.mkdir(parents=True, exist_ok=True)
+            try:
+                configure(component, directory)
+            except BaseException:
+                # a failed configure must not leave a tree that looks configured; the
+                # FetchContent state lives in .dependencies/ and survives for the retry
+                if fresh:
+                    import shutil
+                    shutil.rmtree(directory, ignore_errors=True)
+                raise
+    return directory
+
+
+def build_target(component, target):
+    directory = ensure_configured(component)
+    if os.environ.get("HYPERDRONE_SKIP_BUILD"):
+        return directory
+    with lock(component):
+        run(["cmake", "--build", directory, "--target", target, "-j", BUILD_JOBS],
+            environment(component))
+    return directory
+
+
+def reconfigure(component):
+    directory = ensure_configured(component)
+    with lock(component):
+        run(["cmake", directory], environment(component))
+    return directory

hyperdrone/_jit/_lock.py

@@ -0,0 +1,23 @@
+"""Advisory file lock serializing configure/build on one build tree across processes
+(vectorized RL workloads routinely spawn many workers that would otherwise race the same
+CMake tree). POSIX flock; a Windows port replaces this module with an msvcrt equivalent."""
+import fcntl
+import os
+
+
+class FileLock:
+    def __init__(self, path):
+        self._path = path
+        self._handle = None
+
+    def __enter__(self):
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        self._handle = os.open(self._path, os.O_CREAT | os.O_RDWR, 0o644)
+        fcntl.flock(self._handle, fcntl.LOCK_EX)
+        return self
+
+    def __exit__(self, *exception):
+        fcntl.flock(self._handle, fcntl.LOCK_UN)
+        os.close(self._handle)
+        self._handle = None
+        return False

hyperdrone/_jit/_patches.py

@@ -0,0 +1,99 @@
+"""Reuse dependencies from sibling JIT build caches.
+
+Git sources use FETCHCONTENT_SOURCE_DIR overrides, with patch scripts applied explicitly.
+Assimp reuses downloaded archives so CMake verifies URL_HASH before extracting them.
+"""
+import re
+import shutil
+import subprocess
+
+from ._workspace import source_root
+
+# FetchContent name -> source directory basename
+SEEDED_SOURCES = {
+    "OWL": "owl-src",
+    "STB": "stb-src",
+    "NLOHMANN_JSON": "nlohmann_json-src",
+}
+
+# FetchContent name -> cmake file declaring its GIT_TAG; seeds are validated against the
+# declared pin so a repin upstream can never be shadowed by a stale seeded checkout
+PINNED_SOURCES = {
+    "OWL": "cmake/optional/optix.cmake",
+}
+
+
+def declared_pin(fetch_name):
+    relative = PINNED_SOURCES.get(fetch_name)
+    if relative is None:
+        return None
+    declaration = source_root() / relative
+    if not declaration.exists():
+        return None
+    match = re.search(
+        r"FetchContent_Declare\(\s*" + fetch_name + r"\b.*?GIT_TAG\s+([0-9a-fA-F]{7,40})",
+        declaration.read_text(), re.IGNORECASE | re.DOTALL,
+    )
+    return match.group(1) if match else None
+
+
+def checkout_head(tree):
+    result = subprocess.run(
+        ["git", "-C", str(tree), "rev-parse", "HEAD"], capture_output=True, text=True,
+    )
+    return result.stdout.strip() if result.returncode == 0 else None
+
+
+def matches_pin(tree, pin):
+    return pin is None or checkout_head(tree) == pin
+
+# cmake -P scripts under <source_root>/cmake/patches, applied to the named FetchContent
+# source dir; each script is idempotent and documents its upstream-removal condition.
+# (Empty since the OWL PinnedHostMem fix landed upstream with the 978a6664 repin.)
+PATCH_SCRIPTS = {}
+
+
+def apply_patch_scripts(seeded):
+    for fetch_name, (script_name, root_variable) in PATCH_SCRIPTS.items():
+        tree = seeded.get(fetch_name)
+        script = source_root() / "cmake" / "patches" / script_name
+        if tree is not None and script.exists():
+            subprocess.run(
+                ["cmake", f"-D{root_variable}={tree}", "-P", str(script)],
+                check=True, capture_output=True,
+            )
+
+
+def seed_dependencies(build_dir_name):
+    """Copy FetchContent sources from a populated sibling; returns {FETCH_NAME: path} for
+    every seeded (or already present) source directory."""
+    from ._workspace import dependencies_root
+    dependency_root = dependencies_root()
+    target_base = dependency_root / build_dir_name
+    donors = sorted(
+        candidate for candidate in dependency_root.glob("*")
+        if candidate.is_dir() and candidate.name != build_dir_name
+    ) if dependency_root.is_dir() else []
+    seeded = {}
+    for donor in donors:
+        for archive in (donor / "assimp-download").glob("*.tar.gz"):
+            target = target_base / "assimp-download" / archive.name
+            if not target.exists():
+                target.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(archive, target)
+    for fetch_name, source_name in SEEDED_SOURCES.items():
+        pin = declared_pin(fetch_name)
+        target = target_base / source_name
+        if target.exists() and not matches_pin(target, pin):
+            shutil.rmtree(target)
+        if not target.exists():
+            for donor in donors:
+                source = donor / source_name
+                if source.is_dir() and matches_pin(source, pin):
+                    target_base.mkdir(parents=True, exist_ok=True)
+                    shutil.copytree(source, target, symlinks=True)
+                    break
+        if target.is_dir():
+            seeded[fetch_name] = target
+    apply_patch_scripts(seeded)
+    return seeded

hyperdrone/_jit/_toolchain.py

@@ -0,0 +1,17 @@
+import os
+import shutil
+from pathlib import Path
+
+
+def cuda_available():
+    return Path("/usr/local/cuda/bin/nvcc").exists() or shutil.which("nvcc") is not None
+
+
+def environment(component):
+    env = os.environ.copy()
+    if component.needs_cuda and "CUDACXX" not in env and "CMAKE_CUDA_COMPILER" not in env:
+        for candidate in ("/usr/local/cuda/bin/nvcc", shutil.which("nvcc")):
+            if candidate and Path(candidate).exists():
+                env["CUDACXX"] = str(candidate)
+                break
+    return env

hyperdrone/_jit/_workspace.py

@@ -0,0 +1,94 @@
+import hashlib
+import os
+import platform
+import subprocess
+import sysconfig
+from pathlib import Path
+
+
+class BuildError(RuntimeError):
+    pass
+
+
+def cache_root():
+    env = os.environ.get("HYPERDRONE_CACHE_DIR")
+    if env:
+        return Path(env)
+    return Path.home() / ".cache" / "hyperdrone"
+
+
+def package_root():
+    return Path(__file__).resolve().parents[1]
+
+
+def native_root():
+    return package_root() / "_native"
+
+
+def source_root():
+    """rl-tools source root: env override, enclosing repository checkout, vendored tree."""
+    env = os.environ.get("HYPERDRONE_RLTOOLS_ROOT")
+    if env:
+        root = Path(env).resolve()
+        if not (root / "include" / "rl_tools").is_dir():
+            raise BuildError(
+                f"hyperdrone: HYPERDRONE_RLTOOLS_ROOT={root} is not an rl-tools source root "
+                "(include/rl_tools missing)"
+            )
+        return root
+    candidate = package_root()
+    for _ in range(8):
+        if (candidate / "include" / "rl_tools").is_dir():
+            return candidate
+        candidate = candidate.parent
+    vendored = package_root() / "_vendor" / "rl-tools"
+    if (vendored / "include" / "rl_tools").is_dir():
+        return vendored
+    raise BuildError(
+        "hyperdrone: rl-tools sources not found (no enclosing repository checkout and no "
+        "vendored tree; set HYPERDRONE_RLTOOLS_ROOT to an rl-tools source root)"
+    )
+
+
+def dependencies_root():
+    """FetchContent sources and build state live under the configured cache root.
+
+    Source checkouts and installed packages may be read-only; neither should accumulate
+    generated files. Keeping dependencies beside the component build trees also makes
+    HYPERDRONE_CACHE_DIR a complete, relocatable build cache.
+    """
+    return cache_root() / ".dependencies"
+
+
+def host_target():
+    try:
+        with open("/proc/cpuinfo") as cpuinfo:
+            for line in cpuinfo:
+                name, separator, value = line.partition(":")
+                if separator and name.strip() in ("flags", "Features"):
+                    return platform.machine() + "|" + " ".join(sorted(value.split()))
+    except OSError:
+        pass
+    return platform.machine() + "|" + platform.processor()
+
+
+def cuda_target():
+    if os.environ.get("CUDAARCHS"):
+        return os.environ["CUDAARCHS"]
+    try:
+        result = subprocess.run(["nvidia-smi", "--query-gpu=compute_cap", "--format=csv,noheader"], capture_output=True, text=True, check=True, timeout=5)
+        return ";".join(sorted(set(result.stdout.split()))) + ("|visible=" + os.environ["CUDA_VISIBLE_DEVICES"] if "CUDA_VISIBLE_DEVICES" in os.environ else "")
+    except (OSError, subprocess.SubprocessError) as error:
+        raise BuildError("hyperdrone: cannot identify the native CUDA target; set CUDAARCHS explicitly") from error
+
+
+def workspace_tag(component=None):
+    from .. import __version__
+    fingerprint = f"{source_root()}|{sysconfig.get_config_var('SOABI')}|{__version__}|{host_target()}"
+    if component is not None and component.needs_cuda:
+        fingerprint += "|" + cuda_target()
+    return hashlib.sha1(fingerprint.encode()).hexdigest()[:8]
+
+
+def build_dir(component):
+    return cache_root() / f"{component.name}-{component.variant}-{workspace_tag(component)}"