ernie-renderer 0.2.0.dev117__tar.gz → 0.2.0.dev121__tar.gz

{ernie_renderer-0.2.0.dev117 → ernie_renderer-0.2.0.dev121}/CMakeLists.txt

@@ -15,7 +15,7 @@ project(
 )
 
 # Pre-release suffix such as .dev12, written by CI.
-set(ERNIE_VERSION_SUFFIX ".dev117" CACHE STRING "Pre-release suffix of Ernie's version")
+set(ERNIE_VERSION_SUFFIX ".dev121" CACHE STRING "Pre-release suffix of Ernie's version")
 set(ERNIE_VERSION "${PROJECT_VERSION}${ERNIE_VERSION_SUFFIX}")
 
 option(ERNIE_BUILD_TESTS "Build Ernie's tests" OFF)
@@ -54,6 +54,7 @@ include(cmake/ErnieFetch.cmake)
 include(cmake/third_party/openexr.cmake)
 include(cmake/third_party/tracy.cmake)
 include(cmake/third_party/slang.cmake)
+include(cmake/ErnieShaders.cmake)
 add_subdirectory(third_party)
 
 add_library(ernie_build_options INTERFACE)

{ernie_renderer-0.2.0.dev117 → ernie_renderer-0.2.0.dev121}/Dockerfile

@@ -17,6 +17,8 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=build /opt/ernie/bin/ernie /usr/local/bin/ernie
+# Its private libraries, as Slang's compiler, where its rpath ($ORIGIN/../lib/ernie) looks.
+COPY --from=build /opt/ernie/lib/ernie /usr/local/lib/ernie
 # Data and the licences of everything the program carries.
 COPY --from=build /opt/ernie/share /usr/local/share
 ENV NVIDIA_DRIVER_CAPABILITIES=graphics,compute,utility

{ernie_renderer-0.2.0.dev117 → ernie_renderer-0.2.0.dev121}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ernie-renderer
-Version: 0.2.0.dev117
+Version: 0.2.0.dev121
 Summary: Efficient Renderer for Natural and Interplanetary Environments: a physically based, spectral Vulkan ray tracer
 Author: Alex Maietta
 License-Expression: BSD-3-Clause

ernie_renderer-0.2.0.dev121/cmake/EmbedShader.cmake

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Embeds one Slang file in a header as an ernie::ShaderModule
+file(READ "${INPUT}" source)
+string(FIND "${source}" ")slang\"" delimiter)
+if(NOT delimiter EQUAL -1)
+    message(FATAL_ERROR "${INPUT} holds )slang\", which ends the raw string it is embedded in")
+endif()
+
+string(REPLACE "_" ";" words "${NAME}")
+set(identifier "k")
+foreach(word IN LISTS words)
+    string(SUBSTRING "${word}" 0 1 first)
+    string(SUBSTRING "${word}" 1 -1 rest)
+    string(TOUPPER "${first}" first)
+    string(APPEND identifier "${first}${rest}")
+endforeach()
+
+set(chunks "")
+string(LENGTH "${source}" length)
+set(chunk 8192)
+set(at 0)
+while(at LESS length)
+    string(SUBSTRING "${source}" ${at} ${chunk} piece)
+    string(APPEND chunks "    R\"slang(${piece})slang\"\n")
+    math(EXPR at "${at} + ${chunk}")
+endwhile()
+if(chunks STREQUAL "")
+    set(chunks "    \"\"\n")
+endif()
+
+file(RELATIVE_PATH from "${ROOT}" "${INPUT}")
+set(header
+    "// Generated from ${from} by cmake/EmbedShader.cmake: edit that file.\n#pragma once\n\n#include \"ernie/device/shader_compiler.h\"\n\nnamespace ernie::shaders {\n\ninline const ShaderModule ${identifier}{\n    \"${NAME}\",\n${chunks}};\n\n}  // namespace ernie::shaders\n"
+)
+# Unchanged headers keep their times, so nothing that includes them rebuilds.
+if(EXISTS "${OUTPUT}")
+    file(READ "${OUTPUT}" previous)
+    if(previous STREQUAL header)
+        return()
+    endif()
+endif()
+file(WRITE "${OUTPUT}" "${header}")

ernie_renderer-0.2.0.dev121/cmake/ErnieShaders.cmake

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Slang shaders kept in .slang files and embedded in the library, so editors highlight and
+# resolve them and the program needs no files beside it.
+#
+#     ernie_add_shaders(ernie_shaders math/affine.slang rt/ray_tracer.slang)
+#     add_dependencies(ernie_render ernie_shaders)
+#     // In C++: #include "ernie/shaders/rt/ray_tracer.h" for ernie::shaders::kRayTracer.
+
+# Adds `target`, which writes each file's header beside its path under the build's include
+# directory. Module names are the files' names, so each must be unique.
+function(ernie_add_shaders target)
+    set(headers "")
+    set(names "")
+    foreach(file IN LISTS ARGN)
+        get_filename_component(source "${file}" ABSOLUTE)
+        get_filename_component(name "${file}" NAME_WE)
+        if(name IN_LIST names)
+            message(FATAL_ERROR "Two shader modules are named ${name}")
+        endif()
+        list(APPEND names "${name}")
+        file(RELATIVE_PATH relative "${PROJECT_SOURCE_DIR}" "${source}")
+        string(REGEX REPLACE "\\.slang$" ".h" relative "${relative}")
+        set(header "${PROJECT_BINARY_DIR}/include/${relative}")
+        add_custom_command(
+            OUTPUT "${header}"
+            COMMAND
+                "${CMAKE_COMMAND}" "-DINPUT=${source}" "-DOUTPUT=${header}" "-DNAME=${name}"
+                "-DROOT=${PROJECT_SOURCE_DIR}" -P "${PROJECT_SOURCE_DIR}/cmake/EmbedShader.cmake"
+            DEPENDS "${source}" "${PROJECT_SOURCE_DIR}/cmake/EmbedShader.cmake"
+            COMMENT "Embedding ${relative}"
+            VERBATIM
+        )
+        list(APPEND headers "${header}")
+    endforeach()
+    add_custom_target(${target} DEPENDS ${headers} SOURCES ${ARGN})
+endfunction()

{ernie_renderer-0.2.0.dev117 → ernie_renderer-0.2.0.dev121}/ernie/CMakeLists.txt

@@ -28,6 +28,7 @@ add_subdirectory(io)
 add_subdirectory(scene)
 add_subdirectory(device)
 add_subdirectory(graph)
+add_subdirectory(shaders)
 add_subdirectory(render)
 
 set(ernie_layers

{ernie_renderer-0.2.0.dev117 → ernie_renderer-0.2.0.dev121}/ernie/render/CMakeLists.txt

@@ -2,15 +2,17 @@
 add_library(
     ernie_render
     OBJECT
-    bottom_levels.cpp
     device_scene.cpp
     geometry.cpp
     instances.cpp
     prim_rows.cpp
+    rt/bottom_levels.cpp
+    rt/ray_tracer.cpp
+    rt/top_levels.cpp
     scene_columns.cpp
-    top_levels.cpp
     transform_graph.cpp
 )
+add_dependencies(ernie_render ernie_shaders)
 target_link_libraries(
     ernie_render
     PUBLIC ernie_geometry ernie_scene ernie_graph

{ernie_renderer-0.2.0.dev117 → ernie_renderer-0.2.0.dev121}/ernie/render/device_scene.cpp

@@ -11,8 +11,7 @@ DeviceScene::DeviceScene(DeviceContext& context, Arenas& arenas, const Scene& sc
       transforms_(context, arenas, scene, compiler), geometry_(context, arenas, scene),
       instances_(context, arenas) {
     if (context.Capabilities().rayQuery) {
-        bottoms_ = std::make_unique<BottomLevels>(context, arenas, geometry_);
-        tops_ = std::make_unique<TopLevels>(context, compiler);
+        tracer_ = std::make_unique<RayTracer>(context, arenas, compiler, geometry_);
     }
 }
 
@@ -26,10 +25,8 @@ Ticket DeviceScene::Execute(const std::function<void(graph::Graph&)>& passes,
     instances_.Apply(changes);
     geometry_.AddUploads(graph);
     instances_.AddUploads(graph);
-    if (bottoms_) {
-        bottoms_->Apply(changes);
-        bottoms_->AddBuilds(graph);
-        tops_->AddBuild(graph, transforms_, instances_, *bottoms_);
+    if (tracer_) {
+        tracer_->AddBuild(graph, changes, transforms_, instances_);
     }
     if (passes) {
         passes(graph);

{ernie_renderer-0.2.0.dev117 → ernie_renderer-0.2.0.dev121}/ernie/render/device_scene.h

@@ -14,11 +14,10 @@
 
 #include "ernie/device/shader_compiler.h"
 #include "ernie/graph/graph.h"
-#include "ernie/render/bottom_levels.h"
 #include "ernie/render/geometry.h"
 #include "ernie/render/instances.h"
+#include "ernie/render/rt/ray_tracer.h"
 #include "ernie/render/scene_columns.h"
-#include "ernie/render/top_levels.h"
 #include "ernie/render/transform_graph.h"
 
 #include <functional>
@@ -30,8 +29,7 @@ namespace ernie {
 /// @brief A scene on a device and the graph each frame runs: the columns bound on the device
 /// follow the scene, world transforms compose from them, the geometry and its acceleration
 /// structures follow, and then the frame's own passes run, finding what to trace as a
-/// `SceneStructure` on the graph's blackboard. Devices without ray queries get no acceleration
-/// structures yet.
+/// `TraceScene` on the graph's blackboard. Devices without ray queries get no ray tracer yet.
 class DeviceScene {
   public:
     /// @param arenas Where the device columns live; `MemoryUse::Shared` to share them with CUDA.
@@ -52,9 +50,9 @@ class DeviceScene {
     InstanceTable& Instances() {
         return instances_;
     }
-    /// @return The bottom levels, or null on devices without ray queries.
-    BottomLevels* Bottoms() {
-        return bottoms_.get();
+    /// @return The ray tracer, or null on devices without ray queries.
+    RayTracer* Tracer() {
+        return tracer_.get();
     }
     /// @return Where the frames' transients come from.
     graph::TransientPool& Pool() {
@@ -76,8 +74,7 @@ class DeviceScene {
     TransformGraph transforms_;
     GeometryPool geometry_;
     InstanceTable instances_;
-    std::unique_ptr<BottomLevels> bottoms_;
-    std::unique_ptr<TopLevels> tops_;
+    std::unique_ptr<RayTracer> tracer_;
 };
 
 }  // namespace ernie

{ernie_renderer-0.2.0.dev117/ernie/render → ernie_renderer-0.2.0.dev121/ernie/render/rt}/bottom_levels.cpp

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 /// @file bottom_levels.cpp
 /// @brief Building, refitting and compacting each geometry's bottom level as frame-graph passes.
-#include "ernie/render/bottom_levels.h"
+#include "ernie/render/rt/bottom_levels.h"
 
 #include "ernie/util/profiler.h"
 

ernie_renderer-0.2.0.dev121/ernie/render/rt/ray_tracer.cpp

@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/// @file ray_tracer.cpp
+/// @brief Keeping the acceleration structures up to date each frame.
+#include "ernie/render/rt/ray_tracer.h"
+
+#include "ernie/shaders/rt/ray_tracer.h"
+#include "ernie/shaders/scene/geometry.h"
+#include "ernie/shaders/scene/shapes.h"
+#include "ernie/util/profiler.h"
+
+#include <array>
+
+namespace ernie {
+
+RayTracer::RayTracer(DeviceContext& context, Arenas& arenas, const ShaderCompiler& compiler,
+                     const GeometryPool& geometry)
+    : geometry_(geometry), bottoms_(context, arenas, geometry), tops_(context, compiler) {}
+
+std::span<const ShaderModule> RayTracer::Modules() {
+    static const std::array modules{shaders::kGeometry, shaders::kShapes, shaders::kRayTracer};
+    return modules;
+}
+
+void RayTracer::AddBuild(graph::Graph& graph, const GeometryPool::Changes& changes,
+                         const TransformGraph& transforms, const InstanceTable& instances) {
+    ERNIE_ZONE("Update the ray tracer");
+    const graph::Graph::ScopeGuard scope = graph.Scope("Ray tracer");
+    bottoms_.Apply(changes);
+    bottoms_.AddBuilds(graph);
+    const graph::Acceleration top = tops_.AddBuild(graph, transforms, instances, bottoms_);
+    const DeviceColumn& records = geometry_.Records();
+    graph.Board().Put(TraceScene{
+        .parameters = {.scene = tops_.Structure()->Address(), .records = records.Address()},
+        .top = top,
+        .records = graph.Import(records, records.Name()),
+        .instances = instances.Count()});
+}
+
+}  // namespace ernie

ernie_renderer-0.2.0.dev121/ernie/render/rt/ray_tracer.h

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/// @file ray_tracer.h
+/// @brief The ray tracer: the acceleration structures the scene is traced through, kept up to date
+/// each frame, and the Slang modules its passes trace them with.
+///
+///     const TraceScene& scene = graph.Board().Get<TraceScene>();
+///     graph.Add("Shade", graph::Kind::Compute,
+///               [&](graph::PassBuilder& pass) { scene.Read(pass); ... },
+///               [&](graph::PassContext& pass) { kernel.Dispatch(..., {scene.parameters, ...}); });
+#pragma once
+
+#include "ernie/device/shader_compiler.h"
+#include "ernie/graph/graph.h"
+#include "ernie/render/geometry.h"
+#include "ernie/render/instances.h"
+#include "ernie/render/rt/bottom_levels.h"
+#include "ernie/render/rt/top_levels.h"
+#include "ernie/render/transform_graph.h"
+
+#include <cstdint>
+#include <span>
+
+namespace ernie {
+
+/// @brief What a tracing kernel hands `RayTracer`, as ray_tracer.slang's `TracerParameters`.
+struct TracerParameters {
+    /// The top level's device address.
+    std::uint64_t scene = 0;
+    /// `GeometryPool::Records`' device address.
+    std::uint64_t records = 0;
+};
+static_assert(sizeof(TracerParameters) == 16);
+
+/// @brief What a frame's passes trace, left on the graph's blackboard by `RayTracer::AddBuild`.
+struct TraceScene {
+    TracerParameters parameters;
+    graph::Acceleration top;
+    graph::Buffer records;
+    std::uint32_t instances = 0;
+
+    /// @brief Declares a pass's reads of everything tracing touches.
+    void Read(graph::PassBuilder& pass) const {
+        pass.Read(top);
+        pass.ReadAddress(records);
+    }
+};
+
+/// @brief The scene's acceleration structures on a device with ray queries: a bottom level for
+/// each geometry, built and compacted as it changes, and a top level over the instances rebuilt
+/// every frame.
+class RayTracer {
+  public:
+    RayTracer(DeviceContext& context, Arenas& arenas, const ShaderCompiler& compiler,
+              const GeometryPool& geometry);
+    RayTracer(const RayTracer&) = delete;
+    RayTracer& operator=(const RayTracer&) = delete;
+
+    /// @return The modules, in import order, a tracing kernel compiles with before its own; it
+    /// then imports `ray_tracer`.
+    static std::span<const ShaderModule> Modules();
+
+    /// @brief Adds the passes that bring the structures up to date with the frame's geometry
+    /// changes, after the transforms and the geometry's and instances' uploads, and puts a
+    /// `TraceScene` on the graph's blackboard.
+    void AddBuild(graph::Graph& graph, const GeometryPool::Changes& changes,
+                  const TransformGraph& transforms, const InstanceTable& instances);
+
+    BottomLevels& Bottoms() {
+        return bottoms_;
+    }
+    const TopLevels& Tops() const {
+        return tops_;
+    }
+
+  private:
+    const GeometryPool& geometry_;
+    BottomLevels bottoms_;
+    TopLevels tops_;
+};
+
+}  // namespace ernie

{ernie_renderer-0.2.0.dev117/ernie/render → ernie_renderer-0.2.0.dev121/ernie/render/rt}/top_levels.cpp

@@ -1,8 +1,9 @@
 // SPDX-License-Identifier: BSD-3-Clause
 /// @file top_levels.cpp
 /// @brief Writing instance records on the device and building the top level over them.
-#include "ernie/render/top_levels.h"
+#include "ernie/render/rt/top_levels.h"
 
+#include "ernie/shaders/rt/write_instances.h"
 #include "ernie/util/profiler.h"
 
 #include <algorithm>
@@ -12,38 +13,6 @@
 namespace ernie {
 namespace {
 
-const ShaderModule kWriteInstances{"write_instances", R"(
-import affine;
-import world;
-/// VkAccelerationStructureInstanceKHR.
-struct Instance { float4 rows[3]; uint indexAndMask; uint offsetAndFlags; uint64_t bottom; };
-struct Parameters {
-    float4* worlds; uint* nodes; uint* geometries; float4* shapes; uint64_t* bottoms;
-    Instance* instances; uint count;
-};
-[[vk::push_constant]] ConstantBuffer<Parameters> parameters;
-
-[shader("compute")]
-[numthreads(64, 1, 1)]
-void main(uint3 id : SV_DispatchThreadID) {
-    uint row = id.x;
-    if (row >= parameters.count) {
-        return;
-    }
-    uint geometry = parameters.geometries[row];
-    Affine m = Mul(LoadWorld(parameters.worlds, parameters.nodes[row]),
-                   Load(parameters.shapes, row));
-    uint64_t bottom = parameters.bottoms[geometry];
-    Instance instance;
-    instance.rows = m.rows;
-    // An instance whose bottom level is not built yet is switched off.
-    instance.indexAndMask = (bottom == 0 ? 0u : 0xff000000u) | (geometry & 0xffffffu);
-    instance.offsetAndFlags = 0;
-    instance.bottom = bottom;
-    parameters.instances[row] = instance;
-}
-)"};
-
 struct WriteParameters {
     VkDeviceAddress worlds, nodes, geometries, shapes, bottoms, instances;
     std::uint32_t count, pad;
@@ -52,7 +21,7 @@ struct WriteParameters {
 std::vector<std::uint32_t> Compile(const ShaderCompiler& compiler) {
     std::vector<ShaderModule> modules(TransformGraph::WorldModules().begin(),
                                       TransformGraph::WorldModules().end());
-    modules.push_back(kWriteInstances);
+    modules.push_back(shaders::kWriteInstances);
     return compiler.Spirv(modules, "main");
 }
 
@@ -68,8 +37,9 @@ TopLevels::TopLevels(DeviceContext& context, const ShaderCompiler& compiler)
                                  std::uint32_t{sizeof(WriteParameters)}),
       pool_(context, "top levels") {}
 
-void TopLevels::AddBuild(graph::Graph& graph, const TransformGraph& transforms,
-                         const InstanceTable& instances, const BottomLevels& bottoms) {
+graph::Acceleration TopLevels::AddBuild(graph::Graph& graph, const TransformGraph& transforms,
+                                        const InstanceTable& instances,
+                                        const BottomLevels& bottoms) {
     ERNIE_ZONE("Prepare the top level");
     const std::uint32_t count = instances.Count();
     const std::uint32_t capacity = top_ ? top_->Sizes().primitives[0] : 0;
@@ -132,7 +102,7 @@ void TopLevels::AddBuild(graph::Graph& graph, const TransformGraph& transforms,
             const graph::BufferSlice space = pass.Get(scratch);
             RecordBuilds(pass.commands, {}, std::span(&recorded, 1), {space.buffer, space.offset});
         });
-    graph.Board().Put(SceneStructure{.top = top, .address = top_->Address(), .instances = count});
+    return top;
 }
 
 }  // namespace ernie

{ernie_renderer-0.2.0.dev117/ernie/render → ernie_renderer-0.2.0.dev121/ernie/render/rt}/top_levels.h

@@ -8,24 +8,14 @@
 #include "ernie/device/kernel.h"
 #include "ernie/device/shader_compiler.h"
 #include "ernie/graph/graph.h"
-#include "ernie/render/bottom_levels.h"
 #include "ernie/render/instances.h"
+#include "ernie/render/rt/bottom_levels.h"
 #include "ernie/render/transform_graph.h"
 
-#include <cstdint>
 #include <memory>
 
 namespace ernie {
 
-/// @brief What a frame's passes trace, left on the graph's blackboard by `TopLevels::AddBuild`.
-struct SceneStructure {
-    /// The top level, for passes to read with `Use::Trace`.
-    graph::Acceleration top;
-    /// Its address, as kernels trace it: `RaytracingAccelerationStructure(address)`.
-    std::uint64_t address = 0;
-    std::uint32_t instances = 0;
-};
-
 /// @brief The scene's top level, rebuilt every frame: a kernel writes each instance's record from
 /// its world transform, its shape transform and its geometry's bottom level, and the build follows.
 /// Instances whose bottom level is not built are switched off. An instance's custom index is its
@@ -38,10 +28,10 @@ class TopLevels {
     TopLevels& operator=(const TopLevels&) = delete;
 
     /// @brief Adds the passes that write the instances and build the top level, after the
-    /// transforms, the instance uploads and the bottom levels' builds, and puts a
-    /// `SceneStructure` on the graph's blackboard.
-    void AddBuild(graph::Graph& graph, const TransformGraph& transforms,
-                  const InstanceTable& instances, const BottomLevels& bottoms);
+    /// transforms, the instance uploads and the bottom levels' builds.
+    /// @return The top level built, for passes to read with `Use::Trace`.
+    graph::Acceleration AddBuild(graph::Graph& graph, const TransformGraph& transforms,
+                                 const InstanceTable& instances, const BottomLevels& bottoms);
 
     /// @return The top level as last built, or null before the first build.
     const AccelerationStructure* Structure() const {