warp-lang 1.5.0__py3-none-manylinux2014_x86_64.whl → 1.6.0__py3-none-manylinux2014_x86_64.whl

warp/native/bvh.h

@@ -11,6 +11,14 @@
 #include "builtin.h"
 #include "intersect.h"
 
+#define BVH_LEAF_SIZE (4)
+#define SAH_NUM_BUCKETS (16)
+#define USE_LOAD4
+
+#define BVH_CONSTRUCTOR_SAH (0)
+#define BVH_CONSTRUCTOR_MEDIAN (1)
+#define BVH_CONSTRUCTOR_LBVH (2)
+
 namespace wp
 {
 
@@ -72,12 +80,38 @@ struct bounds3
 		}
 	}
 
+	CUDA_CALLABLE inline bool overlaps(const vec3& b_lower, const vec3& b_upper) const
+	{
+		if (lower[0] > b_upper[0] ||
+			lower[1] > b_upper[1] ||
+			lower[2] > b_upper[2] ||
+			upper[0] < b_lower[0] ||
+			upper[1] < b_lower[1] ||
+			upper[2] < b_lower[2])
+		{
+			return false;
+		}
+		else
+		{
+			return true;
+		}
+	}
+
 	CUDA_CALLABLE inline void add_point(const vec3& p)
 	{
 		lower = min(lower, p);
 		upper = max(upper, p);
 	}
 
+	CUDA_CALLABLE inline void add_bounds(const vec3& lower_other, const vec3& upper_other)
+	{
+		// lower_other will only impact the lower of the new bounds
+		// upper_other will only impact the upper of the new bounds
+		// this costs only half of the computation of adding lower_other and upper_other separately
+		lower = min(lower, lower_other);
+		upper = max(upper, upper_other);
+	}
+
 	CUDA_CALLABLE inline float area() const
 	{
 		vec3 e = upper-lower;
@@ -108,6 +142,13 @@ struct BVHPackedNodeHalf
 	float x;
 	float y;
 	float z;
+	// For non-leaf nodes:
+	// - 'lower.i' represents the index of the left child node.
+	// - 'upper.i' represents the index of the right child node.
+	//
+	// For leaf nodes:
+	// - 'lower.i' indicates the start index of the primitives in 'primitive_indices'.
+	// - 'upper.i' indicates the index just after the last primitive in 'primitive_indices'
 	unsigned int i : 31;
 	unsigned int b : 1;
 };
@@ -120,11 +161,15 @@ struct BVH
 	// used for fast refits
 	int* node_parents;
 	int* node_counts;
+	// reordered primitive indices corresponds to the ordering of leaf nodes
+	int* primitive_indices;
 	
 	int max_depth;
 	int max_nodes;
 	int num_nodes;
-	
+	// since we use packed leaf nodes, the number of them is no longer the number of items, but variable
+	int num_leaf_nodes;
+
 	// pointer (CPU or GPU) to a single integer index in node_lowers, node_uppers
 	// representing the root of the tree, this is not always the first node
 	// for bottom-up builders
@@ -161,6 +206,24 @@ CUDA_CALLABLE inline void make_node(volatile BVHPackedNodeHalf* n, const vec3& b
     n->b = (unsigned int)(leaf?1:0);
 }
 
+#ifdef __CUDA_ARCH__
+__device__ inline wp::BVHPackedNodeHalf bvh_load_node(const wp::BVHPackedNodeHalf* nodes, int index)
+{
+#ifdef USE_LOAD4
+	//return  (const wp::BVHPackedNodeHalf&)(__ldg((const float4*)(nodes)+index));
+	return  (const wp::BVHPackedNodeHalf&)(*((const float4*)(nodes)+index));
+#else
+	return  nodes[index];
+#endif // USE_LOAD4
+
+}
+#else
+inline wp::BVHPackedNodeHalf bvh_load_node(const wp::BVHPackedNodeHalf* nodes, int index)
+{
+	return  nodes[index];
+}
+#endif // __CUDACC__
+
 CUDA_CALLABLE inline int clz(int x)
 {
     int n;
@@ -215,7 +278,8 @@ struct bvh_query_t
           is_ray(false),
           input_lower(),
           input_upper(),
-          bounds_nr(0)
+          bounds_nr(0),
+		  primitive_counter(-1)
     {}
 
     // Required for adjoint computations.
@@ -230,22 +294,37 @@ struct bvh_query_t
 	int stack[32];
 	int count;
 
+	// >= 0 if currently in a packed leaf node
+	int primitive_counter;
+	
     // inputs
-	bool is_ray;
     wp::vec3 input_lower;	// start for ray
     wp::vec3 input_upper;	// dir for ray
 
 	int bounds_nr;
+	bool is_ray;
 };
 
+CUDA_CALLABLE inline bool bvh_query_intersection_test(const bvh_query_t& query, const vec3& node_lower, const vec3& node_upper)
+{
+	if (query.is_ray)
+	{
+		float t = 0.0f;
+		return intersect_ray_aabb(query.input_lower, query.input_upper, node_lower, node_upper, t);
+	}
+	else
+	{
+		return intersect_aabb_aabb(query.input_lower, query.input_upper, node_lower, node_upper);
+	}
+}
 
 CUDA_CALLABLE inline bvh_query_t bvh_query(
-    uint64_t id, bool is_ray, const vec3& lower, const vec3& upper)
+	uint64_t id, bool is_ray, const vec3& lower, const vec3& upper)
 {
-    // This routine traverses the BVH tree until it finds
+	// This routine traverses the BVH tree until it finds
 	// the first overlapping bound. 
 
-    // initialize empty
+	// initialize empty
 	bvh_query_t query;
 
 	query.bounds_nr = -1;
@@ -255,57 +334,41 @@ CUDA_CALLABLE inline bvh_query_t bvh_query(
 	query.bvh = bvh;
 	query.is_ray = is_ray;
 
-    // optimization: make the latest	
+	// optimization: make the latest	
 	query.stack[0] = *bvh.root;
 	query.count = 1;
-    query.input_lower = lower;
-    query.input_upper = upper;
+	query.input_lower = lower;
+	query.input_upper = upper;
 
-    wp::bounds3 input_bounds(query.input_lower, query.input_upper);
-
-    // Navigate through the bvh, find the first overlapping leaf node.
-    while (query.count)
-    {
+	// Navigate through the bvh, find the first overlapping leaf node.
+	while (query.count)
+	{
 		const int node_index = query.stack[--query.count];
+		BVHPackedNodeHalf node_lower = bvh_load_node(bvh.node_lowers, node_index);
+		BVHPackedNodeHalf node_upper = bvh_load_node(bvh.node_uppers, node_index);
 
-		BVHPackedNodeHalf node_lower = bvh.node_lowers[node_index];
-		BVHPackedNodeHalf node_upper = bvh.node_uppers[node_index];
-
-		wp::vec3 lower_pos(node_lower.x, node_lower.y, node_lower.z);
-		wp::vec3 upper_pos(node_upper.x, node_upper.y, node_upper.z);
-        wp::bounds3 current_bounds(lower_pos, upper_pos);
-
-		if (query.is_ray) 
-		{
-			float t = 0.0f;
-			if (!intersect_ray_aabb(query.input_lower, query.input_upper, current_bounds.lower, current_bounds.upper, t))
-			// Skip this box, it doesn't overlap with our ray.
-				continue;
-		}
-		else 
+		if (!bvh_query_intersection_test(query, (vec3&)node_lower, (vec3&)node_upper))
 		{
-	        if (!input_bounds.overlaps(current_bounds))
-				// Skip this box, it doesn't overlap with our target box.
-				continue;
+			continue;
 		}
 
 		const int left_index = node_lower.i;
 		const int right_index = node_upper.i;
-
-        // Make bounds from this AABB
-        if (node_lower.b)
-        {
-			// found very first leaf index.
+		// Make bounds from this AABB
+		if (node_lower.b)
+		{
+			// Reached a leaf node, point to its first primitive
 			// Back up one level and return 
+			query.primitive_counter = left_index;
 			query.stack[query.count++] = node_index;
 			return query;
-        }
-        else
-        {	
-		  query.stack[query.count++] = left_index;
-		  query.stack[query.count++] = right_index;
 		}
-	}	
+		else
+		{
+			query.stack[query.count++] = left_index;
+			query.stack[query.count++] = right_index;
+		}
+	}
 
 	return query;
 }
@@ -338,52 +401,100 @@ CUDA_CALLABLE inline void adj_bvh_query_ray(uint64_t id, const vec3& start, cons
 
 CUDA_CALLABLE inline bool bvh_query_next(bvh_query_t& query, int& index)
 {
-    BVH bvh = query.bvh;
-	
-	wp::bounds3 input_bounds(query.input_lower, query.input_upper);
-
-    // Navigate through the bvh, find the first overlapping leaf node.
-    while (query.count)
-    {
-        const int node_index = query.stack[--query.count];
-        BVHPackedNodeHalf node_lower = bvh.node_lowers[node_index];
-        BVHPackedNodeHalf node_upper = bvh.node_uppers[node_index];
+	BVH bvh = query.bvh;
 
-        wp::vec3 lower_pos(node_lower.x, node_lower.y, node_lower.z);
-        wp::vec3 upper_pos(node_upper.x, node_upper.y, node_upper.z);
-        wp::bounds3 current_bounds(lower_pos, upper_pos);
+	if (query.primitive_counter != -1)
+		// currently in a leaf node which is the last node in the stack
+	{
+		const int node_index = query.stack[query.count - 1];
+		BVHPackedNodeHalf node_lower = bvh_load_node(bvh.node_lowers, node_index);
+		BVHPackedNodeHalf node_upper = bvh_load_node(bvh.node_uppers, node_index);
 
-		if (query.is_ray) 
+		const int end = node_upper.i;
+		for (int primitive_counter = query.primitive_counter; primitive_counter < end; primitive_counter++)
 		{
-			float t = 0.0f;
-			if (!intersect_ray_aabb(query.input_lower, query.input_upper, current_bounds.lower, current_bounds.upper, t))
-			// Skip this box, it doesn't overlap with our ray.
-				continue;
+			int primitive_index = bvh.primitive_indices[primitive_counter];
+			if (bvh_query_intersection_test(query, bvh.item_lowers[primitive_index], bvh.item_uppers[primitive_index]))
+			{
+				if (primitive_counter < end - 1)
+					// still need to come back to this leaf node for the leftover primitives
+				{
+					query.primitive_counter = primitive_counter + 1;
+				}
+				else
+					// no need to come back to this leaf node
+				{
+					query.count--;
+					query.primitive_counter = -1;
+				}
+				index = primitive_index;
+				query.bounds_nr = primitive_index;
+
+				return true;
+			}
 		}
-		else {
-	        if (!input_bounds.overlaps(current_bounds))
-				// Skip this box, it doesn't overlap with our target box.
-				continue;
+		// if we reach here that means we have finished the current leaf node without finding intersections
+		query.primitive_counter = -1;
+		// remove the leaf node from the back of the stack because it is finished
+		// and continue the bvh traversal
+		query.count--;
+	}
+
+	// Navigate through the bvh, find the first overlapping leaf node.
+	while (query.count)
+	{
+		const int node_index = query.stack[--query.count];
+		BVHPackedNodeHalf node_lower = bvh_load_node(bvh.node_lowers, node_index);
+		BVHPackedNodeHalf node_upper = bvh_load_node(bvh.node_uppers, node_index);
+
+		const int left_index = node_lower.i;
+		const int right_index = node_upper.i;
+
+		wp::vec3 lower_pos(node_lower.x, node_lower.y, node_lower.z);
+		wp::vec3 upper_pos(node_upper.x, node_upper.y, node_upper.z);
+		wp::bounds3 current_bounds(lower_pos, upper_pos);
+
+		if (!bvh_query_intersection_test(query, (vec3&)node_lower, (vec3&)node_upper))
+		{
+			continue;
 		}
 
-        const int left_index = node_lower.i;
-        const int right_index = node_upper.i;
-
-        if (node_lower.b)
-        {
-            // found leaf
-            query.bounds_nr = left_index;
-			index = left_index;
-            return true;
-        }
-        else
-        {
-
-            query.stack[query.count++] = left_index;
-            query.stack[query.count++] = right_index;
-        }
-    }
-    return false;
+		if (node_lower.b)
+		{
+			// found leaf, loop through its content primitives
+			const int start = left_index;
+			const int end = right_index;
+
+			for (int primitive_counter = start; primitive_counter < end; primitive_counter++)
+			{
+				int primitive_index = bvh.primitive_indices[primitive_counter];
+				if (bvh_query_intersection_test(query, bvh.item_lowers[primitive_index], bvh.item_uppers[primitive_index]))
+				{
+					if (primitive_counter < end - 1)
+						// still need to come back to this leaf node for the leftover primitives
+					{
+						query.primitive_counter = primitive_counter + 1;
+						query.stack[query.count++] = node_index;
+					}
+					else
+						// no need to come back to this leaf node
+					{
+						query.primitive_counter = -1;
+					}
+					index = primitive_index;
+					query.bounds_nr = primitive_index;
+
+					return true;
+				}
+			}
+		}
+		else
+		{
+			query.stack[query.count++] = left_index;
+			query.stack[query.count++] = right_index;
+		}
+	}
+	return false;
 }
 
 
@@ -421,7 +532,7 @@ CUDA_CALLABLE void bvh_rem_descriptor(uint64_t id);
 
 #if !__CUDA_ARCH__
 
-void bvh_create_host(vec3* lowers, vec3* uppers, int num_items, BVH& bvh);
+void bvh_create_host(vec3* lowers, vec3* uppers, int num_items,  int constructor_type, BVH& bvh);
 void bvh_destroy_host(wp::BVH& bvh);
 void bvh_refit_host(wp::BVH& bvh);
 
@@ -431,4 +542,3 @@ void bvh_refit_device(uint64_t id);
 #endif
 
 } // namespace wp
-

warp/native/clang/clang.cpp

@@ -218,7 +218,7 @@ static std::unique_ptr<llvm::Module> cuda_to_llvm(const std::string& input_file,
 
 extern "C" {
 
-WP_API int compile_cpp(const char* cpp_src, const char *input_file, const char* include_dir, const char* output_file, bool debug, bool verify_fp)
+WP_API int compile_cpp(const char* cpp_src, const char *input_file, const char* include_dir, const char* output_file, bool debug, bool verify_fp, bool fuse_fp)
 {
     initialize_llvm();
 
@@ -236,6 +236,10 @@ WP_API int compile_cpp(const char* cpp_src, const char *input_file, const char*
     const char* CPU = "generic";
     const char* features = "";
     llvm::TargetOptions target_options;
+    if (fuse_fp)
+        target_options.AllowFPOpFusion = llvm::FPOpFusion::Standard;
+    else
+        target_options.AllowFPOpFusion = llvm::FPOpFusion::Strict;
     llvm::Reloc::Model relocation_model = llvm::Reloc::PIC_;  // Position Independent Code
     llvm::CodeModel::Model code_model = llvm::CodeModel::Large;  // Don't make assumptions about displacement sizes
     llvm::TargetMachine* target_machine = target->createTargetMachine(target_triple, CPU, features, target_options, relocation_model, code_model);

warp/native/coloring.cpp

@@ -590,7 +590,11 @@ extern "C"
         if (num_colors > 1) {
             std::vector<std::vector<int>> color_groups;
             convert_to_color_groups(num_colors, graph.node_colors, color_groups);
-            return balance_color_groups(target_max_min_ratio, graph, color_groups);
+
+            float max_min_ratio = balance_color_groups(target_max_min_ratio, graph, color_groups);
+            memcpy(node_colors.data, graph.node_colors.data(), num_nodes * sizeof(int));
+
+            return max_min_ratio;
         }
         else
         {

warp/native/cuda_util.cpp

@@ -102,6 +102,11 @@ static PFN_cuGraphicsGLRegisterBuffer_v3000 pfn_cuGraphicsGLRegisterBuffer;
 static PFN_cuGraphicsUnregisterResource_v3000 pfn_cuGraphicsUnregisterResource;
 static PFN_cuModuleGetGlobal_v3020 pfn_cuModuleGetGlobal;
 static PFN_cuFuncSetAttribute_v9000 pfn_cuFuncSetAttribute;
+static PFN_cuIpcGetEventHandle_v4010 pfn_cuIpcGetEventHandle;
+static PFN_cuIpcOpenEventHandle_v4010 pfn_cuIpcOpenEventHandle;
+static PFN_cuIpcGetMemHandle_v4010 pfn_cuIpcGetMemHandle;
+static PFN_cuIpcOpenMemHandle_v11000 pfn_cuIpcOpenMemHandle;
+static PFN_cuIpcCloseMemHandle_v4010 pfn_cuIpcCloseMemHandle;
 
 static bool cuda_driver_initialized = false;
 
@@ -120,15 +125,17 @@ static inline int get_minor(int version)
     return (version % 1000) / 10;
 }
 
-static bool get_driver_entry_point(const char* name, void** pfn)
+// Get versioned driver entry point. The version argument should match the function pointer type.
+// For example, to initialize PFN_cuCtxCreate_v3020 use version 3020.
+static bool get_driver_entry_point(const char* name, int version, void** pfn)
 {
     if (!pfn_cuGetProcAddress || !name || !pfn)
         return false;
 
 #if CUDA_VERSION < 12000
-    CUresult r = pfn_cuGetProcAddress(name, pfn, WP_CUDA_DRIVER_VERSION, CU_GET_PROC_ADDRESS_DEFAULT);
+    CUresult r = pfn_cuGetProcAddress(name, pfn, version, CU_GET_PROC_ADDRESS_DEFAULT);
 #else
-    CUresult r = pfn_cuGetProcAddress(name, pfn, WP_CUDA_DRIVER_VERSION, CU_GET_PROC_ADDRESS_DEFAULT, NULL);
+    CUresult r = pfn_cuGetProcAddress(name, pfn, version, CU_GET_PROC_ADDRESS_DEFAULT, NULL);
 #endif
 
     if (r != CUDA_SUCCESS)
@@ -170,7 +177,8 @@ bool init_cuda_driver()
 
     // check the CUDA driver version and report an error if it's too low
     int driver_version = 0;
-    if (get_driver_entry_point("cuDriverGetVersion", &(void*&)pfn_cuDriverGetVersion) && check_cu(pfn_cuDriverGetVersion(&driver_version)))
+    if (get_driver_entry_point("cuDriverGetVersion", 2020, &(void*&)pfn_cuDriverGetVersion) &&
+        check_cu(pfn_cuDriverGetVersion(&driver_version)))
     {
         if (driver_version < WP_CUDA_DRIVER_VERSION)
         {
@@ -186,55 +194,60 @@ bool init_cuda_driver()
     }
 
     // initialize driver entry points
-    get_driver_entry_point("cuGetErrorString", &(void*&)pfn_cuGetErrorString);
-    get_driver_entry_point("cuGetErrorName", &(void*&)pfn_cuGetErrorName);
-    get_driver_entry_point("cuInit", &(void*&)pfn_cuInit);
-    get_driver_entry_point("cuDeviceGet", &(void*&)pfn_cuDeviceGet);
-    get_driver_entry_point("cuDeviceGetCount", &(void*&)pfn_cuDeviceGetCount);
-    get_driver_entry_point("cuDeviceGetName", &(void*&)pfn_cuDeviceGetName);
-    get_driver_entry_point("cuDeviceGetAttribute", &(void*&)pfn_cuDeviceGetAttribute);
-    get_driver_entry_point("cuDeviceGetUuid", &(void*&)pfn_cuDeviceGetUuid);
-    get_driver_entry_point("cuDevicePrimaryCtxRetain", &(void*&)pfn_cuDevicePrimaryCtxRetain);
-    get_driver_entry_point("cuDevicePrimaryCtxRelease", &(void*&)pfn_cuDevicePrimaryCtxRelease);
-    get_driver_entry_point("cuDeviceCanAccessPeer", &(void*&)pfn_cuDeviceCanAccessPeer);
-    get_driver_entry_point("cuMemGetInfo", &(void*&)pfn_cuMemGetInfo);
-    get_driver_entry_point("cuCtxSetCurrent", &(void*&)pfn_cuCtxSetCurrent);
-    get_driver_entry_point("cuCtxGetCurrent", &(void*&)pfn_cuCtxGetCurrent);
-    get_driver_entry_point("cuCtxPushCurrent", &(void*&)pfn_cuCtxPushCurrent);
-    get_driver_entry_point("cuCtxPopCurrent", &(void*&)pfn_cuCtxPopCurrent);
-    get_driver_entry_point("cuCtxSynchronize", &(void*&)pfn_cuCtxSynchronize);
-    get_driver_entry_point("cuCtxGetDevice", &(void*&)pfn_cuCtxGetDevice);
-    get_driver_entry_point("cuCtxCreate", &(void*&)pfn_cuCtxCreate);
-    get_driver_entry_point("cuCtxDestroy", &(void*&)pfn_cuCtxDestroy);
-    get_driver_entry_point("cuCtxEnablePeerAccess", &(void*&)pfn_cuCtxEnablePeerAccess);
-    get_driver_entry_point("cuCtxDisablePeerAccess", &(void*&)pfn_cuCtxDisablePeerAccess);
-    get_driver_entry_point("cuStreamCreate", &(void*&)pfn_cuStreamCreate);
-    get_driver_entry_point("cuStreamDestroy", &(void*&)pfn_cuStreamDestroy);
-    get_driver_entry_point("cuStreamSynchronize", &(void*&)pfn_cuStreamSynchronize);
-    get_driver_entry_point("cuStreamWaitEvent", &(void*&)pfn_cuStreamWaitEvent);
-    get_driver_entry_point("cuStreamGetCtx", &(void*&)pfn_cuStreamGetCtx);
-    get_driver_entry_point("cuStreamGetCaptureInfo", &(void*&)pfn_cuStreamGetCaptureInfo);
-    get_driver_entry_point("cuStreamUpdateCaptureDependencies", &(void*&)pfn_cuStreamUpdateCaptureDependencies);
-    get_driver_entry_point("cuStreamCreateWithPriority", &(void*&)pfn_cuStreamCreateWithPriority);
-    get_driver_entry_point("cuStreamGetPriority", &(void*&)pfn_cuStreamGetPriority);
-    get_driver_entry_point("cuEventCreate", &(void*&)pfn_cuEventCreate);
-    get_driver_entry_point("cuEventDestroy", &(void*&)pfn_cuEventDestroy);
-    get_driver_entry_point("cuEventRecord", &(void*&)pfn_cuEventRecord);
-    get_driver_entry_point("cuEventRecordWithFlags", &(void*&)pfn_cuEventRecordWithFlags);
-    get_driver_entry_point("cuEventSynchronize", &(void*&)pfn_cuEventSynchronize);
-    get_driver_entry_point("cuModuleLoadDataEx", &(void*&)pfn_cuModuleLoadDataEx);
-    get_driver_entry_point("cuModuleUnload", &(void*&)pfn_cuModuleUnload);
-    get_driver_entry_point("cuModuleGetFunction", &(void*&)pfn_cuModuleGetFunction);
-    get_driver_entry_point("cuLaunchKernel", &(void*&)pfn_cuLaunchKernel);
-    get_driver_entry_point("cuMemcpyPeerAsync", &(void*&)pfn_cuMemcpyPeerAsync);
-    get_driver_entry_point("cuPointerGetAttribute", &(void*&)pfn_cuPointerGetAttribute);
-    get_driver_entry_point("cuGraphicsMapResources", &(void*&)pfn_cuGraphicsMapResources);
-    get_driver_entry_point("cuGraphicsUnmapResources", &(void*&)pfn_cuGraphicsUnmapResources);
-    get_driver_entry_point("cuGraphicsResourceGetMappedPointer", &(void*&)pfn_cuGraphicsResourceGetMappedPointer);
-    get_driver_entry_point("cuGraphicsGLRegisterBuffer", &(void*&)pfn_cuGraphicsGLRegisterBuffer);
-    get_driver_entry_point("cuGraphicsUnregisterResource", &(void*&)pfn_cuGraphicsUnregisterResource);
-    get_driver_entry_point("cuModuleGetGlobal", &(void*&)pfn_cuModuleGetGlobal);
-    get_driver_entry_point("cuFuncSetAttribute", &(void*&)pfn_cuFuncSetAttribute);
+    get_driver_entry_point("cuGetErrorString", 6000, &(void*&)pfn_cuGetErrorString);
+    get_driver_entry_point("cuGetErrorName", 6000, &(void*&)pfn_cuGetErrorName);
+    get_driver_entry_point("cuInit", 2000, &(void*&)pfn_cuInit);
+    get_driver_entry_point("cuDeviceGet", 2000, &(void*&)pfn_cuDeviceGet);
+    get_driver_entry_point("cuDeviceGetCount", 2000, &(void*&)pfn_cuDeviceGetCount);
+    get_driver_entry_point("cuDeviceGetName", 2000, &(void*&)pfn_cuDeviceGetName);
+    get_driver_entry_point("cuDeviceGetAttribute", 2000, &(void*&)pfn_cuDeviceGetAttribute);
+    get_driver_entry_point("cuDeviceGetUuid", 110400, &(void*&)pfn_cuDeviceGetUuid);
+    get_driver_entry_point("cuDevicePrimaryCtxRetain", 7000, &(void*&)pfn_cuDevicePrimaryCtxRetain);
+    get_driver_entry_point("cuDevicePrimaryCtxRelease", 11000, &(void*&)pfn_cuDevicePrimaryCtxRelease);
+    get_driver_entry_point("cuDeviceCanAccessPeer", 4000, &(void*&)pfn_cuDeviceCanAccessPeer);
+    get_driver_entry_point("cuMemGetInfo", 3020, &(void*&)pfn_cuMemGetInfo);
+    get_driver_entry_point("cuCtxSetCurrent", 4000, &(void*&)pfn_cuCtxSetCurrent);
+    get_driver_entry_point("cuCtxGetCurrent", 4000, &(void*&)pfn_cuCtxGetCurrent);
+    get_driver_entry_point("cuCtxPushCurrent", 4000, &(void*&)pfn_cuCtxPushCurrent);
+    get_driver_entry_point("cuCtxPopCurrent", 4000, &(void*&)pfn_cuCtxPopCurrent);
+    get_driver_entry_point("cuCtxSynchronize", 2000, &(void*&)pfn_cuCtxSynchronize);
+    get_driver_entry_point("cuCtxGetDevice", 2000, &(void*&)pfn_cuCtxGetDevice);
+    get_driver_entry_point("cuCtxCreate", 3020, &(void*&)pfn_cuCtxCreate);
+    get_driver_entry_point("cuCtxDestroy", 4000, &(void*&)pfn_cuCtxDestroy);
+    get_driver_entry_point("cuCtxEnablePeerAccess", 4000, &(void*&)pfn_cuCtxEnablePeerAccess);
+    get_driver_entry_point("cuCtxDisablePeerAccess", 4000, &(void*&)pfn_cuCtxDisablePeerAccess);
+    get_driver_entry_point("cuStreamCreate", 2000, &(void*&)pfn_cuStreamCreate);
+    get_driver_entry_point("cuStreamDestroy", 4000, &(void*&)pfn_cuStreamDestroy);
+    get_driver_entry_point("cuStreamSynchronize", 2000, &(void*&)pfn_cuStreamSynchronize);
+    get_driver_entry_point("cuStreamWaitEvent", 3020, &(void*&)pfn_cuStreamWaitEvent);
+    get_driver_entry_point("cuStreamGetCtx", 9020, &(void*&)pfn_cuStreamGetCtx);
+    get_driver_entry_point("cuStreamGetCaptureInfo", 11030, &(void*&)pfn_cuStreamGetCaptureInfo);
+    get_driver_entry_point("cuStreamUpdateCaptureDependencies", 11030, &(void*&)pfn_cuStreamUpdateCaptureDependencies);
+    get_driver_entry_point("cuStreamCreateWithPriority", 5050, &(void*&)pfn_cuStreamCreateWithPriority);
+    get_driver_entry_point("cuStreamGetPriority", 5050, &(void*&)pfn_cuStreamGetPriority);
+    get_driver_entry_point("cuEventCreate", 2000, &(void*&)pfn_cuEventCreate);
+    get_driver_entry_point("cuEventDestroy", 4000, &(void*&)pfn_cuEventDestroy);
+    get_driver_entry_point("cuEventRecord", 2000, &(void*&)pfn_cuEventRecord);
+    get_driver_entry_point("cuEventRecordWithFlags", 11010, &(void*&)pfn_cuEventRecordWithFlags);
+    get_driver_entry_point("cuEventSynchronize", 2000, &(void*&)pfn_cuEventSynchronize);
+    get_driver_entry_point("cuModuleLoadDataEx", 2010, &(void*&)pfn_cuModuleLoadDataEx);
+    get_driver_entry_point("cuModuleUnload", 2000, &(void*&)pfn_cuModuleUnload);
+    get_driver_entry_point("cuModuleGetFunction", 2000, &(void*&)pfn_cuModuleGetFunction);
+    get_driver_entry_point("cuLaunchKernel", 4000, &(void*&)pfn_cuLaunchKernel);
+    get_driver_entry_point("cuMemcpyPeerAsync", 4000, &(void*&)pfn_cuMemcpyPeerAsync);
+    get_driver_entry_point("cuPointerGetAttribute", 4000, &(void*&)pfn_cuPointerGetAttribute);
+    get_driver_entry_point("cuGraphicsMapResources", 3000, &(void*&)pfn_cuGraphicsMapResources);
+    get_driver_entry_point("cuGraphicsUnmapResources", 3000, &(void*&)pfn_cuGraphicsUnmapResources);
+    get_driver_entry_point("cuGraphicsResourceGetMappedPointer", 3020, &(void*&)pfn_cuGraphicsResourceGetMappedPointer);
+    get_driver_entry_point("cuGraphicsGLRegisterBuffer", 3000, &(void*&)pfn_cuGraphicsGLRegisterBuffer);
+    get_driver_entry_point("cuGraphicsUnregisterResource", 3000, &(void*&)pfn_cuGraphicsUnregisterResource);
+    get_driver_entry_point("cuModuleGetGlobal", 3020, &(void*&)pfn_cuModuleGetGlobal);
+    get_driver_entry_point("cuFuncSetAttribute", 9000, &(void*&)pfn_cuFuncSetAttribute);
+    get_driver_entry_point("cuIpcGetEventHandle", 4010, &(void*&)pfn_cuIpcGetEventHandle);
+    get_driver_entry_point("cuIpcOpenEventHandle", 4010, &(void*&)pfn_cuIpcOpenEventHandle);
+    get_driver_entry_point("cuIpcGetMemHandle", 4010, &(void*&)pfn_cuIpcGetMemHandle);
+    get_driver_entry_point("cuIpcOpenMemHandle", 11000, &(void*&)pfn_cuIpcOpenMemHandle);
+    get_driver_entry_point("cuIpcCloseMemHandle", 4010, &(void*&)pfn_cuIpcCloseMemHandle);
 
     if (pfn_cuInit)
         cuda_driver_initialized = check_cu(pfn_cuInit(0));
@@ -582,4 +595,29 @@ CUresult cuFuncSetAttribute_f(CUfunction hfunc, CUfunction_attribute attrib, int
     return pfn_cuFuncSetAttribute ? pfn_cuFuncSetAttribute(hfunc, attrib, value) : DRIVER_ENTRY_POINT_ERROR;
 }
 
+CUresult cuIpcGetEventHandle_f(CUipcEventHandle *pHandle, CUevent event)
+{
+    return pfn_cuIpcGetEventHandle ? pfn_cuIpcGetEventHandle(pHandle, event) : DRIVER_ENTRY_POINT_ERROR;
+}
+
+CUresult cuIpcOpenEventHandle_f(CUevent *phEvent, CUipcEventHandle handle)
+{
+    return pfn_cuIpcOpenEventHandle ? pfn_cuIpcOpenEventHandle(phEvent, handle) : DRIVER_ENTRY_POINT_ERROR;
+}
+
+CUresult cuIpcGetMemHandle_f(CUipcMemHandle *pHandle, CUdeviceptr dptr)
+{
+    return pfn_cuIpcGetMemHandle ? pfn_cuIpcGetMemHandle(pHandle, dptr) : DRIVER_ENTRY_POINT_ERROR;
+}
+
+CUresult cuIpcOpenMemHandle_f(CUdeviceptr *pdptr, CUipcMemHandle handle, unsigned int flags)
+{
+    return pfn_cuIpcOpenMemHandle ? pfn_cuIpcOpenMemHandle(pdptr, handle, flags) : DRIVER_ENTRY_POINT_ERROR;
+}
+
+CUresult cuIpcCloseMemHandle_f(CUdeviceptr dptr)
+{
+    return pfn_cuIpcCloseMemHandle ? pfn_cuIpcCloseMemHandle(dptr) : DRIVER_ENTRY_POINT_ERROR;
+}
+
 #endif // WP_ENABLE_CUDA

warp/native/cuda_util.h

@@ -101,6 +101,11 @@ CUresult cuGraphicsGLRegisterBuffer_f(CUgraphicsResource *pCudaResource, unsigne
 CUresult cuGraphicsUnregisterResource_f(CUgraphicsResource resource);
 CUresult cuModuleGetGlobal_f(CUdeviceptr* dptr, size_t* bytes, CUmodule hmod, const char* name );
 CUresult cuFuncSetAttribute_f(CUfunction hfunc, CUfunction_attribute attrib, int value);
+CUresult cuIpcGetEventHandle_f(CUipcEventHandle *pHandle, CUevent event);
+CUresult cuIpcOpenEventHandle_f(CUevent *phEvent, CUipcEventHandle handle);
+CUresult cuIpcGetMemHandle_f(CUipcMemHandle *pHandle, CUdeviceptr dptr);
+CUresult cuIpcOpenMemHandle_f(CUdeviceptr *pdptr, CUipcMemHandle handle, unsigned int flags);
+CUresult cuIpcCloseMemHandle_f(CUdeviceptr dptr);
 
 bool init_cuda_driver();
 bool is_cuda_driver_initialized();