warp-lang 1.0.0b2__py3-none-win_amd64.whl → 1.0.0b6__py3-none-win_amd64.whl

warp/tests/test_math.py

@@ -5,13 +5,13 @@
 # distribution of this software and related documentation without an express
 # license agreement from NVIDIA CORPORATION is strictly prohibited.
 
-from typing import NamedTuple
 import unittest
+from typing import NamedTuple
 
 import numpy as np
 
 import warp as wp
-from warp.tests.test_base import *
+from warp.tests.unittest_utils import *
 
 wp.init()
 
@@ -84,16 +84,110 @@ def test_scalar_math(test, device):
         )
 
 
-def register(parent):
-    devices = get_test_devices()
+def test_vec_type(test, device):
+    vec5 = wp.vec(length=5, dtype=float)
+    v = vec5()
+    w = vec5()
+    a = vec5(1.0)
+    b = vec5(0.0, 0.0, 0.0, 0.0, 0.0)
+    c = vec5(0.0)
+
+    v[0] = 1.0
+    v.x = 0.0
+    v[1:] = [1.0, 1.0, 1.0, 1.0]
+
+    w[0] = 1.0
+    w[1:] = [0.0, 0.0, 0.0, 0.0]
+
+    if v[0] != w[1] or v.x != w.y:
+        raise ValueError("vec setter error")
+
+    for x in v[1:]:
+        if x != 1.0:
+            raise ValueError("vec slicing error")
+
+    if b != c:
+        raise ValueError("vec equality error")
+
+    if str(v) != "[0.0, 1.0, 1.0, 1.0, 1.0]":
+        raise ValueError("vec to string error")
+
+
+def test_mat_type(test, device):
+    mat55 = wp.mat(shape=(5, 5), dtype=float)
+    m1 = mat55()
+    m2 = mat55()
+
+    for i in range(5):
+        for j in range(5):
+            if i == j:
+                m1[i, j] = 1.0
+            else:
+                m1[i, j] = 0.0
+
+    for i in range(5):
+        m2[i] = [1.0, 1.0, 1.0, 1.0, 1.0]
+
+    a = mat55(1.0)
+    b = mat55(
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+    )
+
+    if m1 != b:
+        raise ValueError("mat element setting error")
+
+    if m2 != a:
+        raise ValueError("mat row setting error")
+
+    if m1[0, 0] != 1.0:
+        raise ValueError("mat element getting error")
+
+    if m2[0] != [1.0, 1.0, 1.0, 1.0, 1.0]:
+        raise ValueError("mat row getting error")
+
+    if (
+        str(b)
+        != "[[1.0, 0.0, 0.0, 0.0, 0.0],\n [0.0, 1.0, 0.0, 0.0, 0.0],\n [0.0, 0.0, 1.0, 0.0, 0.0],\n [0.0, 0.0, 0.0, 1.0, 0.0],\n [0.0, 0.0, 0.0, 0.0, 1.0]]"
+    ):
+        raise ValueError("mat to string error")
+
+
+devices = get_test_devices()
+
+
+class TestMath(unittest.TestCase):
+    pass
 
-    class TestMath(parent):
-        pass
 
-    add_function_test(TestMath, "test_scalar_math", test_scalar_math, devices=devices)
-    return TestMath
+add_function_test(TestMath, "test_scalar_math", test_scalar_math, devices=devices)
+add_function_test(TestMath, "test_vec_type", test_vec_type, devices=devices)
+add_function_test(TestMath, "test_mat_type", test_mat_type, devices=devices)
 
 
 if __name__ == "__main__":
-    _ = register(unittest.TestCase)
+    wp.build.clear_kernel_cache()
     unittest.main(verbosity=2)

warp/tests/test_matmul.py

@@ -1,88 +1,107 @@
-import numpy as np
+# Copyright (c) 2022 NVIDIA CORPORATION.  All rights reserved.
+# NVIDIA CORPORATION and its licensors retain all intellectual property
+# and proprietary rights in and to this software, related documentation
+# and any modifications thereto.  Any use, reproduction, disclosure or
+# distribution of this software and related documentation without an express
+# license agreement from NVIDIA CORPORATION is strictly prohibited.
+
 import unittest
 
-import warp as wp
-from warp.tests.test_base import *
+import numpy as np
 
-np.random.seed(0)
+import warp as wp
+from warp.tests.unittest_utils import *
 
 wp.init()
-wp.config.mode = "debug"
+
+from warp.context import runtime  # noqa: E402
 
 
-class GemmTestbedRunner:
+class gemm_test_bed_runner:
     def __init__(self, dtype, device):
         self.dtype = dtype
         self.device = device
 
     def alloc(self, m, n, k, batch_count):
+        rng = np.random.default_rng(42)
         low = -4.5
         high = 3.5
         if batch_count == 1:
             A = wp.array2d(
-                np.ceil(np.random.uniform(low=low, high=high, size=(m, k))), dtype=self.dtype, device=self.device
+                np.ceil(rng.uniform(low=low, high=high, size=(m, k))),
+                dtype=self.dtype,
+                device=self.device,
+                requires_grad=True,
             )
             B = wp.array2d(
-                np.ceil(np.random.uniform(low=low, high=high, size=(k, n))), dtype=self.dtype, device=self.device
+                np.ceil(rng.uniform(low=low, high=high, size=(k, n))),
+                dtype=self.dtype,
+                device=self.device,
+                requires_grad=True,
             )
             C = wp.array2d(
-                np.ceil(np.random.uniform(low=low, high=high, size=(m, n))), dtype=self.dtype, device=self.device
+                np.ceil(rng.uniform(low=low, high=high, size=(m, n))),
+                dtype=self.dtype,
+                device=self.device,
+                requires_grad=True,
             )
-            D = wp.array2d(np.zeros((m, n)), dtype=self.dtype, device=self.device)
-            adj_A = wp.array2d(np.zeros((m, k)), dtype=self.dtype, device=self.device)
-            adj_B = wp.array2d(np.zeros((k, n)), dtype=self.dtype, device=self.device)
-            adj_C = wp.array2d(np.zeros((m, n)), dtype=self.dtype, device=self.device)
-            adj_D = wp.array2d(np.ones((m, n)), dtype=self.dtype, device=self.device)
+            D = wp.array2d(np.zeros((m, n)), dtype=self.dtype, device=self.device, requires_grad=True)
         else:
-            A = wp.array2d(
-                np.ceil(np.random.uniform(low=low, high=high, size=(batch_count, m, k))),
+            A = wp.array3d(
+                np.ceil(rng.uniform(low=low, high=high, size=(batch_count, m, k))),
                 dtype=self.dtype,
                 device=self.device,
+                requires_grad=True,
             )
-            B = wp.array2d(
-                np.ceil(np.random.uniform(low=low, high=high, size=(batch_count, k, n))),
+            B = wp.array3d(
+                np.ceil(rng.uniform(low=low, high=high, size=(batch_count, k, n))),
                 dtype=self.dtype,
                 device=self.device,
+                requires_grad=True,
             )
-            C = wp.array2d(
-                np.ceil(np.random.uniform(low=low, high=high, size=(batch_count, m, n))),
+            C = wp.array3d(
+                np.ceil(rng.uniform(low=low, high=high, size=(batch_count, m, n))),
                 dtype=self.dtype,
                 device=self.device,
+                requires_grad=True,
             )
-            D = wp.array2d(np.zeros((batch_count, m, n)), dtype=self.dtype, device=self.device)
-            adj_A = wp.array2d(np.zeros((batch_count, m, k)), dtype=self.dtype, device=self.device)
-            adj_B = wp.array2d(np.zeros((batch_count, k, n)), dtype=self.dtype, device=self.device)
-            adj_C = wp.array2d(np.zeros((batch_count, m, n)), dtype=self.dtype, device=self.device)
-            adj_D = wp.array2d(np.ones((batch_count, m, n)), dtype=self.dtype, device=self.device)
-        return A, B, C, D, adj_A, adj_B, adj_C, adj_D
+            D = wp.array3d(np.zeros((batch_count, m, n)), dtype=self.dtype, device=self.device, requires_grad=True)
+        return A, B, C, D
 
     def run_and_verify(self, m, n, k, batch_count, alpha, beta):
-        A, B, C, D, adj_A, adj_B, adj_C, adj_D = self.alloc(m, n, k, batch_count)
+        A, B, C, D = self.alloc(m, n, k, batch_count)
+        ones = wp.zeros_like(D)
+        ones.fill_(1.0)
+
         if batch_count == 1:
-            wp.matmul(A, B, C, D, alpha, beta, False, self.device)
+            tape = wp.Tape()
+            with tape:
+                wp.matmul(A, B, C, D, alpha, beta, False, self.device)
+            tape.backward(grads={D: ones})
+
             D_np = alpha * (A.numpy() @ B.numpy()) + beta * C.numpy()
             assert np.array_equal(D_np, D.numpy())
 
-            wp.adj_matmul(A, B, C, adj_A, adj_B, adj_C, adj_D, alpha, beta, False, self.device)
-            adj_A_np = alpha * np.matmul(adj_D.numpy(), B.numpy().transpose())
-            adj_B_np = alpha * (A.numpy().transpose() @ adj_D.numpy())
-            adj_C_np = beta * adj_D.numpy()
+            adj_A_np = alpha * np.matmul(ones.numpy(), B.numpy().transpose())
+            adj_B_np = alpha * (A.numpy().transpose() @ ones.numpy())
+            adj_C_np = beta * ones.numpy()
 
-            assert np.array_equal(adj_A_np, adj_A.numpy())
-            assert np.array_equal(adj_B_np, adj_B.numpy())
-            assert np.array_equal(adj_C_np, adj_C.numpy())
         else:
-            wp.batched_matmul(A, B, C, D, alpha, beta, False, self.device)
+            tape = wp.Tape()
+            with tape:
+                wp.batched_matmul(A, B, C, D, alpha, beta, False, self.device)
+            tape.backward(grads={D: ones})
+
             D_np = alpha * np.matmul(A.numpy(), B.numpy()) + beta * C.numpy()
             assert np.array_equal(D_np, D.numpy())
 
-            wp.adj_batched_matmul(A, B, C, adj_A, adj_B, adj_C, adj_D, alpha, beta, False, self.device)
-            adj_A_np = alpha * np.matmul(adj_D.numpy(), B.numpy().transpose((0, 2, 1)))
-            adj_B_np = alpha * np.matmul(A.numpy().transpose((0, 2, 1)), adj_D.numpy())
-            adj_C_np = beta * adj_D.numpy()
-            assert np.array_equal(adj_A_np, adj_A.numpy())
-            assert np.array_equal(adj_B_np, adj_B.numpy())
-            assert np.array_equal(adj_C_np, adj_C.numpy())
+            adj_A_np = alpha * np.matmul(ones.numpy(), B.numpy().transpose((0, 2, 1)))
+            adj_B_np = alpha * np.matmul(A.numpy().transpose((0, 2, 1)), ones.numpy())
+            adj_C_np = beta * ones.numpy()
+
+        assert np.array_equal(adj_A_np, A.grad.numpy())
+        assert np.array_equal(adj_B_np, B.grad.numpy())
+        assert np.array_equal(adj_C_np, C.grad.numpy())
 
     def run(self):
         Ms = [64, 128, 512]
@@ -100,17 +119,156 @@ class GemmTestbedRunner:
                             self.run_and_verify(m, n, k, batch_count, alpha, beta)
 
 
+class gemm_test_bed_runner_transpose:
+    def __init__(self, dtype, device):
+        self.dtype = dtype
+        self.device = device
+
+    def alloc(self, m, n, k, batch_count):
+        rng = np.random.default_rng(42)
+        low = -4.5
+        high = 3.5
+        if batch_count == 1:
+            A = wp.array2d(
+                np.ceil(rng.uniform(low=low, high=high, size=(m, k))),
+                dtype=self.dtype,
+                device=self.device,
+                requires_grad=True,
+            )
+            B = wp.array2d(
+                np.ceil(rng.uniform(low=low, high=high, size=(k, n))),
+                dtype=self.dtype,
+                device=self.device,
+                requires_grad=True,
+            )
+            C = wp.array2d(
+                np.ceil(rng.uniform(low=low, high=high, size=(m, n))),
+                dtype=self.dtype,
+                device=self.device,
+                requires_grad=True,
+            )
+            D = wp.array2d(np.zeros((m, n)), dtype=self.dtype, device=self.device, requires_grad=True)
+            AT = wp.array2d(A.numpy().transpose([1, 0]), dtype=self.dtype, device=self.device, requires_grad=True)
+            BT = wp.array2d(B.numpy().transpose([1, 0]), dtype=self.dtype, device=self.device, requires_grad=True)
+        else:
+            A = wp.array3d(
+                np.ceil(rng.uniform(low=low, high=high, size=(batch_count, m, k))),
+                dtype=self.dtype,
+                device=self.device,
+                requires_grad=True,
+            )
+            B = wp.array3d(
+                np.ceil(rng.uniform(low=low, high=high, size=(batch_count, k, n))),
+                dtype=self.dtype,
+                device=self.device,
+                requires_grad=True,
+            )
+            C = wp.array3d(
+                np.ceil(rng.uniform(low=low, high=high, size=(batch_count, m, n))),
+                dtype=self.dtype,
+                device=self.device,
+                requires_grad=True,
+            )
+            D = wp.array3d(np.zeros((batch_count, m, n)), dtype=self.dtype, device=self.device, requires_grad=True)
+            AT = wp.array3d(A.numpy().transpose([0, 2, 1]), dtype=self.dtype, device=self.device, requires_grad=True)
+            BT = wp.array3d(B.numpy().transpose([0, 2, 1]), dtype=self.dtype, device=self.device, requires_grad=True)
+        return A, B, C, D, AT, BT
+
+    def run_and_verify(self, m, n, k, batch_count, alpha, beta):
+        A, B, C1, D1, AT1, BT1 = self.alloc(m, n, k, batch_count)
+        C2 = wp.clone(C1)
+        C3 = wp.clone(C1)
+        D2 = wp.clone(D1)
+        D3 = wp.clone(D1)
+        AT2 = wp.clone(AT1)
+        BT2 = wp.clone(BT1)
+        ones1 = wp.zeros_like(D1)
+        ones1.fill_(1.0)
+        ones2 = wp.zeros_like(D2)
+        ones2.fill_(1.0)
+        ones3 = wp.zeros_like(D3)
+        ones3.fill_(1.0)
+
+        if batch_count == 1:
+            ATT1 = AT1.transpose([1, 0])
+            BTT1 = BT1.transpose([1, 0])
+            ATT2 = AT2.transpose([1, 0])
+            BTT2 = BT2.transpose([1, 0])
+            tape = wp.Tape()
+            with tape:
+                wp.matmul(A, BTT1, C1, D1, alpha, beta, False, self.device)
+                wp.matmul(ATT1, B, C2, D2, alpha, beta, False, self.device)
+                wp.matmul(ATT2, BTT2, C3, D3, alpha, beta, False, self.device)
+            tape.backward(grads={D1: ones1, D2: ones2, D3: ones3})
+
+            D_np = alpha * (A.numpy() @ B.numpy()) + beta * C1.numpy()
+            assert np.array_equal(D_np, D1.numpy())
+            assert np.array_equal(D_np, D2.numpy())
+            assert np.array_equal(D_np, D3.numpy())
+
+            adj_A_np = alpha * (ones1.numpy() @ B.numpy().transpose())
+            adj_B_np = alpha * (A.numpy().transpose() @ ones1.numpy())
+            adj_C_np = beta * ones1.numpy()
+
+        else:
+            ATT1 = AT1.transpose([0, 2, 1])
+            BTT1 = BT1.transpose([0, 2, 1])
+            ATT2 = AT2.transpose([0, 2, 1])
+            BTT2 = BT2.transpose([0, 2, 1])
+            tape = wp.Tape()
+            with tape:
+                wp.batched_matmul(A, BTT1, C1, D1, alpha, beta, False, self.device)
+                wp.batched_matmul(ATT1, B, C2, D2, alpha, beta, False, self.device)
+                wp.batched_matmul(ATT2, BTT2, C3, D3, alpha, beta, False, self.device)
+            tape.backward(grads={D1: ones1, D2: ones2, D3: ones3})
+
+            D_np = alpha * np.matmul(A.numpy(), B.numpy()) + beta * C1.numpy()
+            assert np.array_equal(D_np, D1.numpy())
+            assert np.array_equal(D_np, D2.numpy())
+            assert np.array_equal(D_np, D3.numpy())
+
+            adj_A_np = alpha * np.matmul(ones1.numpy(), B.numpy().transpose((0, 2, 1)))
+            adj_B_np = alpha * np.matmul(A.numpy().transpose((0, 2, 1)), ones1.numpy())
+            adj_C_np = beta * ones1.numpy()
+
+        assert np.array_equal(adj_A_np, A.grad.numpy())
+        assert np.array_equal(adj_A_np, ATT1.grad.numpy())
+        assert np.array_equal(adj_A_np, ATT2.grad.numpy())
+        assert np.array_equal(adj_B_np, B.grad.numpy())
+        assert np.array_equal(adj_B_np, BTT1.grad.numpy())
+        assert np.array_equal(adj_B_np, BTT2.grad.numpy())
+        assert np.array_equal(adj_C_np, C1.grad.numpy())
+        assert np.array_equal(adj_C_np, C2.grad.numpy())
+        assert np.array_equal(adj_C_np, C3.grad.numpy())
+
+    def run(self):
+        m = 16
+        n = 32
+        k = 64
+        batch_counts = [1, 4]
+        beta = 1.0
+        alpha = 1.0
+
+        for batch_count in batch_counts:
+            self.run_and_verify(m, n, k, batch_count, alpha, beta)
+
+
 # NOTE: F16 tests are slow due to the performance of the reference numpy F16 matmuls performed on CPU.
 def test_f16(test, device):
-    GemmTestbedRunner(wp.float16, device).run()
+    gemm_test_bed_runner(wp.float16, device).run()
+    gemm_test_bed_runner_transpose(wp.float16, device).run()
 
 
+@unittest.skipUnless(runtime.core.is_cutlass_enabled(), "Warp was not built with CUTLASS support")
 def test_f32(test, device):
-    GemmTestbedRunner(wp.float32, device).run()
+    gemm_test_bed_runner(wp.float32, device).run()
+    gemm_test_bed_runner_transpose(wp.float32, device).run()
 
 
+@unittest.skipUnless(runtime.core.is_cutlass_enabled(), "Warp was not built with CUTLASS support")
 def test_f64(test, device):
-    GemmTestbedRunner(wp.float64, device).run()
+    gemm_test_bed_runner(wp.float64, device).run()
+    gemm_test_bed_runner_transpose(wp.float64, device).run()
 
 
 @wp.kernel
@@ -119,20 +277,22 @@ def matrix_sum_kernel(arr: wp.array2d(dtype=float), loss: wp.array(dtype=float))
     wp.atomic_add(loss, 0, arr[i, j])
 
 
+@unittest.skipUnless(runtime.core.is_cutlass_enabled(), "Warp was not built with CUTLASS support")
 def test_tape(test, device):
+    rng = np.random.default_rng(42)
     low = -4.5
     high = 3.5
     m = 64
     n = 128
     k = 256
     A = wp.array2d(
-        np.ceil(np.random.uniform(low=low, high=high, size=(m, k))), dtype=float, device=device, requires_grad=True
+        np.ceil(rng.uniform(low=low, high=high, size=(m, k))), dtype=float, device=device, requires_grad=True
     )
     B = wp.array2d(
-        np.ceil(np.random.uniform(low=low, high=high, size=(k, n))), dtype=float, device=device, requires_grad=True
+        np.ceil(rng.uniform(low=low, high=high, size=(k, n))), dtype=float, device=device, requires_grad=True
     )
     C = wp.array2d(
-        np.ceil(np.random.uniform(low=low, high=high, size=(m, n))), dtype=float, device=device, requires_grad=True
+        np.ceil(rng.uniform(low=low, high=high, size=(m, n))), dtype=float, device=device, requires_grad=True
     )
     D = wp.array2d(np.zeros((m, n)), dtype=float, device=device, requires_grad=True)
     loss = wp.zeros(1, dtype=float, device=device, requires_grad=True)
@@ -145,6 +305,7 @@ def test_tape(test, device):
 
     tape.backward(loss=loss)
     A_grad = A.grad.numpy()
+    tape.reset()
 
     # test adjoint
     D.grad = wp.array2d(np.ones((m, n)), dtype=float, device=device)
@@ -156,17 +317,19 @@ def test_tape(test, device):
     assert_array_equal(A.grad, wp.zeros_like(A))
 
 
+@unittest.skipUnless(runtime.core.is_cutlass_enabled(), "Warp was not built with CUTLASS support")
 def test_operator(test, device):
+    rng = np.random.default_rng(42)
     low = -4.5
     high = 3.5
     m = 64
     n = 128
     k = 256
     A = wp.array2d(
-        np.ceil(np.random.uniform(low=low, high=high, size=(m, k))), dtype=float, device=device, requires_grad=True
+        np.ceil(rng.uniform(low=low, high=high, size=(m, k))), dtype=float, device=device, requires_grad=True
     )
     B = wp.array2d(
-        np.ceil(np.random.uniform(low=low, high=high, size=(k, n))), dtype=float, device=device, requires_grad=True
+        np.ceil(rng.uniform(low=low, high=high, size=(k, n))), dtype=float, device=device, requires_grad=True
     )
     loss = wp.zeros(1, dtype=float, device=device, requires_grad=True)
 
@@ -180,7 +343,6 @@ def test_operator(test, device):
 
     # test adjoint
     D.grad = wp.array2d(np.ones((m, n)), dtype=float, device=device)
-    # deep copy, needed because transpose data is not contiguous
     B_transpose = wp.array2d(B.transpose().numpy(), dtype=float, device=device)
 
     adj_A = D.grad @ B_transpose
@@ -191,28 +353,101 @@ def test_operator(test, device):
     assert_array_equal(A.grad, wp.zeros_like(A))
 
 
-def register(parent):
-    devices = [d for d in get_test_devices()]
+@unittest.skipUnless(runtime.core.is_cutlass_enabled(), "Warp was not built with CUTLASS support")
+def test_large_batch_count(test, device):
+    rng = np.random.default_rng(42)
+    low = -4.5
+    high = 3.5
+    m = 2
+    n = 3
+    k = 4
+    batch_count = 65535 * 2 + int(65535 / 2)
+    A = wp.array3d(
+        np.ceil(rng.uniform(low=low, high=high, size=(batch_count, m, k))),
+        dtype=float,
+        device=device,
+        requires_grad=True,
+    )
+    B = wp.array3d(
+        np.ceil(rng.uniform(low=low, high=high, size=(batch_count, k, n))),
+        dtype=float,
+        device=device,
+        requires_grad=True,
+    )
+    C = wp.array3d(
+        np.ceil(rng.uniform(low=low, high=high, size=(batch_count, m, n))),
+        dtype=float,
+        device=device,
+        requires_grad=True,
+    )
+    D = wp.array3d(np.zeros((batch_count, m, n)), dtype=float, device=device, requires_grad=True)
+    ones = wp.zeros_like(D)
+    ones.fill_(1.0)
+
+    alpha = 1.0
+    beta = 1.0
 
-    class TestMatmul(parent):
-        pass
+    tape = wp.Tape()
+    with tape:
+        wp.batched_matmul(A, B, C, D, alpha=alpha, beta=beta, allow_tf32x3_arith=False, device=device)
+    tape.backward(grads={D: ones})
 
-    if devices:
-        # check if CUTLASS is available
-        from warp.context import runtime
+    D_np = alpha * np.matmul(A.numpy(), B.numpy()) + beta * C.numpy()
+    assert np.array_equal(D_np, D.numpy())
+
+    adj_A_np = alpha * np.matmul(ones.numpy(), B.numpy().transpose((0, 2, 1)))
+    adj_B_np = alpha * np.matmul(A.numpy().transpose((0, 2, 1)), ones.numpy())
+    adj_C_np = beta * ones.numpy()
+
+    assert np.array_equal(adj_A_np, A.grad.numpy())
+    assert np.array_equal(adj_B_np, B.grad.numpy())
+    assert np.array_equal(adj_C_np, C.grad.numpy())
+
+
+def test_adjoint_accumulation(test, device):
+    a_np = np.ones(shape=(2,3))
+    b_np = np.ones(shape=(3,2))
+    c_np = np.zeros(shape=(2,2))
+    d_np = np.zeros(shape=(2,2))
+
+    a_wp = wp.from_numpy(a_np, dtype=float, requires_grad=True)
+    b_wp = wp.from_numpy(b_np, dtype=float, requires_grad=True)
+    c_wp = wp.from_numpy(c_np, dtype=float, requires_grad=True)
+    d1_wp = wp.from_numpy(d_np, dtype=float, requires_grad=True)
+    d2_wp = wp.from_numpy(d_np, dtype=float, requires_grad=True)
+
+    tape = wp.Tape()
+
+    with tape:
+        wp.matmul(a_wp, b_wp, c_wp, d1_wp, alpha=1.0, beta=1.0)
+        wp.matmul(a_wp, b_wp, d1_wp, d2_wp, alpha=1.0, beta=1.0)
+
+    d_grad = wp.zeros_like(d2_wp)
+    d_grad.fill_(1.)
+    grads = {d2_wp : d_grad}
+    tape.backward(grads=grads) 
+
+    assert np.array_equal(a_wp.grad.numpy(), 4.0 * np.ones(shape=(2,3)))
+    assert np.array_equal(b_wp.grad.numpy(), 4.0 * np.ones(shape=(3,2)))
+    assert np.array_equal(c_wp.grad.numpy(), np.ones(shape=(2,2)))
+
+
+devices = get_test_devices()
+
+
+class TestMatmul(unittest.TestCase):
+    pass
 
-        if runtime.core.is_cutlass_enabled():
-            # add_function_test(TestMatmul, "test_f16", test_f16, devices=devices)
-            add_function_test(TestMatmul, "test_f32", test_f32, devices=devices)
-            add_function_test(TestMatmul, "test_f64", test_f64, devices=devices)
-            add_function_test(TestMatmul, "test_tape", test_tape, devices=devices)
-            add_function_test(TestMatmul, "test_operator", test_operator, devices=devices)
-        else:
-            print("Skipping matmul tests because CUTLASS is not supported in this build")
 
-    return TestMatmul
+# add_function_test(TestMatmul, "test_f16", test_f16, devices=devices)
+add_function_test(TestMatmul, "test_f32", test_f32, devices=devices)
+add_function_test(TestMatmul, "test_f64", test_f64, devices=devices)
+add_function_test(TestMatmul, "test_tape", test_tape, devices=devices)
+add_function_test(TestMatmul, "test_operator", test_operator, devices=devices)
+add_function_test(TestMatmul, "test_large_batch_count", test_large_batch_count, devices=devices)
+add_function_test(TestMatmul, "test_adjoint_accumulation", test_adjoint_accumulation, devices=devices)
 
 
 if __name__ == "__main__":
-    c = register(unittest.TestCase)
+    wp.build.clear_kernel_cache()
     unittest.main(verbosity=2, failfast=False)