ferrotensor 0.4.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/Cargo.lock +18 -36
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/Cargo.toml +2 -2
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/PKG-INFO +2 -2
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/Cargo.toml +1 -1
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/backend/compute.rs +15 -1
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/backend/kernels.rs +101 -27
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/backend/mod.rs +1 -1
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/device.rs +202 -19
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/random.rs +0 -24
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/tensor/core.rs +10 -1
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/tensor/mod.rs +1 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/tensor/raw.rs +29 -1
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/tensor/tests.rs +130 -0
- ferrotensor-0.6.0/ferrotensor/tests/manual_seed.rs +24 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/pyproject.toml +1 -1
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/README.md +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/README.md +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/examples/README.md +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/examples/char_transformer.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/examples/data/sonnets.txt +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/examples/gpu_matmul_attention.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/examples/mlp_two_moons.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/examples/quickstart.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/examples/similarity_search.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/data.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/functional.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/lib.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/nn/activation.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/nn/attention.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/nn/container.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/nn/dropout.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/nn/embedding.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/nn/linear.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/nn/mod.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/nn/module.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/nn/normalization.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/nn/tests.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/optim/adam.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/optim/mod.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/optim/optimizer.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/optim/scheduler.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/optim/sgd.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/optim/tests.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/serialize.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/tensor/autograd.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/tensor/cpu.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/tensor/ops.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor/src/tensor/shape.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/.gitignore +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/Cargo.toml +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/README.md +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/build.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/examples/mlp.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/src/device.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/src/error.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/src/functional.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/src/lib.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/src/nn.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/src/optim.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/src/serialize.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/src/tensor.rs +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/tests/conftest.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/tests/test_autograd.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/tests/test_errors.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/tests/test_functional.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/tests/test_gpu.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/tests/test_nn.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/tests/test_optim.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/tests/test_serialize.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/tests/test_tensor.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/ferrotensor-py/tests/test_train.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/python/ferrotensor/__init__.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/python/ferrotensor/_ferrotensor.pyi +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/python/ferrotensor/nn/__init__.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/python/ferrotensor/nn/functional.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/python/ferrotensor/optim.py +0 -0
- {ferrotensor-0.4.0 → ferrotensor-0.6.0}/python/ferrotensor/py.typed +0 -0
|
@@ -606,7 +606,7 @@ dependencies = [
|
|
|
606
606
|
|
|
607
607
|
[[package]]
|
|
608
608
|
name = "ferrotensor"
|
|
609
|
-
version = "0.
|
|
609
|
+
version = "0.6.0"
|
|
610
610
|
dependencies = [
|
|
611
611
|
"bytemuck",
|
|
612
612
|
"naga",
|
|
@@ -619,7 +619,7 @@ dependencies = [
|
|
|
619
619
|
|
|
620
620
|
[[package]]
|
|
621
621
|
name = "ferrotensor-autograd"
|
|
622
|
-
version = "0.
|
|
622
|
+
version = "0.6.0"
|
|
623
623
|
dependencies = [
|
|
624
624
|
"ferrotensor-backend",
|
|
625
625
|
"pollster",
|
|
@@ -628,11 +628,11 @@ dependencies = [
|
|
|
628
628
|
|
|
629
629
|
[[package]]
|
|
630
630
|
name = "ferrotensor-backend"
|
|
631
|
-
version = "0.
|
|
631
|
+
version = "0.6.0"
|
|
632
632
|
dependencies = [
|
|
633
633
|
"bytemuck",
|
|
634
634
|
"env_logger",
|
|
635
|
-
"
|
|
635
|
+
"half",
|
|
636
636
|
"log",
|
|
637
637
|
"naga",
|
|
638
638
|
"pollster",
|
|
@@ -641,7 +641,7 @@ dependencies = [
|
|
|
641
641
|
|
|
642
642
|
[[package]]
|
|
643
643
|
name = "ferrotensor-benchmarks"
|
|
644
|
-
version = "0.
|
|
644
|
+
version = "0.6.0"
|
|
645
645
|
dependencies = [
|
|
646
646
|
"ferrotensor",
|
|
647
647
|
"ferrotensor-backend",
|
|
@@ -652,7 +652,7 @@ dependencies = [
|
|
|
652
652
|
|
|
653
653
|
[[package]]
|
|
654
654
|
name = "ferrotensor-cli"
|
|
655
|
-
version = "0.
|
|
655
|
+
version = "0.6.0"
|
|
656
656
|
dependencies = [
|
|
657
657
|
"clap",
|
|
658
658
|
"env_logger",
|
|
@@ -670,7 +670,7 @@ dependencies = [
|
|
|
670
670
|
|
|
671
671
|
[[package]]
|
|
672
672
|
name = "ferrotensor-graph"
|
|
673
|
-
version = "0.
|
|
673
|
+
version = "0.6.0"
|
|
674
674
|
dependencies = [
|
|
675
675
|
"bytemuck",
|
|
676
676
|
"env_logger",
|
|
@@ -685,7 +685,7 @@ dependencies = [
|
|
|
685
685
|
|
|
686
686
|
[[package]]
|
|
687
687
|
name = "ferrotensor-hub"
|
|
688
|
-
version = "0.
|
|
688
|
+
version = "0.6.0"
|
|
689
689
|
dependencies = [
|
|
690
690
|
"ferrotensor-autograd",
|
|
691
691
|
"ferrotensor-backend",
|
|
@@ -698,7 +698,7 @@ dependencies = [
|
|
|
698
698
|
|
|
699
699
|
[[package]]
|
|
700
700
|
name = "ferrotensor-loader"
|
|
701
|
-
version = "0.
|
|
701
|
+
version = "0.6.0"
|
|
702
702
|
dependencies = [
|
|
703
703
|
"bytemuck",
|
|
704
704
|
"env_logger",
|
|
@@ -716,7 +716,7 @@ dependencies = [
|
|
|
716
716
|
|
|
717
717
|
[[package]]
|
|
718
718
|
name = "ferrotensor-monitor"
|
|
719
|
-
version = "0.
|
|
719
|
+
version = "0.6.0"
|
|
720
720
|
dependencies = [
|
|
721
721
|
"anyhow",
|
|
722
722
|
"chrono",
|
|
@@ -730,7 +730,7 @@ dependencies = [
|
|
|
730
730
|
|
|
731
731
|
[[package]]
|
|
732
732
|
name = "ferrotensor-py"
|
|
733
|
-
version = "0.
|
|
733
|
+
version = "0.6.0"
|
|
734
734
|
dependencies = [
|
|
735
735
|
"ferrotensor",
|
|
736
736
|
"pyo3",
|
|
@@ -739,7 +739,7 @@ dependencies = [
|
|
|
739
739
|
|
|
740
740
|
[[package]]
|
|
741
741
|
name = "ferrotensor-trainer"
|
|
742
|
-
version = "0.
|
|
742
|
+
version = "0.6.0"
|
|
743
743
|
dependencies = [
|
|
744
744
|
"bytemuck",
|
|
745
745
|
"ferrotensor-autograd",
|
|
@@ -801,37 +801,25 @@ checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"
|
|
|
801
801
|
|
|
802
802
|
[[package]]
|
|
803
803
|
name = "futures-core"
|
|
804
|
-
version = "0.3.
|
|
804
|
+
version = "0.3.34"
|
|
805
805
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
806
|
-
checksum = "
|
|
807
|
-
|
|
808
|
-
[[package]]
|
|
809
|
-
name = "futures-intrusive"
|
|
810
|
-
version = "0.5.0"
|
|
811
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
812
|
-
checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f"
|
|
813
|
-
dependencies = [
|
|
814
|
-
"futures-core",
|
|
815
|
-
"lock_api",
|
|
816
|
-
"parking_lot",
|
|
817
|
-
]
|
|
806
|
+
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
|
|
818
807
|
|
|
819
808
|
[[package]]
|
|
820
809
|
name = "futures-task"
|
|
821
|
-
version = "0.3.
|
|
810
|
+
version = "0.3.34"
|
|
822
811
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
823
|
-
checksum = "
|
|
812
|
+
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
|
824
813
|
|
|
825
814
|
[[package]]
|
|
826
815
|
name = "futures-util"
|
|
827
|
-
version = "0.3.
|
|
816
|
+
version = "0.3.34"
|
|
828
817
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
829
|
-
checksum = "
|
|
818
|
+
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
|
830
819
|
dependencies = [
|
|
831
820
|
"futures-core",
|
|
832
821
|
"futures-task",
|
|
833
822
|
"pin-project-lite",
|
|
834
|
-
"pin-utils",
|
|
835
823
|
"slab",
|
|
836
824
|
]
|
|
837
825
|
|
|
@@ -1486,12 +1474,6 @@ version = "0.2.16"
|
|
|
1486
1474
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1487
1475
|
checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
|
|
1488
1476
|
|
|
1489
|
-
[[package]]
|
|
1490
|
-
name = "pin-utils"
|
|
1491
|
-
version = "0.1.0"
|
|
1492
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1493
|
-
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
|
1494
|
-
|
|
1495
1477
|
[[package]]
|
|
1496
1478
|
name = "pkg-config"
|
|
1497
1479
|
version = "0.3.32"
|
|
@@ -4,10 +4,10 @@ resolver = "2"
|
|
|
4
4
|
|
|
5
5
|
[workspace.package]
|
|
6
6
|
edition = "2021"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.6.0"
|
|
8
8
|
license = "MIT OR Apache-2.0"
|
|
9
9
|
repository = "https://github.com/puranikyashaswin/ferrotensor"
|
|
10
|
-
authors = ["Yashaswin
|
|
10
|
+
authors = ["Puranik Yashaswin Sharma <puranikyashaswinsharma@gmail.com>"]
|
|
11
11
|
rust-version = "1.88"
|
|
12
12
|
homepage = "https://github.com/puranikyashaswin/ferrotensor"
|
|
13
13
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ferrotensor
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Classifier: Programming Language :: Rust
|
|
5
5
|
Classifier: Programming Language :: Python :: 3
|
|
6
6
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
@@ -10,7 +10,7 @@ Provides-Extra: test
|
|
|
10
10
|
Summary: PyTorch-style tensors, autograd and nn on wgpu (Metal, Vulkan, DX12) or the CPU, written in Rust
|
|
11
11
|
Keywords: machine-learning,deep-learning,gpu,autograd,wgpu
|
|
12
12
|
Home-Page: https://github.com/puranikyashaswin/ferrotensor
|
|
13
|
-
Author-email: Yashaswin
|
|
13
|
+
Author-email: Puranik Yashaswin Sharma <puranikyashaswinsharma@gmail.com>
|
|
14
14
|
License-Expression: MIT OR Apache-2.0
|
|
15
15
|
Requires-Python: >=3.10
|
|
16
16
|
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
@@ -9,7 +9,7 @@ use wgpu::{BindGroup, BindGroupLayout, BindingResource, CommandEncoder, ComputeP
|
|
|
9
9
|
use super::KERNELS_WGSL;
|
|
10
10
|
|
|
11
11
|
/// Entry points compiled from [`KERNELS_WGSL`].
|
|
12
|
-
pub(crate) const KERNEL_NAMES: [&str;
|
|
12
|
+
pub(crate) const KERNEL_NAMES: [&str; 15] = [
|
|
13
13
|
"unary",
|
|
14
14
|
"binary",
|
|
15
15
|
"reduce",
|
|
@@ -24,8 +24,14 @@ pub(crate) const KERNEL_NAMES: [&str; 14] = [
|
|
|
24
24
|
"copy_block",
|
|
25
25
|
"cross_entropy_rows",
|
|
26
26
|
"cross_entropy_grad",
|
|
27
|
+
"fused",
|
|
27
28
|
];
|
|
28
29
|
|
|
30
|
+
/// Storage buffers the `fused` kernel binds: seven tensor slots and its
|
|
31
|
+
/// program. Devices with fewer per-stage bindings do not compile it, and
|
|
32
|
+
/// element-wise ops then run unfused.
|
|
33
|
+
pub(crate) const FUSED_STORAGE_BINDINGS: u32 = 8;
|
|
34
|
+
|
|
29
35
|
/// Threads per workgroup of the 1-D kernels.
|
|
30
36
|
const WORKGROUP_1D: u32 = 256;
|
|
31
37
|
/// Largest workgroup count allowed in one grid dimension.
|
|
@@ -52,8 +58,11 @@ impl ComputeEngine {
|
|
|
52
58
|
source: wgpu::ShaderSource::Wgsl(Cow::Borrowed(KERNELS_WGSL)),
|
|
53
59
|
});
|
|
54
60
|
|
|
61
|
+
let fused_fits =
|
|
62
|
+
device.limits().max_storage_buffers_per_shader_stage >= FUSED_STORAGE_BINDINGS;
|
|
55
63
|
let pipelines = KERNEL_NAMES
|
|
56
64
|
.iter()
|
|
65
|
+
.filter(|&&name| name != "fused" || fused_fits)
|
|
57
66
|
.map(|&name| {
|
|
58
67
|
let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
|
|
59
68
|
label: Some(name),
|
|
@@ -69,6 +78,11 @@ impl ComputeEngine {
|
|
|
69
78
|
Self { pipelines }
|
|
70
79
|
}
|
|
71
80
|
|
|
81
|
+
/// Whether `name` was compiled for this device.
|
|
82
|
+
pub(crate) fn has(&self, name: &str) -> bool {
|
|
83
|
+
self.pipelines.contains_key(name)
|
|
84
|
+
}
|
|
85
|
+
|
|
72
86
|
/// Get a compiled pipeline by entry point name.
|
|
73
87
|
pub fn get_pipeline(&self, name: &str) -> Option<&ComputePipeline> {
|
|
74
88
|
self.pipelines.get(name).map(|(pipeline, _)| pipeline)
|
|
@@ -100,16 +100,9 @@ struct UnaryParams {
|
|
|
100
100
|
@group(0) @binding(1) var<storage, read_write> un_out: array<f32>;
|
|
101
101
|
@group(0) @binding(2) var<uniform> un_p: UnaryParams;
|
|
102
102
|
|
|
103
|
-
|
|
104
|
-
fn unary(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
105
|
-
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
106
|
-
let i = flat_index(gid, nwg);
|
|
107
|
-
if (i >= un_p.n) { return; }
|
|
108
|
-
let x = un_in[i];
|
|
109
|
-
let a = un_p.a;
|
|
110
|
-
let b = un_p.b;
|
|
103
|
+
fn unary_apply(op: u32, x: f32, a: f32, b: f32) -> f32 {
|
|
111
104
|
var y = x;
|
|
112
|
-
switch
|
|
105
|
+
switch op {
|
|
113
106
|
case 0u: { y = max(x, 0.0); }
|
|
114
107
|
case 1u: { y = x * sigmoid_f(x); }
|
|
115
108
|
case 2u: { y = gelu_tanh(x); }
|
|
@@ -152,7 +145,15 @@ fn unary(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
|
152
145
|
case 25u: { y = select(a, 1.0, x > 0.0); }
|
|
153
146
|
default: { y = x; }
|
|
154
147
|
}
|
|
155
|
-
|
|
148
|
+
return y;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
@compute @workgroup_size(256)
|
|
152
|
+
fn unary(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
153
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
154
|
+
let i = flat_index(gid, nwg);
|
|
155
|
+
if (i >= un_p.n) { return; }
|
|
156
|
+
un_out[i] = unary_apply(un_p.op, un_in[i], un_p.a, un_p.b);
|
|
156
157
|
}
|
|
157
158
|
|
|
158
159
|
// --- Broadcasting binary ops ------------------------------------------------
|
|
@@ -172,6 +173,25 @@ struct BinaryParams {
|
|
|
172
173
|
@group(0) @binding(3) var<uniform> bin_p: BinaryParams;
|
|
173
174
|
@group(0) @binding(4) var<storage, read> bin_meta: array<u32>;
|
|
174
175
|
|
|
176
|
+
fn binary_apply(op: u32, x: f32, y: f32) -> f32 {
|
|
177
|
+
var r = 0.0;
|
|
178
|
+
switch op {
|
|
179
|
+
case 0u: { r = x + y; }
|
|
180
|
+
case 1u: { r = x - y; }
|
|
181
|
+
case 2u: { r = x * y; }
|
|
182
|
+
case 3u: { r = x / y; }
|
|
183
|
+
case 4u: { r = max(x, y); }
|
|
184
|
+
case 5u: { r = min(x, y); }
|
|
185
|
+
case 6u: { r = select(0.0, 1.0, x > y); }
|
|
186
|
+
case 7u: { r = select(0.0, 1.0, x < y); }
|
|
187
|
+
case 8u: { r = select(0.0, 1.0, x == y); }
|
|
188
|
+
case 9u: { r = select(0.0, 1.0, x >= y); }
|
|
189
|
+
case 10u: { r = select(0.0, 1.0, x <= y); }
|
|
190
|
+
default: { r = x; }
|
|
191
|
+
}
|
|
192
|
+
return r;
|
|
193
|
+
}
|
|
194
|
+
|
|
175
195
|
@compute @workgroup_size(256)
|
|
176
196
|
fn binary(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
177
197
|
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
@@ -189,24 +209,78 @@ fn binary(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
|
189
209
|
ia = ia + idx * bin_meta[nd + axis];
|
|
190
210
|
ib = ib + idx * bin_meta[2u * nd + axis];
|
|
191
211
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
212
|
+
bin_out[i] = binary_apply(bin_p.op, bin_a[ia], bin_b[ib]);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// --- Fused element-wise chains ---------------------------------------------
|
|
216
|
+
// A chain of same-size unary and non-broadcasting binary ops in one dispatch
|
|
217
|
+
// (FERROTENSOR_FUSION=1, see docs/design/op-fusion.md). Each step is 8 words:
|
|
218
|
+
// [kind, op, src0, src1, dst, a bits, b bits, 0]; kind 0 applies a unary op
|
|
219
|
+
// to src0, kind 1 a binary op to src0 and src1. Sources and destinations are
|
|
220
|
+
// binding slots 0..6. Every step stores its result and later steps load it
|
|
221
|
+
// back, as separate dispatches would.
|
|
222
|
+
|
|
223
|
+
struct FusedParams {
|
|
224
|
+
n: u32,
|
|
225
|
+
steps: u32,
|
|
226
|
+
_pad0: u32,
|
|
227
|
+
_pad1: u32,
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
@group(0) @binding(0) var<storage, read_write> fb0: array<f32>;
|
|
231
|
+
@group(0) @binding(1) var<storage, read_write> fb1: array<f32>;
|
|
232
|
+
@group(0) @binding(2) var<storage, read_write> fb2: array<f32>;
|
|
233
|
+
@group(0) @binding(3) var<storage, read_write> fb3: array<f32>;
|
|
234
|
+
@group(0) @binding(4) var<storage, read_write> fb4: array<f32>;
|
|
235
|
+
@group(0) @binding(5) var<storage, read_write> fb5: array<f32>;
|
|
236
|
+
@group(0) @binding(6) var<storage, read_write> fb6: array<f32>;
|
|
237
|
+
@group(0) @binding(7) var<storage, read> fused_prog: array<u32>;
|
|
238
|
+
@group(0) @binding(8) var<uniform> fused_p: FusedParams;
|
|
239
|
+
|
|
240
|
+
fn fused_load(slot: u32, i: u32) -> f32 {
|
|
241
|
+
var v = 0.0;
|
|
242
|
+
switch slot {
|
|
243
|
+
case 0u: { v = fb0[i]; }
|
|
244
|
+
case 1u: { v = fb1[i]; }
|
|
245
|
+
case 2u: { v = fb2[i]; }
|
|
246
|
+
case 3u: { v = fb3[i]; }
|
|
247
|
+
case 4u: { v = fb4[i]; }
|
|
248
|
+
case 5u: { v = fb5[i]; }
|
|
249
|
+
default: { v = fb6[i]; }
|
|
250
|
+
}
|
|
251
|
+
return v;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
fn fused_store(slot: u32, i: u32, v: f32) {
|
|
255
|
+
switch slot {
|
|
256
|
+
case 0u: { fb0[i] = v; }
|
|
257
|
+
case 1u: { fb1[i] = v; }
|
|
258
|
+
case 2u: { fb2[i] = v; }
|
|
259
|
+
case 3u: { fb3[i] = v; }
|
|
260
|
+
case 4u: { fb4[i] = v; }
|
|
261
|
+
case 5u: { fb5[i] = v; }
|
|
262
|
+
default: { fb6[i] = v; }
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
@compute @workgroup_size(256)
|
|
267
|
+
fn fused(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
268
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
269
|
+
let i = flat_index(gid, nwg);
|
|
270
|
+
if (i >= fused_p.n) { return; }
|
|
271
|
+
for (var k = 0u; k < fused_p.steps; k = k + 1u) {
|
|
272
|
+
let w = k * 8u;
|
|
273
|
+
let x = fused_load(fused_prog[w + 2u], i);
|
|
274
|
+
var r = 0.0;
|
|
275
|
+
if (fused_prog[w] == 0u) {
|
|
276
|
+
let a = bitcast<f32>(fused_prog[w + 5u]);
|
|
277
|
+
let b = bitcast<f32>(fused_prog[w + 6u]);
|
|
278
|
+
r = unary_apply(fused_prog[w + 1u], x, a, b);
|
|
279
|
+
} else {
|
|
280
|
+
r = binary_apply(fused_prog[w + 1u], x, fused_load(fused_prog[w + 3u], i));
|
|
281
|
+
}
|
|
282
|
+
fused_store(fused_prog[w + 4u], i, r);
|
|
208
283
|
}
|
|
209
|
-
bin_out[i] = r;
|
|
210
284
|
}
|
|
211
285
|
|
|
212
286
|
// --- Axis reductions --------------------------------------------------------
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
use std::collections::HashMap;
|
|
9
9
|
use std::num::NonZeroU64;
|
|
10
|
+
use std::sync::atomic::{AtomicBool, Ordering};
|
|
10
11
|
use std::sync::mpsc::{Receiver, TryRecvError};
|
|
11
12
|
use std::sync::{Arc, Mutex, MutexGuard};
|
|
12
13
|
use std::time::{Duration, Instant};
|
|
@@ -15,7 +16,8 @@ use wgpu::{
|
|
|
15
16
|
Device as WgpuDevice, Queue,
|
|
16
17
|
};
|
|
17
18
|
|
|
18
|
-
use crate::backend::{ComputeEngine, Dispatch};
|
|
19
|
+
use crate::backend::{ComputeEngine, Dispatch, FUSED_STORAGE_BINDINGS};
|
|
20
|
+
use crate::tensor::Storage;
|
|
19
21
|
|
|
20
22
|
/// Ops recorded before the pending commands are submitted anyway, so the GPU
|
|
21
23
|
/// starts on long op sequences that never read back to the host.
|
|
@@ -25,14 +27,91 @@ const MAX_PENDING_OPS: usize = 256;
|
|
|
25
27
|
/// [`Arg::Meta`]) of one batch.
|
|
26
28
|
const ARGS_BYTES: usize = 256 * 1024;
|
|
27
29
|
|
|
28
|
-
/// How long a wait polls the GPU before
|
|
29
|
-
///
|
|
30
|
-
/// back of a short batch.
|
|
30
|
+
/// How long a wait busy-polls the GPU before sleeping between polls.
|
|
31
|
+
/// Sleeping costs up to its length on every read back of a short batch.
|
|
31
32
|
const SPIN_WAIT: Duration = Duration::from_millis(10);
|
|
33
|
+
/// Sleep between polls once [`SPIN_WAIT`] has passed.
|
|
34
|
+
const POLL_SLEEP: Duration = Duration::from_micros(50);
|
|
32
35
|
|
|
33
36
|
/// Largest total size of freed tensor buffers kept for reuse.
|
|
34
37
|
const POOL_BYTES: u64 = 128 << 20;
|
|
35
38
|
|
|
39
|
+
/// Tensor buffers one `fused` dispatch binds (its last binding is the program).
|
|
40
|
+
const FUSED_SLOTS: usize = (FUSED_STORAGE_BINDINGS - 1) as usize;
|
|
41
|
+
|
|
42
|
+
/// One element-wise op for [`GpuContext::elementwise`].
|
|
43
|
+
pub(crate) struct ElementwiseStep {
|
|
44
|
+
/// 0 for a `unary_op`, 1 for a `binary_op`.
|
|
45
|
+
pub(crate) kind: u32,
|
|
46
|
+
pub(crate) op: u32,
|
|
47
|
+
/// Scalars of a unary op.
|
|
48
|
+
pub(crate) a: f32,
|
|
49
|
+
pub(crate) b: f32,
|
|
50
|
+
/// One input for a unary op, two for a binary op, each with `n` elements.
|
|
51
|
+
pub(crate) inputs: Vec<Arc<Storage>>,
|
|
52
|
+
/// A new buffer of `n` elements.
|
|
53
|
+
pub(crate) output: Arc<Storage>,
|
|
54
|
+
pub(crate) n: usize,
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/// Element-wise steps over the same element count, recorded as one `fused`
|
|
58
|
+
/// dispatch when the group closes. Holding each slot's storage keeps its
|
|
59
|
+
/// buffer out of the pool until then.
|
|
60
|
+
struct FusionGroup {
|
|
61
|
+
n: usize,
|
|
62
|
+
/// The storage bound to each slot, in slot order.
|
|
63
|
+
slots: Vec<Arc<Storage>>,
|
|
64
|
+
/// Eight words per step; see the `fused` kernel.
|
|
65
|
+
program: Vec<u32>,
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
impl FusionGroup {
|
|
69
|
+
fn position(&self, storage: &Arc<Storage>) -> Option<usize> {
|
|
70
|
+
self.slots.iter().position(|s| Arc::ptr_eq(s, storage))
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/// Number of slots the group would use with `step` added.
|
|
74
|
+
fn slots_with(&self, step: &ElementwiseStep) -> usize {
|
|
75
|
+
let mut added: Vec<&Arc<Storage>> = Vec::new();
|
|
76
|
+
for s in step.inputs.iter().chain(std::iter::once(&step.output)) {
|
|
77
|
+
if self.position(s).is_none() && !added.iter().any(|a| Arc::ptr_eq(a, s)) {
|
|
78
|
+
added.push(s);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
self.slots.len() + added.len()
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
fn slot(&mut self, storage: &Arc<Storage>) -> u32 {
|
|
85
|
+
let index = match self.position(storage) {
|
|
86
|
+
Some(index) => index,
|
|
87
|
+
None => {
|
|
88
|
+
self.slots.push(Arc::clone(storage));
|
|
89
|
+
self.slots.len() - 1
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
index as u32
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
fn push(&mut self, step: &ElementwiseStep) {
|
|
96
|
+
let src0 = self.slot(&step.inputs[0]);
|
|
97
|
+
let src1 = match step.inputs.get(1) {
|
|
98
|
+
Some(s) => self.slot(s),
|
|
99
|
+
None => 0,
|
|
100
|
+
};
|
|
101
|
+
let dst = self.slot(&step.output);
|
|
102
|
+
self.program.extend_from_slice(&[
|
|
103
|
+
step.kind,
|
|
104
|
+
step.op,
|
|
105
|
+
src0,
|
|
106
|
+
src1,
|
|
107
|
+
dst,
|
|
108
|
+
step.a.to_bits(),
|
|
109
|
+
step.b.to_bits(),
|
|
110
|
+
0,
|
|
111
|
+
]);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
36
115
|
/// Freed tensor buffers by size, reused by later ops.
|
|
37
116
|
///
|
|
38
117
|
/// Reuse needs no synchronisation: the queue runs the new owner's ops after
|
|
@@ -102,6 +181,11 @@ pub(crate) struct GpuContext {
|
|
|
102
181
|
align: usize,
|
|
103
182
|
pending: Mutex<Pending>,
|
|
104
183
|
pool: Arc<Mutex<BufferPool>>,
|
|
184
|
+
/// Whether element-wise ops are fused (`FERROTENSOR_FUSION=1`).
|
|
185
|
+
fusion: AtomicBool,
|
|
186
|
+
/// Placeholders for the unused slots of a `fused` dispatch: one each, so
|
|
187
|
+
/// no buffer is bound twice as read-write.
|
|
188
|
+
fused_dummies: Vec<Buffer>,
|
|
105
189
|
}
|
|
106
190
|
|
|
107
191
|
/// Work recorded since the last submission.
|
|
@@ -113,6 +197,8 @@ struct Pending {
|
|
|
113
197
|
/// Contents of [`GpuContext::args`] for this batch.
|
|
114
198
|
args: Vec<u8>,
|
|
115
199
|
ops: usize,
|
|
200
|
+
/// Element-wise steps not yet recorded as a `fused` dispatch.
|
|
201
|
+
group: Option<FusionGroup>,
|
|
116
202
|
}
|
|
117
203
|
|
|
118
204
|
impl GpuContext {
|
|
@@ -129,6 +215,18 @@ impl GpuContext {
|
|
|
129
215
|
usage: BufferUsages::UNIFORM | BufferUsages::STORAGE | BufferUsages::COPY_DST,
|
|
130
216
|
mapped_at_creation: false,
|
|
131
217
|
});
|
|
218
|
+
let fused_dummies = (0..FUSED_SLOTS)
|
|
219
|
+
.map(|_| {
|
|
220
|
+
device.create_buffer(&wgpu::BufferDescriptor {
|
|
221
|
+
label: Some("Fused slot placeholder"),
|
|
222
|
+
size: 4,
|
|
223
|
+
usage: BufferUsages::STORAGE,
|
|
224
|
+
mapped_at_creation: false,
|
|
225
|
+
})
|
|
226
|
+
})
|
|
227
|
+
.collect();
|
|
228
|
+
let fusion =
|
|
229
|
+
engine.has("fused") && std::env::var("FERROTENSOR_FUSION").as_deref() == Ok("1");
|
|
132
230
|
Self {
|
|
133
231
|
adapter,
|
|
134
232
|
device,
|
|
@@ -138,9 +236,77 @@ impl GpuContext {
|
|
|
138
236
|
align,
|
|
139
237
|
pending: Mutex::default(),
|
|
140
238
|
pool: Arc::default(),
|
|
239
|
+
fusion: AtomicBool::new(fusion),
|
|
240
|
+
fused_dummies,
|
|
141
241
|
}
|
|
142
242
|
}
|
|
143
243
|
|
|
244
|
+
/// Whether element-wise ops go through [`GpuContext::elementwise`].
|
|
245
|
+
pub(crate) fn fusion(&self) -> bool {
|
|
246
|
+
self.fusion.load(Ordering::Relaxed)
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/// Turn fusion on or off (on only where the `fused` kernel compiled).
|
|
250
|
+
/// Tests use it to compare fused and unfused runs on one device.
|
|
251
|
+
#[cfg(test)]
|
|
252
|
+
pub(crate) fn set_fusion(&self, on: bool) {
|
|
253
|
+
self.fusion
|
|
254
|
+
.store(on && self.engine.has("fused"), Ordering::Relaxed);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/// Record an element-wise op, joining the open fusion group when it has
|
|
258
|
+
/// the same element count and room for the op's buffers.
|
|
259
|
+
pub(crate) fn elementwise(&self, step: ElementwiseStep) {
|
|
260
|
+
let mut pending = self.lock();
|
|
261
|
+
let joins = pending
|
|
262
|
+
.group
|
|
263
|
+
.as_ref()
|
|
264
|
+
.is_some_and(|g| g.n == step.n && g.slots_with(&step) <= FUSED_SLOTS);
|
|
265
|
+
if !joins {
|
|
266
|
+
self.close_group(&mut pending);
|
|
267
|
+
pending.group = Some(FusionGroup {
|
|
268
|
+
n: step.n,
|
|
269
|
+
slots: Vec::new(),
|
|
270
|
+
program: Vec::new(),
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
if let Some(group) = pending.group.as_mut() {
|
|
274
|
+
group.push(&step);
|
|
275
|
+
}
|
|
276
|
+
self.count_op(&mut pending);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/// Record the open fusion group, if any, as one `fused` dispatch.
|
|
280
|
+
fn close_group(&self, pending: &mut Pending) {
|
|
281
|
+
let Some(group) = pending.group.take() else {
|
|
282
|
+
return;
|
|
283
|
+
};
|
|
284
|
+
let buffers: Vec<&Buffer> = group
|
|
285
|
+
.slots
|
|
286
|
+
.iter()
|
|
287
|
+
.map(|s| match &**s {
|
|
288
|
+
Storage::Gpu(buffer) => &**buffer,
|
|
289
|
+
Storage::Cpu(_) => unreachable!("fused step on a CPU tensor"),
|
|
290
|
+
})
|
|
291
|
+
.collect();
|
|
292
|
+
let mut args: Vec<Arg<'_>> = (0..FUSED_SLOTS)
|
|
293
|
+
.map(|i| Arg::Buffer(buffers.get(i).copied().unwrap_or(&self.fused_dummies[i])))
|
|
294
|
+
.collect();
|
|
295
|
+
args.push(Arg::Meta(&group.program));
|
|
296
|
+
args.push(Arg::Params([
|
|
297
|
+
group.n as u32,
|
|
298
|
+
(group.program.len() / 8) as u32,
|
|
299
|
+
0,
|
|
300
|
+
0,
|
|
301
|
+
]));
|
|
302
|
+
self.stage(
|
|
303
|
+
pending,
|
|
304
|
+
"fused",
|
|
305
|
+
&args,
|
|
306
|
+
ComputeEngine::workgroups_1d(group.n),
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
|
|
144
310
|
/// A tensor buffer of `bytes` bytes: a freed one when available, else a
|
|
145
311
|
/// new one made by `create`.
|
|
146
312
|
pub(crate) fn buffer(&self, bytes: u64, create: impl FnOnce() -> Buffer) -> GpuBuffer {
|
|
@@ -173,6 +339,20 @@ impl GpuContext {
|
|
|
173
339
|
kernel: &'static str,
|
|
174
340
|
args: &[Arg<'_>],
|
|
175
341
|
workgroups: (u32, u32, u32),
|
|
342
|
+
) {
|
|
343
|
+
let mut pending = self.lock();
|
|
344
|
+
self.close_group(&mut pending);
|
|
345
|
+
self.stage(&mut pending, kernel, args, workgroups);
|
|
346
|
+
self.count_op(&mut pending);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/// Stage `args` in the batch's argument buffer and queue the dispatch.
|
|
350
|
+
fn stage(
|
|
351
|
+
&self,
|
|
352
|
+
pending: &mut Pending,
|
|
353
|
+
kernel: &'static str,
|
|
354
|
+
args: &[Arg<'_>],
|
|
355
|
+
workgroups: (u32, u32, u32),
|
|
176
356
|
) {
|
|
177
357
|
let padded = |bytes: usize| bytes.div_ceil(self.align) * self.align;
|
|
178
358
|
let needed: usize = args
|
|
@@ -188,9 +368,8 @@ impl GpuContext {
|
|
|
188
368
|
"kernel arguments exceed {ARGS_BYTES} bytes"
|
|
189
369
|
);
|
|
190
370
|
|
|
191
|
-
let mut pending = self.lock();
|
|
192
371
|
if pending.args.len() + needed > ARGS_BYTES {
|
|
193
|
-
self.submit(
|
|
372
|
+
self.submit(pending);
|
|
194
373
|
}
|
|
195
374
|
let staged = &mut pending.args;
|
|
196
375
|
let resources = args
|
|
@@ -216,7 +395,6 @@ impl GpuContext {
|
|
|
216
395
|
.engine
|
|
217
396
|
.dispatch(&self.device, kernel, resources, workgroups);
|
|
218
397
|
pending.dispatches.push(dispatch);
|
|
219
|
-
self.count_op(&mut pending);
|
|
220
398
|
}
|
|
221
399
|
|
|
222
400
|
/// Record a copy of the first `bytes` bytes of `src` into `dst`.
|
|
@@ -243,20 +421,20 @@ impl GpuContext {
|
|
|
243
421
|
}
|
|
244
422
|
|
|
245
423
|
/// Poll the device until a GPU callback sends on `rx`: busy-polling for
|
|
246
|
-
/// up to [`SPIN_WAIT`], then
|
|
424
|
+
/// up to [`SPIN_WAIT`], then sleeping [`POLL_SLEEP`] between polls.
|
|
425
|
+
///
|
|
426
|
+
/// Never blocks in `Maintain::Wait`: wgpu-core 0.19 gives up waiting
|
|
427
|
+
/// there after 5 s and then frees the still-running submission's
|
|
428
|
+
/// resources as if it had finished, which crashes drivers that are still
|
|
429
|
+
/// reading them.
|
|
247
430
|
pub(crate) fn wait_for<T>(&self, rx: &Receiver<T>) -> T {
|
|
248
431
|
let start = Instant::now();
|
|
249
432
|
loop {
|
|
250
|
-
|
|
251
|
-
self.device.poll(if blocking {
|
|
252
|
-
wgpu::Maintain::Wait
|
|
253
|
-
} else {
|
|
254
|
-
wgpu::Maintain::Poll
|
|
255
|
-
});
|
|
433
|
+
self.device.poll(wgpu::Maintain::Poll);
|
|
256
434
|
match rx.try_recv() {
|
|
257
435
|
Ok(value) => return value,
|
|
258
|
-
Err(TryRecvError::Empty) if
|
|
259
|
-
Err(TryRecvError::Empty) =>
|
|
436
|
+
Err(TryRecvError::Empty) if start.elapsed() < SPIN_WAIT => std::thread::yield_now(),
|
|
437
|
+
Err(TryRecvError::Empty) => std::thread::sleep(POLL_SLEEP),
|
|
260
438
|
Err(TryRecvError::Disconnected) => panic!("GPU callback dropped"),
|
|
261
439
|
}
|
|
262
440
|
}
|
|
@@ -269,8 +447,10 @@ impl GpuContext {
|
|
|
269
447
|
}
|
|
270
448
|
}
|
|
271
449
|
|
|
272
|
-
/// The batch's encoder, with the pending dispatches
|
|
450
|
+
/// The batch's encoder, with the pending dispatches (and the open
|
|
451
|
+
/// fusion group) encoded into it.
|
|
273
452
|
fn encoder<'p>(&self, pending: &'p mut Pending) -> &'p mut CommandEncoder {
|
|
453
|
+
self.close_group(pending);
|
|
274
454
|
let encoder = pending.encoder.get_or_insert_with(|| {
|
|
275
455
|
self.device
|
|
276
456
|
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
|
@@ -330,7 +510,9 @@ pub struct Device {
|
|
|
330
510
|
impl Device {
|
|
331
511
|
/// Create a GPU device with the best available backend.
|
|
332
512
|
///
|
|
333
|
-
/// On macOS this is Metal; on Windows/Linux, Vulkan or DX12.
|
|
513
|
+
/// On macOS this is Metal; on Windows/Linux, Vulkan or DX12. Set the
|
|
514
|
+
/// `WGPU_BACKEND` environment variable (e.g. `vulkan`) to restrict the
|
|
515
|
+
/// choice.
|
|
334
516
|
///
|
|
335
517
|
/// # Panics
|
|
336
518
|
/// Panics when no GPU adapter is available. Use [`Device::gpu`] to get an
|
|
@@ -351,8 +533,9 @@ impl Device {
|
|
|
351
533
|
|
|
352
534
|
/// Async version of [`Device::gpu`].
|
|
353
535
|
pub async fn gpu_async() -> Option<Self> {
|
|
536
|
+
// WGPU_BACKEND (e.g. "vulkan", "metal", "dx12", "gl") restricts the backends tried.
|
|
354
537
|
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor {
|
|
355
|
-
backends: wgpu::Backends::all(),
|
|
538
|
+
backends: wgpu::util::backend_bits_from_env().unwrap_or(wgpu::Backends::all()),
|
|
356
539
|
..Default::default()
|
|
357
540
|
});
|
|
358
541
|
let adapter = instance
|
|
@@ -51,27 +51,3 @@ pub(crate) fn with_rng<T>(f: impl FnOnce(&mut dyn RngCore) -> T) -> T {
|
|
|
51
51
|
}
|
|
52
52
|
}
|
|
53
53
|
}
|
|
54
|
-
|
|
55
|
-
#[cfg(test)]
|
|
56
|
-
mod tests {
|
|
57
|
-
use super::*;
|
|
58
|
-
use crate::{Device, Tensor};
|
|
59
|
-
|
|
60
|
-
// One test, so parallel tests can't interleave draws from the shared generator.
|
|
61
|
-
#[test]
|
|
62
|
-
fn manual_seed_makes_every_source_reproducible() {
|
|
63
|
-
let d = Device::cpu();
|
|
64
|
-
let draw = || {
|
|
65
|
-
let r = Tensor::randn(&[8], &d).to_vec();
|
|
66
|
-
let u = Tensor::uniform(&[8], -1.0, 1.0, &d).to_vec();
|
|
67
|
-
let m = crate::functional::dropout(&Tensor::ones(&[64], &d), 0.5, true).to_vec();
|
|
68
|
-
(r, u, m)
|
|
69
|
-
};
|
|
70
|
-
manual_seed(7);
|
|
71
|
-
let first = draw();
|
|
72
|
-
manual_seed(7);
|
|
73
|
-
assert_eq!(first, draw());
|
|
74
|
-
manual_seed(8);
|
|
75
|
-
assert_ne!(first.0, draw().0);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
@@ -97,8 +97,17 @@ impl Tensor {
|
|
|
97
97
|
}
|
|
98
98
|
|
|
99
99
|
pub(crate) fn from_storage(storage: Storage, shape: Vec<usize>, device: &Device) -> Self {
|
|
100
|
+
Self::from_shared_storage(Arc::new(storage), shape, device)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/// A new tensor (no autograd) over storage that is already shared.
|
|
104
|
+
pub(crate) fn from_shared_storage(
|
|
105
|
+
storage: Arc<Storage>,
|
|
106
|
+
shape: Vec<usize>,
|
|
107
|
+
device: &Device,
|
|
108
|
+
) -> Self {
|
|
100
109
|
Self {
|
|
101
|
-
storage
|
|
110
|
+
storage,
|
|
102
111
|
shape,
|
|
103
112
|
device: device.clone(),
|
|
104
113
|
grad: Arc::new(RwLock::new(None)),
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
//! reference kernel in [`super::cpu`]. The public ops in `ops.rs` build on
|
|
6
6
|
//! these and attach backward nodes.
|
|
7
7
|
|
|
8
|
+
use std::sync::Arc;
|
|
8
9
|
use wgpu::util::DeviceExt;
|
|
9
10
|
use wgpu::{Buffer, BufferUsages};
|
|
10
11
|
|
|
@@ -13,7 +14,7 @@ use super::cpu;
|
|
|
13
14
|
use super::shape::{broadcast_shapes, broadcast_strides, contiguous_strides, split_at_dim};
|
|
14
15
|
use super::Tensor;
|
|
15
16
|
use crate::backend::{reduce_op, ComputeEngine};
|
|
16
|
-
use crate::device::{Arg, GpuBuffer, GpuContext};
|
|
17
|
+
use crate::device::{Arg, ElementwiseStep, GpuBuffer, GpuContext};
|
|
17
18
|
|
|
18
19
|
/// A read-only `u32` storage buffer (at least one element).
|
|
19
20
|
fn u32_buffer(ctx: &GpuContext, data: &[u32]) -> Buffer {
|
|
@@ -213,6 +214,19 @@ impl Tensor {
|
|
|
213
214
|
match self.gpu() {
|
|
214
215
|
Some((ctx, input)) if n > 0 => {
|
|
215
216
|
let out = alloc(ctx, n);
|
|
217
|
+
if ctx.fusion() {
|
|
218
|
+
let output = Arc::new(Storage::Gpu(out));
|
|
219
|
+
ctx.elementwise(ElementwiseStep {
|
|
220
|
+
kind: 0,
|
|
221
|
+
op,
|
|
222
|
+
a,
|
|
223
|
+
b,
|
|
224
|
+
inputs: vec![Arc::clone(&self.storage)],
|
|
225
|
+
output: Arc::clone(&output),
|
|
226
|
+
n,
|
|
227
|
+
});
|
|
228
|
+
return Tensor::from_shared_storage(output, self.shape.clone(), &self.device);
|
|
229
|
+
}
|
|
216
230
|
let params = Arg::Params([op, n as u32, a.to_bits(), b.to_bits()]);
|
|
217
231
|
ctx.dispatch(
|
|
218
232
|
"unary",
|
|
@@ -243,6 +257,20 @@ impl Tensor {
|
|
|
243
257
|
match (self.gpu(), other.gpu()) {
|
|
244
258
|
(Some((ctx, a)), Some((_, b))) if n > 0 => {
|
|
245
259
|
let out = alloc(ctx, n);
|
|
260
|
+
// Fusion handles same-shape operands; broadcasting stays unfused.
|
|
261
|
+
if ctx.fusion() && self.shape == other.shape {
|
|
262
|
+
let output = Arc::new(Storage::Gpu(out));
|
|
263
|
+
ctx.elementwise(ElementwiseStep {
|
|
264
|
+
kind: 1,
|
|
265
|
+
op,
|
|
266
|
+
a: 0.0,
|
|
267
|
+
b: 0.0,
|
|
268
|
+
inputs: vec![Arc::clone(&self.storage), Arc::clone(&other.storage)],
|
|
269
|
+
output: Arc::clone(&output),
|
|
270
|
+
n,
|
|
271
|
+
});
|
|
272
|
+
return Tensor::from_shared_storage(output, out_shape, &self.device);
|
|
273
|
+
}
|
|
246
274
|
let params = Arg::Params([op, n as u32, out_shape.len() as u32, 0]);
|
|
247
275
|
let meta = as_u32(&[&out_shape[..], &sa, &sb].concat());
|
|
248
276
|
ctx.dispatch(
|
|
@@ -663,3 +663,133 @@ fn cat_matches_cpu_on_gpu() {
|
|
|
663
663
|
assert_eq!(on(&gpu), on(&cpu), "cat along {dim}");
|
|
664
664
|
}
|
|
665
665
|
}
|
|
666
|
+
|
|
667
|
+
// ============================================================================
|
|
668
|
+
// OP FUSION
|
|
669
|
+
// ============================================================================
|
|
670
|
+
|
|
671
|
+
/// Runs `f` on the GPU with fusion off and then on and checks that the
|
|
672
|
+
/// results are bit-identical.
|
|
673
|
+
fn assert_fusion_matches(name: &str, gpu: &Device, f: impl Fn(&Device) -> Vec<f32>) {
|
|
674
|
+
let ctx = gpu.gpu_ctx().expect("GPU device");
|
|
675
|
+
if !ctx.engine.has("fused") {
|
|
676
|
+
eprintln!("skipping {name}: this GPU cannot bind enough buffers for the fused kernel");
|
|
677
|
+
return;
|
|
678
|
+
}
|
|
679
|
+
let was = ctx.fusion();
|
|
680
|
+
ctx.set_fusion(false);
|
|
681
|
+
let off = f(gpu);
|
|
682
|
+
ctx.set_fusion(true);
|
|
683
|
+
let on = f(gpu);
|
|
684
|
+
ctx.set_fusion(was);
|
|
685
|
+
let bits = |v: &[f32]| v.iter().map(|x| x.to_bits()).collect::<Vec<_>>();
|
|
686
|
+
assert_eq!(off.len(), on.len(), "{name}: lengths differ");
|
|
687
|
+
assert_eq!(
|
|
688
|
+
bits(&on),
|
|
689
|
+
bits(&off),
|
|
690
|
+
"{name}: fused result differs from unfused"
|
|
691
|
+
);
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
fn seeded(shape: &[usize], seed: u64, device: &Device) -> Tensor {
|
|
695
|
+
let mut rng = StdRng::seed_from_u64(seed);
|
|
696
|
+
Tensor::randn_with(shape, &mut rng, device)
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
#[test]
|
|
700
|
+
fn fused_chains_match_unfused_on_gpu() {
|
|
701
|
+
let Some(gpu) = test_gpu() else { return };
|
|
702
|
+
|
|
703
|
+
// The tensor-api benchmark's chain: 100 ops, with x re-read every 4.
|
|
704
|
+
assert_fusion_matches("op chain", &gpu, |d| {
|
|
705
|
+
let x = seeded(&[1000], 1, d);
|
|
706
|
+
let mut y = x.clone();
|
|
707
|
+
for i in 0..25 {
|
|
708
|
+
y = y
|
|
709
|
+
.mul_scalar(1.001)
|
|
710
|
+
.add_scalar(0.01 * i as f32)
|
|
711
|
+
.tanh()
|
|
712
|
+
.add(&x);
|
|
713
|
+
}
|
|
714
|
+
y.to_vec()
|
|
715
|
+
});
|
|
716
|
+
|
|
717
|
+
// Every unary op code that has a public method, chained.
|
|
718
|
+
assert_fusion_matches("unary ops", &gpu, |d| {
|
|
719
|
+
let x = seeded(&[777], 2, d);
|
|
720
|
+
let y = x
|
|
721
|
+
.relu()
|
|
722
|
+
.add(&x.sigmoid())
|
|
723
|
+
.add(&x.tanh())
|
|
724
|
+
.add(&x.silu())
|
|
725
|
+
.add(&x.gelu());
|
|
726
|
+
let z = y
|
|
727
|
+
.abs()
|
|
728
|
+
.add_scalar(0.5)
|
|
729
|
+
.sqrt()
|
|
730
|
+
.log()
|
|
731
|
+
.exp()
|
|
732
|
+
.neg()
|
|
733
|
+
.square()
|
|
734
|
+
.clamp(-2.0, 2.0);
|
|
735
|
+
let w = z
|
|
736
|
+
.mul(&x)
|
|
737
|
+
.sub(&x.sin())
|
|
738
|
+
.add(&x.cos())
|
|
739
|
+
.div(&z.add_scalar(3.0));
|
|
740
|
+
w.maximum(&x).minimum(&y).to_vec()
|
|
741
|
+
});
|
|
742
|
+
|
|
743
|
+
// Groups closed by a matmul, a broadcasting add and a reduction.
|
|
744
|
+
assert_fusion_matches("interrupted", &gpu, |d| {
|
|
745
|
+
let a = seeded(&[16, 32], 3, d);
|
|
746
|
+
let w = seeded(&[32, 8], 4, d);
|
|
747
|
+
let bias = seeded(&[8], 5, d);
|
|
748
|
+
let h = a.mul_scalar(0.5).tanh().matmul(&w);
|
|
749
|
+
let h = h.add(&bias).relu().mul_scalar(2.0);
|
|
750
|
+
let s = h.sum();
|
|
751
|
+
let mut out = h.add_scalar(1.0).to_vec();
|
|
752
|
+
out.push(s.item());
|
|
753
|
+
out
|
|
754
|
+
});
|
|
755
|
+
|
|
756
|
+
// The same buffer on both sides, and a chain longer than a batch.
|
|
757
|
+
assert_fusion_matches("aliasing and length", &gpu, |d| {
|
|
758
|
+
let x = seeded(&[300], 6, d);
|
|
759
|
+
let mut y = x.add(&x).mul(&x);
|
|
760
|
+
for i in 0..300 {
|
|
761
|
+
y = if i % 2 == 0 {
|
|
762
|
+
y.mul_scalar(0.999)
|
|
763
|
+
} else {
|
|
764
|
+
y.add_scalar(0.001)
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
y.to_vec()
|
|
768
|
+
});
|
|
769
|
+
|
|
770
|
+
// Large enough to need the second grid dimension.
|
|
771
|
+
assert_fusion_matches("large", &gpu, |d| {
|
|
772
|
+
let x = seeded(&[70_000], 7, d);
|
|
773
|
+
x.mul_scalar(3.0).add_scalar(-1.0).tanh().mul(&x).to_vec()
|
|
774
|
+
});
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
#[test]
|
|
778
|
+
fn fused_gradients_match_unfused_on_gpu() {
|
|
779
|
+
let Some(gpu) = test_gpu() else { return };
|
|
780
|
+
assert_fusion_matches("gradients", &gpu, |d| {
|
|
781
|
+
let w = seeded(&[64], 8, d).requires_grad_();
|
|
782
|
+
let x = seeded(&[64], 9, d);
|
|
783
|
+
let loss = w
|
|
784
|
+
.mul(&x)
|
|
785
|
+
.tanh()
|
|
786
|
+
.mul_scalar(2.0)
|
|
787
|
+
.add(&w.sigmoid())
|
|
788
|
+
.square()
|
|
789
|
+
.sum();
|
|
790
|
+
loss.backward();
|
|
791
|
+
let mut out = w.grad().expect("gradient").to_vec();
|
|
792
|
+
out.push(loss.item());
|
|
793
|
+
out
|
|
794
|
+
});
|
|
795
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
//! `manual_seed` reproducibility.
|
|
2
|
+
//!
|
|
3
|
+
//! The seeded generator is process-wide, so this lives in its own test binary:
|
|
4
|
+
//! inside the library's unit tests, other tests running in parallel draw from
|
|
5
|
+
//! it between the seed and the draws.
|
|
6
|
+
|
|
7
|
+
use ferrotensor::{functional, manual_seed, Device, Tensor};
|
|
8
|
+
|
|
9
|
+
#[test]
|
|
10
|
+
fn manual_seed_makes_every_source_reproducible() {
|
|
11
|
+
let d = Device::cpu();
|
|
12
|
+
let draw = || {
|
|
13
|
+
let r = Tensor::randn(&[8], &d).to_vec();
|
|
14
|
+
let u = Tensor::uniform(&[8], -1.0, 1.0, &d).to_vec();
|
|
15
|
+
let m = functional::dropout(&Tensor::ones(&[64], &d), 0.5, true).to_vec();
|
|
16
|
+
(r, u, m)
|
|
17
|
+
};
|
|
18
|
+
manual_seed(7);
|
|
19
|
+
let first = draw();
|
|
20
|
+
manual_seed(7);
|
|
21
|
+
assert_eq!(first, draw());
|
|
22
|
+
manual_seed(8);
|
|
23
|
+
assert_ne!(first.0, draw().0);
|
|
24
|
+
}
|
|
@@ -9,7 +9,7 @@ description = "PyTorch-style tensors, autograd and nn on wgpu (Metal, Vulkan, DX
|
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
11
11
|
license = "MIT OR Apache-2.0"
|
|
12
|
-
authors = [{ name = "Yashaswin
|
|
12
|
+
authors = [{ name = "Puranik Yashaswin Sharma", email = "puranikyashaswinsharma@gmail.com" }]
|
|
13
13
|
keywords = ["machine-learning", "deep-learning", "gpu", "autograd", "wgpu"]
|
|
14
14
|
classifiers = [
|
|
15
15
|
"Programming Language :: Rust",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|