ferrotensor 0.3.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/Cargo.lock +11 -23
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/Cargo.toml +1 -1
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/PKG-INFO +1 -1
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/Cargo.toml +1 -1
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/backend/compute.rs +20 -1
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/backend/kernels.rs +338 -27
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/backend/mod.rs +1 -1
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/device.rs +202 -19
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/functional.rs +107 -12
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/random.rs +0 -24
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/tensor/core.rs +10 -1
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/tensor/cpu.rs +96 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/tensor/mod.rs +1 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/tensor/ops.rs +23 -49
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/tensor/raw.rs +274 -1
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/tensor/tests.rs +230 -0
- ferrotensor-0.5.0/ferrotensor/tests/manual_seed.rs +24 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/README.md +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/README.md +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/examples/README.md +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/examples/char_transformer.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/examples/data/sonnets.txt +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/examples/gpu_matmul_attention.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/examples/mlp_two_moons.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/examples/quickstart.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/examples/similarity_search.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/data.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/lib.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/nn/activation.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/nn/attention.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/nn/container.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/nn/dropout.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/nn/embedding.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/nn/linear.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/nn/mod.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/nn/module.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/nn/normalization.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/nn/tests.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/optim/adam.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/optim/mod.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/optim/optimizer.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/optim/scheduler.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/optim/sgd.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/optim/tests.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/serialize.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/tensor/autograd.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor/src/tensor/shape.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/.gitignore +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/Cargo.toml +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/README.md +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/build.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/examples/mlp.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/src/device.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/src/error.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/src/functional.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/src/lib.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/src/nn.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/src/optim.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/src/serialize.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/src/tensor.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/tests/conftest.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/tests/test_autograd.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/tests/test_errors.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/tests/test_functional.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/tests/test_gpu.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/tests/test_nn.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/tests/test_optim.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/tests/test_serialize.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/tests/test_tensor.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/ferrotensor-py/tests/test_train.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/pyproject.toml +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/python/ferrotensor/__init__.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/python/ferrotensor/_ferrotensor.pyi +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/python/ferrotensor/nn/__init__.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/python/ferrotensor/nn/functional.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/python/ferrotensor/optim.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.5.0}/python/ferrotensor/py.typed +0 -0
|
@@ -606,7 +606,7 @@ dependencies = [
|
|
|
606
606
|
|
|
607
607
|
[[package]]
|
|
608
608
|
name = "ferrotensor"
|
|
609
|
-
version = "0.
|
|
609
|
+
version = "0.5.0"
|
|
610
610
|
dependencies = [
|
|
611
611
|
"bytemuck",
|
|
612
612
|
"naga",
|
|
@@ -619,7 +619,7 @@ dependencies = [
|
|
|
619
619
|
|
|
620
620
|
[[package]]
|
|
621
621
|
name = "ferrotensor-autograd"
|
|
622
|
-
version = "0.
|
|
622
|
+
version = "0.5.0"
|
|
623
623
|
dependencies = [
|
|
624
624
|
"ferrotensor-backend",
|
|
625
625
|
"pollster",
|
|
@@ -628,11 +628,10 @@ dependencies = [
|
|
|
628
628
|
|
|
629
629
|
[[package]]
|
|
630
630
|
name = "ferrotensor-backend"
|
|
631
|
-
version = "0.
|
|
631
|
+
version = "0.5.0"
|
|
632
632
|
dependencies = [
|
|
633
633
|
"bytemuck",
|
|
634
634
|
"env_logger",
|
|
635
|
-
"futures-intrusive",
|
|
636
635
|
"log",
|
|
637
636
|
"naga",
|
|
638
637
|
"pollster",
|
|
@@ -641,7 +640,7 @@ dependencies = [
|
|
|
641
640
|
|
|
642
641
|
[[package]]
|
|
643
642
|
name = "ferrotensor-benchmarks"
|
|
644
|
-
version = "0.
|
|
643
|
+
version = "0.5.0"
|
|
645
644
|
dependencies = [
|
|
646
645
|
"ferrotensor",
|
|
647
646
|
"ferrotensor-backend",
|
|
@@ -652,7 +651,7 @@ dependencies = [
|
|
|
652
651
|
|
|
653
652
|
[[package]]
|
|
654
653
|
name = "ferrotensor-cli"
|
|
655
|
-
version = "0.
|
|
654
|
+
version = "0.5.0"
|
|
656
655
|
dependencies = [
|
|
657
656
|
"clap",
|
|
658
657
|
"env_logger",
|
|
@@ -670,7 +669,7 @@ dependencies = [
|
|
|
670
669
|
|
|
671
670
|
[[package]]
|
|
672
671
|
name = "ferrotensor-graph"
|
|
673
|
-
version = "0.
|
|
672
|
+
version = "0.5.0"
|
|
674
673
|
dependencies = [
|
|
675
674
|
"bytemuck",
|
|
676
675
|
"env_logger",
|
|
@@ -685,7 +684,7 @@ dependencies = [
|
|
|
685
684
|
|
|
686
685
|
[[package]]
|
|
687
686
|
name = "ferrotensor-hub"
|
|
688
|
-
version = "0.
|
|
687
|
+
version = "0.5.0"
|
|
689
688
|
dependencies = [
|
|
690
689
|
"ferrotensor-autograd",
|
|
691
690
|
"ferrotensor-backend",
|
|
@@ -698,7 +697,7 @@ dependencies = [
|
|
|
698
697
|
|
|
699
698
|
[[package]]
|
|
700
699
|
name = "ferrotensor-loader"
|
|
701
|
-
version = "0.
|
|
700
|
+
version = "0.5.0"
|
|
702
701
|
dependencies = [
|
|
703
702
|
"bytemuck",
|
|
704
703
|
"env_logger",
|
|
@@ -716,7 +715,7 @@ dependencies = [
|
|
|
716
715
|
|
|
717
716
|
[[package]]
|
|
718
717
|
name = "ferrotensor-monitor"
|
|
719
|
-
version = "0.
|
|
718
|
+
version = "0.5.0"
|
|
720
719
|
dependencies = [
|
|
721
720
|
"anyhow",
|
|
722
721
|
"chrono",
|
|
@@ -730,7 +729,7 @@ dependencies = [
|
|
|
730
729
|
|
|
731
730
|
[[package]]
|
|
732
731
|
name = "ferrotensor-py"
|
|
733
|
-
version = "0.
|
|
732
|
+
version = "0.5.0"
|
|
734
733
|
dependencies = [
|
|
735
734
|
"ferrotensor",
|
|
736
735
|
"pyo3",
|
|
@@ -739,7 +738,7 @@ dependencies = [
|
|
|
739
738
|
|
|
740
739
|
[[package]]
|
|
741
740
|
name = "ferrotensor-trainer"
|
|
742
|
-
version = "0.
|
|
741
|
+
version = "0.5.0"
|
|
743
742
|
dependencies = [
|
|
744
743
|
"bytemuck",
|
|
745
744
|
"ferrotensor-autograd",
|
|
@@ -805,17 +804,6 @@ version = "0.3.31"
|
|
|
805
804
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
806
805
|
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
|
|
807
806
|
|
|
808
|
-
[[package]]
|
|
809
|
-
name = "futures-intrusive"
|
|
810
|
-
version = "0.5.0"
|
|
811
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
812
|
-
checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f"
|
|
813
|
-
dependencies = [
|
|
814
|
-
"futures-core",
|
|
815
|
-
"lock_api",
|
|
816
|
-
"parking_lot",
|
|
817
|
-
]
|
|
818
|
-
|
|
819
807
|
[[package]]
|
|
820
808
|
name = "futures-task"
|
|
821
809
|
version = "0.3.31"
|
|
@@ -9,7 +9,7 @@ use wgpu::{BindGroup, BindGroupLayout, BindingResource, CommandEncoder, ComputeP
|
|
|
9
9
|
use super::KERNELS_WGSL;
|
|
10
10
|
|
|
11
11
|
/// Entry points compiled from [`KERNELS_WGSL`].
|
|
12
|
-
pub(crate) const KERNEL_NAMES: [&str;
|
|
12
|
+
pub(crate) const KERNEL_NAMES: [&str; 15] = [
|
|
13
13
|
"unary",
|
|
14
14
|
"binary",
|
|
15
15
|
"reduce",
|
|
@@ -19,8 +19,19 @@ pub(crate) const KERNEL_NAMES: [&str; 9] = [
|
|
|
19
19
|
"matmul",
|
|
20
20
|
"permute",
|
|
21
21
|
"gather_rows",
|
|
22
|
+
"sort_ids",
|
|
23
|
+
"scatter_rows",
|
|
24
|
+
"copy_block",
|
|
25
|
+
"cross_entropy_rows",
|
|
26
|
+
"cross_entropy_grad",
|
|
27
|
+
"fused",
|
|
22
28
|
];
|
|
23
29
|
|
|
30
|
+
/// Storage buffers the `fused` kernel binds: seven tensor slots and its
|
|
31
|
+
/// program. Devices with fewer per-stage bindings do not compile it, and
|
|
32
|
+
/// element-wise ops then run unfused.
|
|
33
|
+
pub(crate) const FUSED_STORAGE_BINDINGS: u32 = 8;
|
|
34
|
+
|
|
24
35
|
/// Threads per workgroup of the 1-D kernels.
|
|
25
36
|
const WORKGROUP_1D: u32 = 256;
|
|
26
37
|
/// Largest workgroup count allowed in one grid dimension.
|
|
@@ -47,8 +58,11 @@ impl ComputeEngine {
|
|
|
47
58
|
source: wgpu::ShaderSource::Wgsl(Cow::Borrowed(KERNELS_WGSL)),
|
|
48
59
|
});
|
|
49
60
|
|
|
61
|
+
let fused_fits =
|
|
62
|
+
device.limits().max_storage_buffers_per_shader_stage >= FUSED_STORAGE_BINDINGS;
|
|
50
63
|
let pipelines = KERNEL_NAMES
|
|
51
64
|
.iter()
|
|
65
|
+
.filter(|&&name| name != "fused" || fused_fits)
|
|
52
66
|
.map(|&name| {
|
|
53
67
|
let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
|
|
54
68
|
label: Some(name),
|
|
@@ -64,6 +78,11 @@ impl ComputeEngine {
|
|
|
64
78
|
Self { pipelines }
|
|
65
79
|
}
|
|
66
80
|
|
|
81
|
+
/// Whether `name` was compiled for this device.
|
|
82
|
+
pub(crate) fn has(&self, name: &str) -> bool {
|
|
83
|
+
self.pipelines.contains_key(name)
|
|
84
|
+
}
|
|
85
|
+
|
|
67
86
|
/// Get a compiled pipeline by entry point name.
|
|
68
87
|
pub fn get_pipeline(&self, name: &str) -> Option<&ComputePipeline> {
|
|
69
88
|
self.pipelines.get(name).map(|(pipeline, _)| pipeline)
|
|
@@ -100,16 +100,9 @@ struct UnaryParams {
|
|
|
100
100
|
@group(0) @binding(1) var<storage, read_write> un_out: array<f32>;
|
|
101
101
|
@group(0) @binding(2) var<uniform> un_p: UnaryParams;
|
|
102
102
|
|
|
103
|
-
|
|
104
|
-
fn unary(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
105
|
-
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
106
|
-
let i = flat_index(gid, nwg);
|
|
107
|
-
if (i >= un_p.n) { return; }
|
|
108
|
-
let x = un_in[i];
|
|
109
|
-
let a = un_p.a;
|
|
110
|
-
let b = un_p.b;
|
|
103
|
+
fn unary_apply(op: u32, x: f32, a: f32, b: f32) -> f32 {
|
|
111
104
|
var y = x;
|
|
112
|
-
switch
|
|
105
|
+
switch op {
|
|
113
106
|
case 0u: { y = max(x, 0.0); }
|
|
114
107
|
case 1u: { y = x * sigmoid_f(x); }
|
|
115
108
|
case 2u: { y = gelu_tanh(x); }
|
|
@@ -152,7 +145,15 @@ fn unary(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
|
152
145
|
case 25u: { y = select(a, 1.0, x > 0.0); }
|
|
153
146
|
default: { y = x; }
|
|
154
147
|
}
|
|
155
|
-
|
|
148
|
+
return y;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
@compute @workgroup_size(256)
|
|
152
|
+
fn unary(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
153
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
154
|
+
let i = flat_index(gid, nwg);
|
|
155
|
+
if (i >= un_p.n) { return; }
|
|
156
|
+
un_out[i] = unary_apply(un_p.op, un_in[i], un_p.a, un_p.b);
|
|
156
157
|
}
|
|
157
158
|
|
|
158
159
|
// --- Broadcasting binary ops ------------------------------------------------
|
|
@@ -172,6 +173,25 @@ struct BinaryParams {
|
|
|
172
173
|
@group(0) @binding(3) var<uniform> bin_p: BinaryParams;
|
|
173
174
|
@group(0) @binding(4) var<storage, read> bin_meta: array<u32>;
|
|
174
175
|
|
|
176
|
+
fn binary_apply(op: u32, x: f32, y: f32) -> f32 {
|
|
177
|
+
var r = 0.0;
|
|
178
|
+
switch op {
|
|
179
|
+
case 0u: { r = x + y; }
|
|
180
|
+
case 1u: { r = x - y; }
|
|
181
|
+
case 2u: { r = x * y; }
|
|
182
|
+
case 3u: { r = x / y; }
|
|
183
|
+
case 4u: { r = max(x, y); }
|
|
184
|
+
case 5u: { r = min(x, y); }
|
|
185
|
+
case 6u: { r = select(0.0, 1.0, x > y); }
|
|
186
|
+
case 7u: { r = select(0.0, 1.0, x < y); }
|
|
187
|
+
case 8u: { r = select(0.0, 1.0, x == y); }
|
|
188
|
+
case 9u: { r = select(0.0, 1.0, x >= y); }
|
|
189
|
+
case 10u: { r = select(0.0, 1.0, x <= y); }
|
|
190
|
+
default: { r = x; }
|
|
191
|
+
}
|
|
192
|
+
return r;
|
|
193
|
+
}
|
|
194
|
+
|
|
175
195
|
@compute @workgroup_size(256)
|
|
176
196
|
fn binary(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
177
197
|
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
@@ -189,24 +209,78 @@ fn binary(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
|
189
209
|
ia = ia + idx * bin_meta[nd + axis];
|
|
190
210
|
ib = ib + idx * bin_meta[2u * nd + axis];
|
|
191
211
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
212
|
+
bin_out[i] = binary_apply(bin_p.op, bin_a[ia], bin_b[ib]);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// --- Fused element-wise chains ---------------------------------------------
|
|
216
|
+
// A chain of same-size unary and non-broadcasting binary ops in one dispatch
|
|
217
|
+
// (FERROTENSOR_FUSION=1, see docs/design/op-fusion.md). Each step is 8 words:
|
|
218
|
+
// [kind, op, src0, src1, dst, a bits, b bits, 0]; kind 0 applies a unary op
|
|
219
|
+
// to src0, kind 1 a binary op to src0 and src1. Sources and destinations are
|
|
220
|
+
// binding slots 0..6. Every step stores its result and later steps load it
|
|
221
|
+
// back, as separate dispatches would.
|
|
222
|
+
|
|
223
|
+
struct FusedParams {
|
|
224
|
+
n: u32,
|
|
225
|
+
steps: u32,
|
|
226
|
+
_pad0: u32,
|
|
227
|
+
_pad1: u32,
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
@group(0) @binding(0) var<storage, read_write> fb0: array<f32>;
|
|
231
|
+
@group(0) @binding(1) var<storage, read_write> fb1: array<f32>;
|
|
232
|
+
@group(0) @binding(2) var<storage, read_write> fb2: array<f32>;
|
|
233
|
+
@group(0) @binding(3) var<storage, read_write> fb3: array<f32>;
|
|
234
|
+
@group(0) @binding(4) var<storage, read_write> fb4: array<f32>;
|
|
235
|
+
@group(0) @binding(5) var<storage, read_write> fb5: array<f32>;
|
|
236
|
+
@group(0) @binding(6) var<storage, read_write> fb6: array<f32>;
|
|
237
|
+
@group(0) @binding(7) var<storage, read> fused_prog: array<u32>;
|
|
238
|
+
@group(0) @binding(8) var<uniform> fused_p: FusedParams;
|
|
239
|
+
|
|
240
|
+
fn fused_load(slot: u32, i: u32) -> f32 {
|
|
241
|
+
var v = 0.0;
|
|
242
|
+
switch slot {
|
|
243
|
+
case 0u: { v = fb0[i]; }
|
|
244
|
+
case 1u: { v = fb1[i]; }
|
|
245
|
+
case 2u: { v = fb2[i]; }
|
|
246
|
+
case 3u: { v = fb3[i]; }
|
|
247
|
+
case 4u: { v = fb4[i]; }
|
|
248
|
+
case 5u: { v = fb5[i]; }
|
|
249
|
+
default: { v = fb6[i]; }
|
|
250
|
+
}
|
|
251
|
+
return v;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
fn fused_store(slot: u32, i: u32, v: f32) {
|
|
255
|
+
switch slot {
|
|
256
|
+
case 0u: { fb0[i] = v; }
|
|
257
|
+
case 1u: { fb1[i] = v; }
|
|
258
|
+
case 2u: { fb2[i] = v; }
|
|
259
|
+
case 3u: { fb3[i] = v; }
|
|
260
|
+
case 4u: { fb4[i] = v; }
|
|
261
|
+
case 5u: { fb5[i] = v; }
|
|
262
|
+
default: { fb6[i] = v; }
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
@compute @workgroup_size(256)
|
|
267
|
+
fn fused(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
268
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
269
|
+
let i = flat_index(gid, nwg);
|
|
270
|
+
if (i >= fused_p.n) { return; }
|
|
271
|
+
for (var k = 0u; k < fused_p.steps; k = k + 1u) {
|
|
272
|
+
let w = k * 8u;
|
|
273
|
+
let x = fused_load(fused_prog[w + 2u], i);
|
|
274
|
+
var r = 0.0;
|
|
275
|
+
if (fused_prog[w] == 0u) {
|
|
276
|
+
let a = bitcast<f32>(fused_prog[w + 5u]);
|
|
277
|
+
let b = bitcast<f32>(fused_prog[w + 6u]);
|
|
278
|
+
r = unary_apply(fused_prog[w + 1u], x, a, b);
|
|
279
|
+
} else {
|
|
280
|
+
r = binary_apply(fused_prog[w + 1u], x, fused_load(fused_prog[w + 3u], i));
|
|
281
|
+
}
|
|
282
|
+
fused_store(fused_prog[w + 4u], i, r);
|
|
208
283
|
}
|
|
209
|
-
bin_out[i] = r;
|
|
210
284
|
}
|
|
211
285
|
|
|
212
286
|
// --- Axis reductions --------------------------------------------------------
|
|
@@ -547,4 +621,241 @@ fn gather_rows(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
|
547
621
|
}
|
|
548
622
|
gat_out[i] = gat_table[id * gat_p.row + c];
|
|
549
623
|
}
|
|
624
|
+
|
|
625
|
+
// --- Row scatter-add (index_select and embedding backward) -------------------
|
|
626
|
+
// WGSL has no float atomics, so the gradient rows are not added with atomics.
|
|
627
|
+
// Instead the (row, position) pairs of the ids are sorted by row, then by
|
|
628
|
+
// position; head[row] is the index of the row's first pair, or SORT_NONE when
|
|
629
|
+
// no id picks it. One thread per output element adds its row's gradients in
|
|
630
|
+
// position order, as the CPU loop does, so the result is deterministic and
|
|
631
|
+
// matches the CPU bit for bit.
|
|
632
|
+
|
|
633
|
+
const SORT_NONE: u32 = 0xffffffffu;
|
|
634
|
+
|
|
635
|
+
// sort_ids builds the pairs and the head table on the GPU from f32 ids.
|
|
636
|
+
// op 0: pair i = (row of id i, i) for i < n, padded with (SORT_NONE, i) up to
|
|
637
|
+
// a = the power-of-two pair count; b = rows. An id outside [0, rows)
|
|
638
|
+
// gets row SORT_NONE: it sorts last and adds nothing.
|
|
639
|
+
// op 1: one compare-and-swap step of a bitonic sort of n pairs (a = k, b = j).
|
|
640
|
+
// op 2: head[r] = SORT_NONE for r < n.
|
|
641
|
+
// op 3: head[row] = i for each pair i < n that is the first of its row.
|
|
642
|
+
|
|
643
|
+
struct SortParams {
|
|
644
|
+
op: u32,
|
|
645
|
+
n: u32,
|
|
646
|
+
a: u32,
|
|
647
|
+
b: u32,
|
|
648
|
+
};
|
|
649
|
+
|
|
650
|
+
@group(0) @binding(0) var<storage, read> srt_ids: array<f32>;
|
|
651
|
+
@group(0) @binding(1) var<storage, read_write> srt_keys: array<u32>;
|
|
652
|
+
@group(0) @binding(2) var<storage, read_write> srt_head: array<u32>;
|
|
653
|
+
@group(0) @binding(3) var<uniform> srt_p: SortParams;
|
|
654
|
+
|
|
655
|
+
@compute @workgroup_size(256)
|
|
656
|
+
fn sort_ids(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
657
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
658
|
+
let i = flat_index(gid, nwg);
|
|
659
|
+
switch srt_p.op {
|
|
660
|
+
case 0u: {
|
|
661
|
+
if (i >= srt_p.a) { return; }
|
|
662
|
+
var row = SORT_NONE;
|
|
663
|
+
if (i < srt_p.n) {
|
|
664
|
+
let f = srt_ids[i];
|
|
665
|
+
if (f >= 0.0 && f < f32(srt_p.b)) { row = u32(f); }
|
|
666
|
+
}
|
|
667
|
+
srt_keys[2u * i] = row;
|
|
668
|
+
srt_keys[2u * i + 1u] = i;
|
|
669
|
+
}
|
|
670
|
+
case 1u: {
|
|
671
|
+
let l = i ^ srt_p.b;
|
|
672
|
+
if (i >= srt_p.n || l <= i) { return; }
|
|
673
|
+
let ri = srt_keys[2u * i];
|
|
674
|
+
let pi = srt_keys[2u * i + 1u];
|
|
675
|
+
let rl = srt_keys[2u * l];
|
|
676
|
+
let pl = srt_keys[2u * l + 1u];
|
|
677
|
+
let greater = ri > rl || (ri == rl && pi > pl);
|
|
678
|
+
if (greater == ((i & srt_p.a) == 0u)) {
|
|
679
|
+
srt_keys[2u * i] = rl;
|
|
680
|
+
srt_keys[2u * i + 1u] = pl;
|
|
681
|
+
srt_keys[2u * l] = ri;
|
|
682
|
+
srt_keys[2u * l + 1u] = pi;
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
case 2u: {
|
|
686
|
+
if (i >= srt_p.n) { return; }
|
|
687
|
+
srt_head[i] = SORT_NONE;
|
|
688
|
+
}
|
|
689
|
+
default: {
|
|
690
|
+
if (i >= srt_p.n) { return; }
|
|
691
|
+
let row = srt_keys[2u * i];
|
|
692
|
+
if (row == SORT_NONE) { return; }
|
|
693
|
+
if (i == 0u || srt_keys[2u * i - 2u] != row) {
|
|
694
|
+
srt_head[row] = i;
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// The gradient is viewed as [outer, n, row] and the output as
|
|
701
|
+
// [outer, rows, row].
|
|
702
|
+
struct ScatterParams {
|
|
703
|
+
rows: u32,
|
|
704
|
+
row: u32,
|
|
705
|
+
n: u32,
|
|
706
|
+
outer: u32,
|
|
707
|
+
};
|
|
708
|
+
|
|
709
|
+
@group(0) @binding(0) var<storage, read> sct_grad: array<f32>;
|
|
710
|
+
@group(0) @binding(1) var<storage, read> sct_keys: array<u32>;
|
|
711
|
+
@group(0) @binding(2) var<storage, read> sct_head: array<u32>;
|
|
712
|
+
@group(0) @binding(3) var<storage, read_write> sct_out: array<f32>;
|
|
713
|
+
@group(0) @binding(4) var<uniform> sct_p: ScatterParams;
|
|
714
|
+
|
|
715
|
+
@compute @workgroup_size(256)
|
|
716
|
+
fn scatter_rows(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
717
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
718
|
+
let i = flat_index(gid, nwg);
|
|
719
|
+
if (i >= sct_p.outer * sct_p.rows * sct_p.row) { return; }
|
|
720
|
+
let o = i / (sct_p.rows * sct_p.row);
|
|
721
|
+
let r = (i / sct_p.row) % sct_p.rows;
|
|
722
|
+
let c = i % sct_p.row;
|
|
723
|
+
let base = o * sct_p.n * sct_p.row + c;
|
|
724
|
+
var acc = 0.0;
|
|
725
|
+
var t = sct_head[r];
|
|
726
|
+
if (t != SORT_NONE) {
|
|
727
|
+
loop {
|
|
728
|
+
if (t >= sct_p.n || sct_keys[2u * t] != r) { break; }
|
|
729
|
+
acc = acc + sct_grad[base + sct_keys[2u * t + 1u] * sct_p.row];
|
|
730
|
+
t = t + 1u;
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
sct_out[i] = acc;
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
// --- Block copy (cat) --------------------------------------------------------
|
|
737
|
+
// Copies an input viewed as [outer, block_in] into columns
|
|
738
|
+
// [offset, offset + block_in) of an output viewed as [outer, block_out].
|
|
739
|
+
|
|
740
|
+
struct CopyParams {
|
|
741
|
+
n: u32,
|
|
742
|
+
block_in: u32,
|
|
743
|
+
block_out: u32,
|
|
744
|
+
offset: u32,
|
|
745
|
+
};
|
|
746
|
+
|
|
747
|
+
@group(0) @binding(0) var<storage, read> cpy_in: array<f32>;
|
|
748
|
+
@group(0) @binding(1) var<storage, read_write> cpy_out: array<f32>;
|
|
749
|
+
@group(0) @binding(2) var<uniform> cpy_p: CopyParams;
|
|
750
|
+
|
|
751
|
+
@compute @workgroup_size(256)
|
|
752
|
+
fn copy_block(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
753
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
754
|
+
let i = flat_index(gid, nwg);
|
|
755
|
+
if (i >= cpy_p.n) { return; }
|
|
756
|
+
let o = i / cpy_p.block_in;
|
|
757
|
+
cpy_out[o * cpy_p.block_out + cpy_p.offset + i % cpy_p.block_in] = cpy_in[i];
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
// --- Cross-entropy with class-id targets ---------------------------------------
|
|
761
|
+
// Logits are viewed as [rows, classes]; targets hold one f32 class id per row.
|
|
762
|
+
// A target <= -1 (an id that is negative as an integer, such as -100) skips
|
|
763
|
+
// its row: loss 0, valid 0, gradient 0. Otherwise the id is truncated to an
|
|
764
|
+
// integer; one that is not below `classes` (or NaN) gives a NaN loss.
|
|
765
|
+
// cross_entropy_rows: one workgroup per row writes the row's loss
|
|
766
|
+
// (lse - logit[target]), its log-sum-exp and whether it counts.
|
|
767
|
+
// cross_entropy_grad: d loss[r] / d logit[r, c] = softmax[r, c] - [c == target],
|
|
768
|
+
// times the incoming gradient of the row's loss.
|
|
769
|
+
|
|
770
|
+
struct CeParams {
|
|
771
|
+
rows: u32,
|
|
772
|
+
classes: u32,
|
|
773
|
+
_pad0: u32,
|
|
774
|
+
_pad1: u32,
|
|
775
|
+
};
|
|
776
|
+
|
|
777
|
+
@group(0) @binding(0) var<storage, read> ce_logits: array<f32>;
|
|
778
|
+
@group(0) @binding(1) var<storage, read> ce_targets: array<f32>;
|
|
779
|
+
@group(0) @binding(2) var<storage, read_write> ce_loss: array<f32>;
|
|
780
|
+
@group(0) @binding(3) var<storage, read_write> ce_lse: array<f32>;
|
|
781
|
+
@group(0) @binding(4) var<storage, read_write> ce_valid: array<f32>;
|
|
782
|
+
@group(0) @binding(5) var<uniform> ce_p: CeParams;
|
|
783
|
+
|
|
784
|
+
var<workgroup> ce_part: array<f32, 256>;
|
|
785
|
+
|
|
786
|
+
@compute @workgroup_size(256)
|
|
787
|
+
fn cross_entropy_rows(@builtin(local_invocation_index) lid: u32,
|
|
788
|
+
@builtin(workgroup_id) wid: vec3<u32>,
|
|
789
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
790
|
+
let r = wid.x + wid.y * nwg.x;
|
|
791
|
+
// Uniform per workgroup, so every thread reaches the barriers or none.
|
|
792
|
+
if (r >= ce_p.rows) { return; }
|
|
793
|
+
let c = ce_p.classes;
|
|
794
|
+
let base = r * c;
|
|
795
|
+
var m = bitcast<f32>(0xff800000u);
|
|
796
|
+
for (var j = lid; j < c; j = j + 256u) {
|
|
797
|
+
m = max(m, ce_logits[base + j]);
|
|
798
|
+
}
|
|
799
|
+
ce_part[lid] = m;
|
|
800
|
+
workgroupBarrier();
|
|
801
|
+
for (var half = 128u; half > 0u; half = half >> 1u) {
|
|
802
|
+
if (lid < half) {
|
|
803
|
+
ce_part[lid] = max(ce_part[lid], ce_part[lid + half]);
|
|
804
|
+
}
|
|
805
|
+
workgroupBarrier();
|
|
806
|
+
}
|
|
807
|
+
m = ce_part[0];
|
|
808
|
+
workgroupBarrier();
|
|
809
|
+
var s = 0.0;
|
|
810
|
+
for (var j = lid; j < c; j = j + 256u) {
|
|
811
|
+
s = s + exp(ce_logits[base + j] - m);
|
|
812
|
+
}
|
|
813
|
+
ce_part[lid] = s;
|
|
814
|
+
workgroupBarrier();
|
|
815
|
+
for (var half = 128u; half > 0u; half = half >> 1u) {
|
|
816
|
+
if (lid < half) {
|
|
817
|
+
ce_part[lid] = ce_part[lid] + ce_part[lid + half];
|
|
818
|
+
}
|
|
819
|
+
workgroupBarrier();
|
|
820
|
+
}
|
|
821
|
+
if (lid == 0u) {
|
|
822
|
+
let lse = m + log(ce_part[0]);
|
|
823
|
+
let f = ce_targets[r];
|
|
824
|
+
ce_lse[r] = lse;
|
|
825
|
+
if (f <= -1.0) {
|
|
826
|
+
ce_loss[r] = 0.0;
|
|
827
|
+
ce_valid[r] = 0.0;
|
|
828
|
+
} else {
|
|
829
|
+
ce_valid[r] = 1.0;
|
|
830
|
+
if (f < f32(c)) {
|
|
831
|
+
ce_loss[r] = lse - ce_logits[base + u32(max(f, 0.0))];
|
|
832
|
+
} else {
|
|
833
|
+
ce_loss[r] = bitcast<f32>(0x7fc00000u);
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
@group(0) @binding(0) var<storage, read> ceg_logits: array<f32>;
|
|
840
|
+
@group(0) @binding(1) var<storage, read> ceg_targets: array<f32>;
|
|
841
|
+
@group(0) @binding(2) var<storage, read> ceg_lse: array<f32>;
|
|
842
|
+
@group(0) @binding(3) var<storage, read> ceg_grad: array<f32>;
|
|
843
|
+
@group(0) @binding(4) var<storage, read_write> ceg_out: array<f32>;
|
|
844
|
+
@group(0) @binding(5) var<uniform> ceg_p: CeParams;
|
|
845
|
+
|
|
846
|
+
@compute @workgroup_size(256)
|
|
847
|
+
fn cross_entropy_grad(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
848
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
849
|
+
let i = flat_index(gid, nwg);
|
|
850
|
+
if (i >= ceg_p.rows * ceg_p.classes) { return; }
|
|
851
|
+
let r = i / ceg_p.classes;
|
|
852
|
+
let f = ceg_targets[r];
|
|
853
|
+
if (f <= -1.0) {
|
|
854
|
+
ceg_out[i] = 0.0;
|
|
855
|
+
return;
|
|
856
|
+
}
|
|
857
|
+
let p = exp(ceg_logits[i] - ceg_lse[r]);
|
|
858
|
+
let hit = f < f32(ceg_p.classes) && i % ceg_p.classes == u32(max(f, 0.0));
|
|
859
|
+
ceg_out[i] = (p - select(0.0, 1.0, hit)) * ceg_grad[r];
|
|
860
|
+
}
|
|
550
861
|
"#;
|