ferrotensor 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/Cargo.lock +11 -11
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/Cargo.toml +1 -1
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/PKG-INFO +1 -1
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/Cargo.toml +1 -1
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/backend/compute.rs +6 -1
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/backend/kernels.rs +237 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/functional.rs +107 -12
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/tensor/cpu.rs +96 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/tensor/ops.rs +23 -49
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/tensor/raw.rs +245 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/tensor/tests.rs +100 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/README.md +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/README.md +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/examples/README.md +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/examples/char_transformer.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/examples/data/sonnets.txt +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/examples/gpu_matmul_attention.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/examples/mlp_two_moons.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/examples/quickstart.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/examples/similarity_search.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/backend/mod.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/data.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/device.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/lib.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/nn/activation.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/nn/attention.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/nn/container.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/nn/dropout.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/nn/embedding.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/nn/linear.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/nn/mod.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/nn/module.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/nn/normalization.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/nn/tests.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/optim/adam.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/optim/mod.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/optim/optimizer.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/optim/scheduler.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/optim/sgd.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/optim/tests.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/random.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/serialize.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/tensor/autograd.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/tensor/core.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/tensor/mod.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor/src/tensor/shape.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/.gitignore +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/Cargo.toml +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/README.md +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/build.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/examples/mlp.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/src/device.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/src/error.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/src/functional.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/src/lib.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/src/nn.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/src/optim.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/src/serialize.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/src/tensor.rs +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/tests/conftest.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/tests/test_autograd.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/tests/test_errors.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/tests/test_functional.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/tests/test_gpu.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/tests/test_nn.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/tests/test_optim.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/tests/test_serialize.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/tests/test_tensor.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/ferrotensor-py/tests/test_train.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/pyproject.toml +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/python/ferrotensor/__init__.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/python/ferrotensor/_ferrotensor.pyi +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/python/ferrotensor/nn/__init__.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/python/ferrotensor/nn/functional.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/python/ferrotensor/optim.py +0 -0
- {ferrotensor-0.3.0 → ferrotensor-0.4.0}/python/ferrotensor/py.typed +0 -0
|
@@ -606,7 +606,7 @@ dependencies = [
|
|
|
606
606
|
|
|
607
607
|
[[package]]
|
|
608
608
|
name = "ferrotensor"
|
|
609
|
-
version = "0.
|
|
609
|
+
version = "0.4.0"
|
|
610
610
|
dependencies = [
|
|
611
611
|
"bytemuck",
|
|
612
612
|
"naga",
|
|
@@ -619,7 +619,7 @@ dependencies = [
|
|
|
619
619
|
|
|
620
620
|
[[package]]
|
|
621
621
|
name = "ferrotensor-autograd"
|
|
622
|
-
version = "0.
|
|
622
|
+
version = "0.4.0"
|
|
623
623
|
dependencies = [
|
|
624
624
|
"ferrotensor-backend",
|
|
625
625
|
"pollster",
|
|
@@ -628,7 +628,7 @@ dependencies = [
|
|
|
628
628
|
|
|
629
629
|
[[package]]
|
|
630
630
|
name = "ferrotensor-backend"
|
|
631
|
-
version = "0.
|
|
631
|
+
version = "0.4.0"
|
|
632
632
|
dependencies = [
|
|
633
633
|
"bytemuck",
|
|
634
634
|
"env_logger",
|
|
@@ -641,7 +641,7 @@ dependencies = [
|
|
|
641
641
|
|
|
642
642
|
[[package]]
|
|
643
643
|
name = "ferrotensor-benchmarks"
|
|
644
|
-
version = "0.
|
|
644
|
+
version = "0.4.0"
|
|
645
645
|
dependencies = [
|
|
646
646
|
"ferrotensor",
|
|
647
647
|
"ferrotensor-backend",
|
|
@@ -652,7 +652,7 @@ dependencies = [
|
|
|
652
652
|
|
|
653
653
|
[[package]]
|
|
654
654
|
name = "ferrotensor-cli"
|
|
655
|
-
version = "0.
|
|
655
|
+
version = "0.4.0"
|
|
656
656
|
dependencies = [
|
|
657
657
|
"clap",
|
|
658
658
|
"env_logger",
|
|
@@ -670,7 +670,7 @@ dependencies = [
|
|
|
670
670
|
|
|
671
671
|
[[package]]
|
|
672
672
|
name = "ferrotensor-graph"
|
|
673
|
-
version = "0.
|
|
673
|
+
version = "0.4.0"
|
|
674
674
|
dependencies = [
|
|
675
675
|
"bytemuck",
|
|
676
676
|
"env_logger",
|
|
@@ -685,7 +685,7 @@ dependencies = [
|
|
|
685
685
|
|
|
686
686
|
[[package]]
|
|
687
687
|
name = "ferrotensor-hub"
|
|
688
|
-
version = "0.
|
|
688
|
+
version = "0.4.0"
|
|
689
689
|
dependencies = [
|
|
690
690
|
"ferrotensor-autograd",
|
|
691
691
|
"ferrotensor-backend",
|
|
@@ -698,7 +698,7 @@ dependencies = [
|
|
|
698
698
|
|
|
699
699
|
[[package]]
|
|
700
700
|
name = "ferrotensor-loader"
|
|
701
|
-
version = "0.
|
|
701
|
+
version = "0.4.0"
|
|
702
702
|
dependencies = [
|
|
703
703
|
"bytemuck",
|
|
704
704
|
"env_logger",
|
|
@@ -716,7 +716,7 @@ dependencies = [
|
|
|
716
716
|
|
|
717
717
|
[[package]]
|
|
718
718
|
name = "ferrotensor-monitor"
|
|
719
|
-
version = "0.
|
|
719
|
+
version = "0.4.0"
|
|
720
720
|
dependencies = [
|
|
721
721
|
"anyhow",
|
|
722
722
|
"chrono",
|
|
@@ -730,7 +730,7 @@ dependencies = [
|
|
|
730
730
|
|
|
731
731
|
[[package]]
|
|
732
732
|
name = "ferrotensor-py"
|
|
733
|
-
version = "0.
|
|
733
|
+
version = "0.4.0"
|
|
734
734
|
dependencies = [
|
|
735
735
|
"ferrotensor",
|
|
736
736
|
"pyo3",
|
|
@@ -739,7 +739,7 @@ dependencies = [
|
|
|
739
739
|
|
|
740
740
|
[[package]]
|
|
741
741
|
name = "ferrotensor-trainer"
|
|
742
|
-
version = "0.
|
|
742
|
+
version = "0.4.0"
|
|
743
743
|
dependencies = [
|
|
744
744
|
"bytemuck",
|
|
745
745
|
"ferrotensor-autograd",
|
|
@@ -9,7 +9,7 @@ use wgpu::{BindGroup, BindGroupLayout, BindingResource, CommandEncoder, ComputeP
|
|
|
9
9
|
use super::KERNELS_WGSL;
|
|
10
10
|
|
|
11
11
|
/// Entry points compiled from [`KERNELS_WGSL`].
|
|
12
|
-
pub(crate) const KERNEL_NAMES: [&str;
|
|
12
|
+
pub(crate) const KERNEL_NAMES: [&str; 14] = [
|
|
13
13
|
"unary",
|
|
14
14
|
"binary",
|
|
15
15
|
"reduce",
|
|
@@ -19,6 +19,11 @@ pub(crate) const KERNEL_NAMES: [&str; 9] = [
|
|
|
19
19
|
"matmul",
|
|
20
20
|
"permute",
|
|
21
21
|
"gather_rows",
|
|
22
|
+
"sort_ids",
|
|
23
|
+
"scatter_rows",
|
|
24
|
+
"copy_block",
|
|
25
|
+
"cross_entropy_rows",
|
|
26
|
+
"cross_entropy_grad",
|
|
22
27
|
];
|
|
23
28
|
|
|
24
29
|
/// Threads per workgroup of the 1-D kernels.
|
|
@@ -547,4 +547,241 @@ fn gather_rows(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
|
547
547
|
}
|
|
548
548
|
gat_out[i] = gat_table[id * gat_p.row + c];
|
|
549
549
|
}
|
|
550
|
+
|
|
551
|
+
// --- Row scatter-add (index_select and embedding backward) -------------------
|
|
552
|
+
// WGSL has no float atomics, so the gradient rows are not added with atomics.
|
|
553
|
+
// Instead the (row, position) pairs of the ids are sorted by row, then by
|
|
554
|
+
// position; head[row] is the index of the row's first pair, or SORT_NONE when
|
|
555
|
+
// no id picks it. One thread per output element adds its row's gradients in
|
|
556
|
+
// position order, as the CPU loop does, so the result is deterministic and
|
|
557
|
+
// matches the CPU bit for bit.
|
|
558
|
+
|
|
559
|
+
const SORT_NONE: u32 = 0xffffffffu;
|
|
560
|
+
|
|
561
|
+
// sort_ids builds the pairs and the head table on the GPU from f32 ids.
|
|
562
|
+
// op 0: pair i = (row of id i, i) for i < n, padded with (SORT_NONE, i) up to
|
|
563
|
+
// a = the power-of-two pair count; b = rows. An id outside [0, rows)
|
|
564
|
+
// gets row SORT_NONE: it sorts last and adds nothing.
|
|
565
|
+
// op 1: one compare-and-swap step of a bitonic sort of n pairs (a = k, b = j).
|
|
566
|
+
// op 2: head[r] = SORT_NONE for r < n.
|
|
567
|
+
// op 3: head[row] = i for each pair i < n that is the first of its row.
|
|
568
|
+
|
|
569
|
+
struct SortParams {
|
|
570
|
+
op: u32,
|
|
571
|
+
n: u32,
|
|
572
|
+
a: u32,
|
|
573
|
+
b: u32,
|
|
574
|
+
};
|
|
575
|
+
|
|
576
|
+
@group(0) @binding(0) var<storage, read> srt_ids: array<f32>;
|
|
577
|
+
@group(0) @binding(1) var<storage, read_write> srt_keys: array<u32>;
|
|
578
|
+
@group(0) @binding(2) var<storage, read_write> srt_head: array<u32>;
|
|
579
|
+
@group(0) @binding(3) var<uniform> srt_p: SortParams;
|
|
580
|
+
|
|
581
|
+
@compute @workgroup_size(256)
|
|
582
|
+
fn sort_ids(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
583
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
584
|
+
let i = flat_index(gid, nwg);
|
|
585
|
+
switch srt_p.op {
|
|
586
|
+
case 0u: {
|
|
587
|
+
if (i >= srt_p.a) { return; }
|
|
588
|
+
var row = SORT_NONE;
|
|
589
|
+
if (i < srt_p.n) {
|
|
590
|
+
let f = srt_ids[i];
|
|
591
|
+
if (f >= 0.0 && f < f32(srt_p.b)) { row = u32(f); }
|
|
592
|
+
}
|
|
593
|
+
srt_keys[2u * i] = row;
|
|
594
|
+
srt_keys[2u * i + 1u] = i;
|
|
595
|
+
}
|
|
596
|
+
case 1u: {
|
|
597
|
+
let l = i ^ srt_p.b;
|
|
598
|
+
if (i >= srt_p.n || l <= i) { return; }
|
|
599
|
+
let ri = srt_keys[2u * i];
|
|
600
|
+
let pi = srt_keys[2u * i + 1u];
|
|
601
|
+
let rl = srt_keys[2u * l];
|
|
602
|
+
let pl = srt_keys[2u * l + 1u];
|
|
603
|
+
let greater = ri > rl || (ri == rl && pi > pl);
|
|
604
|
+
if (greater == ((i & srt_p.a) == 0u)) {
|
|
605
|
+
srt_keys[2u * i] = rl;
|
|
606
|
+
srt_keys[2u * i + 1u] = pl;
|
|
607
|
+
srt_keys[2u * l] = ri;
|
|
608
|
+
srt_keys[2u * l + 1u] = pi;
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
case 2u: {
|
|
612
|
+
if (i >= srt_p.n) { return; }
|
|
613
|
+
srt_head[i] = SORT_NONE;
|
|
614
|
+
}
|
|
615
|
+
default: {
|
|
616
|
+
if (i >= srt_p.n) { return; }
|
|
617
|
+
let row = srt_keys[2u * i];
|
|
618
|
+
if (row == SORT_NONE) { return; }
|
|
619
|
+
if (i == 0u || srt_keys[2u * i - 2u] != row) {
|
|
620
|
+
srt_head[row] = i;
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// The gradient is viewed as [outer, n, row] and the output as
|
|
627
|
+
// [outer, rows, row].
|
|
628
|
+
struct ScatterParams {
|
|
629
|
+
rows: u32,
|
|
630
|
+
row: u32,
|
|
631
|
+
n: u32,
|
|
632
|
+
outer: u32,
|
|
633
|
+
};
|
|
634
|
+
|
|
635
|
+
@group(0) @binding(0) var<storage, read> sct_grad: array<f32>;
|
|
636
|
+
@group(0) @binding(1) var<storage, read> sct_keys: array<u32>;
|
|
637
|
+
@group(0) @binding(2) var<storage, read> sct_head: array<u32>;
|
|
638
|
+
@group(0) @binding(3) var<storage, read_write> sct_out: array<f32>;
|
|
639
|
+
@group(0) @binding(4) var<uniform> sct_p: ScatterParams;
|
|
640
|
+
|
|
641
|
+
@compute @workgroup_size(256)
|
|
642
|
+
fn scatter_rows(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
643
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
644
|
+
let i = flat_index(gid, nwg);
|
|
645
|
+
if (i >= sct_p.outer * sct_p.rows * sct_p.row) { return; }
|
|
646
|
+
let o = i / (sct_p.rows * sct_p.row);
|
|
647
|
+
let r = (i / sct_p.row) % sct_p.rows;
|
|
648
|
+
let c = i % sct_p.row;
|
|
649
|
+
let base = o * sct_p.n * sct_p.row + c;
|
|
650
|
+
var acc = 0.0;
|
|
651
|
+
var t = sct_head[r];
|
|
652
|
+
if (t != SORT_NONE) {
|
|
653
|
+
loop {
|
|
654
|
+
if (t >= sct_p.n || sct_keys[2u * t] != r) { break; }
|
|
655
|
+
acc = acc + sct_grad[base + sct_keys[2u * t + 1u] * sct_p.row];
|
|
656
|
+
t = t + 1u;
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
sct_out[i] = acc;
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
// --- Block copy (cat) --------------------------------------------------------
|
|
663
|
+
// Copies an input viewed as [outer, block_in] into columns
|
|
664
|
+
// [offset, offset + block_in) of an output viewed as [outer, block_out].
|
|
665
|
+
|
|
666
|
+
struct CopyParams {
|
|
667
|
+
n: u32,
|
|
668
|
+
block_in: u32,
|
|
669
|
+
block_out: u32,
|
|
670
|
+
offset: u32,
|
|
671
|
+
};
|
|
672
|
+
|
|
673
|
+
@group(0) @binding(0) var<storage, read> cpy_in: array<f32>;
|
|
674
|
+
@group(0) @binding(1) var<storage, read_write> cpy_out: array<f32>;
|
|
675
|
+
@group(0) @binding(2) var<uniform> cpy_p: CopyParams;
|
|
676
|
+
|
|
677
|
+
@compute @workgroup_size(256)
|
|
678
|
+
fn copy_block(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
679
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
680
|
+
let i = flat_index(gid, nwg);
|
|
681
|
+
if (i >= cpy_p.n) { return; }
|
|
682
|
+
let o = i / cpy_p.block_in;
|
|
683
|
+
cpy_out[o * cpy_p.block_out + cpy_p.offset + i % cpy_p.block_in] = cpy_in[i];
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
// --- Cross-entropy with class-id targets ---------------------------------------
|
|
687
|
+
// Logits are viewed as [rows, classes]; targets hold one f32 class id per row.
|
|
688
|
+
// A target <= -1 (an id that is negative as an integer, such as -100) skips
|
|
689
|
+
// its row: loss 0, valid 0, gradient 0. Otherwise the id is truncated to an
|
|
690
|
+
// integer; one that is not below `classes` (or NaN) gives a NaN loss.
|
|
691
|
+
// cross_entropy_rows: one workgroup per row writes the row's loss
|
|
692
|
+
// (lse - logit[target]), its log-sum-exp and whether it counts.
|
|
693
|
+
// cross_entropy_grad: d loss[r] / d logit[r, c] = softmax[r, c] - [c == target],
|
|
694
|
+
// times the incoming gradient of the row's loss.
|
|
695
|
+
|
|
696
|
+
struct CeParams {
|
|
697
|
+
rows: u32,
|
|
698
|
+
classes: u32,
|
|
699
|
+
_pad0: u32,
|
|
700
|
+
_pad1: u32,
|
|
701
|
+
};
|
|
702
|
+
|
|
703
|
+
@group(0) @binding(0) var<storage, read> ce_logits: array<f32>;
|
|
704
|
+
@group(0) @binding(1) var<storage, read> ce_targets: array<f32>;
|
|
705
|
+
@group(0) @binding(2) var<storage, read_write> ce_loss: array<f32>;
|
|
706
|
+
@group(0) @binding(3) var<storage, read_write> ce_lse: array<f32>;
|
|
707
|
+
@group(0) @binding(4) var<storage, read_write> ce_valid: array<f32>;
|
|
708
|
+
@group(0) @binding(5) var<uniform> ce_p: CeParams;
|
|
709
|
+
|
|
710
|
+
var<workgroup> ce_part: array<f32, 256>;
|
|
711
|
+
|
|
712
|
+
@compute @workgroup_size(256)
|
|
713
|
+
fn cross_entropy_rows(@builtin(local_invocation_index) lid: u32,
|
|
714
|
+
@builtin(workgroup_id) wid: vec3<u32>,
|
|
715
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
716
|
+
let r = wid.x + wid.y * nwg.x;
|
|
717
|
+
// Uniform per workgroup, so every thread reaches the barriers or none.
|
|
718
|
+
if (r >= ce_p.rows) { return; }
|
|
719
|
+
let c = ce_p.classes;
|
|
720
|
+
let base = r * c;
|
|
721
|
+
var m = bitcast<f32>(0xff800000u);
|
|
722
|
+
for (var j = lid; j < c; j = j + 256u) {
|
|
723
|
+
m = max(m, ce_logits[base + j]);
|
|
724
|
+
}
|
|
725
|
+
ce_part[lid] = m;
|
|
726
|
+
workgroupBarrier();
|
|
727
|
+
for (var half = 128u; half > 0u; half = half >> 1u) {
|
|
728
|
+
if (lid < half) {
|
|
729
|
+
ce_part[lid] = max(ce_part[lid], ce_part[lid + half]);
|
|
730
|
+
}
|
|
731
|
+
workgroupBarrier();
|
|
732
|
+
}
|
|
733
|
+
m = ce_part[0];
|
|
734
|
+
workgroupBarrier();
|
|
735
|
+
var s = 0.0;
|
|
736
|
+
for (var j = lid; j < c; j = j + 256u) {
|
|
737
|
+
s = s + exp(ce_logits[base + j] - m);
|
|
738
|
+
}
|
|
739
|
+
ce_part[lid] = s;
|
|
740
|
+
workgroupBarrier();
|
|
741
|
+
for (var half = 128u; half > 0u; half = half >> 1u) {
|
|
742
|
+
if (lid < half) {
|
|
743
|
+
ce_part[lid] = ce_part[lid] + ce_part[lid + half];
|
|
744
|
+
}
|
|
745
|
+
workgroupBarrier();
|
|
746
|
+
}
|
|
747
|
+
if (lid == 0u) {
|
|
748
|
+
let lse = m + log(ce_part[0]);
|
|
749
|
+
let f = ce_targets[r];
|
|
750
|
+
ce_lse[r] = lse;
|
|
751
|
+
if (f <= -1.0) {
|
|
752
|
+
ce_loss[r] = 0.0;
|
|
753
|
+
ce_valid[r] = 0.0;
|
|
754
|
+
} else {
|
|
755
|
+
ce_valid[r] = 1.0;
|
|
756
|
+
if (f < f32(c)) {
|
|
757
|
+
ce_loss[r] = lse - ce_logits[base + u32(max(f, 0.0))];
|
|
758
|
+
} else {
|
|
759
|
+
ce_loss[r] = bitcast<f32>(0x7fc00000u);
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
@group(0) @binding(0) var<storage, read> ceg_logits: array<f32>;
|
|
766
|
+
@group(0) @binding(1) var<storage, read> ceg_targets: array<f32>;
|
|
767
|
+
@group(0) @binding(2) var<storage, read> ceg_lse: array<f32>;
|
|
768
|
+
@group(0) @binding(3) var<storage, read> ceg_grad: array<f32>;
|
|
769
|
+
@group(0) @binding(4) var<storage, read_write> ceg_out: array<f32>;
|
|
770
|
+
@group(0) @binding(5) var<uniform> ceg_p: CeParams;
|
|
771
|
+
|
|
772
|
+
@compute @workgroup_size(256)
|
|
773
|
+
fn cross_entropy_grad(@builtin(global_invocation_id) gid: vec3<u32>,
|
|
774
|
+
@builtin(num_workgroups) nwg: vec3<u32>) {
|
|
775
|
+
let i = flat_index(gid, nwg);
|
|
776
|
+
if (i >= ceg_p.rows * ceg_p.classes) { return; }
|
|
777
|
+
let r = i / ceg_p.classes;
|
|
778
|
+
let f = ceg_targets[r];
|
|
779
|
+
if (f <= -1.0) {
|
|
780
|
+
ceg_out[i] = 0.0;
|
|
781
|
+
return;
|
|
782
|
+
}
|
|
783
|
+
let p = exp(ceg_logits[i] - ceg_lse[r]);
|
|
784
|
+
let hit = f < f32(ceg_p.classes) && i % ceg_p.classes == u32(max(f, 0.0));
|
|
785
|
+
ceg_out[i] = (p - select(0.0, 1.0, hit)) * ceg_grad[r];
|
|
786
|
+
}
|
|
550
787
|
"#;
|
|
@@ -300,21 +300,47 @@ targets_from_ids!(i32, usize);
|
|
|
300
300
|
/// positions;
|
|
301
301
|
/// - or a `&Tensor` distribution (one-hot or soft labels) with the shape of
|
|
302
302
|
/// `logits`.
|
|
303
|
+
///
|
|
304
|
+
/// With class ids on a GPU, the loss and its gradient are computed on the
|
|
305
|
+
/// device with no copy to the host. An id that is not below the number of
|
|
306
|
+
/// classes panics, except for an id tensor on a GPU, which is not read
|
|
307
|
+
/// back: there it gives a NaN loss.
|
|
303
308
|
pub fn cross_entropy<'a>(logits: &Tensor, targets: impl Into<Targets<'a>>) -> Tensor {
|
|
304
309
|
let targets = targets.into();
|
|
305
|
-
let
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
+
let classes = logits.shape[logits.ndim() - 1];
|
|
311
|
+
let rows = logits.numel() / classes.max(1);
|
|
312
|
+
let ids = match targets {
|
|
313
|
+
Targets::Tensor(t) if t.shape == logits.shape => {
|
|
314
|
+
let rows = rows.max(1);
|
|
315
|
+
return logits
|
|
316
|
+
.log_softmax(-1)
|
|
317
|
+
.mul(t)
|
|
318
|
+
.sum()
|
|
319
|
+
.div_scalar(-(rows as f32));
|
|
310
320
|
}
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
321
|
+
Targets::Tensor(t) => {
|
|
322
|
+
assert_eq!(
|
|
323
|
+
t.shape,
|
|
324
|
+
logits.shape[..logits.ndim() - 1],
|
|
325
|
+
"class-id targets must have the logits shape without the class dim"
|
|
326
|
+
);
|
|
327
|
+
t.clone()
|
|
328
|
+
}
|
|
329
|
+
Targets::Ids(ids) => {
|
|
330
|
+
assert_eq!(ids.len(), rows, "one target per row of log-probabilities");
|
|
331
|
+
for &t in ids.iter() {
|
|
332
|
+
assert!(
|
|
333
|
+
t < classes as i64,
|
|
334
|
+
"target class {t} out of range for {classes} classes"
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
let ids = ids.iter().map(|&t| t.max(-1) as f32).collect();
|
|
338
|
+
Tensor::from_vec(ids, &[rows], logits.device())
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
// Mean over the rows that count; 0 when none does.
|
|
342
|
+
let (losses, valid) = logits.cross_entropy_rows(&ids);
|
|
343
|
+
losses.sum().div(&valid.sum().clamp(1.0, f32::INFINITY))
|
|
318
344
|
}
|
|
319
345
|
|
|
320
346
|
/// Negative log-likelihood of class `targets` (see [`Targets`]) under
|
|
@@ -428,6 +454,7 @@ pub fn accuracy<'a>(logits: &Tensor, targets: impl Into<Targets<'a>>) -> f32 {
|
|
|
428
454
|
mod tests {
|
|
429
455
|
use super::*;
|
|
430
456
|
use crate::tensor::tests::{check_gpu_parity, check_gradients, Case, Init};
|
|
457
|
+
use rand::{Rng, SeedableRng};
|
|
431
458
|
|
|
432
459
|
const U: Init = Init::Uniform(-2.0, 2.0);
|
|
433
460
|
const PROB: Init = Init::Uniform(0.05, 0.95);
|
|
@@ -632,4 +659,72 @@ mod tests {
|
|
|
632
659
|
let t = cpu(&[2.0, -1.0, 0.0], &[3]);
|
|
633
660
|
assert_eq!(one_hot(&t, 3, &d).to_vec(), want);
|
|
634
661
|
}
|
|
662
|
+
|
|
663
|
+
#[test]
|
|
664
|
+
#[should_panic(expected = "out of range")]
|
|
665
|
+
fn cross_entropy_rejects_out_of_range_tensor_targets_on_the_cpu() {
|
|
666
|
+
cross_entropy(&cpu(&[0.0; 6], &[2, 3]), &cpu(&[1.0, 3.0], &[2]));
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
#[test]
|
|
670
|
+
fn cross_entropy_with_every_row_skipped_is_zero() {
|
|
671
|
+
let logits = cpu(&[1.0, 2.0, 3.0, 4.0], &[2, 2]).requires_grad_();
|
|
672
|
+
let loss = cross_entropy(&logits, &[-100, -100]);
|
|
673
|
+
assert_eq!(loss.item(), 0.0);
|
|
674
|
+
loss.backward();
|
|
675
|
+
assert_eq!(logits.grad().unwrap().to_vec(), vec![0.0; 4]);
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
/// With `FERROTENSOR_RUN_GPU_TESTS=1`: class-id cross-entropy on the GPU
|
|
679
|
+
/// (loss and logits gradient) matches the CPU on wide rows, 3-D logits,
|
|
680
|
+
/// skipped and fractional ids, and gives NaN for an id out of range.
|
|
681
|
+
#[test]
|
|
682
|
+
fn cross_entropy_matches_cpu_on_gpu() {
|
|
683
|
+
if std::env::var("FERROTENSOR_RUN_GPU_TESTS").as_deref() != Ok("1") {
|
|
684
|
+
eprintln!("skipping GPU cross_entropy; set FERROTENSOR_RUN_GPU_TESTS=1 to enable");
|
|
685
|
+
return;
|
|
686
|
+
}
|
|
687
|
+
let gpu = Device::gpu().expect("FERROTENSOR_RUN_GPU_TESTS=1 but no GPU is available");
|
|
688
|
+
let mut rng = rand::rngs::StdRng::seed_from_u64(13);
|
|
689
|
+
for shape in [
|
|
690
|
+
&[1, 1][..],
|
|
691
|
+
&[5, 3],
|
|
692
|
+
&[37, 1000],
|
|
693
|
+
&[2, 3, 300],
|
|
694
|
+
&[64, 32_000],
|
|
695
|
+
] {
|
|
696
|
+
let classes = shape[shape.len() - 1];
|
|
697
|
+
let rows: usize = shape[..shape.len() - 1].iter().product();
|
|
698
|
+
let logits: Vec<f32> = (0..rows * classes)
|
|
699
|
+
.map(|_| rng.gen_range(-8.0..8.0))
|
|
700
|
+
.collect();
|
|
701
|
+
let mut targets: Vec<f32> = (0..rows).map(|r| ((r * 7919) % classes) as f32).collect();
|
|
702
|
+
if rows > 2 {
|
|
703
|
+
targets[0] = -100.0;
|
|
704
|
+
targets[1] += 0.5;
|
|
705
|
+
targets[2] = -0.5;
|
|
706
|
+
}
|
|
707
|
+
let run = |d: &Device| {
|
|
708
|
+
let x = Tensor::from_data(&logits, shape, d).requires_grad_();
|
|
709
|
+
let t = Tensor::from_data(&targets, &shape[..shape.len() - 1], d);
|
|
710
|
+
let loss = cross_entropy(&x, &t);
|
|
711
|
+
loss.backward();
|
|
712
|
+
(loss.item(), x.grad().unwrap().to_vec())
|
|
713
|
+
};
|
|
714
|
+
let ((gl, gg), (cl, cg)) = (run(&gpu), run(&Device::cpu()));
|
|
715
|
+
assert!(
|
|
716
|
+
(gl - cl).abs() <= 1e-5 * cl.abs().max(1.0),
|
|
717
|
+
"{shape:?}: {gl} vs {cl}"
|
|
718
|
+
);
|
|
719
|
+
let worst = gg
|
|
720
|
+
.iter()
|
|
721
|
+
.zip(&cg)
|
|
722
|
+
.map(|(a, b)| (a - b).abs())
|
|
723
|
+
.fold(0.0, f32::max);
|
|
724
|
+
assert!(worst <= 1e-6, "{shape:?}: gradient differs by {worst}");
|
|
725
|
+
}
|
|
726
|
+
let x = Tensor::from_data(&[0.0; 6], &[2, 3], &gpu);
|
|
727
|
+
let t = Tensor::from_data(&[1.0, 3.0], &[2], &gpu);
|
|
728
|
+
assert!(cross_entropy(&x, &t).item().is_nan());
|
|
729
|
+
}
|
|
635
730
|
}
|
|
@@ -296,6 +296,102 @@ pub(crate) fn gather_rows(table: &[f32], idx: &[u32], row: usize) -> Vec<f32> {
|
|
|
296
296
|
out
|
|
297
297
|
}
|
|
298
298
|
|
|
299
|
+
/// The class of a cross-entropy target stored as `f32`: `None` to skip the
|
|
300
|
+
/// row (an id that is negative as an integer, such as -100), else the
|
|
301
|
+
/// truncated id, as in the `cross_entropy_rows` kernel.
|
|
302
|
+
///
|
|
303
|
+
/// # Panics
|
|
304
|
+
/// Panics when the id is NaN or not below `classes`.
|
|
305
|
+
pub(crate) fn target_class(f: f32, classes: usize) -> Option<usize> {
|
|
306
|
+
if f <= -1.0 {
|
|
307
|
+
return None;
|
|
308
|
+
}
|
|
309
|
+
assert!(
|
|
310
|
+
f < classes as f32,
|
|
311
|
+
"target class {f} out of range for {classes} classes"
|
|
312
|
+
);
|
|
313
|
+
Some(f.max(0.0) as usize)
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/// Per row of `x` (`[rows, classes]`): the cross-entropy loss against
|
|
317
|
+
/// `targets` (0 for a skipped row), the log-sum-exp, and 1 or 0 for whether
|
|
318
|
+
/// the row counts.
|
|
319
|
+
pub(crate) fn cross_entropy_rows(
|
|
320
|
+
x: &[f32],
|
|
321
|
+
targets: &[f32],
|
|
322
|
+
classes: usize,
|
|
323
|
+
) -> (Vec<f32>, Vec<f32>, Vec<f32>) {
|
|
324
|
+
let rows = targets.len();
|
|
325
|
+
let (mut loss, mut lse, mut valid) = (vec![0.0; rows], vec![0.0; rows], vec![0.0; rows]);
|
|
326
|
+
for r in 0..rows {
|
|
327
|
+
let row = &x[r * classes..][..classes];
|
|
328
|
+
let m = row.iter().fold(f32::NEG_INFINITY, |a, &b| a.max(b));
|
|
329
|
+
let s: f32 = row.iter().map(|&v| (v - m).exp()).sum();
|
|
330
|
+
lse[r] = m + s.ln();
|
|
331
|
+
if let Some(t) = target_class(targets[r], classes) {
|
|
332
|
+
loss[r] = lse[r] - row[t];
|
|
333
|
+
valid[r] = 1.0;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
(loss, lse, valid)
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/// Gradient of [`cross_entropy_rows`]'s loss with respect to `x`, given the
|
|
340
|
+
/// gradient `grad` of each row's loss.
|
|
341
|
+
pub(crate) fn cross_entropy_grad(
|
|
342
|
+
x: &[f32],
|
|
343
|
+
targets: &[f32],
|
|
344
|
+
lse: &[f32],
|
|
345
|
+
grad: &[f32],
|
|
346
|
+
classes: usize,
|
|
347
|
+
) -> Vec<f32> {
|
|
348
|
+
let mut out = vec![0.0; x.len()];
|
|
349
|
+
for (r, &f) in targets.iter().enumerate() {
|
|
350
|
+
if let Some(t) = target_class(f, classes) {
|
|
351
|
+
for (c, o) in out[r * classes..][..classes].iter_mut().enumerate() {
|
|
352
|
+
let p = (x[r * classes + c] - lse[r]).exp();
|
|
353
|
+
*o = (p - if c == t { 1.0 } else { 0.0 }) * grad[r];
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
out
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/// Concatenate `parts` (values, block length), each viewed as
|
|
361
|
+
/// `[outer, block]`, along the block dim.
|
|
362
|
+
pub(crate) fn cat(parts: &[(&[f32], usize)], outer: usize) -> Vec<f32> {
|
|
363
|
+
let mut out = Vec::with_capacity(parts.iter().map(|(d, _)| d.len()).sum());
|
|
364
|
+
for o in 0..outer {
|
|
365
|
+
for &(values, block) in parts {
|
|
366
|
+
out.extend_from_slice(&values[o * block..(o + 1) * block]);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
out
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/// Scatter-add along the middle dim: `grad` viewed as `[outer, n, row]`,
|
|
373
|
+
/// the result as `[outer, rows, row]`, whose row `r` is the sum, in position
|
|
374
|
+
/// order, of the gradient rows `p` of every pair `(p, r)`.
|
|
375
|
+
pub(crate) fn scatter_rows(
|
|
376
|
+
grad: &[f32],
|
|
377
|
+
pairs: &[(usize, usize)],
|
|
378
|
+
outer: usize,
|
|
379
|
+
rows: usize,
|
|
380
|
+
row: usize,
|
|
381
|
+
) -> Vec<f32> {
|
|
382
|
+
let n = grad.len() / (outer * row).max(1);
|
|
383
|
+
let mut out = vec![0.0f32; outer * rows * row];
|
|
384
|
+
for o in 0..outer {
|
|
385
|
+
for &(p, r) in pairs {
|
|
386
|
+
let src = &grad[(o * n + p) * row..][..row];
|
|
387
|
+
for (v, &g) in out[(o * rows + r) * row..][..row].iter_mut().zip(src) {
|
|
388
|
+
*v += g;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
out
|
|
393
|
+
}
|
|
394
|
+
|
|
299
395
|
#[cfg(test)]
|
|
300
396
|
mod tests {
|
|
301
397
|
use super::*;
|