ferrotensor 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/Cargo.lock +18 -105
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/Cargo.toml +3 -4
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/PKG-INFO +7 -4
- {ferrotensor-0.6.0/ferrotensor-py → ferrotensor-0.7.0}/README.md +5 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/Cargo.toml +4 -2
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/examples/README.md +22 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/examples/char_transformer.rs +39 -8
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/examples/gpu_matmul_attention.rs +43 -9
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/examples/mlp_two_moons.rs +23 -2
- ferrotensor-0.7.0/ferrotensor/examples/quickstart.rs +36 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/examples/similarity_search.rs +10 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/backend/compute.rs +9 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/backend/kernels.rs +17 -3
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/data.rs +1 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/device.rs +23 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/functional.rs +9 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/lib.rs +2 -4
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/nn/attention.rs +2 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/nn/container.rs +11 -10
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/nn/embedding.rs +2 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/nn/tests.rs +3 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/optim/tests.rs +3 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/serialize.rs +2 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/tensor/autograd.rs +4 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/tensor/core.rs +25 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/tensor/cpu.rs +4 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/tensor/ops.rs +39 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/tensor/raw.rs +15 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/tensor/tests.rs +17 -2
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/Cargo.toml +0 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0/ferrotensor-py}/README.md +5 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/examples/mlp.py +15 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/src/nn.rs +4 -1
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/tests/conftest.py +8 -3
- ferrotensor-0.7.0/ferrotensor-py/tests/test_autograd.py +324 -0
- ferrotensor-0.7.0/ferrotensor-py/tests/test_errors.py +265 -0
- ferrotensor-0.7.0/ferrotensor-py/tests/test_functional.py +194 -0
- ferrotensor-0.7.0/ferrotensor-py/tests/test_gpu.py +632 -0
- ferrotensor-0.7.0/ferrotensor-py/tests/test_nn.py +282 -0
- ferrotensor-0.7.0/ferrotensor-py/tests/test_optim.py +220 -0
- ferrotensor-0.7.0/ferrotensor-py/tests/test_serialize.py +238 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/tests/test_tensor.py +125 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/tests/test_train.py +16 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/pyproject.toml +1 -1
- ferrotensor-0.6.0/ferrotensor/examples/quickstart.rs +0 -21
- ferrotensor-0.6.0/ferrotensor-py/tests/test_autograd.py +0 -110
- ferrotensor-0.6.0/ferrotensor-py/tests/test_errors.py +0 -84
- ferrotensor-0.6.0/ferrotensor-py/tests/test_functional.py +0 -75
- ferrotensor-0.6.0/ferrotensor-py/tests/test_gpu.py +0 -70
- ferrotensor-0.6.0/ferrotensor-py/tests/test_nn.py +0 -144
- ferrotensor-0.6.0/ferrotensor-py/tests/test_optim.py +0 -90
- ferrotensor-0.6.0/ferrotensor-py/tests/test_serialize.py +0 -60
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/README.md +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/examples/data/sonnets.txt +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/backend/mod.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/nn/activation.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/nn/dropout.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/nn/linear.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/nn/mod.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/nn/module.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/nn/normalization.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/optim/adam.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/optim/mod.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/optim/optimizer.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/optim/scheduler.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/optim/sgd.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/random.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/tensor/mod.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/src/tensor/shape.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor/tests/manual_seed.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/.gitignore +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/build.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/src/device.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/src/error.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/src/functional.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/src/lib.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/src/optim.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/src/serialize.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/ferrotensor-py/src/tensor.rs +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/python/ferrotensor/__init__.py +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/python/ferrotensor/_ferrotensor.pyi +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/python/ferrotensor/nn/__init__.py +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/python/ferrotensor/nn/functional.py +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/python/ferrotensor/optim.py +0 -0
- {ferrotensor-0.6.0 → ferrotensor-0.7.0}/python/ferrotensor/py.typed +0 -0
|
@@ -223,19 +223,6 @@ version = "0.1.1"
|
|
|
223
223
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
224
224
|
checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
|
|
225
225
|
|
|
226
|
-
[[package]]
|
|
227
|
-
name = "chrono"
|
|
228
|
-
version = "0.4.44"
|
|
229
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
230
|
-
checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
|
|
231
|
-
dependencies = [
|
|
232
|
-
"iana-time-zone",
|
|
233
|
-
"js-sys",
|
|
234
|
-
"num-traits",
|
|
235
|
-
"wasm-bindgen",
|
|
236
|
-
"windows-link",
|
|
237
|
-
]
|
|
238
|
-
|
|
239
226
|
[[package]]
|
|
240
227
|
name = "clap"
|
|
241
228
|
version = "4.5.56"
|
|
@@ -606,7 +593,7 @@ dependencies = [
|
|
|
606
593
|
|
|
607
594
|
[[package]]
|
|
608
595
|
name = "ferrotensor"
|
|
609
|
-
version = "0.
|
|
596
|
+
version = "0.7.0"
|
|
610
597
|
dependencies = [
|
|
611
598
|
"bytemuck",
|
|
612
599
|
"naga",
|
|
@@ -619,7 +606,7 @@ dependencies = [
|
|
|
619
606
|
|
|
620
607
|
[[package]]
|
|
621
608
|
name = "ferrotensor-autograd"
|
|
622
|
-
version = "0.
|
|
609
|
+
version = "0.7.0"
|
|
623
610
|
dependencies = [
|
|
624
611
|
"ferrotensor-backend",
|
|
625
612
|
"pollster",
|
|
@@ -628,7 +615,7 @@ dependencies = [
|
|
|
628
615
|
|
|
629
616
|
[[package]]
|
|
630
617
|
name = "ferrotensor-backend"
|
|
631
|
-
version = "0.
|
|
618
|
+
version = "0.7.0"
|
|
632
619
|
dependencies = [
|
|
633
620
|
"bytemuck",
|
|
634
621
|
"env_logger",
|
|
@@ -641,7 +628,7 @@ dependencies = [
|
|
|
641
628
|
|
|
642
629
|
[[package]]
|
|
643
630
|
name = "ferrotensor-benchmarks"
|
|
644
|
-
version = "0.
|
|
631
|
+
version = "0.7.0"
|
|
645
632
|
dependencies = [
|
|
646
633
|
"ferrotensor",
|
|
647
634
|
"ferrotensor-backend",
|
|
@@ -652,40 +639,44 @@ dependencies = [
|
|
|
652
639
|
|
|
653
640
|
[[package]]
|
|
654
641
|
name = "ferrotensor-cli"
|
|
655
|
-
version = "0.
|
|
642
|
+
version = "0.7.0"
|
|
656
643
|
dependencies = [
|
|
657
644
|
"clap",
|
|
658
645
|
"env_logger",
|
|
659
646
|
"ferrotensor-autograd",
|
|
660
647
|
"ferrotensor-backend",
|
|
661
648
|
"ferrotensor-graph",
|
|
649
|
+
"ferrotensor-hub",
|
|
662
650
|
"ferrotensor-loader",
|
|
663
651
|
"ferrotensor-monitor",
|
|
664
652
|
"ferrotensor-trainer",
|
|
665
653
|
"log",
|
|
666
654
|
"pollster",
|
|
655
|
+
"safetensors",
|
|
667
656
|
"serde",
|
|
668
657
|
"serde_json",
|
|
669
658
|
]
|
|
670
659
|
|
|
671
660
|
[[package]]
|
|
672
661
|
name = "ferrotensor-graph"
|
|
673
|
-
version = "0.
|
|
662
|
+
version = "0.7.0"
|
|
674
663
|
dependencies = [
|
|
675
664
|
"bytemuck",
|
|
676
665
|
"env_logger",
|
|
677
666
|
"ferrotensor-autograd",
|
|
678
667
|
"ferrotensor-backend",
|
|
679
668
|
"ferrotensor-loader",
|
|
669
|
+
"half",
|
|
680
670
|
"log",
|
|
681
671
|
"pollster",
|
|
672
|
+
"safetensors",
|
|
682
673
|
"serde",
|
|
683
674
|
"serde_json",
|
|
684
675
|
]
|
|
685
676
|
|
|
686
677
|
[[package]]
|
|
687
678
|
name = "ferrotensor-hub"
|
|
688
|
-
version = "0.
|
|
679
|
+
version = "0.7.0"
|
|
689
680
|
dependencies = [
|
|
690
681
|
"ferrotensor-autograd",
|
|
691
682
|
"ferrotensor-backend",
|
|
@@ -698,7 +689,7 @@ dependencies = [
|
|
|
698
689
|
|
|
699
690
|
[[package]]
|
|
700
691
|
name = "ferrotensor-loader"
|
|
701
|
-
version = "0.
|
|
692
|
+
version = "0.7.0"
|
|
702
693
|
dependencies = [
|
|
703
694
|
"bytemuck",
|
|
704
695
|
"env_logger",
|
|
@@ -716,10 +707,9 @@ dependencies = [
|
|
|
716
707
|
|
|
717
708
|
[[package]]
|
|
718
709
|
name = "ferrotensor-monitor"
|
|
719
|
-
version = "0.
|
|
710
|
+
version = "0.7.0"
|
|
720
711
|
dependencies = [
|
|
721
712
|
"anyhow",
|
|
722
|
-
"chrono",
|
|
723
713
|
"crossterm",
|
|
724
714
|
"ferrotensor-trainer",
|
|
725
715
|
"rand 0.9.5",
|
|
@@ -730,7 +720,7 @@ dependencies = [
|
|
|
730
720
|
|
|
731
721
|
[[package]]
|
|
732
722
|
name = "ferrotensor-py"
|
|
733
|
-
version = "0.
|
|
723
|
+
version = "0.7.0"
|
|
734
724
|
dependencies = [
|
|
735
725
|
"ferrotensor",
|
|
736
726
|
"pyo3",
|
|
@@ -739,7 +729,7 @@ dependencies = [
|
|
|
739
729
|
|
|
740
730
|
[[package]]
|
|
741
731
|
name = "ferrotensor-trainer"
|
|
742
|
-
version = "0.
|
|
732
|
+
version = "0.7.0"
|
|
743
733
|
dependencies = [
|
|
744
734
|
"bytemuck",
|
|
745
735
|
"ferrotensor-autograd",
|
|
@@ -995,30 +985,6 @@ version = "0.2.1"
|
|
|
995
985
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
996
986
|
checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df"
|
|
997
987
|
|
|
998
|
-
[[package]]
|
|
999
|
-
name = "iana-time-zone"
|
|
1000
|
-
version = "0.1.65"
|
|
1001
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1002
|
-
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
|
1003
|
-
dependencies = [
|
|
1004
|
-
"android_system_properties",
|
|
1005
|
-
"core-foundation-sys",
|
|
1006
|
-
"iana-time-zone-haiku",
|
|
1007
|
-
"js-sys",
|
|
1008
|
-
"log",
|
|
1009
|
-
"wasm-bindgen",
|
|
1010
|
-
"windows-core 0.62.2",
|
|
1011
|
-
]
|
|
1012
|
-
|
|
1013
|
-
[[package]]
|
|
1014
|
-
name = "iana-time-zone-haiku"
|
|
1015
|
-
version = "0.1.2"
|
|
1016
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1017
|
-
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
|
1018
|
-
dependencies = [
|
|
1019
|
-
"cc",
|
|
1020
|
-
]
|
|
1021
|
-
|
|
1022
988
|
[[package]]
|
|
1023
989
|
name = "ident_case"
|
|
1024
990
|
version = "1.0.1"
|
|
@@ -2486,25 +2452,12 @@ version = "0.57.0"
|
|
|
2486
2452
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2487
2453
|
checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
|
|
2488
2454
|
dependencies = [
|
|
2489
|
-
"windows-implement
|
|
2490
|
-
"windows-interface
|
|
2491
|
-
"windows-result
|
|
2455
|
+
"windows-implement",
|
|
2456
|
+
"windows-interface",
|
|
2457
|
+
"windows-result",
|
|
2492
2458
|
"windows-targets",
|
|
2493
2459
|
]
|
|
2494
2460
|
|
|
2495
|
-
[[package]]
|
|
2496
|
-
name = "windows-core"
|
|
2497
|
-
version = "0.62.2"
|
|
2498
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2499
|
-
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
|
2500
|
-
dependencies = [
|
|
2501
|
-
"windows-implement 0.60.2",
|
|
2502
|
-
"windows-interface 0.59.3",
|
|
2503
|
-
"windows-link",
|
|
2504
|
-
"windows-result 0.4.1",
|
|
2505
|
-
"windows-strings",
|
|
2506
|
-
]
|
|
2507
|
-
|
|
2508
2461
|
[[package]]
|
|
2509
2462
|
name = "windows-implement"
|
|
2510
2463
|
version = "0.57.0"
|
|
@@ -2516,17 +2469,6 @@ dependencies = [
|
|
|
2516
2469
|
"syn 2.0.114",
|
|
2517
2470
|
]
|
|
2518
2471
|
|
|
2519
|
-
[[package]]
|
|
2520
|
-
name = "windows-implement"
|
|
2521
|
-
version = "0.60.2"
|
|
2522
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2523
|
-
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
|
2524
|
-
dependencies = [
|
|
2525
|
-
"proc-macro2",
|
|
2526
|
-
"quote",
|
|
2527
|
-
"syn 2.0.114",
|
|
2528
|
-
]
|
|
2529
|
-
|
|
2530
2472
|
[[package]]
|
|
2531
2473
|
name = "windows-interface"
|
|
2532
2474
|
version = "0.57.0"
|
|
@@ -2538,17 +2480,6 @@ dependencies = [
|
|
|
2538
2480
|
"syn 2.0.114",
|
|
2539
2481
|
]
|
|
2540
2482
|
|
|
2541
|
-
[[package]]
|
|
2542
|
-
name = "windows-interface"
|
|
2543
|
-
version = "0.59.3"
|
|
2544
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2545
|
-
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
|
2546
|
-
dependencies = [
|
|
2547
|
-
"proc-macro2",
|
|
2548
|
-
"quote",
|
|
2549
|
-
"syn 2.0.114",
|
|
2550
|
-
]
|
|
2551
|
-
|
|
2552
2483
|
[[package]]
|
|
2553
2484
|
name = "windows-link"
|
|
2554
2485
|
version = "0.2.1"
|
|
@@ -2564,24 +2495,6 @@ dependencies = [
|
|
|
2564
2495
|
"windows-targets",
|
|
2565
2496
|
]
|
|
2566
2497
|
|
|
2567
|
-
[[package]]
|
|
2568
|
-
name = "windows-result"
|
|
2569
|
-
version = "0.4.1"
|
|
2570
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2571
|
-
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
|
|
2572
|
-
dependencies = [
|
|
2573
|
-
"windows-link",
|
|
2574
|
-
]
|
|
2575
|
-
|
|
2576
|
-
[[package]]
|
|
2577
|
-
name = "windows-strings"
|
|
2578
|
-
version = "0.5.1"
|
|
2579
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2580
|
-
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
|
|
2581
|
-
dependencies = [
|
|
2582
|
-
"windows-link",
|
|
2583
|
-
]
|
|
2584
|
-
|
|
2585
2498
|
[[package]]
|
|
2586
2499
|
name = "windows-sys"
|
|
2587
2500
|
version = "0.59.0"
|
|
@@ -4,15 +4,14 @@ resolver = "2"
|
|
|
4
4
|
|
|
5
5
|
[workspace.package]
|
|
6
6
|
edition = "2021"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.7.0"
|
|
8
8
|
license = "MIT OR Apache-2.0"
|
|
9
9
|
repository = "https://github.com/puranikyashaswin/ferrotensor"
|
|
10
|
-
authors = ["
|
|
10
|
+
authors = ["Yashaswin Sharma Puranik <puranikyashaswinsharma@gmail.com>"]
|
|
11
11
|
rust-version = "1.88"
|
|
12
|
-
homepage = "https://github.com/puranikyashaswin/ferrotensor"
|
|
13
12
|
|
|
14
13
|
[workspace.lints.rust]
|
|
15
|
-
unsafe_code = "
|
|
14
|
+
unsafe_code = "deny"
|
|
16
15
|
missing_docs = "warn"
|
|
17
16
|
rust_2018_idioms = { level = "warn", priority = -1 }
|
|
18
17
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ferrotensor
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Classifier: Programming Language :: Rust
|
|
5
5
|
Classifier: Programming Language :: Python :: 3
|
|
6
6
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
@@ -9,8 +9,7 @@ Requires-Dist: pytest>=7 ; extra == 'test'
|
|
|
9
9
|
Provides-Extra: test
|
|
10
10
|
Summary: PyTorch-style tensors, autograd and nn on wgpu (Metal, Vulkan, DX12) or the CPU, written in Rust
|
|
11
11
|
Keywords: machine-learning,deep-learning,gpu,autograd,wgpu
|
|
12
|
-
|
|
13
|
-
Author-email: Puranik Yashaswin Sharma <puranikyashaswinsharma@gmail.com>
|
|
12
|
+
Author-email: Yashaswin Sharma Puranik <puranikyashaswinsharma@gmail.com>
|
|
14
13
|
License-Expression: MIT OR Apache-2.0
|
|
15
14
|
Requires-Python: >=3.10
|
|
16
15
|
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
@@ -65,6 +64,7 @@ y.backward()
|
|
|
65
64
|
print(x.grad) # tensor([[7.0, 11.0], [9.0, 13.0]])
|
|
66
65
|
|
|
67
66
|
# A model, a loss and an optimizer: learn XOR
|
|
67
|
+
ft.manual_seed(0) # fixes the initial weights, so the output below repeats
|
|
68
68
|
model = nn.Sequential(nn.Linear(2, 16), nn.Tanh(), nn.Linear(16, 2))
|
|
69
69
|
opt = optim.AdamW(model.parameters(), lr=0.05, weight_decay=0.0)
|
|
70
70
|
inputs = ft.tensor([[0, 0], [0, 1], [1, 0], [1, 1]])
|
|
@@ -155,9 +155,12 @@ naming the shapes involved; the Rust code never panics across the boundary.
|
|
|
155
155
|
|
|
156
156
|
```bash
|
|
157
157
|
pytest # CPU
|
|
158
|
-
|
|
158
|
+
FERROTENSOR_RUN_GPU_TESTS=1 pytest # also GPU parity tests
|
|
159
159
|
```
|
|
160
160
|
|
|
161
|
+
`FERROTENSOR_GPU_TESTS=1` still works as the older name. CI runs the GPU
|
|
162
|
+
tests on software Vulkan (Mesa lavapipe).
|
|
163
|
+
|
|
161
164
|
## License
|
|
162
165
|
|
|
163
166
|
MIT or Apache-2.0, at your option.
|
|
@@ -46,6 +46,7 @@ y.backward()
|
|
|
46
46
|
print(x.grad) # tensor([[7.0, 11.0], [9.0, 13.0]])
|
|
47
47
|
|
|
48
48
|
# A model, a loss and an optimizer: learn XOR
|
|
49
|
+
ft.manual_seed(0) # fixes the initial weights, so the output below repeats
|
|
49
50
|
model = nn.Sequential(nn.Linear(2, 16), nn.Tanh(), nn.Linear(16, 2))
|
|
50
51
|
opt = optim.AdamW(model.parameters(), lr=0.05, weight_decay=0.0)
|
|
51
52
|
inputs = ft.tensor([[0, 0], [0, 1], [1, 0], [1, 1]])
|
|
@@ -136,9 +137,12 @@ naming the shapes involved; the Rust code never panics across the boundary.
|
|
|
136
137
|
|
|
137
138
|
```bash
|
|
138
139
|
pytest # CPU
|
|
139
|
-
|
|
140
|
+
FERROTENSOR_RUN_GPU_TESTS=1 pytest # also GPU parity tests
|
|
140
141
|
```
|
|
141
142
|
|
|
143
|
+
`FERROTENSOR_GPU_TESTS=1` still works as the older name. CI runs the GPU
|
|
144
|
+
tests on software Vulkan (Mesa lavapipe).
|
|
145
|
+
|
|
142
146
|
## License
|
|
143
147
|
|
|
144
148
|
MIT or Apache-2.0, at your option.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "ferrotensor"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.7.0"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
authors.workspace = true
|
|
6
6
|
license.workspace = true
|
|
@@ -8,7 +8,6 @@ repository.workspace = true
|
|
|
8
8
|
rust-version.workspace = true
|
|
9
9
|
description = "GPU-accelerated machine learning library for training and inference in Rust"
|
|
10
10
|
documentation = "https://docs.rs/ferrotensor"
|
|
11
|
-
homepage = "https://github.com/puranikyashaswin/ferrotensor"
|
|
12
11
|
readme = "README.md"
|
|
13
12
|
keywords = ["machine-learning", "deep-learning", "gpu", "neural-networks", "transformer"]
|
|
14
13
|
categories = ["science", "algorithms"]
|
|
@@ -30,3 +29,6 @@ rand = "0.8"
|
|
|
30
29
|
|
|
31
30
|
[dev-dependencies]
|
|
32
31
|
naga = { version = "0.19", features = ["wgsl-in"] }
|
|
32
|
+
|
|
33
|
+
[lints]
|
|
34
|
+
workspace = true
|
|
@@ -23,14 +23,35 @@ The low-level examples for the LLM stack live next to the crates they use:
|
|
|
23
23
|
creation, tensor upload, and individual kernels through `ComputeEngine`
|
|
24
24
|
- [`ferrotensor-graph/examples`](../../ferrotensor-graph/examples): a Llama
|
|
25
25
|
block and causal flash attention checked against a CPU reference
|
|
26
|
+
- [`ferrotensor-trainer/examples`](../../ferrotensor-trainer/examples): the
|
|
27
|
+
training stack on a tiny random model built in a temporary directory, so
|
|
28
|
+
nothing is downloaded. Each needs a GPU, checks its own results and runs
|
|
29
|
+
in CI:
|
|
30
|
+
|
|
31
|
+
| Example | What it shows |
|
|
32
|
+
|---------|---------------|
|
|
33
|
+
| `lora_finetune_tiny` | LoRA training; the loss falls, and a saved adapter reloads into a fresh model with the same loss |
|
|
34
|
+
| `resume_training` | A run stopped halfway, saved with its optimizer state and resumed, matches an unbroken run |
|
|
35
|
+
| `generate_with_kv_cache` | Greedy generation gives the same tokens with and without the KV cache |
|
|
36
|
+
|
|
37
|
+
Run one with `cargo run --release -p ferrotensor-trainer --example <name>`.
|
|
26
38
|
|
|
27
39
|
For fine-tuning a real model, see the
|
|
28
40
|
[LoRA walkthrough](../../docs/examples/lora-sft.md).
|
|
29
41
|
|
|
42
|
+
## Running every example
|
|
43
|
+
|
|
44
|
+
CI runs every example, on the CPU or on software Vulkan, and fails when one
|
|
45
|
+
fails: the examples check their own results with `assert!`. To do the same
|
|
46
|
+
locally, run `scripts/run-examples.sh` (all of them, needs a GPU adapter),
|
|
47
|
+
`scripts/run-examples.sh --cpu` or `scripts/run-examples.sh --gpu`.
|
|
48
|
+
|
|
30
49
|
## Adding an example
|
|
31
50
|
|
|
32
51
|
Create a `.rs` file in this directory; Cargo finds it. Put a doc comment at
|
|
33
|
-
the top with the command to run it and its real output
|
|
52
|
+
the top with the command to run it and its real output, check the result with
|
|
53
|
+
`assert!`, and add it to the list in `scripts/run-examples.sh` (the script
|
|
54
|
+
fails when an example is missing from it).
|
|
34
55
|
|
|
35
56
|
## Troubleshooting
|
|
36
57
|
|
|
@@ -9,6 +9,13 @@
|
|
|
9
9
|
//! Run with: cargo run --release -p ferrotensor --example char_transformer
|
|
10
10
|
//! Set FERROTENSOR_DEVICE=gpu to run on the GPU (the default is the CPU, which is
|
|
11
11
|
//! faster for a model this small).
|
|
12
|
+
//! Set FERROTENSOR_STEPS=<n> to train for n steps instead of 600 (CI uses this
|
|
13
|
+
//! to keep the run short).
|
|
14
|
+
//!
|
|
15
|
+
//! The example exits with an error unless the loss fell during training and
|
|
16
|
+
//! the cached forward pass matches the full one.
|
|
17
|
+
|
|
18
|
+
#![allow(clippy::expect_used)]
|
|
12
19
|
|
|
13
20
|
use std::collections::BTreeMap;
|
|
14
21
|
|
|
@@ -23,7 +30,7 @@ const BATCH: usize = 16;
|
|
|
23
30
|
const DIM: usize = 64;
|
|
24
31
|
const HEADS: usize = 4;
|
|
25
32
|
const LAYERS: usize = 2;
|
|
26
|
-
const
|
|
33
|
+
const DEFAULT_STEPS: usize = 600;
|
|
27
34
|
|
|
28
35
|
/// Pre-norm transformer block: `x + attn(ln(x))`, then `x + mlp(ln(x))`.
|
|
29
36
|
struct Block {
|
|
@@ -221,6 +228,15 @@ fn main() {
|
|
|
221
228
|
vocab.len()
|
|
222
229
|
);
|
|
223
230
|
|
|
231
|
+
let steps = match std::env::var("FERROTENSOR_STEPS") {
|
|
232
|
+
Ok(s) => s
|
|
233
|
+
.parse::<usize>()
|
|
234
|
+
.ok()
|
|
235
|
+
.filter(|&n| n > 0)
|
|
236
|
+
.expect("FERROTENSOR_STEPS must be a positive integer"),
|
|
237
|
+
Err(_) => DEFAULT_STEPS,
|
|
238
|
+
};
|
|
239
|
+
|
|
224
240
|
let model = CharLm::new(vocab.len(), &device);
|
|
225
241
|
let n_params: usize = model.parameters().iter().map(|p| p.numel()).sum();
|
|
226
242
|
println!("model: {LAYERS} layers, dim {DIM}, {HEADS} heads, {n_params} parameters");
|
|
@@ -228,11 +244,13 @@ fn main() {
|
|
|
228
244
|
let mut opt = optim::AdamW::new(model.parameters(), 3e-3)
|
|
229
245
|
.weight_decay(0.01)
|
|
230
246
|
.clip_grad_norm(1.0);
|
|
231
|
-
let schedule = WarmupCosineLr::new(3e-3, 3e-4, 30,
|
|
247
|
+
let schedule = WarmupCosineLr::new(3e-3, 3e-4, 30, steps);
|
|
232
248
|
let mut rng = StdRng::seed_from_u64(42);
|
|
233
249
|
|
|
234
250
|
let start = std::time::Instant::now();
|
|
235
|
-
|
|
251
|
+
let mut first_loss = f32::NAN;
|
|
252
|
+
let mut last_loss = f32::NAN;
|
|
253
|
+
for step in 0..steps {
|
|
236
254
|
schedule.apply(&mut opt, step);
|
|
237
255
|
let (mut xs, mut ys) = (Vec::new(), Vec::new());
|
|
238
256
|
for _ in 0..BATCH {
|
|
@@ -246,20 +264,33 @@ fn main() {
|
|
|
246
264
|
let loss = F::cross_entropy(&logits.reshape(&[BATCH * CONTEXT, vocab.len()]), &targets);
|
|
247
265
|
loss.backward();
|
|
248
266
|
opt.step();
|
|
249
|
-
|
|
250
|
-
|
|
267
|
+
last_loss = loss.item();
|
|
268
|
+
if step == 0 {
|
|
269
|
+
first_loss = last_loss;
|
|
270
|
+
}
|
|
271
|
+
if step % 100 == 0 || step == steps - 1 {
|
|
272
|
+
println!("step {step:>3} loss {last_loss:.3}");
|
|
251
273
|
}
|
|
252
274
|
}
|
|
253
275
|
println!(
|
|
254
|
-
"trained {
|
|
276
|
+
"trained {steps} steps in {:.1}s",
|
|
255
277
|
start.elapsed().as_secs_f32()
|
|
256
278
|
);
|
|
279
|
+
// Losses are from single random batches, so compare with a wide margin.
|
|
280
|
+
assert!(
|
|
281
|
+
last_loss < 0.8 * first_loss,
|
|
282
|
+
"the loss did not fall: {first_loss} at step 0, {last_loss} at the end"
|
|
283
|
+
);
|
|
257
284
|
|
|
258
285
|
let prompt = "Shall I compare";
|
|
259
286
|
let ids: Vec<usize> = prompt.chars().map(|c| vocab[&c]).collect();
|
|
287
|
+
let cache_error = model.cache_error(&data[..CONTEXT]);
|
|
260
288
|
println!(
|
|
261
|
-
"KV cache vs full forward on {CONTEXT} characters: max logit difference {:.1e}"
|
|
262
|
-
|
|
289
|
+
"KV cache vs full forward on {CONTEXT} characters: max logit difference {cache_error:.1e}"
|
|
290
|
+
);
|
|
291
|
+
assert!(
|
|
292
|
+
cache_error < 1e-3,
|
|
293
|
+
"the KV cache changes the logits by {cache_error}"
|
|
263
294
|
);
|
|
264
295
|
let start = std::time::Instant::now();
|
|
265
296
|
let out = model.generate(&ids, 200, 0.8, &mut rng);
|
|
@@ -6,6 +6,13 @@
|
|
|
6
6
|
//! benchmark; see `benchmarks/` for the real harness.
|
|
7
7
|
//!
|
|
8
8
|
//! Run with: cargo run --release -p ferrotensor --example gpu_matmul_attention
|
|
9
|
+
//!
|
|
10
|
+
//! The example exits with an error when the CPU and GPU results differ by more
|
|
11
|
+
//! than 1e-3. Two environment variables are for CI, where the GPU is software
|
|
12
|
+
//! Vulkan on the CPU:
|
|
13
|
+
//!
|
|
14
|
+
//! - FERROTENSOR_EXAMPLE_QUICK=1 runs only the smaller sizes.
|
|
15
|
+
//! - FERROTENSOR_REQUIRE_GPU=1 makes a missing GPU an error instead of a skip.
|
|
9
16
|
|
|
10
17
|
use std::time::Instant;
|
|
11
18
|
|
|
@@ -15,7 +22,15 @@ fn max_abs_diff(a: &[f32], b: &[f32]) -> f32 {
|
|
|
15
22
|
a.iter()
|
|
16
23
|
.zip(b)
|
|
17
24
|
.map(|(x, y)| (x - y).abs())
|
|
18
|
-
.
|
|
25
|
+
// A NaN difference must stay NaN so the check below fails on it.
|
|
26
|
+
.fold(0.0, |m, d| if d.is_nan() || d > m { d } else { m })
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/// Largest allowed difference between the CPU and GPU results.
|
|
30
|
+
const TOLERANCE: f32 = 1e-3;
|
|
31
|
+
|
|
32
|
+
fn env_set(name: &str) -> bool {
|
|
33
|
+
std::env::var(name).as_deref() == Ok("1")
|
|
19
34
|
}
|
|
20
35
|
|
|
21
36
|
/// Best-of-`runs` wall time in milliseconds, after one warm-up call.
|
|
@@ -32,9 +47,14 @@ fn time_ms(runs: usize, mut f: impl FnMut() -> Vec<f32>) -> (f64, Vec<f32>) {
|
|
|
32
47
|
|
|
33
48
|
fn main() {
|
|
34
49
|
let Some(gpu) = Device::gpu() else {
|
|
50
|
+
assert!(
|
|
51
|
+
!env_set("FERROTENSOR_REQUIRE_GPU"),
|
|
52
|
+
"FERROTENSOR_REQUIRE_GPU=1 but no GPU adapter is available"
|
|
53
|
+
);
|
|
35
54
|
println!("no GPU adapter available; nothing to compare");
|
|
36
55
|
return;
|
|
37
56
|
};
|
|
57
|
+
let quick = env_set("FERROTENSOR_EXAMPLE_QUICK");
|
|
38
58
|
let cpu = Device::cpu();
|
|
39
59
|
println!("gpu: {} ({})\n", gpu.name(), gpu.backend());
|
|
40
60
|
|
|
@@ -43,16 +63,25 @@ fn main() {
|
|
|
43
63
|
"{:>6} {:>10} {:>10} {:>9} {:>12}",
|
|
44
64
|
"n", "cpu ms", "gpu ms", "speedup", "max |diff|"
|
|
45
65
|
);
|
|
46
|
-
|
|
66
|
+
let matmul_sizes: &[usize] = if quick {
|
|
67
|
+
&[256, 512]
|
|
68
|
+
} else {
|
|
69
|
+
&[256, 512, 1024, 2048]
|
|
70
|
+
};
|
|
71
|
+
for &n in matmul_sizes {
|
|
47
72
|
let a = Tensor::randn(&[n, n], &cpu);
|
|
48
73
|
let b = Tensor::randn(&[n, n], &cpu);
|
|
49
74
|
let (ga, gb) = (a.to_device(&gpu), b.to_device(&gpu));
|
|
50
75
|
let (t_cpu, r_cpu) = time_ms(2, || a.matmul(&b).to_vec());
|
|
51
76
|
let (t_gpu, r_gpu) = time_ms(5, || ga.matmul(&gb).to_vec());
|
|
77
|
+
let diff = max_abs_diff(&r_cpu, &r_gpu);
|
|
52
78
|
println!(
|
|
53
|
-
"{n:>6} {t_cpu:>10.1} {t_gpu:>10.1} {:>8.1}x {:>12.2e}",
|
|
54
|
-
t_cpu / t_gpu
|
|
55
|
-
|
|
79
|
+
"{n:>6} {t_cpu:>10.1} {t_gpu:>10.1} {:>8.1}x {diff:>12.2e}",
|
|
80
|
+
t_cpu / t_gpu
|
|
81
|
+
);
|
|
82
|
+
assert!(
|
|
83
|
+
diff < TOLERANCE,
|
|
84
|
+
"matmul n={n}: CPU and GPU differ by {diff}"
|
|
56
85
|
);
|
|
57
86
|
}
|
|
58
87
|
|
|
@@ -61,7 +90,8 @@ fn main() {
|
|
|
61
90
|
"{:>6} {:>10} {:>10} {:>9} {:>12}",
|
|
62
91
|
"seq", "cpu ms", "gpu ms", "speedup", "max |diff|"
|
|
63
92
|
);
|
|
64
|
-
|
|
93
|
+
let attention_lengths: &[usize] = if quick { &[128] } else { &[128, 512] };
|
|
94
|
+
for &seq in attention_lengths {
|
|
65
95
|
let shape = [1, 8, seq, 64];
|
|
66
96
|
let (q, k, v) = (
|
|
67
97
|
Tensor::randn(&shape, &cpu),
|
|
@@ -75,10 +105,14 @@ fn main() {
|
|
|
75
105
|
let (t_gpu, r_gpu) = time_ms(5, || {
|
|
76
106
|
F::scaled_dot_product_attention(&gq, &gk, &gv, true).to_vec()
|
|
77
107
|
});
|
|
108
|
+
let diff = max_abs_diff(&r_cpu, &r_gpu);
|
|
78
109
|
println!(
|
|
79
|
-
"{seq:>6} {t_cpu:>10.1} {t_gpu:>10.1} {:>8.1}x {:>12.2e}",
|
|
80
|
-
t_cpu / t_gpu
|
|
81
|
-
|
|
110
|
+
"{seq:>6} {t_cpu:>10.1} {t_gpu:>10.1} {:>8.1}x {diff:>12.2e}",
|
|
111
|
+
t_cpu / t_gpu
|
|
112
|
+
);
|
|
113
|
+
assert!(
|
|
114
|
+
diff < TOLERANCE,
|
|
115
|
+
"attention seq={seq}: CPU and GPU differ by {diff}"
|
|
82
116
|
);
|
|
83
117
|
}
|
|
84
118
|
}
|
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
//!
|
|
3
3
|
//! Run with: cargo run --release -p ferrotensor --example mlp_two_moons
|
|
4
4
|
//! Set FERROTENSOR_DEVICE=gpu to train on the GPU instead of the CPU.
|
|
5
|
+
//!
|
|
6
|
+
//! The example exits with an error unless the loss fell during training and the
|
|
7
|
+
//! model classifies the held-out points well.
|
|
5
8
|
|
|
6
9
|
use ferrotensor::data::{DataLoader, Sample, TensorDataset};
|
|
7
10
|
use ferrotensor::optim::{LrScheduler, WarmupCosineLr};
|
|
@@ -74,6 +77,9 @@ fn main() {
|
|
|
74
77
|
let tx = Tensor::from_data(&flat, &[test_x.len(), 2], &device);
|
|
75
78
|
|
|
76
79
|
let mut step = 0;
|
|
80
|
+
let mut first_loss = f32::NAN;
|
|
81
|
+
let mut last_loss = f32::NAN;
|
|
82
|
+
let mut final_acc = 0.0f32;
|
|
77
83
|
for epoch in 0..epochs {
|
|
78
84
|
let mut total = 0.0;
|
|
79
85
|
for (x, y) in loader.epoch(epoch as u64) {
|
|
@@ -85,13 +91,28 @@ fn main() {
|
|
|
85
91
|
opt.step();
|
|
86
92
|
total += loss.item();
|
|
87
93
|
}
|
|
94
|
+
let mean_loss = total / loader.num_batches() as f32;
|
|
95
|
+
if epoch == 0 {
|
|
96
|
+
first_loss = mean_loss;
|
|
97
|
+
}
|
|
98
|
+
last_loss = mean_loss;
|
|
88
99
|
if epoch % 10 == 0 || epoch == epochs - 1 {
|
|
89
100
|
let acc = F::accuracy(&model.forward(&tx), &test_y);
|
|
101
|
+
final_acc = acc;
|
|
90
102
|
println!(
|
|
91
|
-
"epoch {epoch:>2} train loss {:.4} test accuracy {:.1}%",
|
|
92
|
-
total / loader.num_batches() as f32,
|
|
103
|
+
"epoch {epoch:>2} train loss {mean_loss:.4} test accuracy {:.1}%",
|
|
93
104
|
acc * 100.0
|
|
94
105
|
);
|
|
95
106
|
}
|
|
96
107
|
}
|
|
108
|
+
|
|
109
|
+
assert!(
|
|
110
|
+
last_loss < first_loss,
|
|
111
|
+
"the loss did not fall: {first_loss} at epoch 0, {last_loss} at the end"
|
|
112
|
+
);
|
|
113
|
+
assert!(
|
|
114
|
+
final_acc > 0.9,
|
|
115
|
+
"test accuracy is only {:.1}%",
|
|
116
|
+
final_acc * 100.0
|
|
117
|
+
);
|
|
97
118
|
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
//! Quickstart: a tensor, a matmul, a gradient.
|
|
2
|
+
//!
|
|
3
|
+
//! Run with: cargo run --release -p ferrotensor --example quickstart
|
|
4
|
+
|
|
5
|
+
#![allow(clippy::expect_used)]
|
|
6
|
+
|
|
7
|
+
use ferrotensor::prelude::*;
|
|
8
|
+
|
|
9
|
+
/// Panics unless `got` matches `want` element by element.
|
|
10
|
+
fn assert_close(got: &[f32], want: &[f32]) {
|
|
11
|
+
assert_eq!(got.len(), want.len(), "got {got:?}, expected {want:?}");
|
|
12
|
+
for (g, w) in got.iter().zip(want) {
|
|
13
|
+
assert!((g - w).abs() < 1e-4, "got {got:?}, expected {want:?}");
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
fn main() {
|
|
18
|
+
// A GPU (Metal/Vulkan/DX12) when one is available, else the CPU.
|
|
19
|
+
let device = Device::best();
|
|
20
|
+
println!("device: {} ({})", device.name(), device.backend());
|
|
21
|
+
|
|
22
|
+
// y = (x @ w).sum(), so dy/dw = x^T @ ones.
|
|
23
|
+
let x = Tensor::from_data(&[1., 2., 3., 4., 5., 6.], &[2, 3], &device);
|
|
24
|
+
let w = Tensor::from_data(&[0.5, -1., 2., 0., 1., 1.], &[3, 2], &device).requires_grad_();
|
|
25
|
+
|
|
26
|
+
let y = x.matmul(&w);
|
|
27
|
+
let product = y.to_vec();
|
|
28
|
+
println!("x @ w = {:?} (shape {:?})", product, y.shape);
|
|
29
|
+
assert_close(&product, &[7.5, 2., 18., 2.]);
|
|
30
|
+
|
|
31
|
+
y.sum().backward();
|
|
32
|
+
let grad = w.grad().expect("w requires grad").to_vec();
|
|
33
|
+
println!("dL/dw = {:?}", grad);
|
|
34
|
+
// Row k of dL/dw is the sum of column k of x, repeated for both outputs.
|
|
35
|
+
assert_close(&grad, &[5., 5., 7., 7., 9., 9.]);
|
|
36
|
+
}
|