DLcomm 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dlcomm-0.1.0/DLcomm.egg-info/PKG-INFO +121 -0
- dlcomm-0.1.0/DLcomm.egg-info/SOURCES.txt +47 -0
- dlcomm-0.1.0/DLcomm.egg-info/dependency_links.txt +1 -0
- dlcomm-0.1.0/DLcomm.egg-info/entry_points.txt +2 -0
- dlcomm-0.1.0/DLcomm.egg-info/requires.txt +15 -0
- dlcomm-0.1.0/DLcomm.egg-info/top_level.txt +1 -0
- dlcomm-0.1.0/PKG-INFO +121 -0
- dlcomm-0.1.0/README.md +79 -0
- dlcomm-0.1.0/dl_comm/__init__.py +0 -0
- dlcomm-0.1.0/dl_comm/analysis/__init__.py +16 -0
- dlcomm-0.1.0/dl_comm/analysis/bandwidth.py +151 -0
- dlcomm-0.1.0/dl_comm/analysis/ccl_parser.py +38 -0
- dlcomm-0.1.0/dl_comm/analysis/correctness.py +217 -0
- dlcomm-0.1.0/dl_comm/analysis/correctness_new.py +320 -0
- dlcomm-0.1.0/dl_comm/comm/__init__.py +8 -0
- dlcomm-0.1.0/dl_comm/comm/collectives.py +138 -0
- dlcomm-0.1.0/dl_comm/comm/comm_setup.py +204 -0
- dlcomm-0.1.0/dl_comm/config/__init__.py +7 -0
- dlcomm-0.1.0/dl_comm/config/config.yaml +90 -0
- dlcomm-0.1.0/dl_comm/config/config_old.yaml +43 -0
- dlcomm-0.1.0/dl_comm/config/config_spec.json +31 -0
- dlcomm-0.1.0/dl_comm/config/validation.py +312 -0
- dlcomm-0.1.0/dl_comm/dl_comm_main.py +493 -0
- dlcomm-0.1.0/dl_comm/timer/__init__.py +1 -0
- dlcomm-0.1.0/dl_comm/timer/timer.py +214 -0
- dlcomm-0.1.0/dl_comm/utils/__init__.py +0 -0
- dlcomm-0.1.0/dl_comm/utils/utility.py +120 -0
- dlcomm-0.1.0/pyproject.toml +66 -0
- dlcomm-0.1.0/setup.cfg +4 -0
- dlcomm-0.1.0/setup.py +41 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: DLcomm
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Distributed GPU Communication Benchmarking Framework for Deep Learning
|
|
5
|
+
Author: Musa Cim
|
|
6
|
+
Author-email: Musa Cim <musaoguzhancim@gmail.com>, Kaushik Velusamy <kaushikvelusamy@gmail.com>
|
|
7
|
+
Project-URL: Homepage, https://github.com/argonne-lcf/DLcomm_benchmark
|
|
8
|
+
Project-URL: Documentation, https://github.com/argonne-lcf/DLcomm_benchmark
|
|
9
|
+
Project-URL: Repository, https://github.com/argonne-lcf/DLcomm_benchmark
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/argonne-lcf/DLcomm_benchmark/issues
|
|
11
|
+
Keywords: deep-learning,gpu,communication,benchmark,mpi,distributed,pytorch,ccl
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
23
|
+
Classifier: Topic :: System :: Benchmark
|
|
24
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
25
|
+
Requires-Python: >=3.8
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
Requires-Dist: torch>=1.12.0
|
|
28
|
+
Requires-Dist: mpi4py>=3.1.0
|
|
29
|
+
Requires-Dist: hydra-core>=1.1.0
|
|
30
|
+
Requires-Dist: omegaconf>=2.1.0
|
|
31
|
+
Requires-Dist: pytz>=2021.1
|
|
32
|
+
Provides-Extra: intel
|
|
33
|
+
Requires-Dist: intel-extension-for-pytorch; extra == "intel"
|
|
34
|
+
Requires-Dist: oneccl-bindings-for-pytorch; extra == "intel"
|
|
35
|
+
Provides-Extra: dev
|
|
36
|
+
Requires-Dist: pytest>=6.0; extra == "dev"
|
|
37
|
+
Requires-Dist: black>=22.0; extra == "dev"
|
|
38
|
+
Requires-Dist: flake8>=4.0; extra == "dev"
|
|
39
|
+
Requires-Dist: mypy>=0.950; extra == "dev"
|
|
40
|
+
Dynamic: author
|
|
41
|
+
Dynamic: requires-python
|
|
42
|
+
|
|
43
|
+
# Deep Learning Communication (DLcomm) Benchmark
|
|
44
|
+
|
|
45
|
+
This README provides an abbreviated documentation of the DL_COMM_code. Please refer to ... for full user documentation.
|
|
46
|
+
|
|
47
|
+
## Overview
|
|
48
|
+
|
|
49
|
+
DL COMM is a lightweight benchmark for testing common communication patterns in large‐scale deep‐learning ( all‐reduce, broadcast, all‐gather). You run it from a single executable and configure everything with a simple YAML file. It’s modular—so adding new frameworks, back-ends, or algorithms is easy. DL COMM reports per‐iteration latency and bandwidth.
|
|
50
|
+
|
|
51
|
+
update - x axis is num_gpus_per_node and y axis is num_compute_nodes
|
|
52
|
+
|
|
53
|
+

|
|
54
|
+
|
|
55
|
+
## Installation and running DLCOMM
|
|
56
|
+
|
|
57
|
+
pip install -r requirements.txt
|
|
58
|
+
|
|
59
|
+
pip install DLcomm
|
|
60
|
+
|
|
61
|
+
## Running the benchmark
|
|
62
|
+
|
|
63
|
+
## YAML configuration file
|
|
64
|
+
|
|
65
|
+
Workload characteristics for DL COMM are specified by a YAML configuration file. Below is an example of a YAML file for a DL COMM run that executes a PyTorch+XCCL ring all-reduce across 4 nodes with 8 GPUs each, sending a 1 MB float32 buffer for 10 iterations:
|
|
66
|
+
|
|
67
|
+
```yaml
|
|
68
|
+
# contents of dl_comm_run.yaml
|
|
69
|
+
framework. : pytorch # tensorflow / jax / titan / monarch
|
|
70
|
+
ccl_backend : xccl # rccl / nccl
|
|
71
|
+
use_profiler: unitrace
|
|
72
|
+
|
|
73
|
+
collective:
|
|
74
|
+
name: allreduce # allgather / reducescatter / broadcast
|
|
75
|
+
op: prod # max / min
|
|
76
|
+
scale_up_algorithm: topo
|
|
77
|
+
scale_out_algorithm: ring # rabinseifner
|
|
78
|
+
iterations: 5
|
|
79
|
+
payload:
|
|
80
|
+
dtype: float32 # float64 / int32 / int64 / bfloat16 / float8
|
|
81
|
+
count: 1024
|
|
82
|
+
buffer_size: 4096 # in Bytes -> float32(4B) x 1024 elements
|
|
83
|
+
|
|
84
|
+
verify_correctness: on
|
|
85
|
+
|
|
86
|
+
comm_group:
|
|
87
|
+
mode: combined # within_node/across_node/combined/flatview -> Only one out of four should be used
|
|
88
|
+
|
|
89
|
+
flatview: off
|
|
90
|
+
|
|
91
|
+
within_node:
|
|
92
|
+
num_compute_nodes: 2
|
|
93
|
+
num_gpus_per_node: 4
|
|
94
|
+
gpu_ids_per_node: [8, 9, 10, 11]
|
|
95
|
+
|
|
96
|
+
across_node:
|
|
97
|
+
num_compute_nodes: 2
|
|
98
|
+
num_gpus_per_node: 2
|
|
99
|
+
gpu_ids_per_node: [0,1]
|
|
100
|
+
|
|
101
|
+
combined:
|
|
102
|
+
within_node:
|
|
103
|
+
num_compute_nodes: 2
|
|
104
|
+
num_gpus_per_node: 12
|
|
105
|
+
gpu_ids_per_node: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
|
|
106
|
+
across_node:
|
|
107
|
+
num_compute_nodes: 2
|
|
108
|
+
num_gpus_per_node: 2
|
|
109
|
+
gpu_ids_per_node: [5, 7]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## How to contribute
|
|
116
|
+
|
|
117
|
+
## Citation and Reference
|
|
118
|
+
|
|
119
|
+
## Acknowledgments
|
|
120
|
+
|
|
121
|
+
## License
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
setup.py
|
|
4
|
+
./dl_comm/__init__.py
|
|
5
|
+
./dl_comm/dl_comm_main.py
|
|
6
|
+
./dl_comm/analysis/__init__.py
|
|
7
|
+
./dl_comm/analysis/bandwidth.py
|
|
8
|
+
./dl_comm/analysis/ccl_parser.py
|
|
9
|
+
./dl_comm/analysis/correctness.py
|
|
10
|
+
./dl_comm/analysis/correctness_new.py
|
|
11
|
+
./dl_comm/comm/__init__.py
|
|
12
|
+
./dl_comm/comm/collectives.py
|
|
13
|
+
./dl_comm/comm/comm_setup.py
|
|
14
|
+
./dl_comm/config/__init__.py
|
|
15
|
+
./dl_comm/config/config.yaml
|
|
16
|
+
./dl_comm/config/config_old.yaml
|
|
17
|
+
./dl_comm/config/config_spec.json
|
|
18
|
+
./dl_comm/config/validation.py
|
|
19
|
+
./dl_comm/timer/__init__.py
|
|
20
|
+
./dl_comm/timer/timer.py
|
|
21
|
+
./dl_comm/utils/__init__.py
|
|
22
|
+
./dl_comm/utils/utility.py
|
|
23
|
+
DLcomm.egg-info/PKG-INFO
|
|
24
|
+
DLcomm.egg-info/SOURCES.txt
|
|
25
|
+
DLcomm.egg-info/dependency_links.txt
|
|
26
|
+
DLcomm.egg-info/entry_points.txt
|
|
27
|
+
DLcomm.egg-info/requires.txt
|
|
28
|
+
DLcomm.egg-info/top_level.txt
|
|
29
|
+
dl_comm/__init__.py
|
|
30
|
+
dl_comm/dl_comm_main.py
|
|
31
|
+
dl_comm/analysis/__init__.py
|
|
32
|
+
dl_comm/analysis/bandwidth.py
|
|
33
|
+
dl_comm/analysis/ccl_parser.py
|
|
34
|
+
dl_comm/analysis/correctness.py
|
|
35
|
+
dl_comm/analysis/correctness_new.py
|
|
36
|
+
dl_comm/comm/__init__.py
|
|
37
|
+
dl_comm/comm/collectives.py
|
|
38
|
+
dl_comm/comm/comm_setup.py
|
|
39
|
+
dl_comm/config/__init__.py
|
|
40
|
+
dl_comm/config/config.yaml
|
|
41
|
+
dl_comm/config/config_old.yaml
|
|
42
|
+
dl_comm/config/config_spec.json
|
|
43
|
+
dl_comm/config/validation.py
|
|
44
|
+
dl_comm/timer/__init__.py
|
|
45
|
+
dl_comm/timer/timer.py
|
|
46
|
+
dl_comm/utils/__init__.py
|
|
47
|
+
dl_comm/utils/utility.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dl_comm
|
dlcomm-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: DLcomm
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Distributed GPU Communication Benchmarking Framework for Deep Learning
|
|
5
|
+
Author: Musa Cim
|
|
6
|
+
Author-email: Musa Cim <musaoguzhancim@gmail.com>, Kaushik Velusamy <kaushikvelusamy@gmail.com>
|
|
7
|
+
Project-URL: Homepage, https://github.com/argonne-lcf/DLcomm_benchmark
|
|
8
|
+
Project-URL: Documentation, https://github.com/argonne-lcf/DLcomm_benchmark
|
|
9
|
+
Project-URL: Repository, https://github.com/argonne-lcf/DLcomm_benchmark
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/argonne-lcf/DLcomm_benchmark/issues
|
|
11
|
+
Keywords: deep-learning,gpu,communication,benchmark,mpi,distributed,pytorch,ccl
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
23
|
+
Classifier: Topic :: System :: Benchmark
|
|
24
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
25
|
+
Requires-Python: >=3.8
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
Requires-Dist: torch>=1.12.0
|
|
28
|
+
Requires-Dist: mpi4py>=3.1.0
|
|
29
|
+
Requires-Dist: hydra-core>=1.1.0
|
|
30
|
+
Requires-Dist: omegaconf>=2.1.0
|
|
31
|
+
Requires-Dist: pytz>=2021.1
|
|
32
|
+
Provides-Extra: intel
|
|
33
|
+
Requires-Dist: intel-extension-for-pytorch; extra == "intel"
|
|
34
|
+
Requires-Dist: oneccl-bindings-for-pytorch; extra == "intel"
|
|
35
|
+
Provides-Extra: dev
|
|
36
|
+
Requires-Dist: pytest>=6.0; extra == "dev"
|
|
37
|
+
Requires-Dist: black>=22.0; extra == "dev"
|
|
38
|
+
Requires-Dist: flake8>=4.0; extra == "dev"
|
|
39
|
+
Requires-Dist: mypy>=0.950; extra == "dev"
|
|
40
|
+
Dynamic: author
|
|
41
|
+
Dynamic: requires-python
|
|
42
|
+
|
|
43
|
+
# Deep Learning Communication (DLcomm) Benchmark
|
|
44
|
+
|
|
45
|
+
This README provides an abbreviated documentation of the DL_COMM_code. Please refer to ... for full user documentation.
|
|
46
|
+
|
|
47
|
+
## Overview
|
|
48
|
+
|
|
49
|
+
DL COMM is a lightweight benchmark for testing common communication patterns in large‐scale deep‐learning ( all‐reduce, broadcast, all‐gather). You run it from a single executable and configure everything with a simple YAML file. It’s modular—so adding new frameworks, back-ends, or algorithms is easy. DL COMM reports per‐iteration latency and bandwidth.
|
|
50
|
+
|
|
51
|
+
update - x axis is num_gpus_per_node and y axis is num_compute_nodes
|
|
52
|
+
|
|
53
|
+

|
|
54
|
+
|
|
55
|
+
## Installation and running DLCOMM
|
|
56
|
+
|
|
57
|
+
pip install -r requirements.txt
|
|
58
|
+
|
|
59
|
+
pip install DLcomm
|
|
60
|
+
|
|
61
|
+
## Running the benchmark
|
|
62
|
+
|
|
63
|
+
## YAML configuration file
|
|
64
|
+
|
|
65
|
+
Workload characteristics for DL COMM are specified by a YAML configuration file. Below is an example of a YAML file for a DL COMM run that executes a PyTorch+XCCL ring all-reduce across 4 nodes with 8 GPUs each, sending a 1 MB float32 buffer for 10 iterations:
|
|
66
|
+
|
|
67
|
+
```yaml
|
|
68
|
+
# contents of dl_comm_run.yaml
|
|
69
|
+
framework. : pytorch # tensorflow / jax / titan / monarch
|
|
70
|
+
ccl_backend : xccl # rccl / nccl
|
|
71
|
+
use_profiler: unitrace
|
|
72
|
+
|
|
73
|
+
collective:
|
|
74
|
+
name: allreduce # allgather / reducescatter / broadcast
|
|
75
|
+
op: prod # max / min
|
|
76
|
+
scale_up_algorithm: topo
|
|
77
|
+
scale_out_algorithm: ring # rabinseifner
|
|
78
|
+
iterations: 5
|
|
79
|
+
payload:
|
|
80
|
+
dtype: float32 # float64 / int32 / int64 / bfloat16 / float8
|
|
81
|
+
count: 1024
|
|
82
|
+
buffer_size: 4096 # in Bytes -> float32(4B) x 1024 elements
|
|
83
|
+
|
|
84
|
+
verify_correctness: on
|
|
85
|
+
|
|
86
|
+
comm_group:
|
|
87
|
+
mode: combined # within_node/across_node/combined/flatview -> Only one out of four should be used
|
|
88
|
+
|
|
89
|
+
flatview: off
|
|
90
|
+
|
|
91
|
+
within_node:
|
|
92
|
+
num_compute_nodes: 2
|
|
93
|
+
num_gpus_per_node: 4
|
|
94
|
+
gpu_ids_per_node: [8, 9, 10, 11]
|
|
95
|
+
|
|
96
|
+
across_node:
|
|
97
|
+
num_compute_nodes: 2
|
|
98
|
+
num_gpus_per_node: 2
|
|
99
|
+
gpu_ids_per_node: [0,1]
|
|
100
|
+
|
|
101
|
+
combined:
|
|
102
|
+
within_node:
|
|
103
|
+
num_compute_nodes: 2
|
|
104
|
+
num_gpus_per_node: 12
|
|
105
|
+
gpu_ids_per_node: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
|
|
106
|
+
across_node:
|
|
107
|
+
num_compute_nodes: 2
|
|
108
|
+
num_gpus_per_node: 2
|
|
109
|
+
gpu_ids_per_node: [5, 7]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## How to contribute
|
|
116
|
+
|
|
117
|
+
## Citation and Reference
|
|
118
|
+
|
|
119
|
+
## Acknowledgments
|
|
120
|
+
|
|
121
|
+
## License
|
dlcomm-0.1.0/README.md
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# Deep Learning Communication (DLcomm) Benchmark
|
|
2
|
+
|
|
3
|
+
This README provides an abbreviated documentation of the DL_COMM_code. Please refer to ... for full user documentation.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
DL COMM is a lightweight benchmark for testing common communication patterns in large‐scale deep‐learning ( all‐reduce, broadcast, all‐gather). You run it from a single executable and configure everything with a simple YAML file. It’s modular—so adding new frameworks, back-ends, or algorithms is easy. DL COMM reports per‐iteration latency and bandwidth.
|
|
8
|
+
|
|
9
|
+
update - x axis is num_gpus_per_node and y axis is num_compute_nodes
|
|
10
|
+
|
|
11
|
+

|
|
12
|
+
|
|
13
|
+
## Installation and running DLCOMM
|
|
14
|
+
|
|
15
|
+
pip install -r requirements.txt
|
|
16
|
+
|
|
17
|
+
pip install DLcomm
|
|
18
|
+
|
|
19
|
+
## Running the benchmark
|
|
20
|
+
|
|
21
|
+
## YAML configuration file
|
|
22
|
+
|
|
23
|
+
Workload characteristics for DL COMM are specified by a YAML configuration file. Below is an example of a YAML file for a DL COMM run that executes a PyTorch+XCCL ring all-reduce across 4 nodes with 8 GPUs each, sending a 1 MB float32 buffer for 10 iterations:
|
|
24
|
+
|
|
25
|
+
```yaml
|
|
26
|
+
# contents of dl_comm_run.yaml
|
|
27
|
+
framework. : pytorch # tensorflow / jax / titan / monarch
|
|
28
|
+
ccl_backend : xccl # rccl / nccl
|
|
29
|
+
use_profiler: unitrace
|
|
30
|
+
|
|
31
|
+
collective:
|
|
32
|
+
name: allreduce # allgather / reducescatter / broadcast
|
|
33
|
+
op: prod # max / min
|
|
34
|
+
scale_up_algorithm: topo
|
|
35
|
+
scale_out_algorithm: ring # rabinseifner
|
|
36
|
+
iterations: 5
|
|
37
|
+
payload:
|
|
38
|
+
dtype: float32 # float64 / int32 / int64 / bfloat16 / float8
|
|
39
|
+
count: 1024
|
|
40
|
+
buffer_size: 4096 # in Bytes -> float32(4B) x 1024 elements
|
|
41
|
+
|
|
42
|
+
verify_correctness: on
|
|
43
|
+
|
|
44
|
+
comm_group:
|
|
45
|
+
mode: combined # within_node/across_node/combined/flatview -> Only one out of four should be used
|
|
46
|
+
|
|
47
|
+
flatview: off
|
|
48
|
+
|
|
49
|
+
within_node:
|
|
50
|
+
num_compute_nodes: 2
|
|
51
|
+
num_gpus_per_node: 4
|
|
52
|
+
gpu_ids_per_node: [8, 9, 10, 11]
|
|
53
|
+
|
|
54
|
+
across_node:
|
|
55
|
+
num_compute_nodes: 2
|
|
56
|
+
num_gpus_per_node: 2
|
|
57
|
+
gpu_ids_per_node: [0,1]
|
|
58
|
+
|
|
59
|
+
combined:
|
|
60
|
+
within_node:
|
|
61
|
+
num_compute_nodes: 2
|
|
62
|
+
num_gpus_per_node: 12
|
|
63
|
+
gpu_ids_per_node: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
|
|
64
|
+
across_node:
|
|
65
|
+
num_compute_nodes: 2
|
|
66
|
+
num_gpus_per_node: 2
|
|
67
|
+
gpu_ids_per_node: [5, 7]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## How to contribute
|
|
74
|
+
|
|
75
|
+
## Citation and Reference
|
|
76
|
+
|
|
77
|
+
## Acknowledgments
|
|
78
|
+
|
|
79
|
+
## License
|
|
File without changes
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Analysis package for DL-COMM benchmarking results.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .ccl_parser import parse_ccl_selection, report_ccl_selection
|
|
6
|
+
from .bandwidth import print_all_bandwidths
|
|
7
|
+
from .correctness import check_group_correctness
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
'parse_ccl_selection',
|
|
11
|
+
'report_ccl_selection',
|
|
12
|
+
'bytes_per_rank',
|
|
13
|
+
'bytes_per_coll',
|
|
14
|
+
'print_all_bandwidths',
|
|
15
|
+
'check_group_correctness'
|
|
16
|
+
]
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from dl_comm.config import parse_buffer_size
|
|
3
|
+
from dl_comm.timer.timer import TIMES
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def calculate_group_bandwidth(group_size, buffer_size, time_seconds):
|
|
7
|
+
total_bytes = group_size * buffer_size
|
|
8
|
+
bandwidth_bytes_per_sec = total_bytes / time_seconds
|
|
9
|
+
return bandwidth_bytes_per_sec
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def print_all_bandwidths(logger, cfg, mpi_size, ranks_responsible_for_logging):
|
|
13
|
+
from mpi4py import MPI
|
|
14
|
+
|
|
15
|
+
mpi_rank = MPI.COMM_WORLD.Get_rank()
|
|
16
|
+
|
|
17
|
+
my_data = None
|
|
18
|
+
if mpi_rank in ranks_responsible_for_logging:
|
|
19
|
+
my_data = {
|
|
20
|
+
'rank': mpi_rank,
|
|
21
|
+
'timers': dict(TIMES)
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
all_data = MPI.COMM_WORLD.gather(my_data, root=0)
|
|
25
|
+
|
|
26
|
+
if mpi_rank == 0:
|
|
27
|
+
logger.output("")
|
|
28
|
+
logger.output("[BANDWIDTH] -------------------------------------------")
|
|
29
|
+
|
|
30
|
+
comm_mode = cfg.comm_group.mode
|
|
31
|
+
group_bandwidths = {}
|
|
32
|
+
|
|
33
|
+
buffer_configs = {}
|
|
34
|
+
if comm_mode == "flatview":
|
|
35
|
+
coll_cfg = cfg.comm_group.flatview.collective
|
|
36
|
+
buffer_in_bytes = parse_buffer_size(coll_cfg.payload.buffer_size)
|
|
37
|
+
buffer_configs['flatview'] = buffer_in_bytes
|
|
38
|
+
elif comm_mode == "within_node":
|
|
39
|
+
coll_cfg = cfg.comm_group.within_node.collective
|
|
40
|
+
buffer_in_bytes = parse_buffer_size(coll_cfg.payload.buffer_size)
|
|
41
|
+
buffer_configs['within'] = buffer_in_bytes
|
|
42
|
+
elif comm_mode == "across_node":
|
|
43
|
+
coll_cfg = cfg.comm_group.across_node.collective
|
|
44
|
+
buffer_in_bytes = parse_buffer_size(coll_cfg.payload.buffer_size)
|
|
45
|
+
buffer_configs['across'] = buffer_in_bytes
|
|
46
|
+
elif comm_mode == "combined":
|
|
47
|
+
coll_within_cfg = cfg.comm_group.combined.within_node.collective
|
|
48
|
+
coll_across_cfg = cfg.comm_group.combined.across_node.collective
|
|
49
|
+
buffer_within_bytes = parse_buffer_size(coll_within_cfg.payload.buffer_size)
|
|
50
|
+
buffer_across_bytes = parse_buffer_size(coll_across_cfg.payload.buffer_size)
|
|
51
|
+
buffer_configs['within'] = buffer_within_bytes
|
|
52
|
+
buffer_configs['across'] = buffer_across_bytes
|
|
53
|
+
|
|
54
|
+
group_timers = {}
|
|
55
|
+
|
|
56
|
+
for data in all_data:
|
|
57
|
+
if data is not None:
|
|
58
|
+
rank = data['rank']
|
|
59
|
+
timers = data['timers']
|
|
60
|
+
|
|
61
|
+
for label, vals in timers.items():
|
|
62
|
+
if "(flatview)" == label.lower():
|
|
63
|
+
group_key = "flatview"
|
|
64
|
+
elif "(within-group-" in label.lower():
|
|
65
|
+
match = re.search(r'\(within-group-(\d+)\)', label.lower())
|
|
66
|
+
group_key = f"within-{match.group(1)}"
|
|
67
|
+
elif "(across-group-" in label.lower():
|
|
68
|
+
match = re.search(r'\(across-group-(\d+)\)', label.lower())
|
|
69
|
+
group_key = f"across-{match.group(1)}"
|
|
70
|
+
else:
|
|
71
|
+
continue
|
|
72
|
+
|
|
73
|
+
if group_key not in group_timers:
|
|
74
|
+
group_timers[group_key] = {}
|
|
75
|
+
|
|
76
|
+
if label not in group_timers[group_key] or rank < group_timers[group_key][label]['rank']:
|
|
77
|
+
group_timers[group_key][label] = {
|
|
78
|
+
'vals': vals,
|
|
79
|
+
'rank': rank
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
for group_key, labels in group_timers.items():
|
|
83
|
+
for label, timer_data in labels.items():
|
|
84
|
+
vals = timer_data['vals']
|
|
85
|
+
rank = timer_data['rank']
|
|
86
|
+
first_iteration_time = vals[0]
|
|
87
|
+
|
|
88
|
+
if group_key == "flatview":
|
|
89
|
+
world_size = mpi_size
|
|
90
|
+
buffer_size = buffer_configs.get('flatview', 0)
|
|
91
|
+
bandwidth = calculate_group_bandwidth(world_size, buffer_size, first_iteration_time)
|
|
92
|
+
group_bandwidths['Flatview'] = {
|
|
93
|
+
'bandwidth': bandwidth,
|
|
94
|
+
'group_size': world_size,
|
|
95
|
+
'buffer_size': buffer_size,
|
|
96
|
+
'time': first_iteration_time,
|
|
97
|
+
'rank': rank
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
elif group_key.startswith("within-"):
|
|
101
|
+
group_id = group_key.split("-")[1]
|
|
102
|
+
within_mode_cfg = cfg.comm_group.combined.within_node if comm_mode == "combined" else cfg.comm_group.within_node
|
|
103
|
+
group_size = within_mode_cfg.num_gpus_per_node
|
|
104
|
+
buffer_size = buffer_configs.get('within', 0)
|
|
105
|
+
bandwidth = calculate_group_bandwidth(group_size, buffer_size, first_iteration_time)
|
|
106
|
+
group_bandwidths[f'Within-Group-{group_id}'] = {
|
|
107
|
+
'bandwidth': bandwidth,
|
|
108
|
+
'group_size': group_size,
|
|
109
|
+
'buffer_size': buffer_size,
|
|
110
|
+
'time': first_iteration_time,
|
|
111
|
+
'rank': rank
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
elif group_key.startswith("across-"):
|
|
115
|
+
group_id = group_key.split("-")[1]
|
|
116
|
+
across_mode_cfg = cfg.comm_group.combined.across_node if comm_mode == "combined" else cfg.comm_group.across_node
|
|
117
|
+
group_size = across_mode_cfg.num_compute_nodes
|
|
118
|
+
buffer_size = buffer_configs.get('across', 0)
|
|
119
|
+
bandwidth = calculate_group_bandwidth(group_size, buffer_size, first_iteration_time)
|
|
120
|
+
group_bandwidths[f'Across-Group-{group_id}'] = {
|
|
121
|
+
'bandwidth': bandwidth,
|
|
122
|
+
'group_size': group_size,
|
|
123
|
+
'buffer_size': buffer_size,
|
|
124
|
+
'time': first_iteration_time,
|
|
125
|
+
'rank': rank
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
logger.output("[BANDWIDTH] Communication Group Bandwidths:")
|
|
129
|
+
logger.output("")
|
|
130
|
+
|
|
131
|
+
for group_name, data in group_bandwidths.items():
|
|
132
|
+
logger.output(f"[BANDWIDTH] {group_name}:")
|
|
133
|
+
logger.output(f"[BANDWIDTH] Group Size : {data['group_size']} GPUs")
|
|
134
|
+
logger.output(f"[BANDWIDTH] Buffer Size : {data['buffer_size']} bytes")
|
|
135
|
+
logger.output(f"[BANDWIDTH] Time (iter 0) : {data['time']:.6f} s")
|
|
136
|
+
logger.output(f"[BANDWIDTH] Bandwidth : {data['bandwidth']:.0f} bytes/s")
|
|
137
|
+
logger.output(f"[BANDWIDTH] Logging Rank : {data['rank']}")
|
|
138
|
+
logger.output("")
|
|
139
|
+
|
|
140
|
+
logger.output("[BANDWIDTH] -------------------------------------------")
|
|
141
|
+
logger.output("")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def parse_ccl_selection(log_path: str, algo_name: str):
|
|
6
|
+
sel = {}
|
|
7
|
+
start_re = re.compile(rf"{re.escape(algo_name)} selection", re.IGNORECASE)
|
|
8
|
+
table_re = re.compile(r'^\s*([a-z ]+table)\s*$', re.IGNORECASE)
|
|
9
|
+
choice_re = re.compile(r'^\s*\[.*?\]\s*:\s*(\S+)\s*$', re.IGNORECASE)
|
|
10
|
+
with open(log_path) as f:
|
|
11
|
+
lines = f.readlines()
|
|
12
|
+
for idx, L in enumerate(lines):
|
|
13
|
+
if start_re.search(L):
|
|
14
|
+
break
|
|
15
|
+
else:
|
|
16
|
+
return sel
|
|
17
|
+
current_table = None
|
|
18
|
+
for L in lines[idx+1:]:
|
|
19
|
+
if re.match(r'^\d{4}:\d{2}.*\|CCL_', L):
|
|
20
|
+
break
|
|
21
|
+
m_table = table_re.match(L)
|
|
22
|
+
m_choice = choice_re.match(L)
|
|
23
|
+
if m_table:
|
|
24
|
+
current_table = m_table.group(1).strip()
|
|
25
|
+
elif m_choice and current_table:
|
|
26
|
+
sel[current_table] = m_choice.group(1).strip()
|
|
27
|
+
return sel
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def report_ccl_selection(log_path: str, algo_name: str, logger):
|
|
31
|
+
|
|
32
|
+
selection = parse_ccl_selection(log_path, algo_name)
|
|
33
|
+
if not selection:
|
|
34
|
+
logger.info(f"No '{algo_name} selection' block found in {log_path}")
|
|
35
|
+
else:
|
|
36
|
+
logger.info(f"[SELECTION] {algo_name} table selection:")
|
|
37
|
+
for tbl, impl in selection.items():
|
|
38
|
+
logger.info(f"[SELECTION] {tbl:15s} → {impl}")
|