DLcomm 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dl_comm/__init__.py +0 -0
- dl_comm/analysis/__init__.py +16 -0
- dl_comm/analysis/bandwidth.py +151 -0
- dl_comm/analysis/ccl_parser.py +38 -0
- dl_comm/analysis/correctness.py +217 -0
- dl_comm/analysis/correctness_new.py +320 -0
- dl_comm/comm/__init__.py +8 -0
- dl_comm/comm/collectives.py +138 -0
- dl_comm/comm/comm_setup.py +204 -0
- dl_comm/config/__init__.py +7 -0
- dl_comm/config/config.yaml +90 -0
- dl_comm/config/config_old.yaml +43 -0
- dl_comm/config/config_spec.json +31 -0
- dl_comm/config/validation.py +312 -0
- dl_comm/dl_comm_main.py +493 -0
- dl_comm/timer/__init__.py +1 -0
- dl_comm/timer/timer.py +214 -0
- dl_comm/utils/__init__.py +0 -0
- dl_comm/utils/utility.py +120 -0
- dlcomm-0.1.0.dist-info/METADATA +121 -0
- dlcomm-0.1.0.dist-info/RECORD +24 -0
- dlcomm-0.1.0.dist-info/WHEEL +5 -0
- dlcomm-0.1.0.dist-info/entry_points.txt +2 -0
- dlcomm-0.1.0.dist-info/top_level.txt +1 -0
dl_comm/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Analysis package for DL-COMM benchmarking results.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .ccl_parser import parse_ccl_selection, report_ccl_selection
|
|
6
|
+
from .bandwidth import print_all_bandwidths
|
|
7
|
+
from .correctness import check_group_correctness
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
'parse_ccl_selection',
|
|
11
|
+
'report_ccl_selection',
|
|
12
|
+
'bytes_per_rank',
|
|
13
|
+
'bytes_per_coll',
|
|
14
|
+
'print_all_bandwidths',
|
|
15
|
+
'check_group_correctness'
|
|
16
|
+
]
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from dl_comm.config import parse_buffer_size
|
|
3
|
+
from dl_comm.timer.timer import TIMES
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def calculate_group_bandwidth(group_size, buffer_size, time_seconds):
|
|
7
|
+
total_bytes = group_size * buffer_size
|
|
8
|
+
bandwidth_bytes_per_sec = total_bytes / time_seconds
|
|
9
|
+
return bandwidth_bytes_per_sec
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def print_all_bandwidths(logger, cfg, mpi_size, ranks_responsible_for_logging):
|
|
13
|
+
from mpi4py import MPI
|
|
14
|
+
|
|
15
|
+
mpi_rank = MPI.COMM_WORLD.Get_rank()
|
|
16
|
+
|
|
17
|
+
my_data = None
|
|
18
|
+
if mpi_rank in ranks_responsible_for_logging:
|
|
19
|
+
my_data = {
|
|
20
|
+
'rank': mpi_rank,
|
|
21
|
+
'timers': dict(TIMES)
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
all_data = MPI.COMM_WORLD.gather(my_data, root=0)
|
|
25
|
+
|
|
26
|
+
if mpi_rank == 0:
|
|
27
|
+
logger.output("")
|
|
28
|
+
logger.output("[BANDWIDTH] -------------------------------------------")
|
|
29
|
+
|
|
30
|
+
comm_mode = cfg.comm_group.mode
|
|
31
|
+
group_bandwidths = {}
|
|
32
|
+
|
|
33
|
+
buffer_configs = {}
|
|
34
|
+
if comm_mode == "flatview":
|
|
35
|
+
coll_cfg = cfg.comm_group.flatview.collective
|
|
36
|
+
buffer_in_bytes = parse_buffer_size(coll_cfg.payload.buffer_size)
|
|
37
|
+
buffer_configs['flatview'] = buffer_in_bytes
|
|
38
|
+
elif comm_mode == "within_node":
|
|
39
|
+
coll_cfg = cfg.comm_group.within_node.collective
|
|
40
|
+
buffer_in_bytes = parse_buffer_size(coll_cfg.payload.buffer_size)
|
|
41
|
+
buffer_configs['within'] = buffer_in_bytes
|
|
42
|
+
elif comm_mode == "across_node":
|
|
43
|
+
coll_cfg = cfg.comm_group.across_node.collective
|
|
44
|
+
buffer_in_bytes = parse_buffer_size(coll_cfg.payload.buffer_size)
|
|
45
|
+
buffer_configs['across'] = buffer_in_bytes
|
|
46
|
+
elif comm_mode == "combined":
|
|
47
|
+
coll_within_cfg = cfg.comm_group.combined.within_node.collective
|
|
48
|
+
coll_across_cfg = cfg.comm_group.combined.across_node.collective
|
|
49
|
+
buffer_within_bytes = parse_buffer_size(coll_within_cfg.payload.buffer_size)
|
|
50
|
+
buffer_across_bytes = parse_buffer_size(coll_across_cfg.payload.buffer_size)
|
|
51
|
+
buffer_configs['within'] = buffer_within_bytes
|
|
52
|
+
buffer_configs['across'] = buffer_across_bytes
|
|
53
|
+
|
|
54
|
+
group_timers = {}
|
|
55
|
+
|
|
56
|
+
for data in all_data:
|
|
57
|
+
if data is not None:
|
|
58
|
+
rank = data['rank']
|
|
59
|
+
timers = data['timers']
|
|
60
|
+
|
|
61
|
+
for label, vals in timers.items():
|
|
62
|
+
if "(flatview)" == label.lower():
|
|
63
|
+
group_key = "flatview"
|
|
64
|
+
elif "(within-group-" in label.lower():
|
|
65
|
+
match = re.search(r'\(within-group-(\d+)\)', label.lower())
|
|
66
|
+
group_key = f"within-{match.group(1)}"
|
|
67
|
+
elif "(across-group-" in label.lower():
|
|
68
|
+
match = re.search(r'\(across-group-(\d+)\)', label.lower())
|
|
69
|
+
group_key = f"across-{match.group(1)}"
|
|
70
|
+
else:
|
|
71
|
+
continue
|
|
72
|
+
|
|
73
|
+
if group_key not in group_timers:
|
|
74
|
+
group_timers[group_key] = {}
|
|
75
|
+
|
|
76
|
+
if label not in group_timers[group_key] or rank < group_timers[group_key][label]['rank']:
|
|
77
|
+
group_timers[group_key][label] = {
|
|
78
|
+
'vals': vals,
|
|
79
|
+
'rank': rank
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
for group_key, labels in group_timers.items():
|
|
83
|
+
for label, timer_data in labels.items():
|
|
84
|
+
vals = timer_data['vals']
|
|
85
|
+
rank = timer_data['rank']
|
|
86
|
+
first_iteration_time = vals[0]
|
|
87
|
+
|
|
88
|
+
if group_key == "flatview":
|
|
89
|
+
world_size = mpi_size
|
|
90
|
+
buffer_size = buffer_configs.get('flatview', 0)
|
|
91
|
+
bandwidth = calculate_group_bandwidth(world_size, buffer_size, first_iteration_time)
|
|
92
|
+
group_bandwidths['Flatview'] = {
|
|
93
|
+
'bandwidth': bandwidth,
|
|
94
|
+
'group_size': world_size,
|
|
95
|
+
'buffer_size': buffer_size,
|
|
96
|
+
'time': first_iteration_time,
|
|
97
|
+
'rank': rank
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
elif group_key.startswith("within-"):
|
|
101
|
+
group_id = group_key.split("-")[1]
|
|
102
|
+
within_mode_cfg = cfg.comm_group.combined.within_node if comm_mode == "combined" else cfg.comm_group.within_node
|
|
103
|
+
group_size = within_mode_cfg.num_gpus_per_node
|
|
104
|
+
buffer_size = buffer_configs.get('within', 0)
|
|
105
|
+
bandwidth = calculate_group_bandwidth(group_size, buffer_size, first_iteration_time)
|
|
106
|
+
group_bandwidths[f'Within-Group-{group_id}'] = {
|
|
107
|
+
'bandwidth': bandwidth,
|
|
108
|
+
'group_size': group_size,
|
|
109
|
+
'buffer_size': buffer_size,
|
|
110
|
+
'time': first_iteration_time,
|
|
111
|
+
'rank': rank
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
elif group_key.startswith("across-"):
|
|
115
|
+
group_id = group_key.split("-")[1]
|
|
116
|
+
across_mode_cfg = cfg.comm_group.combined.across_node if comm_mode == "combined" else cfg.comm_group.across_node
|
|
117
|
+
group_size = across_mode_cfg.num_compute_nodes
|
|
118
|
+
buffer_size = buffer_configs.get('across', 0)
|
|
119
|
+
bandwidth = calculate_group_bandwidth(group_size, buffer_size, first_iteration_time)
|
|
120
|
+
group_bandwidths[f'Across-Group-{group_id}'] = {
|
|
121
|
+
'bandwidth': bandwidth,
|
|
122
|
+
'group_size': group_size,
|
|
123
|
+
'buffer_size': buffer_size,
|
|
124
|
+
'time': first_iteration_time,
|
|
125
|
+
'rank': rank
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
logger.output("[BANDWIDTH] Communication Group Bandwidths:")
|
|
129
|
+
logger.output("")
|
|
130
|
+
|
|
131
|
+
for group_name, data in group_bandwidths.items():
|
|
132
|
+
logger.output(f"[BANDWIDTH] {group_name}:")
|
|
133
|
+
logger.output(f"[BANDWIDTH] Group Size : {data['group_size']} GPUs")
|
|
134
|
+
logger.output(f"[BANDWIDTH] Buffer Size : {data['buffer_size']} bytes")
|
|
135
|
+
logger.output(f"[BANDWIDTH] Time (iter 0) : {data['time']:.6f} s")
|
|
136
|
+
logger.output(f"[BANDWIDTH] Bandwidth : {data['bandwidth']:.0f} bytes/s")
|
|
137
|
+
logger.output(f"[BANDWIDTH] Logging Rank : {data['rank']}")
|
|
138
|
+
logger.output("")
|
|
139
|
+
|
|
140
|
+
logger.output("[BANDWIDTH] -------------------------------------------")
|
|
141
|
+
logger.output("")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def parse_ccl_selection(log_path: str, algo_name: str):
|
|
6
|
+
sel = {}
|
|
7
|
+
start_re = re.compile(rf"{re.escape(algo_name)} selection", re.IGNORECASE)
|
|
8
|
+
table_re = re.compile(r'^\s*([a-z ]+table)\s*$', re.IGNORECASE)
|
|
9
|
+
choice_re = re.compile(r'^\s*\[.*?\]\s*:\s*(\S+)\s*$', re.IGNORECASE)
|
|
10
|
+
with open(log_path) as f:
|
|
11
|
+
lines = f.readlines()
|
|
12
|
+
for idx, L in enumerate(lines):
|
|
13
|
+
if start_re.search(L):
|
|
14
|
+
break
|
|
15
|
+
else:
|
|
16
|
+
return sel
|
|
17
|
+
current_table = None
|
|
18
|
+
for L in lines[idx+1:]:
|
|
19
|
+
if re.match(r'^\d{4}:\d{2}.*\|CCL_', L):
|
|
20
|
+
break
|
|
21
|
+
m_table = table_re.match(L)
|
|
22
|
+
m_choice = choice_re.match(L)
|
|
23
|
+
if m_table:
|
|
24
|
+
current_table = m_table.group(1).strip()
|
|
25
|
+
elif m_choice and current_table:
|
|
26
|
+
sel[current_table] = m_choice.group(1).strip()
|
|
27
|
+
return sel
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def report_ccl_selection(log_path: str, algo_name: str, logger):
|
|
31
|
+
|
|
32
|
+
selection = parse_ccl_selection(log_path, algo_name)
|
|
33
|
+
if not selection:
|
|
34
|
+
logger.info(f"No '{algo_name} selection' block found in {log_path}")
|
|
35
|
+
else:
|
|
36
|
+
logger.info(f"[SELECTION] {algo_name} table selection:")
|
|
37
|
+
for tbl, impl in selection.items():
|
|
38
|
+
logger.info(f"[SELECTION] {tbl:15s} → {impl}")
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
from mpi4py import MPI
|
|
2
|
+
|
|
3
|
+
_before_values = {}
|
|
4
|
+
|
|
5
|
+
def check_group_correctness(context, x, group_type, phase, tensor_list=None, collective_name=None):
|
|
6
|
+
|
|
7
|
+
# Get verify_correctness setting based on mode and group type
|
|
8
|
+
cfg = context['cfg']
|
|
9
|
+
comm_mode = cfg.comm_group.mode
|
|
10
|
+
|
|
11
|
+
# Determine if correctness checking is enabled for this group type
|
|
12
|
+
verify_correctness = False
|
|
13
|
+
if comm_mode == "flatview" and group_type == "flatview":
|
|
14
|
+
verify_correctness = cfg.comm_group.flatview.verify_correctness
|
|
15
|
+
elif comm_mode == "within_node" and group_type == "within":
|
|
16
|
+
verify_correctness = cfg.comm_group.within_node.verify_correctness
|
|
17
|
+
elif comm_mode == "across_node" and group_type == "across":
|
|
18
|
+
verify_correctness = cfg.comm_group.across_node.verify_correctness
|
|
19
|
+
elif comm_mode == "combined":
|
|
20
|
+
if group_type == "within":
|
|
21
|
+
verify_correctness = cfg.comm_group.combined.within_node.verify_correctness
|
|
22
|
+
elif group_type == "across":
|
|
23
|
+
verify_correctness = cfg.comm_group.combined.across_node.verify_correctness
|
|
24
|
+
|
|
25
|
+
if context['iteration'] != 0 or not verify_correctness:
|
|
26
|
+
return
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
mpi_rank = context['mpi_rank']
|
|
30
|
+
log = context['log']
|
|
31
|
+
|
|
32
|
+
group_rank_id = None
|
|
33
|
+
should_log = False
|
|
34
|
+
|
|
35
|
+
if comm_mode == "within_node" and group_type == "within":
|
|
36
|
+
node_id = mpi_rank // cfg.comm_group.within_node.num_gpus_per_node
|
|
37
|
+
rank_id_per_node = mpi_rank % cfg.comm_group.within_node.num_gpus_per_node
|
|
38
|
+
if rank_id_per_node == 0:
|
|
39
|
+
group_rank_id = node_id
|
|
40
|
+
should_log = True
|
|
41
|
+
|
|
42
|
+
elif comm_mode == "across_node" and group_type == "across":
|
|
43
|
+
rank_id_per_node = mpi_rank % cfg.comm_group.across_node.num_gpus_per_node
|
|
44
|
+
node_id = mpi_rank // cfg.comm_group.across_node.num_gpus_per_node
|
|
45
|
+
if node_id == 0:
|
|
46
|
+
group_rank_id = rank_id_per_node
|
|
47
|
+
should_log = True
|
|
48
|
+
|
|
49
|
+
elif comm_mode == "combined" and group_type == "within":
|
|
50
|
+
node_id = mpi_rank // cfg.comm_group.combined.within_node.num_gpus_per_node
|
|
51
|
+
rank_id_per_node = mpi_rank % cfg.comm_group.combined.within_node.num_gpus_per_node
|
|
52
|
+
if rank_id_per_node == 0:
|
|
53
|
+
group_rank_id = node_id
|
|
54
|
+
should_log = True
|
|
55
|
+
|
|
56
|
+
elif comm_mode == "combined" and group_type == "across":
|
|
57
|
+
rank_id_per_node = mpi_rank % cfg.comm_group.combined.across_node.num_gpus_per_node
|
|
58
|
+
node_id = mpi_rank // cfg.comm_group.combined.across_node.num_gpus_per_node
|
|
59
|
+
if node_id == 0:
|
|
60
|
+
group_rank_id = rank_id_per_node
|
|
61
|
+
should_log = True
|
|
62
|
+
|
|
63
|
+
elif comm_mode == "flatview" and group_type == "flatview":
|
|
64
|
+
if mpi_rank == 0:
|
|
65
|
+
group_rank_id = "All"
|
|
66
|
+
should_log = True
|
|
67
|
+
|
|
68
|
+
# Special handling for broadcast - allow all ranks to log their results
|
|
69
|
+
if tensor_list is not None and isinstance(tensor_list, dict) and 'type' in tensor_list and collective_name == "broadcast":
|
|
70
|
+
# For broadcast, we want to see all ranks, not just designated logging ranks
|
|
71
|
+
if comm_mode == "within_node" and group_type == "within":
|
|
72
|
+
node_id = mpi_rank // cfg.comm_group.within_node.num_gpus_per_node
|
|
73
|
+
group_label = f"{group_type.title()}-Group-{node_id}"
|
|
74
|
+
elif comm_mode == "across_node" and group_type == "across":
|
|
75
|
+
rank_id_per_node = mpi_rank % cfg.comm_group.across_node.num_gpus_per_node
|
|
76
|
+
group_label = f"{group_type.title()}-Group-{rank_id_per_node}"
|
|
77
|
+
elif comm_mode == "combined" and group_type == "within":
|
|
78
|
+
node_id = mpi_rank // cfg.comm_group.combined.within_node.num_gpus_per_node
|
|
79
|
+
group_label = f"{group_type.title()}-Group-{node_id}"
|
|
80
|
+
elif comm_mode == "combined" and group_type == "across":
|
|
81
|
+
rank_id_per_node = mpi_rank % cfg.comm_group.combined.across_node.num_gpus_per_node
|
|
82
|
+
group_label = f"{group_type.title()}-Group-{rank_id_per_node}"
|
|
83
|
+
elif comm_mode == "flatview" and group_type == "flatview":
|
|
84
|
+
group_label = f"{group_type.title()}-Group-All"
|
|
85
|
+
|
|
86
|
+
# Log broadcast results for all ranks
|
|
87
|
+
if tensor_list['type'] == 'source':
|
|
88
|
+
sent_sum = tensor_list['sent_sum']
|
|
89
|
+
global_rank = tensor_list['global_rank']
|
|
90
|
+
log.output(f"[CORRECTNESS][{group_label}] Broadcast Source Rank {global_rank}: sent tensor sum = {sent_sum}")
|
|
91
|
+
elif tensor_list['type'] == 'receiver':
|
|
92
|
+
received_sum = tensor_list['received_sum']
|
|
93
|
+
global_rank = tensor_list['global_rank']
|
|
94
|
+
source_rank = tensor_list['source_rank']
|
|
95
|
+
log.output(f"[CORRECTNESS][{group_label}] Broadcast Receiver Rank {global_rank}: received from rank {source_rank}, tensor sum = {received_sum}")
|
|
96
|
+
|
|
97
|
+
return # Exit early for broadcast special handling
|
|
98
|
+
|
|
99
|
+
if should_log and group_rank_id is not None:
|
|
100
|
+
group_label = f"{group_type.title()}-Group-{group_rank_id}"
|
|
101
|
+
tensor_sum = float(x.sum())
|
|
102
|
+
|
|
103
|
+
if phase == "before":
|
|
104
|
+
|
|
105
|
+
_before_values[group_label] = tensor_sum
|
|
106
|
+
|
|
107
|
+
elif phase == "after":
|
|
108
|
+
|
|
109
|
+
if group_label in _before_values:
|
|
110
|
+
before_value = _before_values[group_label]
|
|
111
|
+
after_value = tensor_sum
|
|
112
|
+
|
|
113
|
+
log.output(f"[CORRECTNESS][{group_label}] Tensor sum before collective: {before_value} → after collective: {after_value}")
|
|
114
|
+
|
|
115
|
+
# Check list-based correctness for collectives that return data structures
|
|
116
|
+
if tensor_list is not None and phase == "after":
|
|
117
|
+
if isinstance(tensor_list, list):
|
|
118
|
+
# List validation for AllGather or Gather
|
|
119
|
+
collective_label = collective_name.title() if collective_name else "List-based collective"
|
|
120
|
+
log.output(f"[CORRECTNESS][{group_label}] {collective_label} validation:")
|
|
121
|
+
log.output(f"[CORRECTNESS][{group_label}] - List size: {len(tensor_list)} tensors")
|
|
122
|
+
for i, tensor in enumerate(tensor_list):
|
|
123
|
+
tensor_sum = float(tensor.sum())
|
|
124
|
+
log.output(f"[CORRECTNESS][{group_label}] - Tensor[{i}] sum: {tensor_sum}")
|
|
125
|
+
elif isinstance(tensor_list, dict):
|
|
126
|
+
if 'type' in tensor_list and tensor_list['type'] == 'source' and 'sent_tensor' in tensor_list:
|
|
127
|
+
# Broadcast validation - source rank
|
|
128
|
+
log.output(f"[CORRECTNESS][{group_label}] Broadcast validation (Source):")
|
|
129
|
+
global_rank = tensor_list['global_rank']
|
|
130
|
+
group_rank = tensor_list['group_rank']
|
|
131
|
+
source_rank = tensor_list['source_rank']
|
|
132
|
+
sent_tensor = tensor_list['sent_tensor']
|
|
133
|
+
sent_sum = tensor_list['sent_sum']
|
|
134
|
+
log.output(f"[CORRECTNESS][{group_label}] - Source Global Rank {global_rank} (Group Rank {group_rank})")
|
|
135
|
+
log.output(f"[CORRECTNESS][{group_label}] - Sent tensor: {sent_tensor.flatten()[:10].tolist()}... (showing first 10 elements)")
|
|
136
|
+
log.output(f"[CORRECTNESS][{group_label}] - Sent tensor sum: {sent_sum}")
|
|
137
|
+
elif 'type' in tensor_list and tensor_list['type'] == 'receiver' and 'received_tensor' in tensor_list:
|
|
138
|
+
# Broadcast validation - receiver rank
|
|
139
|
+
log.output(f"[CORRECTNESS][{group_label}] Broadcast validation (Receiver):")
|
|
140
|
+
global_rank = tensor_list['global_rank']
|
|
141
|
+
group_rank = tensor_list['group_rank']
|
|
142
|
+
source_rank = tensor_list['source_rank']
|
|
143
|
+
received_tensor = tensor_list['received_tensor']
|
|
144
|
+
received_sum = tensor_list['received_sum']
|
|
145
|
+
log.output(f"[CORRECTNESS][{group_label}] - Receiver Global Rank {global_rank} (Group Rank {group_rank})")
|
|
146
|
+
log.output(f"[CORRECTNESS][{group_label}] - Source was Global Rank {source_rank}")
|
|
147
|
+
log.output(f"[CORRECTNESS][{group_label}] - Received tensor: {received_tensor.flatten()[:10].tolist()}... (showing first 10 elements)")
|
|
148
|
+
log.output(f"[CORRECTNESS][{group_label}] - Received tensor sum: {received_sum}")
|
|
149
|
+
elif 'type' in tensor_list and tensor_list['type'] == 'source' and 'scattered_data' in tensor_list:
|
|
150
|
+
# Scatter validation - source rank
|
|
151
|
+
log.output(f"[CORRECTNESS][{group_label}] Scatter validation (Source):")
|
|
152
|
+
source_rank = tensor_list['source_global_rank']
|
|
153
|
+
scattered_data = tensor_list['scattered_data']
|
|
154
|
+
log.output(f"[CORRECTNESS][{group_label}] - Source Global Rank: {source_rank}")
|
|
155
|
+
log.output(f"[CORRECTNESS][{group_label}] - Scattered data:")
|
|
156
|
+
for data in scattered_data:
|
|
157
|
+
to_rank = data['to_group_rank']
|
|
158
|
+
value = data['value']
|
|
159
|
+
tensor_sum = data['tensor_sum']
|
|
160
|
+
log.output(f"[CORRECTNESS][{group_label}] → To Group Rank {to_rank}: tensor filled with {value}, sum = {tensor_sum}")
|
|
161
|
+
elif 'type' in tensor_list and tensor_list['type'] == 'receiver' and 'receiver_global_rank' in tensor_list:
|
|
162
|
+
# Scatter validation - receiver rank
|
|
163
|
+
log.output(f"[CORRECTNESS][{group_label}] Scatter validation (Receiver):")
|
|
164
|
+
global_rank = tensor_list['receiver_global_rank']
|
|
165
|
+
group_rank = tensor_list['receiver_group_rank']
|
|
166
|
+
expected = tensor_list['expected_value']
|
|
167
|
+
received_sum = tensor_list['received_tensor_sum']
|
|
168
|
+
is_correct = tensor_list['is_correct']
|
|
169
|
+
status = "✓ CORRECT" if is_correct else "✗ INCORRECT"
|
|
170
|
+
log.output(f"[CORRECTNESS][{group_label}] - Global Rank {global_rank} (Group Rank {group_rank})")
|
|
171
|
+
log.output(f"[CORRECTNESS][{group_label}] - Expected: tensor filled with {expected}")
|
|
172
|
+
log.output(f"[CORRECTNESS][{group_label}] - Received sum: {received_sum} {status}")
|
|
173
|
+
elif 'sent_data' in tensor_list and 'received_data' in tensor_list:
|
|
174
|
+
# AllToAll validation
|
|
175
|
+
log.output(f"[CORRECTNESS][{group_label}] AllToAll validation:")
|
|
176
|
+
global_rank = tensor_list['global_rank']
|
|
177
|
+
group_rank = tensor_list['group_rank']
|
|
178
|
+
sent_data = tensor_list['sent_data']
|
|
179
|
+
received_data = tensor_list['received_data']
|
|
180
|
+
|
|
181
|
+
log.output(f"[CORRECTNESS][{group_label}] - Global Rank {global_rank} (Group Rank {group_rank})")
|
|
182
|
+
log.output(f"[CORRECTNESS][{group_label}] - Sent data:")
|
|
183
|
+
for data in sent_data:
|
|
184
|
+
to_rank = data['to_group_rank']
|
|
185
|
+
sent_val = data['sent_value']
|
|
186
|
+
chunk_sum = data['chunk_sum']
|
|
187
|
+
log.output(f"[CORRECTNESS][{group_label}] → To Group Rank {to_rank}: value {sent_val}, sum = {chunk_sum}")
|
|
188
|
+
|
|
189
|
+
log.output(f"[CORRECTNESS][{group_label}] - Received data:")
|
|
190
|
+
all_correct = True
|
|
191
|
+
for data in received_data:
|
|
192
|
+
from_rank = data['from_group_rank']
|
|
193
|
+
expected_val = data['expected_value']
|
|
194
|
+
expected_sum = data['expected_sum']
|
|
195
|
+
received_sum = data['received_sum']
|
|
196
|
+
is_correct = data['is_correct']
|
|
197
|
+
status = "✓" if is_correct else "✗"
|
|
198
|
+
if not is_correct:
|
|
199
|
+
all_correct = False
|
|
200
|
+
log.output(f"[CORRECTNESS][{group_label}] ← From Group Rank {from_rank}: expected {expected_val} (sum={expected_sum}), got sum={received_sum} {status}")
|
|
201
|
+
|
|
202
|
+
overall_status = "✓ ALL CORRECT" if all_correct else "✗ SOME INCORRECT"
|
|
203
|
+
log.output(f"[CORRECTNESS][{group_label}] - Overall: {overall_status}")
|
|
204
|
+
else:
|
|
205
|
+
# ReduceScatter validation
|
|
206
|
+
log.output(f"[CORRECTNESS][{group_label}] ReduceScatter validation:")
|
|
207
|
+
global_rank = tensor_list['global_rank']
|
|
208
|
+
group_rank = tensor_list['group_rank']
|
|
209
|
+
my_chunk = tensor_list['my_chunk_index']
|
|
210
|
+
expected_value = tensor_list['expected_value']
|
|
211
|
+
log.output(f"[CORRECTNESS][{group_label}] - Global Rank {global_rank} (Group Rank {group_rank}) contributed: ones to all chunks")
|
|
212
|
+
log.output(f"[CORRECTNESS][{group_label}] - Global Rank {global_rank} received chunk index: {my_chunk}")
|
|
213
|
+
log.output(f"[CORRECTNESS][{group_label}] - Expected: each element = {expected_value} (sum of {int(expected_value)} ones)")
|
|
214
|
+
log.output(f"[CORRECTNESS][{group_label}] - Actual result sum: {after_value}")
|
|
215
|
+
|
|
216
|
+
del _before_values[group_label]
|
|
217
|
+
|