DLcomm 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dl_comm/__init__.py ADDED
File without changes
@@ -0,0 +1,16 @@
1
+ """
2
+ Analysis package for DL-COMM benchmarking results.
3
+ """
4
+
5
+ from .ccl_parser import parse_ccl_selection, report_ccl_selection
6
+ from .bandwidth import print_all_bandwidths
7
+ from .correctness import check_group_correctness
8
+
9
+ __all__ = [
10
+ 'parse_ccl_selection',
11
+ 'report_ccl_selection',
12
+ 'bytes_per_rank',
13
+ 'bytes_per_coll',
14
+ 'print_all_bandwidths',
15
+ 'check_group_correctness'
16
+ ]
@@ -0,0 +1,151 @@
1
+ import re
2
+ from dl_comm.config import parse_buffer_size
3
+ from dl_comm.timer.timer import TIMES
4
+
5
+
6
+ def calculate_group_bandwidth(group_size, buffer_size, time_seconds):
7
+ total_bytes = group_size * buffer_size
8
+ bandwidth_bytes_per_sec = total_bytes / time_seconds
9
+ return bandwidth_bytes_per_sec
10
+
11
+
12
+ def print_all_bandwidths(logger, cfg, mpi_size, ranks_responsible_for_logging):
13
+ from mpi4py import MPI
14
+
15
+ mpi_rank = MPI.COMM_WORLD.Get_rank()
16
+
17
+ my_data = None
18
+ if mpi_rank in ranks_responsible_for_logging:
19
+ my_data = {
20
+ 'rank': mpi_rank,
21
+ 'timers': dict(TIMES)
22
+ }
23
+
24
+ all_data = MPI.COMM_WORLD.gather(my_data, root=0)
25
+
26
+ if mpi_rank == 0:
27
+ logger.output("")
28
+ logger.output("[BANDWIDTH] -------------------------------------------")
29
+
30
+ comm_mode = cfg.comm_group.mode
31
+ group_bandwidths = {}
32
+
33
+ buffer_configs = {}
34
+ if comm_mode == "flatview":
35
+ coll_cfg = cfg.comm_group.flatview.collective
36
+ buffer_in_bytes = parse_buffer_size(coll_cfg.payload.buffer_size)
37
+ buffer_configs['flatview'] = buffer_in_bytes
38
+ elif comm_mode == "within_node":
39
+ coll_cfg = cfg.comm_group.within_node.collective
40
+ buffer_in_bytes = parse_buffer_size(coll_cfg.payload.buffer_size)
41
+ buffer_configs['within'] = buffer_in_bytes
42
+ elif comm_mode == "across_node":
43
+ coll_cfg = cfg.comm_group.across_node.collective
44
+ buffer_in_bytes = parse_buffer_size(coll_cfg.payload.buffer_size)
45
+ buffer_configs['across'] = buffer_in_bytes
46
+ elif comm_mode == "combined":
47
+ coll_within_cfg = cfg.comm_group.combined.within_node.collective
48
+ coll_across_cfg = cfg.comm_group.combined.across_node.collective
49
+ buffer_within_bytes = parse_buffer_size(coll_within_cfg.payload.buffer_size)
50
+ buffer_across_bytes = parse_buffer_size(coll_across_cfg.payload.buffer_size)
51
+ buffer_configs['within'] = buffer_within_bytes
52
+ buffer_configs['across'] = buffer_across_bytes
53
+
54
+ group_timers = {}
55
+
56
+ for data in all_data:
57
+ if data is not None:
58
+ rank = data['rank']
59
+ timers = data['timers']
60
+
61
+ for label, vals in timers.items():
62
+ if "(flatview)" == label.lower():
63
+ group_key = "flatview"
64
+ elif "(within-group-" in label.lower():
65
+ match = re.search(r'\(within-group-(\d+)\)', label.lower())
66
+ group_key = f"within-{match.group(1)}"
67
+ elif "(across-group-" in label.lower():
68
+ match = re.search(r'\(across-group-(\d+)\)', label.lower())
69
+ group_key = f"across-{match.group(1)}"
70
+ else:
71
+ continue
72
+
73
+ if group_key not in group_timers:
74
+ group_timers[group_key] = {}
75
+
76
+ if label not in group_timers[group_key] or rank < group_timers[group_key][label]['rank']:
77
+ group_timers[group_key][label] = {
78
+ 'vals': vals,
79
+ 'rank': rank
80
+ }
81
+
82
+ for group_key, labels in group_timers.items():
83
+ for label, timer_data in labels.items():
84
+ vals = timer_data['vals']
85
+ rank = timer_data['rank']
86
+ first_iteration_time = vals[0]
87
+
88
+ if group_key == "flatview":
89
+ world_size = mpi_size
90
+ buffer_size = buffer_configs.get('flatview', 0)
91
+ bandwidth = calculate_group_bandwidth(world_size, buffer_size, first_iteration_time)
92
+ group_bandwidths['Flatview'] = {
93
+ 'bandwidth': bandwidth,
94
+ 'group_size': world_size,
95
+ 'buffer_size': buffer_size,
96
+ 'time': first_iteration_time,
97
+ 'rank': rank
98
+ }
99
+
100
+ elif group_key.startswith("within-"):
101
+ group_id = group_key.split("-")[1]
102
+ within_mode_cfg = cfg.comm_group.combined.within_node if comm_mode == "combined" else cfg.comm_group.within_node
103
+ group_size = within_mode_cfg.num_gpus_per_node
104
+ buffer_size = buffer_configs.get('within', 0)
105
+ bandwidth = calculate_group_bandwidth(group_size, buffer_size, first_iteration_time)
106
+ group_bandwidths[f'Within-Group-{group_id}'] = {
107
+ 'bandwidth': bandwidth,
108
+ 'group_size': group_size,
109
+ 'buffer_size': buffer_size,
110
+ 'time': first_iteration_time,
111
+ 'rank': rank
112
+ }
113
+
114
+ elif group_key.startswith("across-"):
115
+ group_id = group_key.split("-")[1]
116
+ across_mode_cfg = cfg.comm_group.combined.across_node if comm_mode == "combined" else cfg.comm_group.across_node
117
+ group_size = across_mode_cfg.num_compute_nodes
118
+ buffer_size = buffer_configs.get('across', 0)
119
+ bandwidth = calculate_group_bandwidth(group_size, buffer_size, first_iteration_time)
120
+ group_bandwidths[f'Across-Group-{group_id}'] = {
121
+ 'bandwidth': bandwidth,
122
+ 'group_size': group_size,
123
+ 'buffer_size': buffer_size,
124
+ 'time': first_iteration_time,
125
+ 'rank': rank
126
+ }
127
+
128
+ logger.output("[BANDWIDTH] Communication Group Bandwidths:")
129
+ logger.output("")
130
+
131
+ for group_name, data in group_bandwidths.items():
132
+ logger.output(f"[BANDWIDTH] {group_name}:")
133
+ logger.output(f"[BANDWIDTH] Group Size : {data['group_size']} GPUs")
134
+ logger.output(f"[BANDWIDTH] Buffer Size : {data['buffer_size']} bytes")
135
+ logger.output(f"[BANDWIDTH] Time (iter 0) : {data['time']:.6f} s")
136
+ logger.output(f"[BANDWIDTH] Bandwidth : {data['bandwidth']:.0f} bytes/s")
137
+ logger.output(f"[BANDWIDTH] Logging Rank : {data['rank']}")
138
+ logger.output("")
139
+
140
+ logger.output("[BANDWIDTH] -------------------------------------------")
141
+ logger.output("")
142
+
143
+
144
+
145
+
146
+
147
+
148
+
149
+
150
+
151
+
@@ -0,0 +1,38 @@
1
+
2
+ import re
3
+
4
+
5
+ def parse_ccl_selection(log_path: str, algo_name: str):
6
+ sel = {}
7
+ start_re = re.compile(rf"{re.escape(algo_name)} selection", re.IGNORECASE)
8
+ table_re = re.compile(r'^\s*([a-z ]+table)\s*$', re.IGNORECASE)
9
+ choice_re = re.compile(r'^\s*\[.*?\]\s*:\s*(\S+)\s*$', re.IGNORECASE)
10
+ with open(log_path) as f:
11
+ lines = f.readlines()
12
+ for idx, L in enumerate(lines):
13
+ if start_re.search(L):
14
+ break
15
+ else:
16
+ return sel
17
+ current_table = None
18
+ for L in lines[idx+1:]:
19
+ if re.match(r'^\d{4}:\d{2}.*\|CCL_', L):
20
+ break
21
+ m_table = table_re.match(L)
22
+ m_choice = choice_re.match(L)
23
+ if m_table:
24
+ current_table = m_table.group(1).strip()
25
+ elif m_choice and current_table:
26
+ sel[current_table] = m_choice.group(1).strip()
27
+ return sel
28
+
29
+
30
+ def report_ccl_selection(log_path: str, algo_name: str, logger):
31
+
32
+ selection = parse_ccl_selection(log_path, algo_name)
33
+ if not selection:
34
+ logger.info(f"No '{algo_name} selection' block found in {log_path}")
35
+ else:
36
+ logger.info(f"[SELECTION] {algo_name} table selection:")
37
+ for tbl, impl in selection.items():
38
+ logger.info(f"[SELECTION] {tbl:15s} → {impl}")
@@ -0,0 +1,217 @@
1
+ from mpi4py import MPI
2
+
3
+ _before_values = {}
4
+
5
+ def check_group_correctness(context, x, group_type, phase, tensor_list=None, collective_name=None):
6
+
7
+ # Get verify_correctness setting based on mode and group type
8
+ cfg = context['cfg']
9
+ comm_mode = cfg.comm_group.mode
10
+
11
+ # Determine if correctness checking is enabled for this group type
12
+ verify_correctness = False
13
+ if comm_mode == "flatview" and group_type == "flatview":
14
+ verify_correctness = cfg.comm_group.flatview.verify_correctness
15
+ elif comm_mode == "within_node" and group_type == "within":
16
+ verify_correctness = cfg.comm_group.within_node.verify_correctness
17
+ elif comm_mode == "across_node" and group_type == "across":
18
+ verify_correctness = cfg.comm_group.across_node.verify_correctness
19
+ elif comm_mode == "combined":
20
+ if group_type == "within":
21
+ verify_correctness = cfg.comm_group.combined.within_node.verify_correctness
22
+ elif group_type == "across":
23
+ verify_correctness = cfg.comm_group.combined.across_node.verify_correctness
24
+
25
+ if context['iteration'] != 0 or not verify_correctness:
26
+ return
27
+
28
+
29
+ mpi_rank = context['mpi_rank']
30
+ log = context['log']
31
+
32
+ group_rank_id = None
33
+ should_log = False
34
+
35
+ if comm_mode == "within_node" and group_type == "within":
36
+ node_id = mpi_rank // cfg.comm_group.within_node.num_gpus_per_node
37
+ rank_id_per_node = mpi_rank % cfg.comm_group.within_node.num_gpus_per_node
38
+ if rank_id_per_node == 0:
39
+ group_rank_id = node_id
40
+ should_log = True
41
+
42
+ elif comm_mode == "across_node" and group_type == "across":
43
+ rank_id_per_node = mpi_rank % cfg.comm_group.across_node.num_gpus_per_node
44
+ node_id = mpi_rank // cfg.comm_group.across_node.num_gpus_per_node
45
+ if node_id == 0:
46
+ group_rank_id = rank_id_per_node
47
+ should_log = True
48
+
49
+ elif comm_mode == "combined" and group_type == "within":
50
+ node_id = mpi_rank // cfg.comm_group.combined.within_node.num_gpus_per_node
51
+ rank_id_per_node = mpi_rank % cfg.comm_group.combined.within_node.num_gpus_per_node
52
+ if rank_id_per_node == 0:
53
+ group_rank_id = node_id
54
+ should_log = True
55
+
56
+ elif comm_mode == "combined" and group_type == "across":
57
+ rank_id_per_node = mpi_rank % cfg.comm_group.combined.across_node.num_gpus_per_node
58
+ node_id = mpi_rank // cfg.comm_group.combined.across_node.num_gpus_per_node
59
+ if node_id == 0:
60
+ group_rank_id = rank_id_per_node
61
+ should_log = True
62
+
63
+ elif comm_mode == "flatview" and group_type == "flatview":
64
+ if mpi_rank == 0:
65
+ group_rank_id = "All"
66
+ should_log = True
67
+
68
+ # Special handling for broadcast - allow all ranks to log their results
69
+ if tensor_list is not None and isinstance(tensor_list, dict) and 'type' in tensor_list and collective_name == "broadcast":
70
+ # For broadcast, we want to see all ranks, not just designated logging ranks
71
+ if comm_mode == "within_node" and group_type == "within":
72
+ node_id = mpi_rank // cfg.comm_group.within_node.num_gpus_per_node
73
+ group_label = f"{group_type.title()}-Group-{node_id}"
74
+ elif comm_mode == "across_node" and group_type == "across":
75
+ rank_id_per_node = mpi_rank % cfg.comm_group.across_node.num_gpus_per_node
76
+ group_label = f"{group_type.title()}-Group-{rank_id_per_node}"
77
+ elif comm_mode == "combined" and group_type == "within":
78
+ node_id = mpi_rank // cfg.comm_group.combined.within_node.num_gpus_per_node
79
+ group_label = f"{group_type.title()}-Group-{node_id}"
80
+ elif comm_mode == "combined" and group_type == "across":
81
+ rank_id_per_node = mpi_rank % cfg.comm_group.combined.across_node.num_gpus_per_node
82
+ group_label = f"{group_type.title()}-Group-{rank_id_per_node}"
83
+ elif comm_mode == "flatview" and group_type == "flatview":
84
+ group_label = f"{group_type.title()}-Group-All"
85
+
86
+ # Log broadcast results for all ranks
87
+ if tensor_list['type'] == 'source':
88
+ sent_sum = tensor_list['sent_sum']
89
+ global_rank = tensor_list['global_rank']
90
+ log.output(f"[CORRECTNESS][{group_label}] Broadcast Source Rank {global_rank}: sent tensor sum = {sent_sum}")
91
+ elif tensor_list['type'] == 'receiver':
92
+ received_sum = tensor_list['received_sum']
93
+ global_rank = tensor_list['global_rank']
94
+ source_rank = tensor_list['source_rank']
95
+ log.output(f"[CORRECTNESS][{group_label}] Broadcast Receiver Rank {global_rank}: received from rank {source_rank}, tensor sum = {received_sum}")
96
+
97
+ return # Exit early for broadcast special handling
98
+
99
+ if should_log and group_rank_id is not None:
100
+ group_label = f"{group_type.title()}-Group-{group_rank_id}"
101
+ tensor_sum = float(x.sum())
102
+
103
+ if phase == "before":
104
+
105
+ _before_values[group_label] = tensor_sum
106
+
107
+ elif phase == "after":
108
+
109
+ if group_label in _before_values:
110
+ before_value = _before_values[group_label]
111
+ after_value = tensor_sum
112
+
113
+ log.output(f"[CORRECTNESS][{group_label}] Tensor sum before collective: {before_value} → after collective: {after_value}")
114
+
115
+ # Check list-based correctness for collectives that return data structures
116
+ if tensor_list is not None and phase == "after":
117
+ if isinstance(tensor_list, list):
118
+ # List validation for AllGather or Gather
119
+ collective_label = collective_name.title() if collective_name else "List-based collective"
120
+ log.output(f"[CORRECTNESS][{group_label}] {collective_label} validation:")
121
+ log.output(f"[CORRECTNESS][{group_label}] - List size: {len(tensor_list)} tensors")
122
+ for i, tensor in enumerate(tensor_list):
123
+ tensor_sum = float(tensor.sum())
124
+ log.output(f"[CORRECTNESS][{group_label}] - Tensor[{i}] sum: {tensor_sum}")
125
+ elif isinstance(tensor_list, dict):
126
+ if 'type' in tensor_list and tensor_list['type'] == 'source' and 'sent_tensor' in tensor_list:
127
+ # Broadcast validation - source rank
128
+ log.output(f"[CORRECTNESS][{group_label}] Broadcast validation (Source):")
129
+ global_rank = tensor_list['global_rank']
130
+ group_rank = tensor_list['group_rank']
131
+ source_rank = tensor_list['source_rank']
132
+ sent_tensor = tensor_list['sent_tensor']
133
+ sent_sum = tensor_list['sent_sum']
134
+ log.output(f"[CORRECTNESS][{group_label}] - Source Global Rank {global_rank} (Group Rank {group_rank})")
135
+ log.output(f"[CORRECTNESS][{group_label}] - Sent tensor: {sent_tensor.flatten()[:10].tolist()}... (showing first 10 elements)")
136
+ log.output(f"[CORRECTNESS][{group_label}] - Sent tensor sum: {sent_sum}")
137
+ elif 'type' in tensor_list and tensor_list['type'] == 'receiver' and 'received_tensor' in tensor_list:
138
+ # Broadcast validation - receiver rank
139
+ log.output(f"[CORRECTNESS][{group_label}] Broadcast validation (Receiver):")
140
+ global_rank = tensor_list['global_rank']
141
+ group_rank = tensor_list['group_rank']
142
+ source_rank = tensor_list['source_rank']
143
+ received_tensor = tensor_list['received_tensor']
144
+ received_sum = tensor_list['received_sum']
145
+ log.output(f"[CORRECTNESS][{group_label}] - Receiver Global Rank {global_rank} (Group Rank {group_rank})")
146
+ log.output(f"[CORRECTNESS][{group_label}] - Source was Global Rank {source_rank}")
147
+ log.output(f"[CORRECTNESS][{group_label}] - Received tensor: {received_tensor.flatten()[:10].tolist()}... (showing first 10 elements)")
148
+ log.output(f"[CORRECTNESS][{group_label}] - Received tensor sum: {received_sum}")
149
+ elif 'type' in tensor_list and tensor_list['type'] == 'source' and 'scattered_data' in tensor_list:
150
+ # Scatter validation - source rank
151
+ log.output(f"[CORRECTNESS][{group_label}] Scatter validation (Source):")
152
+ source_rank = tensor_list['source_global_rank']
153
+ scattered_data = tensor_list['scattered_data']
154
+ log.output(f"[CORRECTNESS][{group_label}] - Source Global Rank: {source_rank}")
155
+ log.output(f"[CORRECTNESS][{group_label}] - Scattered data:")
156
+ for data in scattered_data:
157
+ to_rank = data['to_group_rank']
158
+ value = data['value']
159
+ tensor_sum = data['tensor_sum']
160
+ log.output(f"[CORRECTNESS][{group_label}] → To Group Rank {to_rank}: tensor filled with {value}, sum = {tensor_sum}")
161
+ elif 'type' in tensor_list and tensor_list['type'] == 'receiver' and 'receiver_global_rank' in tensor_list:
162
+ # Scatter validation - receiver rank
163
+ log.output(f"[CORRECTNESS][{group_label}] Scatter validation (Receiver):")
164
+ global_rank = tensor_list['receiver_global_rank']
165
+ group_rank = tensor_list['receiver_group_rank']
166
+ expected = tensor_list['expected_value']
167
+ received_sum = tensor_list['received_tensor_sum']
168
+ is_correct = tensor_list['is_correct']
169
+ status = "✓ CORRECT" if is_correct else "✗ INCORRECT"
170
+ log.output(f"[CORRECTNESS][{group_label}] - Global Rank {global_rank} (Group Rank {group_rank})")
171
+ log.output(f"[CORRECTNESS][{group_label}] - Expected: tensor filled with {expected}")
172
+ log.output(f"[CORRECTNESS][{group_label}] - Received sum: {received_sum} {status}")
173
+ elif 'sent_data' in tensor_list and 'received_data' in tensor_list:
174
+ # AllToAll validation
175
+ log.output(f"[CORRECTNESS][{group_label}] AllToAll validation:")
176
+ global_rank = tensor_list['global_rank']
177
+ group_rank = tensor_list['group_rank']
178
+ sent_data = tensor_list['sent_data']
179
+ received_data = tensor_list['received_data']
180
+
181
+ log.output(f"[CORRECTNESS][{group_label}] - Global Rank {global_rank} (Group Rank {group_rank})")
182
+ log.output(f"[CORRECTNESS][{group_label}] - Sent data:")
183
+ for data in sent_data:
184
+ to_rank = data['to_group_rank']
185
+ sent_val = data['sent_value']
186
+ chunk_sum = data['chunk_sum']
187
+ log.output(f"[CORRECTNESS][{group_label}] → To Group Rank {to_rank}: value {sent_val}, sum = {chunk_sum}")
188
+
189
+ log.output(f"[CORRECTNESS][{group_label}] - Received data:")
190
+ all_correct = True
191
+ for data in received_data:
192
+ from_rank = data['from_group_rank']
193
+ expected_val = data['expected_value']
194
+ expected_sum = data['expected_sum']
195
+ received_sum = data['received_sum']
196
+ is_correct = data['is_correct']
197
+ status = "✓" if is_correct else "✗"
198
+ if not is_correct:
199
+ all_correct = False
200
+ log.output(f"[CORRECTNESS][{group_label}] ← From Group Rank {from_rank}: expected {expected_val} (sum={expected_sum}), got sum={received_sum} {status}")
201
+
202
+ overall_status = "✓ ALL CORRECT" if all_correct else "✗ SOME INCORRECT"
203
+ log.output(f"[CORRECTNESS][{group_label}] - Overall: {overall_status}")
204
+ else:
205
+ # ReduceScatter validation
206
+ log.output(f"[CORRECTNESS][{group_label}] ReduceScatter validation:")
207
+ global_rank = tensor_list['global_rank']
208
+ group_rank = tensor_list['group_rank']
209
+ my_chunk = tensor_list['my_chunk_index']
210
+ expected_value = tensor_list['expected_value']
211
+ log.output(f"[CORRECTNESS][{group_label}] - Global Rank {global_rank} (Group Rank {group_rank}) contributed: ones to all chunks")
212
+ log.output(f"[CORRECTNESS][{group_label}] - Global Rank {global_rank} received chunk index: {my_chunk}")
213
+ log.output(f"[CORRECTNESS][{group_label}] - Expected: each element = {expected_value} (sum of {int(expected_value)} ones)")
214
+ log.output(f"[CORRECTNESS][{group_label}] - Actual result sum: {after_value}")
215
+
216
+ del _before_values[group_label]
217
+