tt-burnin 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tt_burnin/__init__.py ADDED
@@ -0,0 +1,59 @@
1
+ # SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # Adapted from https://github.com/python-poetry/poetry/issues/273#issuecomment-1877789967
5
+ # This will get the semantic version from the current pyproject package definition.
6
+
7
+ from typing import Any
8
+
9
+ try:
10
+ import importlib.metadata as importlib_metadata
11
+ except ModuleNotFoundError:
12
+ import importlib_metadata
13
+ from pathlib import Path
14
+
15
+ __package_version = "unknown"
16
+
17
+
18
+ def __get_package_version() -> str:
19
+ """Find the version of this package."""
20
+ global __package_version
21
+
22
+ if __package_version != "unknown":
23
+ # We already set it at some point in the past,
24
+ # so return that previous value without any
25
+ # extra work.
26
+ return __package_version
27
+
28
+ try:
29
+ # Try to get the version of the current package if
30
+ # it is running from a distribution.
31
+ __package_version = importlib_metadata.version("tt-burnin")
32
+ except importlib_metadata.PackageNotFoundError:
33
+ # Fall back on getting it from a local pyproject.toml.
34
+ # This works in a development environment where the
35
+ # package has not been installed from a distribution.
36
+ try:
37
+ # This gets added to the standard library as tomllib in Python3.11
38
+ # therefore we expect to hit an ImportError.
39
+ import tomli as toml
40
+ except ImportError:
41
+ import tomllib as toml
42
+
43
+ pyproject_toml_file = Path(__file__).parent.parent / "pyproject.toml"
44
+ if pyproject_toml_file.exists() and pyproject_toml_file.is_file():
45
+ __package_version = toml.loads(pyproject_toml_file.read_text())["project"][
46
+ "version"
47
+ ]
48
+ # Indicate it might be locally modified or unreleased.
49
+ __package_version = __package_version + "+"
50
+
51
+ return __package_version
52
+
53
+
54
+ def __getattr__(name: str) -> Any:
55
+ """Get package attributes."""
56
+ if name == "__version__":
57
+ return __get_package_version()
58
+ else:
59
+ raise AttributeError(f"No attribute {name} in module {__name__}.")
tt_burnin/chip.py ADDED
@@ -0,0 +1,445 @@
1
+ # SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from __future__ import annotations
5
+
6
+ from abc import abstractmethod
7
+ import time
8
+ from typing import Union
9
+ import itertools
10
+ import sys
11
+
12
+ from pyluwen import PciChip, Telemetry
13
+ from pyluwen import detect_chips as luwen_detect_chips
14
+ from pyluwen import detect_chips_fallible as luwen_detect_chips_fallible
15
+
16
+
17
+ class TTChip:
18
+ def __init__(self, chip: PciChip):
19
+ self.luwen_chip = chip
20
+ self.interface_id = chip.pci_interface_id()
21
+
22
+ self._harvesting_bits = None
23
+
24
+ self.telmetry_cache = None
25
+
26
+ self.is_remote = False
27
+
28
+ @abstractmethod
29
+ def arch(self) -> str: ...
30
+
31
+ def reinit(self, callback=None):
32
+ self.luwen_chip = PciChip(self.interface_id)
33
+ self.telmetry_cache = None
34
+
35
+ chip_count = 0
36
+ block_count = 0
37
+ last_draw = time.time()
38
+
39
+ def chip_detect_callback(status):
40
+ nonlocal chip_count, last_draw, block_count
41
+
42
+ if status.new_chip():
43
+ chip_count += 1
44
+ elif status.correct_down():
45
+ chip_count -= 1
46
+ chip_count = max(chip_count, 0)
47
+
48
+ if sys.stdout.isatty():
49
+ current_time = time.time()
50
+ if current_time - last_draw > 0.1:
51
+ last_draw = current_time
52
+
53
+ if block_count > 0:
54
+ print(f"\033[{block_count}A", end="", flush=True)
55
+ print("\033[J", end="", flush=True)
56
+
57
+ print(f"\rDetected Chips: {chip_count}\n", end="", flush=True)
58
+ block_count = 1
59
+
60
+ status_string = status.status_string()
61
+ if status_string is not None:
62
+ for line in status_string.splitlines():
63
+ block_count += 1
64
+ print(f"\r{line}", flush=True)
65
+ else:
66
+ time.sleep(0.01)
67
+
68
+ self.luwen_chip.init(
69
+ callback=chip_detect_callback if callback is None else callback
70
+ )
71
+
72
+ def get_telemetry(self) -> Telemetry:
73
+ self.telmetry_cache = self.luwen_chip.get_telemetry()
74
+ return self.telmetry_cache
75
+
76
+ def get_telemetry_unchanged(self) -> Telemetry:
77
+ if self.telmetry_cache is None:
78
+ self.telmetry_cache = self.luwen_chip.get_telemetry()
79
+
80
+ return self.telmetry_cache
81
+
82
+ def get_harvest_bits(self) -> int:
83
+ if self._harvesting_bits is None:
84
+ # Magic value to get harvesting info
85
+ (bad_row_bits, _) = self.arc_msg(0x57)
86
+ self._harvesting_bits = bad_row_bits
87
+ return self._harvesting_bits
88
+
89
+ def __vnum_to_version(self, version: int) -> tuple[int, int, int, int]:
90
+ return (
91
+ (version >> 24) & 0xFF,
92
+ (version >> 16) & 0xFF,
93
+ (version >> 8) & 0xFF,
94
+ version & 0xFF,
95
+ )
96
+
97
+ def m3_fw_app_version(self):
98
+ telem = self.get_telemetry_unchanged()
99
+ return self.__vnum_to_version(telem.smbus_tx_m3_app_fw_version)
100
+
101
+ def smbus_fw_version(self):
102
+ telem = self.get_telemetry_unchanged()
103
+ return self.__vnum_to_version(telem.smbus_tx_arc1_fw_version)
104
+
105
+ def arc_l2_fw_version(self):
106
+ telem = self.get_telemetry_unchanged()
107
+ return self.__vnum_to_version(telem.smbus_tx_arc0_fw_version)
108
+
109
+ def board_type(self):
110
+ return self.luwen_chip.pci_board_type()
111
+
112
+ def board_id(self):
113
+ telem = self.get_telemetry_unchanged()
114
+ return telem.board_id
115
+
116
+ def noc_read(self, noc: int, x: int, y: int, addr: int, data: bytes):
117
+ self.luwen_chip.noc_read(noc, x, y, addr, data)
118
+
119
+ def noc_read32(self, noc: int, x: int, y: int, addr: int):
120
+ return self.luwen_chip.noc_read32(noc, x, y, addr)
121
+
122
+ def noc_write(self, noc: int, x: int, y: int, addr: int, data: bytes):
123
+ self.luwen_chip.noc_write(noc, x, y, addr, data)
124
+
125
+ def noc_write32(self, noc: int, x: int, y: int, addr: int, data: int):
126
+ self.luwen_chip.noc_write32(noc, x, y, addr, data)
127
+
128
+ def noc_broadcast(self, noc: int, addr: int, data: bytes):
129
+ self.luwen_chip.noc_broadcast(noc, addr, data)
130
+
131
+ def noc_broadcast32(self, noc: int, addr: int, data: int):
132
+ self.luwen_chip.noc_broadcast32(noc, addr, data)
133
+
134
+ def axi_write32(self, addr: int, value: int):
135
+ self.luwen_chip.axi_write32(addr, value)
136
+
137
+ def axi_write(self, addr: int, data: bytes):
138
+ self.luwen_chip.axi_write(addr, data)
139
+
140
+ def axi_read32(self, addr: int) -> int:
141
+ return self.luwen_chip.axi_read32(addr)
142
+
143
+ def axi_read(self, addr: int, size: int) -> bytes:
144
+ data = bytearray(size)
145
+ self.luwen_chip.axi_read(addr, data)
146
+
147
+ return bytes(data)
148
+
149
+ def spi_write(self, addr: int, data: bytes):
150
+ self.luwen_chip.spi_write(addr, data)
151
+
152
+ def spi_read(self, addr: int, size: int) -> bytes:
153
+ data = bytearray(size)
154
+ self.luwen_chip.spi_read(addr, data)
155
+
156
+ return bytes(data)
157
+
158
+ def arc_msg(self, *args, **kwargs):
159
+ return self.luwen_chip.arc_msg(*args, **kwargs)
160
+
161
+ # Given non-negative integer x, return an iterable containing the bits set in x, in increasing order.
162
+ def _int_to_bits(self, x):
163
+ return list(filter(lambda b: x & (1 << b), range(x.bit_length())))
164
+
165
+
166
+ def reverse_mapping_list(l):
167
+ ret = [0] * len(l)
168
+ for idx, val in enumerate(l):
169
+ ret[val] = idx
170
+ return ret
171
+
172
+
173
+ class BhChip(TTChip):
174
+ def noc_coord_flip(self, coord: tuple[int, int]) -> tuple[int, int]:
175
+ return (self.GRID_SIZE_X - coord[0] - 1, self.GRID_SIZE_Y - coord[1] - 1)
176
+
177
+ # Physical rows & columns are defined in Blackhole - NOC Co-ordinates
178
+ def phys_to_noc(self, coord: tuple[int, int], noc_id: int) -> tuple[int, int]:
179
+ noc0 = (self.PHYS_X_TO_NOC_0_X[coord[0]], self.PHYS_Y_TO_NOC_0_y[coord[1]])
180
+ return noc0 if noc_id == 0 else self.noc_coord_flip(noc0)
181
+
182
+ def __init__(self, *args, **kwargs):
183
+ self.GRID_SIZE_X = 17
184
+ self.GRID_SIZE_Y = 12
185
+
186
+ self.NUM_TENSIX_X = 14
187
+ self.NUM_TENSIX_Y = 10
188
+
189
+ self.TENSIX_LOCATIONS = []
190
+ for y in range(2, 12):
191
+ for x in range(1, 8):
192
+ self.TENSIX_LOCATIONS.append((x, y))
193
+ for x in range(10, 17):
194
+ self.TENSIX_LOCATIONS.append((x, y))
195
+
196
+ super().__init__(*args, **kwargs)
197
+
198
+ @property
199
+ def noc_translation_enabled(self) -> bool:
200
+ telemetry = self.get_telemetry_unchanged()
201
+ return telemetry.noc_translation_enabled
202
+
203
+ @property
204
+ def enabled_tensix_columns(self) -> int:
205
+ telemetry = self.get_telemetry_unchanged()
206
+ return telemetry.tensix_enabled_col
207
+
208
+ def get_tensix_locations(self):
209
+ if self.noc_translation_enabled:
210
+ # When translated NOC 0 looks roughly the same as without translation
211
+ # with the minor difference that the harvested tensix are moved to the
212
+ # end (right side) of the grid
213
+ NUM_TENSIX_X = 0
214
+ enabled_tensix_columns_bitmask = self.enabled_tensix_columns
215
+ while enabled_tensix_columns_bitmask != 0:
216
+ if enabled_tensix_columns_bitmask & 0x1 != 0:
217
+ NUM_TENSIX_X += 1
218
+ enabled_tensix_columns_bitmask >>= 1
219
+
220
+ good_cores = []
221
+ for core in self.TENSIX_LOCATIONS:
222
+ if (core[0] <= 7 and core[0] < NUM_TENSIX_X) or (core[0] >= 10 and (core[0] - 2) < NUM_TENSIX_X):
223
+ good_cores.append(core)
224
+ else:
225
+ enabled_tensix_columns_bitmask = self.enabled_tensix_columns
226
+ enabled_tensix_columns = []
227
+ TENSIX_COLS = [1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16]
228
+ for col in range(self.NUM_TENSIX_X):
229
+ if enabled_tensix_columns_bitmask & 0x1 == 1:
230
+ enabled_tensix_columns.append(TENSIX_COLS[col])
231
+ enabled_tensix_columns_bitmask >>= 1
232
+
233
+ enabled_tensix_columns = set(enabled_tensix_columns)
234
+
235
+ good_cores = []
236
+ for core in self.TENSIX_LOCATIONS:
237
+ if core[0] in enabled_tensix_columns:
238
+ good_cores.append(core)
239
+
240
+ return set(good_cores)
241
+
242
+ def coord(self):
243
+ coord = self.luwen_chip.get_local_coord()
244
+ return (coord.shelf_x, coord.shelf_y, coord.rack_x, coord.rack_y)
245
+
246
+ def arch(self):
247
+ return "Blackhole"
248
+
249
+ def __repr__(self):
250
+ return f"Blackhole[{self.interface_id}]"
251
+
252
+
253
+ class WhChip(TTChip):
254
+ def __init__(self, *args, **kwargs):
255
+ self.GRID_SIZE_X = 10
256
+ self.GRID_SIZE_Y = 12
257
+ self.NUM_TENSIX_X = self.GRID_SIZE_X - 2
258
+ self.NUM_TENSIX_Y = self.GRID_SIZE_Y - 2
259
+
260
+ self.PHYS_X_TO_NOC_0_X = [0, 9, 1, 8, 2, 7, 3, 6, 4, 5]
261
+ self.PHYS_Y_TO_NOC_0_Y = [0, 11, 1, 10, 2, 9, 3, 8, 4, 7, 5, 6]
262
+ self.PHYS_X_TO_NOC_1_X = [9, 0, 8, 1, 7, 2, 6, 3, 5, 4]
263
+ self.PHYS_Y_TO_NOC_1_Y = [11, 0, 10, 1, 9, 2, 8, 3, 7, 4, 6, 5]
264
+ self.NOC_0_X_TO_PHYS_X = reverse_mapping_list(self.PHYS_X_TO_NOC_0_X)
265
+ self.NOC_0_Y_TO_PHYS_Y = reverse_mapping_list(self.PHYS_Y_TO_NOC_0_Y)
266
+ self.NOC_1_X_TO_PHYS_X = reverse_mapping_list(self.PHYS_X_TO_NOC_1_X)
267
+ self.NOC_1_Y_TO_PHYS_Y = reverse_mapping_list(self.PHYS_Y_TO_NOC_1_Y)
268
+
269
+ super().__init__(*args, **kwargs)
270
+
271
+ def get_tensix_locations(self):
272
+ all_tensix_rows = [1, 2, 3, 4, 5, 7, 8, 9, 10, 11]
273
+ all_tensix_cols = [1, 2, 3, 4, 6, 7, 8, 9]
274
+
275
+ # Magic value to get harvesting info
276
+ bad_row_bits = self.get_harvest_bits()
277
+
278
+ bad_row_bits = bad_row_bits << 1
279
+
280
+ bad_physical_rows = self._int_to_bits(bad_row_bits)
281
+
282
+ disabled_rows = frozenset(
283
+ map(lambda y: self.PHYS_Y_TO_NOC_0_Y[y], bad_physical_rows)
284
+ )
285
+
286
+ good_rows = filter(lambda y: y not in disabled_rows, all_tensix_rows)
287
+ good_cores = itertools.product(all_tensix_cols, good_rows)
288
+
289
+ return set(good_cores)
290
+
291
+ def arch(self):
292
+ return "Wormhole"
293
+
294
+ def __repr__(self):
295
+ return f"Wormhole[{self.interface_id}]"
296
+
297
+ def coord(self):
298
+ coord = self.luwen_chip.get_local_coord()
299
+ return (coord.shelf_x, coord.shelf_y, coord.rack_x, coord.rack_y)
300
+
301
+
302
+ class RemoteWhChip(WhChip):
303
+ def __init__(self, *args, **kwargs):
304
+ super().__init__(*args, **kwargs)
305
+
306
+ self.is_remote = True
307
+
308
+ def noc_broadcast(self, noc: int, addr: int, data: bytes):
309
+ for core in self.get_tensix_locations():
310
+ self.luwen_chip.noc_write(noc, *core, addr, data)
311
+
312
+ def noc_broadcast32(self, noc: int, addr: int, data: int):
313
+ for core in self.get_tensix_locations():
314
+ self.luwen_chip.noc_write32(noc, *core, addr, data)
315
+
316
+
317
+ class GsChip(TTChip):
318
+ def __init__(self, *args, **kwargs):
319
+ self.GRID_SIZE_X = 13
320
+ self.GRID_SIZE_Y = 12
321
+ self.NUM_TENSIX_X = self.GRID_SIZE_X - 1
322
+ self.NUM_TENSIX_Y = self.GRID_SIZE_Y - 2
323
+
324
+ self.PHYS_X_TO_NOC_0_X = [0, 12, 1, 11, 2, 10, 3, 9, 4, 8, 5, 7, 6]
325
+ self.PHYS_Y_TO_NOC_0_Y = [0, 11, 1, 10, 2, 9, 3, 8, 4, 7, 5, 6]
326
+ self.PHYS_X_TO_NOC_1_X = [12, 0, 11, 1, 10, 2, 9, 3, 8, 4, 7, 5, 6]
327
+ self.PHYS_Y_TO_NOC_1_Y = [11, 0, 10, 1, 9, 2, 8, 3, 7, 4, 6, 5]
328
+ self.NOC_0_X_TO_PHYS_X = reverse_mapping_list(self.PHYS_X_TO_NOC_0_X)
329
+ self.NOC_0_Y_TO_PHYS_Y = reverse_mapping_list(self.PHYS_Y_TO_NOC_0_Y)
330
+ self.NOC_1_X_TO_PHYS_X = reverse_mapping_list(self.PHYS_X_TO_NOC_1_X)
331
+ self.NOC_1_Y_TO_PHYS_Y = reverse_mapping_list(self.PHYS_Y_TO_NOC_1_Y)
332
+
333
+ super().__init__(*args, **kwargs)
334
+
335
+ def get_tensix_locations(self):
336
+ bad_row_bits = self.get_harvest_bits()
337
+ bad_row_bits = bad_row_bits << 1
338
+
339
+ bad_physical_rows = self._int_to_bits(bad_row_bits)
340
+
341
+ disabled_rows = frozenset(
342
+ map(
343
+ lambda y: self.PHYS_Y_TO_NOC_0_Y[self.GRID_SIZE_Y - y - 1],
344
+ bad_physical_rows,
345
+ )
346
+ )
347
+ good_rows = filter(
348
+ lambda y: y not in disabled_rows, [1, 2, 3, 4, 5, 7, 8, 9, 10, 11]
349
+ )
350
+ good_cores = list(
351
+ itertools.product(list(range(1, self.GRID_SIZE_X)), good_rows)
352
+ )
353
+
354
+ return set(good_cores)
355
+
356
+ def coord(self):
357
+ return "N/A"
358
+
359
+ def arch(self):
360
+ return "Grayskull"
361
+
362
+ def __repr__(self):
363
+ return f"Grayskull[{self.interface_id}]"
364
+
365
+
366
+ def detect_local_chips(ignore_ethernet: bool = False) -> list[Union[GsChip, WhChip]]:
367
+ """
368
+ This will create a chip which only gaurentees that you have communication with the chip.
369
+ """
370
+
371
+ chip_count = 0
372
+ block_count = 0
373
+ last_draw = time.time()
374
+
375
+ def chip_detect_callback(status):
376
+ nonlocal chip_count, last_draw, block_count
377
+
378
+ if status.new_chip():
379
+ chip_count += 1
380
+ elif status.correct_down():
381
+ chip_count -= 1
382
+ chip_count = max(chip_count, 0)
383
+
384
+ if sys.stdout.isatty():
385
+ current_time = time.time()
386
+ if current_time - last_draw > 0.1:
387
+ last_draw = current_time
388
+
389
+ if block_count > 0:
390
+ print(f"\033[{block_count}A", end="", flush=True)
391
+ print(f"\033[J", end="", flush=True)
392
+
393
+ print(f"\rDetected Chips: {chip_count}\n", end="", flush=True)
394
+ block_count = 1
395
+
396
+ status_string = status.status_string()
397
+ if status_string is not None:
398
+ for line in status_string.splitlines():
399
+ block_count += 1
400
+ print(f"\r{line}", flush=True)
401
+ else:
402
+ time.sleep(0.01)
403
+
404
+ output = []
405
+ for device in luwen_detect_chips_fallible(
406
+ local_only=True,
407
+ continue_on_failure=False,
408
+ callback=chip_detect_callback,
409
+ noc_safe=ignore_ethernet,
410
+ ):
411
+ if not device.have_comms():
412
+ raise Exception(
413
+ f"Do not have communication with {device}, you should reset or remove this device from your system before continuing."
414
+ )
415
+
416
+ device = device.force_upgrade()
417
+
418
+ if device.as_gs() is not None:
419
+ output.append(GsChip(device.as_gs()))
420
+ elif device.as_wh() is not None:
421
+ output.append(WhChip(device.as_wh()))
422
+ elif device.as_bh() is not None:
423
+ output.append(BhChip(device.as_bh()))
424
+ else:
425
+ raise ValueError("Did not recognize board")
426
+
427
+ return output
428
+
429
+
430
+ def detect_chips(local_only: bool = False) -> list[Union[GsChip, WhChip]]:
431
+ output = []
432
+ for device in luwen_detect_chips(local_only=local_only):
433
+ if device.as_gs() is not None:
434
+ output.append(GsChip(device.as_gs()))
435
+ elif device.as_wh() is not None:
436
+ if device.is_remote():
437
+ output.append(RemoteWhChip(device.as_wh()))
438
+ else:
439
+ output.append(WhChip(device.as_wh()))
440
+ elif device.as_bh() is not None:
441
+ output.append(BhChip(device.as_bh()))
442
+ else:
443
+ raise ValueError("Did not recognize board")
444
+
445
+ return output